diff --git a/.gitignore b/.gitignore index 3bf5223..746101f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ # OS .DS_Store Thumbs.db +.syntext # Environment .env diff --git a/Cargo.lock b/Cargo.lock index 1132b4f..ee35b8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,7 @@ dependencies = [ "crc32fast", "dashmap", "futures", + "getrandom 0.3.4", "hex", "hmac", "httpdate", diff --git a/crates/proxy/Cargo.toml b/crates/proxy/Cargo.toml index d0f503c..aec3999 100644 --- a/crates/proxy/Cargo.toml +++ b/crates/proxy/Cargo.toml @@ -31,6 +31,7 @@ hmac = "0.12" rusqlite = { version = "0.32", features = ["bundled"] } httpdate = "1" dashmap = "6" +getrandom = "0.3" aws-sigv4 = { version = "1.4", features = ["sign-http"] } aws-credential-types = "1.2" aws-smithy-runtime-api = "1" @@ -44,6 +45,10 @@ zeroize = "1" crc32fast = "1" [features] +## Enables BashTool and ReadFileTool in the builtin tool registry. +## These tools execute arbitrary shell commands and read arbitrary files as the +## proxy process user. Do NOT enable in production without sandboxing. +dangerous-builtin-tools = [] redis = ["dep:redis"] qdrant = ["dep:qdrant-client"] otel = [ diff --git a/crates/proxy/src/admin/db.rs b/crates/proxy/src/admin/db.rs index ad6f1ef..d00ac49 100644 --- a/crates/proxy/src/admin/db.rs +++ b/crates/proxy/src/admin/db.rs @@ -169,12 +169,9 @@ pub fn ensure_hmac_secret(conn: &Connection) -> Vec { return secret; } - // Generate 32 random bytes from two UUID v4s. + // Generate 256-bit CSPRNG secret directly. let mut buf = [0u8; 32]; - let a = uuid::Uuid::new_v4(); - let b = uuid::Uuid::new_v4(); - buf[..16].copy_from_slice(a.as_bytes()); - buf[16..].copy_from_slice(b.as_bytes()); + getrandom::fill(&mut buf).expect("CSPRNG failed"); conn.execute( "INSERT INTO settings (key, value) VALUES ('hmac_secret', ?1)", diff --git a/crates/proxy/src/server/middleware.rs b/crates/proxy/src/server/middleware.rs index d30520a..db9fb13 100644 --- a/crates/proxy/src/server/middleware.rs +++ b/crates/proxy/src/server/middleware.rs @@ -285,10 +285,11 @@ pub async fn validate_auth( } } - // RBAC: developer keys cannot access admin endpoints + // RBAC: developer keys cannot access admin endpoints. + // Case-insensitive to prevent bypass via `/Admin/`, `/ADMIN/`, etc. if meta.role == KeyRole::Developer { - let path = request.uri().path(); - if path.starts_with("/admin/") || path.starts_with("/admin") { + let path = request.uri().path().to_ascii_lowercase(); + if path.starts_with("/admin/") || path == "/admin" { let err_body = serde_json::json!({ "error": { "type": "permission_denied", @@ -525,6 +526,17 @@ static TRUST_PROXY_HEADERS: LazyLock = LazyLock::new(|| { .unwrap_or(false) }); +/// Number of trusted proxy hops. The client IP is extracted as the Nth-from-right +/// entry in X-Forwarded-For. Defaults to 1 (single reverse proxy). +/// Set TRUSTED_PROXY_DEPTH=2 for chains like CDN -> LB -> proxy. +static TRUSTED_PROXY_DEPTH: LazyLock = LazyLock::new(|| { + std::env::var("TRUSTED_PROXY_DEPTH") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(1) + .max(1) // minimum 1 +}); + /// Check if an IP address is allowed by the configured allowlist. /// Returns true if no allowlist is set (open access). pub fn is_ip_allowed(ip: std::net::IpAddr) -> bool { @@ -543,14 +555,20 @@ pub fn ip_allowlist_active() -> bool { /// Applied before auth so blocked IPs never reach authentication. pub async fn check_ip_allowlist(request: Request, next: Next) -> Result { // Extract client IP from X-Forwarded-For (if trusted) or connection info. + // TRUSTED_PROXY_DEPTH controls which entry to pick: depth=1 (default) takes + // the rightmost (single proxy), depth=2 takes the second-from-right (two hops), etc. let client_ip = if *TRUST_PROXY_HEADERS { - // Take the *rightmost* IP: a trusted reverse proxy appends the real client IP. - // The leftmost value is attacker-controlled and must not be trusted. + let depth = *TRUSTED_PROXY_DEPTH; request .headers() .get("x-forwarded-for") .and_then(|v| v.to_str().ok()) - .and_then(|s| s.rsplit(',').map(|p| p.trim()).find(|p| !p.is_empty())) + .and_then(|s| { + s.rsplit(',') + .map(|p| p.trim()) + .filter(|p| !p.is_empty()) + .nth(depth - 1) + }) .and_then(|s| s.parse::().ok()) } else { None diff --git a/crates/proxy/src/server/routes.rs b/crates/proxy/src/server/routes.rs index 96704c3..fe7016c 100644 --- a/crates/proxy/src/server/routes.rs +++ b/crates/proxy/src/server/routes.rs @@ -846,63 +846,45 @@ async fn messages( &original_model, ); - // Tool execution: if the response contains tool_use blocks for - // registered tools, execute them and make a follow-up backend call. + // Tool execution: bounded loop with termination guards. let anthropic_resp = if let Some(ref engine) = state.tool_engine { - let tool_calls = - crate::tools::execution::extract_tool_calls(&anthropic_resp); - let (auto_exec, _pass_through) = - crate::tools::execution::partition_tool_calls( - &tool_calls, - &engine.registry, - &engine.policy, - ); - - if !auto_exec.is_empty() { - let results = crate::tools::execution::execute_tool_calls( - &auto_exec, - engine.registry.clone(), - &engine.policy, - &engine.loop_config, - ) - .await; - - let mut follow_up_req = body.clone(); - follow_up_req.messages.push( - crate::tools::execution::response_to_assistant_message( - &anthropic_resp, - ), - ); - follow_up_req.messages.push( - crate::tools::execution::tool_results_to_user_message(&results), - ); - - let mut follow_up_openai = - mapping::message_map::anthropic_to_openai_request(&follow_up_req); - follow_up_openai.model = mapped_model.clone(); - - match client.chat_completion(&follow_up_openai).await { - Ok((follow_up_resp, _, _)) => { - tracing::info!( - tools_executed = results.len(), - "tool execution loop completed" - ); - mapping::message_map::openai_to_anthropic_response( - &follow_up_resp, - &original_model, - ) + let client_for_tools = client.clone(); + let model_for_tools = mapped_model.clone(); + let orig_model_for_tools = original_model.clone(); + let (resp, trace) = crate::tools::execution::maybe_execute_tools( + engine, + &body, + anthropic_resp, + |follow_up_req| { + let c = client_for_tools.clone(); + let m = model_for_tools.clone(); + let om = orig_model_for_tools.clone(); + async move { + let mut oai_req = + mapping::message_map::anthropic_to_openai_request( + &follow_up_req, + ); + oai_req.model = m; + match c.chat_completion(&oai_req).await { + Ok((resp, _, _)) => Ok( + mapping::message_map::openai_to_anthropic_response( + &resp, &om, + ), + ), + Err(e) => Err(format!("{e}")), + } } - Err(e) => { - tracing::warn!( - error = %e, - "follow-up backend call after tool execution failed" - ); - anthropic_resp - } - } - } else { - anthropic_resp - } + }, + ) + .await; + tracing::debug!( + termination_reason = ?trace.termination_reason, + iterations = trace.iterations.len(), + tool_calls = trace.total_tool_calls(), + total_ms = trace.total_duration.as_millis(), + "tool execution loop complete" + ); + resp } else { anthropic_resp }; diff --git a/crates/proxy/src/tools/mod.rs b/crates/proxy/src/tools/mod.rs index ccc3ec2..e1f0716 100644 --- a/crates/proxy/src/tools/mod.rs +++ b/crates/proxy/src/tools/mod.rs @@ -7,7 +7,7 @@ pub mod policy; pub mod registry; pub mod trace; -pub use execution::{LoopConfig, ToolCall, ToolResult}; +pub use execution::{maybe_execute_tools, LoopConfig, ToolCall, ToolResult}; pub use mcp::McpServerManager; pub use policy::{PolicyAction, PolicyRule, ToolExecutionPolicy}; pub use registry::{Tool, ToolRegistry}; diff --git a/repomix-output.xml b/repomix-output.xml deleted file mode 100644 index 5afd9ff..0000000 --- a/repomix-output.xml +++ /dev/null @@ -1,25762 +0,0 @@ -This file is a merged representation of a subset of the codebase, containing files not matching ignore patterns, combined into a single document by Repomix. -The content has been processed where content has been compressed (code blocks are separated by ⋮---- delimiter). - - -This section contains a summary of this file. - - -This file contains a packed representation of the entire repository's contents. -It is designed to be easily consumable by AI systems for analysis, code review, -or other automated processes. - - - -The content is organized as follows: -1. This summary section -2. Repository information -3. Directory structure -4. Repository files (if enabled) -4. Repository files, each consisting of: - - File path as an attribute - - Full contents of the file - - - -- This file should be treated as read-only. Any changes should be made to the - original repository files, not this packed version. -- When processing this file, use the file path to distinguish - between different files in the repository. -- Be aware that this file may contain sensitive information. Handle it with - the same level of security as you would the original repository. - - - -- Some files may have been excluded based on .gitignore rules and Repomix's configuration -- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files -- Files matching these patterns are excluded: docs -- Files matching patterns in .gitignore are excluded -- Files matching default ignore patterns are excluded -- Content has been compressed - code blocks are separated by ⋮---- delimiter -- Files are sorted by Git change count (files with more changes are at the bottom) - - - - - - - - - -.github/ - workflows/ - ci.yml -assets/ - model_pricing.json -crates/ - client/ - examples/ - basic.rs - streaming.rs - tools.rs - src/ - client.rs - error.rs - http.rs - lib.rs - rate_limit.rs - retry.rs - sse.rs - streaming.rs - tools.rs - Cargo.toml - proxy/ - admin-ui/ - index.html - src/ - admin/ - auth.rs - db.rs - keys.rs - mod.rs - routes.rs - spend.rs - state.rs - ws.rs - backend/ - anthropic_client.rs - bedrock_client.rs - gemini_client.rs - mod.rs - openai_client.rs - batch/ - anthropic_batch.rs - db.rs - mod.rs - openai_batch_client.rs - routes.rs - cache/ - memory.rs - mod.rs - redis.rs - semantic.rs - config/ - env_aliases.rs - litellm.rs - mod.rs - model_router.rs - tls.rs - url_validation.rs - cost/ - db.rs - mod.rs - fallback/ - config.rs - mod.rs - integrations/ - langfuse.rs - mod.rs - metrics/ - mod.rs - server/ - audio.rs - bedrock_passthrough.rs - chat_completions.rs - gemini_native.rs - images.rs - middleware.rs - mod.rs - oidc.rs - passthrough.rs - policy.rs - routes.rs - sse.rs - streaming.rs - token_counting.rs - callbacks.rs - lib.rs - main.rs - otel.rs - ratelimit.rs - tests/ - audio_image.rs - batch_api.rs - body_logging.rs - chat_completions.rs - compatibility.rs - embeddings.rs - error_fixtures.rs - fallback.rs - health.rs - live_api.rs - live_azure.rs - live_bedrock.rs - live_responses.rs - multi_backend.rs - shutdown.rs - virtual_keys.rs - Cargo.toml - translator/ - examples/ - reverse_translation.rs - translate_request.rs - src/ - anthropic/ - batch.rs - errors.rs - messages.rs - mod.rs - streaming.rs - gemini/ - mod.rs - request.rs - response.rs - mapping/ - batch_map.rs - errors_map.rs - gemini_message_map.rs - gemini_streaming_map.rs - message_map.rs - mod.rs - responses_message_map.rs - responses_streaming_map.rs - reverse_message_map.rs - reverse_streaming_map.rs - streaming_map.rs - tools_map.rs - usage_map.rs - warnings.rs - middleware/ - client.rs - handler.rs - mod.rs - openai/ - chat_completions.rs - errors.rs - mod.rs - responses.rs - streaming.rs - util/ - ids.rs - json.rs - mod.rs - redact.rs - config.rs - error.rs - lib.rs - translate.rs - tests/ - golden_fixtures.rs - library_usage.rs - middleware_integration.rs - Cargo.toml - README.md -fixtures/ - anthropic/ - claude_code_tool_result.json - claude_code_tool_use.json - error_invalid_request.json - error_rate_limit.json - messages_basic.json - messages_oversized_request.json - messages_tool_use.json - openai/ - chat_completion_basic.json - chat_completion_malformed.json - chat_completion_tool_call.json - claude_code_tool_call.json - error_401.json - error_429.json - error_500.json -scripts/ - ralph/ - .last-branch - condense-openapi.py -tasks/ - prd-anthropic-domain-types.md - prd-compatibility-endpoints.md - prd-end-to-end-validation.md - prd-files-document-blocks.md - prd-hardening-security-observability.md - prd-non-streaming-translation.md - prd-openai-domain-types.md - prd-project-scaffolding.md - prd-proxy-server-routing.md - prd-streaming-sse-translation.md - prd-tool-calling-translation.md -.gitignore -AGENTS.md -Cargo.toml -CLAUDE.md -Dockerfile -LICENSE -README.md - - - -This section contains the contents of the repository's files. - - -//! Non-streaming example: send an Anthropic request and print the response. -//! -//! ```bash -//! CHAT_COMPLETIONS_URL=https://api.openai.com/v1/chat/completions \ -//! OPENAI_API_KEY=sk-... \ -//! cargo run --example basic -p anyllm_client -//! ``` -⋮---- -//! For local Ollama: CHAT_COMPLETIONS_URL=http://localhost:11434/v1/chat/completions -⋮---- -use anyllm_translate::anthropic::MessageCreateRequest; -⋮---- -async fn main() { -if let Err(e) = run().await { -eprintln!("Error: {e}"); -⋮---- -ClientError::Transport(inner) => eprintln!(" transport: {inner}"), -⋮---- -eprintln!(" backend returned HTTP {status}: {body}") -⋮---- -ClientError::Translation(inner) => eprintln!(" translation: {inner}"), -ClientError::Deserialization(msg) => eprintln!(" deserialization: {msg}"), -ClientError::Sse(inner) => eprintln!(" sse: {inner}"), -⋮---- -async fn run() -> Result<(), ClientError> { -⋮---- -.unwrap_or_else(|_| "https://api.openai.com/v1/chat/completions".to_string()); -let api_key = std::env::var("OPENAI_API_KEY").unwrap_or_default(); -⋮---- -// ClientBuilder: quick setup with sensible defaults (10s connect, 900s read, 3 retries). -// For custom TLS/SSRF settings use Client::new(ClientConfig::builder()...). -⋮---- -.base_url(&url) -.api_key(&api_key) -.build()?; -⋮---- -// Construct the request as Anthropic Messages API JSON. The client handles -// translation to OpenAI Chat Completions format before sending. -⋮---- -.expect("static request JSON is valid"); -⋮---- -let response = client.messages(&req).await?; -⋮---- -println!("stop_reason: {:?}", response.stop_reason); -println!("usage: input={} output={}", response.usage.input_tokens, response.usage.output_tokens); -⋮---- -println!("{text}"); -⋮---- -Ok(()) - - - -//! Streaming example: receive tokens incrementally as they arrive. -//! -//! ```bash -//! CHAT_COMPLETIONS_URL=https://api.openai.com/v1/chat/completions \ -//! OPENAI_API_KEY=sk-... \ -//! cargo run --example streaming -p anyllm_client -//! ``` -⋮---- -use futures::StreamExt; -use std::io::Write; -⋮---- -async fn main() { -if let Err(e) = run().await { -eprintln!("Error: {e}"); -⋮---- -async fn run() -> Result<(), ClientError> { -⋮---- -.unwrap_or_else(|_| "https://api.openai.com/v1/chat/completions".to_string()); -let api_key = std::env::var("OPENAI_API_KEY").unwrap_or_default(); -⋮---- -.base_url(&url) -.api_key(&api_key) -.build()?; -⋮---- -.expect("static request JSON is valid"); -⋮---- -// messages_stream() returns (stream, rate_limit_headers). -// The stream yields StreamEvent items translated from the backend's SSE chunks. -let (mut stream, rate_limits) = client.messages_stream(&req).await?; -⋮---- -while let Some(event) = stream.next().await { -⋮---- -// TextDelta carries incremental text. Print immediately without buffering. -⋮---- -print!("{text}"); -std::io::stdout().flush().ok(); -⋮---- -eprintln!("\n[output tokens: {}]", u.output_tokens); -⋮---- -println!(); // ensure a trailing newline -⋮---- -_ => {} // MessageStart, ContentBlockStart/Stop, Ping are informational -⋮---- -eprintln!("[requests remaining: {remaining}]"); -⋮---- -Ok(()) - - - -//! Tool calling example: define a tool, let the model call it, handle the result. -//! -//! ```bash -//! CHAT_COMPLETIONS_URL=https://api.openai.com/v1/chat/completions \ -//! OPENAI_API_KEY=sk-... \ -//! cargo run --example tools -p anyllm_client -//! ``` -⋮---- -use serde_json::json; -⋮---- -async fn main() { -if let Err(e) = run().await { -eprintln!("Error: {e}"); -⋮---- -async fn run() -> Result<(), ClientError> { -⋮---- -.unwrap_or_else(|_| "https://api.openai.com/v1/chat/completions".to_string()); -let api_key = std::env::var("OPENAI_API_KEY").unwrap_or_default(); -⋮---- -.base_url(&url) -.api_key(&api_key) -.build()?; -⋮---- -// Define a tool with ToolBuilder. The input_schema is standard JSON Schema. -⋮---- -.description("Get the current weather for a location") -.input_schema(json!({ -⋮---- -.build(); -⋮---- -// Construct the request with the tool attached. -// tool_choice: auto lets the model decide; use ToolChoiceBuilder::specific("get_weather") -// to force it to call a particular tool. -let req: MessageCreateRequest = serde_json::from_value(json!({ -⋮---- -.expect("request JSON is valid"); -⋮---- -let response = client.messages(&req).await?; -⋮---- -println!("stop_reason: {:?}", response.stop_reason); -⋮---- -println!("text: {text}"); -⋮---- -println!("tool_use: name={name} id={id}"); -println!(" input: {}", serde_json::to_string_pretty(input).unwrap()); -⋮---- -// In a real application: execute the tool here, then send the result -// back in a follow-up request as a ContentBlock::ToolResult. -let _tool_result = call_weather_tool(input); -println!(" (tool execution would happen here)"); -⋮---- -// If stop_reason is ToolUse, send the tool result in a follow-up turn. -if matches!(response.stop_reason, Some(StopReason::ToolUse)) { -println!("\nNext step: send tool result back in a follow-up messages() call."); -⋮---- -Ok(()) -⋮---- -// Stub simulating a real tool execution. -fn call_weather_tool(input: &serde_json::Value) -> String { -⋮---- -.get("location") -.and_then(|v| v.as_str()) -.unwrap_or("unknown"); -format!("Weather in {location}: 22°C, partly cloudy") - - - -// WebSocket endpoint for live admin dashboard updates. -// Auth via the first WebSocket message to avoid leaking the token in URLs/logs. -⋮---- -use crate::admin::state::SharedState; -⋮---- -use std::sync::Arc; -⋮---- -/// GET /admin/ws -- WebSocket for live dashboard updates. -/// Auth via the first WebSocket message to avoid leaking the token in URLs/logs. -/// The client must send `{"token": ""}` as its first message. -pub(crate) async fn ws_handler( -⋮---- -ws.on_upgrade(move |socket| handle_ws(socket, shared, expected_token)) -.into_response() -⋮---- -/// Authenticate via the first WebSocket message, then stream events. -async fn handle_ws(mut socket: WebSocket, shared: SharedState, expected_token: Arc) { -// Wait for the first message containing the auth token. -⋮---- -tokio::time::timeout(std::time::Duration::from_secs(5), socket.recv()).await; -⋮---- -// Accept either raw token string or {"token": "..."} JSON. -let token_str = text.to_string(); -let trimmed = token_str.trim(); -let expected = expected_token.as_str(); -⋮---- -.ok() -.and_then(|v| v.get("token")?.as_str().map(String::from)) -.map(|t| super::auth::constant_time_eq(&t, expected)) -.unwrap_or(false) -⋮---- -.send(Message::Text( -r#"{"error":"authentication required: send token as first message"}"#.into(), -⋮---- -let _ = socket.send(Message::Close(None)).await; -⋮---- -// Send auth success confirmation. -⋮---- -.send(Message::Text(r#"{"status":"authenticated"}"#.into())) -⋮---- -let mut rx = shared.events_tx.subscribe(); -⋮---- -break; // Client disconnected. -⋮---- -break; // Channel closed, server shutting down. -⋮---- -_ => {} // Ignore other messages from client. - - - -// Optional mTLS configuration for the backend connection. -⋮---- -use std::fmt; -⋮---- -/// Optional mTLS configuration for the backend connection. -/// Stores raw certificate bytes so Config remains Clone. -/// Validated at construction time: bad certs cause startup panic. -⋮---- -pub struct TlsConfig { -/// Raw PKCS#12 bytes and password for client certificate authentication. -⋮---- -/// Raw PEM bytes for additional CA certificate to trust. -⋮---- -fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { -f.debug_struct("TlsConfig") -.field( -⋮---- -&self.p12_identity.as_ref().map(|_| "[REDACTED]"), -⋮---- -.as_ref() -.map(|b| format!("{} bytes", b.len())), -⋮---- -.finish() -⋮---- -impl TlsConfig { -/// Load and validate TLS config from file paths. -/// Panics on invalid/missing files or wrong password. -pub fn load(p12_path: Option<&str>, p12_password: Option<&str>, ca_path: Option<&str>) -> Self { -⋮---- -.unwrap_or_else(|e| panic!("failed to read P12 file '{}': {}", path, e)); -⋮---- -// Validate the P12 parses correctly with the given password -reqwest::Identity::from_pkcs12_der(&bytes, password).unwrap_or_else(|e| { -panic!( -⋮---- -Some((bytes, password.to_string())) -⋮---- -panic!("TLS_CLIENT_CERT_P12 is set but TLS_CLIENT_CERT_PASSWORD is missing"); -⋮---- -let ca_cert_pem = ca_path.map(|path| { -⋮---- -.unwrap_or_else(|e| panic!("failed to read CA cert file '{}': {}", path, e)); -⋮---- -// Validate the PEM parses as a certificate -⋮---- -.unwrap_or_else(|e| panic!("invalid CA certificate '{}': {}", path, e)); -⋮---- -/// Load from environment variables. -pub fn from_env() -> Self { -let p12_path = std::env::var("TLS_CLIENT_CERT_P12").ok(); -let p12_password = std::env::var("TLS_CLIENT_CERT_PASSWORD").ok(); -let ca_path = std::env::var("TLS_CA_CERT").ok(); -⋮---- -p12_path.as_deref(), -p12_password.as_deref(), -ca_path.as_deref(), -⋮---- -mod tests { -⋮---- -/// Path to test fixtures relative to the workspace root. -fn fixture_path(name: &str) -> String { -let manifest = env!("CARGO_MANIFEST_DIR"); -format!("{manifest}/tests/fixtures/tls/{name}") -⋮---- -fn tls_config_none_when_no_paths() { -⋮---- -assert!(tls.p12_identity.is_none()); -assert!(tls.ca_cert_pem.is_none()); -⋮---- -fn tls_config_panics_missing_password() { -TlsConfig::load(Some("/any/path.p12"), None, None); -⋮---- -fn tls_config_panics_missing_p12_file() { -TlsConfig::load(Some("/nonexistent/file.p12"), Some("pass"), None); -⋮---- -fn tls_config_loads_valid_p12() { -let path = fixture_path("test-client.p12"); -let tls = TlsConfig::load(Some(&path), Some("test"), None); -assert!(tls.p12_identity.is_some()); -⋮---- -fn tls_config_loads_valid_ca() { -let path = fixture_path("test-ca.pem"); -let tls = TlsConfig::load(None, None, Some(&path)); -⋮---- -assert!(tls.ca_cert_pem.is_some()); -⋮---- -fn tls_config_loads_both() { -let p12 = fixture_path("test-client.p12"); -let ca = fixture_path("test-ca.pem"); -let tls = TlsConfig::load(Some(&p12), Some("test"), Some(&ca)); -⋮---- -fn tls_config_debug_redacts_password() { -⋮---- -let tls = TlsConfig::load(Some(&p12), Some("test"), None); -let debug = format!("{:?}", tls); -assert!(debug.contains("REDACTED")); -assert!(!debug.contains("test")); -⋮---- -fn tls_config_panics_wrong_password() { -⋮---- -TlsConfig::load(Some(&path), Some("wrong-password"), None); - - - -//! Reverse translation example: OpenAI -> Anthropic direction. -//! -//! Demonstrates: -//! - translate_openai_to_anthropic_request: accept an OpenAI Chat Completions request -//! - translate_anthropic_to_openai_response: convert an Anthropic response to OpenAI format -//! - ReverseStreamingTranslator: convert Anthropic SSE events to OpenAI streaming chunks -⋮---- -//! This is the direction used by the proxy's POST /v1/chat/completions endpoint, -//! allowing OpenAI-native clients (LiteLLM, LangChain) to talk to Anthropic backends. -⋮---- -//! ```bash -//! cargo run --example reverse_translation -p anyllm_translate -//! ``` -⋮---- -use anyllm_translate::openai::ChatCompletionRequest; -⋮---- -fn main() { -translate_direction(); -println!(); -streaming_direction(); -⋮---- -// --- Non-streaming: OpenAI request -> Anthropic request -> OpenAI response --- -fn translate_direction() { -println!("=== Non-streaming reverse translation ==="); -⋮---- -// An OpenAI Chat Completions request received from a client. -⋮---- -.expect("mock OpenAI request is valid"); -⋮---- -// translate_openai_to_anthropic_request: converts for forwarding to Anthropic API. -// Warnings collect features dropped in translation (e.g. tool_choice "none"). -⋮---- -let anthropic_req = translate_openai_to_anthropic_request(&openai_req, &mut warnings) -.expect("reverse translation should succeed"); -⋮---- -println!("Anthropic model: {}", anthropic_req.model); -println!("max_tokens: {}", anthropic_req.max_tokens); -if !warnings.is_empty() { -println!("warnings: {:?}", warnings); -⋮---- -// Mock an Anthropic response (normally returned by the upstream Anthropic API). -⋮---- -.expect("mock Anthropic response is valid"); -⋮---- -// translate_anthropic_to_openai_response: return to the OpenAI-native client. -let openai_resp = translate_anthropic_to_openai_response(&anthropic_resp, "gpt-4o"); -println!("OpenAI response id: {}", openai_resp.id); -if let Some(choice) = openai_resp.choices.first() { -⋮---- -println!("text: {text}"); -⋮---- -// --- Streaming: Anthropic SSE events -> OpenAI ChatCompletionChunk objects --- -fn streaming_direction() { -println!("=== Streaming reverse translation ==="); -⋮---- -// Create a translator for a given message ID and model. -// The proxy generates a synthetic message ID; here we use a fixed value. -⋮---- -new_reverse_stream_translator("chatcmpl-xyz".to_string(), "gpt-4o".to_string()); -⋮---- -// Simulate the Anthropic SSE events that would arrive from the upstream. -⋮---- -.expect("mock events are valid"); -⋮---- -let chunks = translator.process_event(event); -⋮---- -// Each chunk is ready to serialize as `data: \n\n` in an SSE stream. -let serialized = serde_json::to_string(&chunk).unwrap(); -println!("chunk: {serialized}"); -⋮---- -println!("done: {}", translator.is_done()); -if translator.is_done() { -// Emit `data: [DONE]\n\n` to signal end of stream to the OpenAI client. -println!("data: [DONE]"); - - - -//! Pure translation example: convert between Anthropic and OpenAI formats with no IO. -//! -//! This demonstrates the `anyllm_translate` crate in isolation. No HTTP, no async. -//! Useful when you want to bring your own HTTP client or test translation logic directly. -⋮---- -//! ```bash -//! cargo run --example translate_request -p anyllm_translate -//! ``` -⋮---- -use anyllm_translate::anthropic::MessageCreateRequest; -use anyllm_translate::openai::ChatCompletionResponse; -⋮---- -fn main() { -// --- 1. Configure model mapping --- -// Map Anthropic model names to backend model names. -// Any unmapped model name is passed through unchanged. -⋮---- -.model_map("claude-haiku-4-5", "gpt-4o-mini") -.model_map("claude-sonnet-4-6", "gpt-4o") -.model_map("claude-opus-4-6", "gpt-4o") -.build(); -⋮---- -// --- 2. Build an Anthropic request --- -⋮---- -.expect("static request JSON is valid"); -⋮---- -// --- 3. Check for lossy features before translating --- -// compute_request_warnings identifies features that will be silently dropped -// (e.g. top_k, thinking_config, document blocks). Use to set x-anyllm-degradation. -let warnings = compute_request_warnings(&req); -if !warnings.is_empty() { -eprintln!("translation warnings: {:?}", warnings); -⋮---- -// --- 4. Translate Anthropic -> OpenAI --- -let openai_req = translate_request(&req, &config).expect("translation should succeed"); -println!("OpenAI model: {}", openai_req.model); // "gpt-4o" -assert_eq!(openai_req.model, "gpt-4o"); -⋮---- -// Inspect how the system prompt was converted (Anthropic -> OpenAI developer role). -if let Some(first_msg) = openai_req.messages.first() { -println!("first OpenAI message role: {:?}", first_msg.role); -⋮---- -println!( -⋮---- -// --- 5. Mock an OpenAI response (normally from your HTTP client) --- -⋮---- -.expect("mock response JSON is valid"); -⋮---- -// --- 6. Translate OpenAI response -> Anthropic --- -// Pass the original Anthropic model name so it appears in the response. -let anthropic_resp = translate_response(&openai_resp, &req.model); -println!("\nAnthropic response model: {}", anthropic_resp.model); -println!("stop_reason: {:?}", anthropic_resp.stop_reason); -⋮---- -println!("text: {text}"); - - - -// Secret redaction for safe logging. Avoids leaking full API keys -// into log output while keeping enough to identify the key. -⋮---- -/// Redact a secret string for logging, showing only the first 4 and last 4 characters. -/// Returns "****" for strings shorter than 12 characters (too short to redact safely). -/// Uses char_indices() instead of byte offsets because API keys may contain -/// multi-byte UTF-8 characters; byte slicing would panic at non-char boundaries. -pub fn redact_secret(s: &str) -> String { -let char_count = s.chars().count(); -⋮---- -"****".to_string() -⋮---- -// Find byte offset of the 4th char boundary for prefix. -let prefix_end = s.char_indices().nth(4).map(|(i, _)| i).unwrap_or(s.len()); -// Find byte offset of the (len-4)th char boundary for suffix. -⋮---- -.char_indices() -.nth(char_count - 4) -.map(|(i, _)| i) -.unwrap_or(0); -format!("{}...{}", &s[..prefix_end], &s[suffix_start..]) -⋮---- -mod tests { -⋮---- -fn empty_string() { -assert_eq!(redact_secret(""), "****"); -⋮---- -fn short_string() { -assert_eq!(redact_secret("abc"), "****"); -⋮---- -fn exactly_11_chars() { -assert_eq!(redact_secret("12345678901"), "****"); -⋮---- -fn exactly_12_chars() { -assert_eq!(redact_secret("123456789012"), "1234...9012"); -⋮---- -fn typical_api_key() { -⋮---- -let redacted = redact_secret(key); -assert_eq!(redacted, "sk-p...mnop"); -⋮---- -fn multibyte_utf8_does_not_panic() { -// 12 chars, but 24 bytes (each char is 2 bytes in UTF-8). -⋮---- -assert_eq!(key.chars().count(), 12); -⋮---- -assert_eq!( -⋮---- -fn multibyte_short_string() { -// 6 chars but 12 bytes -- should still be "****" because char count < 12. -⋮---- -assert_eq!(key.chars().count(), 6); -assert_eq!(redact_secret(key), "****"); - - - -use crate::error::TranslateError; -⋮---- -/// What to do when an Anthropic feature has no backend equivalent -/// (e.g., cache_control, thinking blocks, metadata). -⋮---- -pub enum LossyBehavior { -/// Drop unsupported features silently. -⋮---- -/// Log a warning via `tracing::warn` (current default behavior). -⋮---- -/// Return a `TranslateError::Translation` instead of dropping. -⋮---- -/// Configuration for the translation layer. -/// -/// Controls model name mapping and behavior when Anthropic features -/// have no equivalent in the target API. -⋮---- -pub struct TranslationConfig { -/// Ordered list of (substring, target_model) pairs for model name mapping. -/// Case-insensitive substring match; first hit wins. -⋮---- -/// How to handle Anthropic features with no backend equivalent. -⋮---- -/// If true, models not matching any entry pass through unchanged. -/// If false, unmatched models produce `TranslateError::UnknownModel`. -⋮---- -impl Default for TranslationConfig { -fn default() -> Self { -⋮---- -impl TranslationConfig { -/// Start building a `TranslationConfig`. -pub fn builder() -> TranslationConfigBuilder { -⋮---- -/// Map an Anthropic model name to a backend model name using the configured rules. -⋮---- -/// Performs case-insensitive substring matching in insertion order. -/// Returns the first match, or passthrough/error depending on config. -pub fn map_model(&self, model: &str) -> Result { -let model_bytes = model.as_bytes(); -⋮---- -if contains_ignore_ascii_case(model_bytes, pattern.as_bytes()) { -return Ok(target.clone()); -⋮---- -Ok(model.to_string()) -⋮---- -Err(TranslateError::UnknownModel(model.to_string())) -⋮---- -/// Builder for `TranslationConfig`. -⋮---- -pub struct TranslationConfigBuilder { -⋮---- -impl TranslationConfigBuilder { -/// Add a model mapping rule: if the Anthropic model name contains `pattern` -/// (case-insensitive), map it to `target`. -pub fn model_map(mut self, pattern: impl Into, target: impl Into) -> Self { -self.config.model_map.push((pattern.into(), target.into())); -⋮---- -/// Set the lossy behavior for unsupported features. -pub fn lossy_behavior(mut self, behavior: LossyBehavior) -> Self { -⋮---- -/// Set whether unknown models pass through unchanged or produce an error. -pub fn passthrough_unknown_models(mut self, passthrough: bool) -> Self { -⋮---- -/// Build the `TranslationConfig`. -pub fn build(self) -> TranslationConfig { -⋮---- -fn contains_ignore_ascii_case(haystack: &[u8], needle: &[u8]) -> bool { -if needle.is_empty() { -⋮---- -.windows(needle.len()) -.any(|w| w.eq_ignore_ascii_case(needle)) -⋮---- -mod tests { -⋮---- -fn default_config_passthrough() { -⋮---- -assert_eq!( -⋮---- -fn model_map_substring_match() { -⋮---- -.model_map("haiku", "gpt-4o-mini") -.model_map("sonnet", "gpt-4o") -.model_map("opus", "gpt-4o") -.build(); -⋮---- -assert_eq!(config.map_model("claude-haiku-4-5").unwrap(), "gpt-4o-mini"); -assert_eq!(config.map_model("claude-sonnet-4-6").unwrap(), "gpt-4o"); -assert_eq!(config.map_model("claude-opus-4-6").unwrap(), "gpt-4o"); -⋮---- -fn model_map_case_insensitive() { -⋮---- -assert_eq!(config.map_model("Claude-SONNET-4-6").unwrap(), "gpt-4o"); -⋮---- -fn model_map_first_match_wins() { -⋮---- -.model_map("claude", "first") -.model_map("sonnet", "second") -⋮---- -// "claude-sonnet-4-6" contains both, but "claude" rule is first -assert_eq!(config.map_model("claude-sonnet-4-6").unwrap(), "first"); -⋮---- -fn unknown_model_passthrough() { -⋮---- -assert_eq!(config.map_model("custom-model").unwrap(), "custom-model"); -⋮---- -fn unknown_model_error_when_strict() { -⋮---- -.passthrough_unknown_models(false) -⋮---- -let err = config.map_model("custom-model").unwrap_err(); -assert!(matches!(err, TranslateError::UnknownModel(_))); -⋮---- -fn default_lossy_behavior_is_warn() { -⋮---- -assert_eq!(config.lossy_behavior, LossyBehavior::Warn); -⋮---- -fn builder_sets_lossy_behavior() { -⋮---- -.lossy_behavior(LossyBehavior::Silent) -⋮---- -assert_eq!(config.lossy_behavior, LossyBehavior::Silent); - - - -{ - "request": { - "model": "claude-sonnet-4-20250514", - "max_tokens": 8096, - "messages": [ - { - "role": "user", - "content": "Read the main config file and list all test files." - }, - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I'll read the config file and find the test files in parallel." - }, - { - "type": "tool_use", - "id": "toolu_01UDAtfZkgcGYMBq7Ns84vfN", - "name": "Read", - "input": { - "file_path": "/home/user/project/config.toml" - } - }, - { - "type": "tool_use", - "id": "toolu_01J3KzMqBf9tXyQFhVw2NxRG", - "name": "Glob", - "input": { - "pattern": "**/*test*" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01UDAtfZkgcGYMBq7Ns84vfN", - "content": "[package]\nname = \"my-project\"\nversion = \"0.1.0\"\nedition = \"2021\"" - }, - { - "type": "tool_result", - "tool_use_id": "toolu_01J3KzMqBf9tXyQFhVw2NxRG", - "content": "tests/unit_test.rs\ntests/integration_test.rs\nsrc/lib_test.rs" - } - ] - } - ], - "tools": [ - { - "name": "Read", - "description": "Reads a file from the local filesystem.", - "input_schema": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to read" - } - }, - "required": ["file_path"] - } - }, - { - "name": "Glob", - "description": "Fast file pattern matching tool that works with any codebase size.", - "input_schema": { - "type": "object", - "properties": { - "pattern": { - "type": "string", - "description": "The glob pattern to match files against" - } - }, - "required": ["pattern"] - } - } - ] - }, - "expected_openai_messages": [ - { - "role": "user", - "content": "Read the main config file and list all test files." - }, - { - "role": "assistant", - "content": "I'll read the config file and find the test files in parallel.", - "tool_calls": [ - { - "id": "toolu_01UDAtfZkgcGYMBq7Ns84vfN", - "type": "function", - "function": { - "name": "Read", - "arguments": "{\"file_path\":\"/home/user/project/config.toml\"}" - } - }, - { - "id": "toolu_01J3KzMqBf9tXyQFhVw2NxRG", - "type": "function", - "function": { - "name": "Glob", - "arguments": "{\"pattern\":\"**/*test*\"}" - } - } - ] - }, - { - "role": "tool", - "tool_call_id": "toolu_01UDAtfZkgcGYMBq7Ns84vfN", - "content": "[package]\nname = \"my-project\"\nversion = \"0.1.0\"\nedition = \"2021\"" - }, - { - "role": "tool", - "tool_call_id": "toolu_01J3KzMqBf9tXyQFhVw2NxRG", - "content": "tests/unit_test.rs\ntests/integration_test.rs\nsrc/lib_test.rs" - } - ] -} - - - -{ - "request": { - "model": "claude-sonnet-4-20250514", - "max_tokens": 8096, - "messages": [ - { - "role": "user", - "content": "Read the main config file and list all test files." - } - ], - "tools": [ - { - "name": "Read", - "description": "Reads a file from the local filesystem.", - "input_schema": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to read" - }, - "offset": { - "type": "number", - "description": "The line number to start reading from" - }, - "limit": { - "type": "number", - "description": "The number of lines to read" - } - }, - "required": ["file_path"] - } - }, - { - "name": "Bash", - "description": "Executes a given bash command and returns its output.", - "input_schema": { - "type": "object", - "properties": { - "command": { - "type": "string", - "description": "The command to execute" - }, - "description": { - "type": "string", - "description": "Clear, concise description of what this command does" - }, - "timeout": { - "type": "number", - "description": "Optional timeout in milliseconds (max 600000)" - } - }, - "required": ["command"] - } - }, - { - "name": "Glob", - "description": "Fast file pattern matching tool that works with any codebase size.", - "input_schema": { - "type": "object", - "properties": { - "pattern": { - "type": "string", - "description": "The glob pattern to match files against" - }, - "path": { - "type": "string", - "description": "The directory to search in" - } - }, - "required": ["pattern"] - } - }, - { - "name": "Grep", - "description": "A powerful search tool built on ripgrep.", - "input_schema": { - "type": "object", - "properties": { - "pattern": { - "type": "string", - "description": "The regular expression pattern to search for" - }, - "path": { - "type": "string", - "description": "File or directory to search in" - }, - "output_mode": { - "type": "string", - "enum": ["content", "files_with_matches", "count"], - "description": "Output mode" - } - }, - "required": ["pattern"] - } - }, - { - "name": "Edit", - "description": "Performs exact string replacements in files.", - "input_schema": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to modify" - }, - "old_string": { - "type": "string", - "description": "The text to replace" - }, - "new_string": { - "type": "string", - "description": "The text to replace it with" - }, - "replace_all": { - "type": "boolean", - "default": false, - "description": "Replace all occurrences" - } - }, - "required": ["file_path", "old_string", "new_string"] - } - }, - { - "name": "Write", - "description": "Writes a file to the local filesystem.", - "input_schema": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to write" - }, - "content": { - "type": "string", - "description": "The content to write to the file" - } - }, - "required": ["file_path", "content"] - } - } - ], - "tool_choice": { "type": "auto" } - }, - "response": { - "id": "msg_01XFDUDYJgAACzvnptvVoYEL", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I'll read the config file and find the test files in parallel." - }, - { - "type": "tool_use", - "id": "toolu_01UDAtfZkgcGYMBq7Ns84vfN", - "name": "Read", - "input": { - "file_path": "/home/user/project/config.toml" - } - }, - { - "type": "tool_use", - "id": "toolu_01J3KzMqBf9tXyQFhVw2NxRG", - "name": "Glob", - "input": { - "pattern": "**/*test*" - } - } - ], - "model": "claude-sonnet-4-20250514", - "stop_reason": "tool_use", - "stop_sequence": null, - "usage": { "input_tokens": 1200, "output_tokens": 85 } - } -} - - - -{ - "type": "error", - "error": { - "type": "invalid_request_error", - "message": "max_tokens: Field required" - } -} - - - -{ - "type": "error", - "error": { - "type": "rate_limit_error", - "message": "Number of request tokens has exceeded your per-minute rate limit" - }, - "request_id": "req_01XYZ" -} - - - -{ - "request": { - "model": "claude-opus-4-6", - "max_tokens": 256, - "system": "You are a concise assistant.", - "messages": [ - { "role": "user", "content": "Explain what an LLM proxy does." } - ] - }, - "response": { - "id": "msg_01XFDUDYJgAACzvnptvVoYEL", - "type": "message", - "role": "assistant", - "content": [{ "type": "text", "text": "An LLM proxy sits between a client and a language model API, translating requests and responses between different API formats." }], - "model": "claude-opus-4-6", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { "input_tokens": 25, "output_tokens": 30 } - } -} - - - -{ - "model": "claude-sonnet-4-6", - "max_tokens": 1024, - "messages": [ - { - "role": "user", - "content": "PLACEHOLDER_OVERSIZED" - } - ] -} - - - -{ - "request": { - "model": "claude-opus-4-6", - "max_tokens": 512, - "messages": [ - { "role": "user", "content": "What is the S&P 500 at today?" } - ], - "tools": [ - { - "name": "get_stock_price", - "description": "Get the latest price for an index or ticker.", - "input_schema": { - "type": "object", - "properties": { "ticker": { "type": "string" } }, - "required": ["ticker"] - } - } - ], - "tool_choice": { "type": "any" } - }, - "response": { - "id": "msg_01ABC123", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV", - "name": "get_stock_price", - "input": { "ticker": "^GSPC" } - } - ], - "model": "claude-opus-4-6", - "stop_reason": "tool_use", - "stop_sequence": null, - "usage": { "input_tokens": 50, "output_tokens": 20 } - } -} - - - -{ - "request": { - "model": "gpt-5.4", - "messages": [ - { "role": "developer", "content": "You are a concise assistant." }, - { "role": "user", "content": "Explain what an LLM proxy does." } - ], - "max_tokens": 256, - "temperature": null, - "stream": null - }, - "response": { - "id": "chatcmpl-abc123", - "object": "chat.completion", - "model": "gpt-5.4", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "An LLM proxy sits between a client and a language model API, translating requests and responses between different API formats." - }, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 25, - "completion_tokens": 30, - "total_tokens": 55 - } - } -} - - - -{ - "id": "chatcmpl-abc123", - "object": "chat.completion" -} - - - -{ - "request": { - "model": "gpt-5.4", - "messages": [ - { "role": "user", "content": "What is the S&P 500 at today?" } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_stock_price", - "description": "Get the latest price for an index or ticker.", - "parameters": { - "type": "object", - "properties": { "ticker": { "type": "string" } }, - "required": ["ticker"] - } - } - } - ], - "tool_choice": "required" - }, - "response": { - "id": "chatcmpl-def456", - "object": "chat.completion", - "model": "gpt-5.4", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "tool_calls": [ - { - "id": "call_xyz789", - "type": "function", - "function": { - "name": "get_stock_price", - "arguments": "{\"ticker\":\"^GSPC\"}" - } - } - ] - }, - "finish_reason": "tool_calls" - } - ], - "usage": { - "prompt_tokens": 50, - "completion_tokens": 15, - "total_tokens": 65 - } - } -} - - - -{ - "request": { - "model": "claude-sonnet-4-20250514", - "max_completion_tokens": 8096, - "messages": [ - { - "role": "user", - "content": "Read the main config file and list all test files." - } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "Read", - "description": "Reads a file from the local filesystem.", - "parameters": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to read" - }, - "offset": { - "type": "number", - "description": "The line number to start reading from" - }, - "limit": { - "type": "number", - "description": "The number of lines to read" - } - }, - "required": ["file_path"] - } - } - }, - { - "type": "function", - "function": { - "name": "Bash", - "description": "Executes a given bash command and returns its output.", - "parameters": { - "type": "object", - "properties": { - "command": { - "type": "string", - "description": "The command to execute" - }, - "description": { - "type": "string", - "description": "Clear, concise description of what this command does" - }, - "timeout": { - "type": "number", - "description": "Optional timeout in milliseconds (max 600000)" - } - }, - "required": ["command"] - } - } - }, - { - "type": "function", - "function": { - "name": "Glob", - "description": "Fast file pattern matching tool that works with any codebase size.", - "parameters": { - "type": "object", - "properties": { - "pattern": { - "type": "string", - "description": "The glob pattern to match files against" - }, - "path": { - "type": "string", - "description": "The directory to search in" - } - }, - "required": ["pattern"] - } - } - }, - { - "type": "function", - "function": { - "name": "Grep", - "description": "A powerful search tool built on ripgrep.", - "parameters": { - "type": "object", - "properties": { - "pattern": { - "type": "string", - "description": "The regular expression pattern to search for" - }, - "path": { - "type": "string", - "description": "File or directory to search in" - }, - "output_mode": { - "type": "string", - "enum": ["content", "files_with_matches", "count"], - "description": "Output mode" - } - }, - "required": ["pattern"] - } - } - }, - { - "type": "function", - "function": { - "name": "Edit", - "description": "Performs exact string replacements in files.", - "parameters": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to modify" - }, - "old_string": { - "type": "string", - "description": "The text to replace" - }, - "new_string": { - "type": "string", - "description": "The text to replace it with" - }, - "replace_all": { - "type": "boolean", - "default": false, - "description": "Replace all occurrences" - } - }, - "required": ["file_path", "old_string", "new_string"] - } - } - }, - { - "type": "function", - "function": { - "name": "Write", - "description": "Writes a file to the local filesystem.", - "parameters": { - "type": "object", - "properties": { - "file_path": { - "type": "string", - "description": "The absolute path to the file to write" - }, - "content": { - "type": "string", - "description": "The content to write to the file" - } - }, - "required": ["file_path", "content"] - } - } - } - ], - "tool_choice": "auto" - }, - "response": { - "id": "chatcmpl-llama001", - "object": "chat.completion", - "model": "llama-3.3-70b", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "I'll read the config file and find the test files in parallel.", - "tool_calls": [ - { - "id": "call_read_001", - "type": "function", - "function": { - "name": "Read", - "arguments": "{\"file_path\":\"/home/user/project/config.toml\"}" - } - }, - { - "id": "call_glob_001", - "type": "function", - "function": { - "name": "Glob", - "arguments": "{\"pattern\":\"**/*test*\"}" - } - } - ] - }, - "finish_reason": "tool_calls" - } - ], - "usage": { - "prompt_tokens": 1200, - "completion_tokens": 85, - "total_tokens": 1285 - } - } -} - - - -{ - "error": { - "message": "Incorrect API key provided: sk-...1234. You can find your API key at https://platform.openai.com/account/api-keys.", - "type": "invalid_request_error", - "param": null, - "code": "invalid_api_key" - } -} - - - -{ - "error": { - "message": "Rate limit reached for gpt-4o in organization org-xxx on tokens per min (TPM): Limit 30000, Used 28000, Requested 5000.", - "type": "tokens", - "param": null, - "code": "rate_limit_exceeded" - } -} - - - -{ - "error": { - "message": "The server had an error while processing your request. Sorry about that!", - "type": "server_error", - "param": null, - "code": null - } -} - - - -ralph/anthropic-domain-types - - - -#!/usr/bin/env python3 -"""Condense an OpenAPI spec into an LLM-readable summary. - -Downloads the full spec, strips examples/metadata/verbose descriptions, -and outputs a compact YAML or JSON with endpoints and schema outlines. -Supports filtering by tag to get only relevant sections. - -Requires: pip install pyyaml -""" -⋮---- -DEFAULT_URL = ( -⋮---- -def parse_args() -> argparse.Namespace -⋮---- -p = argparse.ArgumentParser( -⋮---- -def download_spec(url: str, cache_dir: str, no_cache: bool) -> str -⋮---- -"""Download spec with local file caching.""" -cache_path = Path(cache_dir) -# Derive a simple filename from the URL -filename = url.rstrip("/").rsplit("/", 1)[-1] -cached = cache_path / filename -⋮---- -data = resp.read().decode("utf-8") -⋮---- -def short_ref(ref: str) -> str -⋮---- -"""#/components/schemas/Foo -> $Foo, #/components/parameters/Bar -> $params.Bar""" -⋮---- -# Fallback: just the last segment -⋮---- -def extract_ref(obj: dict) -> str | None -⋮---- -"""Pull $ref from a dict, return short form or None.""" -ref = obj.get("$ref") -⋮---- -def raw_ref(obj: dict) -> str | None -⋮---- -"""Pull raw schema name from $ref.""" -ref = obj.get("$ref", "") -⋮---- -def truncate_desc(desc: str | None, max_len: int = 120) -> str | None -⋮---- -"""Keep first sentence, cap at max_len.""" -⋮---- -# First sentence: split on ". " or ".\n" -⋮---- -idx = desc.find(sep) -⋮---- -desc = desc[: idx + 1] -⋮---- -desc = desc.strip() -⋮---- -desc = desc[: max_len - 3] + "..." -⋮---- -def condense_property(prop: dict) -> dict -⋮---- -"""Condense a single schema property to 1-level summary.""" -# Direct $ref -ref = extract_ref(prop) -⋮---- -result = {} -⋮---- -# oneOf / anyOf / allOf at property level -⋮---- -variants = [] -⋮---- -r = extract_ref(v) -⋮---- -ptype = prop.get("type") -⋮---- -# Array items -⋮---- -items = prop["items"] -iref = extract_ref(items) -⋮---- -# Enum values (compact, important for translation work) -⋮---- -# Nullable -⋮---- -# Default (only if simple scalar) -default = prop.get("default") -⋮---- -def condense_schema(name: str, schema: dict) -> dict -⋮---- -"""Condense a schema to type, description, required, 1-level properties.""" -⋮---- -# Handle composition schemas (no own type, just oneOf/anyOf/allOf) -⋮---- -stype = schema.get("type") -⋮---- -desc = truncate_desc(schema.get("description")) -⋮---- -required = schema.get("required") -⋮---- -# Properties (1-level) -props = schema.get("properties", {}) -⋮---- -condensed_props = {} -⋮---- -# Array items at schema level -⋮---- -items = schema["items"] -⋮---- -# additionalProperties -addl = schema.get("additionalProperties") -⋮---- -r = extract_ref(addl) -⋮---- -def extract_body_ref(operation: dict) -> str | None -⋮---- -"""Extract request body schema ref from an operation.""" -rb = operation.get("requestBody", {}) -⋮---- -# Direct $ref on requestBody -ref = extract_ref(rb) -⋮---- -content = rb.get("content", {}) -⋮---- -mt = content.get(media_type, {}) -schema = mt.get("schema", {}) -ref = extract_ref(schema) -⋮---- -def extract_response_refs(operation: dict) -> dict[str, str] -⋮---- -"""Extract response schema refs from an operation.""" -responses = operation.get("responses", {}) -⋮---- -# Response-level $ref -ref = extract_ref(resp) -⋮---- -content = resp.get("content", {}) -⋮---- -def condense_endpoint(path: str, method: str, operation: dict) -> dict -⋮---- -"""Condense a single endpoint.""" -ep = { -⋮---- -op_id = operation.get("operationId") -⋮---- -summary = truncate_desc(operation.get("summary")) -⋮---- -tags = operation.get("tags", []) -⋮---- -# Parameters -params = operation.get("parameters", []) -⋮---- -condensed_params = [] -⋮---- -# Could be a $ref to a shared parameter -ref = extract_ref(p) -⋮---- -cp = {"name": p.get("name", "?"), "in": p.get("in", "?")} -schema = p.get("schema", {}) -⋮---- -body_ref = extract_body_ref(operation) -⋮---- -resp_refs = extract_response_refs(operation) -⋮---- -def collect_refs_from_schema(schema: dict) -> set[str] -⋮---- -"""Collect all raw schema names referenced by a schema dict.""" -refs = set() -⋮---- -def _walk(obj) -⋮---- -r = raw_ref(obj) -⋮---- -def collect_reachable_schemas(roots: set[str], all_schemas: dict) -> set[str] -⋮---- -"""BFS from root schema names to find all transitively referenced schemas.""" -visited = set() -frontier = set(roots) -⋮---- -current = frontier.pop() -⋮---- -schema = all_schemas.get(current, {}) -child_refs = collect_refs_from_schema(schema) -⋮---- -def collect_endpoint_schema_roots(endpoint: dict) -> set[str] -⋮---- -"""Extract raw schema names from a condensed endpoint's refs.""" -roots = set() -⋮---- -def _extract(val) -⋮---- -name = val.lstrip("$") -⋮---- -def condense_spec(raw_yaml: str, tags: list[str] | None) -> dict -⋮---- -"""Main orchestrator: parse, filter, condense.""" -spec = yaml.safe_load(raw_yaml) -⋮---- -info = spec.get("info", {}) -all_schemas = spec.get("components", {}).get("schemas", {}) -paths = spec.get("paths", {}) -⋮---- -# Normalize tag filter to lowercase -tag_filter = None -⋮---- -tag_filter = {t.lower() for t in tags} -⋮---- -# Condense endpoints -endpoints = [] -⋮---- -operation = methods.get(method) -⋮---- -# Tag filter -⋮---- -op_tags = [t.lower() for t in operation.get("tags", [])] -⋮---- -# Collect schema roots from filtered endpoints -schema_roots = set() -⋮---- -# If no tag filter, include all schemas -⋮---- -reachable = collect_reachable_schemas(schema_roots, all_schemas) -⋮---- -reachable = set(all_schemas.keys()) -⋮---- -# Condense schemas (sorted for stable output) -schemas = {} -⋮---- -total_endpoints = sum( -⋮---- -result = { -⋮---- -# Custom YAML representer to keep output clean -def _str_representer(dumper, data) -⋮---- -"""Use literal block style for multiline strings, plain otherwise.""" -⋮---- -def _ordered_dict_representer(dumper, data) -⋮---- -class CleanDumper(yaml.SafeDumper) -⋮---- -def format_output(data: dict, as_json: bool) -> str -⋮---- -"""Format the condensed spec as YAML or JSON.""" -⋮---- -meta = data["meta"] -header_lines = [ -⋮---- -body = yaml.dump( -⋮---- -def main() -⋮---- -args = parse_args() -⋮---- -raw = download_spec(args.url, args.cache_dir, args.no_cache) -condensed = condense_spec(raw, args.tags) -output = format_output(condensed, args.json) -⋮---- -# Print size stats -lines = output.count("\n") - - - -# PRD: Phase 9 - Compatibility Endpoints - -## Introduction - -Implement additional Anthropic API endpoints beyond `/v1/messages` to improve SDK compatibility: `GET /v1/models` for model listing, `POST /v1/messages/count_tokens` for token estimation, and `POST /v1/messages/batches` as an explicit unsupported endpoint. These endpoints round out the API surface so Anthropic SDKs work without unexpected 404s. - -## Goals - -- Implement `GET /v1/models` returning a model list (proxied from OpenAI or static mapping) -- Implement `POST /v1/messages/count_tokens` with local approximation or explicit unsupported error -- Implement `POST /v1/messages/batches` returning explicit unsupported error -- Return proper Anthropic error shapes for unknown routes - -## User Stories - -### US-001: Models list endpoint -**Description:** As a client using the Anthropic SDK, I need `GET /v1/models` to return a list of available models so model discovery works. - -**Acceptance Criteria:** -- [ ] `GET /v1/models` returns 200 with JSON body matching Anthropic models list shape -- [ ] Option A (proxy): fetch OpenAI models and translate names, or -- [ ] Option B (static): return configured model mapping from config/env -- [ ] Response shape: `{data: [{id, display_name, type, ...}], ...}` matching Anthropic format -- [ ] Test: endpoint returns valid model list - -### US-002: Count tokens endpoint -**Description:** As a client using the Anthropic SDK, I need `POST /v1/messages/count_tokens` to either return an approximation or a clear error so the client handles it gracefully. - -**Acceptance Criteria:** -- [ ] `POST /v1/messages/count_tokens` accepts same body as `/v1/messages` -- [ ] Option A (approximate): return rough token count based on character heuristic -- [ ] Option B (unsupported): return 501 or Anthropic error with clear message -- [ ] Response shape matches Anthropic's count_tokens response if implemented -- [ ] Test: endpoint returns expected response or error - -### US-003: Batches endpoint (unsupported) -**Description:** As a client, I need `POST /v1/messages/batches` to return a clear error rather than a generic 404. - -**Acceptance Criteria:** -- [ ] `POST /v1/messages/batches` returns Anthropic error: `{type: "error", error: {type: "not_found_error", message: "Batch API is not supported by this proxy"}}` -- [ ] HTTP status: 404 or 501 -- [ ] Test: endpoint returns expected error shape - -### US-004: Unknown route handling -**Description:** As a client, I need unknown routes to return Anthropic-shaped 404 errors instead of axum defaults. - -**Acceptance Criteria:** -- [ ] Any unmatched route returns `{type: "error", error: {type: "not_found_error", message: "Not found"}}` with 404 status -- [ ] Response has `Content-Type: application/json` -- [ ] Test: GET /v1/nonexistent returns Anthropic 404 - -## Functional Requirements - -- FR-1: `GET /v1/models` returns a valid response (proxied or static) -- FR-2: `POST /v1/messages/count_tokens` returns approximation or explicit unsupported error -- FR-3: `POST /v1/messages/batches` returns explicit unsupported error -- FR-4: All error responses use Anthropic error shape -- FR-5: Fallback handler catches unmatched routes - -## Non-Goals - -- No actual batch processing implementation -- No accurate tokenization (would require a tokenizer library) -- No Anthropic Files API endpoints -- No Anthropic Skills API endpoints - -## Technical Considerations - -- For static model mapping, consider a config file or env var listing supported models -- If proxying OpenAI models, translate model names (e.g., `gpt-4o` -> a mapped Anthropic-style name) or expose OpenAI names directly -- axum fallback handler for unknown routes: `Router::fallback()` -- count_tokens approximation: ~4 chars per token is a rough heuristic, clearly documented as approximate - -## Success Metrics - -- Integration tests for all three endpoints -- Unknown route test returns Anthropic-shaped 404 -- `cargo test` passes - - - -# PRD: Phase 11 - End-to-End Validation - -## Introduction - -Validate the complete proxy against the full Anthropic API surface. Run the entire test suite, verify fixtures against current API documentation, check the compatibility contract, and optionally run live API tests against real OpenAI endpoints. This phase is about confidence, not new features. - -## Goals - -- Full test suite passes: unit, integration, and fixture tests across both crates -- Fixture files validated against current Anthropic and OpenAI API docs -- Compatibility checklist verified (what works, what is approximated, what is rejected) -- End-to-end flows tested: basic text, tool calling, streaming, files, errors -- Optional: live API test against real OpenAI endpoint - -## User Stories - -### US-001: Full test suite green -**Description:** As a developer preparing for release, I need every test in the project to pass, confirming no regressions across all phases. - -**Acceptance Criteria:** -- [ ] `cargo test` passes with zero failures -- [ ] `cargo clippy -- -D warnings` passes with zero warnings -- [ ] `cargo fmt --check` passes -- [ ] No ignored tests without documented reason - -### US-002: Fixture validation -**Description:** As a developer, I need fixture files checked against current API documentation to confirm they reflect real API shapes. - -**Acceptance Criteria:** -- [ ] Review all `fixtures/anthropic/*.json` against current Anthropic Messages API docs -- [ ] Review all `fixtures/openai/*.json` against current OpenAI Chat Completions docs -- [ ] Update any fixtures where API shapes have changed -- [ ] Document any known fixture deviations with comments - -### US-003: Compatibility checklist -**Description:** As an operator, I need a clear compatibility matrix documenting what the proxy supports, what it approximates, and what it rejects. - -**Acceptance Criteria:** -- [ ] Basic messages (text): verified working -- [ ] Streaming text: verified working -- [ ] Tool use (function calling): verified working -- [ ] Image content blocks: verified working -- [ ] Document (PDF) content blocks: verified with status noted -- [ ] Token counting endpoint: status documented (approximate or unsupported) -- [ ] Batch endpoint: documented as unsupported -- [ ] Stop reason mapping: all cases documented -- [ ] Checklist written to `docs/compatibility.md` - -### US-004: E2E flow tests -**Description:** As a developer, I need integration tests that exercise full request/response flows through the proxy with mocked upstream. - -**Acceptance Criteria:** -- [ ] E2E test: basic text message -> response with text content -- [ ] E2E test: message with tools -> response with tool_use -> follow-up with tool_result -> final text -- [ ] E2E test: streaming text message -> correct SSE event sequence -- [ ] E2E test: streaming with tool calls -> correct SSE event sequence -- [ ] E2E test: image content block in request -> correct upstream payload -- [ ] E2E test: upstream 429 -> Anthropic-shaped error (or retry succeeds) -- [ ] E2E test: upstream 500 -> Anthropic-shaped error -- [ ] E2E test: auth failure -> 401 - -### US-005: Optional live API test -**Description:** As a developer with an OpenAI API key, I optionally want to run tests against the real OpenAI API to catch any mismatch between fixtures and reality. - -**Acceptance Criteria:** -- [ ] Live tests gated behind `#[ignore]` attribute or feature flag -- [ ] Requires `OPENAI_API_KEY` env var to run -- [ ] Tests: basic completion, streaming completion, tool call -- [ ] Clearly documented how to run: `cargo test -- --ignored` or `cargo test --features live-tests` -- [ ] Test output includes model used and response validation - -## Functional Requirements - -- FR-1: `cargo test` with no flags runs all non-live tests -- FR-2: `cargo clippy -- -D warnings` treats all warnings as errors -- FR-3: Compatibility checklist is a markdown file in the repo -- FR-4: E2E tests use the same mocking infrastructure as Phase 6/7 integration tests -- FR-5: Live tests are opt-in and never run in CI by default - -## Non-Goals - -- No performance benchmarking (separate concern) -- No load testing -- No fuzzing (mentioned in PLAN.md CI section, but separate task) -- No deployment automation - -## Technical Considerations - -- Use `wiremock` or `httpmock` for E2E mock server -- Live tests should use cheap models (e.g., `gpt-4o-mini`) to minimize cost -- Live tests should have reasonable timeouts (30s per test) -- Consider a test helper that starts the proxy on a random port with a mock backend - -## Success Metrics - -- `cargo test` exits 0 with all tests passing -- `cargo clippy -- -D warnings` exits 0 -- `cargo fmt --check` exits 0 -- Compatibility checklist document exists and is accurate -- E2E tests cover all major flows - - - -# PRD: Phase 8 - Files and Document Blocks - -## Introduction - -Add support for translating Anthropic document and image content blocks to OpenAI's equivalent input formats. Anthropic clients can send PDFs as base64 document blocks and images as base64 or URL sources; these must be translated to OpenAI's `input_file.file_data` (Responses API) or structured content parts (Chat Completions). - -## Goals - -- Translate Anthropic `document` content blocks (base64 PDF) to OpenAI format -- Translate Anthropic `image` content blocks (base64 and URL) to OpenAI format -- Enforce the 32MB size limit on file content -- Support mixed content messages (text + images + documents) - -## User Stories - -### US-001: Image block translation -**Description:** As a client sending vision requests, I need image content blocks translated so OpenAI models can process them. - -**Acceptance Criteria:** -- [ ] Anthropic `{type: "image", source: {type: "base64", media_type: "image/png", data: "..."}}` -> OpenAI Chat Completions `{type: "image_url", image_url: {url: "data:image/png;base64,..."}}` -- [ ] Anthropic `{type: "image", source: {type: "url", url: "https://..."}}` -> OpenAI `{type: "image_url", image_url: {url: "https://..."}}` -- [ ] Supported media types: `image/png`, `image/jpeg`, `image/gif`, `image/webp` -- [ ] Test: base64 image, URL image, different media types - -### US-002: Document block translation -**Description:** As a client sending PDF documents, I need document blocks translated to OpenAI's file input format. - -**Acceptance Criteria:** -- [ ] Anthropic `{type: "document", source: {type: "base64", media_type: "application/pdf", data: "..."}}` -> OpenAI Responses `input_file` with `file_data` containing the base64 data -- [ ] If using Chat Completions backend (no native file support), return an informative error or attempt best-effort text extraction note -- [ ] Test: PDF base64 document block translation - -### US-003: Size limit enforcement -**Description:** As an operator, I need oversized file content rejected before it reaches OpenAI. - -**Acceptance Criteria:** -- [ ] Base64 content decoded size checked against 32MB limit -- [ ] Oversized content -> 413 `request_too_large` error with descriptive message -- [ ] Check happens during translation, before sending to OpenAI -- [ ] Test: content just under limit passes, content over limit rejected - -### US-004: Mixed content messages -**Description:** As a client, I need to send messages with text, images, and documents interleaved in a single message. - -**Acceptance Criteria:** -- [ ] User message with [text, image, text] -> OpenAI message with corresponding content parts array -- [ ] User message with [text, document, text] -> appropriate handling per backend -- [ ] Content block ordering preserved -- [ ] Test: mixed content message with multiple block types - -## Functional Requirements - -- FR-1: Image translation supports both base64 and URL source types -- FR-2: Document translation targets OpenAI Responses API `input_file.file_data` format -- FR-3: Size validation runs before forwarding to prevent wasting upstream bandwidth -- FR-4: Unsupported media types produce clear error messages - -## Non-Goals - -- No file upload to OpenAI Files API (upload + reference workflow) -- No Anthropic beta Files API support -- No text extraction from PDFs -- No image resizing or format conversion - -## Technical Considerations - -- Base64 data URLs for Chat Completions use format `data:{media_type};base64,{data}` -- Size check: base64 string length * 3/4 gives approximate decoded size -- Document support may require the Responses API backend; if Chat Completions is the only backend, document blocks should produce a clear error -- Consider feature-gating document support behind a config flag - -## Success Metrics - -- Fixture tests for image and document block translation -- Integration test: request with image block produces correct OpenAI payload -- Size limit test: oversized content rejected with 413 -- `cargo test` passes - - - -# PRD: Phase 10 - Hardening, Security, Observability - -## Introduction - -Harden the proxy for production use: add retry logic with backoff for transient upstream errors, header filtering and secret redaction, structured logging with request ID correlation, metrics collection, SSRF protection for URL inputs, and concurrency limits. This phase turns a working proxy into a production-grade one. - -## Goals - -- Implement retry with exponential backoff for 429 and 5xx upstream errors -- Filter and redact sensitive headers from logs and forwarded requests -- Add request ID correlation across all log entries -- Add basic metrics (request count, latency, error rate) -- Protect against SSRF when handling URL-based inputs -- Add concurrency limits to prevent self-DoS - -## User Stories - -### US-001: Retry with backoff -**Description:** As an operator, I need the proxy to retry transient OpenAI errors instead of immediately failing, so brief upstream issues don't cascade to clients. - -**Acceptance Criteria:** -- [ ] Retry on HTTP 429 (rate limited) and 5xx (server errors) -- [ ] Respect `Retry-After` header if present -- [ ] Exponential backoff: 1s, 2s, 4s (3 attempts max) -- [ ] Do NOT retry on 4xx (except 429) or client errors -- [ ] Do NOT retry streaming requests (only non-streaming) -- [ ] Log each retry attempt with attempt number and backoff duration -- [ ] Test: mock upstream returning 429 then 200, verify retry succeeds - -### US-002: Header filtering -**Description:** As a security engineer, I need sensitive headers stripped from forwarded requests and responses so secrets don't leak across trust boundaries. - -**Acceptance Criteria:** -- [ ] Inbound `x-api-key` is NOT forwarded to OpenAI -- [ ] Inbound `anthropic-version` and `anthropic-beta` are NOT forwarded -- [ ] OpenAI `Authorization` header is NOT included in response to client -- [ ] OpenAI rate limit headers (`x-ratelimit-*`) optionally translated to Anthropic format or stripped -- [ ] Test: verify no sensitive headers in outbound request or response - -### US-003: Secret redaction in logs -**Description:** As an operator, I need API keys, auth tokens, and other secrets redacted in all log output so log aggregation is safe. - -**Acceptance Criteria:** -- [ ] `Authorization: Bearer sk-...` redacted to `Authorization: Bearer sk-...REDACTED` -- [ ] `x-api-key` value redacted in any log line -- [ ] Request/response body logging (if enabled) redacts bearer tokens -- [ ] Redaction uses the `util/redact.rs` module from translator crate -- [ ] Test: log output does not contain raw API keys - -### US-004: Request ID correlation -**Description:** As an operator debugging production issues, I need every log line for a request to include the same request ID so I can trace a request end to end. - -**Acceptance Criteria:** -- [ ] Request ID (from Phase 6 middleware) is in every tracing span -- [ ] Log lines from upstream client call include the request ID -- [ ] Request ID sent to OpenAI as a custom header for upstream correlation -- [ ] Test: verify request ID appears in structured log output - -### US-005: Metrics collection -**Description:** As an operator, I need basic request metrics so I can monitor proxy health and performance. - -**Acceptance Criteria:** -- [ ] Track: total request count, request latency histogram, error count by status code -- [ ] Metrics stored in-memory (atomic counters / histogram) -- [ ] `GET /metrics` endpoint exposing current values (JSON or Prometheus format) -- [ ] Latency measured from request receipt to response completion -- [ ] Test: make requests, verify metrics reflect them - -### US-006: SSRF protection -**Description:** As a security engineer, I need URL-based inputs (image URLs, file URLs) validated against SSRF before the proxy fetches them. - -**Acceptance Criteria:** -- [ ] Block URLs pointing to private IP ranges (10.x, 172.16-31.x, 192.168.x, 127.x, ::1, link-local) -- [ ] Block URLs with non-HTTP(S) schemes -- [ ] Block URLs resolving to private IPs (DNS rebinding protection via resolution before fetch) -- [ ] Configurable allowlist for internal URLs if needed -- [ ] Test: private IP URL blocked, public URL allowed - -### US-007: Concurrency limits -**Description:** As an operator, I need the proxy to limit concurrent requests so a burst of traffic doesn't exhaust connections or memory. - -**Acceptance Criteria:** -- [ ] Configurable max concurrent requests (default: 256) -- [ ] Requests exceeding the limit -> 429 `rate_limit_error` with `Retry-After` header -- [ ] Uses tower `ConcurrencyLimit` or semaphore -- [ ] Test: exceed limit, verify 429 response - -## Functional Requirements - -- FR-1: Retry logic only applies to non-streaming requests -- FR-2: Header filtering is applied in both directions (client->proxy and proxy->client) -- FR-3: Secret redaction covers all log levels (debug, info, warn, error) -- FR-4: Metrics endpoint does not require authentication -- FR-5: SSRF protection applies to any URL extracted from request content -- FR-6: Concurrency limit applies to all routes except `/health` and `/metrics` - -## Non-Goals - -- No distributed tracing (OpenTelemetry export) -- No rate limiting per API key (simple concurrency limit only) -- No mutual TLS -- No audit logging to external systems -- No WAF-style request inspection - -## Technical Considerations - -- Use `tower::retry` or manual retry loop with `tokio::time::sleep` -- Parse `Retry-After` as either seconds (integer) or HTTP date -- For SSRF: resolve DNS before connecting, check resolved IPs against blocklist -- Metrics: consider `metrics` crate with `metrics-exporter-prometheus` or simple atomic counters -- Concurrency: `tower::limit::ConcurrencyLimitLayer` integrates cleanly with axum - -## Success Metrics - -- Retry test: 429 followed by 200 succeeds without client seeing the 429 -- Header filtering test: no sensitive headers leak -- SSRF test: private IP URLs rejected -- Concurrency test: limit enforced correctly -- `cargo clippy -- -D warnings` passes -- `cargo test` passes - - - -# PRD: Phase 6 - Proxy Server and Routing - -## Introduction - -Wire the translation logic into a working HTTP proxy. Implement the axum routes, middleware (auth, request ID, size limits, logging), the reqwest-based OpenAI client, and the non-streaming `POST /v1/messages` endpoint. This is where the translator library meets the network. - -## Goals - -- Implement `POST /v1/messages` (non-streaming) as a full request/response proxy -- Add authentication middleware that validates `x-api-key` headers -- Add request ID generation and correlation -- Enforce 32MB request size limit -- Implement the OpenAI backend client using reqwest -- Handle upstream errors and translate them to Anthropic error shapes - -## User Stories - -### US-001: POST /v1/messages route (non-streaming) -**Description:** As a client using the Anthropic SDK, I need to POST to `/v1/messages` and get back an Anthropic-shaped response, even though the proxy is calling OpenAI behind the scenes. - -**Acceptance Criteria:** -- [ ] `POST /v1/messages` accepts JSON body matching `AnthropicMessageCreateRequest` -- [ ] Request with `stream: false` or `stream` absent triggers non-streaming path -- [ ] Response is `AnthropicMessageCreateResponse` JSON with correct content-type -- [ ] Request is translated to OpenAI, sent to OpenAI, response translated back -- [ ] Returns 200 on success with valid Anthropic response shape - -### US-002: Configuration -**Description:** As an operator, I need the proxy to read configuration from environment variables so I can deploy it without code changes. - -**Acceptance Criteria:** -- [ ] `OPENAI_API_KEY`: required, used in `Authorization: Bearer` header to OpenAI -- [ ] `OPENAI_BASE_URL`: optional, defaults to `https://api.openai.com` -- [ ] `LISTEN_PORT`: optional, defaults to `3000` -- [ ] `RUST_LOG`: optional, controls tracing filter -- [ ] Missing `OPENAI_API_KEY` at startup logs a warning (still starts, fails on requests) - -### US-003: Authentication middleware -**Description:** As a developer, I need the proxy to validate that incoming requests include the `x-api-key` header, rejecting unauthenticated requests with the correct Anthropic error shape. - -**Acceptance Criteria:** -- [ ] Requests without `x-api-key` header -> 401 `authentication_error` response -- [ ] Requests with empty `x-api-key` -> 401 `authentication_error` response -- [ ] The `x-api-key` value is NOT forwarded to OpenAI (proxy uses its own configured key) -- [ ] `anthropic-version` header is accepted but not enforced (log if missing) -- [ ] Test: missing auth -> 401, present auth -> passes through - -### US-004: Request ID middleware -**Description:** As an operator debugging issues, I need every request to have a unique ID that appears in logs and response headers. - -**Acceptance Criteria:** -- [ ] Generate UUID v4 request ID for each incoming request -- [ ] If client sends `x-request-id` or `request-id` header, use that instead -- [ ] Include request ID in response header `request-id` -- [ ] Request ID available in tracing span for all log lines -- [ ] Test: response includes `request-id` header - -### US-005: Request size limit -**Description:** As an operator, I need requests larger than 32MB rejected to match Anthropic's documented limit and prevent memory issues. - -**Acceptance Criteria:** -- [ ] Requests with `Content-Length` > 32MB -> 413 `request_too_large` error -- [ ] Error response matches Anthropic error shape -- [ ] Test: oversized request gets 413 - -### US-006: OpenAI backend client -**Description:** As a developer, I need a reqwest client that sends translated requests to OpenAI and returns responses. - -**Acceptance Criteria:** -- [ ] `OpenAIClient` struct wrapping `reqwest::Client` -- [ ] `send_chat_completion(&self, req: OpenAIChatCompletionRequest) -> Result` -- [ ] Sets `Authorization: Bearer {api_key}` header -- [ ] Sets `Content-Type: application/json` -- [ ] Configurable base URL (`{base_url}/v1/chat/completions`) -- [ ] Timeouts: connect 10s, read 120s -- [ ] Returns typed error for HTTP errors, connection failures, JSON parse failures - -### US-007: Upstream error handling -**Description:** As a developer, I need upstream OpenAI errors translated to Anthropic error shapes so clients see consistent error responses. - -**Acceptance Criteria:** -- [ ] OpenAI 4xx/5xx -> translated to Anthropic error using `errors_map` from Phase 4 -- [ ] OpenAI error body parsed and message included in Anthropic error -- [ ] Connection timeout -> Anthropic `api_error` (500) -- [ ] JSON parse error -> Anthropic `api_error` (500) with descriptive message -- [ ] Test: mock upstream returning 429, verify proxy returns Anthropic-shaped 429 - -### US-008: Logging middleware -**Description:** As an operator, I need structured request/response logging for observability. - -**Acceptance Criteria:** -- [ ] Log on request: method, path, request ID, content length -- [ ] Log on response: status code, request ID, latency -- [ ] Use `tracing` with structured fields (not string interpolation) -- [ ] Sensitive headers (`x-api-key`, `Authorization`) are NOT logged -- [ ] Controlled by `RUST_LOG` env var - -## Functional Requirements - -- FR-1: `POST /v1/messages` with `stream: false` performs full translate-forward-translate cycle -- FR-2: `GET /health` continues to return 200 (from Phase 1) -- FR-3: All error responses match Anthropic `{type: "error", error: {type, message}}` shape -- FR-4: Request size limit enforced at 32MB -- FR-5: Middleware ordering: size limit -> auth -> request ID -> logging -> route handler - -## Non-Goals - -- No streaming support (Phase 7) -- No model name aliasing/mapping -- No retry logic (Phase 10) -- No rate limit header translation (Phase 10) -- No `/v1/models` or other compatibility endpoints (Phase 9) - -## Technical Considerations - -- Use axum's `DefaultBodyLimit` for size enforcement -- Use `tower` layers for middleware composition -- reqwest client should be shared (connection pooling) via axum state -- Consider `axum::extract::State` for sharing config and client -- Use `tracing::instrument` for automatic span creation on route handlers - -## Success Metrics - -- Integration test: POST valid Anthropic request to proxy with mocked OpenAI backend, get valid Anthropic response -- Integration test: POST without auth -> 401 -- Integration test: POST oversized body -> 413 -- Integration test: Upstream error -> correct Anthropic error shape -- `cargo test` passes for both crates - - - -# PRD: Phase 7 - Streaming SSE Translation - -## Introduction - -Implement streaming support for `POST /v1/messages` with `stream: true`. The proxy must parse OpenAI's streaming chunks (Chat Completions delta format), translate them into Anthropic's SSE event sequence (`message_start` -> `content_block_start` -> `content_block_delta` -> `content_block_stop` -> `message_delta` -> `message_stop`), and deliver them to the client. This is the highest-complexity translation in the project. - -## Goals - -- Parse OpenAI Chat Completions streaming chunks (`chat.completion.chunk` + `data: [DONE]`) -- Emit Anthropic SSE events in the correct documented order -- Implement a streaming state machine that tracks content blocks and accumulates usage -- Handle backpressure via bounded channels -- Handle client disconnect gracefully (abort upstream) -- Support tool call streaming (`input_json_delta`) - -## User Stories - -### US-001: Anthropic SSE emitter -**Description:** As a developer, I need a component that emits properly formatted Anthropic SSE events so streaming clients receive the exact event shapes they expect. - -**Acceptance Criteria:** -- [ ] Emits `event: message_start` with initial message skeleton (empty content, null stop_reason, initial usage) -- [ ] Emits `event: content_block_start` with block index and type -- [ ] Emits `event: content_block_delta` with `text_delta` or `input_json_delta` -- [ ] Emits `event: content_block_stop` when a content block is complete -- [ ] Emits `event: message_delta` with `stop_reason` and final `usage` -- [ ] Emits `event: message_stop` as the final event -- [ ] Each event is formatted as `event: {type}\ndata: {json}\n\n` -- [ ] Test: emit a complete event sequence, verify output matches PLAN.md lines 392-408 - -### US-002: OpenAI Chat Completions chunk parser -**Description:** As a developer, I need a parser that reads OpenAI SSE stream lines and produces typed chunk objects. - -**Acceptance Criteria:** -- [ ] Parses `data: {json}` lines into `OpenAIChatCompletionChunk` structs -- [ ] Handles `data: [DONE]` as stream termination signal -- [ ] `OpenAIChatCompletionChunk` has: `id`, `object`, `model`, `choices` (with `delta` and `finish_reason`), optional `usage` -- [ ] `Delta` struct: optional `role`, `content`, `tool_calls` -- [ ] Ignores empty lines and comment lines (`:` prefix) -- [ ] Test: parse a sequence of chunk lines, verify extracted deltas - -### US-003: Streaming state machine -**Description:** As a developer, I need a state machine in `streaming_map.rs` that transforms a sequence of OpenAI chunks into a sequence of Anthropic SSE events, maintaining the correct block tracking. - -**Acceptance Criteria:** -- [ ] Tracks current content block index (starts at 0) -- [ ] On first text delta: emits `content_block_start(index=0, type=text)` then `content_block_delta` -- [ ] On subsequent text deltas: emits `content_block_delta` only -- [ ] On tool call delta: starts new content block for each tool call, emits `content_block_start(type=tool_use)` then `input_json_delta` deltas -- [ ] On finish_reason received: emits `content_block_stop` for current block, then `message_delta` with mapped stop reason -- [ ] On stream end: emits `message_stop` -- [ ] Accumulates usage from final usage chunk if `stream_options.include_usage` was set -- [ ] Test: text-only stream, tool call stream, mixed text+tool stream - -### US-004: Backpressure and bounded channel -**Description:** As a developer, I need the upstream reader and downstream SSE writer connected by a bounded channel so slow clients don't cause unbounded memory growth. - -**Acceptance Criteria:** -- [ ] Bounded `mpsc` channel (capacity ~64 events) between upstream reader task and SSE response stream -- [ ] If channel is full, upstream reader awaits (backpressure) -- [ ] SSE response uses `axum::response::Sse` with `ReceiverStream` -- [ ] Test: verify bounded channel does not drop events under normal conditions - -### US-005: Client disconnect handling -**Description:** As an operator, I need the proxy to stop reading from OpenAI when a client disconnects, freeing resources. - -**Acceptance Criteria:** -- [ ] When the downstream SSE connection drops, the channel receiver is dropped -- [ ] The upstream reader task detects sender failure and aborts -- [ ] Upstream HTTP connection to OpenAI is closed/dropped -- [ ] No orphaned tasks or connections after client disconnect -- [ ] Test: simulate client disconnect mid-stream, verify upstream abort - -### US-006: Streaming route integration -**Description:** As a client, I need `POST /v1/messages` with `stream: true` to return an SSE response instead of a JSON response. - -**Acceptance Criteria:** -- [ ] Request with `stream: true` triggers streaming path -- [ ] Response has `Content-Type: text/event-stream` -- [ ] Response includes `Cache-Control: no-cache` -- [ ] OpenAI request is sent with `stream: true` and `stream_options: {include_usage: true}` -- [ ] Events arrive incrementally (not buffered until completion) -- [ ] Test: integration test with mocked streaming OpenAI backend, verify Anthropic SSE event sequence - -### US-007: Golden fixture tests -**Description:** As a developer, I need fixture-based tests comparing full SSE transcripts to catch regressions in streaming translation. - -**Acceptance Criteria:** -- [ ] Fixture file: `fixtures/openai/chat_completion_stream_text.txt` (OpenAI SSE transcript) -- [ ] Fixture file: `fixtures/anthropic/stream_message_text.txt` (expected Anthropic SSE transcript) -- [ ] Test feeds OpenAI fixture through state machine, compares output to Anthropic fixture -- [ ] IDs and timestamps are normalized before comparison -- [ ] At least fixtures for: text streaming, tool call streaming - -## Functional Requirements - -- FR-1: Streaming state machine processes OpenAI chunks one at a time, producing zero or more Anthropic events per chunk -- FR-2: Event ordering strictly follows: `message_start` -> (`content_block_start` -> `content_block_delta`* -> `content_block_stop`)+ -> `message_delta` -> `message_stop` -- FR-3: `stream_options: {include_usage: true}` is always set on OpenAI requests to capture final usage -- FR-4: Bounded channel provides backpressure, not event dropping -- FR-5: Client disconnect triggers upstream cleanup within 1 second - -## Non-Goals - -- No OpenAI Responses API streaming (Chat Completions only for now) -- No WebSocket support -- No streaming obfuscation/side-channel mitigation -- No partial response buffering for content moderation - -## Technical Considerations - -- Use `tokio::sync::mpsc` for the bounded channel -- Use `axum::response::Sse` with `futures::stream::Stream` for SSE delivery -- The state machine should be a struct with methods, not a closure chain, for testability -- OpenAI may send multiple tool call deltas interleaved by index; track by tool call index -- The usage chunk may not arrive if the stream is interrupted; handle gracefully - -## Success Metrics - -- Golden fixture tests pass for text and tool call streaming -- Integration test: full streaming round trip with mock backend -- No memory growth under sustained streaming load (bounded channel enforced) -- `cargo test` passes for both crates - - - -# Rust / Cargo -/target/ -**/*.rs.bk -Cargo.lock - -# IDE -.idea/ -.vscode/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Environment -.env -.env.* -!.env.example - -# Debug -*.pdb - -# Admin server artifacts -.admin_token -*.db - -# TLS certificates and keys -*.p12 -*.pfx -*.pem -*.key - -# Tool config -.claude/ -.cook/ - -# Script caches and venv -scripts/.cache/ -scripts/.venv/ - - - -MIT License - -Copyright (c) 2026 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - - -[ - {"model_pattern": "gpt-4o", "input_cost_per_token": 0.0000025, "output_cost_per_token": 0.00001, "provider": "openai"}, - {"model_pattern": "gpt-4o-mini", "input_cost_per_token": 0.00000015, "output_cost_per_token": 0.0000006, "provider": "openai"}, - {"model_pattern": "gpt-4-turbo", "input_cost_per_token": 0.00001, "output_cost_per_token": 0.00003, "provider": "openai"}, - {"model_pattern": "gpt-4", "input_cost_per_token": 0.00003, "output_cost_per_token": 0.00006, "provider": "openai"}, - {"model_pattern": "gpt-3.5-turbo", "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000015, "provider": "openai"}, - {"model_pattern": "o1", "input_cost_per_token": 0.000015, "output_cost_per_token": 0.00006, "provider": "openai"}, - {"model_pattern": "o1-mini", "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000012, "provider": "openai"}, - {"model_pattern": "o3-mini", "input_cost_per_token": 0.0000011, "output_cost_per_token": 0.0000044, "provider": "openai"}, - {"model_pattern": "claude-opus-4-6", "input_cost_per_token": 0.000015, "output_cost_per_token": 0.000075, "provider": "anthropic"}, - {"model_pattern": "claude-sonnet-4-6", "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, "provider": "anthropic"}, - {"model_pattern": "claude-3-5-sonnet", "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, "provider": "anthropic"}, - {"model_pattern": "claude-3-5-haiku", "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.000004, "provider": "anthropic"}, - {"model_pattern": "claude-3-opus", "input_cost_per_token": 0.000015, "output_cost_per_token": 0.000075, "provider": "anthropic"}, - {"model_pattern": "claude-3-sonnet", "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, "provider": "anthropic"}, - {"model_pattern": "claude-3-haiku", "input_cost_per_token": 0.00000025, "output_cost_per_token": 0.00000125, "provider": "anthropic"}, - {"model_pattern": "gemini-2.5-pro", "input_cost_per_token": 0.00000125, "output_cost_per_token": 0.00001, "provider": "google"}, - {"model_pattern": "gemini-2.5-flash", "input_cost_per_token": 0.00000015, "output_cost_per_token": 0.0000006, "provider": "google"}, - {"model_pattern": "gemini-2.0-flash", "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000004, "provider": "google"}, - {"model_pattern": "gemini-1.5-pro", "input_cost_per_token": 0.00000125, "output_cost_per_token": 0.000005, "provider": "google"}, - {"model_pattern": "gemini-1.5-flash", "input_cost_per_token": 0.000000075, "output_cost_per_token": 0.0000003, "provider": "google"}, - {"model_pattern": "text-embedding-3-large", "input_cost_per_token": 0.00000013, "output_cost_per_token": 0.0, "provider": "openai"}, - {"model_pattern": "text-embedding-3-small", "input_cost_per_token": 0.00000002, "output_cost_per_token": 0.0, "provider": "openai"}, - {"model_pattern": "text-embedding-ada-002", "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0, "provider": "openai"} -] - - - -//! Error types for the client crate. -⋮---- -use anyllm_translate::TranslateError; -⋮---- -/// Errors from the high-level [`Client`](crate::Client). -⋮---- -pub enum ClientError { -/// Translation failed (e.g., unsupported feature with `LossyBehavior::Error`). -⋮---- -/// HTTP transport failure (DNS, TLS, connection refused, timeout). -⋮---- -/// Backend returned a non-2xx status with an error body. -⋮---- -/// Response body could not be deserialized. -⋮---- -/// SSE stream error. -⋮---- -impl ClientError { -/// HTTP status code for API errors, or 500 for transport/deserialization errors. -pub fn status_code(&self) -> u16 { - - - -//! Generic retry logic with exponential backoff and jitter. -⋮---- -use reqwest::Client; -use serde::Serialize; -use std::time::Duration; -use tokio::time::sleep; -⋮---- -/// Default maximum number of retries. -⋮---- -/// Default base delay between retries in milliseconds. -⋮---- -/// Backend error types implement this to enable the generic [`send_with_retry`]. -pub trait RetryableError: Sized { -⋮---- -/// Authentication to apply to outgoing requests. -⋮---- -pub enum RequestAuth<'a> { -⋮---- -fn apply_auth(rb: reqwest::RequestBuilder, auth: &RequestAuth<'_>) -> reqwest::RequestBuilder { -⋮---- -RequestAuth::Bearer(token) => rb.bearer_auth(token), -RequestAuth::Header { name, value } => rb.header(*name, *value), -⋮---- -/// Send a POST request with retry on 429/5xx. Returns the raw successful response. -pub async fn send_with_retry( -⋮---- -let rb = apply_auth(client.post(url).json(body), auth); -let response = rb.send().await.map_err(E::from_request)?; -let status = response.status().as_u16(); -⋮---- -if (200..300).contains(&status) { -return Ok(response); -⋮---- -if attempt < MAX_RETRIES && is_retryable(status) { -let retry_after = parse_retry_after(response.headers()); -let delay = backoff_delay(attempt, retry_after); -⋮---- -// Drain the response body before retrying so the HTTP connection -// returns to the pool. Leaving it unread causes connection leaks. -drop(response.bytes().await); -sleep(delay).await; -⋮---- -let text = response.text().await.unwrap_or_else(|e| { -⋮---- -return Err(E::from_api_response(status, &text)); -⋮---- -unreachable!("loop runs MAX_RETRIES+1 times and always returns") -⋮---- -/// Check if a status code is retryable (408, 429, or 5xx). -pub fn is_retryable(status: u16) -> bool { -status == 408 || status == 429 || (500..=599).contains(&status) -⋮---- -/// Parse retry-after header as integer seconds or HTTP date (RFC 7231). -pub fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option { -⋮---- -.get("retry-after") -.and_then(|v| v.to_str().ok()) -.map(|s| s.trim().to_string())?; -// Fast path: integer seconds -⋮---- -return Some(Duration::from_secs(secs)); -⋮---- -// HTTP date (RFC 7231). Past dates return None (no wait needed). -let date = httpdate::parse_http_date(&value).ok()?; -date.duration_since(std::time::SystemTime::now()).ok() -⋮---- -/// Compute backoff delay with jitter. -/// -/// Uses deterministic 25% jitter (upper bound, not random) to keep tests -/// predictable while still spreading retry storms across backends. -pub fn backoff_delay(attempt: u32, retry_after: Option) -> Duration { -⋮---- -let base = Duration::from_millis(BASE_DELAY_MS * 2u64.pow(attempt)); -let jitter_ms = (base.as_millis() as u64) / 4; -⋮---- -mod tests { -⋮---- -fn is_retryable_429() { -assert!(is_retryable(429)); -⋮---- -fn is_retryable_5xx() { -assert!(is_retryable(500)); -assert!(is_retryable(502)); -assert!(is_retryable(503)); -assert!(is_retryable(599)); -⋮---- -fn is_retryable_408() { -assert!(is_retryable(408)); -⋮---- -fn is_not_retryable_4xx() { -assert!(!is_retryable(400)); -assert!(!is_retryable(401)); -assert!(!is_retryable(404)); -assert!(!is_retryable(409)); -⋮---- -fn backoff_respects_retry_after() { -let delay = backoff_delay(0, Some(Duration::from_secs(5))); -assert_eq!(delay, Duration::from_secs(5)); -⋮---- -fn backoff_increases_with_attempt() { -let d0 = backoff_delay(0, None); -let d1 = backoff_delay(1, None); -let d2 = backoff_delay(2, None); -assert!(d1 > d0); -assert!(d2 > d1); -⋮---- -fn parse_retry_after_valid() { -⋮---- -headers.insert("retry-after", "3".parse().unwrap()); -let dur = parse_retry_after(&headers); -assert_eq!(dur, Some(Duration::from_secs(3))); -⋮---- -fn parse_retry_after_missing() { -⋮---- -assert_eq!(parse_retry_after(&headers), None); -⋮---- -fn parse_retry_after_http_date_future() { -⋮---- -headers.insert( -⋮---- -"Wed, 21 Oct 2037 07:28:00 GMT".parse().unwrap(), -⋮---- -assert!(dur.is_some(), "future HTTP date should parse to Some"); -assert!(dur.unwrap().as_secs() > 0); -⋮---- -fn parse_retry_after_http_date_past() { -⋮---- -"Mon, 01 Jan 2024 00:00:00 GMT".parse().unwrap(), -⋮---- -fn parse_retry_after_garbage() { -⋮---- -headers.insert("retry-after", "not-a-date-or-number".parse().unwrap()); - - - -//! Framework-agnostic SSE frame parser. -//! -//! Reads raw bytes from a `reqwest::Response` stream, splits on SSE frame -//! boundaries (`\n\n` or `\r\n\r\n`), and delivers each `data:` line to a -//! caller-supplied callback. No dependency on axum or any web framework. -⋮---- -use bytes::BytesMut; -⋮---- -/// Maximum SSE buffer size (10 MB). Protects against unbounded memory growth -/// if the backend sends data without frame delimiters. -⋮---- -/// Errors from SSE stream parsing. -⋮---- -pub enum SseError { -⋮---- -/// Find the first SSE frame boundary (`\n\n` or `\r\n\r\n`) in a byte slice, -/// starting the search at `start`. Returns `(position, delimiter_length)` so -/// the caller can skip the full delimiter. -pub fn find_double_newline(buf: &[u8], start: usize) -> Option<(usize, usize)> { -let len = buf.len(); -⋮---- -while i < len.saturating_sub(1) { -⋮---- -return Some((i, 2)); -⋮---- -return Some((i, 4)); -⋮---- -/// Read SSE frames from a response stream, calling `on_data` for each `data:` line. -/// -/// Returns `Ok(())` on normal stream completion, or an `SseError` on failure. -/// The `on_data` callback receives the JSON string after `data: ` and returns -/// an optional list of translated events. The `on_events` callback is called -/// with each batch of events from a complete SSE frame. -⋮---- -/// This is the framework-agnostic core of SSE parsing. It does not depend on -/// axum, tokio channels, or any specific event type. -pub async fn read_sse_stream( -⋮---- -G: FnMut(&[T]) -> bool, // returns false if consumer disconnected -⋮---- -use futures::StreamExt; -let mut stream = response.bytes_stream(); -// BytesMut (not String) because TCP chunks may split mid-UTF-8 character. -⋮---- -while let Some(chunk_result) = stream.next().await { -⋮---- -buffer.extend_from_slice(&bytes); -⋮---- -if buffer.len() > MAX_SSE_BUFFER_SIZE { -return Err(SseError::BufferOverflow); -⋮---- -while let Some((pos, delim_len)) = find_double_newline(&buffer, search_from) { -frame_events.clear(); -⋮---- -for line in frame_str.lines() { -let line = line.trim(); -if let Some(json_str) = line.strip_prefix("data: ") { -if let Some(mut events) = on_data(json_str) { -frame_events.append(&mut events); -⋮---- -let _ = buffer.split_to(pos + delim_len); -⋮---- -if !on_events(&frame_events) { -return Ok(()); // consumer disconnected -⋮---- -// Next chunk: resume scanning 3 bytes back from the end. The 4-byte -// delimiter \r\n\r\n could straddle the chunk boundary. -search_from = buffer.len().saturating_sub(3); -⋮---- -Ok(()) -⋮---- -mod tests { -⋮---- -fn find_double_newline_lf() { -⋮---- -let (pos, len) = find_double_newline(buf, 0).unwrap(); -assert_eq!(pos, 11); -assert_eq!(len, 2); -⋮---- -fn find_double_newline_crlf() { -⋮---- -assert_eq!(len, 4); -⋮---- -fn find_double_newline_from_offset() { -⋮---- -let (pos, len) = find_double_newline(buf, 13).unwrap(); -assert_eq!(pos, 24); -⋮---- -fn find_double_newline_none() { -⋮---- -assert!(find_double_newline(buf, 0).is_none()); -⋮---- -fn find_double_newline_empty() { -assert!(find_double_newline(b"", 0).is_none()); -⋮---- -fn find_double_newline_single_newline() { -assert!(find_double_newline(b"\n", 0).is_none()); -⋮---- -fn find_double_newline_just_delimiter() { -let (pos, len) = find_double_newline(b"\n\n", 0).unwrap(); -assert_eq!(pos, 0); - - - -//! SSE streaming translation: reads OpenAI chunks, yields Anthropic [`StreamEvent`]s. -⋮---- -use anyllm_translate::anthropic::streaming::StreamEvent; -use anyllm_translate::mapping; -use anyllm_translate::openai::ChatCompletionChunk; -use futures::Stream; -use pin_project_lite::pin_project; -⋮---- -use crate::error::ClientError; -⋮---- -pin_project! { -/// A stream that reads SSE frames from a reqwest response, translates -/// OpenAI chunks to Anthropic StreamEvents, and yields them. -⋮---- -impl SseTranslatingStream { -pub(crate) fn new(response: reqwest::Response, model: String) -> Self { -⋮---- -// Spawn a task to read SSE frames and translate them. -⋮---- -return Some(translator.finish()); -⋮---- -Ok(chunk) => Some(translator.process_chunk(&chunk)), -⋮---- -// Block on send; if receiver is dropped, stop. -if tx.try_send(Ok(event.clone())).is_err() { -⋮---- -let _ = tx.try_send(Err(ClientError::Sse(e))); -⋮---- -// Stream ended without [DONE]; flush remaining events. -let events = translator.finish(); -⋮---- -if tx.try_send(Ok(event)).is_err() { -⋮---- -impl Stream for SseTranslatingStream { -type Item = Result; -⋮---- -fn poll_next( -⋮---- -self.project().inner.poll_next(cx) - - - -//! Builder helpers for Anthropic tool definitions and tool choice. -//! -//! These builders produce [`Tool`] and [`ToolChoice`] values from -//! `anyllm_translate::anthropic` with a fluent API, avoiding raw JSON -//! construction for common cases. -⋮---- -use serde_json::Value; -⋮---- -/// Fluent builder for an Anthropic [`Tool`] definition. -/// -/// # Examples -⋮---- -/// ``` -/// use anyllm_client::ToolBuilder; -/// use serde_json::json; -⋮---- -/// let tool = ToolBuilder::new("get_weather") -/// .description("Get the current weather for a location") -/// .input_schema(json!({ -/// "type": "object", -/// "properties": { -/// "location": { "type": "string" } -/// }, -/// "required": ["location"] -/// })) -/// .build(); -⋮---- -/// assert_eq!(tool.name, "get_weather"); -⋮---- -pub struct ToolBuilder { -⋮---- -impl ToolBuilder { -/// Start building a tool with the given name. -pub fn new(name: &str) -> Self { -⋮---- -name: name.to_string(), -⋮---- -/// Set the human-readable description shown to the model. -pub fn description(mut self, desc: &str) -> Self { -self.description = Some(desc.to_string()); -⋮---- -/// Set the JSON Schema describing the tool's expected input. -pub fn input_schema(mut self, schema: Value) -> Self { -⋮---- -/// Consume the builder and produce a [`Tool`]. -pub fn build(self) -> Tool { -⋮---- -/// Convenience constructors for [`ToolChoice`] variants. -⋮---- -/// use anyllm_client::ToolChoiceBuilder; -⋮---- -/// let choice = ToolChoiceBuilder::auto(); -/// let specific = ToolChoiceBuilder::specific("get_weather"); -⋮---- -pub struct ToolChoiceBuilder; -⋮---- -impl ToolChoiceBuilder { -/// Let the model decide whether to use tools. -pub fn auto() -> ToolChoice { -⋮---- -/// Force the model to use at least one tool. -pub fn any() -> ToolChoice { -⋮---- -/// Prevent the model from using any tools. -pub fn none() -> ToolChoice { -⋮---- -/// Force the model to use a specific tool by name. -pub fn specific(name: &str) -> ToolChoice { -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn tool_builder_minimal() { -let tool = ToolBuilder::new("test_tool").build(); -assert_eq!(tool.name, "test_tool"); -assert!(tool.description.is_none()); -assert!(tool.input_schema.is_object()); -⋮---- -fn tool_builder_full() { -let schema = json!({ -⋮---- -.description("Search the web") -.input_schema(schema.clone()) -.build(); -⋮---- -assert_eq!(tool.name, "search"); -assert_eq!(tool.description.as_deref(), Some("Search the web")); -assert_eq!(tool.input_schema, schema); -⋮---- -fn tool_choice_auto() { -⋮---- -assert_eq!( -⋮---- -fn tool_choice_any() { -⋮---- -fn tool_choice_none() { -⋮---- -assert_eq!(choice, ToolChoice::None); -⋮---- -fn tool_choice_specific() { -⋮---- -fn tool_serializes_correctly() { -⋮---- -.description("Calculator") -.input_schema(json!({"type": "object"})) -⋮---- -let json = serde_json::to_value(&tool).unwrap(); -assert_eq!(json["name"], "calc"); -assert_eq!(json["description"], "Calculator"); - - - -// Admin token validation middleware. -// All admin routes (except /admin/health) require Authorization: Bearer {token}. -⋮---- -use std::sync::Arc; -use subtle::ConstantTimeEq; -⋮---- -/// Constant-time string comparison to prevent timing side-channels. -pub(super) fn constant_time_eq(a: &str, b: &str) -> bool { -// Length comparison leaks length info, but the Bearer prefix is fixed-length -// and the token format (UUID) is fixed-length, so this is acceptable. -a.len() == b.len() && a.as_bytes().ct_eq(b.as_bytes()).into() -⋮---- -/// Generate a cryptographically random CSRF token (32 bytes of entropy, hex-encoded, 64 chars). -/// Uses two UUID v4 values (each 122 bits of randomness) concatenated, giving ~244 bits. -pub fn generate_csrf_token() -> String { -let a = uuid::Uuid::new_v4().as_simple().to_string(); -let b = uuid::Uuid::new_v4().as_simple().to_string(); -format!("{a}{b}") -⋮---- -/// Extract the csrf_token value from a Cookie header string. -pub fn extract_csrf_cookie(cookie_header: &str) -> Option { -cookie_header.split(';').find_map(|pair| { -let pair = pair.trim(); -pair.strip_prefix("csrf_token=").map(|v| v.to_string()) -⋮---- -/// Constant-time comparison of two CSRF tokens. Returns false for empty tokens. -pub fn validate_csrf_tokens(from_header: &str, from_cookie: &str) -> bool { -if from_header.is_empty() || from_cookie.is_empty() { -⋮---- -constant_time_eq(from_header, from_cookie) -⋮---- -/// Axum middleware that validates the admin bearer token. -pub async fn validate_admin_token( -⋮---- -.headers() -.get("authorization") -.and_then(|v| v.to_str().ok()); -⋮---- -let expected = format!("Bearer {}", token.as_str()); -⋮---- -Some(h) if constant_time_eq(h, &expected) => next.run(req).await, -⋮---- -.into_response(), -⋮---- -mod csrf_tests { -⋮---- -fn generate_csrf_token_has_correct_length() { -let token = generate_csrf_token(); -// 32 random bytes hex-encoded = 64 chars -assert_eq!(token.len(), 64); -assert!(token.chars().all(|c| c.is_ascii_hexdigit())); -⋮---- -fn csrf_tokens_are_unique() { -let a = generate_csrf_token(); -let b = generate_csrf_token(); -assert_ne!(a, b); -⋮---- -fn extract_csrf_cookie_finds_value() { -⋮---- -assert_eq!(extract_csrf_cookie(cookie_header), Some("abc123".to_string())); -⋮---- -fn extract_csrf_cookie_returns_none_when_absent() { -⋮---- -assert_eq!(extract_csrf_cookie(cookie_header), None); -⋮---- -fn validate_csrf_matching_tokens() { -⋮---- -assert!(validate_csrf_tokens(token, token)); -⋮---- -fn validate_csrf_mismatched_tokens() { -⋮---- -assert!(!validate_csrf_tokens(a, b)); -⋮---- -fn validate_csrf_empty_token_fails() { -assert!(!validate_csrf_tokens("", "")); -⋮---- -mod tests { -⋮---- -use tower::ServiceExt; -⋮---- -fn test_app(token: &str) -> Router { -let token = Arc::new(token.to_string()); -⋮---- -.route("/protected", get(|| async { "ok" })) -.layer(middleware::from_fn_with_state( -token.clone(), -⋮---- -.with_state(token) -⋮---- -async fn valid_token_passes() { -let app = test_app("test-token-123"); -⋮---- -.uri("/protected") -.header("authorization", "Bearer test-token-123") -.body(Body::empty()) -.unwrap(); -⋮---- -let resp = app.oneshot(req).await.unwrap(); -assert_eq!(resp.status(), StatusCode::OK); -⋮---- -async fn missing_token_rejected() { -⋮---- -assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); -⋮---- -async fn wrong_token_rejected() { -⋮---- -.header("authorization", "Bearer wrong-token") -⋮---- -async fn bearer_prefix_required() { -⋮---- -.header("authorization", "test-token-123") - - - -// Admin endpoint for per-key spend reporting. -// -// GET /admin/api/keys/{id}/spend returns accumulated cost and token usage -// for a single virtual API key. -⋮---- -use crate::admin::state::SharedState; -⋮---- -/// GET /admin/api/keys/{id}/spend -- per-key cost and usage summary. -pub async fn get_key_spend( -⋮---- -Some(Ok(Some(spend))) => (StatusCode::OK, Json(serde_json::json!(spend))).into_response(), -⋮---- -Json(serde_json::json!({"error": "Key not found"})), -⋮---- -.into_response(), -⋮---- -Json(serde_json::json!({"error": "internal database error"})), -⋮---- -.into_response() -⋮---- -Json(serde_json::json!({"error": "internal error"})), - - - -// Gemini native HTTP client for generateContent / streamGenerateContent endpoints. -// No OpenAI translation: sends and receives Gemini-native JSON directly. -⋮---- -use super::build_http_client; -use crate::config::TlsConfig; -⋮---- -use reqwest::Client; -⋮---- -/// HTTP client for Google Gemini's native generateContent API. -⋮---- -pub struct GeminiNativeClient { -⋮---- -/// Error type for the Gemini native client. -⋮---- -pub enum GeminiClientError { -/// Transport-level error (connection, timeout, DNS). -⋮---- -/// Upstream returned a non-success status. -⋮---- -/// Response body could not be deserialized. -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -Self::Transport(e) => write!(f, "Gemini transport error: {e}"), -⋮---- -write!(f, "Gemini API error (status {status}): {body}") -⋮---- -Self::Deserialize(e) => write!(f, "Gemini deserialization error: {e}"), -⋮---- -impl GeminiNativeClient { -/// Create a new Gemini native client. -/// -/// `base_url` should be the Gemini API root, e.g. -/// `https://generativelanguage.googleapis.com/v1beta`. -pub fn new( -⋮---- -let client = build_http_client(tls); -⋮---- -pub fn big_model(&self) -> &str { -⋮---- -pub fn small_model(&self) -> &str { -⋮---- -/// Map an Anthropic model name to the configured Gemini model. -pub fn map_model(&self, anthropic_model: &str) -> String { -let lower = anthropic_model.to_lowercase(); -if lower.contains("haiku") { -self.small_model.clone() -⋮---- -self.big_model.clone() -⋮---- -/// Build the generateContent URL for a given model. -fn generate_url(&self, model: &str) -> String { -format!( -⋮---- -/// Build the streamGenerateContent URL for a given model. -fn stream_url(&self, model: &str) -> String { -⋮---- -/// Non-streaming: POST generateContent, parse response. -pub async fn generate_content( -⋮---- -let url = self.generate_url(model); -⋮---- -.post(&url) -.header("x-goog-api-key", &self.api_key) -.header("Content-Type", "application/json") -.json(body) -.send() -⋮---- -.map_err(|e| GeminiClientError::Transport(e.to_string()))?; -⋮---- -let status = resp.status(); -if !status.is_success() { -let body_text = resp.text().await.unwrap_or_default(); -return Err(GeminiClientError::ApiError { -status: status.as_u16(), -⋮---- -.map_err(|e| GeminiClientError::Deserialize(e.to_string())) -⋮---- -/// Streaming: POST streamGenerateContent, return raw Response for SSE reading. -pub async fn generate_content_stream( -⋮---- -let url = self.stream_url(model); -⋮---- -Ok(resp) -⋮---- -mod tests { -⋮---- -fn test_client(base_url: &str) -> GeminiNativeClient { -⋮---- -base_url.to_string(), -"test-key".to_string(), -"gemini-2.5-pro".to_string(), -"gemini-2.5-flash".to_string(), -⋮---- -fn generate_url_construction() { -let c = test_client("https://generativelanguage.googleapis.com/v1beta"); -assert_eq!( -⋮---- -fn stream_url_construction() { -⋮---- -fn map_model_haiku_to_small() { -let c = test_client("https://example.com"); -assert_eq!(c.map_model("claude-3-haiku-20240307"), "gemini-2.5-flash"); -assert_eq!(c.map_model("claude-sonnet-4-6"), "gemini-2.5-pro"); -⋮---- -fn map_model_case_insensitive() { -⋮---- -assert_eq!(c.map_model("Claude-3-HAIKU-20240307"), "gemini-2.5-flash"); -⋮---- -fn base_url_trailing_slash_stripped() { -let c = test_client("https://example.com/v1beta/"); -let url = c.generate_url("pro"); -assert!( -⋮---- -assert!(!url.contains("//models"), "double slash in: {url}"); -⋮---- -fn stream_url_trailing_slash_stripped() { -⋮---- -let url = c.stream_url("pro"); -⋮---- -fn error_display_transport() { -let e = GeminiClientError::Transport("connection refused".to_string()); -let s = e.to_string(); -assert!(s.contains("transport"), "got: {s}"); -assert!(s.contains("connection refused"), "got: {s}"); -⋮---- -fn error_display_api() { -⋮---- -body: "rate limited".to_string(), -⋮---- -assert!(s.contains("429"), "got: {s}"); -assert!(s.contains("rate limited"), "got: {s}"); -⋮---- -fn error_display_deserialize() { -let e = GeminiClientError::Deserialize("unexpected token".to_string()); -⋮---- -assert!(s.contains("deserialization"), "got: {s}"); -⋮---- -fn model_accessors() { -⋮---- -assert_eq!(c.big_model(), "gemini-2.5-pro"); -assert_eq!(c.small_model(), "gemini-2.5-flash"); - - - -// YAML-based fallback chain configuration. -// Loaded from the FALLBACK_CONFIG env var (path to a YAML file). -⋮---- -use serde::Deserialize; -use std::collections::HashMap; -⋮---- -/// Top-level fallback configuration, deserialized from YAML. -/// -/// Example: -/// ```yaml -/// fallback_chains: -/// default: -/// - name: azure -/// env_prefix: AZURE_FALLBACK_ -/// - name: openai -/// env_prefix: OPENAI_FALLBACK_ -/// ``` -⋮---- -pub struct FallbackConfig { -⋮---- -/// A single backend entry in a fallback chain. -/// `name` is a human-readable label (used in logs). -/// `env_prefix` identifies which env vars configure this backend. -⋮---- -pub struct BackendSpec { -⋮---- -/// Parse fallback config from a YAML string. -pub fn parse_fallback_config(yaml: &str) -> Result { -⋮---- -/// Load fallback config from the file path in `FALLBACK_CONFIG` env var. -/// Returns `None` if the env var is not set. -/// Returns `Err` if the file cannot be read or parsed. -pub fn load_fallback_config() -> Result, FallbackConfigError> { -⋮---- -Ok(p) if !p.is_empty() => p, -_ => return Ok(None), -⋮---- -let contents = std::fs::read_to_string(&path).map_err(|e| FallbackConfigError::Io { -path: path.clone(), -⋮---- -parse_fallback_config(&contents).map_err(|e| FallbackConfigError::Parse { source: e })?; -⋮---- -Ok(Some(config)) -⋮---- -/// Errors that can occur when loading fallback configuration. -⋮---- -pub enum FallbackConfigError { -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -write!(f, "failed to read fallback config at {path}: {source}") -⋮---- -Self::Parse { source } => write!(f, "failed to parse fallback config YAML: {source}"), -⋮---- -mod tests { -⋮---- -fn parse_valid_config() { -⋮---- -let config = parse_fallback_config(yaml).expect("should parse valid YAML"); -assert_eq!(config.fallback_chains.len(), 2); -⋮---- -assert_eq!(default_chain.len(), 2); -assert_eq!(default_chain[0].name, "azure"); -assert_eq!(default_chain[0].env_prefix, "AZURE_FALLBACK_"); -assert_eq!(default_chain[1].name, "openai"); -assert_eq!(default_chain[1].env_prefix, "OPENAI_FALLBACK_"); -⋮---- -fn parse_empty_chains() { -⋮---- -let config = parse_fallback_config(yaml).expect("should parse empty chains"); -assert!(config.fallback_chains.is_empty()); -⋮---- -fn parse_malformed_yaml() { -⋮---- -let result = parse_fallback_config(yaml); -assert!(result.is_err(), "malformed YAML should fail"); -⋮---- -fn parse_missing_required_fields() { -// Backend spec missing `env_prefix` -⋮---- -assert!(result.is_err(), "missing env_prefix should fail"); - - - -// Backend fallback chain: when the primary backend returns a retryable error, -// iterate through fallback backends until one succeeds or all are exhausted. -⋮---- -pub mod config; -⋮---- -use crate::backend::BackendError; -⋮---- -/// Header set when all backends in the fallback chain have failed. -⋮---- -/// A chain of backend specs to try in order on retryable failures. -⋮---- -pub struct FallbackChain { -⋮---- -/// Outcome of a fallback attempt: either a successful result or the last error -/// after exhausting all backends. -⋮---- -pub struct FallbackOutcome { -/// The successful result, if any backend succeeded. -⋮---- -/// Index of the backend that produced the result (0 = primary, 1+ = fallback). -⋮---- -/// Name of the backend that produced the result. -⋮---- -/// True if all backends were tried and failed. -⋮---- -impl FallbackChain { -/// Create a new fallback chain from a list of backend specs. -pub fn new(backends: Vec) -> Self { -⋮---- -/// Determine whether a failure should trigger fallback to the next backend. -/// -/// Retryable: 429 (rate limit), 500, 502, 503, connection errors, timeouts. -/// Non-retryable: 400, 401, 403, 404, and other 4xx (client errors that won't -/// resolve by switching backends). -pub fn should_fallback(status: u16, is_connection_error: bool) -> bool { -⋮---- -matches!(status, 429 | 500 | 502 | 503) -⋮---- -/// Execute a fallback chain. Calls `try_backend` for each backend in order, -/// stopping on the first success or non-retryable error. -⋮---- -/// `try_backend` receives the backend spec and index, returns the backend result. -/// The caller is responsible for constructing the actual backend client from the spec. -⋮---- -/// For streaming requests where SSE has already started, callers should NOT use -/// this method. Mid-stream failures should terminate the SSE with an error event -/// rather than retrying (the client has already started consuming events). -pub async fn attempt_with_fallback(&self, mut try_backend: F) -> FallbackOutcome -⋮---- -for (i, spec) in self.backends.iter().enumerate() { -⋮---- -last_name.clone_from(&spec.name); -⋮---- -match try_backend(spec, i).await { -⋮---- -result: Ok(result), -⋮---- -backend_name: spec.name.clone(), -⋮---- -let status = e.status_code(); -let is_conn = is_connection_error(&e); -⋮---- -// Non-retryable error: stop immediately, don't try other backends. -⋮---- -result: Err(e), -⋮---- -last_error = Some(e); -⋮---- -// All backends exhausted. -⋮---- -// All backends in the chain returned retryable errors. -// `last_error` is always Some when backends is non-empty. If the chain -// was empty (a misconfiguration), we fabricate a descriptive error. -⋮---- -// Empty chain: no backends to try. Fabricate an error. -⋮---- -message: "fallback chain is empty: no backends configured".to_string(), -error_type: "configuration_error".to_string(), -⋮---- -result: Err(final_err), -⋮---- -/// Check if a `BackendError` represents a connection-level failure (not an HTTP status). -pub fn is_connection_error(err: &BackendError) -> bool { -⋮---- -e.is_connect() || e.is_timeout() -⋮---- -mod tests { -⋮---- -fn should_fallback_retryable_statuses() { -assert!(FallbackChain::should_fallback(500, false)); -assert!(FallbackChain::should_fallback(502, false)); -assert!(FallbackChain::should_fallback(503, false)); -assert!(FallbackChain::should_fallback(429, false)); -⋮---- -fn should_fallback_connection_error() { -// Connection errors should always fallback, regardless of status. -assert!(FallbackChain::should_fallback(0, true)); -assert!(FallbackChain::should_fallback(200, true)); -⋮---- -fn should_not_fallback_client_errors() { -assert!(!FallbackChain::should_fallback(400, false)); -assert!(!FallbackChain::should_fallback(401, false)); -assert!(!FallbackChain::should_fallback(403, false)); -assert!(!FallbackChain::should_fallback(404, false)); -assert!(!FallbackChain::should_fallback(422, false)); -⋮---- -fn should_not_fallback_success() { -assert!(!FallbackChain::should_fallback(200, false)); -assert!(!FallbackChain::should_fallback(201, false)); -⋮---- -async fn fallback_succeeds_on_second_backend() { -let chain = FallbackChain::new(vec![ -⋮---- -.attempt_with_fallback(|spec, _idx| { -let name = spec.name.clone(); -⋮---- -Err(make_api_error(503)) -⋮---- -Ok("success from secondary") -⋮---- -assert!(outcome.result.is_ok()); -assert_eq!(outcome.backend_index, 1); -assert_eq!(outcome.backend_name, "secondary"); -assert!(!outcome.exhausted); -⋮---- -async fn fallback_stops_on_non_retryable() { -⋮---- -Err(make_api_error(400)) -⋮---- -Ok("should not reach here") -⋮---- -assert!(outcome.result.is_err()); -assert_eq!(outcome.backend_index, 0); -assert_eq!(outcome.backend_name, "primary"); -⋮---- -async fn fallback_all_exhausted() { -⋮---- -.attempt_with_fallback(|_spec, _idx| async move { -// Both backends return 503. -⋮---- -assert!(outcome.exhausted); -⋮---- -async fn fallback_first_succeeds() { -⋮---- -.attempt_with_fallback(|_spec, _idx| async move { Ok("immediate success") }) -⋮---- -/// Helper: create a `BackendError::OpenAI(ApiError { .. })` with a given status code. -fn make_api_error(status: u16) -> BackendError { -⋮---- -message: format!("mock error {status}"), -error_type: "test".to_string(), - - - -// Audio passthrough handlers: forward requests to the backend unchanged. -// Supports /v1/audio/transcriptions (multipart) and /v1/audio/speech (JSON -> binary). -⋮---- -use crate::server::routes::AppState; -⋮---- -/// POST /v1/audio/transcriptions -- multipart/form-data passthrough. -/// Forwards the raw body (including multipart boundary) to the backend. -pub async fn audio_transcriptions( -⋮---- -.get(header::CONTENT_TYPE) -.and_then(|v| v.to_str().ok()) -.unwrap_or("multipart/form-data") -.to_string(); -⋮---- -passthrough_response(&state, "/v1/audio/transcriptions", body, &content_type).await -⋮---- -/// POST /v1/audio/speech -- JSON in, binary audio out. -/// Forwards the JSON body to the backend and streams the audio response bytes back. -pub async fn audio_speech( -⋮---- -.unwrap_or("application/json") -⋮---- -passthrough_response(&state, "/v1/audio/speech", body, &content_type).await -⋮---- -/// Shared passthrough logic: forward to backend, return response unchanged. -async fn passthrough_response( -⋮---- -state.metrics.record_request(); -⋮---- -.raw_passthrough(path, body, content_type) -⋮---- -if status.is_success() { -state.metrics.record_success(); -⋮---- -state.metrics.record_error(); -⋮---- -let mut response = (status, resp_body).into_response(); -⋮---- -response.headers_mut().insert(k, v.clone()); -⋮---- -.to_string(), -⋮---- -(StatusCode::INTERNAL_SERVER_ERROR, axum::Json(err)).into_response() - - - -// Gemini native handler: POST /v1/messages -> Gemini generateContent/streamGenerateContent. -// -// Translates Anthropic requests to Gemini native format (not OpenAI-compat), -// calls the Gemini API, and translates responses back to Anthropic format. -⋮---- -use anyllm_translate::anthropic; -use anyllm_translate::gemini::response::GenerateContentResponse; -⋮---- -use anyllm_translate::mapping::gemini_streaming_map::GeminiStreamingTranslator; -⋮---- -use tokio::sync::mpsc; -use tokio_stream::wrappers::ReceiverStream; -⋮---- -/// POST /v1/messages — Gemini native path. -pub(crate) async fn gemini_native_handler( -⋮---- -BackendClient::GeminiNative(c) => c.clone(), -⋮---- -Json(anyllm_translate::mapping::errors_map::create_anthropic_error( -⋮---- -"gemini_native_handler called with non-native backend".to_string(), -⋮---- -.into_response(); -⋮---- -state.metrics.record_request(); -⋮---- -let model = client.map_model(&body.model); -let gemini_req = anthropic_to_gemini_request(&body); -let original_model = body.model.clone(); -⋮---- -if body.stream == Some(true) { -let metrics = state.metrics.clone(); -⋮---- -let resp = match client.generate_content_stream(&gemini_req, &model).await { -⋮---- -metrics.record_error(); -⋮---- -// Send a synthetic error event so the client knows the stream failed. -⋮---- -be.to_string(), -⋮---- -error_type: "api_error".to_string(), -message: err.error.message.clone(), -⋮---- -let _ = send_events(&tx, &[event]).await; -⋮---- -let mut translator = GeminiStreamingTranslator::new(original_model.clone()); -⋮---- -let outcome = read_sse_frames(resp, &tx, &metrics, |data| { -⋮---- -let events = translator.process_response(&gresp); -if events.is_empty() { None } else { Some(events) } -⋮---- -// If the stream ended without a finishReason, flush the translator. -if matches!(outcome, StreamOutcome::Completed) && !translator.is_finished() { -let final_events = translator.finish(); -send_events(&tx, &final_events).await; -⋮---- -metrics.record_success(); -metrics.record_stream_completed(); -⋮---- -metrics.record_stream_client_disconnected(); -⋮---- -metrics.record_stream_failed(); -⋮---- -.keep_alive(KeepAlive::default()) -.into_response() -⋮---- -match client.generate_content(&gemini_req, &model).await { -⋮---- -state.metrics.record_success(); -let anthropic_resp = gemini_to_anthropic_response(&gresp, &original_model); -(StatusCode::OK, Json(anthropic_resp)).into_response() -⋮---- -state.metrics.record_error(); - - - -// Image generation passthrough handler: forward requests to the backend unchanged. -⋮---- -use crate::server::routes::AppState; -⋮---- -/// POST /v1/images/generations -- JSON passthrough. -/// Forwards the request body to the backend and returns the response unchanged. -pub async fn image_generations( -⋮---- -.get(header::CONTENT_TYPE) -.and_then(|v| v.to_str().ok()) -.unwrap_or("application/json") -.to_string(); -⋮---- -state.metrics.record_request(); -⋮---- -.raw_passthrough("/v1/images/generations", body, &content_type) -⋮---- -if status.is_success() { -state.metrics.record_success(); -⋮---- -state.metrics.record_error(); -⋮---- -let mut response = (status, resp_body).into_response(); -⋮---- -response.headers_mut().insert(k, v.clone()); -⋮---- -.to_string(), -⋮---- -(StatusCode::INTERNAL_SERVER_ERROR, axum::Json(err)).into_response() - - - -// SSE responder helpers for Anthropic-format streaming -⋮---- -use anyllm_translate::anthropic::streaming::StreamEvent; -use axum::response::sse::Event; -⋮---- -/// Format a StreamEvent as an axum SSE Event with the correct Anthropic event type name. -/// -/// The event type string matches what Anthropic clients expect: -/// `message_start`, `content_block_start`, `content_block_delta`, etc. -⋮---- -/// Anthropic: -pub fn stream_event_to_sse(event: &StreamEvent) -> Result { -⋮---- -Ok(Event::default().event(event_type).data(data)) -⋮---- -mod tests { -⋮---- -/// Verify the function returns Ok and the JSON data round-trips correctly. -/// We cannot inspect axum Event internals directly, so we test: -/// 1. The event type mapping logic (via match coverage) -/// 2. The JSON serialization is valid -/// 3. The function does not error -⋮---- -fn assert_sse_ok(event: &StreamEvent) { -let _ = stream_event_to_sse(event).expect("stream_event_to_sse should not fail"); -⋮---- -fn message_start_produces_sse() { -⋮---- -id: "msg_test".into(), -msg_type: "message".into(), -role: "assistant".into(), -content: vec![], -model: "gpt-4o".into(), -⋮---- -assert_sse_ok(&event); -// Verify the JSON serialization contains expected fields -let json = serde_json::to_string(&event).unwrap(); -assert!(json.contains("message_start")); -assert!(json.contains("msg_test")); -⋮---- -fn content_block_start_produces_sse() { -⋮---- -fn content_block_delta_text_produces_sse() { -⋮---- -text: "hello".into(), -⋮---- -assert!(json.contains("text_delta")); -assert!(json.contains("hello")); -⋮---- -fn content_block_delta_input_json_produces_sse() { -⋮---- -partial_json: "{\"key\":".into(), -⋮---- -assert!(json.contains("input_json_delta")); -⋮---- -fn content_block_stop_produces_sse() { -⋮---- -fn message_delta_produces_sse() { -⋮---- -stop_reason: Some(StopReason::EndTurn), -⋮---- -usage: Some(DeltaUsage { output_tokens: 42 }), -⋮---- -assert!(json.contains("end_turn")); -assert!(json.contains("42")); -⋮---- -fn message_stop_produces_sse() { -⋮---- -fn ping_produces_sse() { -⋮---- -fn error_produces_sse() { -⋮---- -error_type: "overloaded_error".into(), -message: "Overloaded".into(), -⋮---- -assert!(json.contains("overloaded_error")); -⋮---- -fn event_type_mapping_covers_all_variants() { -// Verify each variant maps to the correct SSE event type string. -// We test the mapping logic by calling the function and checking it doesn't panic. -let events: Vec = vec![ -⋮---- -assert_sse_ok(event); -⋮---- -fn serialized_data_is_valid_json() { -⋮---- -text: "test".into(), -⋮---- -// The data passed to the SSE event is serde_json::to_string output -let data = serde_json::to_string(&event).unwrap(); -let parsed: serde_json::Value = serde_json::from_str(&data).unwrap(); -assert_eq!(parsed["index"], 0); -assert_eq!(parsed["delta"]["text"], "test"); - - - -//! Optional OpenTelemetry OTLP trace export. -//! -//! Enabled by the `otel` cargo feature. All types and functions in this module -//! are gated behind `#[cfg(feature = "otel")]` at the module declaration site -//! in `lib.rs`, so nothing here compiles into the default binary. -⋮---- -//! The OTLP SDK reads configuration from standard env vars: -//! - `OTEL_EXPORTER_OTLP_ENDPOINT` (default `http://localhost:4318` for HTTP) -//! - `OTEL_SERVICE_NAME` -//! - `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` -⋮---- -use opentelemetry_otlp::SpanExporter; -use opentelemetry_sdk::trace::SdkTracerProvider; -⋮---- -/// Holds the [`SdkTracerProvider`] and flushes pending spans on drop. -pub struct OtelGuard { -⋮---- -impl Drop for OtelGuard { -fn drop(&mut self) { -if let Err(e) = self.provider.shutdown() { -eprintln!("otel: tracer provider shutdown error: {e}"); -⋮---- -/// Initialise the OTLP span exporter and return a guard that must live for the -/// duration of `main`. The returned tracer is suitable for -/// [`tracing_opentelemetry::OpenTelemetryLayer::new`]. -/// -/// Panics if the exporter or provider cannot be created (misconfiguration). -pub fn init_otel() -> (OtelGuard, opentelemetry_sdk::trace::Tracer) { -// HTTP/protobuf transport via reqwest (matches Cargo feature flags). -⋮---- -.with_http() -.build() -.expect("failed to create OTLP span exporter"); -⋮---- -.with_batch_exporter(exporter) -.build(); -⋮---- -opentelemetry::global::set_tracer_provider(provider.clone()); -⋮---- -let tracer = provider.tracer("anyllm-proxy"); - - - -// Integration tests for audio and image passthrough endpoints. -// Actual backend calls need a live API; these tests verify routing and 501 behavior. -⋮---- -use anyllm_proxy::server::routes; -⋮---- -use reqwest::Client; -use tokio::net::TcpListener; -⋮---- -fn openai_config_with_base(base_url: &str) -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: base_url.to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: BackendAuth::BearerToken("test-key".into()), -⋮---- -fn anthropic_config() -> Config { -⋮---- -openai_base_url: "https://api.anthropic.com".to_string(), -⋮---- -big_model: "claude-opus-4-6".into(), -small_model: "claude-haiku-4-5".into(), -⋮---- -/// Mock backend that accepts audio and image passthrough endpoints. -async fn spawn_mock_backend() -> String { -⋮---- -.route( -⋮---- -post(|| async { -⋮---- -.status(200) -.header("content-type", "application/json") -.body(axum::body::Body::from( -⋮---- -.unwrap() -⋮---- -.header("content-type", "audio/mpeg") -.body(axum::body::Body::from(vec![0xFF, 0xFB, 0x90, 0x00])) -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -async fn spawn_proxy_with_config(config: Config) -> String { -⋮---- -// --- Audio transcriptions --- -⋮---- -async fn audio_transcriptions_forwarded_to_backend() { -let mock_base = spawn_mock_backend().await; -let proxy_base = spawn_proxy_with_config(openai_config_with_base(&mock_base)).await; -⋮---- -.post(format!("{proxy_base}/v1/audio/transcriptions")) -.header("x-api-key", "test") -.header("content-type", "multipart/form-data; boundary=abc") -.body("--abc\r\ncontent-disposition: form-data; name=\"file\"\r\n\r\nfake\r\n--abc--") -.send() -⋮---- -.unwrap(); -⋮---- -assert_eq!(resp.status(), 200); -let body: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(body["text"], "Hello world"); -⋮---- -async fn audio_transcriptions_not_routed_for_anthropic_backend() { -let proxy_base = spawn_proxy_with_config(anthropic_config()).await; -⋮---- -.header("content-type", "multipart/form-data") -.body("fake") -⋮---- -// Route not registered for Anthropic backend; fallback returns 404. -assert_eq!(resp.status(), 404); -⋮---- -// --- Audio speech --- -⋮---- -async fn audio_speech_forwarded_to_backend() { -⋮---- -.post(format!("{proxy_base}/v1/audio/speech")) -⋮---- -.body(r#"{"model":"tts-1","input":"Hello","voice":"alloy"}"#) -⋮---- -.headers() -.get("content-type") -⋮---- -.to_str() -⋮---- -assert!(ct.contains("audio/mpeg"), "got content-type: {ct}"); -let bytes = resp.bytes().await.unwrap(); -assert_eq!(&bytes[..], &[0xFF, 0xFB, 0x90, 0x00]); -⋮---- -// --- Image generations --- -⋮---- -async fn image_generations_forwarded_to_backend() { -⋮---- -.post(format!("{proxy_base}/v1/images/generations")) -⋮---- -.body(r#"{"model":"dall-e-3","prompt":"a cat","n":1,"size":"1024x1024"}"#) -⋮---- -assert!(body["data"].is_array()); -assert_eq!(body["data"][0]["url"], "https://example.com/image.png"); -⋮---- -async fn image_generations_not_routed_for_anthropic_backend() { -⋮---- -.body(r#"{"model":"dall-e-3","prompt":"a cat"}"#) - - - -use anyllm_proxy::server::routes; -⋮---- -fn test_config_with_logging() -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: "https://api.openai.com".to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: config::BackendAuth::BearerToken("test-key".into()), -⋮---- -async fn server_starts_with_body_logging_enabled() { -let app = routes::app(test_config_with_logging()); -let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -⋮---- -let resp = reqwest::get(format!("http://{addr}/health")).await.unwrap(); -assert_eq!(resp.status(), 200); - - - -// Integration tests for POST /v1/chat/completions (OpenAI-format input). -⋮---- -use anyllm_proxy::server::routes; -⋮---- -use reqwest::Client; -use serde_json::json; -use tokio::net::TcpListener; -⋮---- -fn openai_config_with_base(base_url: &str) -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: base_url.to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: BackendAuth::BearerToken("test-key".into()), -⋮---- -/// Mock backend that returns a fixed OpenAI Chat Completions response. -async fn spawn_mock_chat_backend() -> String { -let app = Router::new().route( -⋮---- -post(|| async { -axum::Json(json!({ -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -async fn spawn_proxy(config: Config) -> String { -⋮---- -async fn chat_completions_non_streaming() { -let mock = spawn_mock_chat_backend().await; -let proxy = spawn_proxy(openai_config_with_base(&mock)).await; -⋮---- -.post(format!("{proxy}/v1/chat/completions")) -.header("x-api-key", "test") -.header("content-type", "application/json") -.json(&json!({ -⋮---- -.send() -⋮---- -.unwrap(); -⋮---- -assert_eq!(resp.status(), 200); -let body: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(body["object"], "chat.completion"); -assert!(body["id"].as_str().unwrap().starts_with("chatcmpl-")); -assert_eq!(body["choices"][0]["message"]["role"], "assistant"); -assert!(body["choices"][0]["message"]["content"].as_str().is_some()); -assert_eq!(body["choices"][0]["finish_reason"], "stop"); -assert!(body["usage"]["prompt_tokens"].as_u64().is_some()); -⋮---- -async fn chat_completions_missing_max_tokens_returns_400() { -⋮---- -assert_eq!(resp.status(), 400); -⋮---- -assert_eq!(body["error"]["type"], "invalid_request_error"); -⋮---- -async fn chat_completions_empty_messages_returns_400() { -⋮---- -async fn chat_completions_degradation_header_on_lossy_fields() { -⋮---- -.headers() -.get("x-anyllm-degradation") -.and_then(|v| v.to_str().ok()) -.unwrap_or(""); -assert!( -⋮---- -async fn chat_completions_with_system_message() { -⋮---- -async fn chat_completions_returns_openai_error_format() { -⋮---- -// Send completely invalid JSON -⋮---- -.body("not json") -⋮---- -// Should have OpenAI error format (error.type, error.message) -assert!(body["error"]["type"].is_string()); -assert!(body["error"]["message"].is_string()); - - - -// Integration tests for POST /v1/embeddings passthrough and x-anyllm-degradation header. -⋮---- -use anyllm_proxy::server::routes; -⋮---- -use reqwest::Client; -use tokio::net::TcpListener; -⋮---- -fn openai_config_with_base(base_url: &str) -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: base_url.to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: BackendAuth::BearerToken("test-key".into()), -⋮---- -fn anthropic_config() -> Config { -⋮---- -openai_base_url: "https://api.anthropic.com".to_string(), -⋮---- -big_model: "claude-opus-4-6".into(), -small_model: "claude-haiku-4-5".into(), -⋮---- -/// Start a mock backend that accepts POST /v1/embeddings and returns a fixed response. -/// Returns the base URL of the mock server. -async fn spawn_mock_backend() -> String { -let app = Router::new().route( -⋮---- -post(|| async { -⋮---- -.status(200) -.header("content-type", "application/json") -.body(axum::body::Body::from( -⋮---- -.unwrap() -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -async fn spawn_proxy_with_config(config: Config) -> String { -⋮---- -async fn embeddings_forwarded_to_backend() { -let mock_base = spawn_mock_backend().await; -let proxy_base = spawn_proxy_with_config(openai_config_with_base(&mock_base)).await; -⋮---- -.post(format!("{proxy_base}/v1/embeddings")) -.header("x-api-key", "test") -⋮---- -.body(r#"{"model":"text-embedding-3-small","input":"hello world"}"#) -.send() -⋮---- -.unwrap(); -⋮---- -assert_eq!(resp.status(), 200); -let body: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(body["object"], "list"); -assert!(body["data"].is_array()); -⋮---- -async fn embeddings_response_content_type_forwarded() { -⋮---- -.body(r#"{"model":"text-embedding-3-small","input":"hello"}"#) -⋮---- -.headers() -.get("content-type") -⋮---- -.to_str() -⋮---- -assert!(ct.contains("application/json"), "got content-type: {ct}"); -⋮---- -async fn embeddings_requires_auth() { -⋮---- -// Temporarily unset open-relay mode so auth is enforced -⋮---- -let mut c = openai_config_with_base(&mock_base); -c.openai_api_key = "sk-real".into(); -spawn_proxy_with_config(c).await -⋮---- -.post(format!("{proxy_strict}/v1/embeddings")) -⋮---- -assert_eq!(resp.status(), 401); -⋮---- -// Restore for other tests -⋮---- -// Silence the unused-variable warning — proxy_base was used above -⋮---- -async fn embeddings_not_routed_for_anthropic_backend() { -// Anthropic backend does not mount /v1/embeddings; route returns 404. -let proxy_base = spawn_proxy_with_config(anthropic_config()).await; -⋮---- -// Route not registered for Anthropic backend; fallback returns 404. -assert_eq!(resp.status(), 404); -⋮---- -async fn degradation_header_present_when_top_k_set() { -// We can't make a full messages round-trip without a real backend, but we can -// verify the compute_request_warnings function directly via the translator crate. -// The proxy-level injection is covered by the inject_degradation_header unit test. -use anyllm_translate::anthropic; -⋮---- -model: "claude-sonnet-4-6".to_string(), -⋮---- -messages: vec![anthropic::InputMessage { -⋮---- -top_k: Some(40), -⋮---- -let header_val = warnings.as_header_value().expect("should have warnings"); -assert!(header_val.contains("top_k"), "got: {header_val}"); -⋮---- -async fn degradation_header_absent_when_no_lossy_features() { -⋮---- -assert!(warnings.is_empty()); -assert!(warnings.as_header_value().is_none()); - - - -fn malformed_openai_response_fails_deserialization() { -let json = include_str!("../../../fixtures/openai/chat_completion_malformed.json"); -⋮---- -assert!( - - - -// Integration tests for the fallback chain module. -// Tests the FallbackChain logic, should_fallback predicate, and config parsing. -⋮---- -/// Helper: create a `BackendError::OpenAI(ApiError { .. })` with a given status code. -fn make_api_error(status: u16) -> anyllm_proxy::backend::BackendError { -⋮---- -message: format!("mock error {status}"), -error_type: "test".to_string(), -⋮---- -// -- Config parsing tests -- -⋮---- -fn config_roundtrip() { -⋮---- -let config = parse_fallback_config(yaml).unwrap(); -⋮---- -assert_eq!(chain.len(), 2); -assert_eq!(chain[0].name, "azure"); -assert_eq!(chain[1].env_prefix, "OPENAI_FB_"); -⋮---- -fn config_empty_chains() { -⋮---- -assert!(config.fallback_chains.is_empty()); -⋮---- -fn config_malformed_yaml_errors() { -⋮---- -assert!(parse_fallback_config(yaml).is_err()); -⋮---- -// -- should_fallback predicate tests -- -⋮---- -fn should_fallback_server_errors() { -assert!(FallbackChain::should_fallback(500, false)); -assert!(FallbackChain::should_fallback(502, false)); -assert!(FallbackChain::should_fallback(503, false)); -⋮---- -fn should_fallback_rate_limit() { -assert!(FallbackChain::should_fallback(429, false)); -⋮---- -fn should_fallback_connection_error() { -assert!(FallbackChain::should_fallback(0, true)); -⋮---- -fn should_not_fallback_client_errors() { -assert!(!FallbackChain::should_fallback(400, false)); -assert!(!FallbackChain::should_fallback(401, false)); -assert!(!FallbackChain::should_fallback(403, false)); -assert!(!FallbackChain::should_fallback(404, false)); -⋮---- -// -- FallbackChain integration tests -- -⋮---- -async fn primary_503_falls_back_to_secondary() { -let chain = FallbackChain::new(vec![ -⋮---- -.attempt_with_fallback(|spec, _| { -let name = spec.name.clone(); -⋮---- -Err(make_api_error(503)) -⋮---- -Ok("ok from secondary") -⋮---- -assert!(outcome.result.is_ok()); -assert_eq!(outcome.backend_name, "secondary"); -assert_eq!(outcome.backend_index, 1); -assert!(!outcome.exhausted); -⋮---- -async fn primary_400_does_not_fallback() { -⋮---- -Err(make_api_error(400)) -⋮---- -Ok("should not reach") -⋮---- -assert!(outcome.result.is_err()); -assert_eq!(outcome.backend_name, "primary"); -assert_eq!(outcome.backend_index, 0); -// Not exhausted because we stopped early on a non-retryable error. -⋮---- -async fn all_backends_fail_sets_exhausted() { -⋮---- -.attempt_with_fallback(|_spec, _| async move { Err(make_api_error(502)) }) -⋮---- -assert!(outcome.exhausted); -// Last backend tried. -assert_eq!(outcome.backend_index, 2); -assert_eq!(outcome.backend_name, "c"); -⋮---- -fn fallback_exhausted_header_value() { -// Verify the header constant is what callers expect. -assert_eq!(FALLBACK_EXHAUSTED_HEADER, "x-anyllm-fallback-exhausted"); - - - -use tokio::net::TcpListener; -⋮---- -async fn health_endpoint_returns_ok() { -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -⋮---- -axum::serve(listener, app).await.unwrap(); -⋮---- -let resp = reqwest::get(format!("http://{addr}/health")).await.unwrap(); -⋮---- -assert_eq!(resp.status(), 200); -⋮---- -let body: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(body, serde_json::json!({ "status": "ok" })); - - - -//! Live API integration tests against real OpenAI endpoints. -//! -//! All tests are `#[ignore]` so they never run in CI or default `cargo test`. -⋮---- -//! Run manually: -//! ```sh -//! OPENAI_API_KEY=sk-... cargo test --test live_api -- --ignored --test-threads=1 -//! ``` -⋮---- -use anyllm_proxy::server::routes; -⋮---- -use tokio::net::TcpListener; -⋮---- -/// Build a Config targeting the real OpenAI Chat Completions API. -/// Reads OPENAI_API_KEY from the environment; panics if absent. -fn test_config() -> Config { -⋮---- -std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set for live tests"); -⋮---- -openai_api_key: api_key.clone(), -openai_base_url: "https://api.openai.com".to_string(), -⋮---- -big_model: "gpt-4o-mini".to_string(), -small_model: "gpt-4o-mini".to_string(), -⋮---- -/// Spawn the proxy on a random port, return the base URL (e.g. "http://127.0.0.1:12345"). -async fn spawn_test_server(config: Config) -> String { -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -⋮---- -axum::serve(listener, app).await.unwrap(); -⋮---- -format!("http://{addr}") -⋮---- -/// Helper: build a reqwest client with the x-api-key header. -fn api_client() -> reqwest::Client { -⋮---- -// --------------------------------------------------------------------------- -// a) Non-streaming text completion -⋮---- -async fn live_openai_text() { -let base = spawn_test_server(test_config()).await; -let client = api_client(); -⋮---- -.post(format!("{base}/v1/messages")) -.header("x-api-key", "test-key") -.header("content-type", "application/json") -.json(&json!({ -⋮---- -.send() -⋮---- -.expect("request failed"); -⋮---- -assert_eq!(resp.status(), 200, "expected 200, got {}", resp.status()); -⋮---- -let body: Value = resp.json().await.expect("response is not valid JSON"); -⋮---- -assert_eq!(body["type"], "message", "type must be 'message'"); -assert_eq!(body["role"], "assistant", "role must be 'assistant'"); -assert_eq!( -⋮---- -// Content array must be non-empty with text. -let content = body["content"].as_array().expect("content must be array"); -assert!(!content.is_empty(), "content array must not be empty"); -⋮---- -assert_eq!(first["type"], "text"); -let text = first["text"].as_str().expect("text must be a string"); -assert!(!text.is_empty(), "text must not be empty"); -⋮---- -// Usage must have positive token counts. -⋮---- -assert!( -⋮---- -// b) Streaming SSE -⋮---- -async fn live_openai_streaming() { -⋮---- -assert_eq!(resp.status(), 200); -⋮---- -.headers() -.get("content-type") -.and_then(|v| v.to_str().ok()) -.unwrap_or(""); -⋮---- -// Read the full SSE body and parse event types. -let body = resp.text().await.expect("failed to read SSE body"); -⋮---- -.lines() -.filter_map(|line| line.strip_prefix("event: ")) -.collect(); -⋮---- -// At least one content_block_delta with text. -let has_delta = event_types.iter().any(|&e| e == "content_block_delta"); -assert!(has_delta, "expected at least one content_block_delta event"); -⋮---- -// Parse the data lines for content_block_delta to confirm text is present. -⋮---- -let mut lines_iter = body.lines().peekable(); -while let Some(line) = lines_iter.next() { -⋮---- -if let Some(data_line) = lines_iter.next() { -if let Some(json_str) = data_line.strip_prefix("data: ") { -⋮---- -&& val["delta"]["text"].as_str().is_some() -⋮---- -// c) Tool call -⋮---- -async fn live_openai_tool_call() { -⋮---- -assert_eq!(body["type"], "message"); -⋮---- -// Find a tool_use content block. -⋮---- -.iter() -.find(|b| b["type"] == "tool_use") -.expect("expected a tool_use content block in response"); -⋮---- -let id = tool_use["id"].as_str().expect("tool_use must have id"); -assert!(!id.is_empty(), "tool_use id must not be empty"); -⋮---- -let name = tool_use["name"].as_str().expect("tool_use must have name"); -assert_eq!(name, "get_weather", "tool name must be get_weather"); -⋮---- -assert!(input.is_object(), "tool_use input must be an object"); -⋮---- -// d) Authentication error -⋮---- -async fn live_openai_error() { -// Build config with an invalid API key. -⋮---- -openai_api_key: bad_key.to_string(), -⋮---- -backend_auth: config::BackendAuth::BearerToken(bad_key.to_string()), -⋮---- -let base = spawn_test_server(config).await; -⋮---- -assert_eq!(resp.status(), 401, "expected 401, got {}", resp.status()); -⋮---- -assert_eq!(body["type"], "error", "response type must be 'error'"); - - - -//! Live integration tests against Azure OpenAI endpoints. -//! -//! All tests are `#[ignore]` so they never run in CI or default `cargo test`. -⋮---- -//! Run manually: -//! ```sh -//! AZURE_OPENAI_API_KEY=... \ -//! AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com \ -//! AZURE_OPENAI_DEPLOYMENT=your-deployment \ -//! cargo test --test live_azure -- --ignored --test-threads=1 -//! ``` -⋮---- -use anyllm_proxy::server::routes; -⋮---- -use tokio::net::TcpListener; -⋮---- -fn azure_test_config() -> Config { -⋮---- -.expect("AZURE_OPENAI_API_KEY must be set for live Azure tests"); -⋮---- -.expect("AZURE_OPENAI_ENDPOINT must be set for live Azure tests"); -⋮---- -.expect("AZURE_OPENAI_DEPLOYMENT must be set for live Azure tests"); -⋮---- -std::env::var("AZURE_OPENAI_API_VERSION").unwrap_or_else(|_| "2024-10-21".to_string()); -⋮---- -let base_url = format!( -⋮---- -big_model: deployment.clone(), -⋮---- -async fn spawn_test_server(config: Config) -> String { -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -⋮---- -axum::serve(listener, app).await.unwrap(); -⋮---- -format!("http://127.0.0.1:{}", addr.port()) -⋮---- -/// Verify a basic non-streaming request through Azure OpenAI. -⋮---- -async fn azure_non_streaming_hello() { -let base = spawn_test_server(azure_test_config()).await; -⋮---- -.post(format!("{base}/v1/messages")) -.header("x-api-key", "test-key") -.header("content-type", "application/json") -.header("anthropic-version", "2023-06-01") -.json(&json!({ -⋮---- -.send() -⋮---- -.unwrap(); -⋮---- -assert_eq!(resp.status(), 200, "body: {}", resp.text().await.unwrap()); -⋮---- -/// Verify streaming through Azure OpenAI produces SSE events. -⋮---- -async fn azure_streaming_hello() { -⋮---- -assert_eq!(resp.status(), 200); -let body = resp.text().await.unwrap(); -assert!( - - - -// Live integration tests for the Bedrock backend. -// Requires AWS credentials in the environment. Run with: -// AWS_REGION=us-east-1 AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... \ -// cargo test --test live_bedrock -- --ignored --test-threads=1 -⋮---- -use reqwest::Client; -use serde_json::json; -⋮---- -/// Start the proxy with Bedrock backend and return the base URL. -fn start_proxy() -> String { -// These tests are run manually with real credentials. -// The proxy must be started externally, or we construct a test server here. -let port = std::env::var("TEST_PROXY_PORT").unwrap_or_else(|_| "3099".to_string()); -format!("http://127.0.0.1:{port}") -⋮---- -async fn bedrock_non_streaming() { -let base = start_proxy(); -⋮---- -.post(format!("{base}/v1/messages")) -.header("x-api-key", "test-key") -.header("content-type", "application/json") -.json(&json!({ -⋮---- -.send() -⋮---- -.expect("request failed"); -⋮---- -let status = resp.status().as_u16(); -let body: serde_json::Value = resp.json().await.expect("invalid JSON response"); -⋮---- -assert_eq!(status, 200, "unexpected status: {body}"); -assert_eq!(body["type"], "message"); -assert!(body["content"].as_array().map_or(false, |a| !a.is_empty())); -⋮---- -async fn bedrock_streaming() { -⋮---- -assert_eq!(status, 200, "expected 200 for streaming"); -⋮---- -let body = resp.text().await.expect("failed to read body"); -assert!( - - - -//! Live integration tests for the OpenAI Responses API backend. -//! -//! All tests are `#[ignore]` so they never run in CI or default `cargo test`. -⋮---- -//! Run manually: -//! ```sh -//! OPENAI_API_KEY=sk-... cargo test --test live_responses -- --ignored --test-threads=1 -//! ``` -⋮---- -use anyllm_proxy::server::routes; -⋮---- -use tokio::net::TcpListener; -⋮---- -/// Build a Config targeting the real OpenAI Responses API. -/// Reads OPENAI_API_KEY from the environment; panics if absent. -fn test_config() -> Config { -⋮---- -std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set for live tests"); -⋮---- -openai_api_key: api_key.clone(), -openai_base_url: "https://api.openai.com".to_string(), -⋮---- -big_model: "gpt-4o-mini".to_string(), -small_model: "gpt-4o-mini".to_string(), -⋮---- -/// Spawn the proxy on a random port, return the base URL (e.g. "http://127.0.0.1:12345"). -async fn spawn_test_server(config: Config) -> String { -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -⋮---- -axum::serve(listener, app).await.unwrap(); -⋮---- -format!("http://{addr}") -⋮---- -/// Helper: build a reqwest client. -fn api_client() -> reqwest::Client { -⋮---- -// --------------------------------------------------------------------------- -// a) Non-streaming text completion via Responses API -⋮---- -async fn responses_api_non_streaming() { -let base = spawn_test_server(test_config()).await; -let client = api_client(); -⋮---- -.post(format!("{base}/v1/messages")) -.header("x-api-key", "test-key") -.header("content-type", "application/json") -.json(&json!({ -⋮---- -.send() -⋮---- -.expect("request failed"); -⋮---- -assert_eq!(resp.status(), 200, "expected 200, got {}", resp.status()); -⋮---- -let body: Value = resp.json().await.expect("response is not valid JSON"); -⋮---- -assert_eq!(body["type"], "message", "type must be 'message'"); -assert_eq!(body["role"], "assistant", "role must be 'assistant'"); -assert_eq!( -⋮---- -// Content array must be non-empty with text. -let content = body["content"].as_array().expect("content must be array"); -assert!(!content.is_empty(), "content array must not be empty"); -⋮---- -assert_eq!(first["type"], "text"); -let text = first["text"].as_str().expect("text must be a string"); -assert!(!text.is_empty(), "text must not be empty"); -⋮---- -// Usage must have positive token counts. -⋮---- -assert!( -⋮---- -// b) Streaming SSE via Responses API -⋮---- -async fn responses_api_streaming() { -⋮---- -assert_eq!(resp.status(), 200); -⋮---- -.headers() -.get("content-type") -.and_then(|v| v.to_str().ok()) -.unwrap_or(""); -⋮---- -// Read the full SSE body and parse event types. -let body = resp.text().await.expect("failed to read SSE body"); -⋮---- -.lines() -.filter_map(|line| line.strip_prefix("event: ")) -.collect(); -⋮---- -// At least one content_block_delta with text. -let has_delta = event_types.iter().any(|&e| e == "content_block_delta"); -assert!(has_delta, "expected at least one content_block_delta event"); -⋮---- -// Parse the data lines for content_block_delta to confirm text is present. -⋮---- -let mut lines_iter = body.lines().peekable(); -while let Some(line) = lines_iter.next() { -⋮---- -if let Some(data_line) = lines_iter.next() { -if let Some(json_str) = data_line.strip_prefix("data: ") { -⋮---- -&& val["delta"]["text"].as_str().is_some() - - - -// Test: server shuts down cleanly on signal, in-flight requests complete. -⋮---- -use anyllm_proxy::server::routes; -use std::time::Duration; -⋮---- -fn test_config() -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: "https://api.openai.com".to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: config::BackendAuth::BearerToken("test-key".into()), -⋮---- -async fn server_shuts_down_cleanly() { -let app = routes::app(test_config()); -let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -⋮---- -.with_graceful_shutdown(async { -shutdown_rx.await.ok(); -⋮---- -.unwrap(); -⋮---- -// Verify server is up -let resp = reqwest::get(format!("http://{addr}/health")).await.unwrap(); -assert_eq!(resp.status(), 200); -⋮---- -// Send shutdown signal -shutdown_tx.send(()).unwrap(); -⋮---- -// Server task should complete within a reasonable time -⋮---- -assert!(result.is_ok(), "server did not shut down within 5 seconds"); -assert!(result.unwrap().is_ok(), "server task panicked"); -⋮---- -async fn in_flight_health_completes_during_shutdown() { -⋮---- -// Start a request -let health_url = format!("http://{addr}/health"); -let resp = reqwest::get(&health_url).await.unwrap(); -⋮---- -// Signal shutdown -⋮---- -// Server should finish -⋮---- -async fn new_connections_refused_after_shutdown() { -⋮---- -// Shut down and wait for server to exit -⋮---- -.unwrap() -⋮---- -// New connection should fail (connection refused) -let result = reqwest::get(format!("http://{addr}/health")).await; -assert!( - - - -// Anthropic error types and status codes -⋮---- -/// Anthropic API error response wrapper. -/// -/// See -⋮---- -pub struct ErrorResponse { -⋮---- -pub response_type: String, // always "error" -⋮---- -/// Inner error object containing the error type and human-readable message. -⋮---- -pub struct ErrorDetail { -⋮---- -/// Anthropic API error type identifiers, mapped to HTTP status codes. -⋮---- -pub enum ErrorType { -/// 400: Issue with the format or content of the request. -⋮---- -/// 401: Issue with the API key. -⋮---- -/// 402: Issue with billing or payment. -⋮---- -/// 403: API key lacks permission for the resource. -⋮---- -/// 404: Requested resource not found. -⋮---- -/// 413: Request exceeds maximum allowed size (32 MB). -⋮---- -/// 429: Account has hit a rate limit. -⋮---- -/// 500: Unexpected internal error. -⋮---- -/// 529: API is temporarily overloaded. -⋮---- -mod tests { -⋮---- -use pretty_assertions::assert_eq; -use serde_json::json; -⋮---- -fn deserialize_error_response() { -let j = json!({ -⋮---- -let err: ErrorResponse = serde_json::from_value(j).unwrap(); -assert_eq!(err.response_type, "error"); -assert_eq!(err.error.error_type, ErrorType::InvalidRequestError); -assert_eq!(err.error.message, "max_tokens is required"); -assert!(err.request_id.is_none()); -⋮---- -fn deserialize_error_with_request_id() { -⋮---- -assert_eq!(err.request_id.as_deref(), Some("req_01234")); -assert_eq!(err.error.error_type, ErrorType::AuthenticationError); -⋮---- -fn round_trip() { -⋮---- -response_type: "error".into(), -⋮---- -message: "Too many requests".into(), -⋮---- -request_id: Some("req_abc".into()), -⋮---- -let serialized = serde_json::to_string(&err).unwrap(); -let deserialized: ErrorResponse = serde_json::from_str(&serialized).unwrap(); -assert_eq!(deserialized.error.error_type, ErrorType::RateLimitError); -assert_eq!(deserialized.error.message, "Too many requests"); -assert_eq!(deserialized.request_id.as_deref(), Some("req_abc")); -⋮---- -fn all_error_type_variants() { -⋮---- -let val = json!(s); -let parsed: ErrorType = serde_json::from_value(val).unwrap(); -assert_eq!(parsed, expected, "failed for {}", s); -⋮---- -// Round-trip: serialize back and compare -let re_serialized = serde_json::to_value(&parsed).unwrap(); -assert_eq!(re_serialized.as_str().unwrap(), s); -⋮---- -fn optional_request_id_omitted_when_none() { -⋮---- -message: "internal".into(), -⋮---- -let j = serde_json::to_value(&err).unwrap(); -assert!(!j.as_object().unwrap().contains_key("request_id")); - - - -// Anthropic SSE streaming event types -⋮---- -/// Top-level SSE event, internally tagged on `"type"`. -/// -/// See -⋮---- -pub enum StreamEvent { -⋮---- -/// Data payload for the message_start event. -⋮---- -pub struct MessageStartData { -⋮---- -/// Content block delta: text or tool input JSON. -⋮---- -pub enum Delta { -⋮---- -/// Signature delta: sent just before content_block_stop for thinking blocks. -/// Used to verify integrity of the thinking block. -⋮---- -/// Top-level message changes (stop_reason, stop_sequence). -⋮---- -pub struct MessageDeltaData { -⋮---- -/// Cumulative output token count in message_delta events. -⋮---- -pub struct DeltaUsage { -⋮---- -/// Error event in the SSE stream. -⋮---- -pub struct StreamError { -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn deserialize_message_start() { -let j = json!({ -⋮---- -let event: StreamEvent = serde_json::from_value(j).unwrap(); -⋮---- -assert_eq!(message.id, "msg_1nZdL29xx5MUA1yADyHTEsnR8uuvGzszyY"); -assert_eq!(message.role, "assistant"); -assert_eq!(message.usage.input_tokens, 25); -⋮---- -other => panic!("expected MessageStart, got {:?}", other), -⋮---- -fn deserialize_content_block_start_text() { -⋮---- -assert_eq!(index, 0); -⋮---- -ContentBlock::Text { text } => assert_eq!(text, ""), -_ => panic!("expected ContentBlock::Text"), -⋮---- -other => panic!("expected ContentBlockStart, got {:?}", other), -⋮---- -fn deserialize_content_block_start_tool_use() { -⋮---- -assert_eq!(index, 1); -⋮---- -assert_eq!(id, "toolu_xyz"); -assert_eq!(name, "get_weather"); -⋮---- -_ => panic!("expected ContentBlock::ToolUse"), -⋮---- -fn deserialize_text_delta() { -⋮---- -Delta::TextDelta { text } => assert_eq!(text, "Hello"), -_ => panic!("expected Delta::TextDelta"), -⋮---- -other => panic!("expected ContentBlockDelta, got {:?}", other), -⋮---- -fn deserialize_input_json_delta() { -⋮---- -assert_eq!(partial_json, "{\"location\":"); -⋮---- -_ => panic!("expected Delta::InputJsonDelta"), -⋮---- -fn deserialize_content_block_stop() { -let j = json!({"type": "content_block_stop", "index": 0}); -⋮---- -StreamEvent::ContentBlockStop { index } => assert_eq!(index, 0), -other => panic!("expected ContentBlockStop, got {:?}", other), -⋮---- -fn deserialize_message_delta() { -⋮---- -assert_eq!(delta.stop_reason, Some(StopReason::EndTurn)); -assert!(delta.stop_sequence.is_none()); -assert_eq!(usage.unwrap().output_tokens, 15); -⋮---- -other => panic!("expected MessageDelta, got {:?}", other), -⋮---- -fn deserialize_message_stop() { -let j = json!({"type": "message_stop"}); -⋮---- -other => panic!("expected MessageStop, got {:?}", other), -⋮---- -fn deserialize_ping() { -let j = json!({"type": "ping"}); -⋮---- -other => panic!("expected Ping, got {:?}", other), -⋮---- -fn deserialize_error_event() { -⋮---- -assert_eq!(error.error_type, "overloaded_error"); -assert_eq!(error.message, "Overloaded"); -⋮---- -other => panic!("expected Error, got {:?}", other), -⋮---- -fn round_trip_text_delta_event() { -⋮---- -text: "world".into(), -⋮---- -let serialized = serde_json::to_string(&event).unwrap(); -let deserialized: StreamEvent = serde_json::from_str(&serialized).unwrap(); -⋮---- -Delta::TextDelta { text } => assert_eq!(text, "world"), -⋮---- -fn thinking_delta_roundtrip() { -⋮---- -assert_eq!(thinking, "Let me think..."); -⋮---- -_ => panic!("expected Delta::ThinkingDelta"), -⋮---- -fn deserialize_signature_delta() { -⋮---- -assert_eq!(signature, "EqQBCgIYAhIM1gbcDa9GJwZA2b3h"); -⋮---- -_ => panic!("expected Delta::SignatureDelta"), - - - -/// Gemini generateContent API request types. -pub mod request; -/// Gemini generateContent API response types. -pub mod response; -⋮---- -// Re-export commonly used types - - - -// Gemini generateContent API response types. -// -// Shared `Content` and `Part` types are imported from `request.rs`. -⋮---- -use super::request::Content; -⋮---- -/// Response from `/models/{model}:generateContent` (and streaming chunks). -⋮---- -pub struct GenerateContentResponse { -⋮---- -/// A single candidate response from the model. -⋮---- -pub struct Candidate { -⋮---- -/// Why the model stopped generating. -⋮---- -pub enum FinishReason { -⋮---- -/// Catch-all for values added by the API in the future. -⋮---- -/// Token usage metadata. -⋮---- -pub struct UsageMetadata { -⋮---- -/// Per-category safety rating returned with each candidate. -⋮---- -pub struct SafetyRating { -⋮---- -mod tests { -⋮---- -use crate::gemini::request::Part; -use serde_json::json; -⋮---- -fn deserialize_basic_text_response() { -let j = json!({ -⋮---- -let resp: GenerateContentResponse = serde_json::from_value(j).unwrap(); -assert_eq!(resp.candidates.len(), 1); -assert_eq!(resp.candidates[0].content.parts[0].text.as_deref(), Some("Hello!")); -assert_eq!(resp.candidates[0].finish_reason, Some(FinishReason::STOP)); -let usage = resp.usage_metadata.unwrap(); -assert_eq!(usage.prompt_token_count, 5); -assert_eq!(usage.candidates_token_count, 3); -assert_eq!(usage.total_token_count, 8); -⋮---- -fn deserialize_tool_call_response() { -⋮---- -let fc = resp.candidates[0].content.parts[0].function_call.as_ref().unwrap(); -assert_eq!(fc.name, "get_weather"); -assert_eq!(fc.args["city"], "London"); -⋮---- -fn deserialize_usage_metadata() { -⋮---- -let u = resp.usage_metadata.unwrap(); -assert_eq!(u.prompt_token_count, 10); -assert_eq!(u.cached_content_token_count, 5); -⋮---- -fn finish_reason_stop_and_max_tokens() { -let stop: FinishReason = serde_json::from_value(json!("STOP")).unwrap(); -let max: FinishReason = serde_json::from_value(json!("MAX_TOKENS")).unwrap(); -assert_eq!(stop, FinishReason::STOP); -assert_eq!(max, FinishReason::MAX_TOKENS); -⋮---- -fn finish_reason_unknown_variant() { -let fr: FinishReason = serde_json::from_value(json!("BLOCKLIST")).unwrap(); -assert_eq!(fr, FinishReason::Unknown); -⋮---- -fn empty_candidates() { -let j = json!({"candidates": []}); -⋮---- -assert!(resp.candidates.is_empty()); -assert!(resp.usage_metadata.is_none()); -⋮---- -fn candidate_without_finish_reason() { -⋮---- -assert!(resp.candidates[0].finish_reason.is_none()); -⋮---- -fn safety_rating_deserialization() { -⋮---- -let sr = &resp.candidates[0].safety_ratings.as_ref().unwrap()[0]; -assert_eq!(sr.category, "HARM_CATEGORY_HATE_SPEECH"); -assert_eq!(sr.probability, "NEGLIGIBLE"); -⋮---- -fn round_trip_response() { -⋮---- -candidates: vec![Candidate { -⋮---- -usage_metadata: Some(UsageMetadata { -⋮---- -model_version: Some("gemini-2.0-flash".into()), -⋮---- -let json_str = serde_json::to_string(&resp).unwrap(); -let back: GenerateContentResponse = serde_json::from_str(&json_str).unwrap(); -assert_eq!(back.candidates.len(), 1); -assert_eq!(back.candidates[0].finish_reason, Some(FinishReason::STOP)); -assert_eq!(back.model_version.as_deref(), Some("gemini-2.0-flash")); -⋮---- -fn streaming_partial_content() { -// Streaming chunks use the same type but may have partial content -⋮---- -assert_eq!(resp.candidates[0].content.parts[0].text.as_deref(), Some("Hello")); - - - -// crates/translator/src/mapping/batch_map.rs -// Pure translation between Anthropic batch JSONL format and OpenAI batch JSONL format. -// No I/O. All functions are deterministic. -⋮---- -use crate::openai; -⋮---- -/// Translate one Anthropic batch request item into an OpenAI JSONL batch line. -/// -/// OpenAI format: `{"custom_id":"…","method":"POST","url":"/v1/chat/completions","body":{…}}` -pub fn batch_request_item_to_openai_jsonl_line(item: &BatchRequestItem) -> String { -let openai_req = anthropic_to_openai_request(&item.params); -⋮---- -serde_json::to_string(&line).expect("infallible") -⋮---- -/// Translate a complete Anthropic batch request into OpenAI JSONL (newline-separated lines). -pub fn translate_batch_to_openai_jsonl(items: &[BatchRequestItem]) -> String { -⋮---- -.iter() -.map(batch_request_item_to_openai_jsonl_line) -⋮---- -.join("\n") -⋮---- -/// Translate one OpenAI batch output JSONL line into an Anthropic result JSONL line. -⋮---- -/// OpenAI output format: -/// `{"id":"br_…","custom_id":"…","response":{"status_code":200,"body":{ChatCompletion}},"error":null}` -⋮---- -/// `model` is the Anthropic model name to embed in the resulting MessageResponse. -pub fn translate_openai_result_line(line: &str, model: &str) -> Result { -⋮---- -serde_json::from_str(line).map_err(|e| format!("JSON parse error: {e}"))?; -⋮---- -.as_str() -.ok_or("missing custom_id field")? -.to_string(); -⋮---- -let variant = if let Some(err) = v.get("error").filter(|e| !e.is_null()) { -⋮---- -.unwrap_or("unknown batch error") -⋮---- -} else if let Some(response) = v.get("response").filter(|r| !r.is_null()) { -let status = response["status_code"].as_u64().unwrap_or(0); -⋮---- -serde_json::from_value(body.clone()) -.map_err(|e| format!("failed to parse ChatCompletionResponse: {e}"))?; -let message = openai_to_anthropic_response(&completion, model); -⋮---- -message: format!("backend status {status}"), -⋮---- -serde_json::to_string(&item).map_err(|e| format!("serialize error: {e}")) -⋮---- -mod tests { -⋮---- -fn make_request_item() -> crate::anthropic::batch::BatchRequestItem { -⋮---- -custom_id: "req-1".to_string(), -⋮---- -.unwrap(), -⋮---- -fn request_item_serializes_to_openai_jsonl_line() { -let item = make_request_item(); -let line = batch_request_item_to_openai_jsonl_line(&item); -let v: serde_json::Value = serde_json::from_str(&line).unwrap(); -assert_eq!(v["custom_id"], "req-1"); -assert_eq!(v["method"], "POST"); -assert_eq!(v["url"], "/v1/chat/completions"); -assert!(v["body"]["messages"].is_array()); -⋮---- -fn translate_openai_success_result_to_anthropic() { -⋮---- -let line = serde_json::to_string(&openai_line).unwrap(); -let result = translate_openai_result_line(&line, "claude-3-5-sonnet-20241022").unwrap(); -let v: serde_json::Value = serde_json::from_str(&result).unwrap(); -⋮---- -assert_eq!(v["result"]["type"], "succeeded"); -assert_eq!(v["result"]["message"]["role"], "assistant"); -⋮---- -fn translate_openai_error_result_to_anthropic() { -⋮---- -assert_eq!(v["custom_id"], "req-2"); -assert_eq!(v["result"]["type"], "errored"); -assert!(v["result"]["error"]["message"].as_str().unwrap().contains("quota")); -⋮---- -fn translate_batch_items_to_openai_jsonl() { -let items = vec![make_request_item(), { -⋮---- -let jsonl = translate_batch_to_openai_jsonl(&items); -let lines: Vec<&str> = jsonl.lines().collect(); -assert_eq!(lines.len(), 2); -let v: serde_json::Value = serde_json::from_str(lines[1]).unwrap(); - - - -// Error and stop_reason mapping -⋮---- -use crate::anthropic; -use crate::openai; -⋮---- -/// Map an HTTP status code from OpenAI to the corresponding Anthropic error type. -/// -/// Anthropic: -/// OpenAI: -pub fn openai_status_to_anthropic_error_type(status: u16) -> anthropic::ErrorType { -⋮---- -// 408 has no direct Anthropic equivalent; OverloadedError tells -// clients to retry with backoff, which is correct for timeouts. -⋮---- -// 529 (Cloudflare overloaded) and 503 both indicate transient -// capacity issues; OverloadedError triggers client-side backoff. -⋮---- -/// Map an Anthropic error type to an HTTP status code. -⋮---- -pub fn anthropic_error_type_to_status(error_type: &anthropic::ErrorType) -> u16 { -⋮---- -/// Convert an HTTP status code and error message to an Anthropic error response. -/// Works for any backend (OpenAI, Gemini, etc.) since it only needs standard HTTP semantics. -pub fn status_to_anthropic_error( -⋮---- -response_type: "error".to_string(), -⋮---- -error_type: openai_status_to_anthropic_error_type(status), -message: message.to_string(), -⋮---- -/// Convert an OpenAI error response to an Anthropic error response. -pub fn openai_to_anthropic_error( -⋮---- -status_to_anthropic_error(status, &openai_err.error.message, request_id) -⋮---- -/// Create an Anthropic error response from scratch. -⋮---- -pub fn create_anthropic_error( -⋮---- -mod tests { -⋮---- -use pretty_assertions::assert_eq; -⋮---- -fn status_to_error_type_known_codes() { -⋮---- -assert_eq!( -⋮---- -fn unknown_status_maps_to_api_error() { -⋮---- -fn error_type_to_status_all_variants() { -⋮---- -assert_eq!(anthropic_error_type_to_status(error_type), *expected_status,); -⋮---- -fn round_trip_error_type_through_status() { -// Every error type should survive a round-trip through status code -// (except OverloadedError: 529 is not in the 500..=502 range, but -// 529 maps back to OverloadedError via the explicit match arm). -⋮---- -let status = anthropic_error_type_to_status(error_type); -let back = openai_status_to_anthropic_error_type(status); -assert_eq!(&back, error_type, "round-trip failed for {:?}", error_type); -⋮---- -fn openai_error_to_anthropic_error() { -⋮---- -message: "Invalid API key".into(), -error_type: "invalid_request_error".into(), -⋮---- -code: Some("invalid_api_key".into()), -⋮---- -let result = openai_to_anthropic_error(&openai_err, 401, Some("req_123".into())); -⋮---- -assert_eq!(result.response_type, "error"); -⋮---- -assert_eq!(result.error.message, "Invalid API key"); -assert_eq!(result.request_id.as_deref(), Some("req_123")); -⋮---- -fn openai_error_to_anthropic_no_request_id() { -⋮---- -message: "Rate limit exceeded".into(), -error_type: "rate_limit_error".into(), -⋮---- -let result = openai_to_anthropic_error(&openai_err, 429, None); -⋮---- -assert!(result.request_id.is_none()); -⋮---- -fn create_anthropic_error_helper() { -let err = create_anthropic_error( -⋮---- -"Model not found".into(), -Some("req_abc".into()), -⋮---- -assert_eq!(err.response_type, "error"); -assert_eq!(err.error.error_type, anthropic::ErrorType::NotFoundError); -assert_eq!(err.error.message, "Model not found"); -assert_eq!(err.request_id.as_deref(), Some("req_abc")); -⋮---- -fn create_anthropic_error_no_request_id() { -⋮---- -"Internal error".into(), -⋮---- -assert_eq!(err.error.error_type, anthropic::ErrorType::ApiError); -assert!(err.request_id.is_none()); -⋮---- -// --- Fixture deserialization tests --- -⋮---- -fn fixture_openai_error_401_deserializes() { -let json = include_str!("../../../../fixtures/openai/error_401.json"); -let err: openai::errors::ErrorResponse = serde_json::from_str(json).unwrap(); -assert_eq!(err.error.code.as_deref(), Some("invalid_api_key")); -⋮---- -fn fixture_openai_error_429_deserializes() { -let json = include_str!("../../../../fixtures/openai/error_429.json"); -⋮---- -assert!(err.error.message.contains("Rate limit")); -⋮---- -fn fixture_openai_error_500_deserializes() { -let json = include_str!("../../../../fixtures/openai/error_500.json"); -⋮---- -assert_eq!(err.error.error_type, "server_error"); -⋮---- -fn fixture_anthropic_error_invalid_request_deserializes() { -let json = include_str!("../../../../fixtures/anthropic/error_invalid_request.json"); -let err: anthropic::errors::ErrorResponse = serde_json::from_str(json).unwrap(); -⋮---- -fn fixture_anthropic_error_rate_limit_deserializes() { -let json = include_str!("../../../../fixtures/anthropic/error_rate_limit.json"); -⋮---- -assert_eq!(err.error.error_type, anthropic::ErrorType::RateLimitError); -assert_eq!(err.request_id.as_deref(), Some("req_01XYZ")); -⋮---- -// --- Fixture translation tests --- -⋮---- -fn fixture_openai_401_translates_to_anthropic_auth_error() { -⋮---- -let openai_err: openai::errors::ErrorResponse = serde_json::from_str(json).unwrap(); -let anthropic_err = openai_to_anthropic_error(&openai_err, 401, Some("req_test".into())); -⋮---- -fn fixture_openai_429_translates_to_anthropic_rate_limit() { -⋮---- -let anthropic_err = openai_to_anthropic_error(&openai_err, 429, None); -⋮---- -fn fixture_openai_500_translates_to_anthropic_api_error() { -⋮---- -let anthropic_err = openai_to_anthropic_error(&openai_err, 500, None); - - - -/// Collects feature degradation notices produced during request translation. -/// -/// Returned to the proxy layer so it can inject an `x-anyllm-degradation` response -/// header for clients to inspect. This makes silent drops visible without changing -/// the Anthropic-compatible response body. -⋮---- -pub struct TranslationWarnings { -⋮---- -impl TranslationWarnings { -pub fn add(&mut self, feature: &'static str) { -self.items.push(feature); -⋮---- -pub fn is_empty(&self) -> bool { -self.items.is_empty() -⋮---- -/// Returns a comma-separated string suitable for an HTTP header value, -/// or `None` if no features were dropped. -pub fn as_header_value(&self) -> Option { -if self.items.is_empty() { -⋮---- -Some(self.items.join(", ")) -⋮---- -mod tests { -⋮---- -fn empty_returns_none() { -⋮---- -assert!(w.is_empty()); -assert!(w.as_header_value().is_none()); -⋮---- -fn single_item() { -⋮---- -w.add("top_k"); -assert!(!w.is_empty()); -assert_eq!(w.as_header_value().unwrap(), "top_k"); -⋮---- -fn multiple_items_comma_separated() { -⋮---- -w.add("cache_control"); -w.add("document_blocks"); -assert_eq!( - - - -// Minimal reqwest forwarding client for the middleware layer. -// No retry logic; users can add their own Tower retry layer or use the proxy crate. -⋮---- -use reqwest::Client; -use thiserror::Error; -⋮---- -/// Errors from the forwarding client. -⋮---- -pub enum ForwardingError { -⋮---- -/// Backend returned a non-2xx status with a body we could read. -⋮---- -impl ForwardingError { -/// Extract (message, HTTP status) if this is an API error. -pub fn api_error_details(&self) -> Option<(&str, u16)> { -⋮---- -Self::ApiError { status, body } => Some((body.as_str(), *status)), -⋮---- -/// Minimal HTTP client that forwards OpenAI Chat Completions requests to a backend URL. -⋮---- -pub struct ForwardingClient { -⋮---- -impl ForwardingClient { -/// Create a client targeting `{backend_url}/v1/chat/completions`. -/// No retry logic; callers can add their own Tower retry layer. -pub fn new(backend_url: &str, api_key: &str) -> Self { -let base = backend_url.trim_end_matches('/'); -⋮---- -chat_completions_url: format!("{base}/v1/chat/completions"), -api_key: api_key.to_string(), -⋮---- -/// Send a non-streaming chat completion request. -pub async fn chat_completion( -⋮---- -.post(&self.chat_completions_url) -.bearer_auth(&self.api_key) -.json(req) -.send() -⋮---- -let status = response.status().as_u16(); -if !response.status().is_success() { -let body = response.text().await.unwrap_or_default(); -return Err(ForwardingError::ApiError { status, body }); -⋮---- -let body = response.text().await?; -⋮---- -Ok((resp, status)) -⋮---- -/// Send a streaming chat completion request, returning the raw response for SSE parsing. -pub async fn chat_completion_stream( -⋮---- -Ok(response) - - - -// Core handler logic for the Anthropic compatibility middleware. -// Shared by both the Router factory and the Tower Layer. -⋮---- -use std::convert::Infallible; -use std::sync::Arc; -⋮---- -use axum::extract::Json; -use axum::http::StatusCode; -⋮---- -use bytes::BytesMut; -use futures::StreamExt; -use tokio::sync::mpsc; -use tokio_stream::wrappers::ReceiverStream; -⋮---- -use crate::anthropic::streaming::StreamEvent; -⋮---- -use crate::translate; -⋮---- -use super::client::ForwardingError; -use super::MiddlewareState; -⋮---- -/// Handle a POST /v1/messages request (streaming or non-streaming). -pub(crate) async fn handle_messages( -⋮---- -if body.stream == Some(true) { -return handle_streaming(state, body).await.into_response(); -⋮---- -handle_non_streaming(state, body).await -⋮---- -async fn handle_non_streaming(state: Arc, body: MessageCreateRequest) -> Response { -let original_model = body.model.clone(); -⋮---- -Err(e) => return translation_error_response(&e.to_string()), -⋮---- -match state.client.chat_completion(&openai_req).await { -⋮---- -(StatusCode::OK, Json(anthropic_resp)).into_response() -⋮---- -Err(e) => forwarding_error_response(e), -⋮---- -async fn handle_streaming( -⋮---- -// Send error as SSE event, then close -let _ = tx.send(Ok(error_to_sse_event(&e.to_string()))).await; -return Sse::new(ReceiverStream::new(rx)).keep_alive(KeepAlive::default()); -⋮---- -match state.client.chat_completion_stream(&openai_req).await { -⋮---- -let completed = read_sse_frames(response, &tx, |json_str| { -⋮---- -return Some(translator.finish()); -⋮---- -return Some(translator.process_chunk(&chunk)); -⋮---- -let events = translator.finish(); -send_events(&tx, &events).await; -⋮---- -Sse::new(ReceiverStream::new(rx)).keep_alive(KeepAlive::default()) -⋮---- -// --- SSE helpers --- -⋮---- -/// Format a StreamEvent as an axum SSE Event with the Anthropic event type name. -pub fn stream_event_to_sse(event: &StreamEvent) -> Result { -⋮---- -Ok(Event::default().event(event_type).data(data)) -⋮---- -fn error_to_sse_event(message: &str) -> Event { -⋮---- -error_type: "api_error".to_string(), -message: message.to_string(), -⋮---- -// Best-effort; if serialization fails, build JSON safely to avoid injection -stream_event_to_sse(&event).unwrap_or_else(|_| { -⋮---- -Event::default().event("error").data(fallback.to_string()) -⋮---- -/// Maximum SSE buffer size (10 MB). Protects against unbounded memory growth. -⋮---- -/// Find the first SSE frame boundary (`\n\n` or `\r\n\r\n`) in a byte slice. -/// Returns `(position, delimiter_length)` so the caller can skip the full delimiter. -fn find_double_newline(buf: &[u8], start: usize) -> Option<(usize, usize)> { -let len = buf.len(); -⋮---- -while i < len.saturating_sub(1) { -⋮---- -return Some((i, 2)); -⋮---- -return Some((i, 4)); -⋮---- -/// Read SSE frames from a response, parse data lines, call `on_data` for each. -/// Returns true if stream completed normally. -async fn read_sse_frames( -⋮---- -let mut stream = response.bytes_stream(); -// Use a byte buffer to avoid corrupting multi-byte UTF-8 characters -// split across TCP chunk boundaries. -⋮---- -while let Some(chunk_result) = stream.next().await { -⋮---- -buffer.extend_from_slice(&bytes); -⋮---- -if buffer.len() > MAX_SSE_BUFFER_SIZE { -⋮---- -while let Some((pos, delim_len)) = find_double_newline(&buffer, search_from) { -frame_events.clear(); -⋮---- -for line in frame_str.lines() { -let line = line.trim(); -if let Some(json_str) = line.strip_prefix("data: ") { -if let Some(mut events) = on_data(json_str) { -frame_events.append(&mut events); -⋮---- -let _ = buffer.split_to(pos + delim_len); -// split_to shifted the buffer; restart search at the beginning -⋮---- -if !send_events(tx, &frame_events).await { -⋮---- -// Next chunk: resume scanning 3 bytes back from the end so a -// 4-byte delimiter (\r\n\r\n) straddling a chunk boundary is found. -search_from = buffer.len().saturating_sub(3); -⋮---- -/// Send translated events through the channel. Returns false if receiver is gone. -async fn send_events(tx: &mpsc::Sender>, events: &[StreamEvent]) -> bool { -⋮---- -if let Ok(sse_event) = stream_event_to_sse(event) { -if tx.send(Ok(sse_event)).await.is_err() { -⋮---- -// --- Error response helpers --- -⋮---- -fn translation_error_response(message: &str) -> Response { -⋮---- -message.to_string(), -⋮---- -(StatusCode::BAD_REQUEST, Json(err)).into_response() -⋮---- -fn forwarding_error_response(error: ForwardingError) -> Response { -if let Some((body, status)) = error.api_error_details() { -// Try to extract a message from the backend's JSON error body -⋮---- -.ok() -.and_then(|v| { -v.get("error") -.and_then(|e| e.get("message")) -.and_then(|m| m.as_str()) -.map(String::from) -⋮---- -.unwrap_or_else(|| body.to_string()); -⋮---- -.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); -return (http_status, Json(anthropic_err)).into_response(); -⋮---- -format!("Upstream error: {error}"), -⋮---- -(StatusCode::INTERNAL_SERVER_ERROR, Json(err)).into_response() - - - -//! Axum middleware for adding Anthropic Messages API compatibility to existing services. -//! -//! Requires the `middleware` feature: `anyllm_translate = { features = ["middleware"] }` -⋮---- -//! # Usage -⋮---- -//! ```rust,no_run -//! use anyllm_translate::TranslationConfig; -//! use anyllm_translate::middleware::{ -//! AnthropicCompatConfig, AnthropicTranslationLayer, anthropic_compat_router, -//! }; -//! use axum::Router; -⋮---- -//! let config = AnthropicCompatConfig::builder() -//! .backend_url("https://api.openai.com") -//! .api_key("sk-...") -//! .translation( -//! TranslationConfig::builder() -//! .model_map("haiku", "gpt-4o-mini") -//! .model_map("sonnet", "gpt-4o") -//! .build() -//! ) -//! .build(); -⋮---- -//! // Option A: Router factory -- adds POST /v1/messages -//! let app: Router = Router::new() -//! .merge(anthropic_compat_router(config.clone())); -⋮---- -//! // Option B: Tower Layer -- intercepts POST /v1/messages, passes other requests through -⋮---- -//! .layer(AnthropicTranslationLayer::new(config)); -//! ``` -⋮---- -mod client; -mod handler; -⋮---- -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -⋮---- -use axum::body::Body; -use axum::extract::Json; -⋮---- -use axum::routing::post; -use axum::Router; -⋮---- -pub use client::ForwardingError; -pub use handler::stream_event_to_sse; -⋮---- -use crate::config::TranslationConfig; -⋮---- -// --- Configuration --- -⋮---- -/// Configuration for the Anthropic compatibility middleware. -⋮---- -pub struct AnthropicCompatConfig { -/// Base URL of the OpenAI-compatible backend (e.g., `https://api.openai.com`). -⋮---- -/// API key for the backend (sent as Bearer token). -⋮---- -/// Translation settings (model mapping, lossy behavior). -⋮---- -impl AnthropicCompatConfig { -/// Create a builder for configuring the middleware. -pub fn builder() -> AnthropicCompatConfigBuilder { -⋮---- -/// Builder for [`AnthropicCompatConfig`]. -pub struct AnthropicCompatConfigBuilder { -⋮---- -impl AnthropicCompatConfigBuilder { -/// Set the base URL of the OpenAI-compatible backend (e.g., `https://api.openai.com`). -pub fn backend_url(mut self, url: impl Into) -> Self { -self.backend_url = url.into(); -⋮---- -/// Set the API key sent as a Bearer token to the backend. -pub fn api_key(mut self, key: impl Into) -> Self { -self.api_key = key.into(); -⋮---- -/// Set translation settings (model mapping, lossy behavior). -pub fn translation(mut self, config: TranslationConfig) -> Self { -⋮---- -/// Build the configuration. Does not validate; invalid URLs will fail at request time. -pub fn build(self) -> AnthropicCompatConfig { -⋮---- -// --- Shared state --- -⋮---- -/// Internal shared state for the middleware handler. -pub(crate) struct MiddlewareState { -⋮---- -fn make_state(config: AnthropicCompatConfig) -> Arc { -⋮---- -// --- Router factory --- -⋮---- -/// Create an axum [`Router`] that handles `POST /v1/messages`. -/// -/// Merge this into your existing router to add Anthropic Messages API compatibility. -/// Incoming Anthropic requests are translated to OpenAI Chat Completions format, -/// forwarded to the configured backend URL, and the response is translated back. -pub fn anthropic_compat_router(config: AnthropicCompatConfig) -> Router { -let state = make_state(config); -⋮---- -Router::new().route( -⋮---- -post( -⋮---- -// --- Tower Layer --- -⋮---- -/// Tower [`Layer`] that intercepts `POST /v1/messages` requests and translates them. -⋮---- -/// Other requests pass through to the inner service unchanged. -⋮---- -pub struct AnthropicTranslationLayer { -⋮---- -impl AnthropicTranslationLayer { -/// Create a new layer that will intercept `POST /v1/messages` and translate. -pub fn new(config: AnthropicCompatConfig) -> Self { -⋮---- -state: make_state(config), -⋮---- -type Service = AnthropicTranslationService; -⋮---- -fn layer(&self, inner: S) -> Self::Service { -⋮---- -/// Tower [`Service`] created by [`AnthropicTranslationLayer`]. -⋮---- -/// Intercepts `POST /v1/messages` and handles translation. -/// All other requests are forwarded to the inner service. -⋮---- -pub struct AnthropicTranslationService { -⋮---- -type Response = Response; -type Error = S::Error; -type Future = Pin> + Send>>; -⋮---- -fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { -self.inner.poll_ready(cx) -⋮---- -fn call(&mut self, req: Request) -> Self::Future { -// Only intercept POST /v1/messages -if req.method() == Method::POST && req.uri().path() == "/v1/messages" { -⋮---- -// Read the full body -⋮---- -match axum::body::to_bytes(req.into_body(), 32 * 1024 * 1024).await { -⋮---- -"Request body too large".to_string(), -⋮---- -return Ok((axum::http::StatusCode::PAYLOAD_TOO_LARGE, Json(err)) -.into_response()); -⋮---- -format!("Invalid JSON: {e}"), -⋮---- -return Ok( -(axum::http::StatusCode::BAD_REQUEST, Json(err)).into_response() -⋮---- -Ok(handler::handle_messages(state, anthropic_req).await) -⋮---- -// Pass through to inner service -let fut = self.inner.call(req); - - - -// OpenAI Chat Completions request/response types -⋮---- -// --- Request types --- -⋮---- -/// OpenAI Chat Completions API request body. -/// -/// See -⋮---- -pub struct ChatCompletionRequest { -⋮---- -/// Maps from Anthropic metadata.user_id. Compat spec: "Ignored". -/// See: https://docs.anthropic.com/en/api/openai-sdk#simple-fields -⋮---- -/// Compat spec: "Fully supported". -⋮---- -/// Captures OpenAI fields we don't need to translate (seed, logprobs, -/// logit_bias, n, reasoning_effort, etc.) and forwards them as-is. -/// Only fields requiring translation logic get explicit struct fields. -⋮---- -/// Options for streaming responses. -⋮---- -pub struct StreamOptions { -⋮---- -/// Stop sequence(s): single string or array. -⋮---- -pub enum Stop { -⋮---- -/// A message in the conversation. -⋮---- -pub struct ChatMessage { -⋮---- -/// Compat spec response: "Always empty". Present to avoid deserialization failure. -/// See: https://docs.anthropic.com/en/api/openai-sdk#response-fields -⋮---- -/// DeepSeek/Qwen thinking model output. Maps to/from Anthropic thinking blocks. -⋮---- -/// Message role: system, developer, user, assistant, or tool. -⋮---- -pub enum ChatRole { -⋮---- -/// Deprecated by OpenAI but still accepted. Compat spec lists function role messages. -/// See: https://docs.anthropic.com/en/api/openai-sdk#messages-array-fields -⋮---- -/// Content can be a plain string or an array of typed content parts (multimodal). -⋮---- -pub enum ChatContent { -⋮---- -/// Typed content part for multimodal messages. -⋮---- -pub enum ChatContentPart { -⋮---- -/// Image URL reference for vision requests. -⋮---- -pub struct ImageUrl { -⋮---- -/// Audio input for audio-capable models. -⋮---- -/// See -⋮---- -pub struct InputAudio { -pub data: String, // base64-encoded audio -pub format: String, // "wav", "mp3", etc. -⋮---- -/// File input for file-capable models. -⋮---- -/// See -⋮---- -pub struct FileInput { -⋮---- -pub file_data: Option, // base64-encoded or data URI -⋮---- -// --- Tool call from assistant --- -⋮---- -/// Tool call from the assistant. -⋮---- -/// See -⋮---- -pub struct ToolCall { -⋮---- -pub call_type: String, // always "function" -⋮---- -/// Function name and JSON arguments string. -⋮---- -pub struct FunctionCall { -⋮---- -pub arguments: String, // JSON string -⋮---- -// --- Tool definition --- -⋮---- -/// Tool definition wrapping a function. -⋮---- -pub struct ChatTool { -⋮---- -pub tool_type: String, // always "function" -⋮---- -/// Function definition with name, description, and parameters schema. -⋮---- -pub struct FunctionDef { -⋮---- -/// Compat spec: "Ignored". OpenAI accepts it; preserved for round-trip fidelity. -/// See: https://docs.anthropic.com/en/api/openai-sdk#tools--functions-fields -⋮---- -/// Tool choice: "auto", "none", "required", or named function. -⋮---- -pub enum ChatToolChoice { -Simple(String), // "auto", "none", "required" -⋮---- -/// Named tool choice specifying a specific function. -⋮---- -pub struct NamedToolChoice { -⋮---- -pub choice_type: String, // "function" -⋮---- -/// Function name for named tool choice. -⋮---- -pub struct NamedFunction { -⋮---- -/// Response format: text, json_object, or json_schema. -⋮---- -pub struct ResponseFormat { -⋮---- -// --- Response types --- -⋮---- -/// OpenAI Chat Completions API response body. -⋮---- -pub struct ChatCompletionResponse { -⋮---- -pub object: String, // "chat.completion" -⋮---- -/// Compat spec response: "Always empty". -⋮---- -/// A completion choice with message and finish reason. -⋮---- -pub struct Choice { -⋮---- -/// Why the model stopped: stop, length, tool_calls, content_filter, or function_call. -⋮---- -pub enum FinishReason { -⋮---- -/// Catch-all for provider-specific finish reasons (e.g. DeepSeek's -/// "insufficient_system_resource"). Serializes as "unknown". -⋮---- -/// Token usage: prompt, completion, and total. -⋮---- -pub struct ChatUsage { -⋮---- -/// Compat spec response: "Always empty". OpenAI returns reasoning_tokens, etc. -⋮---- -/// Compat spec response: "Always empty". OpenAI returns cached_tokens, etc. -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn deserialize_basic_request() { -let raw = json!({ -⋮---- -let req: ChatCompletionRequest = serde_json::from_value(raw).unwrap(); -assert_eq!(req.model, "gpt-4o"); -assert_eq!(req.messages.len(), 1); -assert_eq!(req.messages[0].role, ChatRole::User); -assert!(matches!(&req.messages[0].content, Some(ChatContent::Text(t)) if t == "Hello")); -assert!(req.max_tokens.is_none()); -assert!(req.tools.is_none()); -⋮---- -fn deserialize_request_with_tools() { -⋮---- -let tools = req.tools.unwrap(); -assert_eq!(tools.len(), 1); -assert_eq!(tools[0].function.name, "get_weather"); -assert!(tools[0].function.description.is_some()); -assert!(matches!(&req.tool_choice, Some(ChatToolChoice::Simple(s)) if s == "auto")); -⋮---- -fn deserialize_content_string_vs_parts() { -// String content -let msg_str: ChatMessage = serde_json::from_value(json!({ -⋮---- -.unwrap(); -assert!(matches!(&msg_str.content, Some(ChatContent::Text(t)) if t == "plain text")); -⋮---- -// Array content with text + image -let msg_parts: ChatMessage = serde_json::from_value(json!({ -⋮---- -assert_eq!(parts.len(), 2); -assert!( -⋮---- -other => panic!("expected Parts, got {:?}", other), -⋮---- -fn deserialize_response_with_choices() { -⋮---- -let resp: ChatCompletionResponse = serde_json::from_value(raw).unwrap(); -assert_eq!(resp.id, "chatcmpl-abc123"); -assert_eq!(resp.choices.len(), 1); -assert_eq!(resp.choices[0].finish_reason, Some(FinishReason::Stop)); -let usage = resp.usage.unwrap(); -assert_eq!(usage.prompt_tokens, 10); -assert_eq!(usage.total_tokens, 18); -⋮---- -fn deserialize_response_with_tool_calls() { -⋮---- -let tc = resp.choices[0].message.tool_calls.as_ref().unwrap(); -assert_eq!(tc.len(), 1); -assert_eq!(tc[0].id, "call_abc"); -assert_eq!(tc[0].function.name, "get_weather"); -assert_eq!(tc[0].function.arguments, "{\"location\":\"NYC\"}"); -assert_eq!(resp.choices[0].finish_reason, Some(FinishReason::ToolCalls)); -⋮---- -fn serialize_deserialize_roundtrip() { -⋮---- -model: "gpt-4o".into(), -messages: vec![ChatMessage { -⋮---- -max_tokens: Some(100), -⋮---- -temperature: Some(0.7), -⋮---- -stream: Some(true), -stream_options: Some(StreamOptions { -⋮---- -let json_str = serde_json::to_string(&req).unwrap(); -let roundtrip: ChatCompletionRequest = serde_json::from_str(&json_str).unwrap(); -assert_eq!(roundtrip.model, "gpt-4o"); -assert_eq!(roundtrip.max_tokens, Some(100)); -assert_eq!(roundtrip.stream, Some(true)); -assert!(roundtrip.stream_options.unwrap().include_usage); -⋮---- -fn stop_single_vs_array() { -let single: Stop = serde_json::from_value(json!("END")).unwrap(); -assert!(matches!(single, Stop::Single(s) if s == "END")); -⋮---- -let multi: Stop = serde_json::from_value(json!(["END", "STOP"])).unwrap(); -⋮---- -Stop::Multiple(v) => assert_eq!(v, vec!["END", "STOP"]), -_ => panic!("expected Multiple"), -⋮---- -fn tool_choice_simple_vs_named() { -let simple: ChatToolChoice = serde_json::from_value(json!("auto")).unwrap(); -assert!(matches!(simple, ChatToolChoice::Simple(s) if s == "auto")); -⋮---- -let named: ChatToolChoice = serde_json::from_value(json!({ -⋮---- -assert_eq!(n.choice_type, "function"); -assert_eq!(n.function.name, "my_tool"); -⋮---- -_ => panic!("expected Named"), -⋮---- -fn extra_fields_captured_via_flatten() { -⋮---- -assert_eq!(req.extra.get("logprobs"), Some(&json!(true))); -assert_eq!(req.extra.get("seed"), Some(&json!(42))); -⋮---- -fn reject_malformed_missing_model() { -⋮---- -assert!(result.is_err()); -⋮---- -fn deserialize_realistic_openai_response() { -// Real gpt-4o response with all fields OpenAI returns -⋮---- -assert_eq!(resp.id, "chatcmpl-AKj3MbOpNGPq"); -assert_eq!(resp.service_tier.as_deref(), Some("default")); -assert_eq!(resp.system_fingerprint.as_deref(), Some("fp_a7d06e42a7")); -assert!(resp.choices[0].logprobs.is_none()); -assert!(resp.choices[0].message.refusal.is_none()); -⋮---- -assert_eq!(usage.prompt_tokens, 12); -assert!(usage.completion_tokens_details.is_some()); -assert!(usage.prompt_tokens_details.is_some()); -⋮---- -fn deserialize_function_role_message() { -⋮---- -let msg: ChatMessage = serde_json::from_value(raw).unwrap(); -assert_eq!(msg.role, ChatRole::Function); -⋮---- -fn temperature_clamping_captured_in_request() { -// Verify user and parallel_tool_calls fields serialize correctly -⋮---- -assert_eq!(req.user.as_deref(), Some("user-123")); -assert_eq!(req.parallel_tool_calls, Some(true)); -⋮---- -fn strict_field_on_function_def() { -⋮---- -let tool: ChatTool = serde_json::from_value(raw).unwrap(); -assert_eq!(tool.function.strict, Some(true)); -⋮---- -fn finish_reason_unknown_variant_deserializes() { -// DeepSeek returns "insufficient_system_resource" as a finish_reason -let raw = json!("insufficient_system_resource"); -let reason: FinishReason = serde_json::from_value(raw).unwrap(); -assert_eq!(reason, FinishReason::Unknown); -⋮---- -fn finish_reason_known_variants_unaffected() { -assert_eq!( -⋮---- -fn reasoning_content_deserialized_from_response() { -⋮---- -fn reasoning_content_absent_is_none() { -⋮---- -assert!(msg.reasoning_content.is_none()); - - - -// OpenAI error types and rate limit headers -⋮---- -/// OpenAI API error response wrapper. -/// -/// See -⋮---- -pub struct ErrorResponse { -⋮---- -/// Error details with message, type, param, and code. -⋮---- -pub struct ErrorDetail { -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn deserialize_error_response() { -let raw = json!({ -⋮---- -let err: ErrorResponse = serde_json::from_value(raw).unwrap(); -assert_eq!(err.error.message, "Invalid API key"); -assert_eq!(err.error.error_type, "invalid_request_error"); -assert!(err.error.param.is_none()); -assert_eq!(err.error.code.as_deref(), Some("invalid_api_key")); -⋮---- -fn deserialize_error_minimal() { -⋮---- -assert_eq!(err.error.error_type, "server_error"); -⋮---- -assert!(err.error.code.is_none()); -⋮---- -fn serialize_error_skips_none_fields() { -⋮---- -message: "bad request".into(), -error_type: "invalid_request_error".into(), -⋮---- -let val = serde_json::to_value(&err).unwrap(); -let detail = val.get("error").unwrap(); -assert!(!detail.as_object().unwrap().contains_key("param")); -assert!(!detail.as_object().unwrap().contains_key("code")); -⋮---- -fn roundtrip() { -⋮---- -message: "Rate limit exceeded".into(), -error_type: "rate_limit_error".into(), -param: Some("messages".into()), -code: Some("rate_limit_exceeded".into()), -⋮---- -let json_str = serde_json::to_string(&err).unwrap(); -let roundtrip: ErrorResponse = serde_json::from_str(&json_str).unwrap(); -assert_eq!(roundtrip.error.message, "Rate limit exceeded"); -assert_eq!(roundtrip.error.param.as_deref(), Some("messages")); - - - -/// OpenAI Chat Completions API request and response types. -pub mod chat_completions; -/// OpenAI error response types. -pub mod errors; -/// OpenAI Responses API types. -pub mod responses; -/// OpenAI Chat Completions SSE streaming chunk types. -pub mod streaming; - - - -// OpenAI Responses API request/response types -⋮---- -/// OpenAI Responses API request body. -/// -/// See -⋮---- -pub struct ResponsesRequest { -⋮---- -/// Responses API input: text string or array of items. -⋮---- -pub enum ResponsesInput { -⋮---- -/// OpenAI Responses API response body. -⋮---- -/// See -⋮---- -pub struct ResponsesResponse { -⋮---- -/// Token usage for Responses API. -⋮---- -pub struct ResponsesUsage { -⋮---- -/// OpenAI returns cached_tokens here; mapped to Anthropic cache_read_input_tokens. -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn deserialize_text_input_request() { -let raw = json!({ -⋮---- -let req: ResponsesRequest = serde_json::from_value(raw).unwrap(); -assert_eq!(req.model, "gpt-4o"); -assert!(matches!(req.input, ResponsesInput::Text(ref t) if t == "Tell me a joke")); -⋮---- -fn deserialize_items_input_request() { -⋮---- -assert!(matches!(req.input, ResponsesInput::Items(ref items) if items.len() == 1)); -assert_eq!(req.instructions.as_deref(), Some("Be helpful")); -⋮---- -fn deserialize_response() { -⋮---- -let resp: ResponsesResponse = serde_json::from_value(raw).unwrap(); -assert_eq!(resp.id, "resp_abc"); -assert_eq!(resp.response_type, "response"); -assert_eq!(resp.status, "completed"); -let usage = resp.usage.unwrap(); -assert_eq!(usage.input_tokens, 5); -assert_eq!(usage.total_tokens, 8); -⋮---- -fn extra_fields_preserved() { -⋮---- -assert!(req.extra.contains_key("metadata")); -⋮---- -fn roundtrip_request() { -⋮---- -model: "gpt-4o".into(), -input: ResponsesInput::Text("hello".into()), -instructions: Some("Be concise".into()), -max_output_tokens: Some(200), -⋮---- -let json_str = serde_json::to_string(&req).unwrap(); -let roundtrip: ResponsesRequest = serde_json::from_str(&json_str).unwrap(); -assert_eq!(roundtrip.model, "gpt-4o"); -assert_eq!(roundtrip.max_output_tokens, Some(200)); - - - -// OpenAI SSE streaming types (ChatCompletions chunks + Responses events) -⋮---- -/// A single chunk in a streamed Chat Completions response. -/// -/// See -⋮---- -pub struct ChatCompletionChunk { -⋮---- -pub object: String, // "chat.completion.chunk" -⋮---- -/// Compat spec response: "Always empty". -/// See: https://docs.anthropic.com/en/api/openai-sdk#response-fields -⋮---- -/// A choice within a streaming chunk. -⋮---- -pub struct ChunkChoice { -⋮---- -/// Incremental content delta in a streaming chunk. -⋮---- -pub struct ChunkDelta { -⋮---- -/// DeepSeek/Qwen thinking model output. Maps to Anthropic thinking block deltas. -⋮---- -/// Streaming tool calls arrive incrementally, with partial function arguments. -⋮---- -pub struct ChunkToolCall { -⋮---- -/// Incremental function call data in a streaming chunk. -⋮---- -pub struct ChunkFunctionCall { -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn deserialize_chunk_with_text_delta() { -let raw = json!({ -⋮---- -let chunk: ChatCompletionChunk = serde_json::from_value(raw).unwrap(); -assert_eq!(chunk.id, "chatcmpl-abc"); -assert_eq!(chunk.object, "chat.completion.chunk"); -assert_eq!(chunk.choices[0].delta.content.as_deref(), Some("Hello")); -assert!(chunk.choices[0].finish_reason.is_none()); -⋮---- -fn deserialize_chunk_with_role_delta() { -⋮---- -assert_eq!(chunk.choices[0].delta.role, Some(ChatRole::Assistant)); -assert!(chunk.choices[0].delta.content.is_none()); -⋮---- -fn deserialize_chunk_with_tool_call_delta() { -⋮---- -let tc = &chunk.choices[0].delta.tool_calls.as_ref().unwrap()[0]; -assert_eq!(tc.index, 0); -assert_eq!(tc.id.as_deref(), Some("call_xyz")); -assert_eq!( -⋮---- -fn deserialize_chunk_with_finish_reason() { -⋮---- -assert_eq!(chunk.choices[0].finish_reason, Some(FinishReason::Stop)); -⋮---- -fn deserialize_chunk_with_usage() { -⋮---- -let usage = chunk.usage.unwrap(); -assert_eq!(usage.prompt_tokens, 10); -assert_eq!(usage.completion_tokens, 20); -assert_eq!(usage.total_tokens, 30); -⋮---- -fn roundtrip_chunk() { -⋮---- -id: "chatcmpl-test".into(), -object: "chat.completion.chunk".into(), -model: "gpt-4o".into(), -choices: vec![ChunkChoice { -⋮---- -created: Some(1700000000), -⋮---- -let json_str = serde_json::to_string(&chunk).unwrap(); -let roundtrip: ChatCompletionChunk = serde_json::from_str(&json_str).unwrap(); -assert_eq!(roundtrip.choices[0].delta.content.as_deref(), Some("world")); -assert_eq!(roundtrip.created, Some(1700000000)); -⋮---- -fn deserialize_realistic_streaming_chunk() { -// Real gpt-4o streaming chunk with all fields -⋮---- -assert_eq!(chunk.system_fingerprint.as_deref(), Some("fp_a7d06e42a7")); -assert!(chunk.choices[0].logprobs.is_none()); -assert_eq!(chunk.choices[0].delta.content.as_deref(), Some("Hi")); - - - -// JSON helpers for defensive parsing/serialization during API translation. -// OpenAI function arguments may arrive as malformed JSON; these helpers -// ensure we never panic on bad input. -⋮---- -use serde_json::Value; -⋮---- -/// Try to parse a JSON string into a Value. Returns the original string wrapped -/// in Value::String if parsing fails (defensive handling for potentially invalid -/// OpenAI function arguments). -pub fn parse_json_lenient(s: &str) -> Value { -serde_json::from_str(s).unwrap_or_else(|_| Value::String(s.to_string())) -⋮---- -/// Strip markdown code fences that local LLMs (DeepSeek, Qwen) sometimes wrap -/// around tool call argument JSON. Handles ```json, ```JSON, and bare ```. -fn strip_markdown_code_fence(s: &str) -> &str { -let s = s.trim(); -if let Some(rest) = s.strip_prefix("```") { -// Skip optional language tag on the opening fence line -⋮---- -.strip_prefix("json") -.or_else(|| rest.strip_prefix("JSON")) -.unwrap_or(rest); -// Strip the newline after the opening fence line -⋮---- -.strip_prefix('\n') -.or_else(|| rest.strip_prefix("\r\n")) -⋮---- -// Strip trailing closing fence -let rest = rest.trim_end(); -let rest = rest.strip_suffix("```").unwrap_or(rest); -rest.trim() -⋮---- -/// Parse an OpenAI tool call `arguments` string into a JSON object suitable -/// for Anthropic's `input` field. Unlike `parse_json_lenient`, this guarantees -/// the result is always a JSON object: -/// - Empty/whitespace-only string -> `{}` -/// - Valid JSON object -> the parsed object -/// - Valid JSON non-object (string, number, array, etc.) -> `{"_raw": }` -/// - Invalid JSON -> `{"_raw_error": ""}` -/// -/// Also strips markdown code fences (```json ... ```) that local LLMs -/// (DeepSeek, Qwen, llama-server, ollama) sometimes wrap around arguments. -/// Anthropic's `input` field must always be a JSON object. -pub fn parse_tool_arguments(s: &str) -> Value { -let trimmed = strip_markdown_code_fence(s.trim()); -if trimmed.is_empty() { -⋮---- -map.insert("_raw".to_string(), other); -⋮---- -map.insert("_raw_error".to_string(), Value::String(s.to_string())); -⋮---- -/// Serialize a JSON Value to a string. Returns "{}" if serialization fails. -/// The fallback ensures tool call arguments always have a valid JSON string -/// even if the Value contains types serde_json cannot serialize (shouldn't -/// happen in practice, but defensive). -pub fn value_to_json_string(v: &Value) -> String { -serde_json::to_string(v).unwrap_or_else(|_| "{}".to_string()) -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn parse_valid_object() { -let v = parse_json_lenient(r#"{"key": "value"}"#); -assert_eq!(v, json!({"key": "value"})); -⋮---- -fn parse_valid_nested_object() { -let v = parse_json_lenient(r#"{"a": {"b": [1, 2, 3]}}"#); -assert_eq!(v, json!({"a": {"b": [1, 2, 3]}})); -⋮---- -fn parse_null() { -let v = parse_json_lenient("null"); -assert_eq!(v, Value::Null); -⋮---- -fn parse_invalid_json_returns_string() { -let v = parse_json_lenient("not json at all"); -assert_eq!(v, Value::String("not json at all".to_string())); -⋮---- -fn parse_empty_string_returns_string() { -let v = parse_json_lenient(""); -assert_eq!(v, Value::String(String::new())); -⋮---- -fn value_to_json_string_object() { -let v = json!({"foo": "bar"}); -let s = value_to_json_string(&v); -assert_eq!(s, r#"{"foo":"bar"}"#); -⋮---- -fn value_to_json_string_null() { -let s = value_to_json_string(&Value::Null); -assert_eq!(s, "null"); -⋮---- -// --- strip_markdown_code_fence --- -⋮---- -fn strip_fence_json_tag() { -⋮---- -assert_eq!(strip_markdown_code_fence(input), r#"{"key": "val"}"#); -⋮---- -fn strip_fence_bare() { -⋮---- -fn strip_fence_json_uppercase() { -⋮---- -assert_eq!(strip_markdown_code_fence(input), r#"{"a": 1}"#); -⋮---- -fn strip_fence_trailing_whitespace() { -⋮---- -fn strip_fence_no_closing() { -// Missing closing fence: return content after opening fence -⋮---- -fn no_fence_passthrough() { -⋮---- -assert_eq!(strip_markdown_code_fence(input), input); -⋮---- -fn backticks_inside_json_string_not_stripped() { -// Backticks inside a JSON string value should not trigger stripping -⋮---- -// --- parse_tool_arguments --- -⋮---- -fn parse_tool_arguments_empty_string() { -let v = parse_tool_arguments(""); -assert_eq!(v, json!({})); -⋮---- -fn parse_tool_arguments_whitespace_only() { -let v = parse_tool_arguments(" \n "); -⋮---- -fn parse_tool_arguments_valid_object() { -let v = parse_tool_arguments(r#"{"file_path": "/tmp/test.rs", "limit": 100}"#); -assert_eq!(v, json!({"file_path": "/tmp/test.rs", "limit": 100})); -⋮---- -fn parse_tool_arguments_nested_object() { -let v = parse_tool_arguments(r#"{"a": {"b": [1, 2]}}"#); -assert_eq!(v, json!({"a": {"b": [1, 2]}})); -⋮---- -fn parse_tool_arguments_bare_string() { -let v = parse_tool_arguments(r#""hello""#); -assert_eq!(v, json!({"_raw": "hello"})); -⋮---- -fn parse_tool_arguments_bare_number() { -let v = parse_tool_arguments("42"); -assert_eq!(v, json!({"_raw": 42})); -⋮---- -fn parse_tool_arguments_bare_array() { -let v = parse_tool_arguments("[1, 2, 3]"); -assert_eq!(v, json!({"_raw": [1, 2, 3]})); -⋮---- -fn parse_tool_arguments_invalid_json() { -let v = parse_tool_arguments("not json {at all"); -assert_eq!(v, json!({"_raw_error": "not json {at all"})); -⋮---- -fn parse_tool_arguments_null() { -let v = parse_tool_arguments("null"); -assert_eq!(v, json!({"_raw": null})); -⋮---- -fn parse_tool_arguments_markdown_json_fence() { -let v = parse_tool_arguments("```json\n{\"file_path\": \"test.rs\"}\n```"); -assert_eq!(v, json!({"file_path": "test.rs"})); -⋮---- -fn parse_tool_arguments_markdown_bare_fence() { -let v = parse_tool_arguments("```\n{\"file_path\": \"test.rs\"}\n```"); -⋮---- -fn parse_tool_arguments_markdown_fence_trailing_whitespace() { -let v = parse_tool_arguments("```json\n{\"a\": 1}\n```\n "); -assert_eq!(v, json!({"a": 1})); -⋮---- -fn parse_tool_arguments_backticks_in_json_value() { -// Backticks inside a JSON string value are not code fences -let v = parse_tool_arguments(r#"{"code": "use `foo`"}"#); -assert_eq!(v, json!({"code": "use `foo`"})); - - - -/// UUID-based ID generation for Anthropic message and content block IDs. -pub mod ids; -/// JSON serialization helpers (pretty-print, merge, normalize). -pub mod json; -/// Secret redaction for logging (API keys, tokens). -pub mod redact; - - - -use thiserror::Error; -⋮---- -/// Errors that can occur during API format translation. -⋮---- -pub enum TranslateError { -/// The model name did not match any entry in the translation config. -⋮---- -/// A translation step failed (validation, unsupported feature with strict config, etc.). -⋮---- -/// A required field was missing from the input. - - - -// Golden-file tests: validate that fixture JSON files can be deserialized -// and that translation between formats produces the expected shapes. -⋮---- -fn fixtures_dir() -> std::path::PathBuf { -std::path::Path::new(env!("CARGO_MANIFEST_DIR")) -.parent() -.unwrap() -⋮---- -.join("fixtures") -⋮---- -fn anthropic_basic_request_fixture_deserializes() { -let path = fixtures_dir().join("anthropic/messages_basic.json"); -let content = std::fs::read_to_string(&path).expect("fixture file should exist"); -let fixture: serde_json::Value = serde_json::from_str(&content).unwrap(); -⋮---- -serde_json::from_value(fixture["request"].clone()).unwrap(); -assert_eq!(req.model, "claude-opus-4-6"); -assert_eq!(req.max_tokens, 256); -⋮---- -serde_json::from_value(fixture["response"].clone()).unwrap(); -assert_eq!(resp.stop_reason, Some(anthropic::StopReason::EndTurn)); -⋮---- -fn anthropic_tool_use_fixture_deserializes() { -let path = fixtures_dir().join("anthropic/messages_tool_use.json"); -⋮---- -assert!(req.tools.is_some()); -assert_eq!(req.tools.as_ref().unwrap().len(), 1); -⋮---- -assert_eq!(resp.stop_reason, Some(anthropic::StopReason::ToolUse)); -⋮---- -assert_eq!(name, "get_stock_price"); -⋮---- -other => panic!("expected ToolUse, got {:?}", other), -⋮---- -fn openai_basic_response_fixture_deserializes() { -let path = fixtures_dir().join("openai/chat_completion_basic.json"); -⋮---- -assert_eq!(resp.choices.len(), 1); -assert_eq!( -⋮---- -fn openai_tool_call_response_fixture_deserializes() { -let path = fixtures_dir().join("openai/chat_completion_tool_call.json"); -⋮---- -.as_ref() -.expect("should have tool_calls"); -assert_eq!(tc[0].function.name, "get_stock_price"); -⋮---- -fn translate_anthropic_fixture_to_openai_request() { -⋮---- -// System prompt should become a system message -assert_eq!(openai_req.messages[0].role, openai::ChatRole::System); -// User message should follow -assert_eq!(openai_req.messages[1].role, openai::ChatRole::User); -assert_eq!(openai_req.model, "claude-opus-4-6"); -⋮---- -fn translate_openai_response_to_anthropic() { -⋮---- -assert_eq!(anthropic_resp.model, "claude-opus-4-6"); -⋮---- -assert!(!anthropic_resp.content.is_empty()); -assert_eq!(anthropic_resp.usage.input_tokens, 25); -assert_eq!(anthropic_resp.usage.output_tokens, 30); -⋮---- -fn translate_openai_tool_call_response_to_anthropic() { -⋮---- -assert_eq!(id, "call_xyz789"); -⋮---- -assert_eq!(input["ticker"], "^GSPC"); -⋮---- -// --- Claude Code fixture tests --- -⋮---- -fn claude_code_tool_use_fixture_deserializes() { -let path = fixtures_dir().join("anthropic/claude_code_tool_use.json"); -⋮---- -assert_eq!(req.tools.as_ref().unwrap().len(), 6); -assert_eq!(req.tools.as_ref().unwrap()[0].name, "Read"); -assert_eq!(req.tools.as_ref().unwrap()[1].name, "Bash"); -⋮---- -// Response has text + 2 parallel tool_use blocks -assert_eq!(resp.content.len(), 3); -⋮---- -anthropic::ContentBlock::ToolUse { name, .. } => assert_eq!(name, "Read"), -⋮---- -anthropic::ContentBlock::ToolUse { name, .. } => assert_eq!(name, "Glob"), -⋮---- -fn claude_code_tool_call_fixture_deserializes() { -let path = fixtures_dir().join("openai/claude_code_tool_call.json"); -⋮---- -assert_eq!(tc.len(), 2); -assert_eq!(tc[0].function.name, "Read"); -assert_eq!(tc[1].function.name, "Glob"); -⋮---- -fn claude_code_request_translates_to_openai() { -⋮---- -// 6 Anthropic tools -> 6 OpenAI function tools -let tools = oai_req.tools.as_ref().unwrap(); -assert_eq!(tools.len(), 6); -assert_eq!(tools[0].tool_type, "function"); -assert_eq!(tools[0].function.name, "Read"); -assert_eq!(tools[1].function.name, "Bash"); -⋮---- -// tool_choice auto preserved -match oai_req.tool_choice.as_ref().unwrap() { -openai::ChatToolChoice::Simple(s) => assert_eq!(s, "auto"), -other => panic!("expected Simple(auto), got {:?}", other), -⋮---- -fn claude_code_openai_response_translates_back() { -// Load the OpenAI fixture response, translate to Anthropic, verify parallel tool_use -⋮---- -assert_eq!(anth.stop_reason, Some(anthropic::StopReason::ToolUse)); -// Text content + 2 tool_use blocks -assert_eq!(anth.content.len(), 3); -⋮---- -assert!(text.contains("parallel")); -⋮---- -other => panic!("expected Text, got {:?}", other), -⋮---- -assert_eq!(id, "call_read_001"); -assert_eq!(name, "Read"); -assert_eq!(input["file_path"], "/home/user/project/config.toml"); -⋮---- -assert_eq!(id, "call_glob_001"); -assert_eq!(name, "Glob"); -assert_eq!(input["pattern"], "**/*test*"); -⋮---- -fn claude_code_tool_result_cycle_translates() { -// Full cycle: Anthropic request with tool_results -> OpenAI messages -let path = fixtures_dir().join("anthropic/claude_code_tool_result.json"); -⋮---- -// Expected: user, assistant (with tool_calls), tool, tool -assert_eq!(oai.messages.len(), 4); -assert_eq!(oai.messages[0].role, openai::ChatRole::User); -assert_eq!(oai.messages[1].role, openai::ChatRole::Assistant); -assert_eq!(oai.messages[2].role, openai::ChatRole::Tool); -assert_eq!(oai.messages[3].role, openai::ChatRole::Tool); -⋮---- -// Assistant message has 2 tool_calls -let tc = oai.messages[1].tool_calls.as_ref().unwrap(); -⋮---- -assert_eq!(tc[0].id, "toolu_01UDAtfZkgcGYMBq7Ns84vfN"); -⋮---- -assert_eq!(tc[1].id, "toolu_01J3KzMqBf9tXyQFhVw2NxRG"); -⋮---- -// Tool result messages have correct IDs - - - -//! Integration test: verify the crate works as a standalone library without the proxy. -⋮---- -use anyllm_translate::openai::ChatCompletionResponse; -⋮---- -fn standalone_translate_request() { -⋮---- -.model_map("haiku", "gpt-4o-mini") -.model_map("sonnet", "gpt-4o") -.build(); -⋮---- -.unwrap(); -⋮---- -let openai_req = translate_request(&req, &config).unwrap(); -assert_eq!(openai_req.model, "gpt-4o"); -assert_eq!(openai_req.max_completion_tokens, Some(100)); -⋮---- -fn standalone_translate_response() { -⋮---- -let resp: MessageResponse = translate_response(&openai_resp, "claude-sonnet-4-6"); -assert_eq!(resp.model, "claude-sonnet-4-6"); -assert_eq!( -⋮---- -fn standalone_strict_mode_rejects_unknown() { -⋮---- -.passthrough_unknown_models(false) -⋮---- -let err = translate_request(&req, &config).unwrap_err(); -assert!(matches!(err, TranslateError::UnknownModel(_))); -assert!(err.to_string().contains("unknown-model")); -⋮---- -fn default_config_passthrough() { -⋮---- -assert_eq!(openai_req.model, "any-model-name"); - - - -//! Integration tests for the middleware module. -//! Uses a mock OpenAI backend (axum test server) to verify end-to-end translation. -⋮---- -use axum::extract::Json; -use axum::http::StatusCode; -⋮---- -use axum::response::IntoResponse; -use axum::routing::post; -use axum::Router; -use tokio::net::TcpListener; -⋮---- -use anyllm_translate::TranslationConfig; -⋮---- -// --- Mock OpenAI backend --- -⋮---- -/// Mock handler that returns a canned ChatCompletionResponse. -async fn mock_chat_completion(Json(req): Json) -> impl IntoResponse { -⋮---- -.get("model") -.and_then(|m| m.as_str()) -.unwrap_or("gpt-4o"); -let is_stream = req.get("stream").and_then(|s| s.as_bool()).unwrap_or(false); -⋮---- -return mock_stream_response(model).await.into_response(); -⋮---- -(StatusCode::OK, Json(response)).into_response() -⋮---- -async fn mock_stream_response( -⋮---- -let model = model.to_string(); -⋮---- -// Chunk 1: role -⋮---- -.send(Ok( -Event::default().data(serde_json::to_string(&chunk1).unwrap()) -⋮---- -// Chunk 2: text content -⋮---- -Event::default().data(serde_json::to_string(&chunk2).unwrap()) -⋮---- -// Chunk 3: finish -⋮---- -Event::default().data(serde_json::to_string(&chunk3).unwrap()) -⋮---- -// [DONE] sentinel -let _ = tx.send(Ok(Event::default().data("[DONE]"))).await; -⋮---- -Sse::new(tokio_stream::wrappers::ReceiverStream::new(rx)).keep_alive(KeepAlive::default()) -⋮---- -/// Mock handler that returns a 429 rate limit error. -async fn mock_rate_limit() -> impl IntoResponse { -⋮---- -(StatusCode::TOO_MANY_REQUESTS, Json(body)) -⋮---- -/// Spin up a mock server and return its base URL. -async fn start_mock_server(router: Router) -> String { -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -⋮---- -axum::serve(listener, router).await.unwrap(); -⋮---- -format!("http://{addr}") -⋮---- -fn test_config(backend_url: &str) -> AnthropicCompatConfig { -⋮---- -.backend_url(backend_url) -.api_key("test-key") -.translation( -⋮---- -.model_map("haiku", "gpt-4o-mini") -.model_map("sonnet", "gpt-4o") -.build(), -⋮---- -.build() -⋮---- -// --- Tests --- -⋮---- -async fn non_streaming_request_translates_roundtrip() { -let mock = Router::new().route("/v1/chat/completions", post(mock_chat_completion)); -let base_url = start_mock_server(mock).await; -⋮---- -let config = test_config(&base_url); -let app = anthropic_compat_router(config); -⋮---- -axum::serve(listener, app).await.unwrap(); -⋮---- -.post(format!("http://{addr}/v1/messages")) -.json(&serde_json::json!({ -⋮---- -.send() -⋮---- -.unwrap(); -⋮---- -assert_eq!(resp.status(), 200); -let body: serde_json::Value = resp.json().await.unwrap(); -⋮---- -// Response should be in Anthropic format -assert_eq!(body["type"], "message"); -assert_eq!(body["model"], "claude-sonnet-4-6"); // original model preserved -assert_eq!(body["role"], "assistant"); -assert_eq!(body["content"][0]["type"], "text"); -assert_eq!(body["content"][0]["text"], "Hello from mock!"); -assert_eq!(body["usage"]["input_tokens"], 10); -assert_eq!(body["usage"]["output_tokens"], 5); -assert_eq!(body["stop_reason"], "end_turn"); -⋮---- -async fn streaming_request_produces_anthropic_sse_events() { -⋮---- -// Collect SSE events from the response body -let body = resp.text().await.unwrap(); -⋮---- -// Should contain Anthropic event types -assert!( -⋮---- -// The text delta should contain "Hello" -assert!(body.contains("Hello"), "missing text content: {body}"); -⋮---- -async fn backend_error_translated_to_anthropic_format() { -let mock = Router::new().route("/v1/chat/completions", post(mock_rate_limit)); -⋮---- -assert_eq!(resp.status(), 429); -⋮---- -// Should be Anthropic error format -assert_eq!(body["type"], "error"); -assert!(body["error"]["type"].as_str().is_some()); -assert!(body["error"]["message"] -⋮---- -async fn tower_layer_intercepts_messages_passes_through_others() { -⋮---- -// An existing service with a custom route -⋮---- -.route( -⋮---- -.layer(AnthropicTranslationLayer::new(config)); -⋮---- -// Custom route still works -⋮---- -.get(format!("http://{addr}/custom")) -⋮---- -assert_eq!(resp.text().await.unwrap(), "custom response"); -⋮---- -// /v1/messages is intercepted by the layer -⋮---- -async fn invalid_json_returns_anthropic_error() { -⋮---- -.header("content-type", "application/json") -.body("not json") -⋮---- -// Should get an error response (400 or 422 depending on axum's JSON extractor) -assert!(resp.status().is_client_error()); -⋮---- -async fn model_mapping_applied() { -// Mock that echoes the model it received -async fn echo_model(Json(req): Json) -> impl IntoResponse { -⋮---- -.unwrap_or("unknown"); -⋮---- -Json(response) -⋮---- -let mock = Router::new().route("/v1/chat/completions", post(echo_model)); -⋮---- -// The mock echoes the model it received; should be the mapped model -let text = body["content"][0]["text"].as_str().unwrap(); -⋮---- -// But the Anthropic response should have the original model name -assert_eq!(body["model"], "claude-haiku-3"); - - - -[package] -name = "anyllm_translate" -description = "Pure translation layer between Anthropic Messages API and OpenAI Chat Completions" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -categories = ["api-bindings", "web-programming"] -keywords = ["anthropic", "openai", "translation", "llm", "api"] -readme = "README.md" - -[features] -default = [] -middleware = [ - "dep:axum", - "dep:reqwest", - "dep:tokio", - "dep:tokio-stream", - "dep:futures", - "dep:bytes", - "dep:tower", -] - -[dependencies] -serde = { version = "1", features = ["derive"] } -serde_json = "1" -thiserror = "2" -tracing = "0.1" -uuid = { version = "1", features = ["v4"] } - -# Optional: middleware feature deps -axum = { version = "0.8", optional = true } -reqwest = { version = "0.12", optional = true, default-features = false, features = ["json", "stream"] } -tokio = { version = "1", optional = true, features = ["rt", "sync"] } -tokio-stream = { version = "0.1", optional = true } -futures = { version = "0.3", optional = true } -bytes = { version = "1", optional = true } -tower = { version = "0.5", optional = true, features = ["util"] } - -[dev-dependencies] -pretty_assertions = "1" -axum = "0.8" -reqwest = { version = "0.12", default-features = false, features = ["json", "stream"] } -tokio = { version = "1", features = ["full"] } -tokio-stream = "0.1" -futures = "0.3" -bytes = "1" -tower = { version = "0.5", features = ["util"] } - - - -# anyllm_translate - -Pure, IO-free translation between Anthropic Messages API and OpenAI Chat Completions / Responses API formats. Also supports Google Gemini native API translation. - -No HTTP clients, no async runtime, no network calls. Just `fn(A) -> B` transformations. - -## Quick Start - -```rust -use anyllm_translate::{TranslationConfig, translate_request, translate_response}; -use anyllm_translate::anthropic::MessageCreateRequest; - -let config = TranslationConfig::builder() - .model_map("haiku", "gpt-4o-mini") - .model_map("sonnet", "gpt-4o") - .model_map("opus", "gpt-4o") - .build(); - -let req: MessageCreateRequest = serde_json::from_str(r#"{ - "model": "claude-sonnet-4-6", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hello"}] -}"#).unwrap(); - -let openai_req = translate_request(&req, &config).unwrap(); -assert_eq!(openai_req.model, "gpt-4o"); - -// Send openai_req to OpenAI, get response, then: -// let anthropic_resp = translate_response(&openai_resp, &req.model); -``` - -## Supported APIs - -- **Anthropic Messages API** (request/response types, streaming SSE events) -- **OpenAI Chat Completions API** (request/response types, streaming chunks) -- **OpenAI Responses API** (request/response types, streaming events) -- **Google Gemini native API** (generateContent types, streaming) - -## Features - -- **Default**: Pure translation types and mapping functions -- **`middleware`**: Adds an axum middleware layer that intercepts Anthropic-format requests, translates them, forwards to a configurable backend, and translates responses back - -```toml -[dependencies] -anyllm_translate = "0.1" - -# With middleware support: -anyllm_translate = { version = "0.1", features = ["middleware"] } -``` - -## Modules - -| Module | Description | -|--------|-------------| -| `anthropic` | Anthropic Messages API types (request, response, streaming, errors) | -| `openai` | OpenAI Chat Completions and Responses API types | -| `gemini` | Google Gemini native API types | -| `mapping` | Stateless conversion functions between APIs | -| `config` | Translation configuration (model mapping, lossy behavior) | -| `translate` | Convenience wrappers combining config with mapping | -| `middleware` | Axum middleware layer (requires `middleware` feature) | - -## Translation Coverage - -- Text messages, multi-turn conversations -- System prompts (Anthropic `system` to OpenAI `developer` role or Responses `instructions`) -- Tool definitions, tool calls, tool results -- Image and document content blocks (documents degrade to text notes) -- Streaming SSE event translation (state machines for each backend) -- Token usage mapping -- Error type/status code translation -- Extended thinking (stripped when targeting OpenAI) - -## Related - -This crate is part of [anyllm-proxy](https://github.com/whit3rabbit/anyllm-proxy), which also includes a standalone HTTP proxy server. - - - -# PRD: Phase 2 - Anthropic Domain Types - -## Introduction - -Define the Rust types that model the Anthropic Messages API surface: request, response, content blocks, errors, and headers. These are the "frontend" types the proxy accepts from clients. They must serialize/deserialize correctly against real Anthropic API payloads. - -## Goals - -- Model `AnthropicMessageCreateRequest` and `AnthropicMessageCreateResponse` as strongly typed Rust structs -- Support all content block types: text, image, document, tool_use, tool_result -- Define error types matching Anthropic's `{type: "error", error: {type, message}}` shape -- Parse Anthropic-specific headers: `x-api-key`, `anthropic-version`, `anthropic-beta` -- Validate with round-trip serde tests and fixture-based golden tests - -## User Stories - -### US-001: Message request types -**Description:** As a developer working on translation logic, I need typed structs for Anthropic message create requests so I can pattern-match on fields instead of working with raw JSON. - -**Acceptance Criteria:** -- [ ] `AnthropicMessageCreateRequest` struct with fields: `model`, `max_tokens`, `messages`, `system`, `temperature`, `top_p`, `stop_sequences`, `tools`, `tool_choice`, `metadata`, `stream` -- [ ] `#[serde(flatten)] extra: Map` for forward compatibility -- [ ] `AnthropicInputMessage` with `role` (user/assistant) and `content` (string or array of content blocks) -- [ ] `AnthropicSystem` supporting both string and array-of-blocks forms -- [ ] Deserializes from PLAN.md example JSON (lines 207-214) without error - -### US-002: Content block types -**Description:** As a developer, I need typed enums for every Anthropic content block so tool_use, images, and documents are first-class types, not untyped JSON. - -**Acceptance Criteria:** -- [ ] `AnthropicContentBlock` enum with variants: `Text`, `Image`, `Document`, `ToolUse`, `ToolResult` -- [ ] `Text` variant: `{type: "text", text: String}` -- [ ] `Image` variant: `{type: "image", source: ImageSource}` with base64 and url source types -- [ ] `Document` variant: `{type: "document", source: DocumentSource}` with base64 PDF support -- [ ] `ToolUse` variant: `{type: "tool_use", id: String, name: String, input: Value}` -- [ ] `ToolResult` variant: `{type: "tool_result", tool_use_id: String, content: ToolResultContent}` -- [ ] Serde tags use `#[serde(tag = "type")]` for correct JSON representation - -### US-003: Message response types -**Description:** As a developer, I need response types so the proxy can construct valid Anthropic-shaped responses from translated OpenAI data. - -**Acceptance Criteria:** -- [ ] `AnthropicMessageCreateResponse` with fields: `id`, `type` (always "message"), `role`, `content` (Vec of content blocks), `model`, `stop_reason`, `stop_sequence`, `usage` -- [ ] `AnthropicUsage` struct: `input_tokens`, `output_tokens`, optional `cache_creation_input_tokens`, `cache_read_input_tokens` -- [ ] `StopReason` enum: `end_turn`, `max_tokens`, `stop_sequence`, `tool_use` -- [ ] Serializes to match PLAN.md example JSON (lines 220-228) - -### US-004: Error types -**Description:** As a developer, I need Anthropic error types so the proxy returns errors in the exact shape Anthropic clients expect. - -**Acceptance Criteria:** -- [ ] `AnthropicError` struct: `{type: "error", error: AnthropicErrorBody}` -- [ ] `AnthropicErrorBody`: `{type: String, message: String}` -- [ ] Error type constants: `invalid_request_error`, `authentication_error`, `permission_error`, `not_found_error`, `request_too_large`, `rate_limit_error`, `api_error`, `overloaded_error` -- [ ] HTTP status code mapping function: error type -> status code (400, 401, 403, 404, 413, 429, 500, 529) - -### US-005: Header parsing -**Description:** As a developer working on middleware, I need utilities to extract and validate Anthropic-specific headers from incoming requests. - -**Acceptance Criteria:** -- [ ] Function to extract `x-api-key` from request headers, returning error if missing -- [ ] Function to extract `anthropic-version` header -- [ ] Function to extract optional `anthropic-beta` header (comma-separated list) -- [ ] Unit tests for present, missing, and malformed header cases - -### US-006: Serde round-trip and fixture tests -**Description:** As a developer, I need confidence that types serialize and deserialize correctly against real API shapes, not just compile. - -**Acceptance Criteria:** -- [ ] Golden fixture files in `fixtures/anthropic/`: `messages_basic.json`, `messages_tool_use.json`, `messages_image.json`, `messages_document.json` -- [ ] Tests that deserialize each fixture into typed structs and re-serialize, comparing output -- [ ] Test that malformed requests (missing `max_tokens`, invalid `role`) are rejected by serde -- [ ] `cargo test -p anyllm_translate` passes - -## Functional Requirements - -- FR-1: All Anthropic content block types defined in PLAN.md lines 64-76 are representable as typed Rust structs -- FR-2: Deserialization uses `#[serde(tag = "type")]` for content blocks so `{"type": "text", "text": "..."}` maps to the `Text` variant -- FR-3: Unknown fields are captured in `extra` maps, not rejected, for forward compatibility -- FR-4: Error types can be constructed from an error type string and message, producing valid JSON matching Anthropic's documented shape -- FR-5: `AnthropicMetadata` includes `user_id` field - -## Non-Goals - -- No streaming event types (Phase 7) -- No translation to/from OpenAI types (Phase 4) -- No HTTP handling or middleware (Phase 6) -- No beta Files or Skills API types - -## Technical Considerations - -- Use `#[serde(rename_all = "snake_case")]` where field names match -- Use `#[serde(skip_serializing_if = "Option::is_none")]` to keep serialized output clean -- `tool_use.input` is `serde_json::Value` (arbitrary JSON object), not a typed struct -- `tool_result.content` can be a string or array of content blocks; model with an enum - -## Success Metrics - -- All fixture files deserialize without error -- Round-trip (deserialize then serialize) produces semantically identical JSON -- `cargo test -p anyllm_translate` passes with zero failures - - - -# PRD: Phase 4 - Non-Streaming Message Translation - -## Introduction - -Implement the core translation logic that converts Anthropic Messages API requests into OpenAI Chat Completions requests, and converts OpenAI responses back into Anthropic response shapes. This is pure `fn(A) -> B` logic with no IO, covering message mapping, tool definition translation, usage field mapping, error mapping, and stop reason mapping. - -## Goals - -- Translate Anthropic request fields to OpenAI Chat Completions fields -- Translate OpenAI Chat Completions responses back to Anthropic response shapes -- Map system prompts, roles, sampling params, stop reasons, and usage correctly -- Handle edge cases: temperature clamping, stop sequence truncation, missing fields -- All translation logic is stateless, IO-free, and testable without mocks - -## User Stories - -### US-001: Message and role mapping -**Description:** As a developer, I need a function that converts Anthropic messages (with system as top-level field) into OpenAI messages (with system/developer as a role in the messages array). - -**Acceptance Criteria:** -- [ ] `map_anthropic_to_openai_request(req: &AnthropicMessageCreateRequest) -> OpenAIChatCompletionRequest` -- [ ] Anthropic `system` string -> OpenAI `developer` role message at position 0 -- [ ] Anthropic `system` array of text blocks -> concatenated into single `developer` message -- [ ] Anthropic user/assistant messages -> OpenAI user/assistant messages with correct content -- [ ] Model name passed through (no mapping in this phase) -- [ ] Test: basic text request with system prompt produces correct OpenAI shape - -### US-002: Sampling parameter translation -**Description:** As a developer, I need sampling parameters translated with proper clamping and defaults. - -**Acceptance Criteria:** -- [ ] `temperature`: passed through; values > 1.0 clamped to 1.0 with a warning (Anthropic max is 1.0, OpenAI allows up to 2.0) -- [ ] `top_p`: passed through directly -- [ ] `max_tokens`: mapped directly to OpenAI `max_tokens` -- [ ] `stop_sequences`: mapped to OpenAI `stop`; truncated to first 4 entries (OpenAI limit) with warning -- [ ] Test: temperature 0.5 passes through, temperature 1.5 clamps to 1.0 - -### US-003: Tool definition mapping -**Description:** As a developer, I need Anthropic tool definitions translated to OpenAI function tool definitions. - -**Acceptance Criteria:** -- [ ] Anthropic `tools[].input_schema` -> OpenAI `tools[].function.parameters` -- [ ] Anthropic `tools[].name` -> OpenAI `tools[].function.name` -- [ ] Anthropic `tools[].description` -> OpenAI `tools[].function.description` -- [ ] OpenAI tool wrapper `{type: "function", function: {...}}` added -- [ ] `tool_choice` mapping: Anthropic `any` -> OpenAI `auto`, `none` -> `none`, `{type: "tool", name: X}` -> `{type: "function", function: {name: X}}` -- [ ] Test: single tool def, multiple tool defs, tool_choice variants - -### US-004: Response translation (OpenAI -> Anthropic) -**Description:** As a developer, I need a function that converts an OpenAI Chat Completions response into an Anthropic Messages response. - -**Acceptance Criteria:** -- [ ] `map_openai_to_anthropic_response(resp: &OpenAIChatCompletionResponse, model: &str) -> AnthropicMessageCreateResponse` -- [ ] OpenAI `choices[0].message.content` -> Anthropic `content: [{type: "text", text: ...}]` -- [ ] Generate Anthropic-style `id` (e.g., `msg_` + uuid) -- [ ] `type` always set to `"message"`, `role` always `"assistant"` -- [ ] Test: basic text response produces valid Anthropic shape - -### US-005: Usage field mapping -**Description:** As a developer, I need token usage fields translated between the two APIs. - -**Acceptance Criteria:** -- [ ] OpenAI `prompt_tokens` -> Anthropic `input_tokens` -- [ ] OpenAI `completion_tokens` -> Anthropic `output_tokens` -- [ ] Cache fields default to 0 or absent -- [ ] Test: usage fields map correctly - -### US-006: Stop reason mapping -**Description:** As a developer, I need finish/stop reasons mapped between OpenAI and Anthropic conventions. - -**Acceptance Criteria:** -- [ ] OpenAI `stop` -> Anthropic `end_turn` -- [ ] OpenAI `length` -> Anthropic `max_tokens` -- [ ] OpenAI `tool_calls` -> Anthropic `tool_use` -- [ ] OpenAI `content_filter` -> Anthropic `end_turn` (best effort, no exact equivalent) -- [ ] Null/missing finish_reason -> Anthropic `end_turn` (default) -- [ ] Test: each mapping case - -### US-007: Error status code mapping -**Description:** As a developer, I need OpenAI HTTP error status codes translated to Anthropic error types. - -**Acceptance Criteria:** -- [ ] OpenAI 400 -> Anthropic `invalid_request_error` (400) -- [ ] OpenAI 401 -> Anthropic `authentication_error` (401) -- [ ] OpenAI 403 -> Anthropic `permission_error` (403) -- [ ] OpenAI 404 -> Anthropic `not_found_error` (404) -- [ ] OpenAI 429 -> Anthropic `rate_limit_error` (429) -- [ ] OpenAI 500/502/503 -> Anthropic `api_error` (500) -- [ ] Test: each status code maps correctly - -## Functional Requirements - -- FR-1: `message_map` module converts between message formats bidirectionally -- FR-2: `tools_map` module converts tool definitions and tool_choice -- FR-3: `usage_map` module converts token usage fields -- FR-4: `errors_map` module converts HTTP status codes and error shapes -- FR-5: All mapping functions are pure (no IO, no state, no async) -- FR-6: Comprehensive field mapping table from PLAN.md lines 964-977 is implemented - -## Non-Goals - -- No tool call/result translation in conversation history (Phase 5) -- No streaming translation (Phase 7) -- No HTTP requests or proxy wiring (Phase 6) -- No model name mapping (configuration concern, Phase 6) - -## Technical Considerations - -- Functions take references and return owned types to avoid lifetime complexity -- Use `serde_json::Value` for passthrough of unknown fields -- Temperature clamping should log a warning (return clamped value + optional warning) -- Stop sequence truncation should be documented in output (first 4 of N used) - -## Success Metrics - -- All mapping functions have corresponding unit tests -- Golden fixture tests: load Anthropic fixture, translate to OpenAI, compare against OpenAI fixture -- `cargo test -p anyllm_translate` passes with all new tests green - - - -# PRD: Phase 3 - OpenAI Domain Types - -## Introduction - -Define the Rust types that model the OpenAI Chat Completions and Responses APIs: requests, responses, tool types, errors, and rate limit headers. These are the "backend" types the proxy uses when communicating with OpenAI. - -## Goals - -- Model `OpenAIChatCompletionRequest` and `OpenAIChatCompletionResponse` structs -- Model `OpenAIResponsesRequest` and `OpenAIResponsesResponse` structs -- Define OpenAI error types and rate limit header parsing -- Define OpenAI tool/function calling types -- Validate with round-trip serde tests and fixture-based golden tests - -## User Stories - -### US-001: Chat Completions request types -**Description:** As a developer working on the backend client, I need typed structs for OpenAI Chat Completions requests so I can construct valid payloads to send to OpenAI. - -**Acceptance Criteria:** -- [ ] `OpenAIChatCompletionRequest` struct with fields: `model`, `messages`, `max_tokens`, `temperature`, `top_p`, `stop`, `tools`, `tool_choice`, `stream`, `stream_options` -- [ ] `#[serde(flatten)] extra` for forward compatibility -- [ ] `OpenAIChatMessage` with roles: `system`, `user`, `assistant`, `developer`, `tool` -- [ ] `OpenAIStop` supporting both single string and string array forms -- [ ] `OpenAIStreamOptions` with `include_usage: bool` -- [ ] Serializes to match PLAN.md example JSON (lines 237-246) - -### US-002: Chat Completions response types -**Description:** As a developer, I need typed response structs so the translator can extract content, tool calls, usage, and stop reasons from OpenAI responses. - -**Acceptance Criteria:** -- [ ] `OpenAIChatCompletionResponse` with fields: `id`, `object`, `model`, `choices`, `usage` -- [ ] `OpenAIChatChoice` with `index`, `message`, `finish_reason` -- [ ] `OpenAIChatResponseMessage` with `role`, `content` (Option), `tool_calls` (Option) -- [ ] `FinishReason` enum: `stop`, `length`, `tool_calls`, `content_filter` -- [ ] `OpenAIUsage` struct: `prompt_tokens`, `completion_tokens`, `total_tokens` -- [ ] Deserializes from PLAN.md example JSON (lines 249-265) - -### US-003: OpenAI tool types -**Description:** As a developer, I need typed tool definitions matching OpenAI's function-calling schema so tool translation is type-safe. - -**Acceptance Criteria:** -- [ ] `OpenAITool` struct with `type` (always "function") and `function: OpenAIFunction` -- [ ] `OpenAIFunction` with `name`, `description`, `parameters` (Value for JSON Schema) -- [ ] `OpenAIToolCall` with `id`, `type`, `function: OpenAIFunctionCall` -- [ ] `OpenAIFunctionCall` with `name`, `arguments` (String, since OpenAI sends JSON as string) -- [ ] `OpenAIToolChoice` supporting string forms ("auto", "none", "required") and object form `{type, function: {name}}` - -### US-004: Responses API types (basic) -**Description:** As a developer, I need basic types for the OpenAI Responses API to support it as an alternative backend, especially for file/document inputs. - -**Acceptance Criteria:** -- [ ] `OpenAIResponsesRequest` with `model`, `input` (string or array of input items), `instructions`, `max_output_tokens`, `tools`, `stream` -- [ ] `OpenAIResponsesInputItem` enum supporting text, image, and `input_file` (with `file_data`, `file_id`, `file_url` variants) -- [ ] `OpenAIResponsesResponse` with `id`, `output`, `status`, `usage` -- [ ] Basic serialization/deserialization tests - -### US-005: Error types and rate limit headers -**Description:** As a developer, I need OpenAI error types and rate limit header parsing so the proxy can handle upstream failures and map them to Anthropic error shapes. - -**Acceptance Criteria:** -- [ ] `OpenAIError` struct matching OpenAI's `{error: {message, type, param, code}}` shape -- [ ] Rate limit header parsing: `x-ratelimit-limit-requests`, `x-ratelimit-remaining-requests`, `x-ratelimit-limit-tokens`, `x-ratelimit-remaining-tokens`, `x-ratelimit-reset-requests`, `x-ratelimit-reset-tokens` -- [ ] Struct `OpenAIRateLimitInfo` holding parsed values -- [ ] Unit tests for error deserialization and header parsing - -### US-006: Fixture tests -**Description:** As a developer, I need golden-file tests to confirm OpenAI types match real API shapes. - -**Acceptance Criteria:** -- [ ] Fixture files in `fixtures/openai/`: `chat_completion_basic.json`, `chat_completion_tool_call.json`, `responses_basic.json` -- [ ] Tests that deserialize each fixture into typed structs without error -- [ ] Round-trip serialization produces semantically identical JSON -- [ ] `cargo test -p anyllm_translate` passes - -## Functional Requirements - -- FR-1: All OpenAI Chat Completions fields from PLAN.md lines 78-87 are modeled -- FR-2: All OpenAI Responses fields from PLAN.md lines 89-94 are modeled (basic subset) -- FR-3: `OpenAIChatMessage.content` supports both string and structured content (array of parts with text/image_url) -- FR-4: `OpenAIToolCall.function.arguments` is `String` (not parsed JSON) to match OpenAI's wire format -- FR-5: Error types support deserialization from OpenAI error JSON responses - -## Non-Goals - -- No streaming chunk types (Phase 7) -- No WebSocket mode types -- No MCP tool types -- No file upload multipart handling - -## Technical Considerations - -- OpenAI `arguments` is a JSON string that may be malformed; keep as `String`, parse defensively in mapping layer -- `finish_reason` may be null in streaming chunks; use `Option` -- Content in assistant messages is `Option` (null when tool_calls present) -- Rate limit headers use different reset formats (absolute timestamp vs relative seconds) - -## Success Metrics - -- All fixture files deserialize without error -- Round-trip serde produces equivalent JSON -- `cargo test -p anyllm_translate` passes - - - -# PRD: Phase 1 - Project Scaffolding - -## Introduction - -Set up the Cargo workspace, dependencies, and directory structure for the Anthropic-to-OpenAI translation proxy. This is the foundation everything else builds on. Two crates: `translator` (pure library, no IO) and `proxy` (axum binary). - -## Goals - -- Establish a working Cargo workspace with two crates -- Pin initial dependency versions for tokio, axum, reqwest, serde, serde_json -- Create the directory tree matching PLAN.md lines 606-665 -- Verify the build compiles and a health endpoint responds 200 - -## User Stories - -### US-001: Create Cargo workspace -**Description:** As a developer, I need a workspace root `Cargo.toml` that declares `translator` and `proxy` as members so I can build both crates with a single command. - -**Acceptance Criteria:** -- [ ] `Cargo.toml` at repo root with `[workspace]` declaring `crates/translator` and `crates/proxy` -- [ ] `cargo build` succeeds with no errors -- [ ] `cargo test` runs (even if no tests yet) - -### US-002: Scaffold translator crate -**Description:** As a developer, I need the `anyllm_translate` library crate with the module structure defined in PLAN.md so subsequent phases have a place to add types and mapping logic. - -**Acceptance Criteria:** -- [ ] `crates/translator/Cargo.toml` with `serde`, `serde_json`, `uuid`, `thiserror` dependencies -- [ ] `src/lib.rs` with module declarations for `anthropic`, `openai`, `mapping`, `util` -- [ ] Subdirectories: `anthropic/`, `openai/`, `mapping/`, `util/` with `mod.rs` stubs -- [ ] `cargo build -p anyllm_translate` succeeds - -### US-003: Scaffold proxy crate -**Description:** As a developer, I need the `anyllm_proxy` binary crate with axum server skeleton and a health endpoint so I can verify the server starts. - -**Acceptance Criteria:** -- [ ] `crates/proxy/Cargo.toml` with `tokio`, `axum`, `reqwest`, `tracing`, `tracing-subscriber` dependencies -- [ ] `src/main.rs` starts a tokio runtime and binds axum router -- [ ] `src/config.rs` reads `OPENAI_API_KEY`, `OPENAI_BASE_URL`, `LISTEN_PORT` from env -- [ ] `src/server/routes.rs` with `GET /health` returning 200 `{"status":"ok"}` -- [ ] Module stubs for `server/middleware.rs`, `server/sse.rs`, `backend/openai_client.rs`, `metrics/mod.rs` - -### US-004: Health endpoint integration test -**Description:** As a developer, I need an integration test proving the server starts and the health endpoint responds, confirming the scaffolding works end to end. - -**Acceptance Criteria:** -- [ ] `crates/proxy/tests/` directory with at least one integration test -- [ ] Test starts the server on a random port, hits `GET /health`, asserts 200 -- [ ] `cargo test health_endpoint` passes - -### US-005: Create fixture directories -**Description:** As a developer, I need `fixtures/anthropic/` and `fixtures/openai/` directories for golden-file testing in later phases. - -**Acceptance Criteria:** -- [ ] `fixtures/anthropic/` and `fixtures/openai/` directories exist -- [ ] At least one placeholder `.json` file in each (can be empty object) - -## Functional Requirements - -- FR-1: Workspace builds with `cargo build` producing no errors or warnings -- FR-2: `cargo test` runs and passes (even with zero test assertions initially) -- FR-3: `cargo run -p anyllm_proxy` starts a server on the configured port -- FR-4: `GET /health` returns HTTP 200 with JSON body `{"status":"ok"}` -- FR-5: Environment variables `OPENAI_API_KEY`, `OPENAI_BASE_URL`, `LISTEN_PORT` are read with sensible defaults - -## Non-Goals - -- No translation logic -- No OpenAI client calls -- No authentication middleware -- No streaming support -- No CI/CD pipeline (handled separately) - -## Technical Considerations - -- Use `tokio` with `rt-multi-thread` and `macros` features -- axum 0.7+ for the server framework -- reqwest with `rustls-tls` (avoid openssl dep) -- Keep dependency count minimal; only add what Phase 1 needs - -## Success Metrics - -- `cargo build` completes in under 60 seconds on a clean build -- `cargo test` exits 0 -- Health endpoint responds within 10ms locally - - - -# PRD: Phase 5 - Tool Calling Translation - -## Introduction - -Implement the conversation history translation for tool calling: converting Anthropic `tool_use` content blocks and `tool_result` content blocks into OpenAI's `tool_calls` in assistant messages and `tool` role messages. This phase handles the structural mismatch where Anthropic uses content blocks within messages while OpenAI uses separate message roles and a JSON-string arguments format. - -## Goals - -- Translate assistant `tool_use` content blocks to OpenAI `tool_calls` in assistant messages -- Translate user `tool_result` content blocks to OpenAI `tool` role messages -- Handle the JSON object (Anthropic `input`) vs JSON string (OpenAI `arguments`) conversion -- Preserve tool call IDs across the round trip without server-side state -- Support multi-turn conversations with interleaved tool calls - -## User Stories - -### US-001: Stateless ID bridge -**Description:** As a developer, I need tool call IDs to pass through unchanged so the proxy requires no session storage for tool call tracking. - -**Acceptance Criteria:** -- [ ] OpenAI `tool_call.id` is used directly as Anthropic `tool_use.id` -- [ ] Client-provided `tool_result.tool_use_id` is used directly as OpenAI `tool_call_id` -- [ ] No ID rewriting, mapping table, or server-side storage -- [ ] Test: ID `"call_abc123"` survives a full round trip - -### US-002: Conversation history walker -**Description:** As a developer, I need the message mapper to walk Anthropic conversation history and correctly split content blocks into OpenAI message structures. - -**Acceptance Criteria:** -- [ ] Anthropic assistant message with `tool_use` blocks -> OpenAI assistant message with `tool_calls` array -- [ ] Anthropic assistant message with mixed text + `tool_use` -> OpenAI assistant message with `content` (text) AND `tool_calls` -- [ ] Anthropic user message with `tool_result` blocks -> one OpenAI `tool` role message per tool_result -- [ ] Anthropic user message with mixed text + `tool_result` -> OpenAI text user message + separate tool messages -- [ ] Multi-turn: correctly handles sequences of user -> assistant(tool_use) -> user(tool_result) -> assistant(text) -- [ ] Test: multi-turn conversation with 2+ tool calls produces correct OpenAI message sequence - -### US-003: JSON object to JSON string conversion -**Description:** As a developer, I need Anthropic `tool_use.input` (JSON object) converted to OpenAI `arguments` (JSON string) and vice versa, handling edge cases. - -**Acceptance Criteria:** -- [ ] Anthropic `input: {"ticker": "AAPL"}` -> OpenAI `arguments: "{\"ticker\":\"AAPL\"}"` -- [ ] OpenAI `arguments: "{\"ticker\":\"AAPL\"}"` -> Anthropic `input: {"ticker": "AAPL"}` -- [ ] Invalid JSON in OpenAI `arguments` -> preserve as-is in a best-effort manner (log warning) -- [ ] Empty input `{}` -> `"{}"` and back -- [ ] Test: valid JSON, empty JSON, nested objects - -### US-004: Tool result content translation -**Description:** As a developer, I need tool result content blocks translated to OpenAI tool message content. - -**Acceptance Criteria:** -- [ ] Anthropic `tool_result` with string content -> OpenAI tool message with string content -- [ ] Anthropic `tool_result` with array of content blocks -> concatenated text content in OpenAI tool message -- [ ] Anthropic `tool_result` with `is_error: true` -> OpenAI tool message content (error info preserved as text) -- [ ] Test: string result, structured result, error result - -### US-005: Response tool call translation (OpenAI -> Anthropic) -**Description:** As a developer, I need OpenAI assistant responses containing `tool_calls` translated back to Anthropic `tool_use` content blocks. - -**Acceptance Criteria:** -- [ ] OpenAI `tool_calls` array -> Anthropic `content` array with `tool_use` blocks -- [ ] OpenAI `arguments` (JSON string) -> Anthropic `input` (JSON object) -- [ ] Mixed content + tool_calls -> Anthropic content array with text block followed by tool_use blocks -- [ ] `finish_reason: "tool_calls"` -> Anthropic `stop_reason: "tool_use"` -- [ ] Test: single tool call, multiple parallel tool calls, mixed text + tool calls - -## Functional Requirements - -- FR-1: Tool call ID is passed through without modification (Anthropic `tool_use.id` = OpenAI `tool_call.id`) -- FR-2: Conversation walker processes messages in order, splitting/combining as needed -- FR-3: JSON string/object conversion uses `serde_json::to_string` / `serde_json::from_str` with defensive error handling -- FR-4: All functions remain pure (no IO, no state beyond the conversation being translated) -- FR-5: Tool definitions already handled by Phase 4; this phase focuses on tool call/result instances - -## Non-Goals - -- No streaming tool input deltas (Phase 7) -- No tool definition translation (already in Phase 4) -- No parallel tool use policy enforcement -- No tool result validation against tool schemas - -## Technical Considerations - -- The conversation walker must handle messages where a single Anthropic message contains both text and tool_use/tool_result blocks, splitting them into multiple OpenAI messages -- Order matters: OpenAI tool messages must appear after the assistant message that generated the tool_calls -- OpenAI `arguments` can be partial or malformed JSON (especially in streaming, but handle defensively even in non-streaming) -- Consider using `serde_json::from_str` with a fallback that wraps malformed JSON in an error object - -## Success Metrics - -- Golden fixture test: multi-turn tool conversation from PLAN.md lines 273-386 translates correctly in both directions -- Round-trip: Anthropic tool conversation -> OpenAI -> back to Anthropic produces equivalent structure -- `cargo test -p anyllm_translate` passes with all new tests - - - -# AGENTS.md - -This file provides guidance to Codex (Codex.ai/code) when working with code in this repository. - -## What This Is - -An Anthropic-to-OpenAI API translation proxy in Rust. Accepts Anthropic Messages API requests, translates them to OpenAI Chat Completions format, forwards to OpenAI, and translates back. Supports streaming SSE, tool calling, file/document blocks. - -See PLAN.md for the full specification and TASKS.md for phased implementation status (all 11 phases complete). - -## Current Status - -**Working (verified):** -- Build: `cargo build` clean, `cargo clippy -- -D warnings` clean -- Tests: ~371 tests passing (273 translator, 98 proxy) -- Full Anthropic Messages API translation: non-streaming, streaming SSE, tool calling, file/document blocks -- Proxy middleware: health, auth, request ID, size limits, concurrency limits, retry with backoff -- Compatibility endpoints: /v1/models, count_tokens (approximate via tiktoken), batches (stub) -- Model mapping and lossy-translation warnings - -**Not fully validated:** -- OpenAI Responses API backend: wired up via `OPENAI_API_FORMAT=responses` but not tested against live API -- Live API integration tests exist (`crates/proxy/tests/live_api.rs`) but are `#[ignore]` by default; run with `OPENAI_API_KEY=sk-... cargo test --test live_api -- --ignored --test-threads=1` -- Metrics endpoint exists (GET /metrics returns JSON counters) but streaming requests only track total count, not success/error - -## Build and Test - -```bash -cargo build # build everything -cargo test # run all tests (~395 tests) -cargo test -p anyllm_translate # translator crate only -cargo test -p anyllm_proxy # proxy crate only -cargo test health_endpoint # single test by name -cargo clippy -- -D warnings # lint -cargo fmt --check # format check -``` - -Run the proxy (requires OPENAI_API_KEY): -```bash -OPENAI_API_KEY=sk-... cargo run -p anyllm_proxy -# Listens on 0.0.0.0:3000, health at GET /health -``` - -## Environment Variables - -- `BACKEND`: Backend provider: `openai` (default), `vertex`, or `gemini` -- `OPENAI_API_KEY`: OpenAI API key (required when BACKEND=openai, empty default) -- `OPENAI_BASE_URL`: OpenAI base URL (default: `https://api.openai.com`) -- `OPENAI_API_FORMAT`: OpenAI API format: `chat` (default, Chat Completions) or `responses` (Responses API). Only relevant when BACKEND=openai. -- `LISTEN_PORT`: Server port (default: `3000`) -- `BIG_MODEL`: Backend model for sonnet/opus requests (default: `gpt-4o` for OpenAI, `gemini-2.5-pro` for Vertex/Gemini) -- `SMALL_MODEL`: Backend model for haiku requests (default: `gpt-4o-mini` for OpenAI, `gemini-2.5-flash` for Vertex/Gemini) -- `RUST_LOG`: Tracing filter (e.g., `info`, `anyllm_proxy=debug`) -- `TLS_CLIENT_CERT_P12`: Path to PKCS#12 (.p12/.pfx) client certificate for mTLS to the backend (optional) -- `TLS_CLIENT_CERT_PASSWORD`: Password to decrypt the P12 file (required if P12 is set) -- `TLS_CA_CERT`: Path to PEM-encoded CA certificate for verifying the backend server (optional) -- `VERTEX_PROJECT`: GCP project ID (required when BACKEND=vertex) -- `VERTEX_REGION`: GCP region, e.g. `us-central1` (required when BACKEND=vertex) -- `VERTEX_API_KEY`: Google API key for Vertex AI (one of VERTEX_API_KEY or GOOGLE_ACCESS_TOKEN required when BACKEND=vertex) -- `GOOGLE_ACCESS_TOKEN`: OAuth bearer token for Vertex AI (alternative to VERTEX_API_KEY) -- `GEMINI_API_KEY`: Google API key for Gemini Developer API (required when BACKEND=gemini) -- `GEMINI_BASE_URL`: Gemini API base URL (default: `https://generativelanguage.googleapis.com/v1beta`) -- `LOG_BODIES`: Enable request/response body logging at debug level (`true` or `1`, default: disabled) - -## Architecture - -Cargo workspace with two crates: - -### `crates/translator` (lib: `anyllm_translate`) -Pure translation logic, no IO. Four modules: -- **`anthropic/`**: Anthropic Messages API types (request, response, streaming events, errors) -- **`openai/`**: OpenAI types for both Chat Completions and Responses APIs -- **`mapping/`**: Stateless conversion functions between the two APIs - - `message_map`: Message/content block translation (system prompt -> developer role) - - `tools_map`: Tool definitions and tool_use/tool_call translation - - `usage_map`: Token usage field mapping - - `errors_map`: HTTP status and error shape translation - - `streaming_map`: SSE event stream translation state machine - - `responses_message_map`: Anthropic to/from OpenAI Responses API mapping - - `responses_streaming_map`: Responses API SSE event stream translation state machine -- **`util/`**: JSON helpers, ID generation (uuid v4), secret redaction - -### `crates/proxy` (bin: `anyllm_proxy`) -HTTP proxy built on axum + reqwest: -- **`config.rs`**: Env-based configuration -- **`server/routes.rs`**: Axum router (POST /v1/messages, GET /health, GET /metrics, GET /v1/models, stubs for count_tokens and batches) -- **`server/middleware.rs`**: Auth validation (x-api-key), request ID injection, 32MB size limit, concurrency limit, logging -- **`server/sse.rs`**: SSE response helpers for Anthropic-format streaming -- **`backend/mod.rs`**: `BackendClient` enum (OpenAI/OpenAIResponses/Vertex/Gemini), `BackendError`, shared retry helpers -- **`backend/openai_client.rs`**: reqwest client calling OpenAI Chat Completions with retry/backoff on 429/5xx -- **`backend/gemini_client.rs`**: reqwest client calling Gemini native `generateContent`/`streamGenerateContent` with retry/backoff -- **`metrics/`**: Request count, success/error tracking, exposed via GET /metrics - -### Data Flow -``` -Client (Anthropic format) -> proxy (axum) - -> translator: anthropic types -> mapping -> openai types - -> backend: reqwest -> OpenAI Chat Completions - -> translator: openai types -> mapping -> anthropic types - -> proxy (axum) -> Client (Anthropic format) -``` - -## Key Design Decisions - -- The translator crate is deliberately IO-free: all mapping is pure `fn(A) -> B`. This makes it testable without mocks. -- Tool call IDs pass through directly (Anthropic tool_use.id = OpenAI tool_call.id). -- OpenAI `arguments` is a JSON string; Anthropic `input` is a JSON object. The mapping layer handles serialization. -- Streaming uses a state machine in `streaming_map.rs` that transforms OpenAI chunk events into Anthropic SSE events, with bounded channel (32) for backpressure. -- JSON fixtures in `fixtures/anthropic/` and `fixtures/openai/` are used for golden-file testing (4 fixture files). -- Retry logic: 3 retries with exponential backoff + 25% jitter, respects retry-after header. -- Backoff jitter is deterministic (upper bound, not random) to keep tests predictable. -- `ChatCompletionRequest` uses `#[serde(flatten)] pub extra: serde_json::Map` to capture unknown OpenAI fields (e.g., `seed`, `logprobs`, `logit_bias`, `n`, `reasoning_effort`). These pass through to OpenAI without typed handling. Only fields that require translation logic (not just forwarding) need explicit struct fields. - -## Conventions - -- Most source files reference their PLAN.md line ranges in a comment at the top. -- Test files live alongside source (`#[cfg(test)]` modules) and in `crates/proxy/tests/` for integration tests. -- Error types use `thiserror` derive macros. -- Test distribution: translator (~224 tests), proxy (~79 tests including integration/compatibility). - -## References - -- OpenAI API spec: https://github.com/openai/openai-openapi/blob/manual_spec/openapi.yaml (very large, ~70k+ lines). See https://simonwillison.net/2024/Dec/22/openai-openapi/ for context on the spec's size and structure. Do not attempt to load the full spec into context; reference specific sections as needed. - - - -[workspace] -members = ["crates/translator", "crates/client", "crates/proxy"] -resolver = "2" - -[workspace.package] -version = "0.1.0" -edition = "2021" -license = "MIT" -repository = "https://github.com/whit3rabbit/anyllm-proxy" - - - -FROM rust:1.85-slim AS builder -RUN apt-get update && apt-get install -y --no-install-recommends libssl-dev pkg-config && rm -rf /var/lib/apt/lists/* -WORKDIR /app -COPY Cargo.toml Cargo.lock ./ -COPY crates crates -RUN cargo build --release -p anyllm_proxy - -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates libssl3 && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/anyllm_proxy /usr/local/bin/ -EXPOSE 3000 -ENTRYPOINT ["anyllm_proxy"] - - - -name: CI - -on: - push: - branches: [main] - tags: ["v*"] - pull_request: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: Check formatting - run: cargo fmt --check - - name: Clippy - run: cargo clippy -- -D warnings - - name: Build - run: cargo build - - name: Test - run: cargo test - - name: Security audit - run: | - cargo install cargo-audit --locked --quiet - cargo audit - - build-release: - name: Build (${{ matrix.target }}) - needs: test - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - include: - # Linux x86_64 - - os: ubuntu-latest - target: x86_64-unknown-linux-gnu - binary: anyllm_proxy - # Linux ARM64 (native GitHub runner) - - os: ubuntu-24.04-arm - target: aarch64-unknown-linux-gnu - binary: anyllm_proxy - # macOS Apple Silicon - - os: macos-latest - target: aarch64-apple-darwin - binary: anyllm_proxy - # macOS Intel - - os: macos-latest - target: x86_64-apple-darwin - binary: anyllm_proxy - # Windows x86_64 - - os: windows-latest - target: x86_64-pc-windows-msvc - binary: anyllm_proxy.exe - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 - with: - key: ${{ matrix.target }} - - name: Build - run: cargo build --release -p anyllm_proxy --target ${{ matrix.target }} - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: anyllm_proxy-${{ matrix.target }} - path: target/${{ matrix.target }}/release/${{ matrix.binary }} - - publish: - name: Publish to crates.io - needs: [test, build-release] - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/v') - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - # Publish in dependency order. --no-verify skips re-building from the - # packed tarball; the test job already verified the build. - # Sleeps give the crates.io index time to propagate before dependents publish. - - name: Publish anyllm_translate - run: cargo publish -p anyllm_translate - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - - name: Wait for index propagation - run: sleep 30 - - name: Publish anyllm_client - run: cargo publish -p anyllm_client --no-verify - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - - name: Wait for index propagation - run: sleep 30 - - name: Publish anyllm_proxy - run: cargo publish -p anyllm_proxy --no-verify - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - - - -//! HTTP client builder with optional mTLS, custom CA, and SSRF-safe DNS resolution. -⋮---- -use reqwest::Client; -use std::net::IpAddr; -use std::time::Duration; -⋮---- -/// Configuration for building an HTTP client. -⋮---- -pub struct HttpClientConfig { -/// PKCS#12 identity bytes and password for mTLS. -⋮---- -/// PEM-encoded CA certificate for verifying the backend server. -⋮---- -/// Connection timeout (default: 10s). -⋮---- -/// Read timeout (default: 900s, generous for reasoning models). -⋮---- -/// TCP keepalive interval (default: 60s). -⋮---- -/// Enable SSRF-safe DNS resolution (default: true when `ssrf-protection` feature enabled). -⋮---- -impl HttpClientConfig { -pub fn new() -> Self { -⋮---- -ssrf_protection: cfg!(feature = "ssrf-protection"), -⋮---- -/// Build a reqwest HTTP client from configuration. -/// -/// Includes hardened defaults: 10s connect timeout, 900s read timeout (for slow -/// reasoning models like o1/o3), 60s TCP keepalive, and SSRF-safe DNS resolution. -pub fn build_http_client(config: &HttpClientConfig) -> Client { -⋮---- -.expect("P12 identity was validated at startup"); -builder = builder.identity(identity); -⋮---- -reqwest::Certificate::from_pem(ca_pem).expect("CA cert was validated at startup"); -builder = builder.add_root_certificate(cert); -⋮---- -let connect_timeout = config.connect_timeout.unwrap_or(Duration::from_secs(10)); -let read_timeout = config.read_timeout.unwrap_or(Duration::from_secs(900)); -let tcp_keepalive = config.tcp_keepalive.unwrap_or(Duration::from_secs(60)); -⋮---- -.connect_timeout(connect_timeout) -.read_timeout(read_timeout) -.tcp_keepalive(tcp_keepalive); -⋮---- -builder = builder.dns_resolver(std::sync::Arc::new(SsrfSafeDnsResolver)); -⋮---- -builder.build().expect("failed to build HTTP client") -⋮---- -/// DNS resolver that rejects private/loopback IPs at connection time, -/// preventing DNS rebinding attacks where a domain resolves to a public IP -/// at startup validation but later resolves to a private/metadata IP. -⋮---- -struct SsrfSafeDnsResolver; -⋮---- -fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { -⋮---- -let name_str = name.as_str().to_string(); -// DNS resolution (ToSocketAddrs) blocks the calling thread. -// Must run on the blocking threadpool to avoid stalling the -// async runtime and all other in-flight requests. -⋮---- -use std::net::ToSocketAddrs; -// Port 0 is a placeholder; reqwest replaces it with the actual port. -let lookup = format!("{name_str}:0"); -Ok(lookup.to_socket_addrs()?.collect()) -⋮---- -.map_err(|e| -> Box { Box::new(e) })? -.map_err( -⋮---- -// Filter out private/loopback IPs to prevent SSRF attacks where -// an attacker-controlled DNS record resolves to internal endpoints -// (e.g., cloud metadata at 169.254.169.254). -⋮---- -.into_iter() -.filter(|addr| !is_private_ip(addr.ip())) -.collect(); -⋮---- -if safe.is_empty() { -return Err(Box::new(std::io::Error::new( -⋮---- -"DNS resolved only to private/loopback IPs (SSRF blocked)".to_string(), -⋮---- -Ok(Box::new(safe.into_iter()) as Box + Send>) -⋮---- -/// Returns true for loopback, private (RFC 1918), link-local, and -/// cloud metadata IPs (169.254.169.254). -pub fn is_private_ip(ip: IpAddr) -> bool { -⋮---- -v4.is_loopback() -|| v4.is_private() -|| v4.is_link_local() -|| v4.is_broadcast() -|| v4.is_unspecified() -// AWS/GCP/Azure metadata endpoint. SSRF to this IP lets -// attackers exfiltrate instance credentials. -⋮---- -let seg0 = v6.segments()[0]; -v6.is_loopback() -|| v6.is_unspecified() -// Unique Local Addresses (fc00::/7): covers fc00:: through fdff:: -⋮---- -// Link-Local addresses (fe80::/10): covers fe80:: through febf:: -⋮---- -// IPv4-mapped (::ffff:x.x.x.x): check recursively against IPv4 rules -|| matches!(v6.to_ipv4_mapped(), Some(v4) if is_private_ip(IpAddr::V4(v4))) -⋮---- -mod tests { -⋮---- -fn private_ipv4_loopback() { -assert!(is_private_ip("127.0.0.1".parse().unwrap())); -⋮---- -fn private_ipv4_rfc1918() { -assert!(is_private_ip("10.0.0.1".parse().unwrap())); -assert!(is_private_ip("172.16.0.1".parse().unwrap())); -assert!(is_private_ip("192.168.1.1".parse().unwrap())); -⋮---- -fn private_ipv4_link_local() { -assert!(is_private_ip("169.254.1.1".parse().unwrap())); -⋮---- -fn private_ipv4_metadata() { -assert!(is_private_ip("169.254.169.254".parse().unwrap())); -⋮---- -fn private_ipv4_unspecified() { -assert!(is_private_ip("0.0.0.0".parse().unwrap())); -⋮---- -fn public_ipv4() { -assert!(!is_private_ip("8.8.8.8".parse().unwrap())); -assert!(!is_private_ip("1.1.1.1".parse().unwrap())); -⋮---- -fn private_ipv6_loopback() { -assert!(is_private_ip("::1".parse().unwrap())); -⋮---- -fn private_ipv6_mapped_private() { -// ::ffff:192.168.1.1 -assert!(is_private_ip("::ffff:192.168.1.1".parse().unwrap())); -⋮---- -fn public_ipv6() { -assert!(!is_private_ip("2001:4860:4860::8888".parse().unwrap())); -⋮---- -fn private_ipv6_ula() { -// fc00::/7 covers fc00:: through fdff:: -assert!(is_private_ip("fc00::1".parse().unwrap())); -assert!(is_private_ip("fd12:3456:789a:1::1".parse().unwrap())); -assert!(is_private_ip("fdff:ffff:ffff:ffff::1".parse().unwrap())); -⋮---- -fn private_ipv6_link_local() { -// fe80::/10 covers fe80:: through febf:: -assert!(is_private_ip("fe80::1".parse().unwrap())); -assert!(is_private_ip("fe80::dead:beef".parse().unwrap())); -assert!(is_private_ip("febf::1".parse().unwrap())); -⋮---- -fn default_config_has_ssrf_protection() { -⋮---- -assert_eq!(config.ssrf_protection, cfg!(feature = "ssrf-protection")); -⋮---- -fn build_client_default_config() { -⋮---- -ssrf_protection: false, // avoid DNS in tests -⋮---- -let _client = build_http_client(&config); - - - -//! # anyllm_client -//! -//! Async HTTP client for Anthropic-to-OpenAI API translation. -⋮---- -//! Accepts Anthropic Messages API requests, translates them to OpenAI Chat Completions -//! format, sends them to an OpenAI-compatible backend, and translates the response back. -//! Supports non-streaming and streaming (SSE) modes, retry with exponential backoff, -//! SSRF-safe DNS resolution, and mTLS. -⋮---- -//! # Quick start -⋮---- -//! ```rust,no_run -//! use anyllm_client::{Client, ClientConfig, Auth}; -//! use anyllm_translate::TranslationConfig; -//! use anyllm_translate::anthropic::MessageCreateRequest; -⋮---- -//! # async fn example() -> Result<(), anyllm_client::ClientError> { -//! let config = ClientConfig::builder() -//! .backend_url("https://api.openai.com/v1/chat/completions") -//! .auth(Auth::Bearer("sk-...".into())) -//! .translation( -//! TranslationConfig::builder() -//! .model_map("haiku", "gpt-4o-mini") -//! .model_map("sonnet", "gpt-4o") -//! .build() -//! ) -//! .build(); -⋮---- -//! let client = Client::new(config); -⋮---- -//! let req: MessageCreateRequest = serde_json::from_str(r#"{ -//! "model": "claude-sonnet-4-6", -//! "max_tokens": 100, -//! "messages": [{"role": "user", "content": "Hello"}] -//! }"#).unwrap(); -⋮---- -//! let response = client.messages(&req).await?; -//! println!("{:?}", response); -//! # Ok(()) -//! # } -//! ``` -⋮---- -//! # Modules -⋮---- -//! - [`client`] -- High-level `Client` and [`ClientBuilder`] for Anthropic-in, Anthropic-out API calls -//! - [`tools`] -- Builder helpers for [`Tool`] definitions and [`ToolChoice`] -//! - [`http`] -- HTTP client builder with TLS and SSRF protection -//! - [`retry`] -- Generic retry logic with exponential backoff -//! - [`rate_limit`] -- Rate limit header extraction and format conversion -//! - [`sse`] -- Framework-agnostic SSE frame parser -//! - [`error`] -- Error types -⋮---- -pub mod client; -pub mod error; -pub mod http; -pub mod rate_limit; -pub mod retry; -pub mod sse; -pub(crate) mod streaming; -pub mod tools; -⋮---- -// Convenience re-exports -⋮---- -pub use error::ClientError; -⋮---- -pub use rate_limit::RateLimitHeaders; -⋮---- -// Re-export key types from the translator crate so downstream users -// do not need a direct dependency on `anyllm_translate`. -pub use anyllm_translate::anthropic::streaming::StreamEvent; - - - -[package] -name = "anyllm_client" -description = "Async HTTP client for Anthropic-to-OpenAI translation with retry, SSRF protection, and SSE streaming" -version = "0.2.0" -edition.workspace = true -license.workspace = true -repository.workspace = true -categories = ["api-bindings", "web-programming::http-client"] -keywords = ["anthropic", "openai", "translation", "llm", "client"] - -[features] -default = ["ssrf-protection"] -ssrf-protection = [] - -[dependencies] -anyllm_translate = { path = "../translator", version = "0.1.0" } -reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "native-tls", "http2"] } -tokio = { version = "1", features = ["rt", "sync", "time"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -tracing = "0.1" -bytes = "1" -futures = "0.3" -httpdate = "1" -thiserror = "2" -pin-project-lite = "0.2" - -[dev-dependencies] -tokio = { version = "1", features = ["full"] } -pretty_assertions = "1" - - - -// SQLite CRUD for batch_file and batch_job tables. -⋮---- -use crate::admin::db::now_iso8601; -⋮---- -/// Mapping from our Anthropic batch ID to the upstream OpenAI batch ID. -pub struct AnthropicBatchMap { -⋮---- -/// Create the anthropic_batch_map table if it doesn't exist. -pub fn init_anthropic_batch_map_table(conn: &Connection) -> rusqlite::Result<()> { -conn.execute_batch( -⋮---- -/// Store an Anthropic->OpenAI batch id mapping. -pub fn insert_anthropic_batch_map( -⋮---- -conn.execute( -⋮---- -params![our_batch_id, openai_batch_id], -⋮---- -Ok(()) -⋮---- -/// Look up a mapping by our batch ID. -pub fn get_anthropic_batch_map( -⋮---- -let mut stmt = conn.prepare( -⋮---- -let mut rows = stmt.query(params![our_batch_id])?; -if let Some(row) = rows.next()? { -Ok(Some(AnthropicBatchMap { -our_batch_id: row.get(0)?, -openai_batch_id: row.get(1)?, -openai_output_file_id: row.get(2)?, -model: row.get(3)?, -⋮---- -Ok(None) -⋮---- -/// Update the output_file_id once the batch completes. -pub fn set_anthropic_batch_output_file( -⋮---- -params![output_file_id, our_batch_id], -⋮---- -/// Create the batch_file and batch_job tables if they do not exist. -pub fn init_batch_tables(conn: &Connection) -> rusqlite::Result<()> { -⋮---- -// Also create the Anthropic batch ID mapping table. -init_anthropic_batch_map_table(conn)?; -⋮---- -/// Insert a new batch file record. -⋮---- -pub fn insert_batch_file( -⋮---- -params![ -⋮---- -/// Check if a batch file exists by file_id. Returns (byte_size, line_count, created_at) if found. -pub fn get_batch_file_meta( -⋮---- -.prepare("SELECT byte_size, line_count, created_at FROM batch_file WHERE file_id = ?1")?; -let mut rows = stmt.query_map(params![file_id], |row| { -Ok((row.get(0)?, row.get(1)?, row.get(2)?)) -⋮---- -rows.next().transpose() -⋮---- -/// Insert a new batch job record. -pub fn insert_batch_job( -⋮---- -let meta_str = metadata.map(|m| serde_json::to_string(m).unwrap_or_default()); -⋮---- -/// Fetch a single batch job by batch_id. -pub fn get_batch_job(conn: &Connection, batch_id: &str) -> rusqlite::Result> { -⋮---- -let mut rows = stmt.query_map(params![batch_id], row_to_batch_job)?; -⋮---- -/// Update the status (and optional completion fields) of a batch job. -pub fn update_batch_job_status( -⋮---- -let completed_at = if matches!(status, BatchStatus::Completed | BatchStatus::Failed) { -Some(now_iso8601()) -⋮---- -let changed = conn.execute( -⋮---- -Ok(changed > 0) -⋮---- -/// List batch jobs, optionally filtered by key_id, with cursor pagination. -pub fn list_batch_jobs( -⋮---- -sql.push_str(" AND key_id = ?"); -param_values.push(Box::new(kid)); -⋮---- -sql.push_str(" AND batch_id < ?"); -param_values.push(Box::new(cursor.to_string())); -⋮---- -sql.push_str(" ORDER BY id DESC LIMIT ?"); -param_values.push(Box::new(limit)); -⋮---- -param_values.iter().map(|p| p.as_ref()).collect(); -⋮---- -let mut stmt = conn.prepare(&sql)?; -let rows = stmt.query_map(params_refs.as_slice(), row_to_batch_job)?; -rows.collect() -⋮---- -/// Map a SQLite row to a BatchJob. -fn row_to_batch_job(row: &rusqlite::Row) -> rusqlite::Result { -let status_str: String = row.get(3)?; -let created_at_str: String = row.get(9)?; -let metadata_str: Option = row.get(12)?; -⋮---- -Ok(BatchJob { -id: row.get(0)?, -object: "batch".to_string(), -endpoint: "/v1/chat/completions".to_string(), -⋮---- -input_file_id: row.get(1)?, -completion_window: "24h".to_string(), -created_at: iso8601_to_epoch(&created_at_str), -⋮---- -total: row.get(4)?, -completed: row.get(5)?, -failed: row.get(6)?, -⋮---- -metadata: metadata_str.and_then(|s| serde_json::from_str(&s).ok()), -output_file_id: row.get(7)?, -error_file_id: row.get(8)?, -⋮---- -.map(|s| iso8601_to_epoch(&s)), -⋮---- -/// Approximate conversion from ISO 8601 string to unix epoch seconds. -/// Falls back to 0 on parse failure (non-critical metadata field). -fn iso8601_to_epoch(s: &str) -> i64 { -// Parse "YYYY-MM-DDTHH:MM:SSZ" manually (no chrono dependency). -let parts: Vec<&str> = s.split('T').collect(); -if parts.len() != 2 { -⋮---- -let date_parts: Vec = parts[0].split('-').filter_map(|p| p.parse().ok()).collect(); -let time_str = parts[1].trim_end_matches('Z'); -let time_parts: Vec = time_str.split(':').filter_map(|p| p.parse().ok()).collect(); -⋮---- -if date_parts.len() != 3 || time_parts.len() != 3 { -⋮---- -// Days from epoch using the inverse of the Howard Hinnant algorithm. -⋮---- -mod tests { -⋮---- -use crate::admin::db::init_db; -⋮---- -fn test_db() -> Connection { -let conn = Connection::open_in_memory().unwrap(); -init_db(&conn).unwrap(); -init_batch_tables(&conn).unwrap(); -⋮---- -fn insert_and_get_batch_file() { -let conn = test_db(); -insert_batch_file( -⋮---- -Some("test.jsonl"), -⋮---- -.unwrap(); -⋮---- -let meta = get_batch_file_meta(&conn, "file-abc123").unwrap(); -assert!(meta.is_some()); -let (size, count, _created) = meta.unwrap(); -assert_eq!(size, 1024); -assert_eq!(count, 10); -⋮---- -fn insert_and_get_batch_job() { -⋮---- -insert_batch_file(&conn, "file-input1", None, "batch", None, 512, 5, b"data").unwrap(); -⋮---- -insert_batch_job(&conn, "batch-job1", None, "file-input1", "openai", 5, None).unwrap(); -⋮---- -let job = get_batch_job(&conn, "batch-job1").unwrap(); -assert!(job.is_some()); -let job = job.unwrap(); -assert_eq!(job.id, "batch-job1"); -assert_eq!(job.status, BatchStatus::Validating); -assert_eq!(job.request_counts.total, 5); -assert_eq!(job.input_file_id, "file-input1"); -⋮---- -fn update_batch_job_status_works() { -⋮---- -insert_batch_file(&conn, "file-u1", None, "batch", None, 100, 2, b"d").unwrap(); -insert_batch_job(&conn, "batch-u1", None, "file-u1", "openai", 2, None).unwrap(); -⋮---- -let ok = update_batch_job_status( -⋮---- -Some(2), -Some(0), -Some("file-out1"), -⋮---- -assert!(ok); -⋮---- -let job = get_batch_job(&conn, "batch-u1").unwrap().unwrap(); -assert_eq!(job.status, BatchStatus::Completed); -assert_eq!(job.request_counts.completed, 2); -assert!(job.output_file_id.is_some()); -assert!(job.completed_at.is_some()); -⋮---- -fn list_batch_jobs_with_pagination() { -⋮---- -insert_batch_file(&conn, "file-l1", None, "batch", None, 10, 1, b"d").unwrap(); -⋮---- -insert_batch_job( -⋮---- -&format!("batch-l{i}"), -Some(1), -⋮---- -let all = list_batch_jobs(&conn, Some(1), 10, None).unwrap(); -assert_eq!(all.len(), 5); -⋮---- -let page = list_batch_jobs(&conn, Some(1), 2, None).unwrap(); -assert_eq!(page.len(), 2); -⋮---- -// Different key_id returns nothing -let empty = list_batch_jobs(&conn, Some(999), 10, None).unwrap(); -assert!(empty.is_empty()); -⋮---- -fn get_nonexistent_job() { -⋮---- -let job = get_batch_job(&conn, "batch-nope").unwrap(); -assert!(job.is_none()); -⋮---- -fn iso8601_round_trip() { -// 2026-03-22T10:30:00Z -let epoch = iso8601_to_epoch("2026-03-22T10:30:00Z"); -assert!(epoch > 0); -⋮---- -fn anthropic_batch_map_round_trip() { -let conn = rusqlite::Connection::open_in_memory().unwrap(); -⋮---- -init_anthropic_batch_map_table(&conn).unwrap(); -⋮---- -insert_anthropic_batch_map(&conn, "msgbatch_our1", "batch_openai1").unwrap(); -let record = get_anthropic_batch_map(&conn, "msgbatch_our1") -.unwrap() -⋮---- -assert_eq!(record.openai_batch_id, "batch_openai1"); -assert!(record.openai_output_file_id.is_none()); -⋮---- -set_anthropic_batch_output_file(&conn, "msgbatch_our1", "file-output1").unwrap(); -let record2 = get_anthropic_batch_map(&conn, "msgbatch_our1") -⋮---- -assert_eq!(record2.openai_output_file_id.as_deref(), Some("file-output1")); - - - -// crates/proxy/src/batch/openai_batch_client.rs -// HTTP client for OpenAI batch API endpoints (/v1/files, /v1/batches). -// Stateless: each method takes the credentials stored in OpenAIBatchClient. -⋮---- -/// Thin HTTP client for OpenAI file and batch APIs. -/// -/// Constructed once per request from `AppState` config values. -pub struct OpenAIBatchClient { -⋮---- -impl OpenAIBatchClient { -pub fn new(api_key: String, base_url: String) -> Self { -⋮---- -pub fn files_url(&self) -> String { -format!("{}/v1/files", self.base_url.trim_end_matches('/')) -⋮---- -pub fn batches_url(&self) -> String { -format!("{}/v1/batches", self.base_url.trim_end_matches('/')) -⋮---- -fn batch_url(&self, batch_id: &str) -> String { -format!("{}/{}", self.batches_url(), batch_id) -⋮---- -fn file_content_url(&self, file_id: &str) -> String { -format!("{}/{}/content", self.files_url(), file_id) -⋮---- -/// Upload a JSONL string as a file with purpose=batch. -/// Returns the OpenAI file_id (e.g. "file-abc123"). -pub async fn upload_jsonl_file(&self, jsonl: &str) -> Result { -let part = multipart::Part::text(jsonl.to_string()) -.file_name("batch.jsonl") -.mime_str("application/jsonl") -.map_err(|e| format!("mime error: {e}"))?; -⋮---- -.text("purpose", "batch") -.part("file", part); -⋮---- -.post(self.files_url()) -.header("Authorization", format!("Bearer {}", self.api_key)) -.multipart(form) -.send() -⋮---- -.map_err(|e| format!("upload request failed: {e}"))?; -⋮---- -if !resp.status().is_success() { -let body = resp.text().await.unwrap_or_default(); -return Err(format!("file upload failed: {body}")); -⋮---- -let v: serde_json::Value = resp.json().await.map_err(|e| format!("parse error: {e}"))?; -⋮---- -.as_str() -.map(|s| s.to_string()) -.ok_or_else(|| "missing id in file upload response".to_string()) -⋮---- -/// Create an OpenAI batch job from an uploaded file. -/// Returns the OpenAI batch_id (e.g. "batch_abc123"). -pub async fn create_batch(&self, input_file_id: &str) -> Result { -⋮---- -.post(self.batches_url()) -⋮---- -.header("Content-Type", "application/json") -.json(&body) -⋮---- -.map_err(|e| format!("batch create failed: {e}"))?; -⋮---- -return Err(format!("batch creation failed: {body}")); -⋮---- -.ok_or_else(|| "missing id in batch response".to_string()) -⋮---- -/// Poll status of an OpenAI batch job. Returns the raw JSON value. -pub async fn get_batch_status( -⋮---- -.get(self.batch_url(openai_batch_id)) -⋮---- -.map_err(|e| format!("get batch failed: {e}"))?; -⋮---- -if resp.status() == StatusCode::NOT_FOUND { -return Err("batch not found at OpenAI".to_string()); -⋮---- -return Err(format!("get batch failed: {body}")); -⋮---- -resp.json().await.map_err(|e| format!("parse error: {e}")) -⋮---- -/// Download the content of an OpenAI file by file_id. Returns raw bytes as String. -pub async fn get_file_content(&self, file_id: &str) -> Result { -⋮---- -.get(self.file_content_url(file_id)) -⋮---- -.map_err(|e| format!("file download failed: {e}"))?; -⋮---- -return Err(format!("file download failed: {body}")); -⋮---- -resp.text() -⋮---- -.map_err(|e| format!("read body failed: {e}")) -⋮---- -/// Map an OpenAI batch status string to Anthropic ProcessingStatus. -pub fn openai_status_to_processing_status(status: &str) -> ProcessingStatus { -⋮---- -_ => ProcessingStatus::Ended, // completed, failed, expired, cancelled -⋮---- -/// Map an OpenAI batch JSON response to an Anthropic MessageBatch. -pub fn openai_batch_to_message_batch(our_batch_id: &str, v: &serde_json::Value) -> MessageBatch { -⋮---- -.duration_since(UNIX_EPOCH) -.unwrap_or_default() -.as_secs() as i64; -⋮---- -let status_str = v["status"].as_str().unwrap_or("in_progress"); -let processing_status = openai_status_to_processing_status(status_str); -⋮---- -processing: v["request_counts"]["in_progress"].as_u64().unwrap_or(0) as u32 -+ v["request_counts"]["validating"].as_u64().unwrap_or(0) as u32, -succeeded: v["request_counts"]["completed"].as_u64().unwrap_or(0) as u32, -errored: v["request_counts"]["failed"].as_u64().unwrap_or(0) as u32, -canceled: v["request_counts"]["cancelled"].as_u64().unwrap_or(0) as u32, -expired: v["request_counts"]["expired"].as_u64().unwrap_or(0) as u32, -⋮---- -let ended_at = if matches!(processing_status, ProcessingStatus::Ended) { -v["completed_at"].as_i64().or(Some(now)) -⋮---- -id: our_batch_id.to_string(), -type_: "message_batch".to_string(), -⋮---- -created_at: v["created_at"].as_i64().unwrap_or(now), -expires_at: v["expires_at"].as_i64().unwrap_or(now + 86400), -⋮---- -mod tests { -⋮---- -fn build_file_upload_url() { -⋮---- -"sk-test".to_string(), -"https://api.openai.com".to_string(), -⋮---- -assert_eq!(c.files_url(), "https://api.openai.com/v1/files"); -assert_eq!(c.batches_url(), "https://api.openai.com/v1/batches"); -⋮---- -fn parse_openai_batch_status_to_anthropic() { -⋮---- -assert!(matches!( - - - -// Axum handlers for batch file upload and job management. -// POST /v1/files, POST /v1/batches, GET /v1/batches/{id}, GET /v1/batches -⋮---- -use super::db; -use super::validate_jsonl; -use crate::backend::BackendClient; -use crate::server::routes::AppState; -use anyllm_translate::anthropic; -use anyllm_translate::mapping::errors_map::create_anthropic_error; -⋮---- -use serde::Deserialize; -⋮---- -/// POST /v1/files - Upload a JSONL batch file via multipart/form-data. -/// -/// Expects fields: `purpose` (must be "batch") and `file` (the JSONL content). -pub async fn upload_file(State(state): State, mut multipart: Multipart) -> Response { -let db = match state.shared.as_ref().map(|s| s.db.clone()) { -⋮---- -None => return service_unavailable("Batch storage not available"), -⋮---- -while let Ok(Some(field)) = multipart.next_field().await { -let field_name = field.name().unwrap_or("").to_string(); -match field_name.as_str() { -⋮---- -purpose = field.text().await.ok(); -⋮---- -filename = field.file_name().map(|s| s.to_string()); -file_data = field.bytes().await.ok(); -⋮---- -let purpose = match purpose.as_deref() { -⋮---- -return bad_request(&format!( -⋮---- -return bad_request("Missing required field 'purpose'"); -⋮---- -Some(d) if !d.is_empty() => d, -⋮---- -return bad_request("Missing or empty 'file' field"); -⋮---- -// Validate JSONL structure -let validated = match validate_jsonl(BufReader::new(Cursor::new(data.as_ref()))) { -⋮---- -return bad_request(&format!("Invalid JSONL: {e}")); -⋮---- -let file_id = format!("file-{}", uuid::Uuid::new_v4()); -let byte_size = data.len() as i64; -⋮---- -// Insert into SQLite on the blocking threadpool -let file_id_clone = file_id.clone(); -let filename_clone = filename.clone(); -let data_ref = data.as_ref().to_vec(); -⋮---- -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -filename_clone.as_deref(), -⋮---- -.duration_since(std::time::UNIX_EPOCH) -.unwrap_or_default() -.as_secs() as i64; -⋮---- -object: "file".to_string(), -⋮---- -purpose: purpose.to_string(), -⋮---- -(StatusCode::OK, Json(file_obj)).into_response() -⋮---- -internal_error("Failed to store file") -⋮---- -internal_error("Internal error") -⋮---- -/// Request body for POST /v1/batches. -⋮---- -pub struct CreateBatchRequest { -⋮---- -fn default_endpoint() -> String { -"/v1/chat/completions".to_string() -⋮---- -fn default_completion_window() -> String { -"24h".to_string() -⋮---- -/// POST /v1/batches - Create a new batch job. -⋮---- -/// Returns 501 for unsupported backends (vertex, gemini, anthropic, bedrock). -pub async fn create_batch( -⋮---- -// Check backend support: only openai and azure are supported -if !is_batch_supported(&state.backend) { -return not_implemented(&format!( -⋮---- -let input_file_id = req.input_file_id.clone(); -let batch_id = format!("batch-{}", uuid::Uuid::new_v4()); -let backend_name = state.backend_name.clone(); -let metadata = req.metadata.clone(); -⋮---- -// Verify input file exists and get line count -let batch_id_clone = batch_id.clone(); -⋮---- -return Ok(None); -⋮---- -metadata.as_ref(), -⋮---- -Ok(Ok(Some(job))) => (StatusCode::OK, Json(job)).into_response(), -Ok(Ok(None)) => bad_request(&format!("Input file '{}' not found", req.input_file_id)), -⋮---- -internal_error("Failed to create batch job") -⋮---- -/// GET /v1/batches/{batch_id} - Retrieve a batch job by ID. -pub async fn get_batch(State(state): State, Path(batch_id): Path) -> Response { -⋮---- -let err = create_anthropic_error( -⋮---- -"Batch not found".to_string(), -⋮---- -(StatusCode::NOT_FOUND, Json(err)).into_response() -⋮---- -internal_error("Failed to fetch batch job") -⋮---- -/// Query parameters for GET /v1/batches. -⋮---- -pub struct ListBatchesQuery { -⋮---- -fn default_limit() -> u32 { -⋮---- -/// GET /v1/batches - List batch jobs with cursor pagination. -pub async fn list_batches( -⋮---- -let limit = query.limit.min(100); -let after = query.after.clone(); -⋮---- -db::list_batch_jobs(&conn, None, limit, after.as_deref()) -⋮---- -let has_more = jobs.len() as u32 == limit; -let last_id = jobs.last().map(|j| j.id.clone()); -⋮---- -(StatusCode::OK, Json(response)).into_response() -⋮---- -internal_error("Failed to list batch jobs") -⋮---- -/// Check if the backend supports batch processing (OpenAI and Azure only). -fn is_batch_supported(backend: &BackendClient) -> bool { -matches!( -⋮---- -fn bad_request(msg: &str) -> Response { -⋮---- -msg.to_string(), -⋮---- -(StatusCode::BAD_REQUEST, Json(err)).into_response() -⋮---- -fn not_implemented(msg: &str) -> Response { -⋮---- -(StatusCode::NOT_IMPLEMENTED, Json(err)).into_response() -⋮---- -fn service_unavailable(msg: &str) -> Response { -let err = create_anthropic_error(anthropic::ErrorType::ApiError, msg.to_string(), None); -(StatusCode::SERVICE_UNAVAILABLE, Json(err)).into_response() -⋮---- -fn internal_error(msg: &str) -> Response { -⋮---- -(StatusCode::INTERNAL_SERVER_ERROR, Json(err)).into_response() - - - -//! In-memory cache backend using moka's async cache. -//! -//! moka provides a concurrent, lock-free cache with TTL-based expiration -//! and bounded capacity (LRU eviction when full). -⋮---- -//! Per-entry TTL is enforced via moka's `Expiry` trait. Each `CacheEntry` -//! carries an optional `ttl_secs` override; when absent, the cache-level -//! default applies. -⋮---- -use moka::Expiry; -use std::time::Duration; -⋮---- -/// Per-entry expiry policy. Reads `CacheEntry::ttl_secs` to decide lifetime; -/// falls back to `default_ttl` when the entry has no override. -struct EntryExpiry { -⋮---- -fn expire_after_create( -⋮---- -Some(ttl) -⋮---- -/// In-memory cache backed by moka::future::Cache. -/// -/// Configured with a default TTL and max entry count. Per-request TTL -/// overrides are enforced via the `EntryExpiry` implementation of moka's -/// `Expiry` trait. -pub struct MemoryCache { -⋮---- -/// Default TTL applied when the request does not specify cache_ttl_secs. -⋮---- -impl MemoryCache { -/// Create a new in-memory cache from the provided configuration. -pub fn new(config: &CacheConfig) -> Self { -⋮---- -.max_capacity(config.max_entries) -.expire_after(EntryExpiry { default_ttl }) -.build(); -⋮---- -impl CacheBackend for MemoryCache { -async fn get(&self, key: &str) -> Option { -self.inner.get(key).await -⋮---- -async fn put(&self, key: &str, entry: CacheEntry, _ttl_secs: u64) { -// Per-entry TTL is now handled by EntryExpiry reading entry.ttl_secs. -// The _ttl_secs parameter from CacheBackend::put is unused; the entry -// itself carries the authoritative TTL override. -self.inner.insert(key.to_string(), entry).await; -⋮---- -mod tests { -⋮---- -use bytes::Bytes; -use std::time::Instant; -⋮---- -fn test_config() -> CacheConfig { -⋮---- -fn test_entry(body: &str) -> CacheEntry { -⋮---- -response_body: Bytes::from(body.to_string()), -model: "test-model".to_string(), -⋮---- -fn test_entry_with_ttl(body: &str, ttl: u64) -> CacheEntry { -⋮---- -ttl_secs: Some(ttl), -⋮---- -async fn put_and_get() { -let cache = MemoryCache::new(&test_config()); -let entry = test_entry(r#"{"id":"msg_1"}"#); -cache.put("test:abc123", entry.clone(), 60).await; -let got = cache.get("test:abc123").await; -assert!(got.is_some()); -assert_eq!(got.unwrap().response_body, entry.response_body); -⋮---- -async fn get_miss() { -⋮---- -let got = cache.get("test:nonexistent").await; -assert!(got.is_none()); -⋮---- -async fn ttl_expiry() { -⋮---- -cache.put("test:expire", test_entry("data"), 1).await; -⋮---- -// Entry should be present immediately -assert!(cache.get("test:expire").await.is_some()); -⋮---- -// Wait for TTL to expire -⋮---- -assert!(cache.get("test:expire").await.is_none()); -⋮---- -async fn max_capacity_eviction() { -⋮---- -cache.put("k1", test_entry("v1"), 300).await; -cache.put("k2", test_entry("v2"), 300).await; -cache.put("k3", test_entry("v3"), 300).await; -⋮---- -// moka eviction is async; run pending maintenance -cache.inner.run_pending_tasks().await; -⋮---- -// moka eviction is best-effort and async; just verify it does not -// grow unbounded beyond the configured max_capacity. -assert!(cache.inner.entry_count() <= 3); -⋮---- -async fn per_entry_ttl_shorter_than_default() { -// Default TTL is 10s, but entry requests 1s. -⋮---- -let entry = test_entry_with_ttl("short-lived", 1); -cache.put("test:short", entry, 1).await; -⋮---- -// Present immediately -assert!(cache.get("test:short").await.is_some()); -⋮---- -// Expired after 1.5s (entry TTL = 1s) -⋮---- -assert!( -⋮---- -async fn per_entry_ttl_longer_than_default() { -// Default TTL is 1s, but entry requests 3s. -⋮---- -let entry = test_entry_with_ttl("long-lived", 3); -cache.put("test:long", entry, 3).await; -⋮---- -// Still alive after 1.5s (past the default 1s) -⋮---- -// Expired after 3.5s - - - -//! Redis L2 cache backend. -//! -//! Provides a `RedisCache` that implements `CacheBackend` for use as an -//! L2 cache behind the in-memory moka cache. Feature-gated behind `redis`. -⋮---- -//! Graceful fallback: if Redis is unreachable, operations return None/no-op -//! and log a warning. The in-memory cache still serves as L1. -⋮---- -use redis::aio::ConnectionManager; -⋮---- -use super::CacheEntry; -⋮---- -/// Redis-backed response cache using SETEX for per-entry TTL. -⋮---- -pub struct RedisCache { -⋮---- -/// Key prefix to namespace cache entries. -⋮---- -impl RedisCache { -/// Create a new Redis cache from an existing connection manager. -pub fn new(conn: ConnectionManager) -> Self { -⋮---- -prefix: "anyllm:cache:".to_string(), -⋮---- -/// Connect to Redis and create a cache. -pub async fn connect(redis_url: &str) -> Result { -⋮---- -Ok(Self::new(conn)) -⋮---- -fn redis_key(&self, key: &str) -> String { -format!("{}{}", self.prefix, key) -⋮---- -/// Get a cached entry from Redis. -pub async fn get(&self, key: &str) -> Option { -let redis_key = self.redis_key(key); -let mut conn = self.conn.clone(); -⋮---- -.arg(&redis_key) -.query_async(&mut conn) -⋮---- -.ok() -.map(|v| CacheEntry { -⋮---- -ttl_secs: None, // Redis manages its own TTL via SETEX -⋮---- -/// Store a cache entry in Redis with the given TTL. -pub async fn put(&self, key: &str, entry: &CacheEntry, ttl_secs: u64) { -⋮---- -response_body: String::from_utf8_lossy(&entry.response_body).to_string(), -model: entry.model.clone(), -⋮---- -.arg(ttl_secs) -.arg(&json) -⋮---- -/// Serializable value stored in Redis. -⋮---- -struct RedisCacheValue { - - - -//! Semantic cache backed by Qdrant vector store. -//! -//! Requires `--features qdrant` and `QDRANT_URL` env var. -//! When `QDRANT_URL` is not set, `SemanticCache::new()` returns `None` -//! and the proxy falls back to exact-match caching only. -⋮---- -//! The caller is responsible for generating embeddings (via the backend's -//! embedding endpoint). This module handles only vector store operations. -⋮---- -/// Semantic cache that stores and searches response embeddings in Qdrant. -pub struct SemanticCache { -⋮---- -/// Whether the collection has been verified/created. -⋮---- -impl SemanticCache { -/// Create a new semantic cache connected to Qdrant. -/// -/// Returns `None` if `QDRANT_URL` is not set, enabling graceful -/// degradation: the proxy starts without semantic caching. -pub fn new() -> Option { -let url = std::env::var("QDRANT_URL").ok()?; -⋮---- -std::env::var("QDRANT_COLLECTION").unwrap_or_else(|_| "anyllm_cache".to_string()); -⋮---- -.ok() -.and_then(|v| v.parse().ok()) -.unwrap_or(0.95); -⋮---- -let client = Qdrant::from_url(&url).build().ok()?; -⋮---- -Some(Self { -⋮---- -/// Ensure the Qdrant collection exists with the right vector dimensions. -/// Called lazily on first use to avoid blocking startup. -pub async fn ensure_collection(&self, vector_size: u64) -> Result<(), String> { -if self.collection_ready.load(Ordering::Acquire) { -return Ok(()); -⋮---- -// Check if collection exists -⋮---- -.collection_exists(&self.collection) -⋮---- -.map_err(|e| format!("Qdrant collection_exists check failed: {e}"))?; -⋮---- -// Concurrent callers may both attempt create_collection. -// Treat "already exists" as success to avoid TOCTOU race. -⋮---- -.create_collection( -⋮---- -.vectors_config(VectorParamsBuilder::new(vector_size, Distance::Cosine)), -⋮---- -// If another caller already created it, that's fine. -let msg = e.to_string(); -if !msg.contains("already exists") { -return Err(format!("Qdrant create_collection failed: {e}")); -⋮---- -self.collection_ready.store(true, Ordering::Release); -Ok(()) -⋮---- -/// Search for a semantically similar cached response. -⋮---- -/// Returns the cached response body and model if the top result's similarity -/// score meets or exceeds the configured threshold. -pub async fn search(&self, embedding: &[f32]) -> Option { -use qdrant_client::qdrant::with_payload_selector::SelectorOptions; -⋮---- -.search_points( -SearchPointsBuilder::new(&self.collection, embedding.to_vec(), 1) -.with_payload(SelectorOptions::Enable(true)), -⋮---- -.ok()?; -⋮---- -let point = results.result.first()?; -⋮---- -// Extract string values from Qdrant payload (protobuf Value type). -⋮---- -let response_body = extract_string_value(payload.get("response_body")?)?; -let model = extract_string_value(payload.get("model")?)?; -⋮---- -Some(super::CacheEntry { -⋮---- -ttl_secs: None, // Semantic cache does not use per-entry TTL -⋮---- -/// Store a response with its embedding vector in Qdrant. -pub async fn store(&self, embedding: &[f32], entry: &super::CacheEntry, cache_key: &str) { -⋮---- -uuid::Uuid::new_v4().to_string(), -embedding.to_vec(), -⋮---- -.upsert_points(UpsertPointsBuilder::new(&self.collection, vec![point])) -⋮---- -/// Extract a string from a Qdrant protobuf Value. -fn extract_string_value(value: &qdrant_client::qdrant::Value) -> Option { -use qdrant_client::qdrant::value::Kind; -⋮---- -Some(Kind::StringValue(s)) => Some(s.clone()), -⋮---- -/// Generate an embedding for the given text using the backend's embeddings endpoint. -⋮---- -/// Calls `embeddings_passthrough` on the backend client with an OpenAI-format -/// embedding request. Returns `None` if the backend doesn't support embeddings -/// or if the request fails. -pub async fn embed_text( -⋮---- -let bytes = serde_json::to_vec(&body).ok()?; -⋮---- -.embeddings_passthrough(bytes::Bytes::from(bytes), "application/json") -⋮---- -if !status.is_success() { -⋮---- -// OpenAI embeddings response: { "data": [{ "embedding": [...] }] } -// Deserialize into a minimal struct to avoid cloning the full JSON value. -⋮---- -struct EmbeddingData { -⋮---- -struct EmbeddingResponse { -⋮---- -let resp: EmbeddingResponse = serde_json::from_slice(&resp_body).ok()?; -resp.data.into_iter().next().map(|d| d.embedding) -⋮---- -/// Extract the last user message text from an Anthropic MessageCreateRequest -/// for use as the semantic cache key. -pub fn extract_last_user_text( -⋮---- -for msg in request.messages.iter().rev() { -⋮---- -if !text.is_empty() { -return Some(text.clone()); -⋮---- -text_parts.push(text.as_str()); -⋮---- -if !text_parts.is_empty() { -return Some(text_parts.join(" ")); -⋮---- -mod tests { -⋮---- -fn new_returns_none_without_env() { -// Ensure QDRANT_URL is not set for this test. -if std::env::var("QDRANT_URL").is_ok() { -⋮---- -assert!( -⋮---- -fn extract_last_user_text_finds_text() { -⋮---- -serde_json::from_value(j).unwrap(); -assert_eq!( -⋮---- -fn extract_last_user_text_empty_messages() { -⋮---- -assert_eq!(extract_last_user_text(&request), None); -⋮---- -fn parse_embedding_response() { -⋮---- -resp.get("data") -.unwrap() -.get(0) -⋮---- -.get("embedding") -⋮---- -.clone(), -⋮---- -.unwrap(); -assert_eq!(embedding, vec![0.1, 0.2, 0.3]); - - - -// URL validation for upstream backend targets. -// Security-critical: prevents SSRF via private/loopback/metadata IPs. -⋮---- -use std::net::IpAddr; -use url::Url; -⋮---- -// Re-export is_private_ip from the client crate (canonical location). -pub use anyllm_client::http::is_private_ip; -⋮---- -/// Validate that a base URL is safe to use as an upstream target. -/// Rejects non-http(s) schemes, private/loopback IPs, and link-local addresses. -/// For domain names, also resolves DNS and validates all resolved IPs to prevent -/// DNS rebinding attacks (where a domain initially resolves to a public IP but -/// later changes to a private/metadata IP). -pub fn validate_base_url(raw: &str) -> Result<(), String> { -let parsed = Url::parse(raw).map_err(|e| format!("invalid URL: {e}"))?; -⋮---- -match parsed.scheme() { -⋮---- -other => return Err(format!("scheme '{other}' not allowed, use http or https")), -⋮---- -match parsed.host() { -None => return Err("URL has no host".to_string()), -⋮---- -if is_private_ip(ip) { -return Err(format!("private/loopback IP {ip} not allowed")); -⋮---- -let lower = domain.to_ascii_lowercase(); -⋮---- -|| lower.ends_with(".localhost") -⋮---- -|| lower.ends_with(".internal") -⋮---- -return Err(format!("hostname '{domain}' not allowed")); -⋮---- -// Resolve DNS at startup and validate all resolved IPs. -// This catches domains that currently resolve to private/metadata IPs. -// Note: does not prevent post-startup DNS rebinding; for full protection, -// restrict outbound traffic at the network level. -⋮---- -.port() -.unwrap_or(if parsed.scheme() == "https" { 443 } else { 80 }); -let lookup = format!("{domain}:{port}"); -⋮---- -if is_private_ip(addr.ip()) { -return Err(format!( -⋮---- -// Allow through: the domain may not be resolvable in the -// build/test environment but will work at runtime. The -// runtime SsrfSafeDnsResolver provides connection-time protection. -⋮---- -Ok(()) -⋮---- -mod tests { -⋮---- -fn valid_https_url() { -assert!(validate_base_url("https://api.openai.com").is_ok()); -⋮---- -fn valid_http_url() { -assert!(validate_base_url("http://my-proxy.example.com").is_ok()); -⋮---- -fn rejects_ftp_scheme() { -let err = validate_base_url("ftp://evil.com").unwrap_err(); -assert!(err.contains("scheme")); -⋮---- -fn rejects_localhost() { -let err = validate_base_url("http://localhost:8080").unwrap_err(); -assert!(err.contains("not allowed")); -⋮---- -fn rejects_loopback_ip() { -let err = validate_base_url("http://127.0.0.1:8080").unwrap_err(); -assert!(err.contains("private/loopback")); -⋮---- -fn rejects_private_10_range() { -let err = validate_base_url("http://10.0.0.1").unwrap_err(); -⋮---- -fn rejects_private_172_range() { -let err = validate_base_url("http://172.16.0.1").unwrap_err(); -⋮---- -fn rejects_private_192_range() { -let err = validate_base_url("http://192.168.1.1").unwrap_err(); -⋮---- -fn rejects_cloud_metadata() { -let err = validate_base_url("http://169.254.169.254").unwrap_err(); -⋮---- -fn rejects_metadata_hostname() { -let err = validate_base_url("http://metadata.google.internal").unwrap_err(); -⋮---- -fn rejects_ipv6_loopback() { -let err = validate_base_url("http://[::1]:8080").unwrap_err(); -⋮---- -fn rejects_unspecified() { -let err = validate_base_url("http://0.0.0.0").unwrap_err(); -⋮---- -fn rejects_invalid_url() { -let err = validate_base_url("not a url").unwrap_err(); -assert!(err.contains("invalid URL")); - - - -// Request metrics: count, latency, error rates -⋮---- -use std::sync::Arc; -⋮---- -/// Simple in-memory metrics counters. -/// For production, replace with prometheus or similar. -⋮---- -pub struct Metrics { -⋮---- -struct MetricsInner { -⋮---- -impl Metrics { -/// Create a new zero-valued metrics counter. -pub fn new() -> Self { -⋮---- -// Relaxed ordering: these are independent counters with no cross-counter -// invariants, so no synchronization is needed. Relaxed is fastest. -⋮---- -/// Increment the total request counter. Called once per proxied request. -pub fn record_request(&self) { -self.inner.requests_total.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment the success counter (backend returned 2xx). -pub fn record_success(&self) { -self.inner.requests_success.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment the error counter (backend returned non-2xx or transport failure). -pub fn record_error(&self) { -self.inner.requests_error.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment when an SSE stream begins sending events to the client. -pub fn record_stream_started(&self) { -self.inner.streams_started.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment when an SSE stream completes normally (backend sent all data). -pub fn record_stream_completed(&self) { -self.inner.streams_completed.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment when an SSE stream fails due to an upstream error. -pub fn record_stream_failed(&self) { -self.inner.streams_failed.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment when the downstream client disconnects before the stream finishes. -pub fn record_stream_client_disconnected(&self) { -⋮---- -.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Take a point-in-time snapshot of all counters for the GET /metrics endpoint. -pub fn snapshot(&self) -> MetricsSnapshot { -⋮---- -requests_total: self.inner.requests_total.load(Ordering::Relaxed), -requests_success: self.inner.requests_success.load(Ordering::Relaxed), -requests_error: self.inner.requests_error.load(Ordering::Relaxed), -streams_started: self.inner.streams_started.load(Ordering::Relaxed), -streams_completed: self.inner.streams_completed.load(Ordering::Relaxed), -streams_failed: self.inner.streams_failed.load(Ordering::Relaxed), -⋮---- -.load(Ordering::Relaxed), -⋮---- -/// Point-in-time snapshot of counters, serialized as JSON for GET /metrics. -⋮---- -pub struct MetricsSnapshot { -/// Total proxied requests (success + error + in-flight). -⋮---- -/// Requests where the backend returned a 2xx status. -⋮---- -/// Requests that failed (non-2xx status or transport error). -⋮---- -/// SSE streams that began sending events to the client. -⋮---- -/// SSE streams that completed normally. -⋮---- -/// SSE streams that failed due to upstream errors. -⋮---- -/// SSE streams where the client disconnected early. -⋮---- -impl MetricsSnapshot { -/// Fraction of requests that resulted in errors (0.0 when no requests). -pub fn error_rate(&self) -> f64 { -⋮---- -mod tests { -⋮---- -fn metrics_counting() { -⋮---- -m.record_request(); -⋮---- -m.record_success(); -m.record_error(); -⋮---- -let s = m.snapshot(); -assert_eq!(s.requests_total, 2); -assert_eq!(s.requests_success, 1); -assert_eq!(s.requests_error, 1); -⋮---- -fn streaming_metrics_counting() { -⋮---- -m.record_stream_started(); -⋮---- -m.record_stream_completed(); -m.record_stream_failed(); -m.record_stream_client_disconnected(); -⋮---- -assert_eq!(s.streams_started, 3); -assert_eq!(s.streams_completed, 1); -assert_eq!(s.streams_failed, 1); -assert_eq!(s.streams_client_disconnected, 1); -⋮---- -fn metrics_clone_shares_state() { -⋮---- -let m2 = m.clone(); -⋮---- -m2.record_request(); -assert_eq!(m.snapshot().requests_total, 2); - - - -//! OIDC/JWT authentication support. -//! -//! When `OIDC_ISSUER_URL` is set, the proxy fetches the OpenID Connect -//! discovery document and JWKS at startup. Incoming Bearer tokens that -//! look like JWTs (contain two dots) are validated against the JWKS. -//! On validation failure, auth falls through to static/virtual key checks. -⋮---- -use crate::config::validate_base_url; -⋮---- -/// Claims extracted from a validated JWT. Inserted into request extensions. -⋮---- -pub struct JwtClaims { -⋮---- -/// Catch-all for custom claims. -⋮---- -/// OIDC configuration loaded at startup from the discovery endpoint. -pub struct OidcConfig { -⋮---- -/// JWKS keys indexed by kid. Protected by RwLock for background refresh. -⋮---- -/// Reused for JWKS refresh calls. -⋮---- -struct JwkEntry { -⋮---- -/// OpenID Connect discovery document (only fields we need). -⋮---- -struct OidcDiscovery { -⋮---- -/// JWKS response. -⋮---- -struct JwksResponse { -⋮---- -/// Individual JWK (RSA, EC, and OKP/EdDSA supported). -⋮---- -struct JwkKey { -⋮---- -/// RSA modulus -⋮---- -/// RSA exponent -⋮---- -/// EC curve. Deserialized but unused; avoids serde unknown-field rejection. -⋮---- -/// EC x coordinate -⋮---- -/// EC y coordinate -⋮---- -impl OidcConfig { -/// Discover OIDC configuration from the issuer URL. -/// Fetches `.well-known/openid-configuration` and then the JWKS. -/// Both the issuer URL and the discovered JWKS URI are validated against -/// private/loopback/metadata IP ranges before any network call is made. -pub async fn discover(issuer_url: &str, audience: &str) -> Result { -// Validate issuer URL before making any network call. -validate_base_url(issuer_url) -.map_err(|e| OidcError::Http(format!("OIDC issuer URL rejected (SSRF risk): {e}")))?; -⋮---- -let client = build_http_client(&HttpClientConfig { -⋮---- -connect_timeout: Some(std::time::Duration::from_secs(10)), -⋮---- -let discovery_url = format!( -⋮---- -.get(&discovery_url) -.send() -⋮---- -.map_err(|e| OidcError::Http(format!("OIDC discovery fetch failed: {e}")))? -.json() -⋮---- -.map_err(|e| OidcError::Http(format!("OIDC discovery parse failed: {e}")))?; -⋮---- -// Validate the JWKS URI from the discovery document before fetching it. -// A compromised or MITM'd discovery endpoint could redirect to an internal service. -validate_base_url(&discovery.jwks_uri).map_err(|e| { -OidcError::Http(format!( -⋮---- -audience: audience.to_string(), -⋮---- -config.refresh_jwks().await?; -Ok(config) -⋮---- -/// Re-fetch the JWKS from the provider. Called periodically in the background. -pub async fn refresh_jwks(&self) -> Result<(), OidcError> { -⋮---- -.get(&self.jwks_uri) -⋮---- -.map_err(|e| OidcError::Http(format!("JWKS fetch failed: {e}")))? -⋮---- -.map_err(|e| OidcError::Http(format!("JWKS parse failed: {e}")))?; -⋮---- -entries.push(entry); -⋮---- -if entries.is_empty() { -return Err(OidcError::NoUsableKeys); -⋮---- -let mut guard = self.keys.write().unwrap_or_else(|e| e.into_inner()); -⋮---- -Ok(()) -⋮---- -fn parse_jwk(key: &JwkKey) -> Option { -let kid = key.kid.clone().unwrap_or_default(); -let algorithm = match key.alg.as_deref() { -⋮---- -// Default RSA keys without alg to RS256 (most common). -⋮---- -let decoding_key = match key.kty.as_str() { -⋮---- -let n = key.n.as_ref()?; -let e = key.e.as_ref()?; -DecodingKey::from_rsa_components(n, e).ok()? -⋮---- -let x = key.x.as_ref()?; -let y = key.y.as_ref()?; -DecodingKey::from_ec_components(x, y).ok()? -⋮---- -DecodingKey::from_ed_components(x).ok()? -⋮---- -Some(JwkEntry { -⋮---- -/// Validate a JWT token against the cached JWKS. -/// Returns claims on success, error on failure. -pub fn validate_token(&self, token: &str) -> Result { -⋮---- -decode_header(token).map_err(|e| OidcError::Validation(format!("bad header: {e}")))?; -⋮---- -let keys = self.keys.read().unwrap_or_else(|e| e.into_inner()); -⋮---- -// Find matching key by kid, or try all keys if no kid in header. -⋮---- -keys.iter().filter(|k| k.kid == *kid).collect() -⋮---- -keys.iter().collect() -⋮---- -if candidates.is_empty() { -return Err(OidcError::Validation( -"no matching key found in JWKS".to_string(), -⋮---- -validation.set_issuer(&[&self.issuer]); -validation.set_audience(&[&self.audience]); -// Accept reasonable clock skew (60 seconds). -⋮---- -let mut v = validation.clone(); -v.algorithms = vec![candidate.algorithm]; -⋮---- -Ok(data) => return Ok(data.claims), -⋮---- -Err(OidcError::Validation( -"token validation failed against all matching keys".to_string(), -⋮---- -/// Check if a credential looks like a JWT (three base64url segments separated by dots). -pub fn looks_like_jwt(credential: &str) -> bool { -credential.matches('.').count() == 2 && credential.len() > 32 -⋮---- -pub enum OidcError { -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -Self::Http(msg) => write!(f, "OIDC HTTP error: {msg}"), -Self::NoUsableKeys => write!(f, "OIDC: no usable keys in JWKS"), -Self::Validation(msg) => write!(f, "JWT validation failed: {msg}"), -⋮---- -mod tests { -⋮---- -fn oidc_url_rejects_private_ips() { -// Cloud metadata endpoint. -assert!(validate_base_url("http://169.254.169.254/oidc").is_err()); -// Loopback. -assert!(validate_base_url("http://127.0.0.1/oidc").is_err()); -// RFC 1918 private range. -assert!(validate_base_url("http://10.0.0.1/oidc").is_err()); -assert!(validate_base_url("http://192.168.1.1/oidc").is_err()); -⋮---- -fn oidc_url_accepts_public_https() { -assert!(validate_base_url("https://accounts.google.com").is_ok()); -assert!(validate_base_url("https://login.microsoftonline.com/tenant/v2.0").is_ok()); -⋮---- -fn looks_like_jwt_detects_jwt_shape() { -// Typical JWT: header.payload.signature -⋮---- -assert!(looks_like_jwt(jwt)); -⋮---- -fn looks_like_jwt_rejects_api_keys() { -assert!(!looks_like_jwt("sk-1234567890abcdef")); -assert!(!looks_like_jwt("sk-vk-abcdef1234567890abcdef")); -assert!(!looks_like_jwt("")); // empty -assert!(!looks_like_jwt("a.b")); // only one dot -⋮---- -fn looks_like_jwt_rejects_short_dot_strings() { -// Two dots but too short to be a real JWT. -assert!(!looks_like_jwt("a.b.c")); -⋮---- -fn parse_rsa_jwk() { -⋮---- -kid: Some("test-kid".to_string()), -kty: "RSA".to_string(), -alg: Some("RS256".to_string()), -// Valid base64url-encoded RSA components (minimal test values). -n: Some("0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw".to_string()), -e: Some("AQAB".to_string()), -⋮---- -assert!(entry.is_some()); -let entry = entry.unwrap(); -assert_eq!(entry.kid, "test-kid"); -assert!(matches!(entry.algorithm, Algorithm::RS256)); -⋮---- -fn parse_unknown_kty_returns_none() { -⋮---- -kid: Some("test".to_string()), -kty: "oct".to_string(), // Symmetric keys, genuinely unsupported -alg: Some("HS256".to_string()), -⋮---- -assert!(OidcConfig::parse_jwk(&key).is_none()); -⋮---- -fn parse_eddsa_jwk() { -⋮---- -kid: Some("ed-key".to_string()), -kty: "OKP".to_string(), -alg: Some("EdDSA".to_string()), -⋮---- -crv: Some("Ed25519".to_string()), -x: Some("11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHURo".to_string()), -⋮---- -assert!(entry.is_some(), "EdDSA/OKP keys should be supported"); -⋮---- -assert_eq!(entry.kid, "ed-key"); -assert!(matches!(entry.algorithm, Algorithm::EdDSA)); - - - -//! Request policy enforcement. -//! -//! Enforces per-key restrictions (e.g., model allowlists) before the request -//! reaches the backend. Policies are optional: absent means "allow all". -⋮---- -/// Check if a model name is allowed by the key's policy. -/// Returns true if no policy is set (all models allowed). -/// -/// Supported patterns: -/// - `"*"` -- allows any model -/// - `"claude-*"` -- prefix wildcard (matches `claude-3-opus`, `claude-sonnet-4-6`, etc.) -/// - `"gpt-4o"` -- exact match -pub fn is_model_allowed(model: &str, allowed_models: &Option>) -> bool { -⋮---- -if let Some(prefix) = pattern.strip_suffix('*') { -if model.starts_with(prefix) { -⋮---- -mod tests { -⋮---- -fn model_allowed_when_no_policy() { -assert!(is_model_allowed("anything", &None)); -⋮---- -fn model_allowed_exact_match() { -let policy = Some(vec!["gpt-4o".to_string(), "gpt-4o-mini".to_string()]); -assert!(is_model_allowed("gpt-4o", &policy)); -assert!(is_model_allowed("gpt-4o-mini", &policy)); -assert!(!is_model_allowed("gpt-4", &policy)); -⋮---- -fn model_allowed_wildcard() { -let policy = Some(vec!["claude-*".to_string()]); -assert!(is_model_allowed("claude-sonnet-4-6", &policy)); -assert!(is_model_allowed("claude-3-opus", &policy)); -assert!(!is_model_allowed("gpt-4o", &policy)); -⋮---- -fn model_allowed_star_allows_all() { -let policy = Some(vec!["*".to_string()]); -assert!(is_model_allowed("literally-anything", &policy)); -⋮---- -fn model_denied_when_not_in_list() { -let policy = Some(vec!["gpt-4o".to_string()]); -assert!(!is_model_allowed("gpt-4o-mini", &policy)); -assert!(!is_model_allowed("claude-sonnet-4-6", &policy)); -⋮---- -fn model_empty_allowlist_denies_all() { -let policy = Some(vec![]); -⋮---- -fn model_multiple_patterns() { -let policy = Some(vec!["gpt-4o".to_string(), "claude-*".to_string()]); - - - -// Phase 9-10: compatibility endpoint and hardening integration tests -// Phase 19: token counting integration tests -⋮---- -use anyllm_proxy::server::routes; -use reqwest::Client; -⋮---- -fn test_config() -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: "https://api.openai.com".to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: config::BackendAuth::BearerToken("test-key".into()), -⋮---- -async fn spawn_test_server() -> String { -// Enable open-relay mode for tests (no PROXY_API_KEYS configured). -⋮---- -let app = routes::app(test_config()); -let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -async fn models_endpoint() { -let base = spawn_test_server().await; -⋮---- -.get(format!("{base}/v1/models")) -.header("x-api-key", "test") -.send() -⋮---- -.unwrap(); -assert_eq!(resp.status(), 200); -let body: serde_json::Value = resp.json().await.unwrap(); -assert!(body["data"].is_array()); -assert!(!body["data"].as_array().unwrap().is_empty()); -⋮---- -async fn count_tokens_returns_count() { -⋮---- -.post(format!("{base}/v1/messages/count_tokens")) -⋮---- -.header("content-type", "application/json") -.body(r#"{"model":"claude-sonnet-4-6","max_tokens":1024,"messages":[{"role":"user","content":"Hello, world!"}]}"#) -⋮---- -let tokens = body["input_tokens"].as_u64().unwrap(); -assert!(tokens > 0, "expected positive token count, got {tokens}"); -⋮---- -async fn count_tokens_with_empty_messages() { -⋮---- -.body(r#"{"model":"claude-sonnet-4-6","max_tokens":1024,"messages":[]}"#) -⋮---- -assert!(body["input_tokens"].is_u64()); -⋮---- -async fn count_tokens_with_tools() { -⋮---- -.body(r#"{"model":"claude-sonnet-4-6","max_tokens":1024,"messages":[{"role":"user","content":"Use the tool"}],"tools":[{"name":"get_weather","description":"Get current weather","input_schema":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}]}"#) -⋮---- -// Tool definitions add tokens beyond just the message text -assert!( -⋮---- -async fn count_tokens_invalid_body() { -⋮---- -.body("{}") -⋮---- -// Missing required fields -> 400 invalid_request_error (Anthropic error shape) -assert_eq!(resp.status(), 400); -⋮---- -assert_eq!(body["type"], "error"); -assert_eq!(body["error"]["type"], "invalid_request_error"); -⋮---- -async fn batches_returns_unsupported() { -⋮---- -.post(format!("{base}/v1/messages/batches")) -⋮---- -async fn auth_required_for_api_routes() { -⋮---- -// No auth header -⋮---- -assert_eq!(resp.status(), 401); -⋮---- -async fn health_no_auth_required() { -⋮---- -// No auth header - should still work -let resp = client.get(format!("{base}/health")).send().await.unwrap(); -⋮---- -async fn metrics_endpoint_requires_auth() { -⋮---- -// No auth header -- should be rejected -let resp = client.get(format!("{base}/metrics")).send().await.unwrap(); -⋮---- -async fn metrics_endpoint_returns_counters() { -⋮---- -// Auth required for metrics -⋮---- -.get(format!("{base}/metrics")) -⋮---- -// Multi-backend metrics format: { "backends": {...}, "total": {...} } -assert_eq!(body["total"]["requests_total"], 0); -assert_eq!(body["total"]["requests_success"], 0); -assert_eq!(body["total"]["requests_error"], 0); -assert!(body["backends"].is_object()); -⋮---- -async fn unknown_route_returns_anthropic_not_found() { -⋮---- -.get(format!("{base}/v1/nonexistent")) -⋮---- -assert_eq!(resp.status(), 404); -⋮---- -assert_eq!(body["error"]["type"], "not_found_error"); -assert_eq!(body["error"]["message"], "Not found"); -⋮---- -async fn malformed_json_returns_anthropic_error() { -⋮---- -.post(format!("{base}/v1/messages")) -⋮---- -.body("not valid json") -⋮---- -// SSRF protection: validate_base_url rejects private/loopback targets -⋮---- -fn ssrf_blocks_loopback() { -assert!(config::validate_base_url("http://127.0.0.1:8080").is_err()); -⋮---- -fn ssrf_blocks_private_network() { -assert!(config::validate_base_url("http://10.0.0.1").is_err()); -assert!(config::validate_base_url("http://172.16.0.1").is_err()); -assert!(config::validate_base_url("http://192.168.1.1").is_err()); -⋮---- -fn ssrf_blocks_localhost() { -assert!(config::validate_base_url("http://localhost").is_err()); -⋮---- -fn ssrf_blocks_cloud_metadata() { -assert!(config::validate_base_url("http://169.254.169.254").is_err()); -assert!(config::validate_base_url("http://metadata.google.internal").is_err()); -⋮---- -fn ssrf_allows_public_url() { -assert!(config::validate_base_url("https://api.openai.com").is_ok()); - - - -// Integration tests for multi-backend path-prefix routing. -⋮---- -use anyllm_proxy::config::MultiConfig; -use anyllm_proxy::server::routes; -use reqwest::Client; -⋮---- -fn test_multi_config() -> MultiConfig { -// Two backends: openai (default) and gemini -⋮---- -async fn spawn_multi_server() -> String { -// Enable open-relay mode for tests (no PROXY_API_KEYS configured). -⋮---- -let app = routes::app_multi(test_multi_config()); -let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -async fn health_works_with_multi_backend() { -let base = spawn_multi_server().await; -⋮---- -let resp = client.get(format!("{base}/health")).send().await.unwrap(); -assert_eq!(resp.status(), 200); -⋮---- -async fn default_route_requires_auth() { -⋮---- -// /v1/messages without auth should return 401 -⋮---- -.post(format!("{base}/v1/messages")) -.header("content-type", "application/json") -.body(r#"{"model":"claude-sonnet-4-6","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#) -.send() -⋮---- -.unwrap(); -assert_eq!(resp.status(), 401); -⋮---- -async fn prefixed_route_requires_auth() { -⋮---- -// /openai/v1/messages without auth should return 401 -⋮---- -.post(format!("{base}/openai/v1/messages")) -⋮---- -async fn prefixed_models_endpoint_works() { -⋮---- -// /openai/v1/models should work with auth -⋮---- -.get(format!("{base}/openai/v1/models")) -.header("x-api-key", "any-key") -⋮---- -let body: serde_json::Value = resp.json().await.unwrap(); -assert!(body["data"].is_array()); -⋮---- -async fn gemini_prefixed_models_endpoint_works() { -⋮---- -.get(format!("{base}/gemini/v1/models")) -⋮---- -async fn unknown_prefix_returns_404() { -⋮---- -.get(format!("{base}/unknown/v1/models")) -⋮---- -assert_eq!(resp.status(), 404); -⋮---- -async fn metrics_shows_per_backend_breakdown() { -⋮---- -.get(format!("{base}/metrics")) -⋮---- -// Should have per-backend metrics -assert!(body["backends"]["openai"].is_object()); -assert!(body["backends"]["gemini"].is_object()); -// And totals -assert_eq!(body["total"]["requests_total"], 0); -⋮---- -async fn default_route_count_tokens_works() { -⋮---- -.post(format!("{base}/v1/messages/count_tokens")) -⋮---- -.body(r#"{"model":"claude-sonnet-4-6","max_tokens":100,"messages":[{"role":"user","content":"hello world"}]}"#) -⋮---- -assert!(body["input_tokens"].as_u64().unwrap() > 0); -⋮---- -async fn prefixed_count_tokens_works() { -⋮---- -.post(format!("{base}/openai/v1/messages/count_tokens")) - - - -// crates/translator/src/anthropic/batch.rs -// Anthropic Message Batches API types. -// See https://docs.anthropic.com/en/api/creating-message-batches -⋮---- -use crate::anthropic::errors::ErrorDetail; -⋮---- -/// One entry in a POST /v1/messages/batches request body. -⋮---- -pub struct BatchRequestItem { -⋮---- -/// POST /v1/messages/batches request body. -⋮---- -pub struct CreateBatchRequest { -⋮---- -/// Processing lifecycle of a message batch. -⋮---- -pub enum ProcessingStatus { -⋮---- -/// Per-status request counts within a batch. -⋮---- -pub struct BatchRequestCounts { -⋮---- -/// Message batch object returned by the API. -⋮---- -pub struct MessageBatch { -⋮---- -/// Result type for a single batch request. -⋮---- -pub enum BatchResultVariant { -⋮---- -/// One line in the results JSONL file. -⋮---- -pub struct BatchResultItem { -⋮---- -mod tests { -⋮---- -fn deserialize_create_batch_request() { -⋮---- -let req: CreateBatchRequest = serde_json::from_value(json).unwrap(); -assert_eq!(req.requests.len(), 1); -assert_eq!(req.requests[0].custom_id, "req-1"); -assert_eq!(req.requests[0].params.model, "claude-3-5-sonnet-20241022"); -⋮---- -fn serialize_message_batch_in_progress() { -⋮---- -id: "msgbatch_abc".to_string(), -type_: "message_batch".to_string(), -⋮---- -let v = serde_json::to_value(&batch).unwrap(); -assert_eq!(v["id"], "msgbatch_abc"); -assert_eq!(v["processing_status"], "in_progress"); -assert_eq!(v["request_counts"]["processing"], 2); -⋮---- -fn serialize_batch_result_succeeded() { -// BatchResultVariant::Succeeded wraps a MessageResponse. -// The "type" discriminant should serialize to "succeeded". -let v = serde_json::to_value(BatchResultVariant::Canceled).unwrap(); -assert_eq!(v["type"], "canceled"); - - - -// Anthropic Messages API request/response types -⋮---- -// --- Request types --- -⋮---- -/// Anthropic Messages API request body. -/// -/// See -⋮---- -pub struct MessageCreateRequest { -⋮---- -/// Forward-compatible extension: captures unknown Anthropic fields via -/// serde flatten so newer API versions work without code changes. -⋮---- -/// System prompt: plain string or array of text blocks (with optional cache_control). -⋮---- -pub enum System { -⋮---- -/// System prompt text block with optional cache control. -⋮---- -pub struct SystemBlock { -⋮---- -pub block_type: String, // always "text" -⋮---- -/// Cache control directive for prompt caching. -⋮---- -pub struct CacheControl { -⋮---- -/// A single message in the conversation (user or assistant). -⋮---- -pub struct InputMessage { -⋮---- -/// Message role: user or assistant. -⋮---- -pub enum Role { -⋮---- -/// Message content: plain string or array of typed content blocks. -⋮---- -pub enum Content { -⋮---- -/// Typed content block within a message. -⋮---- -pub enum ContentBlock { -⋮---- -/// Redacted thinking block: encrypted content returned when safety systems -/// flag extended thinking. Must be passed back to the API for continuity. -⋮---- -/// Tool result content: plain string or array of content blocks. -⋮---- -pub enum ToolResultContent { -⋮---- -/// Image source for image content blocks (base64 or URL). -⋮---- -pub struct ImageSource { -⋮---- -pub source_type: String, // "base64" or "url" -⋮---- -/// Document source for PDF content blocks (base64). -⋮---- -pub struct DocumentSource { -⋮---- -pub source_type: String, // "base64" -pub media_type: String, // "application/pdf" -pub data: String, // base64-encoded data -⋮---- -// --- Tool types --- -⋮---- -/// Tool definition with name, description, and JSON schema. -⋮---- -pub struct Tool { -⋮---- -/// How the model should use tools: auto, any, none, or specific tool. -⋮---- -pub enum ToolChoice { -⋮---- -/// Disable parallel tool use. Maps to OpenAI `parallel_tool_calls: false`. -⋮---- -/// Request metadata for abuse detection. -⋮---- -pub struct Metadata { -⋮---- -/// Extended thinking configuration. -⋮---- -pub enum ThinkingConfig { -⋮---- -// --- Response types --- -⋮---- -/// Anthropic Messages API response body. -⋮---- -pub struct MessageResponse { -⋮---- -pub response_type: String, // always "message" -⋮---- -/// Why the model stopped generating. -⋮---- -pub enum StopReason { -⋮---- -/// Token usage counts for the request and response. -⋮---- -pub struct Usage { -⋮---- -mod tests { -⋮---- -use pretty_assertions::assert_eq; -use serde_json::json; -⋮---- -fn deserialize_basic_text_request() { -let j = json!({ -⋮---- -let req: MessageCreateRequest = serde_json::from_value(j).unwrap(); -assert_eq!(req.model, "claude-3-5-sonnet-20241022"); -assert_eq!(req.max_tokens, 1024); -assert_eq!(req.messages.len(), 1); -assert_eq!(req.messages[0].role, Role::User); -⋮---- -Content::Text(s) => assert_eq!(s, "Hello, world"), -_ => panic!("expected Content::Text"), -⋮---- -fn deserialize_system_as_string() { -⋮---- -match req.system.unwrap() { -System::Text(s) => assert_eq!(s, "You are a helpful assistant."), -_ => panic!("expected System::Text"), -⋮---- -fn deserialize_system_as_blocks() { -⋮---- -assert_eq!(blocks.len(), 2); -assert_eq!(blocks[0].text, "Be concise."); -assert!(blocks[0].cache_control.is_none()); -assert_eq!(blocks[1].text, "Respond in JSON."); -assert_eq!( -⋮---- -_ => panic!("expected System::Blocks"), -⋮---- -fn deserialize_tools_and_tool_choice() { -⋮---- -let tools = req.tools.unwrap(); -assert_eq!(tools.len(), 1); -assert_eq!(tools[0].name, "get_weather"); -assert!(tools[0].description.is_some()); -match req.tool_choice.unwrap() { -⋮---- -other => panic!("expected ToolChoice::Auto, got {:?}", other), -⋮---- -fn deserialize_tool_choice_specific_tool() { -let j = json!({"type": "tool", "name": "get_weather"}); -let tc: ToolChoice = serde_json::from_value(j).unwrap(); -⋮---- -ToolChoice::Tool { name } => assert_eq!(name, "get_weather"), -other => panic!("expected ToolChoice::Tool, got {:?}", other), -⋮---- -fn content_as_string_vs_blocks() { -// String form -let j = json!("just a string"); -let c: Content = serde_json::from_value(j).unwrap(); -⋮---- -Content::Text(s) => assert_eq!(s, "just a string"), -⋮---- -// Blocks form -let j = json!([{"type": "text", "text": "hello"}]); -⋮---- -assert_eq!(blocks.len(), 1); -⋮---- -ContentBlock::Text { text } => assert_eq!(text, "hello"), -_ => panic!("expected ContentBlock::Text"), -⋮---- -_ => panic!("expected Content::Blocks"), -⋮---- -fn deserialize_tool_use_block() { -⋮---- -let block: ContentBlock = serde_json::from_value(j).unwrap(); -⋮---- -assert_eq!(id, "toolu_01A09q90qw90lq917835lqs136"); -assert_eq!(name, "get_weather"); -assert_eq!(input["location"], "San Francisco, CA"); -⋮---- -_ => panic!("expected ContentBlock::ToolUse"), -⋮---- -fn deserialize_tool_result_block() { -⋮---- -assert_eq!(tool_use_id, "toolu_01A09q90qw90lq917835lqs136"); -match content.unwrap() { -ToolResultContent::Text(s) => assert_eq!(s, "72°F, sunny"), -_ => panic!("expected ToolResultContent::Text"), -⋮---- -assert!(is_error.is_none()); -⋮---- -_ => panic!("expected ContentBlock::ToolResult"), -⋮---- -fn deserialize_tool_result_error() { -⋮---- -assert_eq!(is_error, Some(true)); -⋮---- -fn message_response_round_trip() { -⋮---- -id: "msg_01XFDUDYJgAACzvnptvVoYEL".into(), -response_type: "message".into(), -⋮---- -content: vec![ContentBlock::Text { -⋮---- -model: "claude-3-5-sonnet-20241022".into(), -stop_reason: Some(StopReason::EndTurn), -⋮---- -let serialized = serde_json::to_string(&resp).unwrap(); -let deserialized: MessageResponse = serde_json::from_str(&serialized).unwrap(); -assert_eq!(deserialized.id, resp.id); -assert_eq!(deserialized.stop_reason, Some(StopReason::EndTurn)); -assert_eq!(deserialized.usage.input_tokens, 10); -assert_eq!(deserialized.usage.output_tokens, 5); -⋮---- -fn reject_missing_max_tokens() { -⋮---- -assert!(result.is_err(), "should fail without max_tokens"); -⋮---- -fn extra_fields_captured() { -⋮---- -assert_eq!(req.top_k, Some(40)); -assert!(req.extra.get("top_k").is_none()); -assert_eq!(req.extra.get("unknown_field").unwrap(), &json!("value")); -⋮---- -fn stop_reason_variants() { -⋮---- -fn usage_optional_cache_fields_omitted() { -⋮---- -let j = serde_json::to_value(&usage).unwrap(); -assert!(!j -⋮---- -fn thinking_config_enabled_roundtrip() { -⋮---- -let json = serde_json::to_value(&cfg).unwrap(); -assert_eq!(json, json!({"type": "enabled", "budget_tokens": 8192})); -let parsed: ThinkingConfig = serde_json::from_value(json).unwrap(); -assert!(matches!( -⋮---- -fn thinking_config_disabled_roundtrip() { -⋮---- -assert_eq!(json, json!({"type": "disabled"})); -⋮---- -assert!(matches!(parsed, ThinkingConfig::Disabled)); -⋮---- -fn thinking_content_block_roundtrip() { -⋮---- -thinking: "Let me reason about this...".into(), -signature: Some("sig_abc".into()), -⋮---- -let json = serde_json::to_value(&block).unwrap(); -assert_eq!(json["type"], "thinking"); -assert_eq!(json["thinking"], "Let me reason about this..."); -assert_eq!(json["signature"], "sig_abc"); -let parsed: ContentBlock = serde_json::from_value(json).unwrap(); -assert!(matches!(parsed, ContentBlock::Thinking { .. })); -⋮---- -fn request_with_thinking_deserializes() { -⋮---- -fn deserialize_redacted_thinking_block() { -⋮---- -assert_eq!(data, "EqQBCgIYAhIM1gbcDa9GJwZA2b3h"); -⋮---- -_ => panic!("expected ContentBlock::RedactedThinking"), -⋮---- -fn redacted_thinking_round_trip() { -⋮---- -data: "encrypted_data_here".into(), -⋮---- -let serialized = serde_json::to_string(&block).unwrap(); -assert!(serialized.contains("\"redacted_thinking\"")); -let deserialized: ContentBlock = serde_json::from_str(&serialized).unwrap(); -⋮---- -assert_eq!(data, "encrypted_data_here"); - - - -/// Anthropic Message Batches API request/response types. -pub mod batch; -/// Anthropic error response types (`ErrorResponse`, `ErrorType`, `ErrorDetail`). -pub mod errors; -/// Anthropic Messages API request and response types. -pub mod messages; -/// Anthropic SSE streaming event types (`StreamEvent`, `Delta`). -pub mod streaming; -⋮---- -// Re-export primary types - - - -// Gemini generateContent API request types. -// -// Pure data types with serde, no IO. Field names use camelCase to match -// the Gemini REST API (https://ai.google.dev/api/generate-content). -⋮---- -// --- Request types --- -⋮---- -/// POST body for `/models/{model}:generateContent` and `:streamGenerateContent`. -⋮---- -pub struct GenerateContentRequest { -⋮---- -/// A conversation turn: role + parts. -/// -/// `role` is optional because `systemInstruction` omits it. -⋮---- -pub struct Content { -⋮---- -/// A single content part. Gemini discriminates by field presence, so we use -/// a struct with optional fields rather than an enum. -⋮---- -pub struct Part { -⋮---- -/// True for thought parts produced by thinking models (e.g., Gemini 2.5 Pro/Flash). -/// Never set in requests; only appears in model responses. -⋮---- -impl Part { -pub fn text(t: impl Into) -> Self { -⋮---- -text: Some(t.into()), -⋮---- -pub fn function_call(name: impl Into, args: serde_json::Value) -> Self { -⋮---- -function_call: Some(FunctionCallData { -name: name.into(), -⋮---- -pub fn function_response(name: impl Into, response: serde_json::Value) -> Self { -⋮---- -function_response: Some(FunctionResponseData { -⋮---- -pub fn inline_data(mime_type: impl Into, data: impl Into) -> Self { -⋮---- -inline_data: Some(InlineData { -mime_type: mime_type.into(), -data: data.into(), -⋮---- -/// Base64-encoded inline binary data. -⋮---- -pub struct InlineData { -⋮---- -/// Base64-encoded bytes. -⋮---- -/// Reference to a file stored via the Gemini File API. -⋮---- -pub struct FileData { -⋮---- -/// A function call emitted by the model. -⋮---- -pub struct FunctionCallData { -⋮---- -/// JSON object (not a string like OpenAI). -⋮---- -/// The result of executing a function, sent back to the model. -⋮---- -pub struct FunctionResponseData { -⋮---- -/// Sampling and output configuration. -⋮---- -pub struct GenerationConfig { -⋮---- -/// Configures extended thinking for supported models (e.g., Gemini 2.5 Pro/Flash). -⋮---- -/// Configures extended thinking for Gemini 2.5 thinking models. -⋮---- -pub struct ThinkingConfig { -/// Token budget for the thinking phase. Maps from Anthropic `budget_tokens`. -⋮---- -/// Whether to include thought parts in the response. Set to `true` to -/// surface thinking content as `ContentBlock::Thinking` in the translation. -⋮---- -/// Wrapper for function declarations provided to the model. -⋮---- -pub struct Tool { -⋮---- -/// A single function the model may call. -⋮---- -pub struct FunctionDeclaration { -⋮---- -/// Controls how and whether the model calls functions. -⋮---- -pub struct ToolConfig { -⋮---- -/// Function calling mode: AUTO, NONE, or ANY. -⋮---- -pub struct FunctionCallingConfig { -⋮---- -/// Per-category safety threshold. -⋮---- -pub struct SafetySetting { -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn serialize_basic_request() { -⋮---- -contents: vec![Content { -⋮---- -let j = serde_json::to_value(&req).unwrap(); -assert_eq!(j["contents"][0]["role"], "user"); -assert_eq!(j["contents"][0]["parts"][0]["text"], "Hello"); -// Optional fields must be absent -assert!(j.get("systemInstruction").is_none()); -assert!(j.get("tools").is_none()); -⋮---- -fn serialize_with_system_instruction() { -⋮---- -contents: vec![], -system_instruction: Some(Content { -⋮---- -parts: vec![Part::text("You are helpful.")], -⋮---- -assert_eq!(j["systemInstruction"]["parts"][0]["text"], "You are helpful."); -// systemInstruction should not have a role field -assert!(j["systemInstruction"].get("role").is_none()); -⋮---- -fn serialize_with_tools() { -⋮---- -tools: Some(vec![Tool { -⋮---- -assert_eq!(j["tools"][0]["functionDeclarations"][0]["name"], "get_weather"); -⋮---- -fn serialize_with_generation_config() { -⋮---- -generation_config: Some(GenerationConfig { -temperature: Some(0.7), -max_output_tokens: Some(1024), -top_k: Some(40), -⋮---- -let temp = gc["temperature"].as_f64().unwrap(); -assert!((temp - 0.7).abs() < 0.001, "temperature was {temp}"); -assert_eq!(gc["maxOutputTokens"], 1024); -assert_eq!(gc["topK"], 40); -// Unset fields must be absent -assert!(gc.get("topP").is_none()); -assert!(gc.get("seed").is_none()); -⋮---- -fn part_text_constructor() { -⋮---- -assert_eq!(p.text.as_deref(), Some("hello")); -assert!(p.inline_data.is_none()); -assert!(p.function_call.is_none()); -⋮---- -fn part_function_call_constructor() { -let p = Part::function_call("calc", json!({"expr": "1+1"})); -let fc = p.function_call.unwrap(); -assert_eq!(fc.name, "calc"); -assert_eq!(fc.args, json!({"expr": "1+1"})); -assert!(p.text.is_none()); -⋮---- -fn part_function_response_constructor() { -let p = Part::function_response("calc", json!({"result": 2})); -let fr = p.function_response.unwrap(); -assert_eq!(fr.name, "calc"); -assert_eq!(fr.response, json!({"result": 2})); -⋮---- -fn part_inline_data_constructor() { -⋮---- -let id = p.inline_data.unwrap(); -assert_eq!(id.mime_type, "image/png"); -assert_eq!(id.data, "iVBOR..."); -⋮---- -fn round_trip_request() { -⋮---- -temperature: Some(0.5), -⋮---- -let json_str = serde_json::to_string(&req).unwrap(); -let back: GenerateContentRequest = serde_json::from_str(&json_str).unwrap(); -assert_eq!(back.contents.len(), 1); -assert_eq!( -⋮---- -fn generation_config_camel_case() { -⋮---- -max_output_tokens: Some(100), -stop_sequences: Some(vec!["END".into()]), -response_mime_type: Some("application/json".into()), -⋮---- -let j = serde_json::to_value(&gc).unwrap(); -assert!(j.get("maxOutputTokens").is_some()); -assert!(j.get("stopSequences").is_some()); -assert!(j.get("responseMimeType").is_some()); -// snake_case keys must not appear -assert!(j.get("max_output_tokens").is_none()); -⋮---- -fn tool_config_serializes_correctly() { -⋮---- -mode: "ANY".into(), -⋮---- -let j = serde_json::to_value(&tc).unwrap(); -assert_eq!(j["functionCallingConfig"]["mode"], "ANY"); -⋮---- -fn empty_optional_fields_omitted() { -⋮---- -let obj = j.as_object().unwrap(); -assert_eq!(obj.len(), 1); // only "contents" -⋮---- -fn content_with_user_and_model_roles() { -⋮---- -role: Some("user".into()), -parts: vec![Part::text("question")], -⋮---- -role: Some("model".into()), -parts: vec![Part::text("answer")], -⋮---- -let j_user = serde_json::to_value(&user).unwrap(); -let j_model = serde_json::to_value(&model).unwrap(); -assert_eq!(j_user["role"], "user"); -assert_eq!(j_model["role"], "model"); -⋮---- -fn function_call_args_is_json_object() { -⋮---- -name: "f".into(), -args: json!({"a": 1}), -⋮---- -assert!(fc.args.is_object()); -let j = serde_json::to_value(&fc).unwrap(); -assert!(j["args"].is_object()); -⋮---- -fn safety_setting_serializes() { -⋮---- -category: "HARM_CATEGORY_DANGEROUS_CONTENT".into(), -threshold: "BLOCK_ONLY_HIGH".into(), -⋮---- -let j = serde_json::to_value(&ss).unwrap(); -assert_eq!(j["category"], "HARM_CATEGORY_DANGEROUS_CONTENT"); -assert_eq!(j["threshold"], "BLOCK_ONLY_HIGH"); - - - -// Anthropic Messages API <-> Gemini generateContent API message mapping. -// -// Pure translation functions, no IO. Converts Anthropic request types into -// Gemini request types and Gemini response types back into Anthropic responses. -⋮---- -use std::collections::HashMap; -⋮---- -use crate::mapping::tools_map::sanitize_schema_for_gemini; -⋮---- -// --------------------------------------------------------------------------- -// Request direction: Anthropic -> Gemini -⋮---- -/// Convert an Anthropic `MessageCreateRequest` into a Gemini `GenerateContentRequest`. -/// -/// Maps `thinking_config` to `generationConfig.thinkingConfig` for Gemini 2.5 -/// thinking models. Drops unsupported features (thinking content blocks in prior -/// messages, document blocks, cache_control) and merges consecutive same-role -/// messages to satisfy Gemini's strict alternation requirement. -pub fn anthropic_to_gemini_request( -⋮---- -let tool_id_map = build_tool_id_map(&req.messages); -⋮---- -// System instruction -let system_instruction = req.system.as_ref().map(|sys| { -⋮---- -anthropic::System::Text(s) => s.clone(), -⋮---- -.iter() -.map(|b| b.text.as_str()) -⋮---- -.join("\n"), -⋮---- -parts: vec![gemini::Part::text(text)], -⋮---- -// Convert messages -⋮---- -let parts = content_blocks_to_parts(&msg.content, &tool_id_map); -if !parts.is_empty() { -contents.push(gemini::Content { -role: Some(role.to_string()), -⋮---- -contents = merge_consecutive_roles(contents); -⋮---- -// Tools -let tools = req.tools.as_ref().map(|tools| { -vec![gemini::Tool { -⋮---- -// Tool config -let tool_config = req.tool_choice.as_ref().map(|tc| { -⋮---- -// Gemini has no forced-specific-tool mode; fall back to AUTO. -⋮---- -mode: mode.to_string(), -⋮---- -// Generation config -⋮---- -Some(gemini::ThinkingConfig { -⋮---- -include_thoughts: Some(true), -⋮---- -max_output_tokens: Some(req.max_tokens), -⋮---- -stop_sequences: req.stop_sequences.clone(), -⋮---- -Some(gc) -⋮---- -/// Build a map from Anthropic tool_use IDs to tool names. -⋮---- -/// Scans all messages for `ToolUse` blocks so that `ToolResult` translation can -/// look up the function name Gemini expects. -pub fn build_tool_id_map(messages: &[anthropic::InputMessage]) -> HashMap { -⋮---- -map.insert(id.clone(), name.clone()); -⋮---- -/// Merge consecutive same-role `Content` entries by concatenating their parts. -⋮---- -/// Gemini requires strict user/model role alternation. When the Anthropic -/// conversation has two consecutive user (or model) turns, this merges them -/// into a single turn. -pub fn merge_consecutive_roles(contents: Vec) -> Vec { -let mut merged: Vec = Vec::with_capacity(contents.len()); -⋮---- -if let Some(last) = merged.last_mut() { -⋮---- -last.parts.extend(c.parts); -⋮---- -merged.push(c); -⋮---- -/// Convert Anthropic message content into a vec of Gemini Parts. -fn content_blocks_to_parts( -⋮---- -anthropic::Content::Text(s) => vec![gemini::Part::text(s.clone())], -⋮---- -.filter_map(|block| content_block_to_part(block, tool_id_map)) -.collect(), -⋮---- -/// Convert a single Anthropic ContentBlock to a Gemini Part, or None if dropped. -fn content_block_to_part( -⋮---- -anthropic::ContentBlock::Text { text } => Some(gemini::Part::text(text.clone())), -⋮---- -// Gemini only supports inline base64 data, not URLs. -⋮---- -let mime = source.media_type.clone().unwrap_or_else(|| "image/png".into()); -let data = source.data.clone().unwrap_or_default(); -Some(gemini::Part::inline_data(mime, data)) -⋮---- -// URL-type images cannot be sent as inline_data; drop. -⋮---- -// Strip the Anthropic tool_use id; Gemini uses name-based correlation. -Some(gemini::Part::function_call(name.clone(), input.clone())) -⋮---- -.get(tool_use_id) -.cloned() -.unwrap_or_else(|| "unknown_tool".into()); -⋮---- -let response_value = tool_result_to_json(content, *is_error); -Some(gemini::Part::function_response(name, response_value)) -⋮---- -// Thinking, RedactedThinking, Document: not supported by Gemini, drop. -⋮---- -/// Convert Anthropic ToolResult content into a JSON value for Gemini FunctionResponse. -fn tool_result_to_json( -⋮---- -Some(anthropic::ToolResultContent::Text(s)) => s.clone(), -⋮---- -// Concatenate text blocks; other block types become "[non-text]". -⋮---- -.map(|b| match b { -anthropic::ContentBlock::Text { text } => text.clone(), -_ => "[non-text]".into(), -⋮---- -.join("\n") -⋮---- -if is_error == Some(true) { -⋮---- -// Response direction: Gemini -> Anthropic -⋮---- -/// Convert a Gemini `GenerateContentResponse` into an Anthropic `MessageResponse`. -⋮---- -/// Uses only the first candidate. Synthesizes Anthropic-format tool IDs for any -/// function calls. -pub fn gemini_to_anthropic_response( -⋮---- -let candidate = resp.candidates.first(); -⋮---- -.map(|c| { -⋮---- -.filter_map(gemini_part_to_content_block) -⋮---- -.unwrap_or_default(); -⋮---- -.any(|b| matches!(b, anthropic::ContentBlock::ToolUse { .. })); -⋮---- -.and_then(|c| c.finish_reason.as_ref()) -.map(|fr| match fr { -⋮---- -// SAFETY, RECITATION, LANGUAGE, OTHER, Unknown all map to EndTurn. -⋮---- -// No finish_reason at all (e.g. empty candidates) -> EndTurn. -.or(if candidate.is_some() { -Some(anthropic::StopReason::EndTurn) -⋮---- -.as_ref() -.map(|u| anthropic::Usage { -⋮---- -id: generate_message_id(), -response_type: "message".into(), -⋮---- -model: model.to_string(), -⋮---- -/// Convert a single Gemini Part to an Anthropic ContentBlock, or None if not mappable. -fn gemini_part_to_content_block(part: &gemini::Part) -> Option { -// Thought parts from thinking models map to Anthropic thinking blocks. -if part.thought == Some(true) { -return part.text.as_ref().map(|text| anthropic::ContentBlock::Thinking { -thinking: text.clone(), -⋮---- -return Some(anthropic::ContentBlock::Text { text: text.clone() }); -⋮---- -return Some(anthropic::ContentBlock::ToolUse { -id: generate_tool_use_id(), -name: fc.name.clone(), -input: fc.args.clone(), -⋮---- -// inline_data, file_data, function_response: not expected in model output, -// or have no Anthropic equivalent. Drop. -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -// Helper: build a minimal Anthropic request for testing. -fn make_request(messages: Vec) -> anthropic::MessageCreateRequest { -⋮---- -model: "claude-3-5-sonnet-20241022".into(), -⋮---- -fn user_text(text: &str) -> anthropic::InputMessage { -⋮---- -content: anthropic::Content::Text(text.into()), -⋮---- -fn user_blocks(blocks: Vec) -> anthropic::InputMessage { -⋮---- -fn assistant_blocks(blocks: Vec) -> anthropic::InputMessage { -⋮---- -// ----------------------------------------------------------------------- -// Request mapping tests -⋮---- -fn simple_text_message_maps_correctly() { -let req = make_request(vec![user_text("Hello")]); -let gem = anthropic_to_gemini_request(&req); -assert_eq!(gem.contents.len(), 1); -assert_eq!(gem.contents[0].role.as_deref(), Some("user")); -assert_eq!(gem.contents[0].parts[0].text.as_deref(), Some("Hello")); -⋮---- -fn system_prompt_text_extracted_to_system_instruction() { -let mut req = make_request(vec![user_text("Hi")]); -req.system = Some(anthropic::System::Text("Be helpful.".into())); -⋮---- -let si = gem.system_instruction.unwrap(); -assert!(si.role.is_none(), "systemInstruction should have no role"); -assert_eq!(si.parts[0].text.as_deref(), Some("Be helpful.")); -⋮---- -fn system_blocks_concatenated() { -⋮---- -req.system = Some(anthropic::System::Blocks(vec![ -⋮---- -assert_eq!(si.parts[0].text.as_deref(), Some("First.\nSecond.")); -⋮---- -fn assistant_role_maps_to_model() { -let req = make_request(vec![ -⋮---- -assert_eq!(gem.contents[1].role.as_deref(), Some("model")); -⋮---- -fn image_content_maps_to_inline_data() { -let req = make_request(vec![user_blocks(vec![anthropic::ContentBlock::Image { -⋮---- -let id = part.inline_data.as_ref().unwrap(); -assert_eq!(id.mime_type, "image/jpeg"); -assert_eq!(id.data, "abc123=="); -⋮---- -fn url_image_dropped() { -⋮---- -// The URL image is dropped; no parts remain, so the content entry is empty -// and filtered out. -assert!(gem.contents.is_empty() || gem.contents[0].parts.is_empty()); -⋮---- -fn tool_use_maps_to_function_call_id_stripped() { -let req = make_request(vec![assistant_blocks(vec![ -⋮---- -let fc = part.function_call.as_ref().unwrap(); -assert_eq!(fc.name, "get_weather"); -assert_eq!(fc.args, json!({"city": "London"})); -// No id field on Gemini FunctionCallData. -⋮---- -fn tool_result_maps_to_function_response_with_name_lookup() { -⋮---- -// Second content (user role) should have a function_response part -⋮---- -.find(|c| c.role.as_deref() == Some("user")) -.unwrap(); -let fr = user_content.parts[0].function_response.as_ref().unwrap(); -assert_eq!(fr.name, "get_weather"); -assert_eq!(fr.response, json!({"result": "72F sunny"})); -⋮---- -fn tool_result_error_wraps_in_error_key() { -⋮---- -assert_eq!(fr.response, json!({"error": "timeout"})); -⋮---- -fn tool_result_unknown_id_uses_unknown_tool() { -let req = make_request(vec![user_blocks(vec![ -⋮---- -assert_eq!(fr.name, "unknown_tool"); -⋮---- -fn thinking_blocks_dropped() { -⋮---- -assert_eq!(gem.contents[0].parts.len(), 1); -assert_eq!(gem.contents[0].parts[0].text.as_deref(), Some("Answer")); -⋮---- -fn redacted_thinking_blocks_dropped() { -⋮---- -assert_eq!(gem.contents[0].parts[0].text.as_deref(), Some("Visible")); -⋮---- -fn document_blocks_dropped() { -⋮---- -assert_eq!( -⋮---- -fn tools_mapped_to_function_declarations() { -let mut req = make_request(vec![user_text("weather?")]); -req.tools = Some(vec![anthropic::Tool { -⋮---- -let decls = &gem.tools.unwrap()[0].function_declarations; -assert_eq!(decls.len(), 1); -assert_eq!(decls[0].name, "get_weather"); -assert_eq!(decls[0].description.as_deref(), Some("Get weather info")); -assert!(decls[0].parameters.is_some()); -⋮---- -fn tool_schemas_sanitized() { -let mut req = make_request(vec![user_text("test")]); -⋮---- -let tools = gem.tools.unwrap(); -⋮---- -// sanitize_schema_for_gemini strips $schema and additionalProperties -assert!(params.get("$schema").is_none()); -assert!(params.get("additionalProperties").is_none()); -assert_eq!(params["type"], "object"); -⋮---- -fn tool_choice_auto_maps() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::Auto { -⋮---- -fn tool_choice_any_maps() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::Any { -⋮---- -fn tool_choice_none_maps() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::None); -⋮---- -fn tool_choice_specific_tool_maps_to_auto() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::Tool { -name: "get_weather".into(), -⋮---- -// Gemini has no forced-tool mode, so we fall back to AUTO. -⋮---- -fn generation_config_fields_mapped() { -⋮---- -req.temperature = Some(0.7); -req.top_p = Some(0.9); -req.top_k = Some(40); -req.stop_sequences = Some(vec!["STOP".into()]); -⋮---- -let gc = gem.generation_config.unwrap(); -assert_eq!(gc.max_output_tokens, Some(2048)); -let temp = gc.temperature.unwrap(); -assert!((temp - 0.7).abs() < 0.001); -let top_p = gc.top_p.unwrap(); -assert!((top_p - 0.9).abs() < 0.001); -assert_eq!(gc.top_k, Some(40)); -assert_eq!(gc.stop_sequences, Some(vec!["STOP".into()])); -⋮---- -fn consecutive_user_messages_merged() { -let req = make_request(vec![user_text("first"), user_text("second")]); -⋮---- -assert_eq!(gem.contents.len(), 1, "should merge into one content"); -assert_eq!(gem.contents[0].parts.len(), 2); -assert_eq!(gem.contents[0].parts[0].text.as_deref(), Some("first")); -assert_eq!(gem.contents[0].parts[1].text.as_deref(), Some("second")); -⋮---- -fn user_user_model_becomes_user_model() { -⋮---- -assert_eq!(gem.contents.len(), 2); -⋮---- -fn empty_messages_list() { -let req = make_request(vec![]); -⋮---- -assert!(gem.contents.is_empty()); -⋮---- -fn content_text_shorthand_works() { -// Content::Text(string) is a shorthand accepted by Anthropic API -let req = make_request(vec![anthropic::InputMessage { -⋮---- -assert_eq!(gem.contents[0].parts[0].text.as_deref(), Some("shorthand")); -⋮---- -// build_tool_id_map tests -⋮---- -fn build_tool_id_map_finds_tool_uses() { -let messages = vec![ -⋮---- -let map = build_tool_id_map(&messages); -assert_eq!(map.get("toolu_1").unwrap(), "calc"); -assert_eq!(map.get("toolu_2").unwrap(), "search"); -⋮---- -fn build_tool_id_map_empty_on_no_tool_use() { -let messages = vec![user_text("no tools here")]; -⋮---- -assert!(map.is_empty()); -⋮---- -// merge_consecutive_roles tests -⋮---- -fn merge_consecutive_roles_no_op_for_alternating() { -let contents = vec![ -⋮---- -let merged = merge_consecutive_roles(contents); -assert_eq!(merged.len(), 2); -⋮---- -fn merge_consecutive_roles_merges_same_role() { -⋮---- -assert_eq!(merged[0].parts.len(), 2); -⋮---- -// Response mapping tests -⋮---- -fn make_gemini_response( -⋮---- -candidates: vec![gemini_resp::Candidate { -⋮---- -usage_metadata: Some(gemini_resp::UsageMetadata { -⋮---- -fn simple_text_response() { -let resp = make_gemini_response(vec![gemini::Part::text("Hello!")], Some(gemini_resp::FinishReason::STOP)); -let msg = gemini_to_anthropic_response(&resp, "gemini-2.5-flash"); -assert_eq!(msg.content.len(), 1); -⋮---- -anthropic::ContentBlock::Text { text } => assert_eq!(text, "Hello!"), -_ => panic!("expected Text block"), -⋮---- -assert_eq!(msg.model, "gemini-2.5-flash"); -assert_eq!(msg.stop_reason, Some(anthropic::StopReason::EndTurn)); -⋮---- -fn function_call_maps_to_tool_use_with_synthesized_id() { -let resp = make_gemini_response( -vec![gemini::Part::function_call("get_weather", json!({"city": "NYC"}))], -Some(gemini_resp::FinishReason::STOP), -⋮---- -assert!(id.starts_with("toolu_"), "synthesized ID should have toolu_ prefix"); -assert_eq!(name, "get_weather"); -assert_eq!(input, &json!({"city": "NYC"})); -⋮---- -_ => panic!("expected ToolUse block"), -⋮---- -fn mixed_text_and_function_call() { -⋮---- -vec![ -⋮---- -assert_eq!(msg.content.len(), 2); -assert!(matches!(&msg.content[0], anthropic::ContentBlock::Text { .. })); -assert!(matches!(&msg.content[1], anthropic::ContentBlock::ToolUse { .. })); -// Has function call + STOP -> ToolUse stop reason. -assert_eq!(msg.stop_reason, Some(anthropic::StopReason::ToolUse)); -⋮---- -fn finish_reason_stop_without_tools_maps_to_end_turn() { -let resp = make_gemini_response(vec![gemini::Part::text("done")], Some(gemini_resp::FinishReason::STOP)); -let msg = gemini_to_anthropic_response(&resp, "test"); -⋮---- -fn finish_reason_stop_with_function_call_maps_to_tool_use() { -⋮---- -vec![gemini::Part::function_call("f", json!({}))], -⋮---- -fn finish_reason_max_tokens_maps_to_max_tokens() { -let resp = make_gemini_response(vec![gemini::Part::text("trunc")], Some(gemini_resp::FinishReason::MAX_TOKENS)); -⋮---- -assert_eq!(msg.stop_reason, Some(anthropic::StopReason::MaxTokens)); -⋮---- -fn finish_reason_safety_maps_to_end_turn() { -let resp = make_gemini_response(vec![], Some(gemini_resp::FinishReason::SAFETY)); -⋮---- -fn usage_metadata_mapped() { -let resp = make_gemini_response(vec![gemini::Part::text("ok")], Some(gemini_resp::FinishReason::STOP)); -⋮---- -assert_eq!(msg.usage.input_tokens, 10); -assert_eq!(msg.usage.output_tokens, 20); -⋮---- -fn empty_candidates_gives_empty_content() { -⋮---- -candidates: vec![], -⋮---- -assert!(msg.content.is_empty()); -assert!(msg.stop_reason.is_none()); -assert_eq!(msg.usage, anthropic::Usage::default()); -⋮---- -fn no_finish_reason_defaults_to_end_turn() { -let resp = make_gemini_response(vec![gemini::Part::text("partial")], None); -⋮---- -fn multiple_text_parts_become_separate_blocks() { -⋮---- -vec![gemini::Part::text("one"), gemini::Part::text("two")], -⋮---- -assert_eq!(a, "one"); -assert_eq!(b, "two"); -⋮---- -_ => panic!("expected two Text blocks"), -⋮---- -fn model_name_passed_through() { -let resp = make_gemini_response(vec![gemini::Part::text("x")], Some(gemini_resp::FinishReason::STOP)); -let msg = gemini_to_anthropic_response(&resp, "gemini-2.5-pro"); -assert_eq!(msg.model, "gemini-2.5-pro"); -⋮---- -fn message_id_has_correct_format() { -⋮---- -assert!(msg.id.starts_with("msg_")); -assert_eq!(msg.response_type, "message"); -assert_eq!(msg.role, anthropic::Role::Assistant); -⋮---- -fn response_without_usage_metadata_gives_zero_usage() { -let mut resp = make_gemini_response(vec![gemini::Part::text("x")], Some(gemini_resp::FinishReason::STOP)); -⋮---- -assert_eq!(msg.usage.input_tokens, 0); -assert_eq!(msg.usage.output_tokens, 0); -⋮---- -// Thinking config tests -⋮---- -fn thinking_config_enabled_sets_gemini_thinking_config() { -let mut req = make_request(vec![user_text("think hard")]); -req.thinking = Some(anthropic::ThinkingConfig::Enabled { budget_tokens: 8192 }); -⋮---- -let tc = gc.thinking_config.expect("thinkingConfig should be set"); -assert_eq!(tc.thinking_budget, 8192); -assert_eq!(tc.include_thoughts, Some(true)); -⋮---- -fn thinking_config_disabled_no_thinking_config() { -let mut req = make_request(vec![user_text("hi")]); -req.thinking = Some(anthropic::ThinkingConfig::Disabled); -⋮---- -assert!(gc.thinking_config.is_none(), "disabled should not set thinkingConfig"); -⋮---- -fn thinking_config_absent_no_thinking_config() { -let req = make_request(vec![user_text("hi")]); -⋮---- -assert!(gc.thinking_config.is_none()); -⋮---- -fn gemini_thought_parts_become_thinking_blocks() { -⋮---- -thought: Some(true), -text: Some("Let me reason...".into()), -⋮---- -vec![thought_part, gemini::Part::text("Answer")], -⋮---- -assert_eq!(thinking, "Let me reason...") -⋮---- -_ => panic!("expected Thinking block first"), -⋮---- -anthropic::ContentBlock::Text { text } => assert_eq!(text, "Answer"), -_ => panic!("expected Text block second"), -⋮---- -fn gemini_thought_only_no_text() { -⋮---- -text: Some("Only thinking".into()), -⋮---- -let resp = make_gemini_response(vec![thought_part], Some(gemini_resp::FinishReason::STOP)); -⋮---- -assert!(matches!(&msg.content[0], anthropic::ContentBlock::Thinking { .. })); - - - -// Gemini streaming state machine: full-response diffing -> Anthropic SSE events. -// -// Gemini's streamGenerateContent sends FULL accumulated GenerateContentResponse -// objects per SSE event (not incremental deltas like OpenAI). This state machine -// diffs each response against the previous state to produce Anthropic-format -// delta events. -⋮---- -use crate::anthropic; -⋮---- -use crate::util; -⋮---- -/// State machine that converts Gemini streaming responses (full accumulated text) -/// into Anthropic SSE delta events by diffing against previous state. -pub struct GeminiStreamingTranslator { -⋮---- -/// Length of text already emitted as deltas. Used to diff full-text responses. -⋮---- -/// Number of tool calls already processed. -⋮---- -/// Whether a thinking content block is currently open. -⋮---- -/// Length of thought text already emitted as deltas. -⋮---- -impl GeminiStreamingTranslator { -pub fn new(model: String) -> Self { -⋮---- -/// Process one streaming GenerateContentResponse and emit Anthropic events. -/// -/// Each Gemini streaming event contains the FULL accumulated response so far, -/// so we diff against `prev_text_len` to produce incremental deltas. -pub fn process_response( -⋮---- -// Emit message_start on first call -⋮---- -events.push(self.make_message_start()); -⋮---- -let candidate = match resp.candidates.first() { -⋮---- -// Separate thought parts (thinking models) from answer parts. -⋮---- -if part.thought == Some(true) { -⋮---- -current_thought.push_str(t); -⋮---- -current_text.push_str(t); -⋮---- -// Thought delta: diff against what we already emitted. -if current_thought.len() > self.prev_thought_len { -⋮---- -events.push(anthropic::StreamEvent::ContentBlockStart { -⋮---- -events.push(anthropic::StreamEvent::ContentBlockDelta { -⋮---- -thinking: delta_thought.to_string(), -⋮---- -self.prev_thought_len = current_thought.len(); -⋮---- -// Text delta: diff against what we already emitted. -if current_text.len() > self.prev_text_len { -// Close the thought block before opening the text block. -⋮---- -events.push(anthropic::StreamEvent::ContentBlockStop { -⋮---- -text: delta_text.to_string(), -⋮---- -self.prev_text_len = current_text.len(); -⋮---- -// Tool calls: count function_call parts -⋮---- -.iter() -.filter(|p| p.function_call.is_some()) -.collect(); -let tool_count = tool_calls.len(); -⋮---- -// Close open text block before emitting tool calls -⋮---- -// Emit events for each new tool call -⋮---- -let fc = tc_part.function_call.as_ref().unwrap(); -⋮---- -name: fc.name.clone(), -⋮---- -let args_json = serde_json::to_string(&fc.args).unwrap_or_default(); -⋮---- -// Extract usage metadata -⋮---- -// Finish detection -⋮---- -self.emit_finish(reason, tool_count > 0, &mut events); -⋮---- -/// Finalize the stream when no more events are expected (e.g., connection drop -/// without a finishReason from Gemini). -pub fn finish(&mut self) -> Vec { -⋮---- -// Close any open thought block -⋮---- -// Close any open text block -⋮---- -events.push(anthropic::StreamEvent::MessageDelta { -⋮---- -stop_reason: Some(anthropic::StopReason::EndTurn), -⋮---- -usage: Some(DeltaUsage { -⋮---- -events.push(anthropic::StreamEvent::MessageStop {}); -⋮---- -pub fn is_finished(&self) -> bool { -⋮---- -fn make_message_start(&self) -> anthropic::StreamEvent { -⋮---- -id: self.message_id.clone(), -msg_type: "message".to_string(), -role: "assistant".to_string(), -content: vec![], -model: self.model.clone(), -⋮---- -usage: self.usage.clone(), -⋮---- -fn emit_finish( -⋮---- -// SAFETY, RECITATION, LANGUAGE, OTHER, Unknown all map to EndTurn -⋮---- -stop_reason: Some(stop_reason), -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -// --- Test helpers --- -⋮---- -fn make_text_response( -⋮---- -candidates: vec![Candidate { -⋮---- -fn make_text_response_with_usage( -⋮---- -let mut resp = make_text_response(text, finish_reason); -resp.usage_metadata = Some(usage); -⋮---- -fn make_tool_call_response( -⋮---- -fn make_mixed_response( -⋮---- -let mut parts = vec![Part::text(text)]; -⋮---- -parts.push(Part::function_call(*name, args.clone())); -⋮---- -fn make_empty_response() -> GenerateContentResponse { -⋮---- -candidates: vec![], -⋮---- -fn count_event_type(events: &[anthropic::StreamEvent], type_name: &str) -> usize { -⋮---- -.filter(|e| match (e, type_name) { -⋮---- -.count() -⋮---- -fn extract_text_deltas(events: &[anthropic::StreamEvent]) -> Vec { -⋮---- -.filter_map(|e| match e { -⋮---- -} => Some(text.clone()), -⋮---- -.collect() -⋮---- -// --- Tests --- -⋮---- -fn text_only_stream_multi_event() { -let mut t = GeminiStreamingTranslator::new("gemini-2.5-pro".into()); -⋮---- -// Event 1: "Hello" -let events1 = t.process_response(&make_text_response("Hello", None)); -assert_eq!(count_event_type(&events1, "message_start"), 1); -assert_eq!(count_event_type(&events1, "content_block_start"), 1); -assert_eq!(extract_text_deltas(&events1), vec!["Hello"]); -⋮---- -// Event 2: "Hello world" (accumulated) -let events2 = t.process_response(&make_text_response("Hello world", None)); -assert_eq!(count_event_type(&events2, "message_start"), 0); -assert_eq!(count_event_type(&events2, "content_block_start"), 0); -assert_eq!(extract_text_deltas(&events2), vec![" world"]); -⋮---- -// Event 3: "Hello world!" with STOP -let events3 = t.process_response(&make_text_response("Hello world!", Some(FinishReason::STOP))); -assert_eq!(extract_text_deltas(&events3), vec!["!"]); -assert_eq!(count_event_type(&events3, "content_block_stop"), 1); -assert_eq!(count_event_type(&events3, "message_delta"), 1); -assert_eq!(count_event_type(&events3, "message_stop"), 1); -assert!(t.is_finished()); -⋮---- -fn single_event_with_finish() { -let mut t = GeminiStreamingTranslator::new("gemini-2.5-flash".into()); -let events = t.process_response(&make_text_response("Done.", Some(FinishReason::STOP))); -⋮---- -assert_eq!(count_event_type(&events, "message_start"), 1); -assert_eq!(count_event_type(&events, "content_block_start"), 1); -assert_eq!(extract_text_deltas(&events), vec!["Done."]); -assert_eq!(count_event_type(&events, "content_block_stop"), 1); -assert_eq!(count_event_type(&events, "message_delta"), 1); -assert_eq!(count_event_type(&events, "message_stop"), 1); -⋮---- -fn tool_call_stream() { -⋮---- -// Event 1: text appears -let events1 = t.process_response(&make_text_response("Let me check", None)); -assert_eq!(extract_text_deltas(&events1), vec!["Let me check"]); -⋮---- -// Event 2: text + tool call with STOP -let resp2 = make_mixed_response( -⋮---- -&[("get_weather", json!({"city": "London"}))], -Some(FinishReason::STOP), -⋮---- -let events2 = t.process_response(&resp2); -// No text delta (same text length) -assert!(extract_text_deltas(&events2).is_empty()); -// Text block closed, tool block started/deltad/stopped -assert_eq!(count_event_type(&events2, "content_block_stop"), 2); // text close + tool close -assert_eq!(count_event_type(&events2, "content_block_start"), 1); // tool start -⋮---- -// Should finish with tool_use stop reason -let delta_event = events2.iter().find(|e| matches!(e, anthropic::StreamEvent::MessageDelta { .. })); -⋮---- -assert_eq!(delta.stop_reason, Some(anthropic::StopReason::ToolUse)); -⋮---- -panic!("expected MessageDelta"); -⋮---- -fn tool_call_only_no_text() { -⋮---- -let resp = make_tool_call_response("search", json!({"q": "rust"}), Some(FinishReason::STOP)); -let events = t.process_response(&resp); -⋮---- -// No text block should be opened -assert!(extract_text_deltas(&events).is_empty()); -// Tool call events -let tool_starts: Vec<_> = events.iter().filter(|e| matches!(e, anthropic::StreamEvent::ContentBlockStart { content_block: anthropic::ContentBlock::ToolUse { .. }, .. })).collect(); -assert_eq!(tool_starts.len(), 1); -⋮---- -// Verify tool name -⋮---- -assert_eq!(name, "search"); -⋮---- -fn multiple_tool_calls() { -⋮---- -let resp = make_mixed_response( -⋮---- -("get_weather", json!({"city": "London"})), -("get_time", json!({"tz": "UTC"})), -⋮---- -assert_eq!(tool_starts.len(), 2); -⋮---- -fn empty_response_no_candidates() { -⋮---- -let events = t.process_response(&make_empty_response()); -// Only message_start -⋮---- -assert_eq!(events.len(), 1); -assert!(!t.is_finished()); -⋮---- -fn safety_stop() { -⋮---- -let events = t.process_response(&make_text_response("I can't", Some(FinishReason::SAFETY))); -⋮---- -let delta_event = events.iter().find(|e| matches!(e, anthropic::StreamEvent::MessageDelta { .. })); -⋮---- -assert_eq!(delta.stop_reason, Some(anthropic::StopReason::EndTurn)); -⋮---- -fn max_tokens_stop() { -⋮---- -let events = t.process_response(&make_text_response("truncated", Some(FinishReason::MAX_TOKENS))); -⋮---- -assert_eq!(delta.stop_reason, Some(anthropic::StopReason::MaxTokens)); -⋮---- -fn no_new_content_between_events() { -⋮---- -// Same text, no new content -let events2 = t.process_response(&make_text_response("Hello", None)); -// Only message-level events, no deltas or block starts -⋮---- -fn usage_metadata_extracted() { -⋮---- -let resp = make_text_response_with_usage("done", Some(FinishReason::STOP), usage); -⋮---- -assert_eq!(u.output_tokens, 25); -⋮---- -panic!("expected MessageDelta with usage"); -⋮---- -fn finish_called_without_finish_reason() { -⋮---- -let _ = t.process_response(&make_text_response("partial", None)); -⋮---- -let events = t.finish(); -⋮---- -// Double finish should be no-op -let events2 = t.finish(); -assert!(events2.is_empty()); -⋮---- -fn first_event_emits_message_start() { -⋮---- -let events = t.process_response(&make_text_response("hi", None)); -⋮---- -assert!(message.id.starts_with("msg_")); -assert_eq!(message.model, "gemini-2.5-pro"); -assert_eq!(message.role, "assistant"); -assert!(message.content.is_empty()); -assert!(message.stop_reason.is_none()); -⋮---- -other => panic!("expected MessageStart, got {:?}", other), -⋮---- -fn text_block_opened_lazily() { -⋮---- -// Empty text part should not open a text block -⋮---- -// Only message_start, no content block start -assert_eq!(count_event_type(&events, "content_block_start"), 0); -assert!(!t.text_block_open); -⋮---- -fn tool_id_has_toolu_prefix() { -⋮---- -let resp = make_tool_call_response("test_fn", json!({}), Some(FinishReason::STOP)); -⋮---- -let tool_start = events.iter().find(|e| matches!(e, anthropic::StreamEvent::ContentBlockStart { content_block: anthropic::ContentBlock::ToolUse { .. }, .. })); -⋮---- -assert!(id.starts_with("toolu_"), "tool ID should start with toolu_, got: {id}"); -⋮---- -panic!("expected tool use content block start"); -⋮---- -fn mixed_text_and_tool_call_sequence() { -⋮---- -// Event 1: text starts -let events1 = t.process_response(&make_text_response("I'll help.", None)); -⋮---- -assert_eq!(extract_text_deltas(&events1), vec!["I'll help."]); -⋮---- -// Event 2: same text + tool call + finish -⋮---- -&[("lookup", json!({"id": 42}))], -⋮---- -// Text block closed, tool block opened/deltad/stopped, then finish -assert_eq!(count_event_type(&events2, "content_block_stop"), 2); // text + tool -assert_eq!(count_event_type(&events2, "content_block_start"), 1); // tool -⋮---- -// Verify InputJsonDelta was emitted for the tool -let json_deltas: Vec<_> = events2.iter().filter_map(|e| match e { -anthropic::StreamEvent::ContentBlockDelta { delta: anthropic::streaming::Delta::InputJsonDelta { partial_json }, .. } => Some(partial_json.clone()), -⋮---- -}).collect(); -assert_eq!(json_deltas.len(), 1); -assert!(json_deltas[0].contains("42")); -⋮---- -fn content_block_indices_increment_correctly() { -⋮---- -&[("fn_a", json!({})), ("fn_b", json!({}))], -⋮---- -// Collect all indices from ContentBlockStart events -let start_indices: Vec = events.iter().filter_map(|e| match e { -anthropic::StreamEvent::ContentBlockStart { index, .. } => Some(*index), -⋮---- -// text block at 0, fn_a at 1, fn_b at 2 -assert_eq!(start_indices, vec![0, 1, 2]); -⋮---- -fn recitation_stop_maps_to_end_turn() { -⋮---- -let events = t.process_response(&make_text_response("x", Some(FinishReason::RECITATION))); -⋮---- -fn unknown_finish_reason_maps_to_end_turn() { -⋮---- -let events = t.process_response(&make_text_response("x", Some(FinishReason::Unknown))); -⋮---- -fn finish_on_empty_stream() { -// finish() on a translator that never received any events -⋮---- -// No text block was open, so just message_delta + message_stop -assert_eq!(count_event_type(&events, "content_block_stop"), 0); -⋮---- -fn stop_with_tool_calls_gives_tool_use_reason() { -⋮---- -let resp = make_tool_call_response("run", json!({}), Some(FinishReason::STOP)); -⋮---- -fn tool_call_args_serialized_as_json() { -⋮---- -let args = json!({"city": "Paris", "units": "celsius"}); -let resp = make_tool_call_response("weather", args.clone(), Some(FinishReason::STOP)); -⋮---- -let json_deltas: Vec<_> = events.iter().filter_map(|e| match e { -⋮---- -// Parse back and verify -let parsed: serde_json::Value = serde_json::from_str(&json_deltas[0]).unwrap(); -assert_eq!(parsed, args); -⋮---- -fn process_after_finished_is_noop() { -⋮---- -let _ = t.process_response(&make_text_response("done", Some(FinishReason::STOP))); -⋮---- -// Further process calls should still work but emit no finish events again -let events = t.process_response(&make_text_response("extra", None)); -// finished flag prevents double-finish -assert_eq!(count_event_type(&events, "message_delta"), 0); -assert_eq!(count_event_type(&events, "message_stop"), 0); -⋮---- -fn message_id_is_unique_per_translator() { -let t1 = GeminiStreamingTranslator::new("m".into()); -let t2 = GeminiStreamingTranslator::new("m".into()); -assert_ne!(t1.message_id, t2.message_id); -⋮---- -fn text_delta_with_multibyte_characters() { -⋮---- -// Append unicode characters -let events2 = t.process_response(&make_text_response("Hello, world!", None)); -assert_eq!(extract_text_deltas(&events2), vec![", world!"]); -⋮---- -fn incremental_tool_calls_across_events() { -⋮---- -// Event 1: first tool call -⋮---- -let events1 = t.process_response(&resp1); -let tool_starts_1: usize = events1.iter().filter(|e| matches!(e, anthropic::StreamEvent::ContentBlockStart { content_block: anthropic::ContentBlock::ToolUse { .. }, .. })).count(); -assert_eq!(tool_starts_1, 1); -⋮---- -// Event 2: two tool calls (first is same, second is new) -⋮---- -let tool_starts_2: usize = events2.iter().filter(|e| matches!(e, anthropic::StreamEvent::ContentBlockStart { content_block: anthropic::ContentBlock::ToolUse { .. }, .. })).count(); -// Only the new tool call should produce a start event -assert_eq!(tool_starts_2, 1); -⋮---- -fn all_events_serialize_to_valid_json() { -⋮---- -let events = t.process_response(&make_text_response("test", Some(FinishReason::STOP))); -⋮---- -assert!(json.is_ok(), "event should serialize: {:?}", event); -⋮---- -// --- Thinking / extended thinking tests --- -⋮---- -fn make_thought_response( -⋮---- -thought: Some(true), -text: Some(thought.into()), -⋮---- -fn streaming_thought_emits_thinking_delta() { -⋮---- -let resp = make_thought_response("Reasoning here", "", None); -⋮---- -.filter(|e| { -matches!( -⋮---- -assert_eq!(thinking_starts.len(), 1, "should open one thinking block"); -⋮---- -assert_eq!(thinking_deltas.len(), 1); -⋮---- -assert_eq!(thinking, "Reasoning here"); -⋮---- -fn streaming_thought_block_closed_before_text_block() { -⋮---- -let resp = make_thought_response("thought", "answer", Some(FinishReason::STOP)); -⋮---- -// Find sequence: ContentBlockStop for thought, then ContentBlockStart for text. -⋮---- -assert!(saw_thought_stop, "thinking block should be closed"); -assert!(saw_text_start_after_stop, "text block should start after thinking stop"); -⋮---- -// No thinking block should remain open at message_stop. -⋮---- -assert!(!t.thought_block_open); -⋮---- -fn streaming_incremental_thought_diffs() { -⋮---- -// First chunk: partial thought -let resp1 = make_thought_response("Step 1", "", None); -⋮---- -.filter_map(|e| { -⋮---- -Some(thinking.as_str()) -⋮---- -assert_eq!(delta1, vec!["Step 1"]); -⋮---- -// Second chunk: more thought added -let resp2 = make_thought_response("Step 1 Step 2", "", Some(FinishReason::STOP)); -⋮---- -// Only the NEW portion should appear as a delta -assert_eq!(delta2, vec![" Step 2"]); - - - -// Phase 22: Anthropic <-> OpenAI Responses API message mapping -// -// Pure translation between Anthropic Messages API and OpenAI Responses API. -// The Responses API uses `input` (text or items) instead of `messages[]`, -// `instructions` instead of system messages, and `output[]` instead of `choices[]`. -⋮---- -use crate::anthropic; -use crate::mapping::message_map::extract_system_text; -⋮---- -use crate::util; -⋮---- -/// Convert an Anthropic MessageCreateRequest to an OpenAI Responses API request. -/// -/// Anthropic: -/// OpenAI Responses: -pub fn anthropic_to_responses_request(req: &anthropic::MessageCreateRequest) -> ResponsesRequest { -let instructions = req.system.as_ref().map(extract_system_text); -⋮---- -let input = build_input_items(&req.messages); -⋮---- -let tools = req.tools.as_ref().map(|tools| { -⋮---- -.iter() -.map(|t| { -let mut tool = json!({ -⋮---- -tool["description"] = json!(desc); -⋮---- -.collect() -⋮---- -if req.top_k.is_some() { -⋮---- -if req.thinking.is_some() { -⋮---- -if req.metadata.is_some() { -⋮---- -extra.insert("top_p".into(), json!(top_p)); -⋮---- -anthropic::ToolChoice::Auto { .. } => json!("auto"), -anthropic::ToolChoice::Any { .. } => json!("required"), -anthropic::ToolChoice::None => json!("none"), -anthropic::ToolChoice::Tool { name } => json!({ -⋮---- -extra.insert("tool_choice".into(), mapped); -⋮---- -if seqs.len() > 4 { -⋮---- -let capped: Vec<&str> = seqs.iter().take(4).map(|s| s.as_str()).collect(); -extra.insert("stop".into(), json!(capped)); -⋮---- -model: req.model.clone(), -⋮---- -max_output_tokens: Some(req.max_tokens), -temperature: req.temperature.map(|t| t.clamp(0.0, 1.0)), -⋮---- -/// Build Responses API input items from Anthropic messages. -⋮---- -/// The Responses API models tool calls as first-class items in the input -/// array, not as message content blocks. Anthropic tool_use blocks become -/// function_call items at the root level; tool_result blocks become -/// function_call_output items. This flattened structure is required by the -/// Responses API schema. -fn build_input_items(messages: &[anthropic::InputMessage]) -> ResponsesInput { -⋮---- -items.push(json!({ -⋮---- -convert_blocks_to_items(blocks, role, &mut items); -⋮---- -/// Convert Anthropic content blocks into Responses API input items. -⋮---- -/// Text/image blocks are grouped into a message item. -/// Tool results become separate `function_call_output` items. -/// Tool use blocks (in assistant messages) become `function_call` items. -fn convert_blocks_to_items(blocks: &[anthropic::ContentBlock], role: &str, items: &mut Vec) { -⋮---- -content_parts.push(json!({"type": "input_text", "text": text})); -⋮---- -content_parts.push(json!({ -⋮---- -let mt = source.media_type.as_deref().unwrap_or("image/png"); -⋮---- -tool_calls.push(json!({ -⋮---- -let output = tool_result_to_string(content.as_ref()); -tool_results.push(json!({ -⋮---- -// Silently dropped, same as Chat Completions path -⋮---- -if !content_parts.is_empty() { -⋮---- -items.extend(tool_calls); -items.extend(tool_results); -⋮---- -/// Extract text from an Anthropic tool result content. -fn tool_result_to_string(content: Option<&anthropic::messages::ToolResultContent>) -> String { -⋮---- -Some(anthropic::messages::ToolResultContent::Text(t)) => t.clone(), -⋮---- -parts.push(text.as_str()); -⋮---- -parts.join("\n") -⋮---- -/// Convert an OpenAI Responses API response to an Anthropic MessageResponse. -⋮---- -/// OpenAI Responses: -⋮---- -pub fn responses_to_anthropic_response( -⋮---- -let stop_reason = match resp.status.as_str() { -"completed" => Some(anthropic::StopReason::EndTurn), -"incomplete" => Some(anthropic::StopReason::MaxTokens), -"failed" => Some(anthropic::StopReason::EndTurn), -⋮---- -Some(anthropic::StopReason::EndTurn) -⋮---- -extract_output_item(item, &mut content); -⋮---- -if content.is_empty() { -content.push(anthropic::ContentBlock::Text { -⋮---- -.as_ref() -.map_or_else(anthropic::Usage::default, |u| { -⋮---- -super::usage_map::extract_cached_tokens(u.input_token_details.as_ref()); -⋮---- -response_type: "message".to_string(), -⋮---- -model: original_model.to_string(), -⋮---- -/// Extract content blocks from a single output item (JSON value). -fn extract_output_item(item: &Value, content: &mut Vec) { -let item_type = item.get("type").and_then(|v| v.as_str()).unwrap_or(""); -⋮---- -// OutputMessage: has content[] array -if let Some(parts) = item.get("content").and_then(|v| v.as_array()) { -⋮---- -let part_type = part.get("type").and_then(|v| v.as_str()).unwrap_or(""); -⋮---- -if let Some(text) = part.get("text").and_then(|v| v.as_str()) { -⋮---- -text: text.to_string(), -⋮---- -if let Some(refusal) = part.get("refusal").and_then(|v| v.as_str()) { -⋮---- -// FunctionToolCall output item -> tool_use content block -let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); -let name = item.get("name").and_then(|v| v.as_str()).unwrap_or(""); -⋮---- -.get("arguments") -.and_then(|v| v.as_str()) -.unwrap_or("{}"); -⋮---- -if name.is_empty() { -⋮---- -let id = if call_id.is_empty() { -⋮---- -call_id.to_string() -⋮---- -content.push(anthropic::ContentBlock::ToolUse { -⋮---- -name: name.to_string(), -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn simple_request() -> anthropic::MessageCreateRequest { -serde_json::from_value(json!({ -⋮---- -.unwrap() -⋮---- -fn basic_text_request() { -let req = simple_request(); -let responses_req = anthropic_to_responses_request(&req); -⋮---- -assert_eq!(responses_req.model, "claude-sonnet-4-6"); -assert_eq!(responses_req.max_output_tokens, Some(1024)); -assert!(responses_req.instructions.is_none()); -⋮---- -assert_eq!(items.len(), 1); -assert_eq!(items[0]["type"], "message"); -assert_eq!(items[0]["role"], "user"); -assert_eq!(items[0]["content"][0]["type"], "input_text"); -assert_eq!(items[0]["content"][0]["text"], "Hello"); -⋮---- -_ => panic!("expected Items input"), -⋮---- -fn system_prompt_to_instructions() { -let req: anthropic::MessageCreateRequest = serde_json::from_value(json!({ -⋮---- -.unwrap(); -⋮---- -assert_eq!(responses_req.instructions, Some("You are helpful".into())); -⋮---- -fn multi_turn_conversation() { -⋮---- -assert_eq!(items.len(), 3); -⋮---- -assert_eq!(items[1]["role"], "assistant"); -assert_eq!(items[2]["role"], "user"); -⋮---- -fn tool_definitions_mapping() { -⋮---- -let tools = responses_req.tools.unwrap(); -assert_eq!(tools.len(), 1); -assert_eq!(tools[0]["type"], "function"); -assert_eq!(tools[0]["name"], "get_weather"); -assert_eq!(tools[0]["description"], "Get weather for a city"); -assert!(tools[0]["parameters"]["properties"]["city"].is_object()); -⋮---- -fn tool_use_in_assistant_message() { -⋮---- -// user message, function_call item, function_call_output item -⋮---- -assert_eq!(items[1]["type"], "function_call"); -assert_eq!(items[1]["call_id"], "toolu_123"); -assert_eq!(items[1]["name"], "get_weather"); -assert_eq!(items[2]["type"], "function_call_output"); -assert_eq!(items[2]["call_id"], "toolu_123"); -assert_eq!(items[2]["output"], "72F sunny"); -⋮---- -fn temperature_clamped() { -⋮---- -assert_eq!(responses_req.temperature, Some(1.0)); -⋮---- -fn stop_sequences_truncated() { -⋮---- -let stop = responses_req.extra.get("stop").unwrap().as_array().unwrap(); -assert_eq!(stop.len(), 4); -⋮---- -fn basic_text_response() { -let resp: ResponsesResponse = serde_json::from_value(json!({ -⋮---- -let anthropic_resp = responses_to_anthropic_response(&resp, "claude-sonnet-4-6"); -assert_eq!(anthropic_resp.model, "claude-sonnet-4-6"); -assert_eq!(anthropic_resp.role, anthropic::Role::Assistant); -assert_eq!( -⋮---- -assert_eq!(anthropic_resp.usage.input_tokens, 10); -assert_eq!(anthropic_resp.usage.output_tokens, 5); -⋮---- -anthropic::ContentBlock::Text { text } => assert_eq!(text, "Hello!"), -_ => panic!("expected text block"), -⋮---- -fn response_with_function_call() { -⋮---- -assert_eq!(id, "call_123"); -assert_eq!(name, "get_weather"); -assert_eq!(input["city"], "NYC"); -⋮---- -_ => panic!("expected tool_use block"), -⋮---- -fn response_incomplete_status() { -⋮---- -fn response_no_usage() { -⋮---- -assert_eq!(anthropic_resp.usage.input_tokens, 0); -assert_eq!(anthropic_resp.usage.output_tokens, 0); -⋮---- -fn response_mixed_text_and_tool_calls() { -⋮---- -assert_eq!(anthropic_resp.content.len(), 2); -assert!( -⋮---- -fn empty_output_gets_empty_text() { -⋮---- -assert_eq!(anthropic_resp.content.len(), 1); -⋮---- -fn tool_choice_mapping() { -⋮---- -assert_eq!(responses_req.extra["tool_choice"], "required"); -⋮---- -fn image_block_mapping() { -⋮---- -let content = items[0]["content"].as_array().unwrap(); -assert_eq!(content.len(), 2); -assert_eq!(content[0]["type"], "input_text"); -assert_eq!(content[1]["type"], "input_image"); -assert_eq!(content[1]["image_url"], "https://example.com/img.png"); - - - -//! Token usage field mapping between Anthropic and OpenAI APIs. -⋮---- -use crate::anthropic; -use crate::openai; -⋮---- -/// Extract `cached_tokens` from an OpenAI token details JSON object. -/// -/// Used by both Chat Completions (`prompt_tokens_details`) and Responses API -/// (`input_token_details`) paths to map to Anthropic's `cache_read_input_tokens`. -pub(crate) fn extract_cached_tokens(details: Option<&serde_json::Value>) -> Option { -⋮---- -.and_then(|d| d.get("cached_tokens")) -.and_then(|v| v.as_u64()) -.map(|n| n as u32) -⋮---- -/// Convert OpenAI token usage to Anthropic usage format. -⋮---- -/// Maps `prompt_tokens` to `input_tokens`, `completion_tokens` to `output_tokens`, -/// and extracts `cached_tokens` from `prompt_tokens_details` into `cache_read_input_tokens`. -⋮---- -/// OpenAI usage: -/// Anthropic usage: -pub fn openai_to_anthropic_usage(usage: &openai::ChatUsage) -> anthropic::Usage { -// OpenAI reports cached tokens in prompt_tokens_details.cached_tokens; -// Anthropic calls the same concept cache_read_input_tokens. -let cache_read_input_tokens = extract_cached_tokens(usage.prompt_tokens_details.as_ref()); -⋮---- -/// Convert Anthropic usage to OpenAI usage. -⋮---- -/// Anthropic: -/// OpenAI: -pub fn anthropic_to_openai_usage(usage: &anthropic::Usage) -> openai::ChatUsage { -⋮---- -// Compat spec response: "Always empty". No Anthropic equivalent. -// See: https://docs.anthropic.com/en/api/openai-sdk#response-fields -⋮---- -mod tests { -⋮---- -fn openai_to_anthropic_basic() { -⋮---- -let anth = openai_to_anthropic_usage(&oai); -assert_eq!(anth.input_tokens, 100); -assert_eq!(anth.output_tokens, 50); -assert!(anth.cache_creation_input_tokens.is_none()); -assert!(anth.cache_read_input_tokens.is_none()); -⋮---- -fn anthropic_to_openai_basic() { -⋮---- -cache_creation_input_tokens: Some(10), -cache_read_input_tokens: Some(5), -⋮---- -let oai = anthropic_to_openai_usage(&anth); -assert_eq!(oai.prompt_tokens, 200); -assert_eq!(oai.completion_tokens, 80); -assert_eq!(oai.total_tokens, 280); -⋮---- -fn zero_values() { -⋮---- -assert_eq!(anth.input_tokens, 0); -assert_eq!(anth.output_tokens, 0); -⋮---- -let back = anthropic_to_openai_usage(&anth); -assert_eq!(back.prompt_tokens, 0); -assert_eq!(back.completion_tokens, 0); -assert_eq!(back.total_tokens, 0); -⋮---- -fn total_tokens_computed_from_parts() { -// OpenAI total_tokens is ignored when converting to Anthropic and back; -// the round-trip recomputes it from input + output. -⋮---- -assert_eq!(oai.total_tokens, 50); -⋮---- -fn cache_fields_dropped_on_conversion() { -// Cache fields exist in Anthropic but not OpenAI; verify they survive round-trip -// only as None on the way back. -⋮---- -cache_creation_input_tokens: Some(3), -cache_read_input_tokens: Some(7), -⋮---- -let back = openai_to_anthropic_usage(&oai); -assert_eq!(back.input_tokens, 10); -assert_eq!(back.output_tokens, 5); -assert!(back.cache_creation_input_tokens.is_none()); -assert!(back.cache_read_input_tokens.is_none()); -⋮---- -fn openai_to_anthropic_with_cached_tokens() { -⋮---- -prompt_tokens_details: Some( -⋮---- -assert_eq!(anth.cache_read_input_tokens, Some(42)); -⋮---- -fn openai_to_anthropic_zero_cached_tokens() { -⋮---- -prompt_tokens_details: Some(serde_json::json!({"cached_tokens": 0})), -⋮---- -assert_eq!(anth.cache_read_input_tokens, Some(0)); - - - -// ID generation utilities for Anthropic-format identifiers. -// Uses UUID v4 (simple/no-hyphen format) with a domain prefix to match the -// {prefix}_{hex} pattern that Anthropic SDKs and clients expect when parsing -// response IDs. See: https://docs.anthropic.com/en/api/messages -⋮---- -/// Generate a message ID in Anthropic format (msg_ prefix + uuid v4 without hyphens). -pub fn generate_message_id() -> String { -format!("msg_{}", uuid::Uuid::new_v4().as_simple()) -⋮---- -/// Generate a content block ID in Anthropic format. -pub fn generate_content_block_id() -> String { -format!("block_{}", uuid::Uuid::new_v4().as_simple()) -⋮---- -/// Generate a tool use ID in Anthropic format (toolu_ prefix). -pub fn generate_tool_use_id() -> String { -format!("toolu_{}", uuid::Uuid::new_v4().as_simple()) -⋮---- -/// Generate a raw UUID v4 without hyphens (for custom prefix use). -pub fn generate_uuid() -> String { -uuid::Uuid::new_v4().as_simple().to_string() -⋮---- -mod tests { -⋮---- -fn message_id_has_correct_prefix_and_length() { -let id = generate_message_id(); -assert!(id.starts_with("msg_"), "expected msg_ prefix, got: {id}"); -// "msg_" (4) + 32 hex chars = 36 -assert_eq!(id.len(), 36, "unexpected length: {id}"); -⋮---- -fn content_block_id_has_correct_prefix_and_length() { -let id = generate_content_block_id(); -assert!( -⋮---- -// "block_" (6) + 32 hex chars = 38 -assert_eq!(id.len(), 38, "unexpected length: {id}"); -⋮---- -fn tool_use_id_has_correct_prefix_and_length() { -let id = generate_tool_use_id(); -⋮---- -// "toolu_" (6) + 32 hex chars = 38 -⋮---- -fn ids_are_unique() { -let a = generate_message_id(); -let b = generate_message_id(); -assert_ne!(a, b); - - - -//! High-level async client: Anthropic request in, Anthropic response out. -//! -//! Combines translation, HTTP, retry, and SSE streaming into a single ergonomic API. -⋮---- -use anyllm_translate::anthropic::messages::MessageResponse; -use anyllm_translate::anthropic::streaming::StreamEvent; -use anyllm_translate::anthropic::MessageCreateRequest; -⋮---- -use futures::Stream; -⋮---- -use crate::error::ClientError; -⋮---- -use crate::rate_limit::RateLimitHeaders; -⋮---- -use crate::streaming::SseTranslatingStream; -⋮---- -/// Authentication for the backend API. -⋮---- -pub enum Auth { -/// Bearer token (e.g., OpenAI API key). -⋮---- -/// Custom header (e.g., `x-goog-api-key` for Google). -⋮---- -/// Configuration for the [`Client`]. -⋮---- -pub struct ClientConfig { -/// URL for the chat completions endpoint (e.g., `https://api.openai.com/v1/chat/completions`). -⋮---- -/// Authentication credentials. -⋮---- -/// HTTP client configuration (TLS, timeouts, SSRF protection). -⋮---- -/// Translation configuration (model mapping, lossy behavior). -⋮---- -impl ClientConfig { -pub fn builder() -> ClientConfigBuilder { -⋮---- -/// Builder for [`ClientConfig`]. -⋮---- -pub struct ClientConfigBuilder { -⋮---- -impl ClientConfigBuilder { -/// Set the chat completions endpoint URL. -/// For OpenAI: `https://api.openai.com/v1/chat/completions` -pub fn backend_url(mut self, url: impl Into) -> Self { -self.backend_url = url.into(); -⋮---- -/// Set authentication credentials. -pub fn auth(mut self, auth: Auth) -> Self { -self.auth = Some(auth); -⋮---- -/// Set HTTP client configuration. Uses secure defaults if not specified. -pub fn http(mut self, http: HttpClientConfig) -> Self { -self.http = Some(http); -⋮---- -/// Set translation configuration. -pub fn translation(mut self, translation: TranslationConfig) -> Self { -self.translation = Some(translation); -⋮---- -pub fn build(self) -> ClientConfig { -⋮---- -auth: self.auth.unwrap_or(Auth::Bearer(String::new())), -http: self.http.unwrap_or_default(), -translation: self.translation.unwrap_or_default(), -⋮---- -/// Internal error type implementing [`RetryableError`] for the generic retry loop. -⋮---- -enum InternalError { -⋮---- -impl RetryableError for InternalError { -fn from_request(e: reqwest::Error) -> Self { -⋮---- -fn from_api_response(status: u16, body: &str) -> Self { -⋮---- -body: body.to_string(), -⋮---- -fn from(e: InternalError) -> Self { -⋮---- -message: format!("Backend returned status {status}"), -⋮---- -/// Simplified builder for [`Client`] with sensible defaults. -/// -/// Use this when you want a quick client without manually wiring -/// [`ClientConfig`], [`HttpClientConfig`], and [`TranslationConfig`]. -⋮---- -/// # Examples -⋮---- -/// ```rust,no_run -/// use anyllm_client::ClientBuilder; -⋮---- -/// # fn example() -> Result<(), anyllm_client::ClientError> { -/// let client = ClientBuilder::new() -/// .base_url("https://api.openai.com/v1/chat/completions") -/// .api_key("sk-...") -/// .build()?; -/// # Ok(()) -/// # } -/// ``` -pub struct ClientBuilder { -⋮---- -impl ClientBuilder { -/// Create a new builder with all fields unset. -pub fn new() -> Self { -⋮---- -/// Set the backend URL (e.g., `https://api.openai.com/v1/chat/completions`). -pub fn base_url(mut self, url: &str) -> Self { -self.base_url = Some(url.to_string()); -⋮---- -/// Set the API key used as a Bearer token. -pub fn api_key(mut self, key: &str) -> Self { -self.api_key = Some(key.to_string()); -⋮---- -/// Set the connection timeout (default: 10s). -pub fn timeout(mut self, duration: std::time::Duration) -> Self { -self.timeout = Some(duration); -⋮---- -/// Set the read timeout (default: 900s). -pub fn read_timeout(mut self, duration: std::time::Duration) -> Self { -self.read_timeout = Some(duration); -⋮---- -/// Set the maximum number of retries on 429/5xx (default: 3). -⋮---- -/// Note: this value is stored for forward compatibility but the current -/// retry implementation uses the crate-level [`MAX_RETRIES`](crate::retry::MAX_RETRIES) constant. -pub fn max_retries(mut self, n: u32) -> Self { -self.max_retries = Some(n); -⋮---- -/// Build the [`Client`], returning an error if `base_url` is missing. -pub fn build(self) -> Result { -let base_url = self.base_url.ok_or_else(|| ClientError::ApiError { -⋮---- -message: "ClientBuilder: base_url is required".to_string(), -⋮---- -auth: Auth::Bearer(self.api_key.unwrap_or_default()), -⋮---- -Ok(Client::new(config)) -⋮---- -impl Default for ClientBuilder { -fn default() -> Self { -⋮---- -/// Async HTTP client for Anthropic-to-OpenAI translation. -⋮---- -/// Accepts Anthropic Messages API requests, translates to OpenAI format, -/// sends to the configured backend, and translates the response back. -⋮---- -/// use anyllm_client::{Client, ClientConfig, Auth}; -⋮---- -/// let config = ClientConfig::builder() -/// .backend_url("https://api.openai.com/v1/chat/completions") -/// .auth(Auth::Bearer("sk-...".into())) -/// .build(); -/// let client = Client::new(config); -⋮---- -pub struct Client { -⋮---- -impl Client { -/// Create a new client from configuration. -pub fn new(config: ClientConfig) -> Self { -let http = build_http_client(&config.http); -⋮---- -/// Return a [`ClientBuilder`] for simplified construction. -⋮---- -/// use anyllm_client::Client; -⋮---- -/// let client = Client::builder() -⋮---- -pub fn builder() -> ClientBuilder { -⋮---- -/// Create from an existing reqwest client and configuration. -/// Useful when you want to share an HTTP client across multiple instances. -pub fn with_http_client(http: reqwest::Client, config: ClientConfig) -> Self { -⋮---- -fn auth(&self) -> retry::RequestAuth<'_> { -⋮---- -/// Send an Anthropic Messages API request and get an Anthropic response. -⋮---- -/// Translates the request to OpenAI format, sends it, and translates the -/// response back. Retries on 429/5xx with exponential backoff. -pub async fn messages( -⋮---- -let openai_req = translate_request(req, &self.config.translation)?; -let (resp, _status, _rate_limits) = self.chat_completion(&openai_req).await?; -let anthropic_resp = translate_response(&resp, &req.model); -Ok(anthropic_resp) -⋮---- -/// Send an Anthropic Messages API request and get a stream of Anthropic SSE events. -⋮---- -/// The returned stream yields `StreamEvent` items. Translation happens -/// incrementally as chunks arrive from the backend. -pub async fn messages_stream( -⋮---- -let mut openai_req = translate_request(req, &self.config.translation)?; -openai_req.stream = Some(true); -let (response, rate_limits) = self.chat_completion_stream_raw(&openai_req).await?; -⋮---- -let model = req.model.clone(); -⋮---- -Ok((stream, rate_limits)) -⋮---- -/// Send a pre-translated OpenAI Chat Completion request. -⋮---- -/// Useful when you want to handle translation yourself and just need the -/// HTTP client with retry logic. -pub async fn chat_completion( -⋮---- -&self.auth(), -⋮---- -.map_err(ClientError::from)?; -⋮---- -let status = response.status().as_u16(); -let rate_limits = RateLimitHeaders::from_openai_headers(response.headers()); -⋮---- -.map_err(|e| ClientError::Deserialization(e.to_string()))?; -Ok((body, status, rate_limits)) -⋮---- -/// Send a streaming Chat Completion request and get the raw response. -async fn chat_completion_stream_raw( -⋮---- -Ok((response, rate_limits)) -⋮---- -mod tests { -⋮---- -fn client_config_builder_defaults() { -⋮---- -.backend_url("https://api.openai.com/v1/chat/completions") -.auth(Auth::Bearer("sk-test".into())) -.build(); -⋮---- -assert_eq!( -⋮---- -assert!(matches!(config.auth, Auth::Bearer(ref s) if s == "sk-test")); -⋮---- -fn client_config_builder_with_translation() { -⋮---- -.model_map("haiku", "gpt-4o-mini") -.model_map("sonnet", "gpt-4o") -⋮---- -.translation(translation) -⋮---- -assert!(config.translation.map_model("claude-3-haiku").is_ok()); -⋮---- -fn client_creates_without_panic() { -⋮---- -.http(HttpClientConfig { -⋮---- -fn client_builder_success() { -⋮---- -.base_url("https://api.openai.com/v1/chat/completions") -.api_key("sk-test") -.timeout(std::time::Duration::from_secs(5)) -.read_timeout(std::time::Duration::from_secs(30)) -.max_retries(2) -⋮---- -assert!(client.is_ok()); -⋮---- -fn client_builder_missing_url() { -let result = ClientBuilder::new().api_key("sk-test").build(); -assert!(result.is_err()); -⋮---- -fn client_builder_default_api_key() { -// No api_key set: should still build (empty bearer token). -let client = ClientBuilder::new().base_url("https://example.com").build(); -⋮---- -fn client_builder_via_client() { -let client = Client::builder().base_url("https://example.com").build(); -⋮---- -fn client_builder_default_trait() { -⋮---- -assert!(builder.base_url.is_none()); - - - -//! Rate limit header extraction, format conversion, and duration parsing. -//! -//! Converts between OpenAI-style `x-ratelimit-*` headers and Anthropic-style -//! `anthropic-ratelimit-*` headers. OpenAI uses relative durations ("1s", "500ms") -//! for reset fields; Anthropic uses ISO 8601 UTC timestamps. -⋮---- -use std::time::Duration; -⋮---- -/// Rate limit headers extracted from backend responses. -/// Forwarded to clients as Anthropic-style `anthropic-ratelimit-*` headers. -/// See: -⋮---- -pub struct RateLimitHeaders { -/// Maximum requests allowed in the current window. -⋮---- -/// Requests remaining before rate limiting kicks in. -⋮---- -/// Reset value for request limits (raw from backend). -⋮---- -/// Maximum tokens allowed in the current window. -⋮---- -/// Tokens remaining before rate limiting kicks in. -⋮---- -/// Reset value for token limits (raw from backend). -⋮---- -/// Seconds to wait before retrying (from `retry-after` header on 429s). -⋮---- -/// Anthropic organization ID from `anthropic-organization-id` response header. -⋮---- -/// Extract a header value as a trimmed string. -fn header_str(headers: &reqwest::header::HeaderMap, name: &str) -> Option { -⋮---- -.get(name) -.and_then(|v| v.to_str().ok()) -.map(|s| s.trim().to_string()) -⋮---- -impl RateLimitHeaders { -/// Extract rate limit headers from an OpenAI (or Vertex) response. -pub fn from_openai_headers(headers: &reqwest::header::HeaderMap) -> Self { -⋮---- -requests_limit: header_str(headers, "x-ratelimit-limit-requests"), -requests_remaining: header_str(headers, "x-ratelimit-remaining-requests"), -requests_reset: header_str(headers, "x-ratelimit-reset-requests"), -tokens_limit: header_str(headers, "x-ratelimit-limit-tokens"), -tokens_remaining: header_str(headers, "x-ratelimit-remaining-tokens"), -tokens_reset: header_str(headers, "x-ratelimit-reset-tokens"), -retry_after: header_str(headers, "retry-after"), -⋮---- -/// Extract rate limit headers from an Anthropic response. -/// Anthropic uses `anthropic-ratelimit-*` headers natively. -pub fn from_anthropic_headers(headers: &reqwest::header::HeaderMap) -> Self { -⋮---- -requests_limit: header_str(headers, "anthropic-ratelimit-requests-limit"), -requests_remaining: header_str(headers, "anthropic-ratelimit-requests-remaining"), -requests_reset: header_str(headers, "anthropic-ratelimit-requests-reset"), -tokens_limit: header_str(headers, "anthropic-ratelimit-tokens-limit"), -tokens_remaining: header_str(headers, "anthropic-ratelimit-tokens-remaining"), -tokens_reset: header_str(headers, "anthropic-ratelimit-tokens-reset"), -⋮---- -organization_id: header_str(headers, "anthropic-organization-id"), -⋮---- -/// Inject Anthropic-format response headers (rate limits + version) into an -/// `axum::http::HeaderMap`. The `*_reset` fields are converted from OpenAI's -/// relative duration format (e.g., "1s") to Anthropic's ISO 8601 UTC timestamp. -/// Falls back to the raw value with a warning if parsing fails. -/// -/// This method accepts generic `http::HeaderMap` (used by both reqwest and axum). -pub fn inject_anthropic_response_headers(&self, map: &mut http_types::HeaderMap) { -set_if_some( -⋮---- -let req_reset = convert_reset_duration(&self.requests_reset, "requests_reset"); -set_if_some(map, "anthropic-ratelimit-requests-reset", &req_reset); -set_if_some(map, "anthropic-ratelimit-tokens-limit", &self.tokens_limit); -⋮---- -let tok_reset = convert_reset_duration(&self.tokens_reset, "tokens_reset"); -set_if_some(map, "anthropic-ratelimit-tokens-reset", &tok_reset); -set_if_some(map, "retry-after", &self.retry_after); -set_if_some(map, "anthropic-organization-id", &self.organization_id); -map.insert( -⋮---- -// Use the http crate types that reqwest re-exports, avoiding an extra dependency. -mod http_types { -⋮---- -/// Set a header on a HeaderMap if the value is Some. -fn set_if_some(map: &mut http_types::HeaderMap, name: &str, value: &Option) { -⋮---- -http_types::HeaderName::from_bytes(name.as_bytes()), -⋮---- -map.insert(header_name, header_value); -⋮---- -/// Convert an OpenAI relative duration to ISO 8601, falling back to the raw -/// value with a warning if parsing fails. -fn convert_reset_duration(raw: &Option, field: &str) -> Option { -raw.as_deref().map(|v| { -openai_duration_to_iso8601(v).unwrap_or_else(|| { -⋮---- -v.to_string() -⋮---- -/// Convert an OpenAI relative duration string to an ISO 8601 UTC timestamp. -fn openai_duration_to_iso8601(s: &str) -> Option { -openai_duration_to_iso8601_at(s, std::time::SystemTime::now()) -⋮---- -/// Convert an OpenAI relative duration string to an ISO 8601 UTC timestamp -/// by adding it to the given anchor time. Testable variant. -pub fn openai_duration_to_iso8601_at(s: &str, anchor: std::time::SystemTime) -> Option { -let dur = parse_openai_duration(s)?; -⋮---- -.duration_since(std::time::UNIX_EPOCH) -.ok()? -.as_secs(); -Some(epoch_to_iso8601(secs)) -⋮---- -/// Parse OpenAI's duration format (e.g., "6ms", "1s", "1m30s", "2m") into a -/// [`Duration`]. Returns `None` for unrecognized formats. -pub fn parse_openai_duration(s: &str) -> Option { -let s = s.trim(); -if s.is_empty() { -⋮---- -let bytes = s.as_bytes(); -⋮---- -while i < bytes.len() { -⋮---- -if c.is_ascii_digit() || c == b'.' { -if num_start.is_none() { -num_start = Some(i); -⋮---- -} else if c.is_ascii_alphabetic() { -⋮---- -while i < bytes.len() && bytes[i].is_ascii_alphabetic() { -⋮---- -let value: f64 = num_str.parse().ok()?; -⋮---- -total_ms += ms.round() as u64; -⋮---- -// Trailing number with no unit is invalid -if num_start.is_some() { -⋮---- -Some(Duration::from_millis(total_ms)) -⋮---- -/// Convert epoch seconds to ISO 8601 UTC string (e.g., "2025-06-16T12:00:01Z"). -pub fn epoch_to_iso8601(epoch: u64) -> String { -⋮---- -let (year, month, day) = days_to_ymd(days); -⋮---- -format!( -⋮---- -/// Days since 1970-01-01 to (year, month, day). -/// Algorithm from -fn days_to_ymd(days: u64) -> (u64, u64, u64) { -⋮---- -mod tests { -⋮---- -fn from_openai_headers_extracts_all() { -⋮---- -headers.insert("x-ratelimit-limit-requests", "100".parse().unwrap()); -headers.insert("x-ratelimit-remaining-requests", "99".parse().unwrap()); -headers.insert("x-ratelimit-reset-requests", "1s".parse().unwrap()); -headers.insert("x-ratelimit-limit-tokens", "40000".parse().unwrap()); -headers.insert("x-ratelimit-remaining-tokens", "39500".parse().unwrap()); -headers.insert("x-ratelimit-reset-tokens", "500ms".parse().unwrap()); -headers.insert("retry-after", "2".parse().unwrap()); -⋮---- -assert_eq!(rl.requests_limit.as_deref(), Some("100")); -assert_eq!(rl.requests_remaining.as_deref(), Some("99")); -assert_eq!(rl.requests_reset.as_deref(), Some("1s")); -assert_eq!(rl.tokens_limit.as_deref(), Some("40000")); -assert_eq!(rl.tokens_remaining.as_deref(), Some("39500")); -assert_eq!(rl.tokens_reset.as_deref(), Some("500ms")); -assert_eq!(rl.retry_after.as_deref(), Some("2")); -⋮---- -fn from_openai_headers_missing_are_none() { -⋮---- -assert!(rl.requests_limit.is_none()); -assert!(rl.requests_remaining.is_none()); -assert!(rl.requests_reset.is_none()); -assert!(rl.tokens_limit.is_none()); -assert!(rl.tokens_remaining.is_none()); -assert!(rl.tokens_reset.is_none()); -assert!(rl.retry_after.is_none()); -⋮---- -fn inject_anthropic_response_headers_sets_values() { -⋮---- -requests_limit: Some("100".into()), -tokens_remaining: Some("39500".into()), -retry_after: Some("3".into()), -⋮---- -rl.inject_anthropic_response_headers(&mut map); -⋮---- -assert_eq!( -⋮---- -assert_eq!(map.get("retry-after").unwrap(), "3"); -assert_eq!(map.get("anthropic-version").unwrap(), "2023-06-01"); -assert!(map.get("anthropic-ratelimit-requests-remaining").is_none()); -assert!(map.get("anthropic-ratelimit-tokens-limit").is_none()); -⋮---- -fn inject_anthropic_response_headers_default_sets_version_only() { -⋮---- -assert_eq!(map.len(), 1); -⋮---- -fn inject_anthropic_response_headers_converts_reset_to_iso8601() { -⋮---- -requests_reset: Some("1s".into()), -tokens_reset: Some("500ms".into()), -⋮---- -.get("anthropic-ratelimit-requests-reset") -.unwrap() -.to_str() -.unwrap(); -⋮---- -.get("anthropic-ratelimit-tokens-reset") -⋮---- -assert!( -⋮---- -fn parse_openai_duration_various_formats() { -assert_eq!(parse_openai_duration("6ms"), Some(Duration::from_millis(6))); -⋮---- -fn parse_openai_duration_invalid() { -assert_eq!(parse_openai_duration(""), None); -assert_eq!(parse_openai_duration("abc"), None); -assert_eq!(parse_openai_duration("123"), None); -assert_eq!(parse_openai_duration("1x"), None); -⋮---- -fn openai_duration_to_iso8601_at_pinned_time() { -⋮---- -fn openai_duration_to_iso8601_invalid_returns_none() { -assert!(openai_duration_to_iso8601("garbage").is_none()); -assert!(openai_duration_to_iso8601("").is_none()); -⋮---- -fn epoch_to_iso8601_unix_epoch() { -assert_eq!(epoch_to_iso8601(0), "1970-01-01T00:00:00Z"); -⋮---- -fn from_anthropic_headers_extracts_all() { -⋮---- -headers.insert("anthropic-ratelimit-requests-limit", "100".parse().unwrap()); -headers.insert( -⋮---- -"99".parse().unwrap(), -⋮---- -"2025-01-01T00:00:00Z".parse().unwrap(), -⋮---- -headers.insert("retry-after", "5".parse().unwrap()); -⋮---- -assert_eq!(rl.requests_reset.as_deref(), Some("2025-01-01T00:00:00Z")); -assert_eq!(rl.retry_after.as_deref(), Some("5")); -assert!(rl.organization_id.is_none()); -⋮---- -fn from_anthropic_headers_parses_organization_id() { -⋮---- -"org-abc123".parse().unwrap(), -⋮---- -assert_eq!(rl.organization_id.as_deref(), Some("org-abc123")); -⋮---- -fn inject_anthropic_response_headers_sets_organization_id() { -⋮---- -organization_id: Some("org-xyz".into()), - - - -/// Token-based authentication for admin endpoints. -pub mod auth; -/// SQLite persistence for request logs and config overrides. -pub mod db; -/// Virtual API key generation, hashing, and rate limit state. -pub mod keys; -/// Admin HTTP router: config management, request log queries, metrics. -pub mod routes; -/// Per-key spend queries for cost tracking. -pub mod spend; -/// Shared mutable state between proxy handlers and admin server. -pub mod state; -/// WebSocket handler for live admin event streaming. -pub(crate) mod ws; - - - -// Passthrough client for forwarding requests to the real Anthropic API. -// No translation: receives Anthropic-format request bytes, returns Anthropic-format response. -⋮---- -use reqwest::Client; -use tokio::time::sleep; -⋮---- -/// HTTP client that forwards Anthropic requests as-is to the upstream Anthropic API. -⋮---- -pub struct AnthropicClient { -⋮---- -/// Error type for the Anthropic passthrough client. -⋮---- -pub enum AnthropicClientError { -/// Transport-level error (connection, timeout, DNS). -⋮---- -/// Upstream returned a non-success status. Body is raw bytes for passthrough. -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -Self::Transport(msg) => write!(f, "Anthropic transport error: {msg}"), -Self::ApiError { status, .. } => write!(f, "Anthropic API error (status {status})"), -⋮---- -impl AnthropicClient { -/// Create from a BackendConfig (used in multi-backend mode). -pub fn from_backend_config(bc: &BackendConfig) -> Self { -let client = build_http_client(&bc.tls); -let messages_url = format!("{}/v1/messages", bc.base_url.trim_end_matches('/')); -⋮---- -api_key: bc.api_key.clone(), -⋮---- -/// Create from raw parts (used in legacy single-backend mode). -pub fn new(base_url: &str, api_key: &str, tls: &TlsConfig) -> Self { -let client = build_http_client(tls); -let messages_url = format!("{}/v1/messages", base_url.trim_end_matches('/')); -⋮---- -api_key: api_key.to_string(), -⋮---- -/// Apply required Anthropic authentication headers. -/// x-api-key and anthropic-version are mandatory per the Anthropic API spec; -/// without the version header, the API rejects requests. -fn auth_request(&self, rb: reqwest::RequestBuilder) -> reqwest::RequestBuilder { -rb.header("x-api-key", &self.api_key) -.header("anthropic-version", "2023-06-01") -⋮---- -/// Forward a non-streaming request. Returns raw response body and rate limit headers. -/// `extra_headers` are forwarded verbatim to upstream without modification. -pub async fn forward( -⋮---- -let response = self.send_with_retry(body, false, extra_headers).await?; -let rate_limits = RateLimitHeaders::from_anthropic_headers(response.headers()); -⋮---- -.bytes() -⋮---- -.map_err(|e| AnthropicClientError::Transport(e.to_string()))?; -Ok((resp_body, rate_limits)) -⋮---- -/// Forward a streaming request. Returns the raw response for SSE piping. -⋮---- -pub async fn forward_stream( -⋮---- -let response = self.send_with_retry(body, true, extra_headers).await?; -⋮---- -Ok((response, rate_limits)) -⋮---- -/// Send with retry on 429/5xx. For passthrough, we retry the raw body bytes. -async fn send_with_retry( -⋮---- -.post(&self.messages_url) -.header("content-type", content_type) -.body(body.clone()); -let rb = self.auth_request(rb); -// Tell upstream we expect SSE format; the Anthropic routing layer -// may use this hint to optimize response handling. -⋮---- -rb.header("accept", "text/event-stream") -⋮---- -.iter() -.fold(rb, |rb, &(k, v)| rb.header(k, v)); -⋮---- -.send() -⋮---- -let status = response.status().as_u16(); -⋮---- -if (200..300).contains(&status) { -return Ok(response); -⋮---- -let retry_after = super::parse_retry_after(response.headers()); -⋮---- -// Drain body so connection returns to pool -drop(response.bytes().await); -sleep(delay).await; -⋮---- -let resp_body = response.bytes().await.unwrap_or_default(); -return Err(AnthropicClientError::ApiError { -⋮---- -unreachable!("loop runs MAX_RETRIES+1 times and always returns") - - - -// crates/proxy/src/batch/anthropic_batch.rs -// Route handlers for Anthropic native batch API. -// -// POST /v1/messages/batches — translate and submit to OpenAI batch API -// GET /v1/messages/batches/{id} — poll status -// GET /v1/messages/batches/{id}/results — download and translate output JSONL -⋮---- -use super::db; -⋮---- -use crate::backend::BackendClient; -⋮---- -use anyllm_translate::anthropic::batch::CreateBatchRequest; -use anyllm_translate::anthropic::errors::ErrorType; -⋮---- -use anyllm_translate::mapping::errors_map::create_anthropic_error; -⋮---- -/// Extract the OpenAI API key and base URL from the backend client. -/// Returns None when the backend does not support batch (Vertex, Gemini, Anthropic, Bedrock). -fn extract_openai_credentials(backend: &BackendClient) -> Option<(String, String)> { -⋮---- -| BackendClient::OpenAIResponses(c) => Some((c.api_key(), c.base_url_for_batch())), -⋮---- -/// POST /v1/messages/batches -pub(crate) async fn create_anthropic_batch( -⋮---- -if req.requests.is_empty() { -return error_response( -⋮---- -let (api_key, base_url) = match extract_openai_credentials(&state.backend) { -⋮---- -let db = match state.shared.as_ref().map(|s| s.db.clone()) { -⋮---- -// Derive model name from first request (all should use the same model after mapping). -let model = req.requests[0].params.model.clone(); -⋮---- -// Translate Anthropic JSONL to OpenAI JSONL. -let openai_jsonl = translate_batch_to_openai_jsonl(&req.requests); -⋮---- -// Upload translated JSONL to OpenAI. -let openai_file_id = match batch_client.upload_jsonl_file(&openai_jsonl).await { -⋮---- -return error_response(StatusCode::BAD_GATEWAY, ErrorType::ApiError, &e); -⋮---- -// Create OpenAI batch job. -let openai_batch_id = match batch_client.create_batch(&openai_file_id).await { -⋮---- -// Generate our Anthropic-format batch ID. -let our_batch_id = format!("msgbatch_{}", uuid::Uuid::new_v4().as_simple()); -⋮---- -// Store the mapping in SQLite. -let our_id = our_batch_id.clone(); -let oai_id = openai_batch_id.clone(); -let model_clone = model.clone(); -⋮---- -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -// Store model alongside mapping for result translation. -conn.execute( -⋮---- -// Poll OpenAI to get the initial batch status and translate it. -match batch_client.get_batch_status(&openai_batch_id).await { -⋮---- -let batch = openai_batch_to_message_batch(&our_batch_id, &v); -(StatusCode::OK, Json(batch)).into_response() -⋮---- -// Return a synthetic in-progress response — the batch was submitted successfully. -⋮---- -.duration_since(std::time::UNIX_EPOCH) -.unwrap_or_default() -.as_secs() as i64; -⋮---- -type_: "message_batch".to_string(), -⋮---- -processing: req.requests.len() as u32, -⋮---- -/// GET /v1/messages/batches/{id} -pub(crate) async fn get_anthropic_batch( -⋮---- -let batch_id_clone = batch_id.clone(); -⋮---- -match batch_client.get_batch_status(&map.openai_batch_id).await { -⋮---- -let batch = openai_batch_to_message_batch(&batch_id, &v); -⋮---- -error_response(StatusCode::BAD_GATEWAY, ErrorType::ApiError, &e) -⋮---- -/// GET /v1/messages/batches/{id}/results -/// -/// Downloads the OpenAI output JSONL and streams Anthropic-format result lines. -pub(crate) async fn get_anthropic_batch_results( -⋮---- -// Determine output file id: use cached one or poll OpenAI. -⋮---- -let status = match batch_client.get_batch_status(&map.openai_batch_id).await { -⋮---- -match status["output_file_id"].as_str() { -Some(fid) => fid.to_string(), -⋮---- -// Download the output JSONL from OpenAI. -let openai_jsonl = match batch_client.get_file_content(&output_file_id).await { -⋮---- -// Translate each line to Anthropic format. -let model = if map.model.is_empty() { -⋮---- -.lines() -.filter(|l| !l.trim().is_empty()) -.map(|line| { -translate_openai_result_line(line, model).unwrap_or_else(|e| { -⋮---- -// Emit an errored result for lines that fail translation. -⋮---- -.to_string() -⋮---- -.collect(); -⋮---- -let body = anthropic_lines.join("\n"); -⋮---- -.status(StatusCode::OK) -.header("content-type", "application/x-jsonl") -.body(axum::body::Body::from(body)) -.unwrap() -.into_response() -⋮---- -fn error_response(status: StatusCode, error_type: ErrorType, msg: &str) -> Response { -let err = create_anthropic_error(error_type, msg.to_string(), None); -(status, Json(err)).into_response() - - - -//! Response caching for non-streaming requests. -//! -//! Cache keys are SHA-256 hashes of the canonical (sorted-key) JSON of -//! request fields that affect the response: model, messages, temperature, -//! top_p, max_tokens, stop, tools, tool_choice. -⋮---- -//! Two namespaces avoid cross-endpoint collisions: -//! - `anth:` for /v1/messages -//! - `oai:` for /v1/chat/completions -⋮---- -pub mod memory; -/// Redis L2 cache backend (requires `redis` feature). -pub mod redis; -/// Semantic cache backed by Qdrant vector store (requires `qdrant` feature). -⋮---- -pub mod semantic; -⋮---- -use bytes::Bytes; -⋮---- -use std::collections::BTreeMap; -use std::time::Instant; -⋮---- -/// Maximum allowed value for per-request `cache_ttl_secs`. -⋮---- -/// Cached response entry stored in any cache backend. -⋮---- -pub struct CacheEntry { -/// Serialized response body (JSON bytes). -⋮---- -/// Model name from the response, for diagnostics/logging. -⋮---- -/// When this entry was created (wall-clock, not persisted to Redis). -⋮---- -/// Per-entry TTL override in seconds. When set, moka's Expiry trait -/// uses this instead of the cache-level default. -⋮---- -/// Namespace prefix for cache keys, preventing cross-endpoint collisions. -⋮---- -pub enum CacheNamespace { -/// Anthropic /v1/messages endpoint. -⋮---- -/// OpenAI /v1/chat/completions endpoint. -⋮---- -impl CacheNamespace { -fn prefix(self) -> &'static str { -⋮---- -/// Pluggable cache backend trait. Implementations must be Send + Sync -/// for use behind Arc in axum handlers. -pub trait CacheBackend: Send + Sync { -/// Look up a cached response by key. Returns None on miss. -⋮---- -/// Store a response in the cache with the given TTL. -⋮---- -/// Compute a deterministic cache key for a request body. -/// -/// Extracts the fields that affect response content, sorts them via BTreeMap, -/// serializes to canonical JSON, SHA-256 hashes the result, and prepends the -/// namespace prefix. -pub fn cache_key_for_request(body: &serde_json::Value, ns: CacheNamespace) -> String { -// Fields that affect the backend response. Order does not matter because -// BTreeMap sorts keys alphabetically before serialization. -⋮---- -if let Some(obj) = body.as_object() { -⋮---- -if let Some(val) = obj.get(field) { -// Skip null values so absent fields and explicit null produce the same key. -if !val.is_null() { -canonical.insert(field, val.clone()); -⋮---- -// serde_json serializes BTreeMap in key order, giving us canonical JSON. -let json = serde_json::to_string(&canonical).unwrap_or_default(); -let hash = Sha256::digest(json.as_bytes()); -⋮---- -format!("{}:{}", ns.prefix(), hex) -⋮---- -/// Parse the optional `cache_ttl_secs` field from a request body. -⋮---- -/// Returns: -/// - `Ok(None)` if the field is absent or null (use default TTL). -/// - `Ok(Some(0))` if explicitly 0 (bypass cache). -/// - `Ok(Some(n))` for valid positive values up to MAX_TTL_SECS. -/// - `Err(message)` for negative values, values > MAX_TTL_SECS, or non-numeric. -pub fn parse_cache_ttl(body: &serde_json::Value) -> Result, String> { -let Some(val) = body.get("cache_ttl_secs") else { -return Ok(None); -⋮---- -if val.is_null() { -⋮---- -if let Some(n) = val.as_u64() { -⋮---- -return Err(format!("cache_ttl_secs must be <= {MAX_TTL_SECS}, got {n}")); -⋮---- -return Ok(Some(n)); -⋮---- -if let Some(n) = val.as_i64() { -// Negative values are invalid -return Err(format!("cache_ttl_secs must be non-negative, got {n}")); -⋮---- -if let Some(n) = val.as_f64() { -⋮---- -return Err(format!( -⋮---- -return Ok(Some(truncated)); -⋮---- -Err(format!("cache_ttl_secs must be a number, got {}", val)) -⋮---- -/// Configuration for the cache subsystem. -⋮---- -pub struct CacheConfig { -/// Default TTL in seconds for cached responses. -⋮---- -/// Maximum number of entries in the in-memory cache. -⋮---- -/// Optional Redis URL. Used by the Redis L2 cache backend (requires `redis` feature) -/// and distributed rate limiting. When set, responses are cached in Redis in addition -/// to the in-memory L1 cache, and rate limit state is shared across proxy instances. -⋮---- -impl Default for CacheConfig { -fn default() -> Self { -⋮---- -impl CacheConfig { -/// Load cache configuration from environment variables. -pub fn from_env() -> Self { -⋮---- -.ok() -.and_then(|v| v.parse().ok()) -.unwrap_or(300); -⋮---- -.unwrap_or(10_000); -let redis_url = std::env::var("REDIS_URL").ok(); -⋮---- -mod tests { -⋮---- -fn cache_key_deterministic_same_fields() { -⋮---- -let key1 = cache_key_for_request(&body, CacheNamespace::Anthropic); -let key2 = cache_key_for_request(&body, CacheNamespace::Anthropic); -assert_eq!(key1, key2); -assert!(key1.starts_with("anth:")); -⋮---- -fn cache_key_different_for_different_temperature() { -⋮---- -let key1 = cache_key_for_request(&body1, CacheNamespace::Anthropic); -let key2 = cache_key_for_request(&body2, CacheNamespace::Anthropic); -assert_ne!(key1, key2); -⋮---- -fn cache_key_ignores_field_order() { -// JSON object field order should not affect the key because we -// extract into a BTreeMap. -⋮---- -let key1 = cache_key_for_request(&body1, CacheNamespace::OpenAI); -let key2 = cache_key_for_request(&body2, CacheNamespace::OpenAI); -⋮---- -fn cache_key_ignores_non_cache_fields() { -⋮---- -fn cache_key_namespace_differs() { -⋮---- -let anth = cache_key_for_request(&body, CacheNamespace::Anthropic); -let oai = cache_key_for_request(&body, CacheNamespace::OpenAI); -assert_ne!(anth, oai); -assert!(anth.starts_with("anth:")); -assert!(oai.starts_with("oai:")); -⋮---- -fn cache_key_null_field_same_as_absent() { -⋮---- -fn parse_cache_ttl_absent() { -⋮---- -assert_eq!(parse_cache_ttl(&body).unwrap(), None); -⋮---- -fn parse_cache_ttl_null() { -⋮---- -fn parse_cache_ttl_zero() { -⋮---- -assert_eq!(parse_cache_ttl(&body).unwrap(), Some(0)); -⋮---- -fn parse_cache_ttl_valid() { -⋮---- -assert_eq!(parse_cache_ttl(&body).unwrap(), Some(600)); -⋮---- -fn parse_cache_ttl_max() { -⋮---- -assert_eq!(parse_cache_ttl(&body).unwrap(), Some(86400)); -⋮---- -fn parse_cache_ttl_over_max() { -⋮---- -assert!(parse_cache_ttl(&body).is_err()); -⋮---- -fn parse_cache_ttl_negative() { -⋮---- -fn parse_cache_ttl_string() { - - - -/// Model-level routing table for LiteLLM-style model_list configs. -/// -/// Maps virtual model names to one or more backend deployments. -/// Uses lock-free atomics for round-robin counters and approximate -/// RPM/TPM tracking (60-second tumbling windows). -⋮---- -/// Supports multiple routing strategies: round-robin (default), -/// least-busy (lowest in-flight), latency-based (lowest EWMA), -/// weighted round-robin, and cost-based (lowest price-per-token). -use std::collections::HashMap; -⋮---- -use std::sync::Arc; -⋮---- -/// Routing strategy for selecting among multiple deployments. -⋮---- -pub enum RoutingStrategy { -/// Round-robin with RPM-aware skip (default, existing behavior). -⋮---- -/// Pick deployment with lowest in-flight request count. -⋮---- -/// Pick deployment with lowest latency EWMA. -⋮---- -/// Weighted round-robin using per-deployment weight field. -⋮---- -/// Pick deployment with lowest cost per token from the bundled model pricing table. -/// Falls back to round-robin if none of the deployments have known pricing. -⋮---- -/// A single backend deployment that can serve a model name. -pub struct Deployment { -/// Key into MultiConfig.backends. -⋮---- -/// Model name to send to the backend (the actual provider model). -⋮---- -/// Per-deployment requests-per-minute limit (from LiteLLM config). -⋮---- -/// Per-deployment tokens-per-minute limit (from LiteLLM config). -⋮---- -/// Static weight for weighted routing (default 1). -⋮---- -// Approximate 60s tumbling window counters. -⋮---- -// Tracking for least-busy and latency-based routing. -⋮---- -/// Exponentially-weighted moving average of response latency in ms. -⋮---- -impl Deployment { -pub fn new( -⋮---- -pub fn with_weight( -⋮---- -weight: weight.max(1), // floor at 1 -⋮---- -window_start_ms: AtomicU64::new(now_ms()), -⋮---- -/// Check and reset the window if >60s have elapsed. Returns true if reset occurred. -fn maybe_reset_window(&self) -> bool { -let now = now_ms(); -let start = self.window_start_ms.load(Ordering::Relaxed); -if now.saturating_sub(start) > 60_000 { -// CAS to avoid double-reset from concurrent callers. -⋮---- -.compare_exchange(start, now, Ordering::Relaxed, Ordering::Relaxed) -.is_ok() -⋮---- -self.rpm_used.store(0, Ordering::Relaxed); -self.tpm_used.store(0, Ordering::Relaxed); -⋮---- -/// Returns true if this deployment is under its RPM limit (or has no limit). -fn under_rpm_limit(&self) -> bool { -self.maybe_reset_window(); -⋮---- -Some(limit) => self.rpm_used.load(Ordering::Relaxed) < limit, -⋮---- -/// Increment RPM counter. Called when a request is routed here. -fn record_request(&self) { -self.rpm_used.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Increment TPM counter. Called after response with actual token count. -pub fn record_tokens(&self, tokens: u64) { -self.tpm_used.fetch_add(tokens, Ordering::Relaxed); -⋮---- -/// Mark a request as dispatched. Call before sending to backend. -pub fn record_start(&self) { -self.in_flight.fetch_add(1, Ordering::Relaxed); -⋮---- -/// Mark a request as completed. Updates in-flight count and latency EWMA. -/// Call after response (or error) with wall-clock elapsed ms. -pub fn record_finish(&self, latency_ms: u64) { -self.in_flight.fetch_sub(1, Ordering::Relaxed); -// EWMA with alpha=0.3: new = 0.3 * sample + 0.7 * old. -// CAS loop for lock-free update. Approximate is fine. -⋮---- -let old = self.latency_ewma_ms.load(Ordering::Relaxed); -⋮---- -.compare_exchange(old, new_val, Ordering::Relaxed, Ordering::Relaxed) -⋮---- -/// Current in-flight request count. -pub fn in_flight_count(&self) -> u32 { -self.in_flight.load(Ordering::Relaxed) -⋮---- -/// Current latency EWMA in ms. -pub fn latency_ms(&self) -> u64 { -self.latency_ewma_ms.load(Ordering::Relaxed) -⋮---- -/// Result of a routing decision. -pub struct RoutedDeployment<'a> { -⋮---- -/// Maps virtual model names to backend deployments with configurable routing. -pub struct ModelRouter { -/// model_name -> list of deployments (order = config order). -⋮---- -/// Round-robin counters per model name (used by RoundRobin and Weighted). -⋮---- -/// Routing strategy applied to all models. -⋮---- -impl ModelRouter { -pub fn new(routes: HashMap>>) -> Self { -⋮---- -pub fn with_strategy( -⋮---- -.keys() -.map(|k| (k.clone(), AtomicUsize::new(0))) -.collect(); -⋮---- -/// Pick the next available deployment for a model name. -⋮---- -/// Dispatches to the configured routing strategy. All strategies -/// skip deployments that are at their RPM limit. -/// Returns None if the model is unknown or all deployments are at limit. -pub fn route(&self, model_name: &str) -> Option> { -⋮---- -RoutingStrategy::RoundRobin => self.route_round_robin(model_name), -RoutingStrategy::LeastBusy => self.route_least_busy(model_name), -RoutingStrategy::LatencyBased => self.route_latency_based(model_name), -RoutingStrategy::Weighted => self.route_weighted(model_name), -RoutingStrategy::CostBased => self.route_cost_based(model_name), -⋮---- -/// Round-robin with RPM-aware skip. -fn route_round_robin(&self, model_name: &str) -> Option> { -let deployments = self.routes.get(model_name)?; -let counter = self.counters.get(model_name)?; -let len = deployments.len(); -⋮---- -let start = counter.fetch_add(1, Ordering::Relaxed) % len; -⋮---- -if d.under_rpm_limit() { -d.record_request(); -return Some(RoutedDeployment { -⋮---- -/// Pick deployment with lowest in-flight count (ties broken by config order). -fn route_least_busy(&self, model_name: &str) -> Option> { -⋮---- -if deployments.is_empty() { -⋮---- -for (i, d) in deployments.iter().enumerate() { -if !d.under_rpm_limit() { -⋮---- -let count = d.in_flight_count(); -if best.is_none() || count < best.unwrap().1 { -best = Some((i, count)); -⋮---- -best.map(|(idx, _)| { -⋮---- -/// Pick deployment with lowest latency EWMA. Zero (no data yet) is naturally -/// the minimum, so unknown deployments get tried first for warmup. -fn route_latency_based(&self, model_name: &str) -> Option> { -⋮---- -let lat = d.latency_ms(); -if best.is_none() || lat < best.unwrap().1 { -best = Some((i, lat)); -⋮---- -/// Weighted round-robin. Deployments with weight=3 get 3x traffic vs weight=1. -/// Uses a virtual counter that expands by total weight per cycle. -fn route_weighted(&self, model_name: &str) -> Option> { -⋮---- -// Build expanded index: deployment i appears weight[i] times. -let total_weight: usize = deployments.iter().map(|d| d.weight as usize).sum(); -⋮---- -let tick = counter.fetch_add(1, Ordering::Relaxed) % total_weight; -⋮---- -// Find which deployment this tick maps to. -⋮---- -// Try starting at the weighted pick, then scan others if RPM-limited. -⋮---- -/// Pick deployment with the lowest combined cost per token (input + output). -⋮---- -/// Uses the global model pricing table. Deployments with unknown pricing are -/// treated as having infinite cost and are skipped in favour of priced ones. -/// If no deployment has known pricing, falls back to round-robin. -fn route_cost_based(&self, model_name: &str) -> Option> { -⋮---- -if let Some((input, output)) = pricing.price_for_model(&d.actual_model) { -⋮---- -if best.is_none() || score < best.unwrap().1 { -best = Some((i, score)); -⋮---- -// No deployment has known pricing; fall back to round-robin. -⋮---- -return self.route_round_robin(model_name); -⋮---- -/// Check if a model name exists in the routing table. -pub fn has_model(&self, model_name: &str) -> bool { -self.routes.contains_key(model_name) -⋮---- -/// Return all known model names (for /v1/models enrichment). -pub fn known_models(&self) -> Vec<&str> { -self.routes.keys().map(|s| s.as_str()).collect() -⋮---- -/// Current routing strategy. -pub fn strategy(&self) -> RoutingStrategy { -⋮---- -/// Add a deployment for a model name (for dynamic model management). -pub fn add_deployment(&mut self, model_name: String, deployment: Arc) { -let deps = self.routes.entry(model_name.clone()).or_default(); -deps.push(deployment); -⋮---- -.entry(model_name) -.or_insert_with(|| AtomicUsize::new(0)); -⋮---- -/// Remove all deployments for a model name. Returns true if the model existed. -pub fn remove_model(&mut self, model_name: &str) -> bool { -let removed = self.routes.remove(model_name).is_some(); -self.counters.remove(model_name); -⋮---- -/// List all models with their deployment counts (for admin API). -pub fn list_models(&self) -> Vec<(&str, usize)> { -⋮---- -.iter() -.map(|(name, deps)| (name.as_str(), deps.len())) -.collect() -⋮---- -fn now_ms() -> u64 { -⋮---- -mod tests { -⋮---- -fn make_deployments(specs: &[(&str, &str, Option)]) -> Vec> { -⋮---- -.map(|(backend, model, rpm)| { -⋮---- -backend.to_string(), -model.to_string(), -⋮---- -fn make_weighted(specs: &[(&str, &str, u32)]) -> Vec> { -⋮---- -.map(|(backend, model, weight)| { -⋮---- -fn round_robin_across_deployments() { -let deps = make_deployments(&[ -⋮---- -routes.insert("gpt-4o".to_string(), deps); -⋮---- -let r0 = router.route("gpt-4o").unwrap(); -let r1 = router.route("gpt-4o").unwrap(); -let r2 = router.route("gpt-4o").unwrap(); -let r3 = router.route("gpt-4o").unwrap(); -⋮---- -// Should cycle through all three backends -assert_eq!(r0.backend_name, "azure_0"); -assert_eq!(r1.backend_name, "openai_0"); -assert_eq!(r2.backend_name, "azure_1"); -assert_eq!(r3.backend_name, "azure_0"); // wraps around -⋮---- -fn rpm_aware_skip() { -⋮---- -("backend_a", "model-x", Some(2)), -("backend_b", "model-x", None), // unlimited -⋮---- -routes.insert("model-x".to_string(), deps); -⋮---- -// Round-robin: 0->a, 1->b, 2->a, 3->b (all under limit so far) -let r0 = router.route("model-x").unwrap(); -assert_eq!(r0.backend_name, "backend_a"); -let r1 = router.route("model-x").unwrap(); -assert_eq!(r1.backend_name, "backend_b"); -let r2 = router.route("model-x").unwrap(); -assert_eq!(r2.backend_name, "backend_a"); // backend_a now at limit (2 requests) -let r3 = router.route("model-x").unwrap(); -assert_eq!(r3.backend_name, "backend_b"); // normal round-robin -⋮---- -// Request 4 would go to backend_a (index 0) but it's at limit, skip to backend_b -let r4 = router.route("model-x").unwrap(); -assert_eq!(r4.backend_name, "backend_b"); -⋮---- -fn all_at_limit_returns_none() { -let deps = make_deployments(&[("only", "m", Some(1))]); -⋮---- -routes.insert("m".to_string(), deps); -⋮---- -assert!(router.route("m").is_some()); // first request ok -assert!(router.route("m").is_none()); // at limit -⋮---- -fn unknown_model_returns_none() { -⋮---- -assert!(router.route("nonexistent").is_none()); -⋮---- -fn has_model_check() { -let deps = make_deployments(&[("b", "m", None)]); -⋮---- -assert!(router.has_model("gpt-4o")); -assert!(!router.has_model("gpt-3.5")); -⋮---- -fn single_deployment() { -let deps = make_deployments(&[("sole", "the-model", None)]); -⋮---- -routes.insert("alias".to_string(), deps); -⋮---- -let r = router.route("alias").unwrap(); -assert_eq!(r.backend_name, "sole"); -assert_eq!(r.actual_model, "the-model"); -⋮---- -fn known_models_returns_all() { -⋮---- -routes.insert("gpt-4o".to_string(), make_deployments(&[("b", "m", None)])); -routes.insert( -"claude-3".to_string(), -make_deployments(&[("b", "m", None)]), -⋮---- -let mut models = router.known_models(); -models.sort(); -assert_eq!(models, vec!["claude-3", "gpt-4o"]); -⋮---- -// ---- Least-busy strategy tests ---- -⋮---- -fn least_busy_picks_lowest_in_flight() { -let deps = make_deployments(&[("a", "m", None), ("b", "m", None), ("c", "m", None)]); -// Simulate: a has 5 in-flight, b has 1, c has 3. -deps[0].in_flight.store(5, Ordering::Relaxed); -deps[1].in_flight.store(1, Ordering::Relaxed); -deps[2].in_flight.store(3, Ordering::Relaxed); -⋮---- -let r = router.route("m").unwrap(); -assert_eq!(r.backend_name, "b"); -⋮---- -fn least_busy_skips_rpm_limited() { -let deps = make_deployments(&[("a", "m", Some(1)), ("b", "m", None)]); -deps[1].in_flight.store(100, Ordering::Relaxed); -⋮---- -// First request goes to a (lowest in-flight=0) -let r0 = router.route("m").unwrap(); -assert_eq!(r0.backend_name, "a"); -// a is now at RPM limit (1), next goes to b despite high in-flight -let r1 = router.route("m").unwrap(); -assert_eq!(r1.backend_name, "b"); -⋮---- -// ---- Latency-based strategy tests ---- -⋮---- -fn latency_based_picks_lowest_latency() { -⋮---- -deps[0].latency_ewma_ms.store(50, Ordering::Relaxed); -deps[1].latency_ewma_ms.store(500, Ordering::Relaxed); -deps[2].latency_ewma_ms.store(200, Ordering::Relaxed); -⋮---- -assert_eq!(r.backend_name, "fast"); -⋮---- -fn latency_based_prefers_unknown_for_warmup() { -let deps = make_deployments(&[("known", "m", None), ("unknown", "m", None)]); -deps[0].latency_ewma_ms.store(100, Ordering::Relaxed); -// deps[1] stays at 0 (unknown) -⋮---- -assert_eq!(r.backend_name, "unknown"); // prefer unknown to warm it up -⋮---- -// ---- Weighted strategy tests ---- -⋮---- -fn weighted_distributes_by_weight() { -let deps = make_weighted(&[("heavy", "m", 3), ("light", "m", 1)]); -⋮---- -// Over 4 requests (total weight=4): heavy gets 3, light gets 1. -⋮---- -*counts.entry(r.backend_name).or_default() += 1; -⋮---- -assert_eq!(counts["heavy"], 3); -assert_eq!(counts["light"], 1); -⋮---- -fn weighted_falls_back_when_rpm_limited() { -let deps = vec![ -⋮---- -Some(1), // rpm limit of 1 -⋮---- -// First request hits heavy -⋮---- -assert_eq!(r0.backend_name, "heavy"); -// Heavy is now at RPM limit; remaining 3 ticks all fall to light -⋮---- -assert_eq!(r1.backend_name, "light"); -let r2 = router.route("m").unwrap(); -assert_eq!(r2.backend_name, "light"); -⋮---- -// ---- record_start / record_finish tests ---- -⋮---- -fn in_flight_tracking() { -let d = Deployment::new("b".into(), "m".into(), None, None); -assert_eq!(d.in_flight_count(), 0); -⋮---- -d.record_start(); -⋮---- -assert_eq!(d.in_flight_count(), 2); -⋮---- -d.record_finish(100); -assert_eq!(d.in_flight_count(), 1); -⋮---- -d.record_finish(200); -⋮---- -fn latency_ewma_converges() { -⋮---- -assert_eq!(d.latency_ms(), 0); -⋮---- -// First sample sets the EWMA directly. -⋮---- -assert_eq!(d.latency_ms(), 100); -⋮---- -// Second sample: 0.3 * 200 + 0.7 * 100 = 60 + 70 = 130. -d.record_start(); // increment to avoid underflow -⋮---- -assert_eq!(d.latency_ms(), 130); -⋮---- -// ---- Cost-based strategy tests ---- -⋮---- -fn cost_based_picks_cheapest_model() { -// gpt-4o-mini is cheaper than gpt-4o (both are in the bundled pricing table). -let deps = make_deployments(&[("expensive", "gpt-4o", None), ("cheap", "gpt-4o-mini", None)]); -⋮---- -routes.insert("my-model".to_string(), deps); -⋮---- -// Should always pick the cheaper deployment. -⋮---- -let r = router.route("my-model").unwrap(); -assert_eq!(r.backend_name, "cheap"); -⋮---- -fn cost_based_skips_rpm_limited() { -⋮---- -("cheap-limited", "gpt-4o-mini", Some(1)), -⋮---- -// First request: cheap-limited is available and cheapest. -⋮---- -assert_eq!(r0.backend_name, "cheap-limited"); -// cheap-limited now at RPM limit; must use expensive-open. -⋮---- -assert_eq!(r1.backend_name, "expensive-open"); -⋮---- -fn cost_based_falls_back_to_round_robin_for_unknown_models() { -// Unknown model names have no pricing entry; should fall back to round-robin. -let deps = make_deployments(&[("a", "no-such-model-xyz", None), ("b", "no-such-model-xyz", None)]); -⋮---- -// Should not panic and should return some deployment. -⋮---- -// Round-robin order: a, b. -⋮---- -// ---- Mutation method tests ---- -⋮---- -fn add_deployment_to_existing_model() { -⋮---- -let d = Arc::new(Deployment::new("b1".into(), "m1".into(), None, None)); -router.add_deployment("my-model".to_string(), d); -⋮---- -assert!(router.has_model("my-model")); -⋮---- -assert_eq!(r.backend_name, "b1"); -⋮---- -fn remove_model_works() { -⋮---- -routes.insert("x".to_string(), deps); -⋮---- -assert!(router.has_model("x")); -assert!(router.remove_model("x")); -assert!(!router.has_model("x")); -assert!(!router.remove_model("x")); // idempotent -⋮---- -fn list_models_reports_counts() { -⋮---- -"a".to_string(), -make_deployments(&[("b1", "m", None), ("b2", "m", None)]), -⋮---- -routes.insert("b".to_string(), make_deployments(&[("b1", "m", None)])); -⋮---- -let mut list = router.list_models(); -list.sort_by_key(|(name, _)| *name); -assert_eq!(list, vec![("a", 2), ("b", 1)]); - - - -// SQLite persistence for per-key cost tracking. -// -// Accumulates spend and token counts on the virtual_api_key table. -// Reads are used by the admin spend endpoint. -⋮---- -/// Increment spend counters for a virtual key in SQLite. -pub fn accumulate_spend( -⋮---- -conn.execute( -⋮---- -params![cost_usd, input_tokens as i64, output_tokens as i64, key_id], -⋮---- -Ok(()) -⋮---- -/// Atomically reset the period budget in SQLite when a new budget period begins. -/// Called when `check_and_reset_period` triggers a rollover; must run before -/// `accumulate_spend` so the running total starts from zero for the new period. -pub fn reset_period_spend( -⋮---- -params![new_period_start, key_id], -⋮---- -/// Per-key spend summary returned by the admin endpoint. -⋮---- -pub struct KeySpend { -⋮---- -/// Fetch spend data for a single virtual key. -pub fn get_key_spend(conn: &Connection, key_id: i64) -> rusqlite::Result> { -let mut stmt = conn.prepare( -⋮---- -let mut rows = stmt.query_map(params![key_id], |row| { -Ok(KeySpend { -key_id: row.get(0)?, -key_prefix: row.get(1)?, -total_cost_usd: row.get::<_, f64>(2).unwrap_or(0.0), -total_input_tokens: row.get::<_, i64>(3).unwrap_or(0), -total_output_tokens: row.get::<_, i64>(4).unwrap_or(0), -request_count: row.get::<_, i64>(5).unwrap_or(0), -period_cost_usd: row.get::<_, f64>(6).unwrap_or(0.0), -max_budget_usd: row.get::<_, Option>(7).unwrap_or(None), -period_start: row.get::<_, Option>(8).unwrap_or(None), -budget_duration: row.get::<_, Option>(9).unwrap_or(None), -⋮---- -rows.next().transpose() -⋮---- -mod tests { -⋮---- -fn test_db() -> Connection { -let conn = Connection::open_in_memory().unwrap(); -crate::admin::db::init_db(&conn).unwrap(); -⋮---- -fn accumulate_spend_increments_totals() { -let conn = test_db(); -// Insert a key -⋮---- -description: Some("test"), -⋮---- -spend_limit: Some(100.0), -⋮---- -max_budget_usd: Some(100.0), -⋮---- -.unwrap(); -⋮---- -// Accumulate first request -accumulate_spend(&conn, id, 0.05, 1000, 500).unwrap(); -⋮---- -let spend = get_key_spend(&conn, id).unwrap().unwrap(); -assert_eq!(spend.key_id, id); -assert!((spend.total_cost_usd - 0.05).abs() < 1e-10); -assert_eq!(spend.total_input_tokens, 1000); -assert_eq!(spend.total_output_tokens, 500); -assert_eq!(spend.request_count, 1); -⋮---- -// Accumulate second request -accumulate_spend(&conn, id, 0.03, 800, 200).unwrap(); -⋮---- -assert!((spend.total_cost_usd - 0.08).abs() < 1e-10); -assert_eq!(spend.total_input_tokens, 1800); -assert_eq!(spend.total_output_tokens, 700); -assert_eq!(spend.request_count, 2); -assert!((spend.period_cost_usd - 0.08).abs() < 1e-10); -assert!((spend.max_budget_usd.unwrap() - 100.0).abs() < 1e-10); -⋮---- -fn get_key_spend_not_found() { -⋮---- -let result = get_key_spend(&conn, 9999).unwrap(); -assert!(result.is_none()); -⋮---- -fn reset_period_spend_zeroes_and_updates_start() { -⋮---- -description: Some("period-reset-test"), -⋮---- -max_budget_usd: Some(10.0), -budget_duration: Some("monthly"), -⋮---- -// Accumulate spend in the old period -accumulate_spend(&conn, id, 7.50, 1000, 500).unwrap(); -⋮---- -assert!((spend.period_cost_usd - 7.50).abs() < 1e-10); -⋮---- -// Simulate period rollover -reset_period_spend(&conn, id, "2026-04-01T00:00:00Z").unwrap(); -⋮---- -assert!((spend.period_cost_usd - 0.0).abs() < 1e-10); -assert_eq!(spend.period_start.as_deref(), Some("2026-04-01T00:00:00Z")); -⋮---- -// Accumulate in the new period: must be 1.25, NOT 7.50 + 1.25 -accumulate_spend(&conn, id, 1.25, 200, 100).unwrap(); -⋮---- -assert!((spend.period_cost_usd - 1.25).abs() < 1e-10); -// total_spend is cumulative across periods -assert!((spend.total_cost_usd - 8.75).abs() < 1e-10); - - - -// Named integration registry. -// -// Each named integration is initialized at startup and called -// in the same fire-and-forget path as webhook URL callbacks. -⋮---- -pub mod langfuse; -⋮---- -pub use langfuse::LangfuseClient; -⋮---- -/// A named (non-URL) callback integration. -⋮---- -pub enum NamedIntegration { -⋮---- -impl NamedIntegration { -/// Send a request log entry to the integration. Fire-and-forget. -pub fn notify(&self, entry: &crate::admin::state::RequestLogEntry) { -⋮---- -NamedIntegration::Langfuse(client) => client.send(entry), -⋮---- -mod tests { -⋮---- -fn named_integration_dispatches() { -// Smoke test: NamedIntegration enum compiles and notify() is callable. -// Actual LangfuseClient behavior is tested in integrations::langfuse::tests. - - - -// Token counting endpoint and helpers. -⋮---- -use anyllm_translate::anthropic; -⋮---- -use std::sync::LazyLock; -use tiktoken_rs::CoreBPE; -⋮---- -use super::routes::AnthropicJson; -⋮---- -/// GPT-4o tokenizer (o200k_base), the closest available approximation to -/// Anthropic's tokenizer. This endpoint is inherently approximate since we -/// use tiktoken, not the real Anthropic tokenizer. -⋮---- -LazyLock::new(|| tiktoken_rs::o200k_base().expect("failed to load o200k_base tokenizer")); -⋮---- -pub(crate) async fn count_tokens( -⋮---- -// Offload to blocking threadpool: tokenization is CPU-intensive and -// would stall the async runtime, blocking other request handlers. -match tokio::task::spawn_blocking(move || count_request_tokens(&body)).await { -⋮---- -Json(serde_json::json!({ "input_tokens": token_count })), -⋮---- -.into_response(); -// Token counts use o200k_base (GPT-4o) which may differ significantly -// from the target model's tokenizer, especially for CJK text. -resp.headers_mut().insert( -⋮---- -Json(serde_json::json!({ "error": "token counting failed" })), -⋮---- -.into_response(), -⋮---- -/// Count tokens across all text segments of an Anthropic request. -/// Counts each segment independently to avoid a single large concatenation. -/// Per-segment counting may differ slightly from concatenated counting at BPE -/// boundaries, but this endpoint is already approximate (tiktoken, not the real -/// Anthropic tokenizer). -fn count_request_tokens(req: &anthropic::MessageCreateRequest) -> usize { -⋮---- -anthropic::System::Text(t) => total += count_segment(t), -⋮---- -total += count_segment(&b.text); -⋮---- -total += count_content(&msg.content); -⋮---- -total += count_segment(&tool.name); -⋮---- -total += count_segment(desc); -⋮---- -total += count_segment(&schema); -⋮---- -/// Tokenize a single text segment and return its token count. -fn count_segment(text: &str) -> usize { -TOKENIZER.encode_with_special_tokens(text).len() -⋮---- -fn count_content(content: &anthropic::Content) -> usize { -⋮---- -anthropic::Content::Text(t) => count_segment(t), -⋮---- -anthropic::ContentBlock::Text { text } => total += count_segment(text), -⋮---- -total += count_segment(name); -⋮---- -total += count_segment(&s); -⋮---- -// The translation layer prepends "Error: " for error -// tool results (message_map.rs), so count that prefix. -if *is_error == Some(true) { -total += count_segment("Error: "); -⋮---- -total += count_segment(t); -⋮---- -total += count_segment(text); -⋮---- -total += count_segment(thinking); -⋮---- -// Images and documents have their own token costs in -// the actual APIs, which we can't compute client-side. - - - -// Integration tests for batch processing endpoints (T026-T036). -// Tests file upload, batch creation, status retrieval, listing, and 501 on unsupported backends. -⋮---- -use anyllm_proxy::admin; -⋮---- -use anyllm_proxy::server::routes; -⋮---- -fn test_config() -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: "https://api.openai.com".to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: config::BackendAuth::BearerToken("test-key".into()), -⋮---- -/// Spawn a test server with SharedState (needed for batch DB access). -async fn spawn_test_server_with_shared() -> String { -⋮---- -let config = test_config(); -⋮---- -// Initialize batch tables in the test DB -⋮---- -let conn = shared.db.lock().unwrap(); -anyllm_proxy::batch::db::init_batch_tables(&conn).unwrap(); -⋮---- -let app = routes::app_multi_with_shared(multi, Some(shared), None); -let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -fn valid_jsonl() -> &'static str { -⋮---- -async fn upload_file_and_create_batch() { -let base = spawn_test_server_with_shared().await; -⋮---- -// Step 1: Upload a valid JSONL file -let form = multipart::Form::new().text("purpose", "batch").part( -⋮---- -multipart::Part::bytes(valid_jsonl().as_bytes().to_vec()) -.file_name("test.jsonl") -.mime_str("application/jsonl") -.unwrap(), -⋮---- -.post(format!("{base}/v1/files")) -.header("x-api-key", "test") -.multipart(form) -.send() -⋮---- -.unwrap(); -assert_eq!(resp.status(), 200); -⋮---- -let file_obj: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(file_obj["object"], "file"); -assert_eq!(file_obj["purpose"], "batch"); -assert!(file_obj["id"].as_str().unwrap().starts_with("file-")); -let file_id = file_obj["id"].as_str().unwrap().to_string(); -⋮---- -// Step 2: Create a batch job -⋮---- -.post(format!("{base}/v1/batches")) -⋮---- -.json(&serde_json::json!({ -⋮---- -let batch_obj: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(batch_obj["object"], "batch"); -assert!(batch_obj["id"].as_str().unwrap().starts_with("batch-")); -assert_eq!(batch_obj["status"], "validating"); -assert_eq!(batch_obj["input_file_id"], file_id); -assert_eq!(batch_obj["request_counts"]["total"], 2); -let batch_id = batch_obj["id"].as_str().unwrap().to_string(); -⋮---- -// Step 3: Get batch status -⋮---- -.get(format!("{base}/v1/batches/{batch_id}")) -⋮---- -let fetched: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(fetched["id"], batch_id); -⋮---- -// Step 4: List batches -⋮---- -.get(format!("{base}/v1/batches")) -⋮---- -let list: serde_json::Value = resp.json().await.unwrap(); -assert_eq!(list["object"], "list"); -assert!(!list["data"].as_array().unwrap().is_empty()); -⋮---- -async fn upload_invalid_jsonl_returns_400() { -⋮---- -multipart::Part::bytes(b"not valid json".to_vec()).file_name("bad.jsonl"), -⋮---- -assert_eq!(resp.status(), 400); -⋮---- -async fn create_batch_with_missing_file_returns_400() { -⋮---- -async fn get_nonexistent_batch_returns_404() { -⋮---- -.get(format!("{base}/v1/batches/batch-does-not-exist")) -⋮---- -assert_eq!(resp.status(), 404); -⋮---- -async fn unsupported_backend_returns_501() { -⋮---- -// Create a config with an Anthropic backend (unsupported for batches) -⋮---- -openai_base_url: "https://api.anthropic.com".to_string(), -⋮---- -big_model: "claude-sonnet-4-6".into(), -small_model: "claude-haiku-4-5".into(), -⋮---- -let base = format!("http://{addr}"); -⋮---- -assert_eq!(resp.status(), 501); -⋮---- -async fn anthropic_batch_rejects_empty_requests() { -⋮---- -.post(format!("{base}/v1/messages/batches")) -⋮---- -.header("content-type", "application/json") -.body(serde_json::to_string(&serde_json::json!({"requests": []})).unwrap()) - - - -// Phase 22: Responses API streaming state machine -// -// Converts OpenAI Responses API SSE events into Anthropic SSE events. -// The Responses API emits typed events (response.created, response.output_text.delta, -// response.completed, etc.) rather than partial JSON chunks like Chat Completions. -// There is no [DONE] sentinel; the stream ends with response.completed. -⋮---- -use crate::anthropic; -use crate::util; -⋮---- -/// A single SSE event from the Responses API streaming endpoint. -/// -/// The `type` field identifies the event kind. Additional fields vary by event type. -/// We keep the structure flexible with a flattened map for the varying fields. -⋮---- -/// OpenAI Responses streaming: -⋮---- -pub struct ResponsesStreamEvent { -⋮---- -/// All other fields, varying by event type. -⋮---- -/// State machine that converts Responses API streaming events into Anthropic SSE events. -⋮---- -/// Feed events via `process_event`, then call `finish` after the stream ends. -/// Each call returns zero or more Anthropic SSE events to forward to the client. -pub struct ResponsesStreamingTranslator { -⋮---- -/// Number of currently open function call items (used to gate content_part behavior). -⋮---- -impl ResponsesStreamingTranslator { -/// Create a new translator for the given model name. -/// Generates a fresh Anthropic message ID for the translated stream. -pub fn new(model: String) -> Self { -⋮---- -/// Process one Responses API streaming event. -/// Returns zero or more Anthropic SSE events. -pub fn process_event(&mut self, event: &ResponsesStreamEvent) -> Vec { -⋮---- -match event.event_type.as_str() { -"response.created" => self.handle_created(), -"response.output_item.added" => self.handle_output_item_added(event), -"response.content_part.added" => self.handle_content_part_added(event), -"response.output_text.delta" => self.handle_text_delta(event), -"response.output_text.done" => Vec::new(), // We already streamed the text via deltas -"response.content_part.done" => self.handle_content_part_done(), -"response.output_item.done" => self.handle_output_item_done(event), -"response.function_call_arguments.delta" => self.handle_function_call_delta(event), -"response.function_call_arguments.done" => Vec::new(), // Handled in output_item.done -"response.completed" => self.handle_completed(event), -"response.failed" | "response.cancelled" => self.handle_error(event), -⋮---- -/// Call after all events have been processed (stream ended without response.completed). -pub fn finish(&mut self) -> Vec { -⋮---- -// Close any open content block -⋮---- -events.push(anthropic::StreamEvent::ContentBlockStop { -⋮---- -// Emit message_delta with stop_reason -events.push(anthropic::StreamEvent::MessageDelta { -⋮---- -stop_reason: Some(anthropic::StopReason::EndTurn), -⋮---- -usage: Some(anthropic::streaming::DeltaUsage { -⋮---- -events.push(anthropic::StreamEvent::MessageStop {}); -⋮---- -fn ensure_started(&mut self) -> Vec { -⋮---- -vec![self.make_message_start()] -⋮---- -fn handle_created(&mut self) -> Vec { -self.ensure_started() -⋮---- -fn handle_output_item_added( -⋮---- -let mut events = self.ensure_started(); -⋮---- -// Check if this is a function_call item -if let Some(item) = event.data.get("item") { -if item.get("type").and_then(|v| v.as_str()) == Some("function_call") { -// Close any open text block first -⋮---- -let name = item.get("name").and_then(|v| v.as_str()).unwrap_or(""); -let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); -⋮---- -let id = if call_id.is_empty() { -⋮---- -call_id.to_string() -⋮---- -events.push(anthropic::StreamEvent::ContentBlockStart { -⋮---- -name: name.to_string(), -⋮---- -fn handle_content_part_added( -⋮---- -// Close previous block if open (text block ending, new one starting) -⋮---- -// First content part -⋮---- -fn handle_text_delta(&mut self, event: &ResponsesStreamEvent) -> Vec { -⋮---- -.get("delta") -.and_then(|v| v.as_str()) -.unwrap_or(""); -if delta.is_empty() { -⋮---- -vec![anthropic::StreamEvent::ContentBlockDelta { -⋮---- -fn handle_content_part_done(&mut self) -> Vec { -⋮---- -let events = vec![anthropic::StreamEvent::ContentBlockStop { -⋮---- -fn handle_output_item_done( -⋮---- -// If this was a function_call item, close its content block -⋮---- -self.tool_call_depth = self.tool_call_depth.saturating_sub(1); -⋮---- -fn handle_function_call_delta( -⋮---- -fn handle_completed(&mut self, event: &ResponsesStreamEvent) -> Vec { -⋮---- -// Extract usage from the completed response -if let Some(response) = event.data.get("response") { -if let Some(usage) = response.get("usage") { -⋮---- -.get("input_tokens") -.and_then(|v| v.as_u64()) -.unwrap_or(0) as u32; -⋮---- -.get("output_tokens") -⋮---- -// Determine stop reason -let stop_reason = if let Some(response) = event.data.get("response") { -match response.get("status").and_then(|v| v.as_str()) { -⋮---- -stop_reason: Some(stop_reason), -⋮---- -fn handle_error(&mut self, event: &ResponsesStreamEvent) -> Vec { -self.finished = true; // Error is terminal; prevent finish() from emitting closure events -⋮---- -.get("response") -.and_then(|r| r.get("status_details")) -.and_then(|d| d.get("error")) -.and_then(|e| e.get("message")) -.and_then(|m| m.as_str()) -.unwrap_or("Unknown error"); -⋮---- -vec![anthropic::StreamEvent::Error { -⋮---- -/// Return accumulated usage if any tokens were counted, None otherwise. -/// Only populated after a `response.completed` event has been processed. -pub fn usage(&self) -> Option<&anthropic::Usage> { -⋮---- -Some(&self.usage) -⋮---- -fn make_message_start(&self) -> anthropic::StreamEvent { -⋮---- -id: self.message_id.clone(), -msg_type: "message".to_string(), -role: "assistant".to_string(), -⋮---- -model: self.model.clone(), -⋮---- -usage: self.usage.clone(), -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn make_event(event_type: &str, data: serde_json::Value) -> ResponsesStreamEvent { -⋮---- -event_type: event_type.to_string(), -⋮---- -fn created_emits_message_start() { -let mut t = ResponsesStreamingTranslator::new("gpt-4o".into()); -let events = t.process_event(&make_event("response.created", json!({}))); -assert_eq!(events.len(), 1); -assert!(matches!( -⋮---- -fn text_delta_emits_content_block_delta() { -⋮---- -t.process_event(&make_event("response.created", json!({}))); -t.process_event(&make_event( -⋮---- -json!({"item": {"type": "message"}}), -⋮---- -json!({"part": {"type": "output_text"}}), -⋮---- -let events = t.process_event(&make_event( -⋮---- -json!({"delta": "Hello"}), -⋮---- -assert!(matches!(delta, anthropic::Delta::TextDelta { text } if text == "Hello")); -⋮---- -_ => panic!("expected ContentBlockDelta"), -⋮---- -fn completed_emits_final_events() { -⋮---- -t.process_event(&make_event("response.content_part.added", json!({}))); -⋮---- -json!({"delta": "Hi"}), -⋮---- -t.process_event(&make_event("response.content_part.done", json!({}))); -⋮---- -json!({ -⋮---- -// Should have MessageDelta + MessageStop (content block already closed) -assert!(events -⋮---- -fn incomplete_status_maps_to_max_tokens() { -⋮---- -let events = t.process_event(&make_event("response.completed", json!({ -⋮---- -.iter() -.find(|e| matches!(e, anthropic::StreamEvent::MessageDelta { .. })); -⋮---- -assert_eq!(delta.stop_reason, Some(anthropic::StopReason::MaxTokens)); -⋮---- -_ => panic!("expected MessageDelta"), -⋮---- -fn function_call_streaming() { -⋮---- -// Function call item added -⋮---- -assert!(events.iter().any(|e| matches!(e, anthropic::StreamEvent::ContentBlockStart { content_block: anthropic::ContentBlock::ToolUse { name, .. }, .. } if name == "get_weather"))); -⋮---- -// Function call arguments delta -⋮---- -json!({"delta": "{\"city\":"}), -⋮---- -assert!(events.iter().any(|e| matches!( -⋮---- -// Output item done -⋮---- -fn finish_without_completed_event() { -⋮---- -// Stream ends without response.completed (connection dropped, etc.) -let events = t.finish(); -⋮---- -fn empty_delta_ignored() { -⋮---- -json!({"delta": ""}), -⋮---- -assert!(events.is_empty()); -⋮---- -fn error_event_produces_stream_error() { -⋮---- -assert!( -⋮---- -fn double_finish_is_noop() { -⋮---- -t.process_event(&make_event("response.completed", json!({ -⋮---- -fn translator_usage_returns_none_before_any_events() { -let t = ResponsesStreamingTranslator::new("gpt-4o".into()); -assert!(t.usage().is_none()); -⋮---- -fn translator_usage_returns_tokens_after_completed_event() { -⋮---- -let completed = make_event( -⋮---- -t.process_event(&completed); -let usage = t.usage().expect("usage should be Some after completed event"); -assert_eq!(usage.input_tokens, 42); -assert_eq!(usage.output_tokens, 17); - - - -// Reverse message mapping: OpenAI Chat Completions -> Anthropic Messages -// -// Converts OpenAI-format requests to Anthropic format (for accepting OpenAI -// input) and Anthropic responses back to OpenAI format. -⋮---- -use crate::anthropic; -use crate::error::TranslateError; -⋮---- -use crate::openai; -use crate::util; -⋮---- -/// Convert an OpenAI ChatCompletionRequest to an Anthropic MessageCreateRequest. -/// -/// Returns an error if `max_tokens` and `max_completion_tokens` are both absent -/// (Anthropic requires `max_tokens`). -pub fn openai_to_anthropic_request( -⋮---- -// max_tokens is required in Anthropic; reject if absent -⋮---- -.or(req.max_tokens) -.ok_or_else(|| { -TranslateError::MissingField("max_tokens or max_completion_tokens is required".into()) -⋮---- -// Extract system messages into the Anthropic system field. -// Multiple system messages are concatenated. -let text = extract_text_content(&msg.content); -if !text.is_empty() { -⋮---- -existing.push('\n'); -existing.push_str(&text); -⋮---- -system = Some(anthropic::System::Text(text)); -⋮---- -let content = convert_openai_content_to_anthropic(&msg.content); -messages.push(anthropic::InputMessage { -⋮---- -let content = convert_assistant_to_anthropic(msg); -⋮---- -// Tool role messages become Anthropic tool_result blocks -// on a user message (Anthropic requires tool results in user turn) -⋮---- -let tool_use_id = msg.tool_call_id.clone().unwrap_or_default(); -⋮---- -content: if text.is_empty() { -⋮---- -Some(anthropic::ToolResultContent::Text(text)) -⋮---- -content: anthropic::Content::Blocks(vec![content_block]), -⋮---- -// Deprecated function role: treat as tool -⋮---- -let tool_use_id = msg.name.clone().unwrap_or_default(); -⋮---- -.as_ref() -.map(|t| tools_map::openai_tools_to_anthropic(t)); -⋮---- -.map(tools_map::openai_tool_choice_to_anthropic); -⋮---- -let stop_sequences = req.stop.as_ref().map(|s| match s { -openai::Stop::Single(s) => vec![s.clone()], -openai::Stop::Multiple(v) => v.clone(), -⋮---- -let metadata = req.user.as_ref().map(|u| anthropic::Metadata { -user_id: Some(u.clone()), -⋮---- -if req.presence_penalty.is_some() { -warnings.add("presence_penalty"); -⋮---- -if req.frequency_penalty.is_some() { -warnings.add("frequency_penalty"); -⋮---- -if req.response_format.is_some() { -warnings.add("response_format"); -⋮---- -if req.extra.contains_key("logprobs") { -warnings.add("logprobs"); -⋮---- -if req.extra.contains_key("n") { -warnings.add("n"); -⋮---- -if req.extra.contains_key("seed") { -warnings.add("seed"); -⋮---- -if req.stream_options.is_some() { -warnings.add("stream_options"); -⋮---- -Some(anthropic::ToolChoice::Auto { -disable_parallel_tool_use: Some(true), -⋮---- -Some(anthropic::ToolChoice::Any { -⋮---- -Ok(anthropic::MessageCreateRequest { -model: req.model.clone(), -⋮---- -/// Convert an Anthropic MessageResponse to an OpenAI ChatCompletionResponse. -pub fn anthropic_to_openai_response( -⋮---- -text_parts.push(text.clone()); -⋮---- -tool_calls.push(openai::ToolCall { -id: id.clone(), -call_type: "function".to_string(), -⋮---- -name: name.clone(), -⋮---- -existing.push_str(thinking); -⋮---- -reasoning_content = Some(thinking.clone()); -⋮---- -let content = if text_parts.is_empty() { -⋮---- -Some(openai::ChatContent::Text(text_parts.join(""))) -⋮---- -.map(anthropic_stop_reason_to_openai); -⋮---- -let id = format!("chatcmpl-{}", util::ids::generate_uuid()); -⋮---- -object: "chat.completion".to_string(), -model: model.to_string(), -choices: vec![openai::Choice { -⋮---- -usage: Some(usage), -⋮---- -/// Map Anthropic stop_reason to OpenAI finish_reason. -pub fn anthropic_stop_reason_to_openai( -⋮---- -/// Compute warnings for an OpenAI request about features that will be dropped. -pub fn compute_openai_request_warnings(req: &openai::ChatCompletionRequest) -> TranslationWarnings { -⋮---- -openai_to_anthropic_request(req, &mut w).ok(); -⋮---- -// --- Helper functions --- -⋮---- -fn extract_text_content(content: &Option) -> String { -⋮---- -Some(openai::ChatContent::Text(s)) => s.clone(), -⋮---- -.iter() -.filter_map(|p| match p { -openai::ChatContentPart::Text { text } => Some(text.as_str()), -⋮---- -.join(""), -⋮---- -fn convert_openai_content_to_anthropic( -⋮---- -Some(openai::ChatContent::Text(s)) => anthropic::Content::Text(s.clone()), -⋮---- -blocks.push(anthropic::ContentBlock::Text { text: text.clone() }); -⋮---- -// Parse data URIs back to base64 + media_type -let source = url_to_image_source(&image_url.url); -blocks.push(anthropic::ContentBlock::Image { source }); -⋮---- -// InputAudio and File have no Anthropic equivalent; drop them -⋮---- -if blocks.is_empty() { -⋮---- -fn convert_assistant_to_anthropic(msg: &openai::ChatMessage) -> anthropic::Content { -⋮---- -// Map reasoning_content to thinking block -⋮---- -if !reasoning.is_empty() { -blocks.push(anthropic::ContentBlock::Thinking { -thinking: reasoning.clone(), -⋮---- -// Map text content -⋮---- -// Map tool calls to tool_use blocks -⋮---- -blocks.push(anthropic::ContentBlock::ToolUse { -id: tc.id.clone(), -name: tc.function.name.clone(), -⋮---- -} else if blocks.len() == 1 { -⋮---- -return anthropic::Content::Text(text.clone()); -⋮---- -/// Parse a URL string into an Anthropic ImageSource. -/// Handles both data URIs (data:image/png;base64,...) and regular URLs. -fn url_to_image_source(url: &str) -> anthropic::ImageSource { -if let Some(rest) = url.strip_prefix("data:") { -// Parse data URI: data:media_type;base64,data -if let Some((meta, data)) = rest.split_once(',') { -let media_type = meta.strip_suffix(";base64").unwrap_or(meta); -⋮---- -source_type: "base64".to_string(), -media_type: Some(media_type.to_string()), -data: Some(data.to_string()), -⋮---- -// Regular URL -⋮---- -source_type: "url".to_string(), -⋮---- -url: Some(url.to_string()), -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -fn make_basic_request() -> openai::ChatCompletionRequest { -serde_json::from_value(json!({ -⋮---- -.unwrap() -⋮---- -fn basic_message_conversion() { -let req = make_basic_request(); -⋮---- -let result = openai_to_anthropic_request(&req, &mut w).unwrap(); -assert_eq!(result.model, "claude-sonnet-4-20250514"); -assert_eq!(result.max_tokens, 100); -assert_eq!(result.messages.len(), 1); -assert_eq!(result.messages[0].role, anthropic::Role::User); -⋮---- -fn system_message_extraction() { -let req: openai::ChatCompletionRequest = serde_json::from_value(json!({ -⋮---- -.unwrap(); -⋮---- -assert!( -⋮---- -assert_eq!(result.messages.len(), 1); // system not in messages -⋮---- -fn developer_role_maps_to_system() { -⋮---- -fn missing_max_tokens_rejected() { -⋮---- -let result = openai_to_anthropic_request(&req, &mut w); -assert!(result.is_err()); -⋮---- -fn max_completion_tokens_used_as_fallback() { -⋮---- -assert_eq!(result.max_tokens, 200); -⋮---- -fn tool_call_conversion() { -⋮---- -assert_eq!(result.messages.len(), 3); -assert!(result.tools.is_some()); -// Second message (assistant) should have tool_use block -⋮---- -_ => panic!("expected blocks"), -⋮---- -// Third message (tool result) should be user with tool_result -assert_eq!(result.messages[2].role, anthropic::Role::User); -⋮---- -fn lossy_fields_generate_warnings() { -⋮---- -openai_to_anthropic_request(&req, &mut w).unwrap(); -let header = w.as_header_value().unwrap(); -assert!(header.contains("presence_penalty")); -assert!(header.contains("frequency_penalty")); -assert!(header.contains("logprobs")); -assert!(header.contains("seed")); -⋮---- -fn stop_sequences_mapping() { -⋮---- -assert_eq!( -⋮---- -// --- Response tests --- -⋮---- -fn basic_response_conversion() { -⋮---- -id: "msg_123".to_string(), -response_type: "message".to_string(), -⋮---- -content: vec![anthropic::ContentBlock::Text { -⋮---- -model: "claude-sonnet-4-20250514".to_string(), -stop_reason: Some(anthropic::StopReason::EndTurn), -⋮---- -created: Some(1700000000), -⋮---- -let result = anthropic_to_openai_response(&resp, "claude-sonnet-4-20250514"); -assert_eq!(result.object, "chat.completion"); -assert!(result.id.starts_with("chatcmpl-")); -assert_eq!(result.choices.len(), 1); -⋮---- -Some(openai::ChatContent::Text(s)) => assert_eq!(s, "Hello!"), -other => panic!("expected Text, got {:?}", other), -⋮---- -let usage = result.usage.unwrap(); -assert_eq!(usage.prompt_tokens, 10); -assert_eq!(usage.completion_tokens, 5); -⋮---- -fn tool_use_response_conversion() { -⋮---- -id: "msg_456".to_string(), -⋮---- -content: vec![anthropic::ContentBlock::ToolUse { -⋮---- -stop_reason: Some(anthropic::StopReason::ToolUse), -⋮---- -let tc = result.choices[0].message.tool_calls.as_ref().unwrap(); -assert_eq!(tc.len(), 1); -assert_eq!(tc[0].id, "call_1"); -assert_eq!(tc[0].function.name, "get_weather"); -⋮---- -fn thinking_block_maps_to_reasoning_content() { -⋮---- -id: "msg_789".to_string(), -⋮---- -content: vec![ -⋮---- -Some(openai::ChatContent::Text(s)) => assert_eq!(s, "The answer is 4."), -⋮---- -fn stop_reason_mapping() { -⋮---- -fn data_uri_image_parsing() { -let source = url_to_image_source("data:image/png;base64,iVBORw0KGgo="); -assert_eq!(source.source_type, "base64"); -assert_eq!(source.media_type.as_deref(), Some("image/png")); -assert_eq!(source.data.as_deref(), Some("iVBORw0KGgo=")); -assert!(source.url.is_none()); -⋮---- -fn regular_url_image_source() { -let source = url_to_image_source("https://example.com/img.png"); -assert_eq!(source.source_type, "url"); -assert_eq!(source.url.as_deref(), Some("https://example.com/img.png")); -assert!(source.data.is_none()); -⋮---- -fn user_field_maps_to_metadata() { -⋮---- -fn parallel_tool_calls_false_maps_to_disable() { -⋮---- -assert!(matches!( - - - -// Reverse streaming: Anthropic SSE events -> OpenAI ChatCompletionChunk SSE -// -// Consumes Anthropic StreamEvent items and emits OpenAI ChatCompletionChunk -// objects. This is the inverse of StreamingTranslator in streaming_map.rs. -⋮---- -use crate::anthropic; -use crate::mapping::reverse_message_map::anthropic_stop_reason_to_openai; -use crate::openai; -⋮---- -/// Sentinel value returned by `process_event` to signal the stream is done. -/// The caller should emit `data: [DONE]\n\n` when it sees this. -⋮---- -/// State machine that converts Anthropic SSE events into OpenAI ChatCompletionChunk objects. -/// -/// Feed events via `process_event`, which returns zero or more chunks to send. -/// When `message_stop` is received, `is_done()` returns true and the caller -/// should emit `data: [DONE]\n\n`. -pub struct ReverseStreamingTranslator { -⋮---- -impl ReverseStreamingTranslator { -pub fn new(id: String, model: String) -> Self { -⋮---- -.duration_since(std::time::UNIX_EPOCH) -.unwrap_or_default() -.as_secs(), -⋮---- -pub fn is_done(&self) -> bool { -⋮---- -/// Process a single Anthropic StreamEvent and return zero or more OpenAI chunks. -pub fn process_event(&mut self, event: &anthropic::StreamEvent) -> Vec { -⋮---- -self.input_tokens = Some(message.usage.input_tokens); -⋮---- -// Emit first chunk with role -vec![self.make_chunk( -⋮---- -id: Some(id.clone()), -call_type: Some("function".to_string()), -function: Some(ChunkFunctionCall { -name: Some(name.clone()), -arguments: Some(String::new()), -⋮---- -// Text and Thinking blocks emit their content via deltas -_ => vec![], -⋮---- -return vec![]; -⋮---- -arguments: Some(partial_json.clone()), -⋮---- -anthropic::streaming::Delta::SignatureDelta { .. } => vec![], -⋮---- -anthropic::StreamEvent::ContentBlockStop { .. } => vec![], -⋮---- -self.output_tokens = Some(u.output_tokens); -⋮---- -.as_ref() -.map(anthropic_stop_reason_to_openai); -let mut chunks = vec![self.make_chunk(ChunkDelta::default(), finish_reason)]; -// Emit usage chunk if we have token counts -⋮---- -chunks.push(ChatCompletionChunk { -id: self.message_id.clone(), -object: "chat.completion.chunk".to_string(), -model: self.model.clone(), -choices: vec![], -usage: Some(openai::ChatUsage { -⋮---- -created: Some(self.created), -⋮---- -vec![] -⋮---- -anthropic::StreamEvent::Ping {} => vec![], -⋮---- -fn make_chunk( -⋮---- -choices: vec![ChunkChoice { -⋮---- -mod tests { -⋮---- -fn make_translator() -> ReverseStreamingTranslator { -ReverseStreamingTranslator::new("chatcmpl-test".to_string(), "gpt-4o".to_string()) -⋮---- -fn message_start_emits_role_chunk() { -let mut t = make_translator(); -⋮---- -id: "msg_123".to_string(), -msg_type: "message".to_string(), -role: "assistant".to_string(), -content: vec![], -model: "claude-sonnet".to_string(), -⋮---- -created: Some(1700000000), -⋮---- -let chunks = t.process_event(&event); -assert_eq!(chunks.len(), 1); -assert_eq!( -⋮---- -assert!(chunks[0].choices[0].finish_reason.is_none()); -⋮---- -fn text_delta_emits_content_chunk() { -⋮---- -text: "Hello".to_string(), -⋮---- -assert_eq!(chunks[0].choices[0].delta.content.as_deref(), Some("Hello")); -⋮---- -fn tool_use_streaming() { -⋮---- -// Start tool use block -⋮---- -id: "call_123".to_string(), -name: "get_weather".to_string(), -⋮---- -let chunks = t.process_event(&start); -⋮---- -let tc = &chunks[0].choices[0].delta.tool_calls.as_ref().unwrap()[0]; -assert_eq!(tc.id.as_deref(), Some("call_123")); -⋮---- -// Delta with args -⋮---- -partial_json: "{\"loc".to_string(), -⋮---- -let chunks = t.process_event(&delta); -⋮---- -assert_eq!(tc.index, 0); -assert!(tc.id.is_none()); // Only first chunk has id -⋮---- -fn thinking_delta_emits_reasoning_content() { -⋮---- -thinking: "Let me think...".to_string(), -⋮---- -fn message_delta_emits_finish_reason_and_usage() { -⋮---- -// Set input tokens via message_start -⋮---- -id: "msg_1".to_string(), -⋮---- -model: "claude".to_string(), -⋮---- -t.process_event(&start); -⋮---- -stop_reason: Some(StopReason::EndTurn), -⋮---- -usage: Some(DeltaUsage { output_tokens: 5 }), -⋮---- -assert_eq!(chunks.len(), 2); // finish chunk + usage chunk -⋮---- -let usage = chunks[1].usage.as_ref().unwrap(); -assert_eq!(usage.prompt_tokens, 10); -assert_eq!(usage.completion_tokens, 5); -assert_eq!(usage.total_tokens, 15); -⋮---- -fn message_stop_sets_done() { -⋮---- -assert!(!t.is_done()); -t.process_event(&StreamEvent::MessageStop {}); -assert!(t.is_done()); -⋮---- -fn ping_produces_no_chunks() { -⋮---- -let chunks = t.process_event(&StreamEvent::Ping {}); -assert!(chunks.is_empty()); -⋮---- -fn multiple_tool_calls_track_index() { -⋮---- -// First tool -⋮---- -id: "call_1".to_string(), -name: "fn_a".to_string(), -⋮---- -let chunks = t.process_event(&start1); -⋮---- -// Second tool -⋮---- -id: "call_2".to_string(), -name: "fn_b".to_string(), -⋮---- -let chunks = t.process_event(&start2); - - - -// Phase 21a: Convenience wrappers for the translation layer. -// -// Thin functions combining TranslationConfig with the stateless mapping functions. -// The raw mapping API (crate::mapping::*) remains available for advanced use. -⋮---- -use crate::config::TranslationConfig; -use crate::error::TranslateError; -use crate::gemini::request::GenerateContentRequest; -use crate::gemini::response::GenerateContentResponse; -pub use crate::mapping::warnings::TranslationWarnings; -⋮---- -/// Compute degradation warnings for a request — features that will be dropped in translation. -/// -/// Call this before translating; inject the result as `x-anyllm-degradation` header. -pub fn compute_request_warnings(req: &MessageCreateRequest) -> TranslationWarnings { -⋮---- -/// Translate an Anthropic request to an OpenAI Chat Completions request. -⋮---- -/// Applies model mapping from config to the resulting request's `model` field. -pub fn translate_request( -⋮---- -openai_req.model = config.map_model(&openai_req.model)?; -Ok(openai_req) -⋮---- -/// Translate an OpenAI Chat Completions response back to an Anthropic response. -⋮---- -/// `original_model` is the Anthropic model name from the original request, -/// used in the response's `model` field. -pub fn translate_response(resp: &ChatCompletionResponse, original_model: &str) -> MessageResponse { -⋮---- -/// Create a new streaming translator for OpenAI Chat Completions chunks. -⋮---- -/// The returned translator is stateful: feed chunks via `process_chunk()`, -/// then call `finish()` to get the final events. -pub fn new_stream_translator(model: String) -> streaming_map::StreamingTranslator { -⋮---- -/// Translate an Anthropic request to an OpenAI Responses API request. -⋮---- -pub fn translate_request_responses( -⋮---- -responses_req.model = config.map_model(&responses_req.model)?; -Ok(responses_req) -⋮---- -/// Translate an OpenAI Responses API response back to an Anthropic response. -⋮---- -/// `original_model` is the Anthropic model name from the original request. -pub fn translate_response_responses( -⋮---- -/// Translate an OpenAI Chat Completions request to an Anthropic request. -⋮---- -/// Returns an error if `max_tokens` / `max_completion_tokens` is absent. -/// Populates `warnings` with features dropped during translation. -pub fn translate_openai_to_anthropic_request( -⋮---- -/// Translate an Anthropic response to an OpenAI Chat Completions response. -⋮---- -/// `model` is used as the response's `model` field. -pub fn translate_anthropic_to_openai_response( -⋮---- -/// Create a new reverse streaming translator (Anthropic SSE -> OpenAI chunks). -⋮---- -/// The returned translator is stateful: feed Anthropic StreamEvent items via -/// `process_event()`, which returns OpenAI ChatCompletionChunk objects. -pub fn new_reverse_stream_translator( -⋮---- -/// Create a new streaming translator for OpenAI Responses API events. -⋮---- -/// Same stateful pattern as `new_stream_translator`. -pub fn new_responses_stream_translator( -⋮---- -/// Translate an Anthropic request to a Gemini native `GenerateContentRequest`. -⋮---- -/// Applies model mapping from config to the resulting request's model selection. -/// The returned request is ready to POST to `models/{model}:generateContent`. -pub fn translate_request_gemini( -⋮---- -let model = config.map_model(&req.model)?; -Ok((gemini_req, model)) -⋮---- -/// Translate a Gemini native `GenerateContentResponse` back to an Anthropic response. -⋮---- -/// `model` is the Anthropic model name from the original request. -pub fn translate_response_gemini( -⋮---- -/// Create a new Gemini streaming translator. -⋮---- -/// The returned translator is stateful: feed full `GenerateContentResponse` -/// objects via `process_response()`, then call `finish()` for final events. -pub fn new_gemini_stream_translator( -⋮---- -mod tests { -⋮---- -use crate::config::LossyBehavior; -⋮---- -fn sample_request() -> MessageCreateRequest { -⋮---- -.unwrap() -⋮---- -fn translate_request_with_default_config() { -⋮---- -let req = sample_request(); -let openai_req = translate_request(&req, &config).unwrap(); -// Default config: empty model_map, passthrough -assert_eq!(openai_req.model, "claude-sonnet-4-6"); -assert_eq!(openai_req.max_completion_tokens, Some(100)); -⋮---- -fn translate_request_with_model_mapping() { -⋮---- -.model_map("haiku", "gpt-4o-mini") -.model_map("sonnet", "gpt-4o") -.build(); -⋮---- -assert_eq!(openai_req.model, "gpt-4o"); -⋮---- -fn translate_request_unknown_model_passthrough() { -⋮---- -.unwrap(); -⋮---- -assert_eq!(openai_req.model, "custom-model"); -⋮---- -fn translate_request_unknown_model_strict() { -⋮---- -.passthrough_unknown_models(false) -⋮---- -let err = translate_request(&req, &config).unwrap_err(); -assert!(matches!(err, TranslateError::UnknownModel(_))); -⋮---- -fn translate_response_roundtrip() { -⋮---- -let anthropic_resp = translate_response(&openai_resp, "claude-sonnet-4-6"); -assert_eq!(anthropic_resp.model, "claude-sonnet-4-6"); -assert_eq!(anthropic_resp.usage.input_tokens, 10); -assert_eq!(anthropic_resp.usage.output_tokens, 5); -⋮---- -fn builder_ergonomics() { -⋮---- -.model_map("haiku", "gemini-2.5-flash") -.model_map("sonnet", "gemini-2.5-pro") -.model_map("opus", "gemini-2.5-pro") -.lossy_behavior(LossyBehavior::Silent) -⋮---- -assert_eq!(config.model_map.len(), 3); -assert_eq!(config.lossy_behavior, LossyBehavior::Silent); -assert!(!config.passthrough_unknown_models); - - - -// AWS Bedrock client with SigV4 request signing. -// Sends Anthropic Messages API requests directly to Bedrock (no OpenAI translation). -// Bedrock streaming uses AWS Event Stream binary framing, not SSE. -⋮---- -use crate::config::TlsConfig; -use aws_credential_types::Credentials; -⋮---- -use aws_sigv4::sign::v4; -use reqwest::Client; -use tokio::time::sleep; -use zeroize::Zeroizing; -⋮---- -/// HTTP client for AWS Bedrock with SigV4 request signing. -/// Secret fields (secret_access_key, session_token) are wrapped in `Zeroizing` -/// so they are zeroed from memory when the client is dropped. -⋮---- -pub struct BedrockClient { -⋮---- -/// Error type for the Bedrock client. -⋮---- -pub enum BedrockClientError { -/// Transport-level error (connection, timeout, DNS). -⋮---- -/// Upstream returned a non-success status. Body is raw bytes for passthrough. -⋮---- -/// SigV4 signing failed. -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -Self::Transport(msg) => write!(f, "Bedrock transport error: {msg}"), -Self::ApiError { status, .. } => write!(f, "Bedrock API error (status {status})"), -Self::Signing(msg) => write!(f, "Bedrock signing error: {msg}"), -⋮---- -impl BedrockClient { -/// Create a new Bedrock client. Decomposes `Credentials` so that secret -/// fields are stored in `Zeroizing` and wiped on drop. -pub fn new( -⋮---- -let client = build_http_client(tls); -let access_key_id = credentials.access_key_id().to_string(); -let secret_access_key = Zeroizing::new(credentials.secret_access_key().to_string()); -⋮---- -.session_token() -.map(|t| Zeroizing::new(t.to_string())); -⋮---- -pub fn big_model(&self) -> &str { -⋮---- -pub fn small_model(&self) -> &str { -⋮---- -/// Build the Bedrock InvokeModel URL for a given model. -fn invoke_url(&self, model_id: &str) -> String { -format!( -⋮---- -/// Build the Bedrock InvokeModelWithResponseStream URL. -fn invoke_stream_url(&self, model_id: &str) -> String { -⋮---- -/// Sign an HTTP request with SigV4 and return headers to add. -fn sign_request( -⋮---- -// Reconstruct Credentials on each call; the struct fields hold the -// canonical copies wrapped in Zeroizing for safe drop. -⋮---- -self.access_key_id.clone(), -self.secret_access_key.as_str(), -self.session_token.as_deref().map(|s| s.to_string()), -None, // expiration -"anyllm", // provider name -⋮---- -let identity: aws_smithy_runtime_api::client::identity::Identity = creds.into(); -⋮---- -.identity(&identity) -.region(&self.region) -.name("bedrock") -.time(std::time::SystemTime::now()) -.settings(settings) -.build() -.map_err(|e| BedrockClientError::Signing(e.to_string()))?; -let signing_params = params.into(); -⋮---- -extra_headers.iter().copied(), -⋮---- -let (instructions, _signature) = sign(signable, &signing_params) -.map_err(|e| BedrockClientError::Signing(e.to_string()))? -.into_parts(); -⋮---- -// Collect signing headers -⋮---- -.headers() -.map(|(k, v)| (k.to_string(), v.to_string())) -.collect(); -Ok(headers) -⋮---- -/// Forward a non-streaming request. Returns raw response body and rate limit headers. -pub async fn forward( -⋮---- -let response = self.send_with_retry(body, model_id, false).await?; -⋮---- -.bytes() -⋮---- -.map_err(|e| BedrockClientError::Transport(e.to_string()))?; -Ok((resp_body, rate_limits)) -⋮---- -/// Forward a streaming request. Returns the raw response for event stream decoding. -pub async fn forward_stream( -⋮---- -let response = self.send_with_retry(body, model_id, true).await?; -⋮---- -Ok((response, rate_limits)) -⋮---- -/// Send with retry on 429/5xx. -async fn send_with_retry( -⋮---- -self.invoke_stream_url(model_id) -⋮---- -self.invoke_url(model_id) -⋮---- -let signing_headers = self.sign_request("POST", &url, &body, &base_headers)?; -⋮---- -.post(&url) -.header("content-type", content_type) -.header("accept", accept) -.body(body.clone()); -⋮---- -rb = rb.header(k.as_str(), v.as_str()); -⋮---- -.send() -⋮---- -let status = response.status().as_u16(); -⋮---- -if (200..300).contains(&status) { -return Ok(response); -⋮---- -let retry_after = super::parse_retry_after(response.headers()); -⋮---- -drop(response.bytes().await); -sleep(delay).await; -⋮---- -let resp_body = response.bytes().await.unwrap_or_default(); -return Err(BedrockClientError::ApiError { -⋮---- -unreachable!("loop runs MAX_RETRIES+1 times and always returns") -⋮---- -// --------------------------------------------------------------------------- -// AWS Event Stream binary frame decoder -⋮---- -/// Decode AWS Event Stream frames from a byte buffer. -/// Each frame: 4-byte total_len | 4-byte headers_len | 4-byte prelude CRC | -/// headers | payload | 4-byte message CRC -/// -/// The payload contains `{"bytes":""}` where base64 decodes to an -/// Anthropic SSE JSON event string. -pub mod eventstream { -use bytes::BytesMut; -⋮---- -/// Minimum frame size: 4 (total_len) + 4 (headers_len) + 4 (prelude CRC) -/// + 0 (headers) + 0 (payload) + 4 (message CRC) = 16 -⋮---- -/// Try to extract one complete event stream frame from the buffer. -/// Returns `Some(payload_bytes)` and advances the buffer past the frame, -/// or `None` if the buffer does not contain a complete frame yet. -pub fn decode_frame(buf: &mut BytesMut) -> Option> { -if buf.len() < MIN_FRAME_SIZE { -⋮---- -if buf.len() < total_len { -return None; // incomplete frame -⋮---- -// Prelude is 8 bytes (total_len + headers_len), then 4-byte prelude CRC -let headers_start = 12; // 4 + 4 + 4 (prelude CRC) -⋮---- -// Message CRC is the last 4 bytes -let payload_end = total_len.saturating_sub(4); -⋮---- -if payload_start > payload_end || payload_end > buf.len() { -// Malformed frame: skip it -let _ = buf.split_to(total_len); -return Some(Vec::new()); -⋮---- -let payload = buf[payload_start..payload_end].to_vec(); -⋮---- -// Advance buffer past this frame -⋮---- -Some(payload) -⋮---- -/// Extract the Anthropic event JSON string from a Bedrock event stream payload. -/// Bedrock wraps the Anthropic event in `{"bytes":""}`. -/// Returns None if the payload is not a chunk event or is malformed. -pub fn extract_event_from_payload(payload: &[u8]) -> Option { -if payload.is_empty() { -⋮---- -// Parse as JSON to extract the base64-encoded bytes field -let parsed: serde_json::Value = serde_json::from_slice(payload).ok()?; -let b64 = parsed.get("bytes")?.as_str()?; -⋮---- -// Base64 decode -use base64::Engine; -let decoded = base64::engine::general_purpose::STANDARD.decode(b64).ok()?; -String::from_utf8(decoded).ok() -⋮---- -mod tests { -use super::eventstream; -⋮---- -/// Build a minimal AWS Event Stream frame with the given payload. -/// Uses zero CRCs (we don't validate CRCs in the decoder). -fn build_frame(headers: &[u8], payload: &[u8]) -> Vec { -let total_len = 12 + headers.len() + payload.len() + 4; // prelude(12) + headers + payload + msg CRC(4) -let headers_len = headers.len(); -⋮---- -frame.extend_from_slice(&(total_len as u32).to_be_bytes()); -frame.extend_from_slice(&(headers_len as u32).to_be_bytes()); -frame.extend_from_slice(&[0u8; 4]); // prelude CRC (not validated) -frame.extend_from_slice(headers); -frame.extend_from_slice(payload); -frame.extend_from_slice(&[0u8; 4]); // message CRC (not validated) -⋮---- -fn decode_frame_empty_payload() { -let frame = build_frame(&[], &[]); -let mut buf = BytesMut::from(frame.as_slice()); -let payload = eventstream::decode_frame(&mut buf).unwrap(); -assert!(payload.is_empty()); -assert!(buf.is_empty()); -⋮---- -fn decode_frame_with_payload() { -⋮---- -let frame = build_frame(&[], payload_data); -⋮---- -assert_eq!(payload, b"hello world"); -⋮---- -fn decode_frame_incomplete() { -let frame = build_frame(&[], b"hello"); -let mut buf = BytesMut::from(&frame[..frame.len() - 2]); // truncate -assert!(eventstream::decode_frame(&mut buf).is_none()); -⋮---- -fn decode_multiple_frames() { -let frame1 = build_frame(&[], b"first"); -let frame2 = build_frame(&[], b"second"); -⋮---- -buf.extend_from_slice(&frame1); -buf.extend_from_slice(&frame2); -⋮---- -let p1 = eventstream::decode_frame(&mut buf).unwrap(); -assert_eq!(p1, b"first"); -let p2 = eventstream::decode_frame(&mut buf).unwrap(); -assert_eq!(p2, b"second"); -⋮---- -fn decode_frame_with_headers() { -⋮---- -let frame = build_frame(headers, payload_data); -⋮---- -assert_eq!(payload, b"data"); -⋮---- -fn extract_event_from_valid_payload() { -⋮---- -let b64 = base64::engine::general_purpose::STANDARD.encode(event_json); -let wrapper = format!(r#"{{"bytes":"{b64}"}}"#); -let result = eventstream::extract_event_from_payload(wrapper.as_bytes()); -assert_eq!(result.unwrap(), event_json); -⋮---- -fn extract_event_empty_payload() { -assert!(eventstream::extract_event_from_payload(&[]).is_none()); -⋮---- -fn extract_event_invalid_json() { -assert!(eventstream::extract_event_from_payload(b"not json").is_none()); -⋮---- -fn extract_event_missing_bytes_field() { -⋮---- -assert!(eventstream::extract_event_from_payload(payload.as_bytes()).is_none()); - - - -// LiteLLM environment variable aliases. -// -// Maps LiteLLM env var names to their anyllm-proxy equivalents. -// Applied once at startup; the real environment always wins (aliases -// only take effect when the target var is not already set). -⋮---- -// Auth -⋮---- -// Config file -⋮---- -// Azure -⋮---- -// AWS Bedrock -⋮---- -// IP allowlisting -⋮---- -/// Compute env var overrides from LiteLLM aliases without mutating the environment. -/// -/// Returns `(target_var, value)` pairs for each alias where the source is set -/// and the target is not. Caller is responsible for applying them via `set_var`. -pub fn compute_env_aliases() -> Vec<(&'static str, String)> { -⋮---- -if std::env::var(to).is_err() { -⋮---- -overrides.push((to, val)); -⋮---- -mod tests { -⋮---- -use std::sync::Mutex; -⋮---- -/// Test-only wrapper: apply computed aliases to the environment. -fn apply_env_aliases() { -for (key, val) in compute_env_aliases() { -⋮---- -// Serial test lock: env var mutations are process-global. -⋮---- -fn compute_returns_overrides_when_target_unset() { -let _lock = ENV_LOCK.lock().unwrap(); -⋮---- -let overrides = compute_env_aliases(); -⋮---- -// Should contain the PROXY_API_KEYS override. -⋮---- -.iter() -.find(|(k, _)| *k == "PROXY_API_KEYS") -.map(|(_, v)| v.as_str()); -assert_eq!(found, Some("sk-test-master")); -⋮---- -// Environment should NOT have been mutated by compute alone. -assert!(std::env::var("PROXY_API_KEYS").is_err()); -⋮---- -// Cleanup -⋮---- -fn compute_skips_when_target_already_set() { -⋮---- -// Should NOT contain PROXY_API_KEYS since the target is already set. -let found = overrides.iter().any(|(k, _)| *k == "PROXY_API_KEYS"); -assert!(!found); -⋮---- -fn apply_env_aliases_sets_vars() { -⋮---- -apply_env_aliases(); -⋮---- -assert_eq!(std::env::var("PROXY_API_KEYS").unwrap(), "sk-test-master"); - - - -/// LiteLLM config.yaml parser. -/// -/// Accepts LiteLLM's YAML config format (model_list, litellm_settings, -/// router_settings, general_settings) and converts it to anyllm-proxy's -/// MultiConfig + ModelRouter. -use std::collections::HashMap; -use std::sync::Arc; -⋮---- -use indexmap::IndexMap; -use serde::Deserialize; -⋮---- -// ---- Serde structs for LiteLLM config.yaml ---- -⋮---- -pub(crate) struct LiteLLMConfig { -⋮---- -struct LiteLLMModelEntry { -⋮---- -struct LiteLLMParams { -⋮---- -// Azure-specific -⋮---- -// Bedrock-specific -⋮---- -// Catch unknown fields silently (LiteLLM has many we don't support). -⋮---- -struct LiteLLMSettings { -⋮---- -struct RouterSettings { -⋮---- -/// Map LiteLLM routing_strategy string to our enum. -fn parse_routing_strategy(s: &str) -> RoutingStrategy { -match s.to_ascii_lowercase().replace('_', "-").as_str() { -⋮---- -struct GeneralSettings { -⋮---- -// ---- Provider parsing ---- -⋮---- -/// Parse LiteLLM's "provider/model_name" format. -/// No prefix defaults to OpenAI (matches LiteLLM behavior). -fn parse_provider_model(model: &str) -> (BackendKind, String) { -let (provider, model_name) = model.split_once('/').unwrap_or(("openai", model)); -let kind = match provider.to_ascii_lowercase().as_str() { -⋮---- -(kind, model_name.to_string()) -⋮---- -// ---- Backend deduplication key ---- -⋮---- -/// Unique identity for a backend: same kind + base_url + api_key share one connection pool. -⋮---- -struct BackendKey { -⋮---- -/// Hash of the API key (not the key itself) to avoid holding secrets in hash keys. -⋮---- -fn hash_string(s: &str) -> u64 { -⋮---- -s.hash(&mut hasher); -hasher.finish() -⋮---- -// ---- Conversion ---- -⋮---- -/// Parse a LiteLLM config.yaml string and produce a MultiConfig + ModelRouter. -⋮---- -/// # Panics -/// On invalid YAML, missing required fields, or unresolvable env var references. -/// Parsed result from a LiteLLM config file. -pub struct LiteLLMParsed { -⋮---- -/// Webhook callback URLs from litellm_settings.callbacks (non-named entries). -⋮---- -/// True when "langfuse" appears in litellm_settings.callbacks. -⋮---- -/// Resolved `general_settings.master_key`, if present. -/// Caller should apply as PROXY_API_KEYS if that var is not already set. -⋮---- -pub fn from_litellm_yaml(yaml: &str) -> (MultiConfig, ModelRouter) { -let parsed = parse_litellm_yaml(yaml); -⋮---- -pub fn parse_litellm_yaml(yaml: &str) -> LiteLLMParsed { -⋮---- -serde_yaml::from_str(yaml).unwrap_or_else(|e| panic!("invalid LiteLLM config YAML: {e}")); -⋮---- -if config.model_list.is_empty() { -panic!("LiteLLM config must define at least one entry in model_list"); -⋮---- -// Resolve general_settings.master_key but do not call set_var here. -// The caller applies it in the consolidated env override block. -⋮---- -let mk = gs.master_key.as_ref().map(|mk| { -resolve_env_value(mk).unwrap_or_else(|e| panic!("general_settings.master_key: {e}")) -⋮---- -// Log unsupported keys at warn. -for key in gs._extra.keys() { -⋮---- -for key in ls._extra.keys() { -⋮---- -for key in rs._extra.keys() { -⋮---- -.ok() -.and_then(|v| v.parse().ok()) -.unwrap_or(3000); -⋮---- -.map(|v| v == "true" || v == "1") -.unwrap_or(false); -⋮---- -// Group model_list entries into deduplicated backends + deployment list. -⋮---- -// model_name -> Vec<(backend_name, actual_model, rpm, tpm)> -⋮---- -let (kind, actual_model) = parse_provider_model(&entry.litellm_params.model); -⋮---- -.as_deref() -.map(|v| resolve_env_value(v).unwrap_or_else(|e| panic!("model_list api_key: {e}"))) -.unwrap_or_default(); -⋮---- -let base_url = resolve_base_url(&kind, params); -⋮---- -kind: format!("{kind:?}"), -base_url: base_url.clone(), -api_key_hash: hash_string(&api_key), -⋮---- -let backend_name = if let Some((name, _)) = backend_map.get(&bk) { -name.clone() -⋮---- -let name = format!("litellm_{backend_counter}"); -⋮---- -let bc = build_backend_config( -⋮---- -backend_map.insert(bk, (name.clone(), bc)); -⋮---- -.entry(entry.model_name.clone()) -.or_default() -.push(DeploymentSpec { -⋮---- -// Build MultiConfig backends (ordered). -⋮---- -for (name, bc) in backend_map.values() { -backends.insert(name.clone(), bc.clone()); -⋮---- -.keys() -.next() -.cloned() -.expect("at least one backend"); -⋮---- -// Determine routing strategy from router_settings. -⋮---- -.as_ref() -.and_then(|rs| rs.routing_strategy.as_deref()) -.map(parse_routing_strategy) -⋮---- -// Build ModelRouter. -⋮---- -.into_iter() -.map(|s| { -⋮---- -s.weight.unwrap_or(1), -⋮---- -.collect(); -routes.insert(model_name, deployments); -⋮---- -.map(|s| s.callbacks.clone()) -⋮---- -let langfuse_requested = callbacks.iter().any(|c| c.eq_ignore_ascii_case("langfuse")); -⋮---- -.filter(|c| !c.eq_ignore_ascii_case("langfuse")) -⋮---- -struct DeploymentSpec { -⋮---- -/// Determine the base URL for a deployment, applying provider-specific defaults. -fn resolve_base_url(kind: &BackendKind, params: &LiteLLMParams) -> String { -⋮---- -resolve_env_value(url).unwrap_or_else(|e| panic!("model_list api_base: {e}")); -⋮---- -BackendKind::OpenAI => "https://api.openai.com".to_string(), -⋮---- -"https://generativelanguage.googleapis.com/v1beta/openai".to_string() -⋮---- -BackendKind::Anthropic => "https://api.anthropic.com".to_string(), -⋮---- -// For Bedrock, base_url stores the region. -⋮---- -.map(|v| v.to_string()) -.or_else(|| std::env::var("AWS_REGION").ok()) -.unwrap_or_else(|| "us-east-1".to_string()) -⋮---- -// Azure and Vertex require api_base in the config. -⋮---- -panic!("api_base is required for azure deployments in model_list") -⋮---- -panic!("api_base is required for vertex deployments in model_list") -⋮---- -/// Build a BackendConfig from LiteLLM model_list params. -⋮---- -fn build_backend_config( -⋮---- -BackendKind::AzureOpenAI => BackendAuth::AzureApiKey(api_key.to_string()), -BackendKind::Gemini | BackendKind::Vertex => BackendAuth::GoogleApiKey(api_key.to_string()), -_ => BackendAuth::BearerToken(api_key.to_string()), -⋮---- -// For Azure, build deployment URL from api_base. -⋮---- -let api_version = params.api_version.as_deref().unwrap_or("2024-10-21"); -// LiteLLM api_base for Azure is the resource endpoint. -// We need to append the deployment path. -if base_url.contains("/openai/deployments/") { -// Already a full deployment URL. -base_url.to_string() -⋮---- -format!( -⋮---- -// Validate non-Azure URLs. -⋮---- -if let Err(e) = validate_base_url(base_url) { -panic!("backend '{name}' base_url rejected: {e}"); -⋮---- -// Bedrock credentials. -⋮---- -.map(|v| resolve_env_value(v).unwrap_or_else(|e| panic!("backend '{name}': {e}"))) -⋮---- -.unwrap_or_else(|| "us-east-1".to_string()); -⋮---- -.or_else(|| std::env::var("AWS_ACCESS_KEY_ID").ok()) -.unwrap_or_else(|| panic!("backend '{name}': aws_access_key_id required for bedrock")); -⋮---- -.or_else(|| std::env::var("AWS_SECRET_ACCESS_KEY").ok()) -.unwrap_or_else(|| { -panic!("backend '{name}': aws_secret_access_key required for bedrock") -⋮---- -// Store region as base_url for Bedrock (matches existing convention). -// The effective_url was already set to the region string. -let _ = region; // region is used as base_url via resolve_base_url -⋮---- -Some(aws_credential_types::Credentials::new( -⋮---- -None, // session token not commonly in LiteLLM configs -⋮---- -// Placeholder model mapping: with model router, these are not used for routing. -// They serve as fallback for Anthropic model name translation if needed. -⋮---- -let _num_retries = config.litellm_settings.as_ref().and_then(|s| s.num_retries); -⋮---- -.and_then(|s| s.request_timeout); -⋮---- -kind: kind.clone(), -api_key: api_key.to_string(), -⋮---- -tls: tls.clone(), -⋮---- -mod tests { -⋮---- -fn parse_provider_model_openai() { -let (kind, model) = parse_provider_model("openai/gpt-4o"); -assert_eq!(kind, BackendKind::OpenAI); -assert_eq!(model, "gpt-4o"); -⋮---- -fn parse_provider_model_azure() { -let (kind, model) = parse_provider_model("azure/gpt-4o-eu"); -assert_eq!(kind, BackendKind::AzureOpenAI); -assert_eq!(model, "gpt-4o-eu"); -⋮---- -fn parse_provider_model_no_prefix() { -let (kind, model) = parse_provider_model("gpt-4o"); -⋮---- -fn parse_provider_model_vertex_ai() { -let (kind, model) = parse_provider_model("vertex_ai/gemini-pro"); -assert_eq!(kind, BackendKind::Vertex); -assert_eq!(model, "gemini-pro"); -⋮---- -fn parse_provider_model_bedrock() { -let (kind, model) = parse_provider_model("bedrock/anthropic.claude-v2"); -assert_eq!(kind, BackendKind::Bedrock); -assert_eq!(model, "anthropic.claude-v2"); -⋮---- -fn parse_provider_model_unknown_treated_as_openai() { -let (kind, model) = parse_provider_model("groq/llama-70b"); -⋮---- -assert_eq!(model, "llama-70b"); -⋮---- -fn minimal_litellm_config() { -⋮---- -let (multi, router) = from_litellm_yaml(yaml); -assert_eq!(multi.backends.len(), 1); -assert!(router.has_model("gpt-4o")); -⋮---- -let routed = router.route("gpt-4o").unwrap(); -assert_eq!(routed.actual_model, "gpt-4o"); -⋮---- -fn multiple_deployments_same_model() { -⋮---- -// Different api_keys = different backends -assert_eq!(multi.backends.len(), 2); -⋮---- -// Should round-robin between the two -let r0 = router.route("gpt-4o").unwrap(); -let r1 = router.route("gpt-4o").unwrap(); -assert_ne!(r0.backend_name, r1.backend_name); -⋮---- -fn backend_deduplication() { -⋮---- -// Same provider + base_url + api_key = one backend -⋮---- -assert!(router.has_model("gpt-4o-mini")); -⋮---- -fn os_environ_syntax_in_litellm_yaml() { -// Set env var for test -⋮---- -let (multi, _) = from_litellm_yaml(yaml); -let bc = multi.backends.values().next().unwrap(); -assert_eq!(bc.api_key, "sk-from-env"); -⋮---- -fn unknown_settings_are_accepted() { -⋮---- -// Should not panic; unknown fields are captured by serde(flatten). -⋮---- -fn routing_strategy_parsed() { -⋮---- -let (_, router) = from_litellm_yaml(yaml); -assert_eq!(router.strategy(), RoutingStrategy::LeastBusy); -⋮---- -fn routing_strategy_latency() { -⋮---- -assert_eq!(router.strategy(), RoutingStrategy::LatencyBased); -⋮---- -fn routing_strategy_cost_based() { -⋮---- -assert_eq!(router.strategy(), RoutingStrategy::CostBased); -⋮---- -fn routing_strategy_defaults_to_round_robin() { -⋮---- -assert_eq!(router.strategy(), RoutingStrategy::RoundRobin); -⋮---- -fn weight_field_parsed() { -⋮---- -assert_eq!(router.strategy(), RoutingStrategy::Weighted); -⋮---- -fn langfuse_callback_sets_flag() { -⋮---- -assert!(parsed.langfuse_requested); -assert!(parsed.callback_urls.is_empty()); // "langfuse" filtered out -⋮---- -fn webhook_url_not_flagged_as_langfuse() { -⋮---- -assert!(!parsed.langfuse_requested); -assert_eq!(parsed.callback_urls.len(), 1); -⋮---- -fn empty_model_list_panics() { -⋮---- -from_litellm_yaml(yaml); -⋮---- -fn gemini_provider() { -⋮---- -assert_eq!(bc.kind, BackendKind::Gemini); -assert!(router.has_model("gemini-pro")); - - - -// Anthropic passthrough handler: forwards raw request bytes to the real Anthropic API. -// No translation: the proxy receives Anthropic format and returns Anthropic format. -⋮---- -use crate::backend::BackendClient; -⋮---- -use super::routes::AppState; -⋮---- -pub(crate) async fn anthropic_passthrough( -⋮---- -state.metrics.record_request(); -⋮---- -"Backend is not configured as anthropic passthrough".to_string(), -⋮---- -return (StatusCode::INTERNAL_SERVER_ERROR, Json(err)).into_response(); -⋮---- -// Collect Anthropic-specific client headers to forward upstream. -// anthropic-beta enables beta features; must reach upstream to take effect. -// x-claude-code-session-id allows upstream and intermediary proxies to correlate sessions. -⋮---- -.iter() -.filter_map(|&name| { -⋮---- -.get(name) -.and_then(|v| v.to_str().ok()) -.map(|v| (name, v)) -⋮---- -.collect(); -⋮---- -// Peek at just the `stream` field instead of parsing the full body. -// Full deserialization would be wasteful for image-heavy requests -// (up to 32MB) when we only need one boolean to choose the handler. -⋮---- -struct StreamPeek { -⋮---- -.map(|p| p.stream) -.unwrap_or(false); -⋮---- -match client.forward_stream(body, &extra_headers).await { -⋮---- -state.metrics.record_success(); -// Pipe the raw SSE stream through to the client -let stream = response.bytes_stream(); -let mut resp = axum::body::Body::from_stream(stream).into_response(); -resp.headers_mut() -.insert("content-type", "text/event-stream".parse().unwrap()); -⋮---- -.insert("cache-control", "no-cache".parse().unwrap()); -rate_limits.inject_anthropic_response_headers(resp.headers_mut()); -⋮---- -state.metrics.record_error(); -passthrough_error_to_response(e) -⋮---- -match client.forward(body, &extra_headers).await { -⋮---- -.into_response(); -⋮---- -/// Convert an AnthropicClientError into a Response. -/// For API errors, return the upstream error body directly (it's already Anthropic format). -fn passthrough_error_to_response( -⋮---- -use crate::backend::anthropic_client::AnthropicClientError; -⋮---- -StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); -(http_status, [("content-type", "application/json")], body).into_response() -⋮---- -.to_string(), -⋮---- -(StatusCode::BAD_GATEWAY, Json(err)).into_response() - - - -// Webhook callback support for request completion notifications. -// -// Fires HTTP POST to configured webhook URLs after each request completes. -// Fire-and-forget: spawned tasks, no impact on request latency. -⋮---- -use crate::admin::state::RequestLogEntry; -use crate::config::validate_base_url; -use crate::integrations::NamedIntegration; -⋮---- -use std::sync::Arc; -⋮---- -/// Configuration for webhook callbacks. -⋮---- -pub struct CallbackConfig { -/// Webhook URLs to POST to on request completion. -⋮---- -/// Named (non-URL) integrations such as Langfuse. -⋮---- -/// Shared HTTP client with timeout. -⋮---- -impl CallbackConfig { -/// Create a new CallbackConfig from a list of webhook URLs. -/// URLs that don't start with http:// or https:// are skipped with a warning. -pub fn new(urls: Vec) -> Option> { -Self::with_named(urls, vec![]) -⋮---- -/// Create a CallbackConfig with both webhook URLs and named integrations. -/// Returns None only when both valid_urls and named are empty. -/// URLs pointing to private/loopback/metadata IP ranges are rejected to prevent SSRF. -pub fn with_named(urls: Vec, named: Vec) -> Option> { -⋮---- -.into_iter() -.filter(|u| { -if !u.starts_with("http://") && !u.starts_with("https://") { -⋮---- -// Reject private/loopback/metadata targets to prevent SSRF. -if let Err(reason) = validate_base_url(u) { -⋮---- -.collect(); -⋮---- -// Warn on plaintext HTTP (all private/loopback URLs already rejected above). -⋮---- -if url.starts_with("http://") { -⋮---- -if valid_urls.is_empty() && named.is_empty() { -⋮---- -let client = build_http_client(&HttpClientConfig { -⋮---- -connect_timeout: Some(std::time::Duration::from_secs(5)), -read_timeout: Some(std::time::Duration::from_secs(10)), -⋮---- -Some(Arc::new(Self { -⋮---- -/// Create from WEBHOOK_URLS env var (comma-separated). -pub fn from_env() -> Option> { -let urls_str = std::env::var("WEBHOOK_URLS").ok()?; -⋮---- -.split(',') -.map(|s| s.trim().to_string()) -.filter(|s| !s.is_empty()) -⋮---- -/// Fire-and-forget: POST the request log entry to all configured webhooks. -pub fn notify(&self, entry: &RequestLogEntry) { -⋮---- -let client = self.client.clone(); -let url = url.clone(); -let payload = payload.clone(); -⋮---- -match client.post(&url).json(&payload).send().await { -⋮---- -if !resp.status().is_success() { -⋮---- -integration.notify(entry); -⋮---- -/// Fire-and-forget: POST an arbitrary JSON payload to all configured webhooks. -/// Used for spend alerts and other event types beyond request completion. -pub fn notify_json(&self, payload: &serde_json::Value) { -⋮---- -/// Number of configured webhook URLs. -pub fn url_count(&self) -> usize { -self.urls.len() -⋮---- -/// Number of configured named integrations. -pub fn named_count(&self) -> usize { -self.named.len() -⋮---- -mod tests { -⋮---- -fn filters_non_url_callbacks() { -// Non-URL strings and localhost/private URLs are all filtered. -let config = CallbackConfig::new(vec![ -⋮---- -"langfuse".to_string(), // not a URL, filtered -"http://localhost:9999/cb".to_string(), // loopback, rejected (SSRF) -⋮---- -let config = config.unwrap(); -// Only the public HTTPS URL survives. -assert_eq!(config.url_count(), 1); -⋮---- -fn rejects_private_and_loopback_webhook_urls() { -⋮---- -"http://169.254.169.254/hook".to_string(), // cloud metadata -"http://10.0.0.1/hook".to_string(), // RFC 1918 -"http://127.0.0.1:9999/hook".to_string(), // loopback -"http://localhost:8080/hook".to_string(), // loopback hostname -⋮---- -// All URLs are private/loopback; no valid URLs remain. -assert!(config.is_none(), "private/loopback webhook URLs must be rejected"); -⋮---- -fn empty_urls_returns_none() { -assert!(CallbackConfig::new(vec![]).is_none()); -assert!(CallbackConfig::new(vec!["langfuse".to_string()]).is_none()); -⋮---- -fn valid_urls_creates_config() { -let config = CallbackConfig::new(vec!["https://hook.example.com".to_string()]); -assert!(config.is_some()); -⋮---- -fn http_plaintext_to_public_host_accepted() { -// Plaintext HTTP to a public hostname is accepted (with a warning). -// Private/loopback URLs are now rejected. -⋮---- -assert_eq!(config.url_count(), 2); -⋮---- -fn with_named_accepts_named_only() { -// with_named with no URLs but a named integration returns Some -// We can't construct LangfuseClient directly in tests easily, -// so just verify with_named([valid_url], []) == new([valid_url]) -⋮---- -vec!["https://example.com/hook".to_string()], -vec![], -⋮---- -let c2 = CallbackConfig::new(vec!["https://example.com/hook".to_string()]); -assert!(c1.is_some()); -assert!(c2.is_some()); -assert_eq!(c1.unwrap().url_count(), c2.unwrap().url_count()); - - - -//! Distributed rate limiting via Redis sorted sets. -//! -//! When `REDIS_URL` is set and the `redis` feature is enabled, RPM/TPM -//! checks are performed against Redis so multiple proxy instances share -//! rate limit state. On Redis failure, the proxy falls back to local -//! in-memory rate limiting (each instance limits independently). -⋮---- -/// Policy for handling Redis rate limiter errors. -⋮---- -pub enum RateLimitFailPolicy { -/// Allow requests when Redis is unavailable (default). -⋮---- -/// Reject requests when Redis is unavailable. -⋮---- -impl RateLimitFailPolicy { -pub fn from_env_str(s: &str) -> Self { -match s.to_lowercase().as_str() { -⋮---- -pub fn from_env() -> Self { -⋮---- -.map(|v| Self::from_env_str(&v)) -.unwrap_or(Self::Open) -⋮---- -use redis::aio::ConnectionManager; -⋮---- -/// Initialize the global Redis rate limiter. Called once from main. -⋮---- -pub fn set_redis_rate_limiter(limiter: RedisRateLimiter) { -let _ = REDIS_RATE_LIMITER.set(limiter); -⋮---- -/// Get the global Redis rate limiter, if initialized. -⋮---- -pub fn get_redis_rate_limiter() -> Option<&'static RedisRateLimiter> { -REDIS_RATE_LIMITER.get() -⋮---- -/// Stub when redis feature is not enabled. -⋮---- -pub fn get_redis_rate_limiter() -> Option<&'static ()> { -⋮---- -/// Redis-backed distributed rate limiter using sorted sets. -/// -/// Keys use the format `anyllm:rl:{key_hash_hex}:rpm` and `anyllm:rl:{key_hash_hex}:tpm`. -/// Each request is a member scored by its timestamp in milliseconds. -/// A Lua script atomically trims expired entries, checks the count/sum, -/// and adds the new entry if within limits. -⋮---- -pub struct RedisRateLimiter { -⋮---- -impl RedisRateLimiter { -/// Connect to Redis and create a rate limiter. -pub async fn new( -⋮---- -Ok(Self { conn, fail_policy }) -⋮---- -/// Get the underlying connection manager for reuse (e.g., by cache layer). -pub fn connection(&self) -> &ConnectionManager { -⋮---- -/// Check RPM limit. Returns Ok(()) if allowed, Err(retry_after_secs) if exceeded. -/// On Redis error, behavior depends on the configured `RateLimitFailPolicy`. -pub async fn check_rpm(&self, key_hash_hex: &str, limit: u32, now_ms: u64) -> Result<(), u64> { -let redis_key = format!("anyllm:rl:{key_hash_hex}:rpm"); -match self.check_rpm_inner(&redis_key, limit, now_ms).await { -⋮---- -Ok(()) -⋮---- -Err(60) -⋮---- -async fn check_rpm_inner( -⋮---- -let mut conn = self.conn.clone(); -let cutoff = now_ms.saturating_sub(60_000); -let member_id = format!("{now_ms}:{}", uuid::Uuid::new_v4().as_simple()); -⋮---- -// Hashed once at first use; avoids re-computing SHA1 per request. -⋮---- -.key(redis_key) -.arg(cutoff) -.arg(limit) -.arg(now_ms) -.arg(&member_id) -.invoke_async(&mut conn) -⋮---- -Ok(Ok(())) -⋮---- -let retry_after_ms = (oldest_ms + 60_000).saturating_sub(now_ms); -Ok(Err((retry_after_ms / 1000).max(1))) -⋮---- -/// Check TPM limit. Returns Ok(()) if allowed, Err(retry_after_secs) if exceeded. -⋮---- -pub async fn check_tpm(&self, key_hash_hex: &str, limit: u32, now_ms: u64) -> Result<(), u64> { -let redis_key = format!("anyllm:rl:{key_hash_hex}:tpm"); -match self.check_tpm_inner(&redis_key, limit, now_ms).await { -⋮---- -async fn check_tpm_inner( -⋮---- -// For TPM, members are scored by timestamp and the member value encodes the token count. -// We sum member names (which are "{tokens}:{uuid}") to get total tokens. -⋮---- -/// Record TPM tokens after a response is received. -pub async fn record_tpm(&self, key_hash_hex: &str, now_ms: u64, tokens: u32) { -⋮---- -let member = format!("{tokens}:{}", uuid::Uuid::new_v4().as_simple()); -⋮---- -.zadd(&redis_key, member, now_ms as f64) -.expire(&redis_key, 120) -.query_async(&mut conn) -⋮---- -mod tests { -use super::RateLimitFailPolicy; -⋮---- -fn get_redis_rate_limiter_returns_none_without_init() { -// When redis feature is not enabled, or when not initialized, -// the function should return None. -assert!(super::get_redis_rate_limiter().is_none()); -⋮---- -fn parse_rate_limit_fail_policy() { -assert!(matches!( -⋮---- -fn fail_policy_defaults_to_open() { -// When RATE_LIMIT_FAIL_POLICY is unset, from_env should return Open. -⋮---- -assert_eq!(policy, RateLimitFailPolicy::Open); - - - -// Streaming state machine: OpenAI chunks -> Anthropic SSE events -⋮---- -use crate::anthropic; -use crate::openai; -use crate::util; -⋮---- -// Safety cap: prevents unbounded Vec growth if a backend sends a -// malformed chunk with an absurdly large tool_call index. -⋮---- -/// State machine that converts OpenAI ChatCompletion chunks into Anthropic SSE events. -/// -/// Feed chunks via `process_chunk`, then call `finish` after the OpenAI `[DONE]` sentinel. -/// Each call returns zero or more Anthropic SSE events to forward to the client. -⋮---- -/// Anthropic: -/// OpenAI: -pub struct StreamingTranslator { -⋮---- -/// Tracks whether a thinking content block is open (for reasoning_content -/// from DeepSeek/Qwen thinking models). -⋮---- -/// Tool calls arrive incrementally across multiple chunks, indexed by -/// position in the OpenAI tool_calls array. We accumulate them here -/// so we can emit Anthropic's strict Start -> Delta* -> Stop sequence -/// per tool when finish_reason arrives. -⋮---- -struct ToolCallAccumulator { -⋮---- -impl StreamingTranslator { -/// Create a new streaming translator for the given model. -⋮---- -pub fn new(model: String) -> Self { -⋮---- -/// Process one OpenAI chunk and return zero or more Anthropic SSE events. -⋮---- -pub fn process_chunk( -⋮---- -// Emit message_start on first chunk -⋮---- -events.push(self.make_message_start()); -⋮---- -// Capture usage from the final chunk (OpenAI sends it with stream_options.include_usage) -⋮---- -usage.prompt_tokens_details.as_ref(), -⋮---- -// Handle reasoning_content (DeepSeek/Qwen thinking models). -// Emitted as a separate Anthropic thinking content block before text. -⋮---- -events.push(anthropic::StreamEvent::ContentBlockStart { -⋮---- -events.push(anthropic::StreamEvent::ContentBlockDelta { -⋮---- -thinking: reasoning.clone(), -⋮---- -// Handle text content deltas -⋮---- -// Close thinking block if transitioning from reasoning to content -⋮---- -events.push(anthropic::StreamEvent::ContentBlockStop { -⋮---- -delta: anthropic::Delta::TextDelta { text: text.clone() }, -⋮---- -// Handle refusals (safety filter triggered during streaming). -// Anthropic has no refusal type; surface as text so the client sees it. -⋮---- -// Handle tool call deltas -⋮---- -self.handle_tool_call_delta(tc, &mut events); -⋮---- -// Handle finish_reason -⋮---- -// Close any open thinking block -⋮---- -// Close any open text content block -⋮---- -// Flush any accumulated tool calls -self.flush_tool_calls(&mut events); -⋮---- -// Map OpenAI finish_reason to Anthropic stop_reason -let stop_reason = map_finish_reason(finish_reason); -⋮---- -events.push(anthropic::StreamEvent::MessageDelta { -⋮---- -stop_reason: Some(stop_reason), -⋮---- -usage: Some(anthropic::streaming::DeltaUsage { -⋮---- -/// Call after all chunks have been processed (when OpenAI sends `[DONE]`). -⋮---- -pub fn finish(&mut self) -> Vec { -⋮---- -events.push(anthropic::StreamEvent::MessageStop {}); -⋮---- -/// Return accumulated usage if any tokens were counted, None otherwise. -pub fn usage(&self) -> Option<&anthropic::Usage> { -⋮---- -Some(&self.usage) -⋮---- -fn make_message_start(&self) -> anthropic::StreamEvent { -⋮---- -id: self.message_id.clone(), -msg_type: "message".to_string(), -role: "assistant".to_string(), -content: vec![], -model: self.model.clone(), -⋮---- -usage: self.usage.clone(), -⋮---- -fn handle_tool_call_delta( -⋮---- -// Determine if this chunk starts a new tool call. OpenAI-compliant backends -// send `id` on the first chunk; local LLMs may omit `id` but include `name`. -let has_id = tc.id.is_some(); -let has_name = tc.function.as_ref().and_then(|f| f.name.as_ref()).is_some(); -⋮---- -// Bug 4 guard: if the accumulator at this index is already open (not closed), -// this is a continuation chunk (e.g., local LLM sending id:"" on every chunk), -// not a genuinely new tool call. -let already_active = self.active_tool_calls.get(idx).is_some_and(|tc| !tc.closed); -⋮---- -// Close any open text content block first -⋮---- -// Close the previous tool call block before starting a new one. -// Anthropic streaming protocol requires sequential: Start -> Delta -> Stop per block. -if let Some(last_tc) = self.active_tool_calls.last_mut() { -⋮---- -.as_ref() -.and_then(|f| f.name.clone()) -.unwrap_or_default(); -// Skip tool calls with empty name (matches non-streaming behavior). -if name.is_empty() { -let id_str = tc.id.as_deref().unwrap_or(""); -⋮---- -// Local LLMs may send empty or missing tool call ID -let tool_id = match tc.id.as_deref() { -Some(id) if !id.is_empty() => id.to_string(), -⋮---- -// OpenAI indexes tool calls within a single chunk (0, 1, 2...); -// Anthropic uses sequential content block indices across the -// entire message. Merge the two index spaces by offsetting. -⋮---- -name: name.clone(), -⋮---- -// Grow the accumulator vec to fit this index. OpenAI chunks may -// report tool calls out of order, so we pre-fill with defaults -// to avoid index-out-of-bounds, then overwrite at [idx]. -while self.active_tool_calls.len() <= idx { -self.active_tool_calls.push(ToolCallAccumulator { -⋮---- -closed: true, // Padding: never opened, so must not emit ContentBlockStop -⋮---- -// Emit argument fragments as input_json_delta events -⋮---- -if idx < self.active_tool_calls.len() { -⋮---- -partial_json: args.clone(), -⋮---- -fn flush_tool_calls(&mut self, events: &mut Vec) { -for tc in self.active_tool_calls.drain(..) { -⋮---- -/// Map OpenAI finish_reason to Anthropic stop_reason. -⋮---- -/// OpenAI: -/// Anthropic: -pub fn map_finish_reason(reason: &openai::FinishReason) -> anthropic::StopReason { -⋮---- -// Anthropic has no content_filter stop reason; EndTurn is the -// closest approximation. Refusal text is already surfaced via -// the refusal handling path above. -⋮---- -// Provider-specific reasons (e.g. DeepSeek "insufficient_system_resource") -⋮---- -mod tests { -⋮---- -/// Helper: build a ChatCompletionChunk with text content. -fn text_chunk(id: &str, model: &str, text: &str) -> ChatCompletionChunk { -⋮---- -id: id.into(), -object: "chat.completion.chunk".into(), -model: model.into(), -choices: vec![ChunkChoice { -⋮---- -/// Helper: build a chunk with only a role delta (first chunk from OpenAI). -fn role_chunk(id: &str, model: &str) -> ChatCompletionChunk { -⋮---- -/// Helper: build a chunk with finish_reason. -fn finish_chunk( -⋮---- -/// Helper: build a chunk with usage info (no choices). -fn usage_chunk(id: &str, model: &str, prompt: u32, completion: u32) -> ChatCompletionChunk { -⋮---- -choices: vec![], -usage: Some(crate::openai::ChatUsage { -⋮---- -/// Helper: build a chunk with a tool call delta. -fn tool_call_chunk( -⋮---- -id: id_str.into(), -⋮---- -fn first_chunk_emits_message_start() { -let mut translator = StreamingTranslator::new("gpt-4o".into()); -let chunk = role_chunk("chatcmpl-1", "gpt-4o"); -let events = translator.process_chunk(&chunk); -⋮---- -assert_eq!(events.len(), 1); -⋮---- -assert!(message.id.starts_with("msg_")); -assert_eq!(message.model, "gpt-4o"); -assert_eq!(message.role, "assistant"); -assert!(message.content.is_empty()); -assert!(message.stop_reason.is_none()); -⋮---- -other => panic!("expected MessageStart, got {:?}", other), -⋮---- -fn text_chunks_emit_block_start_and_deltas() { -⋮---- -// First text chunk: should emit message_start + content_block_start + delta -let events = translator.process_chunk(&text_chunk("c1", "gpt-4o", "Hello")); -assert_eq!(events.len(), 3); -assert!(matches!( -⋮---- -} => assert_eq!(text, "Hello"), -other => panic!("expected TextDelta, got {:?}", other), -⋮---- -// Second text chunk: only delta (no message_start, no block_start) -let events = translator.process_chunk(&text_chunk("c1", "gpt-4o", " world")); -⋮---- -} => assert_eq!(text, " world"), -⋮---- -fn finish_reason_stop_emits_block_stop_and_message_delta() { -⋮---- -translator.process_chunk(&text_chunk("c1", "gpt-4o", "Hi")); -⋮---- -translator.process_chunk(&finish_chunk("c1", "gpt-4o", openai::FinishReason::Stop)); -⋮---- -// Should emit: ContentBlockStop, MessageDelta -assert_eq!(events.len(), 2); -⋮---- -assert_eq!(delta.stop_reason, Some(anthropic::StopReason::EndTurn)); -assert!(usage.is_some()); -⋮---- -other => panic!("expected MessageDelta, got {:?}", other), -⋮---- -fn finish_emits_message_stop() { -⋮---- -let events = translator.finish(); -⋮---- -assert!(matches!(&events[0], anthropic::StreamEvent::MessageStop {})); -⋮---- -// Calling finish again should produce nothing -⋮---- -assert!(events.is_empty()); -⋮---- -fn usage_chunk_updates_token_counts() { -⋮---- -translator.process_chunk(&usage_chunk("c1", "gpt-4o", 10, 5)); -⋮---- -// The MessageDelta should carry the usage from the usage chunk -⋮---- -.iter() -.find(|e| matches!(e, anthropic::StreamEvent::MessageDelta { .. })); -⋮---- -let u = usage.as_ref().unwrap(); -assert_eq!(u.output_tokens, 5); -⋮---- -other => panic!("expected MessageDelta with usage, got {:?}", other), -⋮---- -fn tool_call_chunks_emit_tool_use_events() { -⋮---- -translator.process_chunk(&role_chunk("c1", "gpt-4o")); -⋮---- -// First tool call chunk: has id + name + partial args -let events = translator.process_chunk(&tool_call_chunk( -⋮---- -Some("call_abc"), -Some("get_weather"), -Some("{\"loc"), -⋮---- -// Should emit ContentBlockStart (tool_use) + ContentBlockDelta (input_json_delta) -⋮---- -assert_eq!(id, "call_abc"); -assert_eq!(name, "get_weather"); -⋮---- -other => panic!("expected ToolUse content block, got {:?}", other), -⋮---- -other => panic!("expected ContentBlockStart, got {:?}", other), -⋮---- -} => assert_eq!(partial_json, "{\"loc"), -other => panic!("expected InputJsonDelta, got {:?}", other), -⋮---- -// Continuation chunk: more args -⋮---- -Some("ation\": \"NYC\"}"), -⋮---- -} => assert_eq!(partial_json, "ation\": \"NYC\"}"), -⋮---- -fn tool_call_finish_flushes_and_emits_stop() { -⋮---- -translator.process_chunk(&tool_call_chunk( -⋮---- -Some("{\"location\": \"NYC\"}"), -⋮---- -let events = translator.process_chunk(&finish_chunk( -⋮---- -// Should emit: ContentBlockStop (for tool call), MessageDelta -⋮---- -assert_eq!(delta.stop_reason, Some(anthropic::StopReason::ToolUse)); -⋮---- -fn text_then_tool_call_closes_text_block() { -⋮---- -// Text content first -translator.process_chunk(&text_chunk("c1", "gpt-4o", "Let me check")); -⋮---- -// Then a tool call arrives: should close text block first -⋮---- -Some("call_xyz"), -Some("search"), -Some("{}"), -⋮---- -// ContentBlockStop (text, index 0), ContentBlockStart (tool, index 1), ContentBlockDelta -⋮---- -} => assert_eq!(id, "call_xyz"), -other => panic!( -⋮---- -fn empty_choices_chunk_only_emits_message_start() { -⋮---- -id: "c1".into(), -⋮---- -model: "gpt-4o".into(), -⋮---- -// Only message_start on first call -⋮---- -// Subsequent empty chunk: no events -⋮---- -fn map_finish_reason_length() { -assert_eq!( -⋮---- -fn map_finish_reason_content_filter() { -// Content filter maps to EndTurn (best approximation) -⋮---- -fn full_text_stream_sequence() { -⋮---- -// Simulate a complete text streaming sequence -⋮---- -all_events.extend(translator.process_chunk(&role_chunk("c1", "gpt-4o"))); -all_events.extend(translator.process_chunk(&text_chunk("c1", "gpt-4o", "Hello"))); -all_events.extend(translator.process_chunk(&text_chunk("c1", "gpt-4o", " world"))); -all_events.extend(translator.process_chunk(&usage_chunk("c1", "gpt-4o", 10, 5))); -all_events.extend(translator.process_chunk(&finish_chunk( -⋮---- -all_events.extend(translator.finish()); -⋮---- -// Verify event sequence: MessageStart, ContentBlockStart, TextDelta, TextDelta, -// ContentBlockStop, MessageDelta, MessageStop -⋮---- -.map(|e| match e { -⋮---- -.collect(); -⋮---- -// --- Local LLM robustness --- -⋮---- -fn streaming_tool_call_empty_id_gets_synthetic() { -let mut translator = StreamingTranslator::new("llama".into()); -translator.process_chunk(&role_chunk("c1", "llama")); -⋮---- -// First tool call chunk with empty string ID -⋮---- -Some(""), // empty ID from local LLM -Some("Read"), -Some("{\"file"), -⋮---- -assert!( -⋮---- -assert_eq!(name, "Read"); -⋮---- -other => panic!("expected ContentBlockStart with ToolUse, got {:?}", other), -⋮---- -fn streaming_tool_call_empty_name_skipped() { -⋮---- -// Tool call chunk with no name should be skipped (consistent with non-streaming). -⋮---- -Some("call_1"), -None, // no name from local LLM -⋮---- -// Empty name causes early return -- no ContentBlockStart emitted. -⋮---- -fn streaming_tool_call_none_id_with_name_gets_synthetic() { -// Bug 3: local LLMs may omit id entirely but provide name -⋮---- -// First chunk: id is None, but name is present -⋮---- -None, // no ID at all -⋮---- -// Second chunk: continuation with more arguments (no id, no name) -let events2 = translator.process_chunk(&tool_call_chunk( -⋮---- -Some("ation\"}"), -⋮---- -fn streaming_tool_call_repeated_empty_id_not_corrupted() { -// Bug 4: backend sends id:"" on every chunk of the same tool call; -// only the first chunk should open a new block. -⋮---- -// First chunk with empty id + name: opens a new tool block -let events1 = translator.process_chunk(&tool_call_chunk( -⋮---- -Some(""), -⋮---- -Some("{\"f"), -⋮---- -} => id.clone(), -⋮---- -// Second chunk with empty id again: should NOT open a new block -⋮---- -Some("ile\"}"), -⋮---- -// Should only have the argument delta, no new ContentBlockStart -⋮---- -// Verify no second synthetic ID was generated (only one ContentBlockStart total) -⋮---- -.chain(events2.iter()) -.filter(|e| matches!(e, anthropic::StreamEvent::ContentBlockStart { .. })) -⋮---- -// The synthetic ID from the first chunk should be used -assert!(first_id.starts_with("toolu_")); -⋮---- -fn streaming_refusal_emits_text_delta() { -⋮---- -id: "chatcmpl-1".into(), -⋮---- -// message_start + content_block_start + content_block_delta -⋮---- -match &events[events.len() - 1] { -⋮---- -assert!(text.contains("content policy violation")); -⋮---- -other => panic!("expected TextDelta with refusal, got {:?}", other), -⋮---- -/// Helper: build a chunk with reasoning_content (DeepSeek/Qwen thinking). -fn reasoning_chunk(id: &str, model: &str, reasoning: &str) -> ChatCompletionChunk { -⋮---- -fn reasoning_content_emits_thinking_block() { -let mut translator = StreamingTranslator::new("deepseek-reasoner".into()); -⋮---- -// First reasoning chunk should open a thinking block -⋮---- -translator.process_chunk(&reasoning_chunk("c1", "deepseek-reasoner", "Let me")); -assert_eq!(events.len(), 3); // message_start + content_block_start + thinking_delta -⋮---- -assert_eq!(*index, 0); -⋮---- -assert_eq!(thinking, "Let me"); -⋮---- -other => panic!("expected ThinkingDelta, got {:?}", other), -⋮---- -other => panic!("expected ContentBlockDelta, got {:?}", other), -⋮---- -// Second reasoning chunk continues the thinking block -⋮---- -translator.process_chunk(&reasoning_chunk("c1", "deepseek-reasoner", " think...")); -assert_eq!(events.len(), 1); // just a thinking delta -⋮---- -// Text chunk should close thinking block and open text block -let events = translator.process_chunk(&text_chunk("c1", "deepseek-reasoner", "Answer: 4")); -assert_eq!(events.len(), 3); // content_block_stop (thinking) + content_block_start (text) + text_delta -⋮---- -assert_eq!(*index, 1); -⋮---- -// Finish -⋮---- -// content_block_stop (text) + message_delta -⋮---- -fn reasoning_only_without_text_content() { -// Some thinking models may return only reasoning_content with no text content -⋮---- -translator.process_chunk(&reasoning_chunk("c1", "deepseek-reasoner", "Thinking...")); -⋮---- -// Should close thinking block + message_delta -⋮---- -fn usage_chunk_with_cached_tokens_maps_cache_read() { -⋮---- -prompt_tokens_details: Some(serde_json::json!({"cached_tokens": 42})), -⋮---- -translator.process_chunk(&chunk); -let usage = translator.usage().expect("usage should be present"); -assert_eq!(usage.input_tokens, 100); -assert_eq!(usage.output_tokens, 50); -assert_eq!(usage.cache_read_input_tokens, Some(42)); -⋮---- -fn usage_chunk_without_cached_tokens_leaves_cache_read_none() { -⋮---- -assert!(usage.cache_read_input_tokens.is_none()); - - - -// reqwest client for calling OpenAI endpoints -⋮---- -use anyllm_translate::openai; -use reqwest::Client; -⋮---- -/// HTTP client for OpenAI-compatible Chat Completions APIs with retry logic. -/// Works with both OpenAI and Vertex AI OpenAI-compatible endpoints. -/// -/// OpenAI: -⋮---- -pub struct OpenAIClient { -⋮---- -/// The backend kind, needed for constructing passthrough URLs at runtime. -⋮---- -/// Raw base URL from config, used to build passthrough endpoint URLs. -⋮---- -impl OpenAIClient { -/// Create a new client from proxy configuration. -/// Configures mTLS identity and custom CA cert if present in config. -pub fn new(config: &Config) -> Self { -let client = build_http_client(&config.tls); -⋮---- -// Each provider uses a different URL structure for the same API: -// - OpenAI: {base}/v1/chat/completions (base has no path) -// - Vertex: {base}/chat/completions (base ends at .../openapi) -// - Gemini: {base}/chat/completions (config appends /openai to base) -⋮---- -format!("{}/v1/chat/completions", config.openai_base_url), -format!("{}/v1/responses", config.openai_base_url), -format!("{}/v1/embeddings", config.openai_base_url), -⋮---- -format!("{}/chat/completions", config.openai_base_url), -// Vertex does not support Responses API; URL included for completeness -format!("{}/responses", config.openai_base_url), -format!("{}/embeddings", config.openai_base_url), -⋮---- -// openai_base_url already has /openai appended by config, -// producing .../v1beta/openai/chat/completions -⋮---- -// Gemini embeddings: .../v1beta/openai/embeddings -⋮---- -// Azure URL is pre-constructed in config (includes deployment + api-version). -// Embeddings and Responses URLs are derived by replacing the path component. -⋮---- -.split("/openai/deployments/") -.next() -.unwrap_or(&config.openai_base_url); -⋮---- -.split("api-version=") -.nth(1) -.unwrap_or("2024-10-21"); -⋮---- -.and_then(|s| s.split('/').next()) -.unwrap_or(""); -⋮---- -config.openai_base_url.clone(), -// Azure Responses API is not widely available; provide URL for completeness -format!("{endpoint}/openai/deployments/{deployment}/responses?api-version={api_version}"), -format!("{endpoint}/openai/deployments/{deployment}/embeddings?api-version={api_version}"), -⋮---- -unreachable!("OpenAIClient should not be constructed for Anthropic/Bedrock backend") -⋮---- -auth: config.backend_auth.clone(), -backend_kind: config.backend.clone(), -base_url: config.openai_base_url.clone(), -⋮---- -/// Returns the API key/token for use in batch API calls. -pub fn api_key(&self) -> String { -⋮---- -BackendAuth::BearerToken(k) => k.clone(), -BackendAuth::AzureApiKey(k) => k.clone(), -BackendAuth::GoogleApiKey(k) => k.clone(), -⋮---- -/// Returns the base URL for batch API calls. -⋮---- -/// For Gemini/Vertex the openai_base_url ends in /openai — strip that since the -/// batch endpoint is not on the OpenAI-compat path. -pub fn base_url_for_batch(&self) -> String { -⋮---- -.trim_end_matches("/openai") -.trim_end_matches('/') -.to_string() -⋮---- -/// Fallback error for unparseable error responses. The backend may return -/// HTML error pages (e.g., Cloudflare 502) that don't match ErrorResponse. -fn fallback_error(status: u16) -> openai::errors::ErrorResponse { -⋮---- -message: format!("OpenAI returned status {status}"), -error_type: "api_error".to_string(), -⋮---- -async fn send_with_retry( -⋮---- -/// Send a non-streaming chat completion request with retry on 429/5xx. -⋮---- -pub async fn chat_completion( -⋮---- -let response = self.send_with_retry(req).await?; -let status = response.status().as_u16(); -let rate_limits = RateLimitHeaders::from_openai_headers(response.headers()); -⋮---- -.map_err(OpenAIClientError::Deserialization)?; -Ok((body, status, rate_limits)) -⋮---- -/// Send a streaming chat completion request with retry on 429/5xx. -/// Returns the raw response and rate limit headers for SSE parsing once a -/// successful connection is established. -⋮---- -/// OpenAI: -pub async fn chat_completion_stream( -⋮---- -Ok((response, rate_limits)) -⋮---- -/// Send a non-streaming Responses API request with retry. -⋮---- -/// OpenAI Responses: -pub async fn responses( -⋮---- -/// Send a streaming Responses API request with retry. -/// Returns the raw response for SSE parsing. -⋮---- -/// OpenAI Responses streaming: -pub async fn responses_stream( -⋮---- -/// Build a passthrough URL for the given path suffix (e.g., "/v1/audio/speech"). -/// Adjusts for backend-specific URL schemes (Azure deployments, Vertex/Gemini paths). -pub fn passthrough_url(&self, path: &str) -> String { -⋮---- -BackendKind::OpenAI => format!("{}{}", self.base_url, path), -⋮---- -// Azure: {endpoint}/openai/deployments/{deployment}/{suffix}?api-version=... -⋮---- -.unwrap_or(&self.base_url); -⋮---- -// Strip leading /v1/ to get the resource name (e.g., "audio/speech") -let suffix = path.strip_prefix("/v1/").unwrap_or(path); -format!( -⋮---- -// Vertex/Gemini: base_url already has provider-specific prefix, -// just append the path without /v1 prefix -⋮---- -format!("{}/{}", self.base_url, suffix) -⋮---- -unreachable!("OpenAIClient should not be constructed for Anthropic/Bedrock") -⋮---- -/// Forward a raw request body to an arbitrary backend endpoint. -/// No retry: passthrough requests are forwarded once (callers can retry). -pub async fn raw_passthrough( -⋮---- -.post(url) -.body(body) -.header("content-type", content_type); -⋮---- -BackendAuth::BearerToken(token) => req.bearer_auth(token), -BackendAuth::GoogleApiKey(key) => req.header("x-goog-api-key", key), -BackendAuth::AzureApiKey(key) => req.header("api-key", key), -⋮---- -let response = req.send().await.map_err(OpenAIClientError::Request)?; -let status = axum::http::StatusCode::from_u16(response.status().as_u16()) -.unwrap_or(axum::http::StatusCode::INTERNAL_SERVER_ERROR); -⋮---- -if let Some(ct) = response.headers().get("content-type") { -resp_headers.insert("content-type", ct.clone()); -⋮---- -let resp_body = response.bytes().await.map_err(OpenAIClientError::Request)?; -Ok((status, resp_headers, resp_body)) -⋮---- -/// Forward a raw embeddings request body to the backend embeddings endpoint. -/// No retry: embeddings are idempotent but we keep it simple, callers can retry. -⋮---- -/// OpenAI: -pub async fn embeddings_passthrough( -⋮---- -self.raw_passthrough(&self.embeddings_url, body, content_type) -⋮---- -/// Errors from the OpenAI HTTP client. -⋮---- -pub enum OpenAIClientError { -/// Transport-level failure (DNS, TLS, connection refused, timeout). -⋮---- -/// Backend returned 2xx but the body was not valid ChatCompletionResponse JSON. -⋮---- -/// Backend returned a non-2xx status with a parseable OpenAI error body. -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -Self::Request(e) => write!(f, "request failed: {e}"), -Self::Deserialization(e) => write!(f, "response deserialization failed: {e}"), -⋮---- -write!(f, "OpenAI API error ({status}): {}", error.error.message) -⋮---- -impl OpenAIClientError { -/// HTTP status code from an API error, or 500 for transport/deserialization errors. -pub fn status_code(&self) -> u16 { -⋮---- -impl RetryableError for OpenAIClientError { -fn from_request(e: reqwest::Error) -> Self { -⋮---- -fn from_api_response(status: u16, body: &str) -> Self { -⋮---- -serde_json::from_str::(body).unwrap_or_else(|e| { -⋮---- -mod tests { -⋮---- -use std::time::Duration; -⋮---- -fn is_retryable_429() { -assert!(crate::backend::is_retryable(429)); -⋮---- -fn is_retryable_500() { -assert!(crate::backend::is_retryable(500)); -assert!(crate::backend::is_retryable(502)); -assert!(crate::backend::is_retryable(503)); -assert!(crate::backend::is_retryable(599)); -⋮---- -fn is_retryable_408() { -assert!(crate::backend::is_retryable(408)); -⋮---- -fn is_not_retryable_400() { -assert!(!crate::backend::is_retryable(400)); -assert!(!crate::backend::is_retryable(401)); -assert!(!crate::backend::is_retryable(404)); -assert!(!crate::backend::is_retryable(409)); -⋮---- -fn backoff_respects_retry_after() { -let delay = crate::backend::backoff_delay(0, Some(Duration::from_secs(5))); -assert_eq!(delay, Duration::from_secs(5)); -⋮---- -fn backoff_increases_with_attempt() { -⋮---- -assert!(d1 > d0); -assert!(d2 > d1); -⋮---- -fn parse_retry_after_valid() { -⋮---- -headers.insert("retry-after", "3".parse().unwrap()); -⋮---- -assert_eq!(dur, Some(Duration::from_secs(3))); -⋮---- -fn parse_retry_after_missing() { -⋮---- -assert_eq!(crate::backend::parse_retry_after(&headers), None); -⋮---- -fn parse_retry_after_http_date_future() { -// Use a date far in the future so it's always ahead of now -⋮---- -headers.insert( -⋮---- -"Wed, 21 Oct 2037 07:28:00 GMT".parse().unwrap(), -⋮---- -assert!(dur.is_some(), "future HTTP date should parse to Some"); -assert!(dur.unwrap().as_secs() > 0); -⋮---- -fn parse_retry_after_http_date_past() { -⋮---- -"Mon, 01 Jan 2024 00:00:00 GMT".parse().unwrap(), -⋮---- -// Past date: no wait needed -⋮---- -fn parse_retry_after_garbage() { -⋮---- -headers.insert("retry-after", "not-a-date-or-number".parse().unwrap()); -⋮---- -fn client_builds_without_tls() { -⋮---- -openai_api_key: "test".into(), -openai_base_url: "https://api.openai.com".into(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: BackendAuth::BearerToken("test".into()), -⋮---- -// Should not panic -⋮---- -fn client_builds_vertex_config() { -⋮---- -openai_base_url: "https://us-central1-aiplatform.googleapis.com/v1/projects/my-project/locations/us-central1/endpoints/openapi".into(), -⋮---- -big_model: "gemini-2.5-pro".into(), -small_model: "gemini-2.5-flash".into(), -⋮---- -backend_auth: BackendAuth::GoogleApiKey("test-key".into()), -⋮---- -// Verify URL construction for Vertex (no /v1 prefix) -assert!(client -⋮---- -fn embeddings_url_openai() { -⋮---- -assert_eq!( -⋮---- -fn embeddings_url_vertex() { -⋮---- -assert!( -⋮---- -fn embeddings_url_gemini() { -⋮---- -// Config appends /openai to the base, so this is what arrives here -openai_base_url: "https://generativelanguage.googleapis.com/v1beta/openai".into(), -⋮---- -backend_auth: BackendAuth::GoogleApiKey("test-gemini-key".into()), -⋮---- -fn azure_url_passthrough() { -⋮---- -openai_base_url: "https://myresource.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21".into(), -⋮---- -backend_auth: BackendAuth::AzureApiKey("test-azure-key".into()), -⋮---- -// Chat completions URL is the pre-built URL, unchanged -⋮---- -// Embeddings URL is derived from the endpoint and deployment - - - -// Batch processing types and JSONL validation. -// Implements OpenAI-compatible batch file upload and job management. -⋮---- -pub mod anthropic_batch; -pub mod db; -pub mod openai_batch_client; -pub mod routes; -⋮---- -use std::collections::HashSet; -⋮---- -/// Maximum number of lines in a JSONL batch file. -⋮---- -/// Maximum file size in bytes (100 MB). -⋮---- -/// Maximum length of a custom_id field. -⋮---- -/// A batch input file stored in SQLite. -⋮---- -pub struct BatchFile { -⋮---- -/// A batch processing job. -⋮---- -pub struct BatchJob { -⋮---- -/// Counts of requests within a batch job. -⋮---- -pub struct RequestCounts { -⋮---- -/// Batch job lifecycle status. -⋮---- -pub enum BatchStatus { -⋮---- -impl BatchStatus { -/// Convert from the string stored in SQLite. -pub fn from_str_status(s: &str) -> Self { -⋮---- -/// Convert to the string stored in SQLite. -pub fn as_str(&self) -> &'static str { -⋮---- -/// Result of JSONL validation: line count on success, error message on failure. -⋮---- -pub struct ValidatedJsonl { -⋮---- -/// Validate a JSONL batch file. -/// -/// Each line must be valid JSON with a unique `custom_id` (string, max 64 chars) -/// and a `body` object containing a `model` field. Max 50,000 lines, 100 MB. -⋮---- -/// Takes a `BufRead` to read line-by-line without requiring a contiguous UTF-8 -/// string for the entire file. -pub fn validate_jsonl(mut reader: impl std::io::BufRead) -> Result { -⋮---- -line_buf.clear(); -⋮---- -.read_line(&mut line_buf) -.map_err(|e| format!("Read error: {e}"))?; -⋮---- -break; // EOF -⋮---- -return Err(format!( -⋮---- -let line = line_buf.trim(); -if line.is_empty() { -⋮---- -return Err(format!("File exceeds maximum of {MAX_LINE_COUNT} lines")); -⋮---- -.map_err(|e| format!("Line {raw_line_num}: invalid JSON: {e}"))?; -⋮---- -.as_object() -.ok_or_else(|| format!("Line {raw_line_num}: expected JSON object"))?; -⋮---- -.get("custom_id") -.and_then(|v| v.as_str()) -.ok_or_else(|| format!("Line {raw_line_num}: missing or non-string 'custom_id'"))?; -⋮---- -if custom_id.len() > MAX_CUSTOM_ID_LEN { -⋮---- -if !seen_ids.insert(custom_id.to_string()) { -⋮---- -.get("body") -.and_then(|v| v.as_object()) -.ok_or_else(|| format!("Line {raw_line_num}: missing or non-object 'body'"))?; -⋮---- -if !body.contains_key("model") { -return Err(format!("Line {raw_line_num}: body missing 'model' field")); -⋮---- -return Err("File is empty".to_string()); -⋮---- -Ok(ValidatedJsonl { line_count }) -⋮---- -mod tests { -⋮---- -fn check(data: &str) -> Result { -validate_jsonl(BufReader::new(Cursor::new(data.as_bytes()))) -⋮---- -fn check_bytes(data: &[u8]) -> Result { -validate_jsonl(BufReader::new(Cursor::new(data))) -⋮---- -fn valid_jsonl() { -⋮---- -let result = check(data); -assert!(result.is_ok()); -assert_eq!(result.unwrap().line_count, 2); -⋮---- -fn missing_custom_id() { -⋮---- -assert!(result.is_err()); -assert!(result.unwrap_err().contains("custom_id")); -⋮---- -fn missing_body_model() { -⋮---- -assert!(result.unwrap_err().contains("model")); -⋮---- -fn duplicate_custom_id() { -⋮---- -assert!(result.unwrap_err().contains("duplicate")); -⋮---- -fn oversized_custom_id() { -let long_id = "a".repeat(65); -let data = format!(r#"{{"custom_id": "{long_id}", "body": {{"model": "gpt-4o"}}}}"#); -let result = check(&data); -⋮---- -assert!(result.unwrap_err().contains("maximum length")); -⋮---- -fn empty_file() { -let result = check_bytes(b""); -⋮---- -assert!(result.unwrap_err().contains("empty")); -⋮---- -fn invalid_json_line() { -⋮---- -let result = check_bytes(data); -⋮---- -assert!(result.unwrap_err().contains("invalid JSON")); -⋮---- -fn blank_lines_skipped() { -⋮---- -fn error_reports_absolute_line_number_with_blank_lines() { -// Blank line at position 1, bad JSON at position 2. -// Should report "Line 2", not "Line 1". -⋮---- -let err = check(data).unwrap_err(); -assert!(err.contains("Line 2"), "expected 'Line 2' in: {err}"); - - - -// Bedrock passthrough handler: forwards Anthropic-format requests to AWS Bedrock -// with SigV4 signing and AWS Event Stream decoding for streaming. -⋮---- -use crate::backend::BackendClient; -⋮---- -use bytes::BytesMut; -use futures::StreamExt; -⋮---- -use super::routes::AppState; -⋮---- -/// Bedrock passthrough handler for POST /v1/messages. -/// Strips the `model` field from the body (Bedrock uses it in the URL), -/// adds `anthropic_version`, and handles AWS Event Stream binary framing -/// for streaming responses. -pub(crate) async fn bedrock_passthrough(State(state): State, body: Bytes) -> Response { -state.metrics.record_request(); -⋮---- -BackendClient::Bedrock(c) => c.clone(), -⋮---- -"Backend is not configured as bedrock".to_string(), -⋮---- -return (StatusCode::INTERNAL_SERVER_ERROR, Json(err)).into_response(); -⋮---- -// Parse the body to extract model and stream fields, then rebuild for Bedrock. -⋮---- -format!("invalid JSON: {e}"), -⋮---- -return (StatusCode::BAD_REQUEST, Json(err)).into_response(); -⋮---- -// Extract model for the URL -⋮---- -.get("model") -.and_then(|v| v.as_str()) -.unwrap_or("") -.to_string(); -if model_id.is_empty() { -⋮---- -"model is required".to_string(), -⋮---- -// Map model name through model router or runtime config -let mapped_model = match state.resolve_model(&model_id) { -⋮---- -"all deployments for this model are at their RPM limit".to_string(), -⋮---- -return (StatusCode::TOO_MANY_REQUESTS, Json(err)).into_response(); -⋮---- -.get("stream") -.and_then(|v| v.as_bool()) -.unwrap_or(false); -⋮---- -// Bedrock: model goes in URL, not body. Add anthropic_version. -if let Some(obj) = parsed.as_object_mut() { -obj.remove("model"); -obj.insert( -"anthropic_version".to_string(), -serde_json::Value::String("bedrock-2023-05-31".to_string()), -⋮---- -format!("failed to serialize request: {e}"), -⋮---- -bedrock_stream(state, &client, bedrock_body, &mapped_model).await -⋮---- -bedrock_non_stream(state, &client, bedrock_body, &mapped_model).await -⋮---- -/// Non-streaming Bedrock request. -async fn bedrock_non_stream( -⋮---- -match client.forward(body, model_id).await { -⋮---- -state.metrics.record_success(); -⋮---- -.into_response(); -rate_limits.inject_anthropic_response_headers(resp.headers_mut()); -⋮---- -state.metrics.record_error(); -bedrock_error_to_response(e) -⋮---- -/// Streaming Bedrock request. Decodes AWS Event Stream binary frames into -/// Anthropic SSE events and re-emits them as standard SSE. -async fn bedrock_stream( -⋮---- -let (response, rate_limits) = match client.forward_stream(body, model_id).await { -⋮---- -return bedrock_error_to_response(e); -⋮---- -let metrics = state.metrics.clone(); -⋮---- -let mut byte_stream = response.bytes_stream(); -⋮---- -while let Some(chunk_result) = byte_stream.next().await { -⋮---- -metrics.record_error(); -⋮---- -event_buf.extend_from_slice(&bytes); -⋮---- -if event_buf.len() > crate::backend::MAX_SSE_BUFFER_SIZE { -⋮---- -// Decode all complete frames in the buffer -⋮---- -// Re-emit as SSE: "event: \ndata: \n\n" -// Bedrock events are raw Anthropic JSON; detect the event type. -let event_type = detect_event_type(&event_json); -let sse_line = format!("event: {event_type}\ndata: {event_json}\n\n"); -if tx.send(Ok(sse_line)).await.is_err() { -return; // client disconnected -⋮---- -metrics.record_success(); -⋮---- -.status(StatusCode::OK) -.header("content-type", "text/event-stream") -.header("cache-control", "no-cache") -.body(body) -.unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()); -⋮---- -/// Extract the Anthropic SSE event type from a JSON string. -/// Parses only the top-level `type` field via serde_json to avoid brittle -/// substring matching that fails on whitespace-formatted JSON or nested fields. -/// Falls back to "message" on any parse failure or unrecognized event type. -fn detect_event_type(json: &str) -> &'static str { -⋮---- -struct EventType<'a> { -⋮---- -match parsed.as_ref().map(|e| e.event_type) { -⋮---- -/// Convert a BedrockClientError into a Response. -fn bedrock_error_to_response(error: BedrockClientError) -> Response { -⋮---- -StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); -// Try to return body as-is (Bedrock may return JSON error) -(http_status, [("content-type", "application/json")], body).into_response() -⋮---- -.to_string(), -⋮---- -(StatusCode::BAD_GATEWAY, Json(err)).into_response() -⋮---- -"Failed to sign request for AWS Bedrock.".to_string(), -⋮---- -(StatusCode::INTERNAL_SERVER_ERROR, Json(err)).into_response() -⋮---- -mod tests { -use super::detect_event_type; -⋮---- -fn detect_message_start() { -assert_eq!(detect_event_type(r#"{"type":"message_start","message":{"id":"msg-1"}}"#), "message_start"); -⋮---- -fn detect_content_block_delta() { -assert_eq!( -⋮---- -fn detect_falls_back_for_unknown_type() { -assert_eq!(detect_event_type(r#"{"type":"some_future_event"}"#), "message"); -⋮---- -fn detect_falls_back_on_malformed_json() { -assert_eq!(detect_event_type("not json at all"), "message"); -⋮---- -fn detect_handles_spaced_json() { -assert_eq!(detect_event_type(r#"{ "type" : "message_stop" }"#), "message_stop"); -⋮---- -fn detect_ignores_nested_type_field() { -// Top-level type is content_block_delta; nested delta.type is text_delta. -⋮---- -assert_eq!(detect_event_type(json), "content_block_delta"); - - - -/// Admin server: localhost-only config management, request logging, WebSocket live updates. -pub mod admin; -/// Backend HTTP clients for OpenAI, Vertex, Gemini, and Anthropic passthrough. -pub mod backend; -/// Async batch job submission and management (US3). -pub mod batch; -/// Response caching with in-memory (moka) and optional Redis tier (US1). -pub mod cache; -/// Webhook callback support for request completion notifications. -pub mod callbacks; -/// Named integration registry (Langfuse, etc.). -pub mod integrations; -/// Environment-based configuration, TLS client cert setup, URL validation. -pub mod config; -/// Per-request cost tracking and model pricing (US4). -pub mod cost; -/// Backend fallback chains for transparent failover (US2). -pub mod fallback; -/// Request count, success/error tracking, exposed via GET /metrics. -pub mod metrics; -/// Optional OpenTelemetry OTLP trace export (requires `otel` feature). -⋮---- -pub mod otel; -/// Distributed rate limiting via Redis sorted sets (requires `redis` feature). -pub mod ratelimit; -/// Axum HTTP server: routes, middleware (auth, request ID, size/concurrency limits), SSE streaming. -pub mod server; - - - -/// Anthropic batch JSONL <-> OpenAI batch JSONL translation functions. -pub mod batch_map; -/// HTTP status and error shape translation between APIs. -pub mod errors_map; -/// Anthropic Messages API <-> Gemini generateContent API message mapping. -pub mod gemini_message_map; -/// Gemini streaming: full-response diffing -> Anthropic SSE delta events. -pub mod gemini_streaming_map; -/// Message and content block translation (system prompt, text, images, documents). -pub mod message_map; -/// Anthropic to/from OpenAI Responses API request and response mapping. -pub mod responses_message_map; -/// Responses API SSE event stream translation state machine. -pub mod responses_streaming_map; -/// Reverse message mapping: OpenAI Chat Completions -> Anthropic Messages. -pub mod reverse_message_map; -/// Reverse streaming: Anthropic SSE events -> OpenAI ChatCompletionChunk SSE. -pub mod reverse_streaming_map; -/// Chat Completions SSE event stream translation state machine. -pub mod streaming_map; -/// Tool definitions and tool_use/tool_call translation. -pub mod tools_map; -/// Token usage field mapping between Anthropic and OpenAI formats. -pub mod usage_map; -/// Degradation warning collection for client-visible feature-drop signals. -pub mod warnings; -⋮---- -/// Format an OpenAI refusal string as Anthropic text content. -/// Anthropic has no refusal type, so we surface it as a bracketed text marker. -pub(crate) fn format_refusal(refusal: &str) -> String { -format!("[Refusal: {}]", refusal) - - - -//! # anyllm_translate -//! -//! Pure, IO-free translation between Anthropic Messages API and OpenAI Chat Completions API. -⋮---- -//! # Quick start -⋮---- -//! ```rust -//! use anyllm_translate::{TranslationConfig, translate_request, translate_response}; -//! use anyllm_translate::anthropic::MessageCreateRequest; -⋮---- -//! let config = TranslationConfig::builder() -//! .model_map("haiku", "gpt-4o-mini") -//! .model_map("sonnet", "gpt-4o") -//! .model_map("opus", "gpt-4o") -//! .build(); -⋮---- -//! let req: MessageCreateRequest = serde_json::from_str(r#"{ -//! "model": "claude-sonnet-4-6", -//! "max_tokens": 100, -//! "messages": [{"role": "user", "content": "Hello"}] -//! }"#).unwrap(); -⋮---- -//! let openai_req = translate_request(&req, &config).unwrap(); -//! assert_eq!(openai_req.model, "gpt-4o"); -⋮---- -//! // ... send openai_req to OpenAI, get response ... -//! // let anthropic_resp = translate_response(&openai_resp, &req.model); -//! ``` -⋮---- -//! # Modules -⋮---- -//! - [`anthropic`] -- Anthropic Messages API types -//! - [`openai`] -- OpenAI Chat Completions and Responses API types -//! - [`mapping`] -- Stateless conversion functions between APIs -//! - [`config`] -- Translation configuration (model mapping, lossy behavior) -//! - [`translate`] -- Convenience wrappers combining config with mapping functions -⋮---- -/// Anthropic Messages API types (request, response, streaming events, errors). -pub mod anthropic; -/// Translation configuration: model mapping and lossy-translation behavior. -pub mod config; -/// Error types for translation failures. -pub mod error; -/// Gemini native generateContent API types (request, response). -pub mod gemini; -/// Stateless conversion functions between Anthropic and OpenAI API formats. -pub mod mapping; -/// HTTP middleware for request/response translation (requires `middleware` feature). -⋮---- -pub mod middleware; -/// OpenAI Chat Completions and Responses API types. -pub mod openai; -/// Convenience wrappers combining config with mapping functions. -pub mod translate; -/// Shared utilities: ID generation, JSON helpers, secret redaction. -pub mod util; -⋮---- -// Convenience re-exports -⋮---- -pub use error::TranslateError; -pub use mapping::reverse_streaming_map::ReverseStreamingTranslator; - - - -// Langfuse named callback integration. -// -// Sends LLM generation events to Langfuse's batch ingestion API. -// Activated when LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY are set, -// either explicitly via env vars or when "langfuse" appears in -// litellm_settings.callbacks. -⋮---- -// Fire-and-forget: send() spawns a tokio task and returns immediately. -⋮---- -use crate::admin::state::RequestLogEntry; -use std::sync::Arc; -⋮---- -pub struct LangfuseClient { -⋮---- -impl LangfuseClient { -/// Construct from env vars: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_HOST. -/// Returns None when either required key is absent or empty. -pub fn from_env() -> Option> { -let public_key = std::env::var("LANGFUSE_PUBLIC_KEY").ok()?; -let secret_key = std::env::var("LANGFUSE_SECRET_KEY").ok()?; -if public_key.is_empty() || secret_key.is_empty() { -⋮---- -.unwrap_or_else(|_| "https://cloud.langfuse.com".to_string()); -⋮---- -.timeout(std::time::Duration::from_secs(5)) -.build() -.expect("langfuse http client"); -Some(Arc::new(Self { -⋮---- -/// Fire-and-forget: POST generation event to Langfuse ingestion API. -/// Returns immediately; the HTTP call happens in a spawned tokio task. -pub fn send(&self, entry: &RequestLogEntry) { -let payload = build_generation_payload(entry); -let url = format!("{}/api/public/ingestion", self.host.trim_end_matches('/')); -let auth = format!( -⋮---- -let client = self.client.clone(); -⋮---- -.post(&url) -.header("Authorization", &auth) -.json(&payload) -.send() -⋮---- -if !resp.status().is_success() { -⋮---- -/// Build the Langfuse batch ingestion payload for a single generation. -pub(crate) fn build_generation_payload(entry: &RequestLogEntry) -> serde_json::Value { -⋮---- -let start_time = iso8601_to_epoch_ms(end_time) -.map(|end_ms| { -let start_ms = end_ms.saturating_sub(entry.latency_ms); -⋮---- -.unwrap_or_else(|| end_time.clone()); -⋮---- -.as_deref() -.or(entry.model_requested.as_deref()) -.unwrap_or("unknown"); -⋮---- -let level = if entry.error_message.is_some() { "ERROR" } else { "DEFAULT" }; -⋮---- -/// Encode bytes as standard base64 (RFC 4648). No external dependencies. -pub(crate) fn base64_encode(input: &[u8]) -> String { -⋮---- -let mut out = String::with_capacity(input.len().div_ceil(3) * 4); -⋮---- -while i + 2 < input.len() { -⋮---- -out.push(CHARS[((b >> 18) & 0x3f) as usize] as char); -out.push(CHARS[((b >> 12) & 0x3f) as usize] as char); -out.push(CHARS[((b >> 6) & 0x3f) as usize] as char); -out.push(CHARS[(b & 0x3f) as usize] as char); -⋮---- -let rem = input.len() - i; -⋮---- -out.push('='); -⋮---- -/// Parse ISO 8601 UTC timestamp ("2026-03-27T10:15:30Z") to Unix epoch seconds. -/// Returns None on parse failure. -pub fn iso8601_to_epoch(s: &str) -> Option { -if s.len() < 20 { -⋮---- -let year: i64 = s[0..4].parse().ok()?; -let month: i64 = s[5..7].parse().ok()?; -let day: i64 = s[8..10].parse().ok()?; -let hour: i64 = s[11..13].parse().ok()?; -let min: i64 = s[14..16].parse().ok()?; -let sec: i64 = s[17..19].parse().ok()?; -if !(1..=12).contains(&month) || !(1..=31).contains(&day) { -⋮---- -let days = days_from_civil(year, month, day); -⋮---- -u64::try_from(total).ok() -⋮---- -/// Parse ISO 8601 UTC timestamp to Unix epoch milliseconds, retaining sub-second precision. -/// Handles "2026-03-27T10:15:30Z" (returns seconds * 1000) and -/// "2026-03-27T10:15:30.750Z" (retains the fractional ms component). -pub(crate) fn iso8601_to_epoch_ms(s: &str) -> Option { -let epoch_secs = iso8601_to_epoch(s)?; -let base_ms = epoch_secs.saturating_mul(1000); -⋮---- -// Look for fractional seconds: '.' at position 19 (after "...SS."). -if s.len() > 20 && s.as_bytes().get(19) == Some(&b'.') { -⋮---- -.find(|c: char| !c.is_ascii_digit()) -.map(|i| frac_start + i) -.unwrap_or(s.len()); -⋮---- -if !frac_str.is_empty() { -let ms_digits = match frac_str.len() { -1 => frac_str.parse::().ok()?.saturating_mul(100), -2 => frac_str.parse::().ok()?.saturating_mul(10), -_ => frac_str[..3].parse::().ok()?, -⋮---- -return Some(base_ms + ms_digits); -⋮---- -Some(base_ms) -⋮---- -/// Days from 1970-01-01 to the given date (may be negative for dates before epoch). -/// Algorithm: http://howardhinnant.github.io/date_algorithms.html -fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { -// Shift March 1 to start of year so Feb (with leap day) is last month. -⋮---- -let era = y.div_euclid(400); -let yoe = y - era * 400; // [0, 399] -let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; // [0, 365] -let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096] -⋮---- -mod tests { -⋮---- -use std::sync::Mutex; -⋮---- -// Serialize env-var tests to avoid races with parallel test runner. -⋮---- -fn from_env_returns_none_when_keys_absent() { -let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -assert!(LangfuseClient::from_env().is_none()); -⋮---- -fn from_env_returns_some_when_keys_present() { -⋮---- -assert!(client.is_some()); -⋮---- -fn from_env_uses_custom_host() { -⋮---- -assert_eq!(client.unwrap().host, "https://my-langfuse.example.com"); -⋮---- -fn base64_encode_empty() { -assert_eq!(base64_encode(b""), ""); -⋮---- -fn base64_encode_produces_expected_output() { -let encoded = base64_encode(b"pk-test:sk-test"); -assert_eq!(encoded, "cGstdGVzdDpzay10ZXN0"); -⋮---- -fn base64_encode_one_byte_padding() { -assert_eq!(base64_encode(b"A"), "QQ=="); -⋮---- -fn base64_encode_two_byte_padding() { -assert_eq!(base64_encode(b"AB"), "QUI="); -⋮---- -fn base64_encode_three_bytes_no_padding() { -assert_eq!(base64_encode(b"ABC"), "QUJD"); -⋮---- -fn iso8601_to_epoch_unix_epoch() { -let epoch = iso8601_to_epoch("1970-01-01T00:00:00Z").unwrap(); -assert_eq!(epoch, 0); -⋮---- -fn iso8601_to_epoch_parses_2026() { -let epoch = iso8601_to_epoch("2026-03-27T10:00:00Z").unwrap(); -assert!(epoch > 1_735_689_600); // > 2025-01-01 -assert!(epoch < 1_900_000_000); // < 2030 -⋮---- -fn iso8601_to_epoch_february() { -// 1970-02-28T00:00:00Z = 58 days * 86400 = 5_011_200 -assert_eq!(iso8601_to_epoch("1970-02-28T00:00:00Z"), Some(5_011_200)); -⋮---- -fn iso8601_to_epoch_returns_none_on_short_string() { -assert!(iso8601_to_epoch("not-a-date").is_none()); -assert!(iso8601_to_epoch("").is_none()); -⋮---- -fn iso8601_to_epoch_ms_retains_milliseconds() { -let ms = iso8601_to_epoch_ms("2026-03-27T10:15:30.750Z").unwrap(); -let base_ms = iso8601_to_epoch("2026-03-27T10:15:30Z").unwrap() * 1000; -assert_eq!(ms, base_ms + 750); -⋮---- -fn iso8601_to_epoch_ms_no_fractional() { -let ms = iso8601_to_epoch_ms("2026-03-27T10:15:30Z").unwrap(); -let secs_ms = iso8601_to_epoch("2026-03-27T10:15:30Z").unwrap() * 1000; -assert_eq!(ms, secs_ms); -⋮---- -fn iso8601_to_epoch_ms_microsecond_input() { -// 6-digit fractional: should truncate to ms -let ms = iso8601_to_epoch_ms("2026-03-27T10:15:30.123456Z").unwrap(); -⋮---- -assert_eq!(ms, base_ms + 123); -⋮---- -fn build_payload_structure() { -⋮---- -request_id: "req-123".to_string(), -timestamp: "2026-03-27T10:00:01Z".to_string(), -backend: "openai".to_string(), -model_requested: Some("claude-3-haiku-20240307".to_string()), -model_mapped: Some("gpt-4o-mini".to_string()), -⋮---- -input_tokens: Some(100), -output_tokens: Some(50), -⋮---- -cost_usd: Some(0.001), -⋮---- -let payload = build_generation_payload(&entry); -let batch = payload["batch"].as_array().unwrap(); -assert_eq!(batch.len(), 1); -⋮---- -assert_eq!(event["type"], "generation-create"); -⋮---- -assert_eq!(body["model"], "gpt-4o-mini"); -assert_eq!(body["usage"]["input"], 100); -assert_eq!(body["usage"]["output"], 50); -assert_eq!(body["usage"]["unit"], "TOKENS"); -assert!(body["startTime"].as_str().is_some()); -assert!(body["endTime"].as_str().is_some()); -assert_eq!(body["level"], "DEFAULT"); -assert_eq!(body["metadata"]["backend"], "openai"); -assert_eq!(body["metadata"]["latency_ms"], 500); -⋮---- -fn build_payload_error_entry() { -⋮---- -request_id: "req-err".to_string(), -⋮---- -model_requested: Some("gpt-4o".to_string()), -⋮---- -error_message: Some("internal error".to_string()), -⋮---- -assert_eq!(body["level"], "ERROR"); -assert_eq!(body["metadata"]["error"], "internal error"); -⋮---- -fn build_payload_starttime_has_ms_precision_for_subsecond_latency() { -⋮---- -request_id: "req-ms".to_string(), -⋮---- -let start = body["startTime"].as_str().unwrap(); -let end = body["endTime"].as_str().unwrap(); -assert_ne!(start, end, "sub-second latency should produce different start/end times"); -assert!(start.contains('.'), "startTime should have ms precision: {start}"); - - - -// Anthropic <-> OpenAI message mapping -⋮---- -use crate::anthropic; -⋮---- -use crate::openai; -use crate::util; -⋮---- -/// Extract system prompt text from Anthropic's System type. -/// Warns if cache_control is present (no equivalent in downstream APIs). -pub fn extract_system_text(system: &anthropic::System) -> String { -⋮---- -if blocks.iter().any(|b| b.cache_control.is_some()) { -⋮---- -anthropic::System::Text(s) => s.clone(), -⋮---- -.iter() -.map(|b| b.text.as_str()) -⋮---- -.join("\n"), -⋮---- -/// Compute degradation warnings for an Anthropic request without performing translation. -/// -/// Returns a `TranslationWarnings` value listing every feature that will be silently -/// dropped or degraded when this request is translated to an OpenAI request. -/// The proxy injects these as an `x-anyllm-degradation` response header so clients -/// can detect silent drops without inspecting server logs. -pub fn compute_request_warnings(req: &anthropic::MessageCreateRequest) -> TranslationWarnings { -⋮---- -if req.top_k.is_some() { -w.add("top_k"); -⋮---- -if req.thinking.is_some() { -w.add("thinking_config"); -⋮---- -if seqs.len() > 4 { -w.add("stop_sequences_truncated"); -⋮---- -w.add("cache_control"); -⋮---- -let has_document = req.messages.iter().any(|msg| match &msg.content { -⋮---- -.any(|b| matches!(b, anthropic::ContentBlock::Document { .. })), -⋮---- -w.add("document_blocks"); -⋮---- -/// Convert an Anthropic MessageCreateRequest to an OpenAI ChatCompletionRequest. -⋮---- -/// Anthropic: -/// OpenAI: -pub fn anthropic_to_openai_request( -⋮---- -let text = extract_system_text(system); -// Uses System role instead of Developer for backward compat with -// local LLMs (vLLM, Ollama, llama-server) that don't recognize -// "developer". Trade-off: OpenAI o1/o3 require "developer" and -// reject "system". We chose broader compat over o-series support -// because most proxy users target GPT-4o or local models. -messages.push(openai::ChatMessage { -⋮---- -content: Some(openai::ChatContent::Text(text)), -⋮---- -convert_anthropic_message(msg, &mut messages); -⋮---- -.as_ref() -.map(|t| tools_map::anthropic_tools_to_openai(t)); -⋮---- -.map(tools_map::anthropic_tool_choice_to_openai); -⋮---- -// Map disable_parallel_tool_use to OpenAI parallel_tool_calls. -// Compat spec: "Fully supported". See: https://docs.anthropic.com/en/api/openai-sdk#tools--functions-fields -let parallel_tool_calls = match req.tool_choice.as_ref() { -⋮---- -}) => Some(false), -⋮---- -// Map metadata.user_id to OpenAI user field. -// Compat spec: user is "Ignored", but we forward it for traceability. -// See: https://docs.anthropic.com/en/api/openai-sdk#simple-fields -let user = req.metadata.as_ref().and_then(|m| m.user_id.clone()); -⋮---- -// Thinking config (budget_tokens) has no standard OpenAI equivalent. -// Thinking content blocks in messages ARE mapped to reasoning_content. -⋮---- -// OpenAI caps stop sequences at 4; empty array is invalid (requires 1-4 elements) -let stop = req.stop_sequences.as_ref().and_then(|seqs| { -if seqs.is_empty() { -⋮---- -let capped: Vec = seqs.iter().take(4).cloned().collect(); -Some(if capped.len() == 1 { -openai::Stop::Single(capped.into_iter().next().unwrap()) -⋮---- -model: req.model.clone(), -⋮---- -// Default: set both for local LLM compat (vLLM, ollama only -// recognize max_tokens). Overridden below for o-series models. -max_tokens: Some(req.max_tokens), -max_completion_tokens: Some(req.max_tokens), -// Compat spec: "Between 0 and 1 (inclusive). Values greater than 1 are capped at 1." -⋮---- -temperature: req.temperature.map(|t| t.clamp(0.0, 1.0)), -⋮---- -// Required for the streaming translator: without include_usage=true, -// OpenAI omits the final usage chunk and we cannot report token counts -// back to the Anthropic client. Local LLMs that don't support -// stream_options may reject this with 400. -stream_options: if req.stream == Some(true) { -Some(openai::StreamOptions { -⋮---- -extra: req.extra.clone(), -⋮---- -// Anthropic API returns single completions only; strip n to avoid -// wasting tokens on choices that get discarded (only choices[0] is used). -if let Some(n_val) = oai_req.extra.remove("n") { -if n_val != serde_json::Value::Number(1.into()) { -⋮---- -// o-series reasoning models (o1, o3, o4-mini, etc.) reject requests -// with both max_tokens and max_completion_tokens, require the -// "developer" role instead of "system", and reject non-default -// temperature/top_p values. -if is_o_series_model(&oai_req.model) { -⋮---- -// When a specific tool is forced, enable OpenAI strict structured outputs for it. -// This guarantees the returned JSON exactly matches the schema. -if let Some(forced_name) = req.tool_choice.as_ref().and_then(extract_forced_tool_name) { -⋮---- -apply_strict_mode_to_tool(tools, &forced_name); -⋮---- -/// Extract the forced tool name from an Anthropic ToolChoice, if any. -fn extract_forced_tool_name(tc: &anthropic::ToolChoice) -> Option { -⋮---- -anthropic::ToolChoice::Tool { name } => Some(name.clone()), -⋮---- -/// Set strict=true and normalize the parameter schema for the named tool. -/// Other tools in the vec are left unchanged. -fn apply_strict_mode_to_tool(tools: &mut [openai::ChatTool], forced_name: &str) { -for tool in tools.iter_mut() { -⋮---- -tool.function.strict = Some(true); -if let Some(params) = tool.function.parameters.take() { -⋮---- -Some(tools_map::normalize_schema_for_strict(params)); -⋮---- -// Tool names are unique; stop after the first match. -⋮---- -/// Returns true if the model name matches an OpenAI o-series reasoning model -/// (o1, o3, o4-mini, etc.). Does not match "gpt-4o" where 'o' is a suffix. -/// Update this list when new o-series model families ship. -fn is_o_series_model(model: &str) -> bool { -// Case-insensitive without allocating a lowercase copy. -⋮---- -prefixes.iter().any(|p| { -model.len() >= p.len() -&& model[..p.len()].eq_ignore_ascii_case(p) -&& (model.len() == p.len() || model.as_bytes()[p.len()] == b'-') -⋮---- -/// Convert a single Anthropic InputMessage into one or more OpenAI ChatMessages. -/// An assistant message with tool_use blocks produces tool_calls. -/// A user message with tool_result blocks produces OpenAI tool-role messages. -fn convert_anthropic_message(msg: &anthropic::InputMessage, out: &mut Vec) { -⋮---- -out.push(openai::ChatMessage { -⋮---- -content: Some(openai::ChatContent::Text(text.clone())), -⋮---- -convert_assistant_blocks(blocks, out); -⋮---- -convert_user_blocks(blocks, out); -⋮---- -/// Assistant blocks: text parts become content, tool_use blocks become tool_calls. -fn convert_assistant_blocks( -⋮---- -text_parts.push(text.clone()); -⋮---- -tool_calls.push(openai::ToolCall { -id: id.clone(), -call_type: "function".to_string(), -⋮---- -name: name.clone(), -⋮---- -thinking_parts.push(thinking.clone()); -⋮---- -// RedactedThinking has no meaningful content to forward -⋮---- -let content = if text_parts.is_empty() { -⋮---- -Some(openai::ChatContent::Text(text_parts.join(""))) -⋮---- -let reasoning_content = if thinking_parts.is_empty() { -⋮---- -Some(thinking_parts.join("")) -⋮---- -tool_calls: if tool_calls.is_empty() { -⋮---- -Some(tool_calls) -⋮---- -/// Resolve an Anthropic ImageSource to a URL string (data URI or direct URL). -fn image_source_to_url(source: &anthropic::messages::ImageSource) -> Option { -⋮---- -Some(url.clone()) -⋮---- -let mt = source.media_type.as_deref().unwrap_or("image/png"); -Some(format!("data:{};base64,{}", mt, data)) -⋮---- -/// Simplify a Vec of content parts: use plain Text when there's a single text part, -/// multipart array otherwise. Moves data out of the Vec to avoid cloning. -fn simplify_content_parts(mut parts: Vec) -> openai::ChatContent { -if parts.len() == 1 { -match parts.remove(0) { -⋮---- -other => openai::ChatContent::Parts(vec![other]), -⋮---- -/// User blocks: text/image parts become content, tool_result blocks become -/// separate OpenAI tool-role messages. -fn convert_user_blocks(blocks: &[anthropic::ContentBlock], out: &mut Vec) { -⋮---- -content_parts.push(openai::ChatContentPart::Text { text: text.clone() }); -⋮---- -if let Some(url) = image_source_to_url(source) { -content_parts.push(openai::ChatContentPart::ImageUrl { -⋮---- -parts.push(openai::ChatContentPart::Text { text: s.clone() }); -⋮---- -.push(openai::ChatContentPart::Text { text: text.clone() }); -⋮---- -parts.push(openai::ChatContentPart::ImageUrl { -⋮---- -// Anthropic's is_error flag has no direct OpenAI equivalent. -// We surface it as a text prefix so the backend model sees the -// error context in message history. -if *is_error == Some(true) { -// Prefix the first text part (or add one) with "Error: " -⋮---- -.iter_mut() -.find(|p| matches!(p, openai::ChatContentPart::Text { .. })) -⋮---- -*text = format!("Error: {}", text); -⋮---- -parts.insert( -⋮---- -text: "Error".to_string(), -⋮---- -// Use empty text if no content was provided -if parts.is_empty() { -parts.push(openai::ChatContentPart::Text { -⋮---- -tool_results.push((tool_use_id.clone(), parts)); -⋮---- -// OpenAI Chat Completions has no inline document support; -// the Responses API (input_file.file_data) would be needed -// for full fidelity. Degrade to a text note so the model -// still sees that a document was attached. -let label = title.as_deref().unwrap_or("document"); -⋮---- -let note = format!( -⋮---- -content_parts.push(openai::ChatContentPart::Text { text: note }); -⋮---- -// ToolUse and Thinking blocks don't appear in user messages; ignore if present -⋮---- -// Emit tool results before user content: OpenAI enforces strict turn -// ordering where Tool messages must immediately follow the Assistant -// message that produced the tool_calls. Violating this causes 400s. -⋮---- -let content = Some(simplify_content_parts(parts)); -⋮---- -tool_call_id: Some(tool_call_id), -⋮---- -// Emit user content message after tool results -if !content_parts.is_empty() { -let content = Some(simplify_content_parts(content_parts)); -⋮---- -/// Convert an OpenAI ChatCompletionResponse back to an Anthropic MessageResponse. -⋮---- -/// OpenAI: -⋮---- -pub fn openai_to_anthropic_response( -⋮---- -let choice = resp.choices.first(); -⋮---- -let mut stop_reason = Some(anthropic::StopReason::EndTurn); -⋮---- -.map(streaming_map::map_finish_reason); -⋮---- -// Map reasoning_content (DeepSeek/Qwen thinking) to Anthropic thinking block. -// Thinking blocks precede text content in Anthropic responses. -⋮---- -if !reasoning.is_empty() { -content.push(anthropic::ContentBlock::Thinking { -thinking: reasoning.clone(), -⋮---- -// Map content -⋮---- -if !text.is_empty() { -content.push(anthropic::ContentBlock::Text { text: text.clone() }); -⋮---- -// Map refusal to text block (same pattern as Responses API path) -⋮---- -if !refusal.is_empty() { -content.push(anthropic::ContentBlock::Text { -⋮---- -// Map tool calls with robustness for local LLMs (llama-server, ollama) -// that may produce empty IDs, empty names, or malformed arguments. -⋮---- -if tc.function.name.is_empty() { -⋮---- -let id = if tc.id.is_empty() { -⋮---- -tc.id.clone() -⋮---- -content.push(anthropic::ContentBlock::ToolUse { -⋮---- -name: tc.function.name.clone(), -⋮---- -.map(usage_map::openai_to_anthropic_usage) -.unwrap_or_default(); -⋮---- -response_type: "message".to_string(), -⋮---- -model: model.to_string(), -⋮---- -mod tests { -⋮---- -use serde_json::json; -⋮---- -// --- Helper: build a minimal Anthropic request --- -⋮---- -fn basic_request() -> anthropic::MessageCreateRequest { -⋮---- -model: "claude-3-5-sonnet-20241022".to_string(), -⋮---- -messages: vec![anthropic::InputMessage { -⋮---- -fn basic_openai_response() -> openai::ChatCompletionResponse { -⋮---- -id: "chatcmpl-abc123".to_string(), -object: "chat.completion".to_string(), -model: "gpt-4o".to_string(), -choices: vec![openai::Choice { -⋮---- -usage: Some(openai::ChatUsage { -⋮---- -created: Some(1700000000), -⋮---- -// --- Request translation tests --- -⋮---- -fn basic_text_request() { -let req = basic_request(); -let oai = anthropic_to_openai_request(&req); -⋮---- -assert_eq!(oai.model, "claude-3-5-sonnet-20241022"); -assert_eq!(oai.max_tokens, Some(1024)); -assert_eq!(oai.max_completion_tokens, Some(1024)); -assert_eq!(oai.messages.len(), 1); -assert_eq!(oai.messages[0].role, openai::ChatRole::User); -assert!(matches!( -⋮---- -assert!(oai.tools.is_none()); -assert!(oai.tool_choice.is_none()); -assert!(oai.stream_options.is_none()); -⋮---- -fn system_prompt_string_becomes_developer_message() { -let mut req = basic_request(); -req.system = Some(anthropic::System::Text( -"You are a helpful assistant.".to_string(), -⋮---- -assert_eq!(oai.messages.len(), 2); -assert_eq!(oai.messages[0].role, openai::ChatRole::System); -⋮---- -fn system_prompt_blocks_concatenated_into_developer_message() { -⋮---- -req.system = Some(anthropic::System::Blocks(vec![ -⋮---- -fn tool_definitions_mapped() { -let schema = json!({ -⋮---- -req.tools = Some(vec![anthropic::Tool { -⋮---- -let tools = oai.tools.unwrap(); -assert_eq!(tools.len(), 1); -assert_eq!(tools[0].tool_type, "function"); -assert_eq!(tools[0].function.name, "get_weather"); -assert_eq!( -⋮---- -assert_eq!(tools[0].function.parameters, Some(schema)); -⋮---- -fn tool_choice_auto() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::Auto { -⋮---- -fn tool_choice_any_becomes_required() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::Any { -⋮---- -fn tool_choice_none() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::None); -⋮---- -fn tool_choice_specific_tool() { -⋮---- -req.tool_choice = Some(anthropic::ToolChoice::Tool { -name: "get_weather".to_string(), -⋮---- -assert_eq!(n.choice_type, "function"); -assert_eq!(n.function.name, "get_weather"); -⋮---- -other => panic!("expected Named tool choice, got {:?}", other), -⋮---- -fn disable_parallel_tool_use_sets_parallel_tool_calls_false() { -⋮---- -disable_parallel_tool_use: Some(true), -⋮---- -assert_eq!(oai.parallel_tool_calls, Some(false)); -⋮---- -fn disable_parallel_tool_use_false_leaves_parallel_tool_calls_none() { -⋮---- -disable_parallel_tool_use: Some(false), -⋮---- -assert!(oai.parallel_tool_calls.is_none()); -⋮---- -fn no_tool_choice_leaves_parallel_tool_calls_none() { -⋮---- -fn stop_sequences_capped_at_four() { -⋮---- -req.stop_sequences = Some(vec![ -⋮---- -Some(openai::Stop::Multiple(ref v)) => assert_eq!(v.len(), 4), -other => panic!("expected Multiple stop, got {:?}", other), -⋮---- -fn single_stop_sequence_is_single() { -⋮---- -req.stop_sequences = Some(vec!["END".into()]); -⋮---- -fn empty_stop_sequences_becomes_none() { -⋮---- -req.stop_sequences = Some(vec![]); -⋮---- -assert!( -⋮---- -fn streaming_sets_stream_options() { -⋮---- -req.stream = Some(true); -⋮---- -assert_eq!(oai.stream, Some(true)); -assert!(oai.stream_options.as_ref().unwrap().include_usage); -⋮---- -fn conversation_with_tool_use_and_tool_result() { -⋮---- -req.messages = vec![ -// User asks -⋮---- -// Assistant calls tool -⋮---- -// User provides tool result -⋮---- -// msg 0: user text -⋮---- -// msg 1: assistant with text + tool_calls -assert_eq!(oai.messages[1].role, openai::ChatRole::Assistant); -⋮---- -let tc = oai.messages[1].tool_calls.as_ref().unwrap(); -assert_eq!(tc.len(), 1); -assert_eq!(tc[0].id, "call_001"); -assert_eq!(tc[0].function.name, "get_weather"); -assert_eq!(tc[0].function.arguments, r#"{"location":"NYC"}"#); -⋮---- -// msg 2: tool result -assert_eq!(oai.messages[2].role, openai::ChatRole::Tool); -assert_eq!(oai.messages[2].tool_call_id.as_deref(), Some("call_001")); -⋮---- -fn tool_result_error_prefixed() { -⋮---- -req.messages = vec![anthropic::InputMessage { -⋮---- -assert_eq!(oai.messages[0].role, openai::ChatRole::Tool); -⋮---- -fn image_block_to_image_url_part() { -⋮---- -assert_eq!(parts.len(), 2); -⋮---- -assert_eq!(image_url.url, "data:image/jpeg;base64,abc123"); -⋮---- -other => panic!("expected ImageUrl, got {:?}", other), -⋮---- -other => panic!("expected Parts, got {:?}", other), -⋮---- -fn image_block_with_url_uses_url_directly() { -⋮---- -// Single image part still wrapped in Parts (not a text shortcut) -⋮---- -assert_eq!(parts.len(), 1); -⋮---- -assert_eq!(image_url.url, "https://example.com/img.png"); -⋮---- -fn single_text_block_user_message_flattened() { -// A single text block in user content should produce ChatContent::Text, not Parts. -⋮---- -// --- Response translation tests --- -⋮---- -fn openai_text_response_to_anthropic() { -let resp = basic_openai_response(); -let anth = openai_to_anthropic_response(&resp, "claude-3-5-sonnet-20241022"); -⋮---- -assert!(anth.id.starts_with("msg_")); -assert_eq!(anth.response_type, "message"); -assert_eq!(anth.role, anthropic::Role::Assistant); -assert_eq!(anth.model, "claude-3-5-sonnet-20241022"); -assert_eq!(anth.content.len(), 1); -⋮---- -assert_eq!(anth.stop_reason, Some(anthropic::StopReason::EndTurn)); -assert!(anth.stop_sequence.is_none()); -assert_eq!(anth.usage.input_tokens, 10); -assert_eq!(anth.usage.output_tokens, 5); -⋮---- -fn openai_tool_calls_response_to_anthropic() { -⋮---- -id: "chatcmpl-xyz".to_string(), -⋮---- -assert_eq!(anth.stop_reason, Some(anthropic::StopReason::ToolUse)); -⋮---- -assert_eq!(id, "call_abc"); -assert_eq!(name, "get_weather"); -assert_eq!(input, &json!({"location": "NYC"})); -⋮---- -other => panic!("expected ToolUse, got {:?}", other), -⋮---- -fn stop_reason_mapping() { -let cases = vec![ -⋮---- -id: "x".into(), -object: "chat.completion".into(), -model: "gpt-4o".into(), -⋮---- -let anth = openai_to_anthropic_response(&resp, "m"); -assert_eq!(anth.stop_reason, Some(expected)); -⋮---- -fn empty_content_response() { -⋮---- -// Empty text is not added to content blocks -assert!(anth.content.is_empty()); -⋮---- -fn no_choices_produces_default_response() { -⋮---- -choices: vec![], -⋮---- -// Default stop_reason when no choice -⋮---- -fn missing_usage_produces_defaults() { -⋮---- -assert_eq!(anth.usage.input_tokens, 0); -assert_eq!(anth.usage.output_tokens, 0); -⋮---- -fn tool_result_blocks_content_concatenated() { -⋮---- -// Multiple text blocks are now preserved as separate parts (not concatenated) -// to support mixed text+image content in tool results. -⋮---- -other => panic!("expected Parts with 2 text entries, got {:?}", other), -⋮---- -fn tool_result_none_content_becomes_empty_string() { -⋮---- -fn mixed_user_content_and_tool_results_produces_multiple_messages() { -⋮---- -// Should produce two messages: tool result first (must follow assistant), -// then user text. -⋮---- -assert_eq!(oai.messages[1].role, openai::ChatRole::User); -⋮---- -fn assistant_text_and_tool_use_combined() { -// Assistant message with both text and tool_use should produce a single -// OpenAI message with content + tool_calls. -⋮---- -assert_eq!(oai.messages[0].role, openai::ChatRole::Assistant); -⋮---- -let tc = oai.messages[0].tool_calls.as_ref().unwrap(); -⋮---- -assert_eq!(tc[0].id, "call_1"); -⋮---- -fn openai_response_with_text_and_tool_calls() { -// OpenAI can return both content and tool_calls in a single choice. -⋮---- -assert_eq!(anth.content.len(), 2); -⋮---- -fn malformed_tool_arguments_handled() { -// If OpenAI returns invalid JSON in arguments, parse_json_lenient wraps it as a string. -⋮---- -// parse_tool_arguments wraps invalid JSON in an object -assert_eq!(input, &json!({"_raw_error": "not json"})); -⋮---- -fn document_block_converted_to_text_note() { -⋮---- -model: "claude-opus-4-6".into(), -⋮---- -let openai_req = anthropic_to_openai_request(&req); -// Should produce a single user message with multipart content -assert_eq!(openai_req.messages.len(), 1); -⋮---- -// Second part should be the document note -⋮---- -assert!(text.contains("report.pdf")); -assert!(text.contains("application/pdf")); -⋮---- -panic!("expected text part for document"); -⋮---- -fn created_timestamp_preserved_from_openai() { -⋮---- -assert_eq!(resp.created, Some(1700000000)); -let anth = openai_to_anthropic_response(&resp, "claude-sonnet-4-6"); -assert_eq!(anth.created, Some(1700000000)); -⋮---- -fn created_timestamp_none_when_absent() { -let mut resp = basic_openai_response(); -⋮---- -assert_eq!(anth.created, None); -// Verify None created is omitted from JSON -let json = serde_json::to_string(&anth).unwrap(); -assert!(!json.contains("\"created\"")); -⋮---- -fn thinking_config_stripped_in_translation() { -⋮---- -req.thinking = Some(anthropic::ThinkingConfig::Enabled { -⋮---- -// Thinking has no OpenAI equivalent; verify translation succeeds -// and the OpenAI request has no thinking field (it's not in the struct) -⋮---- -fn thinking_block_mapped_to_reasoning_content_in_assistant_translation() { -⋮---- -// Thinking block mapped to reasoning_content, text block preserved -⋮---- -fn redacted_thinking_block_dropped_in_assistant_translation() { -⋮---- -// RedactedThinking block dropped, text block preserved -⋮---- -fn temperature_clamped_to_zero_one() { -⋮---- -req.temperature = Some(1.5); -⋮---- -assert_eq!(oai.temperature, Some(1.0)); -⋮---- -req.temperature = Some(0.5); -⋮---- -assert_eq!(oai.temperature, Some(0.5)); -⋮---- -req.temperature = Some(-0.1); -⋮---- -assert_eq!(oai.temperature, Some(0.0)); -⋮---- -assert!(oai.temperature.is_none()); -⋮---- -fn metadata_user_id_maps_to_openai_user() { -⋮---- -req.metadata = Some(anthropic::messages::Metadata { -user_id: Some("u-abc123".into()), -⋮---- -assert_eq!(oai.user.as_deref(), Some("u-abc123")); -⋮---- -// No metadata: user is None -⋮---- -assert!(oai.user.is_none()); -⋮---- -// --- Claude Code parallel tool use --- -⋮---- -fn claude_code_parallel_tool_use_request() { -// Assistant message with 2 tool_use blocks -> OpenAI message with 2 tool_calls -⋮---- -// Should produce: user msg, assistant msg with tool_calls -⋮---- -assert_eq!(assistant_msg.role, openai::ChatRole::Assistant); -match assistant_msg.content.as_ref().unwrap() { -openai::ChatContent::Text(t) => assert_eq!(t, "I'll do both."), -other => panic!("expected Text content, got {:?}", other), -⋮---- -let tool_calls = assistant_msg.tool_calls.as_ref().unwrap(); -assert_eq!(tool_calls.len(), 2); -assert_eq!(tool_calls[0].id, "toolu_01A"); -assert_eq!(tool_calls[0].function.name, "Read"); -assert_eq!(tool_calls[1].id, "toolu_01B"); -assert_eq!(tool_calls[1].function.name, "Glob"); -⋮---- -fn claude_code_tool_result_request() { -// User message with tool_result blocks -> OpenAI tool-role messages -⋮---- -// Should produce 2 tool-role messages -⋮---- -assert_eq!(oai.messages[0].tool_call_id.as_deref(), Some("toolu_01A")); -assert_eq!(oai.messages[1].role, openai::ChatRole::Tool); -assert_eq!(oai.messages[1].tool_call_id.as_deref(), Some("toolu_01B")); -⋮---- -fn claude_code_tool_response_roundtrip() { -// OpenAI tool_call response -> Anthropic tool_use, verify fields survive -⋮---- -id: "chatcmpl-llama001".into(), -⋮---- -model: "llama-3.3-70b".into(), -⋮---- -let anth = openai_to_anthropic_response(&resp, "claude-sonnet-4-20250514"); -⋮---- -// First block is text -⋮---- -anthropic::ContentBlock::Text { text } => assert_eq!(text, "Reading file."), -other => panic!("expected Text, got {:?}", other), -⋮---- -// Second and third blocks are tool_use -⋮---- -assert_eq!(id, "call_read_001"); -assert_eq!(name, "Read"); -assert_eq!(input["file_path"], "/config.toml"); -⋮---- -assert_eq!(id, "call_glob_001"); -assert_eq!(name, "Glob"); -assert_eq!(input["pattern"], "**/*test*"); -⋮---- -// --- Local LLM robustness --- -⋮---- -fn tool_call_empty_id_gets_synthetic_id() { -⋮---- -model: "llama".into(), -⋮---- -id: "".into(), // empty ID from local LLM -⋮---- -fn tool_call_empty_arguments_becomes_empty_object() { -⋮---- -arguments: "".into(), // empty args from local LLM -⋮---- -assert_eq!(input, &json!({})); -⋮---- -fn tool_call_missing_name_skipped() { -⋮---- -name: "".into(), // empty name -⋮---- -// Empty-name tool call skipped; text + valid tool call remain -⋮---- -anthropic::ContentBlock::Text { text } => assert_eq!(text, "text"), -⋮---- -anthropic::ContentBlock::ToolUse { name, .. } => assert_eq!(name, "Read"), -⋮---- -fn refusal_mapped_to_text_block() { -⋮---- -id: "chatcmpl-1".into(), -⋮---- -assert!(text.contains("Refusal")); -assert!(text.contains("I cannot help with that request.")); -⋮---- -other => panic!("expected Text with refusal, got {:?}", other), -⋮---- -fn extra_fields_forwarded_to_openai_request() { -⋮---- -.insert("seed".into(), serde_json::Value::Number(42.into())); -⋮---- -.insert("logprobs".into(), serde_json::Value::Bool(true)); -⋮---- -assert_eq!(oai.extra.get("seed"), Some(&json!(42))); -assert_eq!(oai.extra.get("logprobs"), Some(&json!(true))); -⋮---- -fn n_parameter_stripped_from_extra() { -⋮---- -req.extra.insert("n".into(), json!(4)); -req.extra.insert("seed".into(), json!(42)); -⋮---- -assert!(oai.extra.get("n").is_none()); -⋮---- -fn n_parameter_one_stripped_silently() { -⋮---- -req.extra.insert("n".into(), json!(1)); -⋮---- -fn reasoning_content_mapped_to_thinking_block_in_response() { -⋮---- -model: "deepseek-reasoner".into(), -⋮---- -let resp = openai_to_anthropic_response(&oai_resp, "deepseek-reasoner"); -// First block should be thinking, second should be text -assert_eq!(resp.content.len(), 2); -⋮---- -assert_eq!(thinking, "Let me think... 2+2=4"); -assert!(signature.is_none()); -⋮---- -other => panic!("expected Thinking block, got {:?}", other), -⋮---- -assert_eq!(text, "The answer is 4."); -⋮---- -other => panic!("expected Text block, got {:?}", other), -⋮---- -fn thinking_block_mapped_to_reasoning_content_in_request() { -⋮---- -fn unknown_finish_reason_maps_to_end_turn() { -⋮---- -model: "deepseek-chat".into(), -⋮---- -let resp = openai_to_anthropic_response(&oai_resp, "deepseek-chat"); -assert_eq!(resp.stop_reason, Some(anthropic::StopReason::EndTurn)); -⋮---- -fn is_o_series_model_matches() { -assert!(is_o_series_model("o1")); -assert!(is_o_series_model("o3")); -assert!(is_o_series_model("o4")); -assert!(is_o_series_model("o1-mini")); -assert!(is_o_series_model("o1-preview")); -assert!(is_o_series_model("o3-mini")); -assert!(is_o_series_model("o4-mini")); -assert!(is_o_series_model("O1")); // case-insensitive -assert!(is_o_series_model("O3-Mini")); -⋮---- -fn is_o_series_model_rejects() { -assert!(!is_o_series_model("gpt-4o")); -assert!(!is_o_series_model("gpt-4o-mini")); -assert!(!is_o_series_model("gpt-4")); -assert!(!is_o_series_model("claude-3-opus")); -⋮---- -fn make_request(model: &str, system: Option<&str>) -> anthropic::MessageCreateRequest { -⋮---- -model: model.into(), -⋮---- -messages: vec![], -system: system.map(|s| anthropic::System::Text(s.into())), -⋮---- -fn o_series_model_gets_only_max_completion_tokens() { -let req = make_request("o1-mini", Some("You are helpful.")); -⋮---- -// System role should be converted to Developer for o-series. -assert_eq!(oai.messages[0].role, openai::ChatRole::Developer); -⋮---- -fn non_o_series_model_gets_both_max_tokens() { -let req = make_request("gpt-4o", Some("You are helpful.")); -⋮---- -// System role should remain System for non-o-series. -⋮---- -fn o_series_strips_temperature() { -let mut req = make_request("o3-mini", None); -req.temperature = Some(0.7); -⋮---- -fn o_series_strips_top_p() { -let mut req = make_request("o1-preview", None); -req.top_p = Some(0.9); -⋮---- -assert!(oai.top_p.is_none(), "o-series should strip top_p"); -⋮---- -fn non_o_series_preserves_temperature() { -let mut req = make_request("gpt-4o", None); -⋮---- -assert_eq!(oai.temperature, Some(0.7)); -⋮---- -// --- compute_request_warnings --- -⋮---- -fn warnings_empty_for_plain_request() { -⋮---- -let w = compute_request_warnings(&req); -assert!(w.is_empty()); -assert!(w.as_header_value().is_none()); -⋮---- -fn warnings_top_k() { -⋮---- -req.top_k = Some(40); -⋮---- -assert_eq!(w.as_header_value().unwrap(), "top_k"); -⋮---- -fn warnings_thinking_config() { -⋮---- -assert_eq!(w.as_header_value().unwrap(), "thinking_config"); -⋮---- -fn warnings_stop_sequences_truncated_at_5() { -⋮---- -assert_eq!(w.as_header_value().unwrap(), "stop_sequences_truncated"); -⋮---- -fn warnings_stop_sequences_4_is_fine() { -⋮---- -fn warnings_cache_control_on_system() { -⋮---- -req.system = Some(anthropic::System::Blocks(vec![anthropic::SystemBlock { -⋮---- -assert_eq!(w.as_header_value().unwrap(), "cache_control"); -⋮---- -fn warnings_document_blocks() { -⋮---- -assert_eq!(w.as_header_value().unwrap(), "document_blocks"); -⋮---- -fn warnings_multiple_combined() { -⋮---- -req.top_k = Some(10); -⋮---- -let val = w.as_header_value().unwrap(); -assert!(val.contains("top_k"), "missing top_k in: {val}"); -⋮---- -fn forced_tool_choice_enables_strict_mode_in_openai_request() { -⋮---- -.unwrap(); -⋮---- -let openai_req = anthropic_to_openai_request(&anthropic_req); -⋮---- -let tools = openai_req.tools.expect("tools should be present"); -⋮---- -// The tool should have strict: true. -assert_eq!(tools[0].function.strict, Some(true)); -⋮---- -// The schema should have additionalProperties: false. -let params = tools[0].function.parameters.as_ref().expect("parameters should be present"); -assert_eq!(params["additionalProperties"], serde_json::json!(false)); -⋮---- -// required should include both properties. -let required = params["required"].as_array().expect("required should be present"); -assert!(required.iter().any(|v| v == "name")); -assert!(required.iter().any(|v| v == "value")); - - - -// Tool definition and tool_choice mapping -⋮---- -use crate::anthropic; -use crate::openai; -⋮---- -/// Convert Anthropic tool definitions to OpenAI tool definitions. -/// -/// Anthropic: -/// OpenAI: -pub fn anthropic_tools_to_openai(tools: &[anthropic::Tool]) -> Vec { -⋮---- -.iter() -.map(|t| openai::ChatTool { -tool_type: "function".to_string(), -⋮---- -name: t.name.clone(), -description: t.description.clone(), -parameters: Some(t.input_schema.clone()), -// Compat spec: "Ignored". Anthropic has no equivalent. -// See: https://docs.anthropic.com/en/api/openai-sdk#tools--functions-fields -⋮---- -.collect() -⋮---- -/// Convert OpenAI tool definitions back to Anthropic tool definitions. -/// When parameters is None, defaults to `{"type": "object"}` since Anthropic -/// requires input_schema to be present. -⋮---- -pub fn openai_tools_to_anthropic(tools: &[openai::ChatTool]) -> Vec { -⋮---- -.map(|t| anthropic::Tool { -name: t.function.name.clone(), -description: t.function.description.clone(), -⋮---- -.clone() -.unwrap_or_else(|| serde_json::json!({"type": "object"})), -⋮---- -/// JSON Schema keys that Gemini's function-calling API rejects. -/// Gemini supports only the OpenAPI 3.0 subset of JSON Schema. -⋮---- -/// Recursively strip JSON Schema fields that Gemini rejects. -/// Applied to tool `parameters` when the backend is Gemini or Vertex. -pub fn sanitize_schema_for_gemini(schema: serde_json::Value) -> serde_json::Value { -⋮---- -map.remove(*key); -⋮---- -.into_iter() -.map(|(k, v)| (k, sanitize_schema_for_gemini(v))) -.collect(); -⋮---- -serde_json::Value::Array(arr.into_iter().map(sanitize_schema_for_gemini).collect()) -⋮---- -/// Convert Anthropic tool_choice to OpenAI tool_choice. -⋮---- -pub fn anthropic_tool_choice_to_openai(tc: &anthropic::ToolChoice) -> openai::ChatToolChoice { -⋮---- -anthropic::ToolChoice::Auto { .. } => openai::ChatToolChoice::Simple("auto".to_string()), -// Any = "model must call at least one tool". OpenAI's "required" -// is the closest: it forces a tool call when tools are defined. -anthropic::ToolChoice::Any { .. } => openai::ChatToolChoice::Simple("required".to_string()), -anthropic::ToolChoice::None => openai::ChatToolChoice::Simple("none".to_string()), -⋮---- -choice_type: "function".to_string(), -function: openai::chat_completions::NamedFunction { name: name.clone() }, -⋮---- -/// Convert OpenAI tool_choice to Anthropic tool_choice. -⋮---- -pub fn openai_tool_choice_to_anthropic(tc: &openai::ChatToolChoice) -> anthropic::ToolChoice { -⋮---- -openai::ChatToolChoice::Simple(s) => match s.as_str() { -⋮---- -// Default unknown values to Auto for forward compatibility; -// rejecting would break when OpenAI adds new tool_choice variants. -⋮---- -name: named.function.name.clone(), -⋮---- -/// Normalize a JSON Schema for OpenAI strict mode. -⋮---- -/// OpenAI strict mode requires: -/// - All properties of object schemas listed in `required`. -/// - `additionalProperties: false` on all object schemas. -⋮---- -/// Applied recursively to nested object schemas. -/// Non-object schemas are returned unchanged. -pub fn normalize_schema_for_strict(mut schema: serde_json::Value) -> serde_json::Value { -if schema.get("type").and_then(|t| t.as_str()) != Some("object") { -⋮---- -let Some(obj) = schema.as_object_mut() else { -⋮---- -// Set additionalProperties: false. -obj.insert( -"additionalProperties".to_string(), -⋮---- -// Collect all property keys. -⋮---- -.get("properties") -.and_then(|p| p.as_object()) -.map(|p| p.keys().cloned().collect()) -.unwrap_or_default(); -⋮---- -if !prop_keys.is_empty() { -// Merge with any existing required array. -⋮---- -.get("required") -.and_then(|r| r.as_array()) -.map(|arr| { -arr.iter() -.filter_map(|v| v.as_str().map(|s| s.to_string())) -⋮---- -.chain(existing) -⋮---- -merged.sort(); -merged.dedup(); -⋮---- -"required".to_string(), -⋮---- -merged.into_iter().map(serde_json::Value::String).collect(), -⋮---- -// Recurse into nested object properties. -if let Some(props) = obj.get_mut("properties").and_then(|p| p.as_object_mut()) { -for prop_val in props.values_mut() { -if prop_val.get("type").and_then(|t| t.as_str()) == Some("object") { -*prop_val = normalize_schema_for_strict(prop_val.clone()); -⋮---- -serde_json::Value::Object(obj.clone()) -⋮---- -/// Apply strict mode to the single tool that is being forced via tool_choice. -⋮---- -/// Finds the tool whose function name matches `forced_name`, sets `strict: true` -/// on its function object, and normalizes its parameter schema. -⋮---- -/// All other tools are left unchanged. -pub fn apply_strict_to_forced_tool(tools: &mut [serde_json::Value], forced_name: &str) { -for tool in tools.iter_mut() { -let Some(function) = tool.get_mut("function") else { -⋮---- -.get("name") -.and_then(|n| n.as_str()) -== Some(forced_name); -⋮---- -if let Some(obj) = function.as_object_mut() { -obj.insert("strict".to_string(), serde_json::Value::Bool(true)); -⋮---- -// Normalize parameter schema in place. -if let Some(params) = obj.get("parameters").cloned() { -⋮---- -"parameters".to_string(), -normalize_schema_for_strict(params), -⋮---- -// Tool names are unique; stop after the first match. -⋮---- -mod tests { -⋮---- -use pretty_assertions::assert_eq; -use serde_json::json; -⋮---- -fn sample_anthropic_tool() -> anthropic::Tool { -⋮---- -name: "get_weather".into(), -description: Some("Get weather for a location".into()), -input_schema: json!({ -⋮---- -fn sample_openai_tool() -> openai::ChatTool { -⋮---- -tool_type: "function".into(), -⋮---- -parameters: Some(json!({ -⋮---- -// --- Tool definition conversion --- -⋮---- -fn anthropic_to_openai_tool() { -let tools = anthropic_tools_to_openai(&[sample_anthropic_tool()]); -assert_eq!(tools.len(), 1); -assert_eq!(tools[0].tool_type, "function"); -assert_eq!(tools[0].function.name, "get_weather"); -assert_eq!( -⋮---- -fn openai_to_anthropic_tool() { -let tools = openai_tools_to_anthropic(&[sample_openai_tool()]); -⋮---- -assert_eq!(tools[0].name, "get_weather"); -⋮---- -assert_eq!(tools[0].input_schema, sample_anthropic_tool().input_schema); -⋮---- -fn empty_tools_list() { -assert!(anthropic_tools_to_openai(&[]).is_empty()); -assert!(openai_tools_to_anthropic(&[]).is_empty()); -⋮---- -fn tool_without_description() { -⋮---- -name: "no_desc".into(), -⋮---- -input_schema: json!({"type": "object"}), -⋮---- -let openai = anthropic_tools_to_openai(&[tool]); -assert!(openai[0].function.description.is_none()); -⋮---- -// And back -let anthropic = openai_tools_to_anthropic(&openai); -assert!(anthropic[0].description.is_none()); -⋮---- -fn openai_tool_without_parameters_defaults_to_object() { -⋮---- -name: "simple".into(), -⋮---- -let anthropic = openai_tools_to_anthropic(&[tool]); -assert_eq!(anthropic[0].input_schema, json!({"type": "object"})); -⋮---- -fn multiple_tools_preserved() { -let tools = vec![ -⋮---- -let openai = anthropic_tools_to_openai(&tools); -assert_eq!(openai.len(), 2); -assert_eq!(openai[0].function.name, "tool_a"); -assert_eq!(openai[1].function.name, "tool_b"); -⋮---- -let back = openai_tools_to_anthropic(&openai); -assert_eq!(back.len(), 2); -assert_eq!(back[0].name, "tool_a"); -assert_eq!(back[1].name, "tool_b"); -assert_eq!(back[1].input_schema, tools[1].input_schema); -⋮---- -// --- Tool choice mapping --- -⋮---- -fn tool_choice_auto() { -let openai = anthropic_tool_choice_to_openai(&anthropic::ToolChoice::Auto { -⋮---- -assert!(matches!(openai, openai::ChatToolChoice::Simple(ref s) if s == "auto")); -⋮---- -let back = openai_tool_choice_to_anthropic(&openai); -assert!(matches!(back, anthropic::ToolChoice::Auto { .. })); -⋮---- -fn tool_choice_any_to_required() { -let openai = anthropic_tool_choice_to_openai(&anthropic::ToolChoice::Any { -⋮---- -assert!(matches!(openai, openai::ChatToolChoice::Simple(ref s) if s == "required")); -⋮---- -assert!(matches!(back, anthropic::ToolChoice::Any { .. })); -⋮---- -fn tool_choice_none() { -let openai = anthropic_tool_choice_to_openai(&anthropic::ToolChoice::None); -assert!(matches!(openai, openai::ChatToolChoice::Simple(ref s) if s == "none")); -⋮---- -assert!(matches!(back, anthropic::ToolChoice::None)); -⋮---- -fn tool_choice_specific_tool() { -⋮---- -let openai = anthropic_tool_choice_to_openai(&tc); -⋮---- -assert_eq!(named.choice_type, "function"); -assert_eq!(named.function.name, "get_weather"); -⋮---- -_ => panic!("expected Named tool choice"), -⋮---- -anthropic::ToolChoice::Tool { name } => assert_eq!(name, "get_weather"), -other => panic!("expected ToolChoice::Tool, got {:?}", other), -⋮---- -fn openai_unknown_simple_choice_defaults_to_auto() { -// Any unrecognized simple string should map to Auto -let tc = openai::ChatToolChoice::Simple("something_else".into()); -assert!(matches!( -⋮---- -fn disable_parallel_tool_use_roundtrips_via_serde() { -// Ensure the field survives JSON deserialization -⋮---- -let tc: anthropic::ToolChoice = serde_json::from_value(json).unwrap(); -⋮---- -other => panic!( -⋮---- -fn auto_without_disable_parallel_omits_field_in_json() { -⋮---- -let json = serde_json::to_value(&tc).unwrap(); -assert_eq!(json, serde_json::json!({"type": "auto"})); -⋮---- -// --- Claude Code tool schema round-trips --- -⋮---- -fn claude_code_read_tool_roundtrip() { -⋮---- -name: "Read".into(), -description: Some("Reads a file from the local filesystem.".into()), -⋮---- -let openai = anthropic_tools_to_openai(&[tool.clone()]); -⋮---- -assert_eq!(back[0].name, tool.name); -assert_eq!(back[0].description, tool.description); -assert_eq!(back[0].input_schema, tool.input_schema); -⋮---- -fn claude_code_bash_tool_roundtrip() { -⋮---- -name: "Bash".into(), -description: Some("Executes a given bash command and returns its output.".into()), -⋮---- -fn claude_code_edit_tool_roundtrip() { -⋮---- -name: "Edit".into(), -description: Some("Performs exact string replacements in files.".into()), -⋮---- -fn claude_code_grep_tool_with_enum_roundtrip() { -// Grep has an enum field (output_mode) which must survive translation -⋮---- -name: "Grep".into(), -description: Some("A powerful search tool built on ripgrep.".into()), -⋮---- -fn claude_code_all_six_tools_preserved() { -// All 6 core Claude Code tools survive batch translation -⋮---- -.map(|name| anthropic::Tool { -name: (*name).to_string(), -description: Some(format!("{} tool", name)), -⋮---- -assert_eq!(openai.len(), 6); -⋮---- -assert_eq!(back.len(), 6); -for (orig, rt) in tools.iter().zip(back.iter()) { -assert_eq!(orig.name, rt.name); -⋮---- -// --- Gemini schema sanitization --- -⋮---- -fn sanitize_strips_disallowed_top_level_fields() { -⋮---- -let result = sanitize_schema_for_gemini(schema); -assert!(result.get("$schema").is_none()); -assert!(result.get("default").is_none()); -assert!(result.get("$defs").is_none()); -assert!(result.get("additionalProperties").is_none()); -assert_eq!(result["type"], "object"); -⋮---- -fn sanitize_strips_disallowed_nested_fields() { -⋮---- -assert!(name_prop.get("default").is_none()); -assert!(name_prop.get("const").is_none()); -assert_eq!(name_prop["type"], "string"); -⋮---- -assert!(count_prop.get("anyOf").is_none()); -⋮---- -fn sanitize_leaves_valid_schema_unchanged() { -⋮---- -let result = sanitize_schema_for_gemini(schema.clone()); -assert_eq!(result, schema); -⋮---- -mod strict_tests { -⋮---- -fn normalize_adds_required_and_disables_additional_props() { -let schema = json!({ -⋮---- -let normalized = normalize_schema_for_strict(schema); -let required = normalized["required"].as_array().unwrap(); -assert!(required.iter().any(|v| v == "name"), "name should be required"); -assert!(required.iter().any(|v| v == "age"), "age should be required"); -assert_eq!(normalized["additionalProperties"], json!(false)); -⋮---- -fn normalize_nested_object_properties() { -⋮---- -// Nested object must also have required and additionalProperties. -⋮---- -assert_eq!(nested["additionalProperties"], json!(false)); -let nested_required = nested["required"].as_array().unwrap(); -assert!(nested_required.iter().any(|v| v == "street")); -⋮---- -fn normalize_preserves_existing_required() { -⋮---- -// Should merge existing required with all properties. -⋮---- -assert!(required.iter().any(|v| v == "x")); -assert!(required.iter().any(|v| v == "y")); -⋮---- -fn normalize_non_object_schema_unchanged() { -let schema = json!({"type": "string"}); -let normalized = normalize_schema_for_strict(schema.clone()); -assert_eq!(normalized, schema); -⋮---- -fn apply_strict_to_forced_tool_sets_strict_flag() { -let mut tools: Vec = vec![ -⋮---- -apply_strict_to_forced_tool(&mut tools, "send_email"); -⋮---- -// Only send_email should have strict: true. -⋮---- -assert_eq!(send_email["strict"], serde_json::json!(true)); -⋮---- -// get_weather should be unchanged (no strict flag). -⋮---- -assert!(get_weather.get("strict").map(|v| v.is_null() || v == &serde_json::json!(false)).unwrap_or(true)); -⋮---- -fn apply_strict_no_match_does_not_panic() { -let mut tools: Vec = vec![serde_json::json!({ -⋮---- -// Should not panic when tool name is not found. -apply_strict_to_forced_tool(&mut tools, "nonexistent"); - - - -# anyllm-proxy - -An API translation proxy that lets Anthropic-based tools (Claude Code, Cursor, Windsurf, Cline) talk to any OpenAI-compatible backend, local LLM, or alternative provider. - -**[Releases](https://github.com/whit3rabbit/anyllm-proxy/releases)** | **[Library Usage](#using-as-a-library)** - ---- - -## Quick Start - -Download a binary from the [releases page](https://github.com/whit3rabbit/anyllm-proxy/releases), or install from source: - -```bash -cargo install anyllm_proxy -``` - -Create a `.anyllm.env` config file: - -```env -OPENAI_API_KEY=unused -OPENAI_BASE_URL=http://localhost:11434/v1 -BIG_MODEL=qwen2.5-coder:32b -SMALL_MODEL=qwen2.5-coder:32b -``` - -Run the proxy (auto-loads `.anyllm.env` from the current directory): - -```bash -anyllm_proxy -# or: anyllm_proxy --env-file ~/configs/ollama.env -``` - -Point Claude Code at the proxy: - -```bash -ANTHROPIC_BASE_URL=http://localhost:3000 claude -``` - -### Admin Web Interface (optional) - -Pass `--webui` (or `--admin`) to also start the admin dashboard on `127.0.0.1:3001`. The dashboard has the following tabs: - -- **Dashboard:** Live RPM, error rate, P50/P95 latency, per-backend cards, and a filterable live request feed. -- **Request Log:** Historical request log with filters (backend, status, key, date range), paginated, with per-request cost and token detail. -- **Settings:** Mutable config (log level, log_bodies, per-backend model mappings), read-only env vars (secrets masked), and **Export .env** to generate a `.anyllm.env` template. -- **Backends:** Configured backends and their settings. -- **Access Control:** Virtual key CRUD — create, edit (RPM/TPM limits, budget, expiry, model allowlist), and revoke keys without restarting. -- **Models:** Add/remove model routing deployments (LiteLLM config mode only). -- **Audit:** Log of all admin config mutations and key lifecycle events. - -```bash -anyllm_proxy --webui -# Proxy API: http://localhost:3000 -# Admin UI: http://127.0.0.1:3001/admin/?token=$(cat .admin_token) -``` - -The dashboard's Settings tab shows all active environment variables (API keys masked) and has an **Export .env** button that generates a `.anyllm.env` template you can edit and reuse. To use a custom port or a fixed token: - -```bash -ADMIN_PORT=4000 ADMIN_TOKEN=mysecret anyllm_proxy --webui -``` - -To force-disable the admin even when the flag is present (useful in automated environments): - -```bash -DISABLE_ADMIN=1 anyllm_proxy --webui # admin will NOT start -``` - -Additional admin env vars: `ADMIN_DB_PATH` (SQLite file, default: `admin.db`), `ADMIN_TOKEN_PATH` (where the generated token is written, default: `.admin_token`), `ADMIN_LOG_RETENTION_DAYS` (request log retention, default: `7`). - -### Multiple backends on one proxy (recommended) - -A single proxy instance can serve all your backends simultaneously. Each backend gets its own URL path. Use a `config.toml` (see [section 2](#2-multi-routing-and-the-web-interface)): - -```toml -# config.toml -listen_port = 3000 -default_backend = "local" - -[backends.local] -kind = "openai" -api_key = "unused" -base_url = "http://localhost:11434/v1" -big_model = "qwen2.5-coder:32b" -small_model = "qwen2.5-coder:7b" - -[backends.openai] -kind = "openai" -api_key = "sk-..." -base_url = "https://api.openai.com/v1" -big_model = "gpt-4o" -small_model = "gpt-4o-mini" - -[backends.deepseek] -kind = "openai" -api_key = "sk-deepseek-..." -base_url = "https://api.deepseek.com/v1" -big_model = "deepseek-coder" -small_model = "deepseek-chat" -``` - -```bash -PROXY_CONFIG=config.toml anyllm_proxy --webui -``` - -All three backends are live at once: - -| Path | Backend | -|------|---------| -| `http://localhost:3000/v1/messages` | local (default) | -| `http://localhost:3000/openai/v1/messages` | OpenAI | -| `http://localhost:3000/deepseek/v1/messages` | DeepSeek | - -Point different tools at different paths, or switch in Claude Code by changing `ANTHROPIC_BASE_URL`. - -### Coming from LiteLLM? Drop in your config.yaml - -anyllm-proxy accepts LiteLLM `config.yaml` files directly. If you already have a LiteLLM deployment, point the proxy at your existing config: - -```bash -PROXY_CONFIG=config.yaml anyllm_proxy --webui -``` - -A standard LiteLLM config works as-is: - -```yaml -# config.yaml (LiteLLM format) -model_list: - - model_name: gpt-4o - litellm_params: - model: azure/gpt-4o-eu - api_base: https://my-resource.openai.azure.com/ - api_key: os.environ/AZURE_API_KEY - rpm: 6000 - - model_name: gpt-4o - litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY - rpm: 10000 - - model_name: claude-3-opus - litellm_params: - model: anthropic/claude-3-opus-20240229 - api_key: os.environ/ANTHROPIC_API_KEY - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -Multiple deployments of the same model name are load-balanced with round-robin routing. Deployments at their RPM limit are automatically skipped. - -**Env var compatibility:** LiteLLM env var names are accepted as aliases, so you do not need to rename anything: - -| LiteLLM env var | anyllm-proxy equivalent | Notes | -|---|---|---| -| `LITELLM_MASTER_KEY` | `PROXY_API_KEYS` | Admin/auth key | -| `LITELLM_CONFIG` | `PROXY_CONFIG` | Config file path | -| `AZURE_API_KEY` | `AZURE_OPENAI_API_KEY` | Azure auth | -| `AZURE_API_BASE` | `AZURE_OPENAI_ENDPOINT` | Azure endpoint | -| `AZURE_API_VERSION` | `AZURE_OPENAI_API_VERSION` | Azure API version | -| `AWS_REGION_NAME` | `AWS_REGION` | Bedrock region | -| `OPENAI_API_KEY` | `OPENAI_API_KEY` | Same name | -| `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY` | Same name | - -The `os.environ/VAR_NAME` syntax in YAML values is supported alongside anyllm's native `env:VAR_NAME`. See [docs/COMPARISON_LITELLM.md](docs/COMPARISON_LITELLM.md) for a full feature comparison. - -### Multiple separate instances (for isolated deployments) - -For cases where you want completely separate proxy processes (different ports, different machines, different Docker containers), keep one `.env` file per deployment: - -``` -~/proxies/ - ollama.env # local Ollama - openai-prod.env # production OpenAI - deepseek.env # DeepSeek API -``` - -Run any one: - -```bash -anyllm_proxy --env-file ~/proxies/deepseek.env -``` - -Docker-compatible — same file works with `--env-file`: - -```bash -docker run --env-file ~/proxies/openai-prod.env -p 3000:3000 anyllm-proxy -``` - -The admin UI's **Export .env** button (Settings tab) generates a ready-to-edit template from the current configuration. - ---- - -## What, Why, and How? - -### What is it? -A lightweight, fast Rust-based proxy that accepts Anthropic Messages API requests, translates them to OpenAI Chat Completions format, forwards them to any compliant backend, and translates responses back in real-time. Supports streaming SSE, tool calling, and image/document blocks. - -### Why use it? - -- **Local AI Coding:** Run Claude Code against local models (Llama 3, DeepSeek, Qwen) without API credits. -- **Broad Compatibility:** Works with open-weights and alternative models including Qwen and DeepSeek. -- **Multi-Backend Routing:** Route `haiku` requests to a fast local model and `opus` requests to external providers, transparently. -- **Observability:** Built-in admin dashboard for request logs, latency, and live config changes. - -### How to Build from Source - -```bash -cargo build - -# Proxy only (default) -cargo run -p anyllm_proxy - -# Proxy + admin web UI -cargo run -p anyllm_proxy -- --webui -``` - -The proxy listens on `0.0.0.0:3000`. The admin dashboard (opt-in via `--webui`) binds to `127.0.0.1:3001`. - ---- - -## 1. Primary Use Case: Claude Code + Local LLMs - -### Example: Running with Ollama (DeepSeek / Qwen) - -```bash -# 1. Start your local LLM -ollama run qwen2.5-coder:32b & - -# 2. Start the translation proxy -OPENAI_API_KEY=unused \ -OPENAI_BASE_URL=http://localhost:11434/v1 \ -BIG_MODEL=qwen2.5-coder:32b \ -SMALL_MODEL=qwen2.5-coder:32b \ -cargo run -p anyllm_proxy & - -# 3. Use Claude Code targeting the local proxy -ANTHROPIC_BASE_URL=http://localhost:3000 claude -``` - -Use the same pattern for **LM Studio** (default port `1234`) or **vLLM** (default port `8000`) by substituting `OPENAI_BASE_URL`. - ---- - -## 2. Multi-Routing and the Web Interface - -Create a `config.toml` to map different routes to different backends: - -```toml -listen_port = 3000 -default_backend = "local_qwen" - -[backends.local_qwen] -kind = "openai" -api_key = "unused" -base_url = "http://localhost:11434/v1" -big_model = "qwen2.5-coder:32b" -small_model = "qwen2.5-coder:7b" - -[backends.deepseek_api] -kind = "openai" -api_key = "sk-deepseek-..." -base_url = "https://api.deepseek.com/v1" -big_model = "deepseek-coder" -small_model = "deepseek-chat" - -[backends.openrouter] -kind = "openai" -api_key = "sk-or-..." -base_url = "https://openrouter.ai/api/v1" -big_model = "anthropic/claude-3.5-sonnet" -small_model = "google/gemini-2.5-flash" -``` - -```bash -PROXY_CONFIG=config.toml anyllm_proxy --webui -``` - -Additional per-backend fields: `api_format = "chat"` (OpenAI only; `chat` or `responses`), `omit_stream_options = true` (strip `stream_options` for backends that reject it). Top-level `log_bodies = true` enables request/response body logging. Any config value can use `env:VAR_NAME` to read from the environment at startup (e.g., `api_key = "env:OPENAI_API_KEY"`). - -All backends are live at once on a single port. The path prefix matches the backend name in the config: - -| Path | Backend | Notes | -|------|---------|-------| -| `/v1/messages` | `local_qwen` | default | -| `/deepseek_api/v1/messages` | `deepseek_api` | | -| `/openrouter/v1/messages` | `openrouter` | | - -Point Claude Code at a specific backend: -```bash -ANTHROPIC_BASE_URL=http://localhost:3000/deepseek_api claude -``` - -### The Admin Dashboard - -Start the proxy with `--webui`, then open: - -```bash -open http://127.0.0.1:3001/admin/?token=$(cat .admin_token) -``` - -The dashboard tabs are described under [Admin Web Interface](#admin-web-interface-optional) above. When using a LiteLLM config, the **Models** tab lets you add/remove deployments without editing the config file. All config mutations (model changes, key creation/revocation) are recorded in the **Audit** tab. - ---- - -## 3. Commercial APIs (OpenAI, Gemini, OpenRouter) - -**OpenRouter:** -```bash -OPENAI_API_KEY=sk-or-... \ -OPENAI_BASE_URL=https://openrouter.ai/api/v1 \ -BIG_MODEL=anthropic/claude-3.5-sonnet \ -SMALL_MODEL=anthropic/claude-3-haiku \ -cargo run -p anyllm_proxy -``` - -**OpenAI:** -```bash -OPENAI_API_KEY=sk-... \ -BIG_MODEL=gpt-4o \ -SMALL_MODEL=gpt-4o-mini \ -cargo run -p anyllm_proxy -``` - -**Google Gemini:** -```bash -BACKEND=gemini \ -GEMINI_API_KEY=AIza... \ -BIG_MODEL=gemini-2.5-pro \ -SMALL_MODEL=gemini-2.5-flash \ -cargo run -p anyllm_proxy -``` - ---- - -## 4. Additional Backends and Features - -### Azure OpenAI - -```bash -BACKEND=azure \ -AZURE_OPENAI_ENDPOINT=https://myresource.openai.azure.com \ -AZURE_OPENAI_DEPLOYMENT=my-gpt4o \ -AZURE_OPENAI_API_KEY=... \ -cargo run -p anyllm_proxy -``` - -### AWS Bedrock - -```bash -BACKEND=bedrock \ -AWS_REGION=us-east-1 \ -AWS_ACCESS_KEY_ID=AKIA... \ -AWS_SECRET_ACCESS_KEY=... \ -BIG_MODEL=anthropic.claude-3-5-sonnet-20241022-v2:0 \ -SMALL_MODEL=anthropic.claude-3-5-haiku-20241022-v1:0 \ -cargo run -p anyllm_proxy -``` - -### Anthropic Passthrough - -Forwards requests to the Anthropic API with no format translation. Use this when the upstream is already Anthropic and you only need auth, routing, or rate limiting from the proxy. - -```bash -BACKEND=anthropic \ -ANTHROPIC_API_KEY=sk-ant-... \ -cargo run -p anyllm_proxy -``` - -`ANTHROPIC_BASE_URL` overrides the upstream URL (default: `https://api.anthropic.com`). Note: `POST /v1/embeddings` is not available on this backend. - -### OpenAI Chat Completions Input - -The proxy accepts `POST /v1/chat/completions` in OpenAI format and returns OpenAI format. This means any OpenAI-native client (LiteLLM, LangChain, etc.) can route through the proxy unchanged: - -```bash -curl http://localhost:3000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "x-api-key: your-key" \ - -d '{ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hello"}], - "max_tokens": 100 - }' -``` - -### Virtual Key Management - -Create short-lived, rate-limited, or budget-capped API keys without restarting the proxy. Start with `--webui` to enable the admin server, then: - -```bash -# Create a key with RPM/TPM limits, a monthly budget, and a model allowlist -curl -X POST http://localhost:3001/admin/api/keys \ - -H "Authorization: Bearer $(cat .admin_token)" \ - -H "Content-Type: application/json" \ - -d '{ - "description": "dev key", - "rpm_limit": 60, - "tpm_limit": 100000, - "max_budget_usd": 10.00, - "budget_duration": "monthly", - "expires_at": "2026-12-31T00:00:00Z", - "allowed_models": ["claude-*", "gpt-4o"] - }' -# Response: {"id": 1, "key": "sk-vk...", ...} - -# Use the key like any other proxy key -curl http://localhost:3000/v1/messages \ - -H "x-api-key: sk-vk..." \ - -d '{"model": "claude-sonnet-4-20250514", "max_tokens": 100, "messages": [...]}' - -# Update limits on an existing key (no restart needed) -curl -X PUT http://localhost:3001/admin/api/keys/1 \ - -H "Authorization: Bearer $(cat .admin_token)" \ - -H "Content-Type: application/json" \ - -d '{"rpm_limit": 120, "max_budget_usd": 20.00}' - -# Check spend for a key -curl http://localhost:3001/admin/api/keys/1/spend \ - -H "Authorization: Bearer $(cat .admin_token)" - -# Revoke immediately (no restart needed) -curl -X DELETE http://localhost:3001/admin/api/keys/1 \ - -H "Authorization: Bearer $(cat .admin_token)" -``` - -`budget_duration` accepts `daily`, `monthly`, or `lifetime`. `allowed_models` supports exact names and `prefix/*` wildcards. A key at 100% of its budget returns 429 with period reset information. Webhook notifications fire at 80%, 95%, and 100% of the budget via `WEBHOOK_URLS`. - -Requests from unauthenticated clients are rejected by default. For local development, set `PROXY_OPEN_RELAY=true` to accept any non-empty key (insecure, never use in production). - -**Distributed rate limiting (optional):** Build with `--features redis` and set `REDIS_URL=redis://localhost:6379` to use Redis-backed rate limiting across multiple proxy instances. In-process rate limits are per-instance only. `RATE_LIMIT_FAIL_POLICY=open` (default) allows requests when Redis is unavailable; `closed` rejects them with 503. - -### OpenTelemetry Export - -```bash -cargo build -p anyllm_proxy --features otel - -OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \ -OTEL_SERVICE_NAME=anyllm-proxy \ -OPENAI_API_KEY=sk-... \ -./target/debug/anyllm_proxy -``` - -Spans are exported via OTLP HTTP (protobuf). Standard OpenTelemetry SDK environment variables control endpoint, service name, and sampling. The feature adds zero runtime overhead when not compiled in. - ---- - -## Using as a Library - -The translation engine is available as standalone Rust crates. - -``` -crates/translator (lib, IO-free pure translation) - | -crates/client (lib, async HTTP client wrapping translator) - | -crates/proxy (bin, full proxy server) -``` - -| Level | Crate | Use Case | -|---|---|---| -| **Pure translation** | `anyllm_translate` | Stateless type conversion between Anthropic and OpenAI formats. No IO, no HTTP. Bring your own transport. | -| **HTTP client** | `anyllm_client` | `client.messages(req).await` -- send Anthropic requests, get Anthropic responses. Handles translation, HTTP, retry, and streaming internally. | -| **Embedded middleware** | `anyllm_translate` with `middleware` feature | Drop-in axum Router that adds `/v1/messages` to an existing server. | -| **Full proxy** | `anyllm_proxy` | Multi-backend routing, admin UI, metrics, auth. Everything in this README. | - -### Adding as a dependency - -```toml -[dependencies] -# HTTP client (includes translation) -anyllm_client = { git = "https://github.com/whit3rabbit/anyllm-proxy" } - -# Translation only (no HTTP, no async) -anyllm_translate = { git = "https://github.com/whit3rabbit/anyllm-proxy" } - -# With axum middleware support -anyllm_translate = { git = "https://github.com/whit3rabbit/anyllm-proxy", features = ["middleware"] } -``` - -### HTTP Client (translation + transport) - -The simplest path. Send Anthropic requests, get Anthropic responses. Translation, retry, and SSE streaming are handled internally. - -```rust -use anyllm_client::{Client, ClientError}; -use anyllm_translate::anthropic::MessageCreateRequest; - -let client = Client::builder() - .base_url("https://api.openai.com/v1/chat/completions") - .api_key("sk-...") - .build()?; - -let req: MessageCreateRequest = serde_json::from_str(r#"{ - "model": "claude-sonnet-4-6", - "max_tokens": 256, - "messages": [{"role": "user", "content": "Hello"}] -}"#)?; - -let response = client.messages(&req).await?; -``` - -For custom TLS, SSRF protection, or per-model mapping, use `ClientConfig::builder()`: - -```rust -use anyllm_client::{Client, ClientConfig, Auth}; -use anyllm_translate::TranslationConfig; - -let client = Client::new( - ClientConfig::builder() - .backend_url("https://api.openai.com/v1/chat/completions") - .auth(Auth::Bearer("sk-...".into())) - .translation( - TranslationConfig::builder() - .model_map("claude-sonnet-4-6", "gpt-4o") - .model_map("claude-haiku-4-5", "gpt-4o-mini") - .build() - ) - .build() -); -``` - -**Error handling:** - -```rust -match client.messages(&req).await { - Ok(resp) => { /* ... */ } - Err(ClientError::ApiError { status, body, .. }) => eprintln!("HTTP {status}: {body}"), - Err(ClientError::Transport(e)) => eprintln!("network: {e}"), - Err(ClientError::Translation(e)) => eprintln!("translation: {e}"), - Err(e) => eprintln!("{e}"), -} -``` - -**Streaming:** - -```rust -use anyllm_translate::anthropic::{Delta, StreamEvent}; -use futures::StreamExt; - -let (mut stream, _rate_limits) = client.messages_stream(&req).await?; -while let Some(event) = stream.next().await { - if let StreamEvent::ContentBlockDelta { delta: Delta::TextDelta { text }, .. } = event? { - print!("{text}"); - } -} -``` - -**Tool calling:** - -```rust -use anyllm_client::{ToolBuilder, ToolChoiceBuilder}; -use serde_json::json; - -let tool = ToolBuilder::new("get_weather") - .description("Get the current weather for a location") - .input_schema(json!({ - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"] - })) - .build(); -// Attach tool to MessageCreateRequest via serde_json, then call client.messages(). -``` - -Runnable examples: `cargo run --example basic -p anyllm_client`, `streaming`, `tools`. - -### Pure Translation (no IO) - -Use when you want to bring your own HTTP client or embed translation in a non-async context. - -```rust -use anyllm_translate::{TranslationConfig, translate_request, translate_response}; -use anyllm_translate::anthropic::MessageCreateRequest; - -let config = TranslationConfig::builder() - .model_map("claude-sonnet-4-6", "gpt-4o") - .build(); - -let anthropic_req: MessageCreateRequest = serde_json::from_str(&body)?; -let openai_req = translate_request(&anthropic_req, &config)?; -// ... send openai_req with your HTTP client ... -let anthropic_resp = translate_response(&openai_resp, &anthropic_req.model); -``` - -**Streaming (OpenAI chunks → Anthropic SSE events):** - -```rust -use anyllm_translate::new_stream_translator; - -let mut translator = new_stream_translator(model); -// Feed each OpenAI chunk as it arrives: -let events = translator.process_chunk(&chunk); -// After the stream ends: -let final_events = translator.finish(); -``` - -**Reverse direction (OpenAI ← Anthropic), for serving OpenAI-native clients:** - -```rust -use anyllm_translate::{ - translate_openai_to_anthropic_request, - translate_anthropic_to_openai_response, - new_reverse_stream_translator, - TranslationWarnings, -}; - -let mut warnings = TranslationWarnings::default(); -let anthropic_req = translate_openai_to_anthropic_request(&openai_req, &mut warnings)?; -// ... forward to Anthropic API ... -let openai_resp = translate_anthropic_to_openai_response(&anthropic_resp, "gpt-4o"); -``` - -Runnable examples: `cargo run --example translate_request -p anyllm_translate`, `reverse_translation`. - -### Embedded Middleware (for existing axum apps) - -```rust -use anyllm_translate::middleware::{anthropic_compat_router, AnthropicCompatConfig}; - -let config = AnthropicCompatConfig::builder() - .backend_url("https://api.openai.com") - .api_key("sk-...") - .build(); - -let app = Router::new() - .merge(anthropic_compat_router(config)) - .route("/my-other-endpoint", get(handler)); -``` - -For cross-language bindings (FFI, WASM, PyO3), see [docs/library-integration.md](docs/library-integration.md). - ---- - -## Advanced Features - -- **Streaming SSE:** Real-time translation of chunked responses. -- **Tool Calling:** Transparent tool definition and `tool_use`/`tool_result` translation. -- **Image & Document Blocks:** Base64/URL and document block support. -- **Embeddings passthrough:** `POST /v1/embeddings` forwarded as-is to the backend (no translation). Works with OpenAI, Azure, Vertex, Gemini, and vLLM. Not available when `BACKEND=anthropic`. -- **Degradation header:** `x-anyllm-degradation` is set on responses when features are silently dropped during translation (e.g., `top_k`, `cache_control`, `document_blocks`, `thinking_config`). -- **Model allowlist:** Per-virtual-key restriction by exact model name or `prefix/*` wildcard, enforced pre-request. -- **Budget tracking and spend alerts:** Per-key `max_budget_usd` with daily/monthly/lifetime periods. Webhook notifications (via `WEBHOOK_URLS`) fire at 80%, 95%, and 100% of the budget. -- **Audit log:** All admin config mutations and key lifecycle events stored in SQLite, queryable via `GET /admin/api/audit`. -- **OIDC/JWT authentication:** Set `OIDC_ISSUER_URL` (and optionally `OIDC_AUDIENCE`) to accept JWT bearer tokens for proxy authentication. -- **Observability:** SQLite request logging, metrics endpoint, WebSocket live dashboard. -- **Safety:** SSRF protection (including IPv6 ULA/link-local), concurrency limits, exponential backoff retry, CSRF protection on admin endpoints. - -## License - -MIT - - - -// Virtual API key generation, hashing, and rate limit state. -⋮---- -use std::collections::VecDeque; -⋮---- -type HmacSha256 = Hmac; -⋮---- -/// Role assigned to a virtual API key, controlling access scope. -⋮---- -pub enum KeyRole { -⋮---- -impl KeyRole { -pub fn as_str(&self) -> &'static str { -⋮---- -pub fn from_str_or_default(s: &str) -> Self { -match s.to_lowercase().as_str() { -⋮---- -/// Budget reset period for a virtual key. -⋮---- -pub enum BudgetDuration { -⋮---- -impl BudgetDuration { -⋮---- -pub fn parse(s: &str) -> Option { -⋮---- -"daily" => Some(BudgetDuration::Daily), -"monthly" => Some(BudgetDuration::Monthly), -⋮---- -/// Current time as milliseconds since the Unix epoch. Used for rate-limit sliding windows. -pub fn now_ms() -> u64 { -⋮---- -.duration_since(std::time::UNIX_EPOCH) -.unwrap_or_default() -.as_millis() as u64 -⋮---- -/// Generate a new virtual API key, hashed with HMAC-SHA256 using the installation secret. -/// Returns (raw_key, key_prefix, key_hash_hex). -/// The raw_key is shown once at creation; key_prefix is for display; key_hash_hex is stored. -pub fn generate_virtual_key(hmac_secret: &[u8]) -> (String, String, String) { -let a = uuid::Uuid::new_v4().as_simple().to_string(); -let b = uuid::Uuid::new_v4().as_simple().to_string(); -let raw_key = format!("sk-vk{}{}", a, b); -let key_prefix = raw_key[..8].to_string(); -let key_hash_hex = hmac_hash_key(&raw_key, hmac_secret); -⋮---- -/// SHA-256 hash a key string and return hex-encoded result. -/// Used for legacy keys created before HMAC was introduced. -pub fn hash_key(key: &str) -> String { -let hash: [u8; 32] = Sha256::digest(key.as_bytes()).into(); -bytes_to_hex(&hash) -⋮---- -/// HMAC-SHA256 hash a key with a per-installation secret. Returns hex string. -/// Used for all newly created keys. The secret binds hashes to this installation, -/// so a stolen database cannot be used to brute-force keys on a different instance. -pub fn hmac_hash_key(key: &str, secret: &[u8]) -> String { -let mut mac = HmacSha256::new_from_slice(secret).expect("HMAC-SHA256 accepts any key length"); -mac.update(key.as_bytes()); -let result = mac.finalize(); -bytes_to_hex(&result.into_bytes()) -⋮---- -/// Convert a hex-encoded hash to raw bytes. -pub fn hash_from_hex(hex_str: &str) -> Option<[u8; 32]> { -if hex_str.len() != 64 { -⋮---- -arr[i] = u8::from_str_radix(&hex_str[i * 2..i * 2 + 2], 16).ok()?; -⋮---- -Some(arr) -⋮---- -fn bytes_to_hex(bytes: &[u8]) -> String { -bytes.iter().map(|b| format!("{b:02x}")).collect() -⋮---- -/// In-memory metadata for a virtual key (stored in DashMap). -⋮---- -pub struct VirtualKeyMeta { -⋮---- -/// Epoch seconds; None = no expiry. -⋮---- -/// Access role (admin or developer). Defaults to developer. -⋮---- -/// Maximum budget in USD per period. None = unlimited. -⋮---- -/// Budget reset period. None = lifetime budget (no reset). -⋮---- -/// Start of the current budget period (ISO 8601 UTC). -⋮---- -/// Accumulated spend in the current period. -⋮---- -/// Optional model allowlist. None = all models allowed. -/// Supports exact match and prefix wildcard (e.g., `"claude-*"`). -⋮---- -/// Sliding window rate limit state per virtual key. -⋮---- -pub struct RateLimitState { -⋮---- -impl Default for RateLimitState { -fn default() -> Self { -⋮---- -impl RateLimitState { -pub fn new() -> Self { -⋮---- -/// Check if a new request is within the RPM limit. -/// Returns Ok(()) if allowed, Err(retry_after_secs) if exceeded. -pub fn check_rpm(&self, limit: u32, now_ms: u64) -> Result<(), u64> { -let mut window = self.rpm_window.lock().unwrap_or_else(|e| e.into_inner()); -let cutoff = now_ms.saturating_sub(60_000); -// Drain expired entries -while window.front().is_some_and(|&ts| ts < cutoff) { -window.pop_front(); -⋮---- -if window.len() >= limit as usize { -// Compute retry-after: time until the oldest entry expires -let oldest = window.front().copied().unwrap_or(now_ms); -let retry_after_ms = (oldest + 60_000).saturating_sub(now_ms); -return Err((retry_after_ms / 1000).max(1)); -⋮---- -window.push_back(now_ms); -Ok(()) -⋮---- -/// Record a TPM token count for the current request. -pub fn record_tpm(&self, now_ms: u64, tokens: u32) { -let mut window = self.tpm_window.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -while window.front().is_some_and(|&(ts, _)| ts < cutoff) { -⋮---- -window.push_back((now_ms, tokens)); -⋮---- -/// Check if adding `tokens` would exceed the TPM limit. -pub fn check_tpm(&self, limit: u32, now_ms: u64) -> Result<(), u64> { -⋮---- -let total: u64 = window.iter().map(|&(_, t)| t as u64).sum(); -⋮---- -let oldest = window.front().map(|&(ts, _)| ts).unwrap_or(now_ms); -⋮---- -/// Check whether the budget period has elapsed and reset spend if so. -/// Returns true if a reset occurred. -/// Does NOT persist to SQLite; caller should fire-and-forget a DB update. -pub fn check_and_reset_period(meta: &mut VirtualKeyMeta) -> bool { -⋮---- -None => return false, // Lifetime budget, no periodic reset -⋮---- -.as_secs(); -⋮---- -Some(start) => next_period_boundary(start, duration), -⋮---- -// No period_start set yet; initialize it now -meta.period_start = Some(current_period_start(now_epoch, duration)); -⋮---- -/// Compute the epoch timestamp of the next period boundary given a period start ISO string. -fn next_period_boundary(start_iso: &str, duration: BudgetDuration) -> Option { -// Parse the ISO 8601 date to extract year, month, day -// Format: "2026-03-22T00:00:00Z" -if start_iso.len() < 10 { -⋮---- -let year: u64 = start_iso[0..4].parse().ok()?; -let month: u64 = start_iso[5..7].parse().ok()?; -let day: u64 = start_iso[8..10].parse().ok()?; -⋮---- -// Next day at UTC midnight -let start_epoch = ymd_to_epoch(year, month, day); -Some(start_epoch + 86400) -⋮---- -// 1st of next month at UTC midnight -⋮---- -Some(ymd_to_epoch(ny, nm, 1)) -⋮---- -/// Compute the current period start for a given epoch time. -fn current_period_start(now_epoch: u64, duration: BudgetDuration) -> String { -⋮---- -format!("{year:04}-{month:02}-{day:02}T00:00:00Z") -⋮---- -format!("{year:04}-{month:02}-01T00:00:00Z") -⋮---- -/// Convert year/month/day to epoch seconds (UTC midnight). -fn ymd_to_epoch(year: u64, month: u64, day: u64) -> u64 { -// Inverse of the Hinnant algorithm used in db.rs -⋮---- -/// Compute the ISO 8601 string for when the current period resets. -pub fn period_reset_at(meta: &VirtualKeyMeta) -> Option { -⋮---- -let start = meta.period_start.as_ref()?; -let boundary = next_period_boundary(start, duration)?; -Some(super::db::epoch_to_iso8601(boundary)) -⋮---- -/// Row from the virtual_api_key table. -⋮---- -pub struct VirtualKeyRow { -⋮---- -impl VirtualKeyRow { -/// Compute the effective status of a key. -pub fn status(&self) -> &'static str { -if self.revoked_at.is_some() { -⋮---- -mod tests { -⋮---- -fn key_generation_format() { -⋮---- -let (raw, prefix, hash) = generate_virtual_key(secret); -assert!(raw.starts_with("sk-vk")); -assert_eq!(prefix.len(), 8); -assert!(prefix.starts_with("sk-vk")); -assert_eq!(hash.len(), 64); // hex HMAC-SHA256 -⋮---- -fn hash_deterministic() { -let h1 = hash_key("test-key-123"); -let h2 = hash_key("test-key-123"); -assert_eq!(h1, h2); -⋮---- -fn hash_from_hex_roundtrip() { -let hex = hash_key("test"); -let bytes = hash_from_hex(&hex).unwrap(); -assert_eq!(bytes_to_hex(&bytes), hex); -⋮---- -fn rpm_within_limit() { -⋮---- -assert!(state.check_rpm(3, now).is_ok()); -assert!(state.check_rpm(3, now + 1).is_ok()); -assert!(state.check_rpm(3, now + 2).is_ok()); -// 4th request should be rejected -assert!(state.check_rpm(3, now + 3).is_err()); -⋮---- -fn rpm_window_expiry() { -⋮---- -assert!(state.check_rpm(1, now).is_ok()); -assert!(state.check_rpm(1, now + 100).is_err()); -// After 60 seconds, window should clear -assert!(state.check_rpm(1, now + 60_001).is_ok()); -⋮---- -fn tpm_within_limit() { -⋮---- -state.record_tpm(now, 50); -assert!(state.check_tpm(100, now + 1).is_ok()); -state.record_tpm(now + 1, 50); -// At limit -assert!(state.check_tpm(100, now + 2).is_err()); -⋮---- -fn tpm_window_expiry() { -⋮---- -state.record_tpm(now, 100); -assert!(state.check_tpm(100, now + 1).is_err()); -// After 60 seconds -assert!(state.check_tpm(100, now + 60_001).is_ok()); -⋮---- -// -- KeyRole tests -- -⋮---- -fn key_role_roundtrip() { -assert_eq!(KeyRole::Admin.as_str(), "admin"); -assert_eq!(KeyRole::Developer.as_str(), "developer"); -assert_eq!(KeyRole::from_str_or_default("admin"), KeyRole::Admin); -assert_eq!(KeyRole::from_str_or_default("Admin"), KeyRole::Admin); -assert_eq!( -⋮---- -assert_eq!(KeyRole::from_str_or_default("unknown"), KeyRole::Developer); -assert_eq!(KeyRole::from_str_or_default(""), KeyRole::Developer); -⋮---- -// -- BudgetDuration tests -- -⋮---- -fn budget_duration_roundtrip() { -assert_eq!(BudgetDuration::Daily.as_str(), "daily"); -assert_eq!(BudgetDuration::Monthly.as_str(), "monthly"); -assert_eq!(BudgetDuration::parse("daily"), Some(BudgetDuration::Daily)); -⋮---- -assert_eq!(BudgetDuration::parse("weekly"), None); -⋮---- -// -- Period boundary tests -- -⋮---- -fn ymd_to_epoch_known_values() { -// 1970-01-01 = epoch 0 -assert_eq!(ymd_to_epoch(1970, 1, 1), 0); -// 2020-01-01 = 1577836800 -assert_eq!(ymd_to_epoch(2020, 1, 1), 1577836800); -⋮---- -fn next_period_boundary_daily() { -⋮---- -let boundary = next_period_boundary(start, BudgetDuration::Daily).unwrap(); -// Should be 2026-03-26 midnight -let expected = ymd_to_epoch(2026, 3, 26); -assert_eq!(boundary, expected); -⋮---- -fn next_period_boundary_monthly() { -⋮---- -let boundary = next_period_boundary(start, BudgetDuration::Monthly).unwrap(); -// Should be 2026-04-01 midnight -let expected = ymd_to_epoch(2026, 4, 1); -⋮---- -fn next_period_boundary_monthly_december() { -⋮---- -// Should be 2027-01-01 midnight -let expected = ymd_to_epoch(2027, 1, 1); -⋮---- -fn check_and_reset_period_no_duration() { -⋮---- -max_budget_usd: Some(10.0), -budget_duration: None, // lifetime, no reset -period_start: Some("2020-01-01T00:00:00Z".to_string()), -⋮---- -// No reset because no duration -assert!(!check_and_reset_period(&mut meta)); -assert_eq!(meta.period_spend_usd, 5.0); -⋮---- -fn hmac_hash_differs_from_plain_sha256() { -⋮---- -let hmac_hash = hmac_hash_key(key, secret); -let plain_hash = hash_key(key); -assert_ne!(hmac_hash, plain_hash); -⋮---- -fn hmac_hash_differs_with_different_secrets() { -⋮---- -let h1 = hmac_hash_key(key, b"secret-a"); -let h2 = hmac_hash_key(key, b"secret-b"); -assert_ne!(h1, h2); -⋮---- -fn hmac_hash_deterministic() { -⋮---- -assert_eq!(hmac_hash_key(key, secret), hmac_hash_key(key, secret)); -⋮---- -fn check_and_reset_period_resets_when_past_boundary() { -⋮---- -budget_duration: Some(BudgetDuration::Daily), -⋮---- -// Period start is in 2020, so it should reset -assert!(check_and_reset_period(&mut meta)); -assert_eq!(meta.period_spend_usd, 0.0); -assert!(meta.period_start.is_some()); - - - -/// Audio transcription and text-to-speech passthrough handlers. -pub mod audio; -/// AWS Bedrock passthrough handler (SigV4 signing + event stream decoding). -mod bedrock_passthrough; -/// OpenAI Chat Completions input handler (POST /v1/chat/completions). -mod chat_completions; -/// Image generation passthrough handler. -pub mod images; -/// Gemini native generateContent handler (POST /v1/messages when GEMINI_API_FORMAT=native). -mod gemini_native; -/// Auth validation, request ID injection, size limits, concurrency limits, header logging. -pub mod middleware; -/// OIDC/JWT authentication (optional, enabled via OIDC_ISSUER_URL). -pub mod oidc; -/// Anthropic passthrough handler (no translation, forwards as-is). -mod passthrough; -/// Per-key request policy enforcement (model allowlists). -pub mod policy; -/// Axum router setup and request handlers for all API endpoints. -pub mod routes; -/// SSE response helpers for Anthropic-format streaming. -pub mod sse; -/// SSE streaming handler with pre-stream error propagation and backpressure. -mod streaming; -/// Approximate token counting via tiktoken. -mod token_counting; - - - - - - - - -Proxy Admin - - - - -
-
Dashboard
-
Request Log
-
Settings
-
Backends
-
Access Control
-
Models
-
Audit
-
Disconnected
-
- -
-
-
Requests/min
--
-
Error Rate
--
-
P50 Latency
--
-
P95 Latency
--
-
Total Requests
0
-
-
-
Streams Started
0
-
Completed
0
-
Failed
0
-
Client Disconnects
0
-
-
-
-
Live Request Feed
-
- - -
-
-
-
Time
Status
Backend
Model
Latency
In
Out
-
-
- -
-
-
Request Log
-
- - - - - - -
-
-
-
Time
Status
Backend
Key
Model
Latency
In
Out
-
- -
- -
-
-
Mutable Settings (changes take effect immediately)
-
-
-
- - - -
-
Environment (requires restart to change)
-
-
Loading...
-
-
- -
-
Configured Backends
-
-
- -
-
-
Virtual API Keys
- -
- - - -
-
-
Loading...
-
-
-
- -
-
-
Model Routing
- -
- -
-
-
-
Loading...
-
-
-
- -
-
-
Audit Log
-
- - - - - -
-
-
-
-
Loading...
-
-
- -
- - - - - - - -// Shared state between the proxy and admin server. -// RuntimeConfig holds mutable settings; AdminEvent is broadcast to WebSocket clients. -⋮---- -use crate::admin::keys::VirtualKeyMeta; -use crate::config::ModelMapping; -use crate::metrics::Metrics; -use dashmap::DashMap; -use indexmap::IndexMap; -use std::collections::HashMap; -⋮---- -use tokio::sync::broadcast; -⋮---- -/// Type-erased closure that reloads the tracing filter at runtime. -/// Returns true on success, false if the filter string is invalid. -pub type LogReloadFn = Arc bool + Send + Sync>; -⋮---- -/// Shared between proxy handlers and the admin server. -⋮---- -pub struct SharedState { -/// SQLite connection for request logging and config persistence. -/// Uses std::sync::Mutex (not tokio::sync::Mutex) because rusqlite -/// is synchronous; holding a tokio Mutex guard across .await would -/// require the guard to be Send, which std::sync satisfies. -⋮---- -/// Broadcast channel sender for live dashboard updates. -⋮---- -/// Runtime-mutable config read on every proxy request. -/// std::sync::RwLock (not tokio): proxy reads are synchronous and -/// frequent; async locking would add unnecessary overhead. Write -/// contention is negligible since only the admin API writes. -⋮---- -/// Per-backend metrics (same Arc the proxy already uses). -⋮---- -/// Write buffer sender for batched SQLite inserts. -⋮---- -/// Closure to reload tracing filter at runtime. None in tests. -⋮---- -/// Serializes config write operations (Phase 1: SQLite + Phase 2: in-memory) -/// so concurrent PUT /admin/api/config requests cannot interleave. -⋮---- -/// In-memory cache of active virtual API keys, keyed by hash bytes -/// (HMAC-SHA256 for new keys, legacy SHA-256 for pre-HMAC keys). -/// Populated from SQLite at startup; updated on create/revoke via admin API. -⋮---- -/// Per-installation HMAC secret for keyed hashing of virtual API keys. -/// Generated once and persisted in the settings table. -⋮---- -/// Model router for dynamic model management. None unless LiteLLM config is active. -⋮---- -/// Run a synchronous closure against the SQLite connection on the blocking -/// threadpool. Recovers from mutex poisoning (unwrap_or_else on into_inner) -/// because a panic in one request should not permanently lock out the DB. -/// Returns None if spawn_blocking itself panicked (should not happen). -pub async fn with_db(db: &Arc>, f: F) -> Option -⋮---- -let db = db.clone(); -⋮---- -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -f(&conn) -⋮---- -.ok() -⋮---- -/// Runtime-mutable configuration. Changes via admin UI take effect immediately. -/// Env vars are the defaults; overrides from SQLite take precedence. -⋮---- -pub struct RuntimeConfig { -/// Per-backend model mappings (key = backend name). -⋮---- -/// Tracing filter string (e.g., "info", "debug"). -⋮---- -/// Whether to log request/response bodies at debug level. -⋮---- -/// Events broadcast to WebSocket clients for live dashboard updates. -⋮---- -pub enum AdminEvent { -/// Fired after each proxied request completes. -⋮---- -/// Periodic metrics summary. -⋮---- -/// Config changed via admin UI. -⋮---- -/// Data recorded for each proxied request. Stored in SQLite and broadcast -/// to WebSocket clients for the live admin dashboard. -⋮---- -pub struct RequestLogEntry { -⋮---- -/// Model name from the client's Anthropic request (before mapping). -⋮---- -/// Model name actually sent to the backend (after mapping). -⋮---- -/// Whether the request used SSE streaming. -⋮---- -/// Present only when the request failed; contains the error description. -⋮---- -/// Database row ID of the virtual key that authenticated this request. -/// None when the request used a static API key or open relay. -⋮---- -/// Estimated cost in USD for this request, computed from token usage -/// and the model pricing table. None when cost could not be calculated. -⋮---- -impl SharedState { -/// Construct a minimal SharedState for tests (in-memory DB, dummy channel). -pub fn new_for_test() -> Self { -let conn = rusqlite::Connection::open_in_memory().expect("in-memory sqlite"); -crate::admin::db::init_db(&conn).expect("init_db"); -⋮---- -log_level: "info".to_string(), -⋮---- -/// Aggregated metrics for the periodic WebSocket snapshot. -⋮---- -pub struct MetricsSnapshotData { - - - -// Model pricing loader and cost calculation. -// -// Loads pricing data from an embedded JSON file at startup. Calculates per-request -// cost from token counts by matching the backend model name against pricing entries. -⋮---- -pub mod db; -⋮---- -use dashmap::DashMap; -use std::sync::LazyLock; -⋮---- -/// Global pricing data, loaded once from embedded JSON at first access. -⋮---- -/// Tracks the highest alert level sent per key to avoid duplicate alerts. -/// Key: virtual key DB id, Value: highest threshold level (0-3). -⋮---- -/// Returns the spend alert level: 0=none, 1=80%, 2=95%, 3=100%. -pub fn spend_threshold_level(spend: f64, budget: f64) -> u8 { -⋮---- -/// Reset alert tracking for a key (call on budget period rollover). -pub fn reset_alert_level(key_id: i64) { -ALERT_LEVELS.remove(&key_id); -⋮---- -/// Check whether a spend alert should fire and, if so, send it via webhooks. -/// -/// Only fires when the threshold level increases (dedup). The webhook payload -/// includes key metadata and the crossed threshold percentage. -fn maybe_fire_spend_alert( -⋮---- -let level = spend_threshold_level(period_spend_usd, max_budget_usd); -⋮---- -// Check and update dedup map atomically. -⋮---- -let mut entry = ALERT_LEVELS.entry(key_id).or_insert(0); -⋮---- -// Fire webhook if configured (uses the global OnceLock from routes). -⋮---- -cb.notify_json(&payload); -⋮---- -/// Access the global model pricing table. -pub fn pricing() -> &'static ModelPricing { -⋮---- -pub struct ModelPricingEntry { -⋮---- -pub struct ModelPricing { -⋮---- -impl ModelPricing { -/// Load from embedded JSON (compiled into the binary). -pub fn load() -> Self { -let json = include_str!("../../../../assets/model_pricing.json"); -⋮---- -serde_json::from_str(json).expect("invalid model_pricing.json"); -⋮---- -/// Return (input_cost_per_token, output_cost_per_token) for a model, or None if unknown. -⋮---- -/// Same lookup order as cost_for_usage (exact then longest-prefix) but does not log -/// on miss, so it is safe to call during routing decisions. -pub fn price_for_model(&self, model: &str) -> Option<(f64, f64)> { -if let Some(entry) = self.entries.iter().find(|e| e.model_pattern == model) { -return Some((entry.input_cost_per_token, entry.output_cost_per_token)); -⋮---- -if model.starts_with(&entry.model_pattern) && entry.model_pattern.len() > best_len { -best = Some(entry); -best_len = entry.model_pattern.len(); -⋮---- -best.map(|e| (e.input_cost_per_token, e.output_cost_per_token)) -⋮---- -/// Calculate cost for a usage record. -⋮---- -/// Matching strategy: exact match first, then longest prefix match. -/// Returns 0.0 with a warning log if no match found. -pub fn cost_for_usage(&self, model: &str, input_tokens: u64, output_tokens: u64) -> f64 { -// 1. Try exact match -⋮---- -// 2. Try longest prefix match (e.g., "gpt-4o-2024-05-13" matches "gpt-4o") -⋮---- -// 3. No match -⋮---- -/// Record cost for a completed request against a virtual key. -⋮---- -/// Calculates cost from token usage and the resolved model name, then -/// persists the spend to SQLite asynchronously. Returns the computed cost -/// so the caller can set the `x-anyllm-cost-usd` header. -pub fn record_cost( -⋮---- -let cost = pricing().cost_for_usage(model, input_tokens, output_tokens); -⋮---- -let db = shared.db.clone(); -⋮---- -let period_reset = ctx.period_reset.clone(); -// Spawn a blocking task so the response is not delayed by the DB write. -⋮---- -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -// If the budget period rolled over during auth, reset SQLite first so that -// accumulate_spend starts from 0 instead of adding to the stale old-period total. -⋮---- -reset_alert_level(key_id); -⋮---- -// Check spend thresholds after accumulation. -⋮---- -maybe_fire_spend_alert( -⋮---- -spend.budget_duration.as_deref(), -⋮---- -mod tests { -⋮---- -fn test_pricing() -> ModelPricing { -⋮---- -entries: vec![ -⋮---- -fn exact_match() { -let pricing = test_pricing(); -let cost = pricing.cost_for_usage("gpt-4o", 1000, 500); -// 1000 * 0.0000025 + 500 * 0.00001 = 0.0025 + 0.005 = 0.0075 -⋮---- -assert!((cost - expected).abs() < 1e-12); -⋮---- -fn exact_match_prefers_longer() { -⋮---- -// "gpt-4o-mini" should match the gpt-4o-mini entry, not gpt-4o -let cost = pricing.cost_for_usage("gpt-4o-mini", 1000, 500); -⋮---- -fn prefix_match() { -⋮---- -// "gpt-4o-2024-05-13" should match "gpt-4o" by prefix -let cost = pricing.cost_for_usage("gpt-4o-2024-05-13", 1000, 500); -⋮---- -fn prefix_match_longest_wins() { -⋮---- -// "gpt-4o-mini-2024" should match "gpt-4o-mini" (longer prefix) not "gpt-4o" -let cost = pricing.cost_for_usage("gpt-4o-mini-2024", 1000, 500); -⋮---- -fn unknown_model_returns_zero() { -⋮---- -let cost = pricing.cost_for_usage("totally-unknown-model", 1000, 500); -assert_eq!(cost, 0.0); -⋮---- -fn zero_tokens() { -⋮---- -let cost = pricing.cost_for_usage("gpt-4o", 0, 0); -⋮---- -fn load_embedded_pricing() { -// Verify the embedded JSON parses without panic -⋮---- -assert!(!pricing.entries.is_empty()); -⋮---- -fn record_cost_without_shared_state_is_noop() { -// When there is no shared state or virtual key context, record_cost -// should return the computed cost but not attempt any DB write. -let cost = record_cost(&None, &None, "gpt-4o", 1000, 500); -// Should compute cost from global pricing (gpt-4o is in the embedded pricing). -// Exact value depends on the embedded pricing data, but should be > 0. -assert!(cost > 0.0); -⋮---- -fn record_cost_with_shared_state_persists_spend() { -// Build a minimal SharedState with an in-memory SQLite DB to verify -// that record_cost spawns a blocking task that writes to the DB. -⋮---- -use crate::admin::keys::RateLimitState; -use crate::server::middleware::VirtualKeyContext; -⋮---- -let conn = rusqlite::Connection::open_in_memory().unwrap(); -init_db(&conn).unwrap(); -⋮---- -description: Some("cost test"), -⋮---- -max_budget_usd: Some(100.0), -⋮---- -.unwrap(); -⋮---- -db: db.clone(), -⋮---- -log_level: "info".to_string(), -⋮---- -hmac_secret: Arc::new(b"test-secret".to_vec()), -⋮---- -// record_cost uses tokio::task::spawn_blocking, so we need a runtime. -let rt = tokio::runtime::Runtime::new().unwrap(); -rt.block_on(async { -let cost = record_cost(&Some(shared), &Some(vk_ctx), "gpt-4o", 1000, 500); -⋮---- -// Wait for the spawned blocking task to complete. -⋮---- -// Verify the spend was persisted. -let conn = db.lock().unwrap(); -let spend = db::get_key_spend(&conn, key_id).unwrap().unwrap(); -assert!(spend.total_cost_usd > 0.0); -assert_eq!(spend.total_input_tokens, 1000); -assert_eq!(spend.total_output_tokens, 500); -assert_eq!(spend.request_count, 1); -⋮---- -// -- Spend threshold detection tests -- -⋮---- -fn spend_threshold_detection() { -// Zero budget always returns 0 (no alerting). -assert_eq!(spend_threshold_level(50.0, 0.0), 0); -assert_eq!(spend_threshold_level(50.0, -10.0), 0); -⋮---- -// Below 80% -assert_eq!(spend_threshold_level(0.0, 100.0), 0); -assert_eq!(spend_threshold_level(79.99, 100.0), 0); -⋮---- -// At and above 80% -assert_eq!(spend_threshold_level(80.0, 100.0), 1); -assert_eq!(spend_threshold_level(85.0, 100.0), 1); -assert_eq!(spend_threshold_level(94.99, 100.0), 1); -⋮---- -// At and above 95% -assert_eq!(spend_threshold_level(95.0, 100.0), 2); -assert_eq!(spend_threshold_level(99.99, 100.0), 2); -⋮---- -// At and above 100% -assert_eq!(spend_threshold_level(100.0, 100.0), 3); -assert_eq!(spend_threshold_level(150.0, 100.0), 3); -⋮---- -fn spend_threshold_below_80_returns_0() { -// Boundary: 79.999...% is still below 80%. -assert_eq!(spend_threshold_level(79.999, 100.0), 0); -// Small budget, small spend. -assert_eq!(spend_threshold_level(0.79, 1.0), 0); -// Exactly at the boundary: 80/100 = 80%. -assert_eq!(spend_threshold_level(0.80, 1.0), 1); -⋮---- -fn reset_alert_level_clears_map() { -// Insert a tracked level. -ALERT_LEVELS.insert(-999, 2); -assert!(ALERT_LEVELS.contains_key(&-999)); -⋮---- -reset_alert_level(-999); -assert!(!ALERT_LEVELS.contains_key(&-999)); -⋮---- -// Resetting a non-existent key is a no-op (should not panic). -reset_alert_level(-998); -⋮---- -fn alert_dedup_fires_only_on_increase() { -// Use a unique key_id to avoid collisions with other tests. -⋮---- -// Simulate crossing 80% threshold. -// maybe_fire_spend_alert is not easily testable for webhook firing -// (no webhook configured in tests), but we can verify the dedup map. -maybe_fire_spend_alert(key_id, "sk-vktest", 80.0, 100.0, Some("monthly")); -assert_eq!(*ALERT_LEVELS.get(&key_id).unwrap(), 1); -⋮---- -// Same level should not update (still 1). -maybe_fire_spend_alert(key_id, "sk-vktest", 85.0, 100.0, Some("monthly")); -⋮---- -// Higher level (95%) should update. -maybe_fire_spend_alert(key_id, "sk-vktest", 95.0, 100.0, Some("monthly")); -assert_eq!(*ALERT_LEVELS.get(&key_id).unwrap(), 2); -⋮---- -// 100% should update to 3. -maybe_fire_spend_alert(key_id, "sk-vktest", 100.0, 100.0, Some("monthly")); -assert_eq!(*ALERT_LEVELS.get(&key_id).unwrap(), 3); -⋮---- -// Reset and verify re-alerting works. -⋮---- -// Clean up. - - - -// Integration tests for virtual key admin API (T038), rate limiting (T051), -// budget enforcement (US5), and RBAC (US6). -// -// Admin routes require CSRF double-submit cookie protection on POST/PUT/DELETE. -// Tests inject a fixed test token via X-CSRF-Token header + Cookie to satisfy the middleware. -⋮---- -use anyllm_proxy::admin; -⋮---- -use anyllm_proxy::server::routes; -use axum::body::Body; -use axum::extract::connect_info::MockConnectInfo; -use axum::http::Request; -use axum::routing::post; -use axum::Router; -use dashmap::DashMap; -use reqwest::Client; -use serde_json::json; -use std::net::SocketAddr; -⋮---- -use tokio::net::TcpListener; -use tower::ServiceExt; -⋮---- -// --------------------------------------------------------------------------- -// Shared DashMap for tests that need the proxy auth middleware. -// `set_virtual_keys` uses a global OnceLock — whichever test runs first wins. -// All proxy-auth tests share this one Arc so the middleware always -// looks at the same map that the tests populate. -⋮---- -fn shared_vk_map() -> Arc> { -⋮---- -.get_or_init(|| { -⋮---- -anyllm_proxy::server::middleware::set_virtual_keys(map.clone()); -⋮---- -.clone() -⋮---- -fn shared_hmac_secret() -> Arc> { -⋮---- -// Use a fixed test secret so all tests agree on hash values. -let secret = Arc::new(b"test-hmac-secret-for-integration".to_vec()); -anyllm_proxy::server::middleware::set_hmac_secret(secret.clone()); -⋮---- -/// Build a SharedState whose `virtual_keys` is the shared test map. -fn shared_state() -> admin::state::SharedState { -⋮---- -state.virtual_keys = shared_vk_map(); -state.hmac_secret = shared_hmac_secret(); -⋮---- -// Admin API CRUD tests (T038) -⋮---- -/// CSRF token used by unit-level tests (oneshot). Must match the cookie value below. -⋮---- -/// Cookie header value that satisfies the CSRF double-submit check for the token above. -⋮---- -fn test_admin_router() -> (Router, admin::state::SharedState) { -// Raise admin rate limit so parallel tests from 127.0.0.1 don't starve each other. -⋮---- -let state = shared_state(); -let token = Arc::new("test-admin-token".to_string()); -let router = admin::routes::admin_router(state.clone(), token) -// ConnectInfo extractor requires the service to be wrapped with -// into_make_service_with_connect_info in production. In tests we use -// MockConnectInfo so handlers can extract a fake peer address. -.layer(MockConnectInfo(SocketAddr::from(([127, 0, 0, 1], 0)))); -⋮---- -async fn create_key_returns_201_with_raw_key() { -let (app, _state) = test_admin_router(); -⋮---- -.header("host", "localhost:9090") -.header("authorization", "Bearer test-admin-token") -.header("content-type", "application/json") -.header("x-csrf-token", TEST_CSRF_TOKEN) -.header("cookie", TEST_CSRF_COOKIE) -.body(Body::from( -serde_json::to_string(&json!({"description": "test key", "rpm_limit": 60})).unwrap(), -⋮---- -.unwrap(); -⋮---- -let resp = app.oneshot(req).await.unwrap(); -assert_eq!(resp.status(), 201); -⋮---- -&axum::body::to_bytes(resp.into_body(), 1 << 20) -⋮---- -.unwrap(), -⋮---- -assert!(body["key"].as_str().unwrap().starts_with("sk-vk")); -assert!(body["id"].as_i64().is_some()); -assert_eq!(body["description"], "test key"); -assert_eq!(body["rpm_limit"], 60); -⋮---- -async fn list_keys_returns_created_keys() { -⋮---- -serde_json::to_string(&json!({"description": "list-test"})).unwrap(), -⋮---- -let _ = app.clone().oneshot(create_req).await.unwrap(); -⋮---- -.body(Body::empty()) -⋮---- -let resp = app.oneshot(list_req).await.unwrap(); -assert_eq!(resp.status(), 200); -⋮---- -let keys = body["keys"].as_array().unwrap(); -assert!(!keys.is_empty()); -⋮---- -async fn revoke_key_removes_from_dashmap() { -let (app, state) = test_admin_router(); -⋮---- -serde_json::to_string(&json!({"description": "revoke-test"})).unwrap(), -⋮---- -let resp = app.clone().oneshot(create_req).await.unwrap(); -⋮---- -let id = body["id"].as_i64().unwrap(); -let raw_key = body["key"].as_str().unwrap().to_string(); -⋮---- -let hash_bytes = admin::keys::hash_from_hex(&hash).unwrap(); -assert!(state.virtual_keys.contains_key(&hash_bytes)); -⋮---- -let revoke_req = Request::delete(format!("/admin/api/keys/{id}")) -⋮---- -let resp = app.clone().oneshot(revoke_req).await.unwrap(); -⋮---- -assert_eq!(body["status"], "revoked"); -assert!(!state.virtual_keys.contains_key(&hash_bytes)); -⋮---- -async fn revoke_nonexistent_key_returns_404() { -⋮---- -assert_eq!(resp.status(), 404); -⋮---- -// Update key (PUT /admin/api/keys/{id}) tests -⋮---- -async fn update_key_returns_200_with_updated_fields() { -⋮---- -// Create a key first. -⋮---- -serde_json::to_string(&json!({"description": "update-test", "rpm_limit": 10})).unwrap(), -⋮---- -// Update description and rpm_limit. -let update_req = Request::put(format!("/admin/api/keys/{id}")) -⋮---- -serde_json::to_string(&json!({ -⋮---- -let resp = app.clone().oneshot(update_req).await.unwrap(); -⋮---- -assert_eq!(body["description"], "updated-desc"); -assert_eq!(body["rpm_limit"], 200); -let models = body["allowed_models"].as_array().unwrap(); -assert_eq!(models.len(), 2); -assert_eq!(models[0], "gpt-4o"); -assert_eq!(models[1], "claude-*"); -⋮---- -async fn update_nonexistent_key_returns_404() { -⋮---- -serde_json::to_string(&json!({"description": "no-such-key"})).unwrap(), -⋮---- -async fn update_revoked_key_returns_404() { -⋮---- -// Create then revoke. -⋮---- -serde_json::to_string(&json!({"description": "revoke-then-update"})).unwrap(), -⋮---- -// Update should fail with 404. -⋮---- -serde_json::to_string(&json!({"description": "should-fail"})).unwrap(), -⋮---- -let resp = app.oneshot(update_req).await.unwrap(); -⋮---- -async fn update_key_refreshes_dashmap() { -⋮---- -// Create key. -⋮---- -serde_json::to_string(&json!({"description": "dashmap-update-test", "rpm_limit": 10})) -⋮---- -// Update rpm_limit via PUT. -⋮---- -serde_json::to_string(&json!({"rpm_limit": 500})).unwrap(), -⋮---- -// Verify DashMap entry was updated. -⋮---- -.get(&hash_bytes) -.expect("key should exist in DashMap"); -assert_eq!(meta.rpm_limit, Some(500)); -⋮---- -// Helpers for proxy-level tests -⋮---- -fn openai_config_with_base(base_url: &str) -> Config { -⋮---- -openai_api_key: "test-key".to_string(), -openai_base_url: base_url.to_string(), -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -backend_auth: BackendAuth::BearerToken("test-key".into()), -⋮---- -async fn spawn_mock_backend() -> String { -let app = Router::new().route( -⋮---- -post(|| async { -axum::Json(json!({ -⋮---- -let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let addr = listener.local_addr().unwrap(); -tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); -format!("http://{addr}") -⋮---- -/// Spawn a proxy backed by the shared VK map so auth middleware can find virtual keys. -/// Includes a dummy /admin/api/test route behind auth to test RBAC. -async fn spawn_proxy_with_shared_vk(config: Config) -> String { -let state = shared_state(); // must call before building app to ensure set_virtual_keys fires -⋮---- -let base_app = routes::app_multi_with_shared(multi, Some(state), None); -⋮---- -// Add a test /admin/ route behind the same auth middleware so RBAC can be tested. -⋮---- -.route( -⋮---- -axum::routing::get(|| async { axum::Json(json!({"ok": true})) }), -⋮---- -.layer(axum::middleware::from_fn( -⋮---- -let app = base_app.merge(admin_test); -⋮---- -// Virtual key auth lifecycle (T038): create → use → revoke → rejected -⋮---- -async fn virtual_key_auth_and_revocation_lifecycle() { -let mock = spawn_mock_backend().await; -let proxy_url = spawn_proxy_with_shared_vk(openai_config_with_base(&mock)).await; -⋮---- -// Admin server uses shared VK map so create/revoke affect the same DashMap -// the middleware checks. -⋮---- -let admin_app = admin::routes::admin_router(state, Arc::new("admin-token".to_string())); -let admin_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); -let admin_port = admin_listener.local_addr().unwrap().port(); -let admin_url = format!("http://127.0.0.1:{admin_port}"); -⋮---- -.unwrap() -⋮---- -// 1. Create a virtual key -⋮---- -.post(format!("{admin_url}/admin/api/keys")) -.header("host", format!("localhost:{admin_port}")) -.header("authorization", "Bearer admin-token") -⋮---- -.json(&json!({"description": "lifecycle-test"})) -.send() -⋮---- -let body: serde_json::Value = resp.json().await.unwrap(); -⋮---- -let key_id = body["id"].as_i64().unwrap(); -⋮---- -// 2. Use the virtual key to authenticate -⋮---- -.post(format!("{proxy_url}/v1/messages")) -.header("x-api-key", &raw_key) -.json(&json!({ -⋮---- -assert_eq!(resp.status(), 200, "virtual key should authenticate"); -⋮---- -// 3. Revoke the key -⋮---- -.delete(format!("{admin_url}/admin/api/keys/{key_id}")) -⋮---- -// 4. Revoked key must be rejected -⋮---- -assert_eq!(resp.status(), 401, "revoked key should be rejected"); -⋮---- -// RPM rate limiting (T051): create key with rpm_limit:2, 3rd request → 429 -⋮---- -async fn rpm_limit_returns_429_after_exceeded() { -⋮---- -let admin_app = admin::routes::admin_router(state, Arc::new("admin-token2".to_string())); -⋮---- -// Create a key with rpm_limit: 2 -⋮---- -.header("authorization", "Bearer admin-token2") -⋮---- -.json(&json!({"description": "rate-limit-test", "rpm_limit": 2})) -⋮---- -let msg = json!({ -⋮---- -// First 2 requests should succeed -⋮---- -.json(&msg) -⋮---- -// 3rd request must be rate-limited -⋮---- -assert_eq!(resp.status(), 429); -assert!( -⋮---- -// Budget enforcement tests (US5: T046) -⋮---- -/// Helper: create a key via admin API and return (raw_key, key_id). -async fn create_key_via_admin( -⋮---- -.header("authorization", format!("Bearer {admin_token}")) -⋮---- -.json(&body) -⋮---- -assert_eq!(resp.status(), 201, "create key failed"); -⋮---- -/// Helper: spawn admin + proxy servers, returns (proxy_url, admin_url, admin_port). -async fn spawn_test_servers(admin_token: &str) -> (String, String, u16) { -⋮---- -let admin_app = admin::routes::admin_router(state, Arc::new(admin_token.to_string())); -⋮---- -async fn budget_exceeded_returns_429_with_budget_exceeded_type() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("budget-token1").await; -⋮---- -// Create key with a tiny budget ($0.0001) and no duration (lifetime budget) -let (raw_key, _key_id) = create_key_via_admin( -⋮---- -json!({"description": "budget-test", "max_budget_usd": 0.0001}), -⋮---- -// Manually set the key's period_spend above the limit in the DashMap -let vk_map = shared_vk_map(); -let hash = admin::keys::hmac_hash_key(&raw_key, &shared_hmac_secret()); -⋮---- -if let Some(mut meta) = vk_map.get_mut(&hash_bytes) { -meta.period_spend_usd = 1.0; // Way over the $0.0001 limit -⋮---- -assert_eq!(resp.status(), 429, "budget exceeded should return 429"); -⋮---- -assert_eq!( -⋮---- -async fn budget_not_exceeded_allows_request() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("budget-token2").await; -⋮---- -// Create key with generous budget -⋮---- -json!({"description": "budget-ok-test", "spend_limit": 100.0, "budget_duration": "monthly"}), -⋮---- -assert_eq!(resp.status(), 200, "under-budget key should succeed"); -⋮---- -async fn budget_resets_on_new_period() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("budget-token3").await; -⋮---- -// Create key with daily budget -⋮---- -json!({"description": "budget-reset-test", "max_budget_usd": 0.0001, "budget_duration": "daily"}), -⋮---- -// Manually set the key's spend above limit AND set period_start to yesterday -⋮---- -meta.period_spend_usd = 1.0; // Over budget -meta.period_start = Some("2020-01-01T00:00:00Z".to_string()); // Long past -⋮---- -// The lazy reset should kick in and allow the request -⋮---- -assert_eq!(resp.status(), 200, "budget should reset for new period"); -⋮---- -async fn no_duration_budget_stays_blocked() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("budget-token4").await; -⋮---- -// Create key with lifetime budget (no duration) -⋮---- -json!({"description": "lifetime-budget-test", "max_budget_usd": 0.0001}), -⋮---- -// Set spend above limit; no duration means no reset -⋮---- -assert_eq!(resp.status(), 429, "lifetime budget should stay blocked"); -⋮---- -assert_eq!(body["error"]["type"], "budget_exceeded"); -// No budget_duration = lifetime (null in response) -assert!(body["error"]["budget_duration"].is_null()); -⋮---- -// RBAC tests (US6: T050) -⋮---- -async fn developer_key_succeeds_on_v1_messages() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("rbac-token1").await; -⋮---- -// Create developer key (default role) -⋮---- -json!({"description": "dev-key-test"}), -⋮---- -async fn developer_key_gets_403_on_admin() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("rbac-token2").await; -⋮---- -// Create developer key explicitly -⋮---- -json!({"description": "dev-admin-test", "role": "developer"}), -⋮---- -// Access the test /admin/ route on the proxy (added by spawn_proxy_with_shared_vk). -// The auth middleware checks RBAC before the route handler runs. -⋮---- -.get(format!("{proxy_url}/admin/api/test")) -⋮---- -assert_eq!(body["error"]["type"], "permission_denied"); -⋮---- -async fn admin_key_succeeds_on_v1_messages() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("rbac-token3").await; -⋮---- -// Create admin key -⋮---- -json!({"description": "admin-key-test", "role": "admin"}), -⋮---- -async fn admin_key_not_blocked_on_admin_path() { -let (proxy_url, admin_url, admin_port) = spawn_test_servers("rbac-token4").await; -⋮---- -json!({"description": "admin-path-test", "role": "admin"}), -⋮---- -// Admin key should NOT get 403. The test route returns 200. -⋮---- -async fn new_key_defaults_to_developer_role() { -let (_proxy_url, admin_url, admin_port) = spawn_test_servers("rbac-token5").await; -⋮---- -// Create key with no explicit role -⋮---- -json!({"description": "default-role-test"}), -⋮---- -// Check that the in-memory meta has developer role -⋮---- -let meta = vk_map.get(&hash_bytes).expect("key should exist in map"); -⋮---- -// Also check via list endpoint -⋮---- -.get(format!("{admin_url}/admin/api/keys")) -⋮---- -.header("authorization", "Bearer rbac-token5") -⋮---- -// Find our key by description -⋮---- -.iter() -.find(|k| k["description"] == "default-role-test") -.expect("our key should appear in list"); -assert_eq!(our_key["role"], "developer"); - - - -/// Passthrough client forwarding Anthropic requests as-is to upstream Anthropic API. -pub mod anthropic_client; -/// AWS Bedrock client with SigV4 request signing. -pub mod bedrock_client; -/// Gemini native generateContent client (no OpenAI translation layer). -pub mod gemini_client; -/// reqwest client for OpenAI-compatible Chat Completions and Responses APIs with retry/backoff. -pub mod openai_client; -⋮---- -// Re-export from the client crate so existing code paths (streaming, routes, etc.) keep working. -pub use anyllm_client::rate_limit::RateLimitHeaders; -⋮---- -use anyllm_client::http::HttpClientConfig; -⋮---- -/// Build a reqwest HTTP client from proxy TlsConfig (adapter to client crate). -pub(crate) fn build_http_client(tls: &TlsConfig) -> reqwest::Client { -⋮---- -p12_identity: tls.p12_identity.clone(), -ca_cert_pem: tls.ca_cert_pem.clone(), -⋮---- -/// Send a POST request with retry on 429/5xx. Returns the raw successful response. -/// Adapter that maps BackendAuth to the client crate's RequestAuth. -pub(crate) async fn send_with_retry( -⋮---- -/// Backend-agnostic client for dispatching requests to OpenAI, Vertex, Gemini, or Anthropic. -/// Callers pattern-match on the enum variants to access the typed inner clients directly. -⋮---- -pub enum BackendClient { -⋮---- -/// Same HTTP client as OpenAI, but targets the Responses API endpoint -/// with a different request/response shape. Separate variant so callers -/// can pattern-match on the API format. -⋮---- -/// Azure OpenAI: same Chat Completions format, different auth and URL scheme. -⋮---- -/// Gemini via OpenAI-compatible endpoint (reuses OpenAI translation path). -⋮---- -/// Passthrough to real Anthropic API (no translation). -⋮---- -/// AWS Bedrock: sends Anthropic-format requests with SigV4 signing. -⋮---- -/// Gemini native: sends generateContent requests directly (no OpenAI translation). -⋮---- -/// Unified error type for all backend clients. -⋮---- -pub enum BackendError { -⋮---- -impl BackendError { -/// HTTP status code for API errors, None for transport/deserialization errors. -pub fn api_error_status(&self) -> Option { -⋮---- -Self::OpenAI(OpenAIClientError::ApiError { status, .. }) => Some(*status), -Self::Bedrock(BedrockClientError::ApiError { status, .. }) => Some(*status), -Self::Gemini(GeminiClientError::ApiError { status, .. }) => Some(*status), -⋮---- -/// HTTP status code from an API error, or 500 for transport/deserialization errors. -pub fn status_code(&self) -> u16 { -self.api_error_status().unwrap_or(500) -⋮---- -/// Human-readable error message. -pub fn api_error_message(&self) -> String { -⋮---- -Self::OpenAI(e) => e.to_string(), -Self::Anthropic(e) => e.to_string(), -Self::Bedrock(e) => e.to_string(), -Self::Gemini(e) => e.to_string(), -⋮---- -/// Extract the upstream error message and HTTP status for API errors. -/// Returns None for transport/deserialization errors. -pub fn api_error_details(&self) -> Option<(&str, u16)> { -⋮---- -Some((&error.error.message, *status)) -⋮---- -fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -⋮---- -Self::OpenAI(e) => write!(f, "{e}"), -Self::Anthropic(e) => write!(f, "{e}"), -Self::Bedrock(e) => write!(f, "{e}"), -Self::Gemini(e) => write!(f, "{e}"), -⋮---- -fn from(e: OpenAIClientError) -> Self { -⋮---- -fn from(e: AnthropicClientError) -> Self { -⋮---- -fn from(e: BedrockClientError) -> Self { -⋮---- -fn from(e: GeminiClientError) -> Self { -⋮---- -impl BackendClient { -/// Forward a raw request to a passthrough endpoint (audio, images, etc.). -/// Returns `501 Not Implemented` for Anthropic/Bedrock backends. -pub async fn raw_passthrough( -⋮---- -let url = c.passthrough_url(path); -c.raw_passthrough(&url, body, content_type) -⋮---- -.map_err(BackendError::OpenAI) -⋮---- -format!( -⋮---- -let body = serde_json::to_vec(&err).unwrap_or_default(); -Ok(( -⋮---- -/// Forward a raw embeddings request to the backend. No translation — model names pass through. -/// Returns `501 Not Implemented` for the Anthropic backend (no embeddings endpoint). -pub async fn embeddings_passthrough( -⋮---- -.embeddings_passthrough(body, content_type) -⋮---- -.map_err(BackendError::OpenAI), -⋮---- -// Anthropic, Bedrock, and Gemini native have no embeddings passthrough. -⋮---- -"Embeddings are not supported by this backend.".to_string(), -⋮---- -/// Create a backend client from a single-backend [`Config`]. -/// -/// Dispatches on [`Config::backend`] and [`Config::openai_api_format`] to construct -/// the appropriate variant (OpenAI, OpenAIResponses, Vertex, GeminiOpenAI, or Anthropic). -pub fn new(config: &Config) -> Self { -⋮---- -.map(|v| v.to_lowercase() == "native") -.unwrap_or(false); -⋮---- -let base_url = std::env::var("GEMINI_BASE_URL").unwrap_or_else(|_| { -"https://generativelanguage.googleapis.com/v1beta".to_string() -⋮---- -crate::config::BackendAuth::GoogleApiKey(k) => k.clone(), -_ => config.openai_api_key.clone(), -⋮---- -config.model_mapping.big_model.clone(), -config.model_mapping.small_model.clone(), -⋮---- -// Bedrock config is stored in openai_base_url (region) and openai_api_key (unused). -// Credentials come from env vars at Config::from_env time. -unreachable!("Bedrock backend uses from_backend_config, not Config::new") -⋮---- -/// Construct from a per-backend config (multi-backend mode). -pub fn from_backend_config(bc: &BackendConfig) -> Self { -// Build a legacy Config to reuse existing OpenAI constructors. -// This avoids duplicating URL construction logic. -⋮---- -backend: bc.kind.clone(), -openai_api_key: bc.api_key.clone(), -openai_base_url: bc.base_url.clone(), -listen_port: 0, // unused by client constructors -model_mapping: bc.model_mapping.clone(), -tls: bc.tls.clone(), -backend_auth: bc.backend_auth.clone(), -⋮---- -openai_api_format: bc.api_format.clone(), -⋮---- -// bc.base_url has GEMINI_OPENAI_PATH ("/openai") appended; strip it. -⋮---- -.trim_end_matches('/') -.trim_end_matches("/openai") -.to_string(); -⋮---- -_ => bc.api_key.clone(), -⋮---- -bc.model_mapping.big_model.clone(), -bc.model_mapping.small_model.clone(), -⋮---- -bc.base_url.clone(), // region is stored in base_url for Bedrock -⋮---- -.clone() -.expect("Bedrock credentials must be set"), - - - -pub mod env_aliases; -pub mod litellm; -pub mod model_router; -mod tls; -mod url_validation; -⋮---- -pub use tls::TlsConfig; -⋮---- -use indexmap::IndexMap; -use serde::Deserialize; -use std::fmt; -use std::sync::Arc; -⋮---- -/// Path suffix appended to Gemini base URL to reach its OpenAI-compatible endpoint. -⋮---- -/// Which upstream backend the proxy targets. -⋮---- -pub enum BackendKind { -⋮---- -/// Which OpenAI API format to use (only relevant when BACKEND=openai). -⋮---- -pub enum OpenAIApiFormat { -/// Chat Completions API (default) -⋮---- -/// Responses API -⋮---- -/// How the proxy authenticates to the upstream backend. -⋮---- -pub enum BackendAuth { -/// `Authorization: Bearer {token}` (OpenAI, Vertex OAuth) -⋮---- -/// `x-goog-api-key: {key}` (Vertex API key) -⋮---- -/// `api-key: {key}` (Azure OpenAI) -⋮---- -fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { -⋮---- -Self::BearerToken(_) => write!(f, "BearerToken([REDACTED])"), -Self::GoogleApiKey(_) => write!(f, "GoogleApiKey([REDACTED])"), -Self::AzureApiKey(_) => write!(f, "AzureApiKey([REDACTED])"), -⋮---- -/// Proxy configuration loaded from environment variables. -⋮---- -pub struct Config { -⋮---- -/// Enable request/response body logging at debug level. -⋮---- -/// Validate that a GCP identifier (project ID, region) contains only safe characters. -/// Prevents URL injection when these values are interpolated into Vertex AI endpoint URLs. -fn validate_gcp_identifier(name: &str, value: &str) { -if value.is_empty() -⋮---- -.bytes() -.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_' || b == b'.') -⋮---- -panic!( -⋮---- -impl Config { -/// Build configuration from environment variables. Panics on invalid values -/// (unknown backend, bad GCP identifiers) to fail fast at startup. -pub fn from_env() -> Self { -let backend_str = std::env::var("BACKEND").unwrap_or_else(|_| "openai".into()); -let backend = match backend_str.to_ascii_lowercase().as_str() { -⋮---- -panic!("unknown BACKEND value '{other}', expected 'openai', 'azure', 'vertex', 'gemini', 'anthropic', or 'bedrock'") -⋮---- -.ok() -.and_then(|p| p.parse().ok()) -.unwrap_or(3000); -⋮---- -.map(|v| v == "true" || v == "1") -.unwrap_or(false); -⋮---- -.unwrap_or_else(|_| "https://api.openai.com".to_string()); -if let Err(e) = validate_base_url(&base_url) { -panic!("OPENAI_BASE_URL rejected: {e}"); -⋮---- -let api_key = std::env::var("OPENAI_API_KEY").unwrap_or_default(); -let backend_auth = BackendAuth::BearerToken(api_key.clone()); -⋮---- -.unwrap_or_else(|_| "chat".into()) -.to_ascii_lowercase() -.as_str() -⋮---- -other => panic!( -⋮---- -let endpoint = std::env::var("AZURE_OPENAI_ENDPOINT").unwrap_or_else(|_| { -panic!("AZURE_OPENAI_ENDPOINT is required when BACKEND=azure") -⋮---- -let deployment = std::env::var("AZURE_OPENAI_DEPLOYMENT").unwrap_or_else(|_| { -panic!("AZURE_OPENAI_DEPLOYMENT is required when BACKEND=azure") -⋮---- -let api_key = std::env::var("AZURE_OPENAI_API_KEY").unwrap_or_else(|_| { -panic!("AZURE_OPENAI_API_KEY is required when BACKEND=azure") -⋮---- -.unwrap_or_else(|_| "2024-10-21".to_string()); -⋮---- -// Pre-construct the full URL; no suffix is appended by OpenAIClient. -let base_url = format!( -⋮---- -// Validate the endpoint (not the full URL, which has query params) -if let Err(e) = validate_base_url(endpoint.trim_end_matches('/')) { -panic!("AZURE_OPENAI_ENDPOINT rejected: {e}"); -⋮---- -.unwrap_or_else(|_| panic!("VERTEX_PROJECT is required when BACKEND=vertex")); -⋮---- -.unwrap_or_else(|_| panic!("VERTEX_REGION is required when BACKEND=vertex")); -validate_gcp_identifier("VERTEX_PROJECT", &project); -validate_gcp_identifier("VERTEX_REGION", ®ion); -⋮---- -panic!("VERTEX_API_KEY or GOOGLE_ACCESS_TOKEN is required when BACKEND=vertex"); -⋮---- -panic!("Vertex base URL rejected: {e}"); -⋮---- -.unwrap_or_else(|_| panic!("GEMINI_API_KEY is required when BACKEND=gemini")); -⋮---- -let base_url = std::env::var("GEMINI_BASE_URL").unwrap_or_else(|_| { -"https://generativelanguage.googleapis.com/v1beta".to_string() -⋮---- -panic!("Gemini base URL rejected: {e}"); -⋮---- -openai_base_url: format!("{base_url}{GEMINI_OPENAI_PATH}"), -⋮---- -let api_key = std::env::var("ANTHROPIC_API_KEY").unwrap_or_else(|_| { -panic!("ANTHROPIC_API_KEY is required when BACKEND=anthropic") -⋮---- -.unwrap_or_else(|_| "https://api.anthropic.com".to_string()); -⋮---- -panic!("ANTHROPIC_BASE_URL rejected: {e}"); -⋮---- -.unwrap_or_else(|_| panic!("AWS_REGION is required when BACKEND=bedrock")); -validate_gcp_identifier("AWS_REGION", ®ion); // reuse safe-char validation -⋮---- -// Validate credentials are present at startup; the actual values -// are read again when constructing BedrockClient. -let _access_key_id = std::env::var("AWS_ACCESS_KEY_ID").unwrap_or_else(|_| { -panic!("AWS_ACCESS_KEY_ID is required when BACKEND=bedrock") -⋮---- -std::env::var("AWS_SECRET_ACCESS_KEY").unwrap_or_else(|_| { -panic!("AWS_SECRET_ACCESS_KEY is required when BACKEND=bedrock") -⋮---- -let _session_token = std::env::var("AWS_SESSION_TOKEN").ok(); -⋮---- -// Store region in openai_base_url for wrap_config -openai_base_url: region.clone(), -⋮---- -/// Maps Anthropic model names to OpenAI model names. -/// Pattern: "haiku" -> small_model, "sonnet"/"opus" -> big_model. -/// Unrecognized models pass through with a warning. -⋮---- -pub struct ModelMapping { -⋮---- -impl ModelMapping { -/// Load model mapping from `BIG_MODEL` / `SMALL_MODEL` env vars with OpenAI defaults. -⋮---- -/// Load model mapping from env vars, falling back to the provided defaults. -/// Each backend calls this with its own defaults (e.g., Gemini uses `gemini-2.5-pro`). -pub fn from_env_with_defaults(big_default: &str, small_default: &str) -> Self { -⋮---- -big_model: std::env::var("BIG_MODEL").unwrap_or_else(|_| big_default.into()), -small_model: std::env::var("SMALL_MODEL").unwrap_or_else(|_| small_default.into()), -⋮---- -/// Map an Anthropic model name to the configured OpenAI model. -pub fn map_model(&self, model: &str) -> String { -// ASCII case-insensitive substring check avoids allocating a lowercase copy. -let bytes = model.as_bytes(); -if contains_ignore_ascii_case(bytes, b"haiku") { -self.small_model.clone() -} else if contains_ignore_ascii_case(bytes, b"sonnet") -|| contains_ignore_ascii_case(bytes, b"opus") -⋮---- -self.big_model.clone() -⋮---- -model.to_string() -⋮---- -fn contains_ignore_ascii_case(haystack: &[u8], needle: &[u8]) -> bool { -⋮---- -.windows(needle.len()) -.any(|w| w.eq_ignore_ascii_case(needle)) -⋮---- -/// Read AWS credentials from environment variables for the Bedrock backend. -fn bedrock_credentials_from_env() -> aws_credential_types::Credentials { -⋮---- -.unwrap_or_else(|_| panic!("AWS_ACCESS_KEY_ID is required for bedrock")); -⋮---- -.unwrap_or_else(|_| panic!("AWS_SECRET_ACCESS_KEY is required for bedrock")); -let session_token = std::env::var("AWS_SESSION_TOKEN").ok(); -⋮---- -// --------------------------------------------------------------------------- -// Multi-backend configuration -⋮---- -/// Resolve a config value that may reference an env var via `env:VAR_NAME` prefix. -/// This allows TOML config files to reference secrets from the environment -/// without hardcoding them, keeping credentials out of version control. -pub fn resolve_env_value(value: &str) -> Result { -if let Some(var_name) = value.strip_prefix("env:") { -⋮---- -.map_err(|_| format!("env var '{var_name}' referenced in config is not set")) -} else if let Some(var_name) = value.strip_prefix("os.environ/") { -// LiteLLM-compatible syntax: "os.environ/VAR_NAME" -⋮---- -.map_err(|_| format!("env var '{var_name}' (os.environ/ syntax) is not set")) -⋮---- -Ok(value.to_string()) -⋮---- -/// Per-backend configuration. Each entry in `[backends.*]` deserializes into this. -⋮---- -pub struct BackendConfig { -/// Which provider type this backend uses (OpenAI, Vertex, Gemini, Anthropic). -⋮---- -/// API key for authentication. Resolved from env vars via `env:VAR_NAME` syntax. -⋮---- -/// Base URL of the backend API (e.g., `https://api.openai.com`). -⋮---- -/// Which OpenAI API format to use (Chat Completions or Responses). -⋮---- -/// Anthropic-to-backend model name mapping. -⋮---- -/// Optional mTLS and custom CA configuration. -⋮---- -/// How to authenticate to this backend (Bearer token or Google API key). -⋮---- -/// Whether to log request/response bodies at debug level. -⋮---- -/// Strip `stream_options` from streaming requests. Needed for local LLMs -/// (older Ollama, text-generation-webui, LM Studio) that reject unknown -/// fields with HTTP 400. -⋮---- -/// AWS credentials for Bedrock backend. None for all other backends. -⋮---- -/// Top-level multi-backend configuration loaded from TOML. -/// Enables routing requests to different backends by route prefix. -⋮---- -pub struct MultiConfig { -/// Port the proxy listens on (default: 3000). -⋮---- -/// Whether to log request/response bodies at debug level (global default). -⋮---- -/// Backend name used when no route prefix matches. -⋮---- -/// Ordered map: key = route prefix (e.g. "openai"), value = backend config. -⋮---- -// -- TOML deserialization structs (separate from runtime types) -- -⋮---- -struct TomlConfig { -⋮---- -struct TomlBackendConfig { -⋮---- -// Vertex-specific -⋮---- -// Azure-specific -⋮---- -// Optional env var name for Google access token (Vertex) -⋮---- -// Strip stream_options from streaming requests (local LLM compat) -⋮---- -// Bedrock-specific: AWS credentials (support env: prefix for env var resolution) -⋮---- -/// Result of `MultiConfig::load()`. -pub struct LoadResult { -⋮---- -/// Resolved master_key from LiteLLM general_settings, if present. -/// Caller should apply as PROXY_API_KEYS if that var is not already set. -⋮---- -impl MultiConfig { -/// Load configuration. -/// -/// Detection order: -/// 1. `PROXY_CONFIG` with `.yaml`/`.yml` extension: parse as LiteLLM config -/// 2. `PROXY_CONFIG` with any other extension: parse as TOML -/// 3. No `PROXY_CONFIG`: env-var-based single-backend config -⋮---- -/// The model router is only set for LiteLLM configs (model_list routing). -/// `litellm_master_key` is returned (not applied) so the caller can -/// consolidate all `set_var` calls into a single pre-runtime block. -pub fn load() -> LoadResult { -⋮---- -if path.ends_with(".yaml") || path.ends_with(".yml") { -⋮---- -.unwrap_or_else(|e| panic!("failed to read LiteLLM config '{path}': {e}")); -⋮---- -// Wire up webhook callbacks and named integrations from litellm_settings.callbacks. -let mut named = vec![]; -⋮---- -named.push(crate::integrations::NamedIntegration::Langfuse(lf)); -⋮---- -model_router: Some(Arc::new(std::sync::RwLock::new(parsed.router))), -⋮---- -/// Wrap a single-backend Config into a MultiConfig. -/// Used by the legacy `app(config)` path and by `from_legacy_env`. -pub fn from_single_config(config: &Config) -> Self { -⋮---- -/// Wrap the existing single-backend Config into a MultiConfig. -fn from_legacy_env() -> Self { -⋮---- -fn wrap_config(config: &Config) -> Self { -⋮---- -// For Bedrock, read AWS credentials from env vars. -⋮---- -Some(bedrock_credentials_from_env()) -⋮---- -kind: config.backend.clone(), -api_key: config.openai_api_key.clone(), -base_url: config.openai_base_url.clone(), -api_format: config.openai_api_format.clone(), -model_mapping: config.model_mapping.clone(), -tls: config.tls.clone(), -backend_auth: config.backend_auth.clone(), -⋮---- -backends.insert(name.to_string(), bc); -⋮---- -default_backend: name.to_string(), -⋮---- -/// Parse a TOML config file into MultiConfig. -fn from_toml_file(path: &str) -> Self { -⋮---- -.unwrap_or_else(|e| panic!("failed to read config file '{path}': {e}")); -⋮---- -/// Parse TOML string into MultiConfig. Separated from file I/O for testing. -pub fn from_toml_str(toml_str: &str) -> Self { -⋮---- -toml::from_str(toml_str).unwrap_or_else(|e| panic!("invalid TOML config: {e}")); -⋮---- -if raw.backends.is_empty() { -panic!("config must define at least one backend in [backends.*]"); -⋮---- -let listen_port = raw.listen_port.unwrap_or(3000); -let log_bodies = raw.log_bodies.unwrap_or(false); -⋮---- -.unwrap_or_else(|| raw.backends.keys().next().unwrap().clone()); -⋮---- -if !raw.backends.contains_key(&default_backend) { -⋮---- -backends.insert(name.clone(), bc); -⋮---- -fn build_backend_config( -⋮---- -let kind = match tb.kind.to_ascii_lowercase().as_str() { -⋮---- -other => panic!("unknown backend kind '{other}' for backend '{name}'"), -⋮---- -.as_deref() -.map(|v| resolve_env_value(v).unwrap_or_else(|e| panic!("backend '{name}': {e}"))) -.unwrap_or_default(); -⋮---- -.clone() -.unwrap_or_else(|| "https://api.openai.com".to_string()); -⋮---- -panic!("backend '{name}' base_url rejected: {e}"); -⋮---- -let auth = BackendAuth::BearerToken(api_key.clone()); -⋮---- -.unwrap_or("chat") -⋮---- -other => panic!("unknown api_format '{other}' for backend '{name}'"), -⋮---- -big_model: tb.big_model.clone().unwrap_or_else(|| "gpt-4o".to_string()), -⋮---- -.unwrap_or_else(|| "gpt-4o-mini".to_string()), -⋮---- -if api_key.is_empty() { -panic!("backend '{name}': api_key is required for azure"); -⋮---- -let endpoint = tb.endpoint.as_deref().unwrap_or_else(|| { -panic!("backend '{name}': 'endpoint' is required for azure") -⋮---- -let deployment = tb.deployment.as_deref().unwrap_or_else(|| { -panic!("backend '{name}': 'deployment' is required for azure") -⋮---- -let api_version = tb.api_version.as_deref().unwrap_or("2024-10-21"); -⋮---- -panic!("backend '{name}' endpoint rejected: {e}"); -⋮---- -let auth = BackendAuth::AzureApiKey(api_key.clone()); -⋮---- -let project = tb.project.as_deref().unwrap_or_else(|| { -panic!("backend '{name}': 'project' is required for vertex") -⋮---- -.unwrap_or_else(|| panic!("backend '{name}': 'region' is required for vertex")); -validate_gcp_identifier("project", project); -validate_gcp_identifier("region", region); -⋮---- -let base_url = tb.base_url.clone().unwrap_or_else(|| { -format!( -⋮---- -let auth = if !api_key.is_empty() { -BackendAuth::GoogleApiKey(api_key.clone()) -⋮---- -let token = resolve_env_value(token_ref) -.unwrap_or_else(|e| panic!("backend '{name}': {e}")); -⋮---- -panic!("backend '{name}': api_key or access_token is required for vertex"); -⋮---- -.unwrap_or_else(|| "gemini-2.5-pro".to_string()), -⋮---- -.unwrap_or_else(|| "gemini-2.5-flash".to_string()), -⋮---- -panic!("backend '{name}': api_key is required for gemini"); -⋮---- -let auth = BackendAuth::GoogleApiKey(api_key.clone()); -⋮---- -format!("{base_url}{GEMINI_OPENAI_PATH}"), -⋮---- -panic!("backend '{name}': api_key is required for anthropic"); -⋮---- -.unwrap_or_else(|| "https://api.anthropic.com".to_string()); -⋮---- -// Anthropic uses x-api-key header, stored as BearerToken for simplicity -// (the AnthropicClient will apply it correctly) -⋮---- -// No model mapping needed for passthrough -⋮---- -let region = tb.region.as_deref().unwrap_or_else(|| { -panic!("backend '{name}': 'region' is required for bedrock") -⋮---- -// For Bedrock, base_url stores the region (used by BedrockClient to build URLs) -⋮---- -big_model: tb.big_model.clone().unwrap_or_else(|| { -"anthropic.claude-sonnet-4-20250514-v1:0".to_string() -⋮---- -small_model: tb.small_model.clone().unwrap_or_else(|| { -"anthropic.claude-haiku-4-5-20251001-v1:0".to_string() -⋮---- -(region.to_string(), auth, mm, OpenAIApiFormat::Chat) -⋮---- -// Build AWS credentials for Bedrock from TOML fields or env vars. -⋮---- -.unwrap_or_else(|| { -std::env::var("AWS_ACCESS_KEY_ID").unwrap_or_else(|_| { -panic!("backend '{name}': aws_access_key_id or AWS_ACCESS_KEY_ID required") -⋮---- -.or_else(|| std::env::var("AWS_SESSION_TOKEN").ok()); -Some(aws_credential_types::Credentials::new( -⋮---- -tls: tls.clone(), -⋮---- -omit_stream_options: tb.omit_stream_options.unwrap_or(false), -⋮---- -mod tests { -⋮---- -fn model_mapping_haiku() { -⋮---- -big_model: "gpt-4o".into(), -small_model: "gpt-4o-mini".into(), -⋮---- -assert_eq!(m.map_model("claude-3-haiku-20240307"), "gpt-4o-mini"); -assert_eq!(m.map_model("claude-haiku-4-5-20251001"), "gpt-4o-mini"); -⋮---- -fn model_mapping_sonnet() { -⋮---- -assert_eq!(m.map_model("claude-sonnet-4-6"), "gpt-4o"); -assert_eq!(m.map_model("claude-3-5-sonnet-20241022"), "gpt-4o"); -⋮---- -fn model_mapping_opus() { -⋮---- -assert_eq!(m.map_model("claude-opus-4-6"), "gpt-4o"); -⋮---- -fn model_mapping_passthrough() { -⋮---- -// Unrecognized models pass through unchanged -assert_eq!(m.map_model("gpt-4o"), "gpt-4o"); -assert_eq!(m.map_model("custom-model"), "custom-model"); -⋮---- -fn model_mapping_case_insensitive() { -⋮---- -assert_eq!(m.map_model("Claude-Sonnet-4-6"), "gpt-4o"); -assert_eq!(m.map_model("CLAUDE-HAIKU-4-5"), "gpt-4o-mini"); -⋮---- -fn model_mapping_custom_values() { -⋮---- -big_model: "o1-preview".into(), -small_model: "o1-mini".into(), -⋮---- -assert_eq!(m.map_model("claude-sonnet-4-6"), "o1-preview"); -assert_eq!(m.map_model("claude-haiku-4-5-20251001"), "o1-mini"); -⋮---- -// --- Vertex / BackendKind tests --- -⋮---- -fn vertex_url_construction() { -let url = format!( -⋮---- -assert_eq!( -⋮---- -fn vertex_base_url_passes_ssrf() { -⋮---- -assert!(validate_base_url(url).is_ok()); -⋮---- -fn vertex_model_defaults() { -⋮---- -// When BIG_MODEL/SMALL_MODEL env vars are not set, uses Vertex defaults -// (This test works because env vars are unlikely to be set in test environment) -assert_eq!(m.map_model("claude-sonnet-4-6"), "gemini-2.5-pro"); -assert_eq!(m.map_model("claude-haiku-4-5"), "gemini-2.5-flash"); -⋮---- -fn backend_auth_debug_redacts() { -let bearer = BackendAuth::BearerToken("secret-token".into()); -let debug = format!("{:?}", bearer); -assert!(debug.contains("REDACTED")); -assert!(!debug.contains("secret-token")); -⋮---- -let api_key = BackendAuth::GoogleApiKey("secret-key".into()); -let debug = format!("{:?}", api_key); -⋮---- -assert!(!debug.contains("secret-key")); -⋮---- -let azure_key = BackendAuth::AzureApiKey("azure-secret".into()); -let debug = format!("{:?}", azure_key); -⋮---- -assert!(!debug.contains("azure-secret")); -⋮---- -// --- MultiConfig TOML parsing tests --- -⋮---- -fn multi_config_parses_openai_backend() { -⋮---- -assert_eq!(mc.listen_port, 4000); -assert_eq!(mc.default_backend, "openai"); -assert_eq!(mc.backends.len(), 1); -⋮---- -assert_eq!(bc.kind, BackendKind::OpenAI); -assert_eq!(bc.api_key, "sk-test"); -assert_eq!(bc.model_mapping.big_model, "gpt-4o"); -assert_eq!(bc.model_mapping.small_model, "gpt-4o-mini"); -⋮---- -fn multi_config_parses_multiple_backends() { -⋮---- -assert_eq!(mc.backends.len(), 3); -assert_eq!(mc.backends["openai"].kind, BackendKind::OpenAI); -assert_eq!(mc.backends["gemini"].kind, BackendKind::Gemini); -assert_eq!(mc.backends["claude"].kind, BackendKind::Anthropic); -⋮---- -fn multi_config_defaults_first_backend_as_default() { -⋮---- -assert_eq!(mc.default_backend, "gemini"); -⋮---- -fn multi_config_defaults_listen_port() { -⋮---- -assert_eq!(mc.listen_port, 3000); -⋮---- -fn multi_config_openai_defaults_base_url() { -⋮---- -assert_eq!(mc.backends["openai"].base_url, "https://api.openai.com"); -⋮---- -fn multi_config_anthropic_defaults_base_url() { -⋮---- -assert_eq!(mc.backends["claude"].base_url, "https://api.anthropic.com"); -⋮---- -fn multi_config_custom_base_url() { -⋮---- -fn multi_config_api_format_responses() { -⋮---- -assert_eq!(mc.backends["openai"].api_format, OpenAIApiFormat::Responses); -⋮---- -fn multi_config_panics_no_backends() { -⋮---- -fn multi_config_panics_invalid_default() { -⋮---- -fn multi_config_panics_unknown_kind() { -⋮---- -fn multi_config_panics_gemini_no_key() { -⋮---- -fn multi_config_panics_anthropic_no_key() { -⋮---- -fn resolve_env_value_inline() { -assert_eq!(resolve_env_value("my-key").unwrap(), "my-key"); -⋮---- -fn resolve_env_value_from_env() { -⋮---- -fn resolve_env_value_missing_env() { -let err = resolve_env_value("env:NONEXISTENT_VAR_99999").unwrap_err(); -assert!(err.contains("not set")); -⋮---- -fn multi_config_env_prefix_resolves() { -⋮---- -assert_eq!(mc.backends["openai"].api_key, "sk-from-env"); -⋮---- -fn multi_config_log_bodies() { -⋮---- -assert!(mc.log_bodies); -assert!(mc.backends["openai"].log_bodies); -⋮---- -fn multi_config_gemini_defaults() { -⋮---- -assert_eq!(bc.model_mapping.big_model, "gemini-2.5-pro"); -assert_eq!(bc.model_mapping.small_model, "gemini-2.5-flash"); -// /openai is appended to route through Gemini's OpenAI-compatible endpoint -⋮---- -// --- Azure OpenAI tests --- -⋮---- -fn multi_config_parses_azure_backend() { -⋮---- -assert_eq!(bc.kind, BackendKind::AzureOpenAI); -⋮---- -assert!(matches!(bc.backend_auth, BackendAuth::AzureApiKey(_))); -⋮---- -fn multi_config_azure_custom_api_version() { -⋮---- -assert!(bc.base_url.contains("api-version=2025-01-01")); -⋮---- -fn multi_config_panics_azure_no_key() { -⋮---- -fn multi_config_panics_azure_no_endpoint() { -⋮---- -fn multi_config_panics_azure_no_deployment() { - - - -[package] -name = "anyllm_proxy" -description = "HTTP proxy translating Anthropic Messages API to OpenAI Chat Completions" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -anyllm_translate = { path = "../translator", version = "0.1.0" } -anyllm_client = { path = "../client", version = "0.2.0" } -axum = { version = "0.8", features = ["ws", "multipart"] } -tokio = { version = "1", features = ["full"] } -reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "native-tls", "http2", "multipart"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -futures = "0.3" -tokio-stream = "0.1" -tower = "0.5" -url = "2" -tracing = "0.1" -tiktoken-rs = "0.9" -tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -uuid = { version = "1", features = ["v4"] } -toml = "1.0.7" -indexmap = { version = "2.13.0", features = ["serde"] } -bytes = "1.11.1" -subtle = "2" -sha2 = "0.10" -hmac = "0.12" -rusqlite = { version = "0.32", features = ["bundled"] } -httpdate = "1" -dashmap = "6" -aws-sigv4 = { version = "1.4", features = ["sign-http"] } -aws-credential-types = "1.2" -aws-smithy-runtime-api = "1" -base64 = "0.22" -hex = "0.4" -moka = { version = "0.12", features = ["future"] } -serde_yaml = "0.9" -jsonwebtoken = "10" -ipnetwork = "0.20" -zeroize = "1" - -[features] -redis = ["dep:redis"] -qdrant = ["dep:qdrant-client"] -otel = [ - "opentelemetry", - "opentelemetry_sdk", - "opentelemetry-otlp", - "tracing-opentelemetry", -] - -[dependencies.opentelemetry] -version = "0.31" -optional = true - -[dependencies.opentelemetry_sdk] -version = "0.31" -optional = true - -[dependencies.opentelemetry-otlp] -version = "0.31" -default-features = false -features = ["trace", "http-proto", "reqwest-client"] -optional = true - -[dependencies.tracing-opentelemetry] -version = "0.32" -optional = true - -[dependencies.redis] -version = "0.27" -features = ["tokio-comp", "connection-manager"] -optional = true - -[dependencies.qdrant-client] -version = "1" -optional = true - -[dev-dependencies] -pretty_assertions = "1" -reqwest = { version = "0.12", default-features = false, features = ["json", "native-tls", "multipart"] } -tokio = { version = "1", features = ["full"] } - - - -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## What This Is - -**anyllm-proxy** is an API translation proxy in Rust. Accepts Anthropic Messages API requests and OpenAI Chat Completions requests, translates between formats, forwards to any supported backend, and translates back. Supports streaming SSE, tool calling, file/document blocks, virtual key management, and optional OpenTelemetry export. - -All implementation phases are complete. - -## Current Status - -**Working (verified):** -- Build: `cargo build` clean, `cargo clippy -- -D warnings` clean -- Tests: ~906 tests passing, 8 ignored (live API) -- Full Anthropic Messages API translation: non-streaming, streaming SSE, tool calling, file/document blocks -- `POST /v1/chat/completions` input: accepts OpenAI Chat Completions format, returns OpenAI format (unblocks all OpenAI-native clients) -- Azure OpenAI backend: `BACKEND=azure` with deployment-scoped URL and `api-key` header -- Virtual key management: admin API to create/list/revoke keys stored in SQLite, with DashMap cache for auth; no proxy restart required -- Per-key rate limiting: RPM/TPM sliding window per virtual key, returns 429 with `retry-after` on excess -- Rust client library v0.2.0: `ClientBuilder`, `ToolBuilder`, `messages_stream()` returning `impl Stream` -- Optional OpenTelemetry export: `--features otel` enables OTLP span export; zero overhead when feature is off -- Proxy middleware: health, auth (env-var keys + virtual keys), request ID, size limits, concurrency limits, retry with backoff -- Compatibility endpoints: /v1/models, count_tokens (approximate via tiktoken) -- Anthropic batch API: `/v1/messages/batches` (create, get, list, cancel, results) translated to/from OpenAI batch format -- Gemini native path: direct `generateContent` API, non-streaming + streaming SSE with full-response diffing -- Strict tool calling: sets `strict: true` on the forced tool when `tool_choice: {type: "tool", name: "X"}` -- Langfuse integration: native tracing when `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` set, or via config `callbacks: ["langfuse"]` -- CSRF protection: admin state-mutating endpoints require `X-CSRF-Token` header (double-submit cookie pattern) -- Per-entry cache TTL: `MemoryCache` enforces per-entry TTL via moka `Expiry` trait -- Configurable Redis fail policy: `RATE_LIMIT_FAIL_POLICY=open|closed` (default: open) -- Cost tracking: `record_cost()` wired into all paths; `key_id` + `cost_usd` in request log -- Audit log: admin config mutations recorded in SQLite `audit_log` table -- Spend alerts: webhook notifications at 80% / 95% / 100% of key budget -- Model allowlist: per-key policy with exact match and `prefix/*` wildcard -- Admin UI: login form (sessionStorage), virtual keys tab, models tab, request detail view, cost column, feed pause + filter -- Security hardening: plaintext HTTP startup warning, 1MB admin body limit, CSP header, model name validation -- Security fixes (2026-03-30 audit): `AWS_ACCESS_KEY_ID`/`GOOGLE_ACCESS_TOKEN` redacted in env endpoint; admin rate limiter uses sliding window; all audit entries include `source_ip`; OIDC discovery and webhook callbacks use SSRF-safe HTTP client and validate URLs against private IP ranges; CSRF public-route decision documented; non-Unix token file warning already present -- Model mapping and lossy-translation warnings -- `POST /v1/embeddings` passthrough: forwards directly to the backend with no translation; works with OpenAI, Vertex, Gemini (`gemini-embedding-exp-03-07`), and vLLM/HuggingFace models. Not mounted for the Anthropic passthrough backend. -- `x-anyllm-degradation` response header: set when features are silently dropped during translation (e.g., `top_k`, `thinking_config`, `cache_control`, `document_blocks`, `stop_sequences_truncated`) - -**Not fully validated:** -- OpenAI Responses API backend: wired up via `OPENAI_API_FORMAT=responses` but not tested against live API -- AWS Bedrock backend: wired up via `BACKEND=bedrock` with SigV4 signing and Event Stream decoding; not tested against live API. Run with `AWS_REGION=... AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... cargo test --test live_bedrock -- --ignored --test-threads=1` -- Azure OpenAI backend: wired up via `BACKEND=azure`; not tested against live API. Run with `AZURE_OPENAI_API_KEY=... cargo test --test live_azure -- --ignored --test-threads=1` -- Live API integration tests exist (`crates/proxy/tests/live_api.rs`) but are `#[ignore]` by default; run with `OPENAI_API_KEY=sk-... cargo test --test live_api -- --ignored --test-threads=1` - -## Build and Test - -```bash -cargo build # build everything -cargo build --features otel # with OpenTelemetry support -cargo test # run all tests (~906 tests, 8 ignored) -cargo test -p anyllm_client # client crate only -cargo test -p anyllm_translate # translator crate only -cargo test -p anyllm_proxy # proxy crate only -cargo test health_endpoint # single test by name -cargo test --test virtual_keys # virtual key + rate limit integration tests -cargo clippy -- -D warnings # lint -cargo fmt --check # format check -``` - -Run the proxy (requires OPENAI_API_KEY): -```bash -OPENAI_API_KEY=sk-... cargo run -p anyllm_proxy -# Listens on 0.0.0.0:3000, health at GET /health -``` - -## Environment Variables - -- `BACKEND`: Backend provider: `openai` (default), `azure`, `vertex`, `gemini`, `anthropic` (passthrough), or `bedrock` (SigV4-signed, Anthropic format) -- `OPENAI_API_KEY`: OpenAI API key (required when BACKEND=openai, empty default) -- `OPENAI_BASE_URL`: OpenAI base URL (default: `https://api.openai.com`) -- `OPENAI_API_FORMAT`: OpenAI API format: `chat` (default, Chat Completions) or `responses` (Responses API). Only relevant when BACKEND=openai. -- `LISTEN_PORT`: Server port (default: `3000`) -- `BIG_MODEL`: Backend model for sonnet/opus requests (default: `gpt-4o` for OpenAI, `gemini-2.5-pro` for Vertex/Gemini) -- `SMALL_MODEL`: Backend model for haiku requests (default: `gpt-4o-mini` for OpenAI, `gemini-2.5-flash` for Vertex/Gemini) -- `RUST_LOG`: Tracing filter (e.g., `info`, `anyllm_proxy=debug`) -- `TLS_CLIENT_CERT_P12`: Path to PKCS#12 (.p12/.pfx) client certificate for mTLS to the backend (optional) -- `TLS_CLIENT_CERT_PASSWORD`: Password to decrypt the P12 file (required if P12 is set) -- `TLS_CA_CERT`: Path to PEM-encoded CA certificate for verifying the backend server (optional) -- `VERTEX_PROJECT`: GCP project ID (required when BACKEND=vertex) -- `VERTEX_REGION`: GCP region, e.g. `us-central1` (required when BACKEND=vertex) -- `VERTEX_API_KEY`: Google API key for Vertex AI (one of VERTEX_API_KEY or GOOGLE_ACCESS_TOKEN required when BACKEND=vertex) -- `GOOGLE_ACCESS_TOKEN`: OAuth bearer token for Vertex AI (alternative to VERTEX_API_KEY) -- `GEMINI_API_KEY`: Google API key for Gemini Developer API (required when BACKEND=gemini) -- `GEMINI_BASE_URL`: Gemini API base URL (default: `https://generativelanguage.googleapis.com/v1beta`) -- `AWS_REGION`: AWS region for Bedrock (required when BACKEND=bedrock) -- `AWS_ACCESS_KEY_ID`: AWS access key ID for SigV4 signing (required when BACKEND=bedrock) -- `AWS_SECRET_ACCESS_KEY`: AWS secret access key for SigV4 signing (required when BACKEND=bedrock) -- `AWS_SESSION_TOKEN`: Temporary session token for STS credentials (optional, BACKEND=bedrock) -- `AZURE_OPENAI_ENDPOINT`: Azure OpenAI resource endpoint, e.g. `https://myresource.openai.azure.com` (required when BACKEND=azure) -- `AZURE_OPENAI_DEPLOYMENT`: Deployment name, e.g. `gpt4o` (required when BACKEND=azure) -- `AZURE_OPENAI_API_KEY`: Azure API key (required when BACKEND=azure) -- `AZURE_OPENAI_API_VERSION`: API version (default: `2024-10-21`, optional when BACKEND=azure) -- `PROXY_API_KEYS`: Comma-separated list of allowed API keys for proxy authentication (optional; if unset and PROXY_OPEN_RELAY is not set, all requests are rejected) -- `PROXY_OPEN_RELAY`: Set to `true` or `1` to accept any non-empty key (insecure, for local dev only) -- `LOG_BODIES`: Enable request/response body logging at debug level (`true` or `1`, default: disabled) -- `OTEL_EXPORTER_OTLP_ENDPOINT`: OTLP collector endpoint (default: `http://localhost:4318`). Only effective when built with `--features otel`. -- `OTEL_SERVICE_NAME`: Service name for exported traces. Only effective when built with `--features otel`. -- `OTEL_TRACES_SAMPLER`: Sampling strategy (default: `parentbased_always_on`). Only effective when built with `--features otel`. -- `PROXY_CONFIG`: Path to config file. TOML for multi-backend config, or `.yaml`/`.yml` for LiteLLM-compatible config with model_list routing. -- `IP_ALLOWLIST`: Comma-separated CIDR ranges for IP allowlisting (e.g., `192.168.1.0/24,10.0.0.0/8`). Bare IPs also accepted. When set, only matching IPs can access the proxy. -- `TRUST_PROXY_HEADERS`: Set to `true` or `1` to use `X-Forwarded-For` header for client IP when behind a reverse proxy. Only effective when `IP_ALLOWLIST` is set. -- `WEBHOOK_URLS`: Comma-separated webhook URLs for request completion notifications. Fire-and-forget HTTP POST with `RequestLogEntry` JSON payload. -- `RATE_LIMIT_FAIL_POLICY`: Behavior when Redis rate limiter is unavailable: `open` (default, allow requests) or `closed`/`deny` (reject with 503 and retry-after 60s). - -### LiteLLM env var aliases - -These LiteLLM env var names are accepted as aliases at startup (target takes precedence if already set): -- `LITELLM_MASTER_KEY` -> `PROXY_API_KEYS` -- `LITELLM_CONFIG` -> `PROXY_CONFIG` -- `AZURE_API_KEY` -> `AZURE_OPENAI_API_KEY` -- `AZURE_API_BASE` -> `AZURE_OPENAI_ENDPOINT` -- `AZURE_API_VERSION` -> `AZURE_OPENAI_API_VERSION` -- `AWS_REGION_NAME` -> `AWS_REGION` -- `LITELLM_IP_ALLOWLIST` -> `IP_ALLOWLIST` - -## Architecture - -Cargo workspace with three crates: - -### `crates/client` (lib: `anyllm_client`) v0.2.0 -High-level async HTTP client (Anthropic-in, Anthropic-out). Depends on `anyllm_translate` for translation logic. Key modules: -- **`client.rs`**: `Client` struct; `ClientBuilder` with method chaining (base_url, api_key, timeout, max_retries, tls_config); `messages()` for non-streaming, `messages_stream()` returning `impl Stream>` -- **`tools.rs`**: `ToolBuilder` (name, description, input_schema) and `ToolChoiceBuilder` (auto/any/none/specific) -- **`http.rs`**: reqwest client builder with optional SSRF-safe DNS resolution and mTLS (PKCS#12) -- **`retry.rs`**: Generic retry with exponential backoff + jitter; `is_retryable`, `send_with_retry` -- **`rate_limit.rs`**: Parses `x-ratelimit-*` / `retry-after` headers into a typed struct -- **`sse.rs`**: Framework-agnostic SSE frame parser (`find_double_newline`) -- **`error.rs`**: `ClientError` enum - -### `crates/translator` (lib: `anyllm_translate`) -Pure translation logic, no IO. Key modules: -- **`anthropic/`**: Anthropic Messages API types (request, response, streaming events, errors) -- **`openai/`**: OpenAI types for both Chat Completions and Responses APIs -- **`mapping/`**: Stateless conversion functions between the two APIs - - `message_map`: Message/content block translation (system prompt -> developer role); also `openai_to_anthropic_request` and `anthropic_to_openai_response` for reverse direction - - `tools_map`: Tool definitions and tool_use/tool_call translation - - `usage_map`: Token usage field mapping - - `errors_map`: HTTP status and error shape translation - - `streaming_map`: SSE event stream translation state machine (OpenAI chunks -> Anthropic events) - - `reverse_streaming_map`: `ReverseStreamingTranslator` (Anthropic SSE events -> OpenAI ChatCompletionChunk) - - `responses_message_map`: Anthropic to/from OpenAI Responses API mapping - - `responses_streaming_map`: Responses API SSE event stream translation state machine - - `warnings`: `TranslationWarnings` collector; lossy drops are surfaced via `x-anyllm-degradation` response header -- **`middleware/`**: Request/response handler orchestrating translation and backend calls -- **`util/`**: JSON helpers, ID generation (uuid v4), secret redaction -- **`config.rs`**: Translator-level configuration, **`error.rs`**: Error types, **`translate.rs`**: Top-level translation entry points - -### `crates/proxy` (bin: `anyllm_proxy`) -HTTP proxy built on axum + reqwest: -- **`config/`**: Env-based configuration (`mod.rs`), TLS client cert setup (`tls.rs`), URL validation (`url_validation.rs`) -- **`server/routes.rs`**: Axum router (POST /v1/messages, POST /v1/chat/completions, GET /health, GET /metrics, GET /v1/models, stub for count_tokens, POST /v1/messages/batches and related batch endpoints); `record_vk_tpm` for post-response TPM recording -- **`server/chat_completions.rs`**: Handler for POST /v1/chat/completions (OpenAI format in, OpenAI format out); uses `ReverseStreamingTranslator` for streaming -- **`server/middleware.rs`**: Auth validation (env-var keys + virtual key DashMap), RPM/TPM pre-check, request ID injection, 32MB size limit, concurrency limit, `VirtualKeyContext` extension for TPM recording -- **`server/sse.rs`**: SSE response helpers for Anthropic-format streaming -- **`server/streaming.rs`**: SSE streaming handler with pre-stream error propagation and backpressure -- **`server/passthrough.rs`**: Anthropic passthrough handler (no translation, forwards as-is) -- **`server/bedrock_passthrough.rs`**: Bedrock handler (SigV4 signing, model-in-URL, Event Stream decoding for streaming) -- **`server/token_counting.rs`**: Approximate token counting via tiktoken -- **`backend/mod.rs`**: `BackendClient` enum (OpenAI/AzureOpenAI/OpenAIResponses/Vertex/GeminiOpenAI/Anthropic/Bedrock), `BackendError`, shared retry helpers -- **`backend/openai_client.rs`**: reqwest client calling OpenAI-compatible Chat Completions with retry/backoff on 429/5xx (used for OpenAI, Azure, Vertex, and Gemini backends) -- **`backend/anthropic_client.rs`**: Passthrough client forwarding Anthropic requests as-is to upstream Anthropic API (no translation) -- **`backend/bedrock_client.rs`**: AWS Bedrock client with SigV4 signing, AWS Event Stream binary frame decoder for streaming -- **`admin/`**: Admin server (localhost-only) with config management, WebSocket live updates (`ws.rs`), token auth (`auth.rs`, `db.rs`, `mod.rs`, `routes.rs`, `state.rs`) -- **`admin/keys.rs`**: Virtual key generation (SHA-256 hashed, `sk-vk` prefix), `VirtualKeyMeta`, `RateLimitState` (sliding window RPM/TPM) -- **`admin/routes.rs`**: Admin API endpoints including POST/GET/DELETE `/admin/api/keys` for virtual key CRUD -- **`admin-ui/`**: Static admin UI served by the admin server (`index.html`) -- **`metrics/`**: Request count, success/error tracking, exposed via GET /metrics -- **`otel.rs`**: OpenTelemetry initialization behind `#[cfg(feature = "otel")]`; `OtelGuard` shuts down the provider on drop - -### Data Flow -``` -Client (Anthropic format) -> proxy (axum) - -> translator: anthropic types -> mapping -> openai types - -> backend: reqwest -> OpenAI Chat Completions - -> translator: openai types -> mapping -> anthropic types - -> proxy (axum) -> Client (Anthropic format) -``` - -## Key Design Decisions - -- The translator crate is deliberately IO-free: all mapping is pure `fn(A) -> B`. This makes it testable without mocks. -- Tool call IDs pass through directly (Anthropic tool_use.id = OpenAI tool_call.id). -- OpenAI `arguments` is a JSON string; Anthropic `input` is a JSON object. The mapping layer handles serialization. -- Streaming uses a state machine in `streaming_map.rs` that transforms OpenAI chunk events into Anthropic SSE events, with bounded channel (32) for backpressure. -- JSON fixtures in `fixtures/anthropic/` and `fixtures/openai/` are used for golden-file testing (14 fixture files). -- Retry logic: 3 retries with exponential backoff + 25% jitter, respects retry-after header. -- Backoff jitter is deterministic (upper bound, not random) to keep tests predictable. -- `ChatCompletionRequest` uses `#[serde(flatten)] pub extra: serde_json::Map` to capture unknown OpenAI fields (e.g., `seed`, `logprobs`, `logit_bias`, `n`, `reasoning_effort`). These pass through to OpenAI without typed handling. Only fields that require translation logic (not just forwarding) need explicit struct fields. -- DeepSeek/Qwen thinking model support: `reasoning_content` on `ChatMessage` and `ChunkDelta` maps bidirectionally to Anthropic thinking blocks. Request direction: Anthropic `Thinking` content blocks become `reasoning_content` on the assistant message. Response direction: `reasoning_content` becomes an Anthropic `Thinking` block preceding the text content. Streaming: `reasoning_content` deltas open a thinking content block, which is closed when regular `content` deltas begin. The `thinking` config (`budget_tokens`) is stripped with a warning since it has no standard OpenAI equivalent. -- Local LLM compatibility: streaming tool calls handle missing/empty IDs by generating synthetic `toolu_` IDs. `FinishReason::Unknown` (serde catch-all) maps to `end_turn` for providers like DeepSeek that use non-standard finish reasons (e.g., `insufficient_system_resource`). - -## Conventions - -- Some source files reference PLAN.md line ranges in a comment at the top (historical; PLAN.md has been removed). -- Test files live alongside source (`#[cfg(test)]` modules) and in `crates/proxy/tests/` for integration tests. -- Error types use `thiserror` derive macros. -- Test distribution: translator (~305 tests including reverse translation), proxy + client (~240 tests including virtual key CRUD + rate limiting integration), plus doc tests. Counts shift as features are added. -- Virtual key CRUD integration tests are in `crates/proxy/tests/virtual_keys.rs`. They use a shared `OnceLock` to avoid fighting over the global `set_virtual_keys` OnceLock. -- The `PROXY_OPEN_RELAY=true` env var enables dev mode (any non-empty key accepted). Without it and without `PROXY_API_KEYS`, the proxy rejects all requests. - -## References - -- OpenAI API spec: https://github.com/openai/openai-openapi/blob/manual_spec/openapi.yaml (very large, ~70k+ lines). See https://simonwillison.net/2024/Dec/22/openai-openapi/ for context on the spec's size and structure. Do not attempt to load the full spec into context; reference specific sections as needed. - -## Recent Changes -- 001-litellm-parity: Added Rust stable (1.83+, workspace edition 2021) -- 20260325-120000-litellm-gap-fill: Added POST /v1/chat/completions (OpenAI format input), Azure OpenAI backend (BACKEND=azure), virtual key management (admin API + DashMap cache), per-key RPM/TPM rate limiting, Rust client v0.2.0 (ClientBuilder + ToolBuilder + messages_stream), optional OpenTelemetry export (--features otel), ReverseStreamingTranslator in translator crate, reverse translation functions openai_to_anthropic_request / anthropic_to_openai_response -- parity-gaps: Routing strategies (least-busy, latency-based, weighted), dynamic model management admin API, /v1/models enrichment, IP allowlisting (CIDR, X-Forwarded-For), webhook callbacks -- 20260327: Gemini native generateContent path; Anthropic batch API (/v1/messages/batches); strict tool calling; Langfuse integration; CSRF protection; per-entry cache TTL; Redis fail policy; cost tracking + audit log + spend alerts + model allowlist; admin UI overhaul; security hardening; jsonwebtoken CVE fix - -## Active Technologies -- Rust stable (1.83+, workspace edition 2021) (001-litellm-parity) -- SQLite (existing, extended with new tables); Redis (optional Tier 1 cache); Qdran (001-litellm-parity) - - - -// SSE streaming infrastructure and the messages_stream handler. -⋮---- -use crate::metrics::Metrics; -⋮---- -use bytes::BytesMut; -use futures::stream::Stream; -use tokio::sync::mpsc; -use tokio_stream::wrappers::ReceiverStream; -⋮---- -/// Send translated stream events over the SSE channel. Returns false if client disconnected. -pub(super) async fn send_events( -⋮---- -if tx.send(Ok(sse)).await.is_err() { -⋮---- -/// Why the SSE stream ended. -pub(super) enum StreamOutcome { -/// Backend stream completed normally. -⋮---- -/// Downstream client disconnected before the stream finished. -⋮---- -/// Backend stream failed (error already recorded in metrics). -⋮---- -impl StreamOutcome { -/// Record metrics and return (HTTP status, error message) for logging. -fn record(&self, metrics: &Metrics) -> (u16, Option) { -⋮---- -metrics.record_success(); -metrics.record_stream_completed(); -⋮---- -metrics.record_stream_client_disconnected(); -(499, Some("client disconnected".into())) -⋮---- -metrics.record_stream_failed(); -(502, Some("stream interrupted".into())) -⋮---- -/// Read SSE bytes from a response, parse frames, and call `on_data` for each data line. -pub(super) async fn read_sse_frames( -⋮---- -use futures::StreamExt; -let mut stream = response.bytes_stream(); -// BytesMut (not String) because TCP chunks may split mid-UTF-8 character. -// String::from_utf8_lossy would permanently replace partial trailing bytes -// with U+FFFD, corrupting the JSON payload. -⋮---- -// Reuse a single events buffer across all frames to avoid per-frame allocation -⋮---- -// Track where to start the next delimiter search so we don't rescan -// already-inspected bytes when a large SSE event spans many TCP chunks. -⋮---- -while let Some(chunk_result) = stream.next().await { -⋮---- -metrics.record_error(); -⋮---- -buffer.extend_from_slice(&bytes); -⋮---- -// Guard against unbounded buffer growth from a misbehaving backend. -if buffer.len() > MAX_SSE_BUFFER_SIZE { -⋮---- -while let Some((pos, delim_len)) = find_double_newline(&buffer, search_from) { -frame_events.clear(); -// Convert the complete frame bytes to UTF-8. A frame ending at -// a double-newline boundary should always be valid UTF-8; if not, -// skip the malformed frame rather than injecting replacement chars. -⋮---- -for line in frame_str.lines() { -let line = line.trim(); -if let Some(json_str) = line.strip_prefix("data: ") { -if let Some(mut events) = on_data(json_str) { -frame_events.append(&mut events); -⋮---- -let _ = buffer.split_to(pos + delim_len); -// split_to shifted the buffer; restart search at the beginning -⋮---- -if !send_events(tx, &frame_events).await { -⋮---- -// Next chunk: resume scanning 3 bytes back from the end. The 4-byte -// delimiter \r\n\r\n could straddle the chunk boundary (e.g., \r\n at -// end of this chunk, \r\n at start of the next). -search_from = buffer.len().saturating_sub(3); -⋮---- -/// Build an SSE response that streams Anthropic events translated from backend chunks. -/// Returns rate limit headers alongside the SSE stream so the caller can inject them. -/// Pre-stream backend errors (e.g., 401, 429, 500 before any data) are returned as -/// `Err(BackendError)` so the caller can respond with a proper HTTP status code. -/// Logging is deferred: each spawned task logs after the stream completes with actual -/// latency, status, and token counts. -pub(crate) async fn messages_stream( -⋮---- -let metrics = state.metrics.clone(); -let log_shared = state.shared.clone(); -let log_backend_name = state.backend_name.clone(); -⋮---- -let client = client.clone(); -⋮---- -// Strip Gemini-incompatible JSON Schema keywords from tool parameters. -if matches!( -⋮---- -if let Some(tools) = openai_req.tools.take() { -openai_req.tools = Some( -⋮---- -.into_iter() -.map(|mut t| { -if let Some(params) = t.function.parameters.take() { -t.function.parameters = Some( -⋮---- -.collect(), -⋮---- -openai_req.model = mapped_model.clone(); -let model = body.model.clone(); -let permit = concurrency_permit.clone(); -⋮---- -// Hold concurrency permit until the stream completes, not just -// until headers are sent, so the semaphore accurately bounds -// concurrent streaming connections. -⋮---- -metrics.record_stream_started(); -match client.chat_completion_stream(&openai_req).await { -⋮---- -rl_tx.send(Ok(rate_limits)).ok(); -⋮---- -let outcome = read_sse_frames(response, &tx, &metrics, |json_str| { -⋮---- -let events = translator.finish(); -return Some(events); -⋮---- -Ok(chunk) => Some(translator.process_chunk(&chunk)), -⋮---- -if matches!(outcome, StreamOutcome::Completed) && !done { -⋮---- -send_events(&tx, &events).await; -⋮---- -let usage = translator.usage(); -let tokens = usage.map(|u| (u.input_tokens as u64, u.output_tokens as u64)); -// Record cost for virtual key spend tracking. -⋮---- -Some(crate::cost::record_cost( -⋮---- -let (status, err) = outcome.record(&metrics); -log_request( -⋮---- -ctx.log_entry_with_attribution( -⋮---- -Some(mapped_model), -⋮---- -let status = e.status_code(); -let err_msg = e.to_string(); -⋮---- -Some(err_msg), -⋮---- -// Send the error through the oneshot so the caller can -// return a proper HTTP error response instead of 200 OK. -let _ = rl_tx.send(Err(crate::backend::BackendError::from(e))); -⋮---- -responses_req.model = mapped_model.clone(); -responses_req.stream = Some(true); -⋮---- -match client.responses_stream(&responses_req).await { -⋮---- -Ok(event) => Some(translator.process_event(&event)), -⋮---- -if matches!(outcome, StreamOutcome::Completed) { -⋮---- -drop(rl_tx); -⋮---- -.send(Ok(Event::default().data( -⋮---- -Ok(Ok(rate_limits)) => Ok(( -⋮---- -Sse::new(ReceiverStream::new(rx)).keep_alive(KeepAlive::default()), -⋮---- -Ok(Err(backend_err)) => Err(backend_err), -// Sender dropped without sending (e.g., Anthropic passthrough branch or task panic). -// Default to empty rate limits and let the stream deliver whatever it has. -Err(_) => Ok(( - - - -// SQLite schema, migrations, queries, and write buffer for request logging. -⋮---- -use crate::admin::state::RequestLogEntry; -⋮---- -use tokio::sync::mpsc; -⋮---- -/// Run an ALTER TABLE ADD COLUMN statement, ignoring "duplicate column" errors -/// so migrations are idempotent across restarts. -fn idempotent_add_column(conn: &Connection, stmt: &str) -> rusqlite::Result<()> { -match conn.execute_batch(stmt) { -Ok(()) => Ok(()), -Err(e) if e.to_string().contains("duplicate column") => Ok(()), -Err(e) => Err(e), -⋮---- -/// Initialize the SQLite database: create tables and indexes. -pub fn init_db(conn: &Connection) -> rusqlite::Result<()> { -// WAL mode: better read concurrency (proxy reads while admin writes) -// and crash recovery compared to the default rollback journal. -conn.execute_batch("PRAGMA journal_mode=WAL;")?; -conn.execute_batch( -⋮---- -// Schema migrations for virtual_api_key new columns (idempotent via IF NOT EXISTS). -// SQLite 3.37+ supports ADD COLUMN IF NOT EXISTS. -⋮---- -idempotent_add_column(conn, stmt)?; -⋮---- -// request_log migrations: add key_id and cost_usd for request attribution. -⋮---- -// Index on key_id for filtering requests by virtual key. -⋮---- -Ok(()) -⋮---- -/// Ensure an HMAC secret exists in the settings table. Creates one if missing. -/// Returns the 32-byte secret used for HMAC-SHA256 key hashing. -/// The secret is generated from two UUID v4s (uuid is already a dep) to avoid -/// adding a CSPRNG dependency; the entropy is sufficient for HMAC keying. -pub fn ensure_hmac_secret(conn: &Connection) -> Vec { -⋮---- -.expect("create settings table"); -⋮---- -.query_row( -⋮---- -|row| row.get(0), -⋮---- -.ok(); -⋮---- -// Generate 32 random bytes from two UUID v4s. -⋮---- -buf[..16].copy_from_slice(a.as_bytes()); -buf[16..].copy_from_slice(b.as_bytes()); -⋮---- -conn.execute( -⋮---- -.expect("insert hmac_secret"); -⋮---- -buf.to_vec() -⋮---- -/// Insert a single request log entry. -pub fn insert_request_log(conn: &Connection, entry: &RequestLogEntry) -> rusqlite::Result<()> { -⋮---- -params![ -⋮---- -/// Query request log with optional filters and pagination. -/// Typed status code filter -- prevents SQL injection by construction. -/// Only valid patterns are representable; invalid input is rejected at parse time. -enum StatusFilter { -⋮---- -impl StatusFilter { -fn parse(s: &str) -> Option { -⋮---- -"2xx" => Some(Self::Class2xx), -"4xx" => Some(Self::Class4xx), -"5xx" => Some(Self::Class5xx), -other => other.parse::().ok().map(Self::Exact), -⋮---- -fn apply_to_query( -⋮---- -sql.push_str(" AND status_code = ?"); -params.push(Box::new(*code as i64)); -⋮---- -Self::Class2xx => sql.push_str(" AND status_code >= 200 AND status_code < 300"), -Self::Class4xx => sql.push_str(" AND status_code >= 400 AND status_code < 500"), -Self::Class5xx => sql.push_str(" AND status_code >= 500 AND status_code < 600"), -⋮---- -pub fn query_request_log( -⋮---- -sql.push_str(" AND backend = ?"); -param_values.push(Box::new(b.to_string())); -⋮---- -sql.push_str(" AND timestamp >= ?"); -param_values.push(Box::new(s.to_string())); -⋮---- -sql.push_str(" AND timestamp <= ?"); -param_values.push(Box::new(u.to_string())); -⋮---- -parsed.apply_to_query(&mut sql, &mut param_values); -⋮---- -// Invalid filter silently ignored -⋮---- -sql.push_str(" AND key_id = ?"); -param_values.push(Box::new(kid)); -⋮---- -sql.push_str(" ORDER BY id DESC LIMIT ? OFFSET ?"); -param_values.push(Box::new(limit)); -param_values.push(Box::new(offset)); -⋮---- -param_values.iter().map(|p| p.as_ref()).collect(); -⋮---- -let mut stmt = conn.prepare(&sql)?; -let rows = stmt.query_map(params_refs.as_slice(), row_to_request_log)?; -⋮---- -rows.collect() -⋮---- -/// Map a SQLite row to a RequestLogEntry. Column order must match the SELECT -/// used in query_request_log and get_request_by_id. -fn row_to_request_log(row: &rusqlite::Row) -> rusqlite::Result { -Ok(RequestLogEntry { -request_id: row.get(0)?, -timestamp: row.get(1)?, -backend: row.get(2)?, -model_requested: row.get(3)?, -model_mapped: row.get(4)?, -⋮---- -input_tokens: row.get::<_, Option>(7)?.map(|v| v as u64), -output_tokens: row.get::<_, Option>(8)?.map(|v| v as u64), -⋮---- -error_message: row.get(10)?, -key_id: row.get(11)?, -cost_usd: row.get(12)?, -⋮---- -/// Get a single request log entry by request_id. -pub fn get_request_by_id( -⋮---- -let mut stmt = conn.prepare( -⋮---- -let mut rows = stmt.query_map(params![request_id], row_to_request_log)?; -rows.next().transpose() -⋮---- -// -- Config overrides -- -⋮---- -/// Get all config overrides from SQLite. -pub fn get_config_overrides(conn: &Connection) -> rusqlite::Result> { -⋮---- -conn.prepare("SELECT key, value, updated_at FROM config_override ORDER BY key")?; -let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?; -⋮---- -/// Set a config override (upsert). -pub fn set_config_override(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> { -let now = chrono_now(); -⋮---- -params![key, value, now], -⋮---- -/// Delete a config override. -pub fn delete_config_override(conn: &Connection, key: &str) -> rusqlite::Result { -let changed = conn.execute("DELETE FROM config_override WHERE key = ?1", params![key])?; -Ok(changed > 0) -⋮---- -/// Delete request log entries older than the given number of days. -pub fn purge_old_logs(conn: &Connection, retention_days: u32) -> rusqlite::Result { -// SQLite datetime comparison: delete rows where timestamp < cutoff -⋮---- -.duration_since(std::time::UNIX_EPOCH) -.unwrap_or_default() -.as_secs() -.saturating_sub(retention_days as u64 * 86400); -let cutoff_iso = epoch_to_iso8601(cutoff); -let changed = conn.execute( -⋮---- -params![cutoff_iso], -⋮---- -Ok(changed) -⋮---- -/// Count request log entries with a timestamp >= `since_epoch` (Unix seconds). -/// Used to compute requests-per-second for the metrics dashboard. -pub fn count_requests_since(conn: &Connection, since_epoch: u64) -> rusqlite::Result { -let since_iso = epoch_to_iso8601(since_epoch); -let count: i64 = conn.query_row( -⋮---- -Ok(count.max(0) as u64) -⋮---- -/// Spawn the write buffer background task. Returns the sender for proxy handlers. -/// Flushes every 100ms or 100 rows, whichever comes first. -pub fn spawn_write_buffer(db: Arc>) -> mpsc::Sender { -⋮---- -// Channel closed, flush remaining and exit. -⋮---- -async fn flush_buffer(db: &Arc>, buf: &mut Vec) { -⋮---- -let db = db.clone(); -// Run SQLite IO on the blocking threadpool to avoid stalling the tokio executor. -// On failure, return the entries so they can be re-queued for retry. -⋮---- -// Mutex poisoning recovery: if a prior request panicked while holding the lock, -// we recover the inner value rather than permanently locking the database. -// This is safe because SQLite transactions provide ACID guarantees -- a panic -// mid-transaction means the transaction was rolled back by SQLite. -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -let tx = conn.unchecked_transaction()?; -⋮---- -insert_request_log(&tx, entry)?; -⋮---- -tx.commit()?; -⋮---- -Some(entries) -⋮---- -// On failure, re-queue entries so they can be retried on the next flush. -⋮---- -buf.append(&mut entries); -// Cap retry buffer to prevent unbounded growth on persistent DB failure. -⋮---- -if buf.len() > MAX_RETRY_BUFFER { -let dropped = buf.len() - MAX_RETRY_BUFFER; -buf.drain(..dropped); -⋮---- -/// ISO 8601 UTC timestamp for "now". -fn chrono_now() -> String { -// Use std only, no chrono dependency. Format: 2026-03-22T10:15:30Z -use std::time::SystemTime; -⋮---- -.duration_since(SystemTime::UNIX_EPOCH) -.unwrap(); -epoch_to_iso8601(dur.as_secs()) -⋮---- -/// Convert unix epoch seconds to ISO 8601 string (UTC, second precision). -pub(crate) fn epoch_to_iso8601(epoch: u64) -> String { -// Manual conversion without chrono. -⋮---- -// Days since 1970-01-01 to year/month/day. -let (year, month, day) = days_to_ymd(days); -⋮---- -format!( -⋮---- -/// Convert unix epoch milliseconds to ISO 8601 string with millisecond precision. -/// Format: "2026-03-27T10:15:30.500Z" -pub(crate) fn epoch_to_iso8601_ms(epoch_ms: u64) -> String { -⋮---- -let base = epoch_to_iso8601(secs); -// epoch_to_iso8601 returns "YYYY-MM-DDTHH:MM:SSZ"; strip the Z, append .mmmZ -let without_z = base.trim_end_matches('Z'); -format!("{}.{:03}Z", without_z, ms) -⋮---- -/// Convert days since 1970-01-01 to (year, month, day). -pub(crate) fn days_to_ymd(days: u64) -> (u64, u64, u64) { -// Algorithm from http://howardhinnant.github.io/date_algorithms.html -⋮---- -/// Get the current time as ISO 8601 UTC string. -pub fn now_iso8601() -> String { -chrono_now() -⋮---- -// --- Virtual API Key CRUD --- -⋮---- -use super::keys::VirtualKeyRow; -⋮---- -/// Parameters for creating a new virtual key. -pub struct InsertVirtualKeyParams<'a> { -⋮---- -/// Insert a new virtual API key. -pub fn insert_virtual_key(conn: &Connection, p: &InsertVirtualKeyParams) -> rusqlite::Result { -let now = now_iso8601(); -⋮---- -// Set period_start to now if budget_duration is set -⋮---- -Ok(conn.last_insert_rowid()) -⋮---- -/// Map a SQLite row to a VirtualKeyRow. -fn row_to_virtual_key(row: &rusqlite::Row) -> rusqlite::Result { -Ok(VirtualKeyRow { -id: row.get(0)?, -key_hash: row.get(1)?, -key_prefix: row.get(2)?, -description: row.get(3)?, -created_at: row.get(4)?, -expires_at: row.get(5)?, -revoked_at: row.get(6)?, -rpm_limit: row.get::<_, Option>(7)?.map(|v| v as u32), -tpm_limit: row.get::<_, Option>(8)?.map(|v| v as u32), -spend_limit: row.get(9)?, -total_spend: row.get::<_, f64>(10).unwrap_or(0.0), -total_requests: row.get::<_, i64>(11).unwrap_or(0), -total_tokens: row.get::<_, i64>(12).unwrap_or(0), -⋮---- -.unwrap_or_else(|_| "developer".into()), -max_budget_usd: row.get(14).unwrap_or(None), -budget_duration: row.get(15).unwrap_or(None), -period_start: row.get(16).unwrap_or(None), -period_spend_usd: row.get::<_, f64>(17).unwrap_or(0.0), -total_input_tokens: row.get::<_, i64>(18).unwrap_or(0), -total_output_tokens: row.get::<_, i64>(19).unwrap_or(0), -⋮---- -.unwrap_or(None) -.and_then(|s| serde_json::from_str(&s).ok()), -⋮---- -/// List all virtual keys (active, expired, revoked). -pub fn list_virtual_keys(conn: &Connection) -> rusqlite::Result> { -let sql = format!("SELECT {VIRTUAL_KEY_COLUMNS} FROM virtual_api_key ORDER BY id DESC"); -⋮---- -let rows = stmt.query_map([], row_to_virtual_key)?; -⋮---- -/// Revoke a virtual key by setting revoked_at. Returns the row if found. -pub fn revoke_virtual_key(conn: &Connection, id: i64) -> rusqlite::Result> { -⋮---- -let updated = conn.execute( -⋮---- -params![now, id], -⋮---- -return Ok(None); -⋮---- -let sql = format!("SELECT {VIRTUAL_KEY_COLUMNS} FROM virtual_api_key WHERE id = ?1"); -⋮---- -stmt.query_row(params![id], |row| Ok(Some(row_to_virtual_key(row)?))) -⋮---- -/// Parameters for updating an existing virtual key (all fields are optional; None = clear). -pub struct UpdateVirtualKeyParams<'a> { -⋮---- -/// Update an existing virtual key. Returns the updated row, or None if not found / revoked. -/// When `budget_duration` is provided, the budget period is reset (period_start = NULL, -/// period_spend_usd = 0) so the new window starts fresh. -pub fn update_virtual_key( -⋮---- -// When changing budget_duration, reset the spend period so the new window starts clean. -⋮---- -if p.budget_duration.is_some() { -sql.push_str(", period_start = NULL, period_spend_usd = 0.0"); -⋮---- -sql.push_str(" WHERE id = ?1 AND revoked_at IS NULL"); -⋮---- -/// Load all active (non-revoked, non-expired) virtual keys from the database. -pub fn load_active_virtual_keys(conn: &Connection) -> rusqlite::Result> { -⋮---- -let sql = format!( -⋮---- -let rows = stmt.query_map(params![now], row_to_virtual_key)?; -⋮---- -// --- Audit Log --- -⋮---- -/// A single audit log entry recording an admin mutation. -⋮---- -pub struct AuditEntry { -⋮---- -/// Insert an audit log entry with current UTC timestamp. -pub fn insert_audit_entry(conn: &Connection, entry: &AuditEntry) -> rusqlite::Result<()> { -let ts = chrono_now(); -⋮---- -/// Query the audit log, returning entries in reverse chronological order. -pub fn query_audit_log( -⋮---- -sql.push_str(" AND action = ?"); -param_values.push(Box::new(a.to_string())); -⋮---- -sql.push_str(" AND target_type = ?"); -param_values.push(Box::new(t.to_string())); -⋮---- -param_values.iter().map(|v| v.as_ref()).collect(); -let rows = stmt.query_map(param_refs.as_slice(), |row| { -Ok(AuditEntry { -id: Some(row.get(0)?), -timestamp: Some(row.get(1)?), -action: row.get(2)?, -target_type: row.get(3)?, -target_id: row.get(4)?, -detail: row.get(5)?, -source_ip: row.get(6)?, -⋮---- -mod tests { -⋮---- -fn in_memory_db() -> Connection { -let conn = Connection::open_in_memory().unwrap(); -init_db(&conn).unwrap(); -⋮---- -fn sample_entry() -> RequestLogEntry { -⋮---- -request_id: "test-123".into(), -timestamp: "2099-01-01T00:00:00Z".into(), -backend: "openai".into(), -model_requested: Some("claude-sonnet-4-6".into()), -model_mapped: Some("gpt-4o".into()), -⋮---- -input_tokens: Some(150), -output_tokens: Some(87), -⋮---- -fn init_db_creates_tables() { -let conn = in_memory_db(); -// Verify tables exist by querying them. -⋮---- -.query_row("SELECT COUNT(*) FROM request_log", [], |r| r.get(0)) -⋮---- -assert_eq!(count, 0); -⋮---- -.query_row("SELECT COUNT(*) FROM config_override", [], |r| r.get(0)) -⋮---- -fn insert_and_query_request_log() { -⋮---- -let entry = sample_entry(); -insert_request_log(&conn, &entry).unwrap(); -⋮---- -let results = query_request_log(&conn, 10, 0, None, None, None, None, None).unwrap(); -assert_eq!(results.len(), 1); -assert_eq!(results[0].request_id, "test-123"); -assert_eq!(results[0].status_code, 200); -assert_eq!(results[0].latency_ms, 342); -assert_eq!(results[0].input_tokens, Some(150)); -⋮---- -fn query_with_backend_filter() { -⋮---- -insert_request_log(&conn, &sample_entry()).unwrap(); -⋮---- -let mut entry2 = sample_entry(); -entry2.request_id = "test-456".into(); -entry2.backend = "gemini".into(); -insert_request_log(&conn, &entry2).unwrap(); -⋮---- -let results = query_request_log(&conn, 10, 0, Some("gemini"), None, None, None, None).unwrap(); -⋮---- -assert_eq!(results[0].backend, "gemini"); -⋮---- -fn query_with_status_filter() { -⋮---- -let mut err_entry = sample_entry(); -err_entry.request_id = "test-err".into(); -⋮---- -insert_request_log(&conn, &err_entry).unwrap(); -⋮---- -let results = query_request_log(&conn, 10, 0, None, None, None, Some("5xx"), None).unwrap(); -⋮---- -assert_eq!(results[0].status_code, 500); -⋮---- -let results = query_request_log(&conn, 10, 0, None, None, None, Some("2xx"), None).unwrap(); -⋮---- -fn query_pagination() { -⋮---- -let mut entry = sample_entry(); -entry.request_id = format!("test-{i}"); -⋮---- -let page1 = query_request_log(&conn, 2, 0, None, None, None, None, None).unwrap(); -assert_eq!(page1.len(), 2); -⋮---- -let page2 = query_request_log(&conn, 2, 2, None, None, None, None, None).unwrap(); -assert_eq!(page2.len(), 2); -⋮---- -let page3 = query_request_log(&conn, 2, 4, None, None, None, None, None).unwrap(); -assert_eq!(page3.len(), 1); -⋮---- -fn get_request_by_id_found() { -⋮---- -let result = get_request_by_id(&conn, "test-123").unwrap(); -assert!(result.is_some()); -assert_eq!(result.unwrap().request_id, "test-123"); -⋮---- -fn get_request_by_id_not_found() { -⋮---- -let result = get_request_by_id(&conn, "nonexistent").unwrap(); -assert!(result.is_none()); -⋮---- -fn config_override_crud() { -⋮---- -// Set -set_config_override(&conn, "log_level", "debug").unwrap(); -let overrides = get_config_overrides(&conn).unwrap(); -assert_eq!(overrides.len(), 1); -assert_eq!(overrides[0].0, "log_level"); -assert_eq!(overrides[0].1, "debug"); -⋮---- -// Update (upsert) -set_config_override(&conn, "log_level", "trace").unwrap(); -⋮---- -assert_eq!(overrides[0].1, "trace"); -⋮---- -// Delete -let deleted = delete_config_override(&conn, "log_level").unwrap(); -assert!(deleted); -⋮---- -assert!(overrides.is_empty()); -⋮---- -// Delete non-existent -let deleted = delete_config_override(&conn, "nonexistent").unwrap(); -assert!(!deleted); -⋮---- -fn purge_old_logs_removes_old_entries() { -⋮---- -// Insert an old entry (timestamp in 2020). -let mut old = sample_entry(); -old.timestamp = "2020-01-01T00:00:00Z".into(); -insert_request_log(&conn, &old).unwrap(); -⋮---- -// Insert a recent entry. -⋮---- -let purged = purge_old_logs(&conn, 1).unwrap(); -assert_eq!(purged, 1); -⋮---- -let remaining = query_request_log(&conn, 10, 0, None, None, None, None, None).unwrap(); -assert_eq!(remaining.len(), 1); -assert_eq!(remaining[0].request_id, "test-123"); -⋮---- -fn epoch_to_iso8601_known_value() { -// 2026-03-22T00:00:00Z = 1774070400 (approximate) -let result = epoch_to_iso8601(0); -assert_eq!(result, "1970-01-01T00:00:00Z"); -⋮---- -fn epoch_to_iso8601_ms_formats_fractional_seconds() { -assert_eq!(epoch_to_iso8601_ms(500), "1970-01-01T00:00:00.500Z"); -assert_eq!(epoch_to_iso8601_ms(1000), "1970-01-01T00:00:01.000Z"); -assert_eq!(epoch_to_iso8601_ms(1001), "1970-01-01T00:00:01.001Z"); -let result = epoch_to_iso8601_ms(1774070400000); -assert!(result.ends_with(".000Z"), "got: {result}"); -⋮---- -fn init_db_idempotent() { -⋮---- -// Running again should not error. -⋮---- -fn insert_and_query_with_key_id_and_cost() { -⋮---- -entry.key_id = Some(42); -entry.cost_usd = Some(0.0075); -⋮---- -// Query without key_id filter returns all. -⋮---- -assert_eq!(results[0].key_id, Some(42)); -assert!((results[0].cost_usd.unwrap() - 0.0075).abs() < 1e-12); -⋮---- -// Query with matching key_id filter. -let results = query_request_log(&conn, 10, 0, None, None, None, None, Some(42)).unwrap(); -⋮---- -// Query with non-matching key_id filter. -let results = query_request_log(&conn, 10, 0, None, None, None, None, Some(99)).unwrap(); -assert!(results.is_empty()); -⋮---- -// get_request_by_id also returns the new fields. -let found = get_request_by_id(&conn, "test-123").unwrap().unwrap(); -assert_eq!(found.key_id, Some(42)); -assert!((found.cost_usd.unwrap() - 0.0075).abs() < 1e-12); -⋮---- -fn insert_without_attribution_fields() { -// Entries without key_id/cost_usd should still work (NULL columns). -⋮---- -assert_eq!(results[0].key_id, None); -assert_eq!(results[0].cost_usd, None); -⋮---- -fn audit_log_insert_and_query() { -⋮---- -action: "key_created".into(), -target_type: "virtual_key".into(), -target_id: Some("42".into()), -detail: Some("description=test key, prefix=sk-vk-abc".into()), -source_ip: Some("127.0.0.1".into()), -⋮---- -action: "key_revoked".into(), -⋮---- -insert_audit_entry(&conn, &entry1).unwrap(); -insert_audit_entry(&conn, &entry2).unwrap(); -⋮---- -let results = query_audit_log(&conn, 50, 0, None, None, None, None).unwrap(); -assert_eq!(results.len(), 2); -// Reverse chronological: most recent first. -assert_eq!(results[0].action, "key_revoked"); -assert_eq!(results[1].action, "key_created"); -assert!(results[0].id.unwrap() > results[1].id.unwrap()); -// Timestamps are filled in by the insert function. -assert!(results[0].timestamp.is_some()); -assert_eq!(results[1].target_id.as_deref(), Some("42")); -assert_eq!( -⋮---- -assert_eq!(results[1].source_ip.as_deref(), Some("127.0.0.1")); -⋮---- -fn audit_log_empty_returns_empty_vec() { -⋮---- -fn audit_log_pagination() { -⋮---- -insert_audit_entry( -⋮---- -action: format!("action_{i}"), -target_type: "test".into(), -⋮---- -let page1 = query_audit_log(&conn, 2, 0, None, None, None, None).unwrap(); -⋮---- -let page2 = query_audit_log(&conn, 2, 2, None, None, None, None).unwrap(); -⋮---- -let page3 = query_audit_log(&conn, 2, 4, None, None, None, None).unwrap(); -⋮---- -fn status_filter_parses_valid_inputs() { -assert!(StatusFilter::parse("200").is_some()); -assert!(StatusFilter::parse("2xx").is_some()); -assert!(StatusFilter::parse("4xx").is_some()); -assert!(StatusFilter::parse("5xx").is_some()); -assert!(StatusFilter::parse("404").is_some()); -⋮---- -fn status_filter_rejects_invalid_inputs() { -assert!(StatusFilter::parse("abc").is_none()); -assert!(StatusFilter::parse("2xx; DROP TABLE").is_none()); -assert!(StatusFilter::parse("").is_none()); -assert!(StatusFilter::parse("99999").is_none()); // overflows u16 -assert!(StatusFilter::parse("-1").is_none()); -⋮---- -fn status_filter_exact_code_query() { -⋮---- -insert_request_log(&conn, &sample_entry()).unwrap(); // status 200 -⋮---- -err_entry.request_id = "test-404".into(); -⋮---- -// Exact code filter should match only the 404 entry. -let results = query_request_log(&conn, 10, 0, None, None, None, Some("404"), None).unwrap(); -⋮---- -assert_eq!(results[0].status_code, 404); -⋮---- -fn status_filter_invalid_ignored() { -⋮---- -// Invalid filter should be silently ignored, returning all rows. -⋮---- -query_request_log(&conn, 10, 0, None, None, None, Some("garbage"), None).unwrap(); -⋮---- -// --- update_virtual_key tests --- -⋮---- -fn sample_key_params() -> InsertVirtualKeyParams<'static> { -⋮---- -description: Some("test key"), -⋮---- -rpm_limit: Some(100), -⋮---- -max_budget_usd: Some(10.0), -budget_duration: Some("monthly"), -⋮---- -fn update_virtual_key_returns_updated_row() { -⋮---- -let id = insert_virtual_key(&conn, &sample_key_params()).unwrap(); -⋮---- -description: Some("updated desc"), -⋮---- -rpm_limit: Some(200), -⋮---- -let row = update_virtual_key(&conn, id, ¶ms).unwrap(); -assert!(row.is_some()); -let row = row.unwrap(); -assert_eq!(row.description.as_deref(), Some("updated desc")); -assert_eq!(row.rpm_limit, Some(200)); -⋮---- -fn update_virtual_key_on_revoked_returns_none() { -⋮---- -revoke_virtual_key(&conn, id).unwrap(); -⋮---- -description: Some("should not apply"), -⋮---- -assert!(row.is_none()); -⋮---- -fn update_virtual_key_allowed_models_roundtrip() { -⋮---- -let models_json = serde_json::to_string(&["gpt-4o", "claude-*"]).unwrap(); -⋮---- -allowed_models: Some(models_json), -⋮---- -let row = update_virtual_key(&conn, id, ¶ms).unwrap().unwrap(); -// row.allowed_models is parsed from JSON into Vec. -⋮---- -fn update_virtual_key_budget_duration_resets_period() { -⋮---- -// Insert with a non-null period_spend_usd to verify reset. -⋮---- -params![id], -⋮---- -budget_duration: Some("daily"), -⋮---- -update_virtual_key(&conn, id, ¶ms).unwrap(); -⋮---- -// period_spend_usd should be reset to 0, period_start to NULL. -⋮---- -|r| Ok((r.get(0)?, r.get(1)?)), -⋮---- -assert_eq!(spend, 0.0); -assert!(start.is_none()); -⋮---- -// --- query_audit_log filter tests --- -⋮---- -fn insert_audit(conn: &Connection, action: &str, target_type: &str, ts: &str) { -⋮---- -params![ts, action, target_type], -⋮---- -fn audit_filter_by_action() { -⋮---- -insert_audit(&conn, "key_created", "virtual_key", "2099-01-01T00:00:00Z"); -insert_audit(&conn, "key_revoked", "virtual_key", "2099-01-02T00:00:00Z"); -⋮---- -query_audit_log(&conn, 10, 0, Some("key_created"), None, None, None).unwrap(); -⋮---- -assert_eq!(results[0].action, "key_created"); -⋮---- -fn audit_filter_by_target_type() { -⋮---- -insert_audit(&conn, "config_changed", "config", "2099-01-02T00:00:00Z"); -⋮---- -query_audit_log(&conn, 10, 0, None, Some("config"), None, None).unwrap(); -⋮---- -assert_eq!(results[0].target_type, "config"); -⋮---- -fn audit_filter_since_until() { -⋮---- -insert_audit(&conn, "key_revoked", "virtual_key", "2099-01-03T00:00:00Z"); -insert_audit(&conn, "key_updated", "virtual_key", "2099-01-05T00:00:00Z"); -⋮---- -// since + until window should return only the middle entry. -let results = query_audit_log( -⋮---- -Some("2099-01-02T00:00:00Z"), -Some("2099-01-04T00:00:00Z"), -⋮---- -fn count_requests_since_returns_zero_on_empty_log() { -⋮---- -let count = count_requests_since(&conn, 0).unwrap(); -⋮---- -fn count_requests_since_counts_recent_entries() { -⋮---- -// Insert a recent entry (sample_entry uses current time). -let recent = sample_entry(); -insert_request_log(&conn, &recent).unwrap(); -⋮---- -// Insert an old entry. -⋮---- -old.request_id = "old-req".to_string(); -old.timestamp = "2020-01-01T00:00:00Z".to_string(); -⋮---- -// Count since 2025-01-01 should include only the recent entry. -let since_2025: u64 = 1735689600; // 2025-01-01T00:00:00Z -let count = count_requests_since(&conn, since_2025).unwrap(); -assert_eq!(count, 1); - - - -// OpenAI Chat Completions input handler. -// -// Accepts POST /v1/chat/completions in OpenAI format, translates through -// the Anthropic pipeline, returns OpenAI-format responses. -⋮---- -use bytes::BytesMut; -use futures::StreamExt; -⋮---- -/// OpenAI-shaped error response body. -fn openai_error_response(message: &str, error_type: &str, status: StatusCode) -> Response { -⋮---- -(status, Json(body)).into_response() -⋮---- -/// Convert a BackendError into an OpenAI-shaped error response. -fn backend_error_to_openai_response(error: BackendError) -> Response { -if let Some((message, status)) = error.api_error_details() { -⋮---- -let http_status = StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); -return openai_error_response(message, error_type, http_status); -⋮---- -openai_error_response( -⋮---- -/// Handler for POST /v1/chat/completions (non-streaming and streaming). -pub(crate) async fn chat_completions( -⋮---- -let vk_ctx = vk_ctx.map(|axum::Extension(c)| c); -⋮---- -return openai_error_response( -&e.body_text(), -⋮---- -let permit = permit.map(|axum::Extension(p)| p); -⋮---- -.get("x-request-id") -.and_then(|v| v.to_str().ok()) -.unwrap_or("unknown") -.to_string(), -⋮---- -model_requested: body.model.clone(), -⋮---- -state.metrics.record_request(); -⋮---- -// Enforce model allowlist policy for virtual keys. -⋮---- -&format!("Model '{}' is not allowed for this API key.", body.model), -⋮---- -// Translate OpenAI request -> Anthropic request -⋮---- -let anthropic_req = match translate_openai_to_anthropic_request(&body, &mut warnings) { -⋮---- -&e.to_string(), -⋮---- -if anthropic_req.messages.is_empty() { -⋮---- -let is_streaming = body.stream == Some(true); -let original_model = body.model.clone(); -⋮---- -let mut response = chat_completions_stream( -⋮---- -response.headers_mut().insert( -⋮---- -// Non-streaming path: check cache before calling backend. -let body_value = serde_json::to_value(&body).unwrap_or_default(); -⋮---- -return openai_error_response(&msg, "invalid_request_error", StatusCode::BAD_REQUEST); -⋮---- -let bypass_cache = cache_ttl == Some(0); -⋮---- -Some(cache::cache_key_for_request( -⋮---- -// Check cache on non-bypass requests -⋮---- -if let Some(entry) = c.get(key).await { -⋮---- -.status(StatusCode::OK) -.header("content-type", "application/json") -.header("x-anyllm-cache", "hit") -.body(axum::body::Body::from(entry.response_body)) -.unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()); -inject_degradation_header(response.headers_mut(), &warnings); -⋮---- -// Resolve model routing (may switch to a different backend). -let (mapped_model, effective, deployment) = match state.resolve_model_and_state(&original_model) -⋮---- -d.record_start(); -⋮---- -// Non-streaming path -⋮---- -// Gemini/Vertex rejects standard JSON Schema keywords; sanitize tool schemas. -if matches!( -⋮---- -if let Some(tools) = openai_req.tools.take() { -openai_req.tools = Some( -⋮---- -.into_iter() -.map(|mut t| { -if let Some(params) = t.function.parameters.take() { -t.function.parameters = Some( -⋮---- -.collect(), -⋮---- -openai_req.model = mapped_model.clone(); -let mapped_model = openai_req.model.clone(); -⋮---- -match client.chat_completion(&openai_req).await { -⋮---- -d.record_finish(backend_start.elapsed().as_millis() as u64); -⋮---- -state.metrics.record_success(); -// Translate Anthropic response back to OpenAI format -⋮---- -translate_anthropic_to_openai_response(&anthropic_resp, &original_model); -⋮---- -log_request( -⋮---- -ctx.log_entry_with_attribution( -⋮---- -Some(mapped_model), -⋮---- -Some(( -⋮---- -Some(cost), -⋮---- -original_model.clone(), -⋮---- -let mut response = (StatusCode::OK, Json(oai_response)).into_response(); -rate_limits.inject_anthropic_response_headers(response.headers_mut()); -⋮---- -response.headers_mut().insert("x-anyllm-cache", cache_hv); -⋮---- -state.metrics.record_error(); -let status = e.status_code(); -⋮---- -Some(e.to_string()), -⋮---- -backend_error_to_openai_response(BackendError::from(e)) -⋮---- -responses_req.model = mapped_model.clone(); -let mapped_model = responses_req.model.clone(); -⋮---- -match client.responses(&responses_req).await { -⋮---- -BackendClient::Anthropic(_) | BackendClient::Bedrock(_) | BackendClient::GeminiNative(_) => openai_error_response( -⋮---- -/// Streaming handler for POST /v1/chat/completions with stream: true. -/// -/// Translates the Anthropic request to OpenAI, streams the backend response, -/// then uses ReverseStreamingTranslator to convert Anthropic SSE events back -/// to OpenAI ChatCompletionChunk SSE format. -async fn chat_completions_stream( -⋮---- -match state.resolve_model_and_state(&original_model) { -⋮---- -// Translate to OpenAI format for the backend -⋮---- -openai_req.stream = Some(true); -⋮---- -openai_req.stream_options = Some(openai::StreamOptions { -⋮---- -| BackendClient::OpenAIResponses(c) => c.clone(), -⋮---- -// Start the backend request -let response = match client.chat_completion_stream(&openai_req).await { -⋮---- -// Build the SSE response with OpenAI chunk format -⋮---- -let metrics = state.metrics.clone(); -let log_shared = state.shared.clone(); -let log_backend_name = state.backend_name.clone(); -let model_for_translator = original_model.clone(); -let cost_model = mapped_model.clone(); -⋮---- -metrics.record_stream_started(); -⋮---- -format!("chatcmpl-{}", uuid::Uuid::new_v4().as_simple()), -model_for_translator.clone(), -⋮---- -mapping::streaming_map::StreamingTranslator::new(model_for_translator.clone()); -⋮---- -let mut byte_stream = resp.bytes_stream(); -⋮---- -while let Some(chunk_result) = byte_stream.next().await { -⋮---- -metrics.record_error(); -metrics.record_stream_failed(); -⋮---- -buffer.extend_from_slice(&bytes); -⋮---- -if buffer.len() > MAX_SSE_BUFFER_SIZE { -⋮---- -while let Some((pos, delim_len)) = find_double_newline(&buffer, search_from) { -⋮---- -for line in frame_str.lines() { -let line = line.trim(); -if let Some(json_str) = line.strip_prefix("data: ") { -⋮---- -// Emit [DONE] for OpenAI clients -let _ = tx.send(Ok("data: [DONE]\n\n".to_string())).await; -⋮---- -// Parse OpenAI chunk, translate to Anthropic events, -// then reverse-translate to OpenAI chunks -⋮---- -stream_translator.process_chunk(&chunk); -⋮---- -let oai_chunks = translator.process_event(event); -⋮---- -let sse_line = format!("data: {}\n\n", json); -if tx.send(Ok(sse_line)).await.is_err() { -metrics.record_stream_client_disconnected(); -return; // Client disconnected -⋮---- -let _ = buffer.split_to(pos + delim_len); -⋮---- -search_from = buffer.len().saturating_sub(3); -⋮---- -// Emit any remaining finish events -let finish_events = stream_translator.finish(); -⋮---- -let _ = tx.send(Ok(format!("data: {}\n\n", json))).await; -⋮---- -if !translator.is_done() { -⋮---- -// Extract token counts from the stream translator for cost tracking. -let usage = stream_translator.usage(); -let tokens = usage.map(|u| (u.input_tokens as u64, u.output_tokens as u64)); -⋮---- -Some(crate::cost::record_cost( -⋮---- -metrics.record_success(); -metrics.record_stream_completed(); -⋮---- -// Build the SSE response using raw text/event-stream -⋮---- -.header("content-type", "text/event-stream") -.header("cache-control", "no-cache") -.header("connection", "keep-alive") -.body(body) -⋮---- -e.status_code(), - - - -// Auth, logging, and request size limit middleware -⋮---- -use anyllm_translate::anthropic; -use anyllm_translate::mapping::errors_map::create_anthropic_error; -⋮---- -use dashmap::DashMap; -⋮---- -use subtle::ConstantTimeEq; -⋮---- -/// Per-installation HMAC secret for virtual key hashing. -/// Set once during startup alongside the virtual keys DashMap. -⋮---- -/// Initialize the global HMAC secret. Called once from main. -pub fn set_hmac_secret(secret: Arc>) { -let _ = HMAC_SECRET.set(secret); -⋮---- -/// Build a 429 rate-limit error response with retry-after header. -fn rate_limit_response(message: &str, retry_after: u64) -> Response { -let err = create_anthropic_error( -⋮---- -message.to_string(), -⋮---- -let mut resp = (StatusCode::TOO_MANY_REQUESTS, Json(err)).into_response(); -if let Ok(val) = axum::http::HeaderValue::from_str(&retry_after.to_string()) { -resp.headers_mut().insert("retry-after", val); -⋮---- -/// Context passed from auth middleware to handlers for post-response TPM and cost recording. -/// Inserted into request extensions when a virtual key is used. -⋮---- -pub struct VirtualKeyContext { -/// Database row ID for the virtual key (used for cost accumulation). -⋮---- -/// Optional model allowlist from the virtual key policy. -⋮---- -/// Set to the new period_start ISO string when a budget period was reset -/// during this request's auth check. Signals `record_cost` to call -/// `reset_period_spend` before `accumulate_spend` so SQLite stays in sync. -⋮---- -/// Controls which authentication paths are active. -⋮---- -pub enum AuthMode { -/// Only accept static and virtual API keys. JWTs are not checked. -⋮---- -/// Only accept JWT tokens. Static and virtual keys are rejected. -⋮---- -/// Try JWT first, fall through to keys on failure (default). -⋮---- -impl AuthMode { -/// Parse an AUTH_MODE string. Accepts both new names (oidc, oidc-only, keys, -/// keys-only, both) and legacy names (jwt_only, keys_only, jwt_or_keys). -pub fn from_env_str(s: &str) -> Self { -match s.to_lowercase().as_str() { -⋮---- -/// Read AUTH_MODE from the environment. Defaults to Both for backward compatibility. -pub fn from_env() -> Self { -⋮---- -.map(|v| Self::from_env_str(&v)) -.unwrap_or(Self::Both) -⋮---- -pub fn allows_key_auth(&self) -> bool { -matches!(self, AuthMode::KeysOnly | AuthMode::Both) -⋮---- -pub fn allows_oidc(&self) -> bool { -matches!(self, AuthMode::OidcOnly | AuthMode::Both) -⋮---- -/// Global reference to the virtual keys DashMap, set once during startup. -/// Checked during auth after the static ALLOWED_KEY_HASHES check. -⋮---- -/// Global OIDC config, set once during startup when OIDC_ISSUER_URL is configured. -⋮---- -/// Initialize the global OIDC config. Called once from main when OIDC is enabled. -pub fn set_oidc_config(config: Arc) { -let _ = OIDC_CONFIG.set(config); -⋮---- -/// Initialize the global virtual keys reference. Called once from main. -pub fn set_virtual_keys(keys: Arc>) { -let _ = VIRTUAL_KEYS.set(keys); -⋮---- -/// Pre-hashed allowed API keys for constant-time comparison without -/// leaking key length via timing. Each key is SHA-256 hashed at startup. -⋮---- -.unwrap_or_default() -.split(',') -.map(|s| s.trim().to_string()) -.filter(|s| !s.is_empty()) -.collect(); -if keys.is_empty() { -⋮---- -.map(|v| v == "true" || v == "1") -.unwrap_or(false); -⋮---- -keys.iter() -.map(|k| Sha256::digest(k.as_bytes()).into()) -.collect() -⋮---- -/// Whether open-relay mode is explicitly enabled via PROXY_OPEN_RELAY=true. -⋮---- -ALLOWED_KEY_HASHES.is_empty() -⋮---- -.unwrap_or(false) -⋮---- -/// Validate that the request carries a valid API key. -/// If `PROXY_API_KEYS` is set, the caller's key must be in the allowlist. -/// Otherwise, any non-empty key is accepted (backward-compatible open mode). -/// -/// Anthropic: -pub async fn validate_auth( -⋮---- -.get("x-api-key") -.and_then(|v| v.to_str().ok()) -.map(|s| s.to_string()); -⋮---- -.get("authorization") -⋮---- -.and_then(|v| { -let lower = v.to_lowercase(); -if lower.starts_with("bearer ") { -Some(v[7..].trim().to_string()) -⋮---- -let credential = api_key.or(bearer_token); -⋮---- -Some(c) if !c.is_empty() => c, -⋮---- -"Missing authentication. Provide x-api-key or Authorization header.".to_string(), -⋮---- -return Err((StatusCode::UNAUTHORIZED, Json(err)).into_response()); -⋮---- -// Check 0: OIDC/JWT validation (if configured and mode allows it). -⋮---- -if auth_mode.allows_oidc() { -if let Some(oidc) = OIDC_CONFIG.get() { -⋮---- -match oidc.validate_token(&credential) { -⋮---- -request.extensions_mut().insert(claims); -return Ok(next.run(request).await); -⋮---- -"JWT validation failed.".to_string(), -⋮---- -"JWT required but credential is not a valid JWT format.".to_string(), -⋮---- -"Server misconfigured: JWT auth required but OIDC not configured.".to_string(), -⋮---- -// Compare SHA-256 hashes of the credential against pre-hashed allowed keys. -// Hashing eliminates the timing side-channel on key length: all comparisons -// operate on fixed-size 32-byte digests regardless of original key length. -let credential_hash: [u8; 32] = Sha256::digest(credential.as_bytes()).into(); -⋮---- -// Check 1: static env-var keys (constant-time comparison) -⋮---- -.iter() -.any(|h| bool::from(h.ct_eq(&credential_hash))); -⋮---- -// Check 2: virtual keys from DashMap (with per-key rate limiting, budget, RBAC) -// Dual-mode lookup: try HMAC-SHA256 hash first (new keys), fall back to legacy SHA-256 (old keys). -if let Some(map) = VIRTUAL_KEYS.get() { -let hmac_hash: Option<[u8; 32]> = HMAC_SECRET.get().and_then(|secret| { -⋮---- -.and_then(|h| map.get_mut(&h)) -.or_else(|| map.get_mut(&credential_hash)); -⋮---- -// Reject expired virtual keys at auth time (lazy eviction). -⋮---- -let now_secs = (now_ms() / 1000) as i64; -⋮---- -drop(meta); -// Remove expired key from cache so future lookups skip it. -⋮---- -map.remove(&h); -⋮---- -map.remove(&credential_hash); -⋮---- -return Err((StatusCode::UNAUTHORIZED, Json(err_body)).into_response()); -⋮---- -// RBAC: developer keys cannot access admin endpoints -⋮---- -let path = request.uri().path(); -if path.starts_with("/admin/") || path.starts_with("/admin") { -⋮---- -return Err((StatusCode::FORBIDDEN, Json(err_body)).into_response()); -⋮---- -let now_ms = now_ms(); -⋮---- -// Enforce RPM limit if configured -⋮---- -credential_hash.iter().map(|b| format!("{b:02x}")).collect(); -⋮---- -redis_limiter.check_rpm(&hash_hex, rpm_limit, now_ms).await -⋮---- -return Err(rate_limit_response( -⋮---- -if let Err(retry_after) = meta.rate_state.check_rpm(rpm_limit, now_ms) { -⋮---- -// Enforce TPM limit pre-check -⋮---- -redis_limiter.check_tpm(&hash_hex, tpm_limit, now_ms).await -⋮---- -if let Err(retry_after) = meta.rate_state.check_tpm(tpm_limit, now_ms) { -⋮---- -// Budget enforcement: lazy period reset then check -⋮---- -if meta.max_budget_usd.is_some() { -let did_reset = check_and_reset_period(&mut meta); -⋮---- -period_reset = meta.period_start.clone(); -⋮---- -let reset_at = period_reset_at(&meta); -⋮---- -return Err((StatusCode::TOO_MANY_REQUESTS, Json(err_body)).into_response()); -⋮---- -// Always insert context for post-response TPM recording and cost tracking. -request.extensions_mut().insert(VirtualKeyContext { -⋮---- -rate_state: meta.rate_state.clone(), -allowed_models: meta.allowed_models.clone(), -⋮---- -// Check 3: open-relay mode (any non-empty key accepted) -⋮---- -// No match found: reject -let message = if ALLOWED_KEY_HASHES.is_empty() { -⋮---- -Err((StatusCode::UNAUTHORIZED, Json(err)).into_response()) -⋮---- -/// Attach a request ID to the request and echo it on the response. -/// Uses the incoming x-request-id if present, otherwise generates a UUID v4. -⋮---- -/// Anthropic: -pub async fn add_request_id(mut request: Request, next: Next) -> Response { -⋮---- -.headers() -.get("x-request-id") -⋮---- -.map(|s| s.to_string()) -.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); -⋮---- -// Replace invalid request IDs with UUIDs to prevent header injection. -// Client-provided IDs may contain characters illegal in HTTP headers. -let header_value: axum::http::HeaderValue = request_id.parse().unwrap_or_else(|_| { -⋮---- -.to_string() -.parse() -.expect("UUID is always a valid header value") -⋮---- -.headers_mut() -.insert("x-request-id", header_value.clone()); -⋮---- -let mut response = next.run(request).await; -response.headers_mut().insert("x-request-id", header_value); -⋮---- -/// Log Anthropic-specific headers without rejecting requests that lack them. -/// Claude Code CLI and other Anthropic SDK clients send these headers. -pub async fn log_anthropic_headers(request: Request, next: Next) -> Response { -⋮---- -.get("anthropic-version") -⋮---- -.get("anthropic-beta") -⋮---- -// Claude Code v2.1.86+ sends this for proxy-side session routing/aggregation. -⋮---- -.get("x-claude-code-session-id") -⋮---- -next.run(request).await -⋮---- -/// Maximum request body size (32 MB, matching Anthropic's Messages endpoint limit). -⋮---- -/// Maximum concurrent requests to prevent self-DOS under 429 incidents. -⋮---- -// ---- IP allowlisting ---- -⋮---- -/// Parsed CIDR allowlist from IP_ALLOWLIST env var. None means allow all. -⋮---- -std::env::var("IP_ALLOWLIST").ok().map(|v| { -v.split(',') -.map(|s| s.trim()) -⋮---- -.map(|s| { -// Accept bare IPs (e.g., "127.0.0.1") by appending /32 or /128. -if !s.contains('/') { -⋮---- -.unwrap_or_else(|e| panic!("invalid IP_ALLOWLIST entry '{s}': {e}")); -⋮---- -.unwrap_or_else(|e| panic!("invalid IP_ALLOWLIST CIDR '{s}': {e}")) -⋮---- -/// Whether to trust X-Forwarded-For for IP allowlisting (production behind reverse proxy). -⋮---- -/// Check if an IP address is allowed by the configured allowlist. -/// Returns true if no allowlist is set (open access). -pub fn is_ip_allowed(ip: std::net::IpAddr) -> bool { -match IP_ALLOWLIST.as_ref() { -⋮---- -Some(networks) => networks.iter().any(|net| net.contains(ip)), -⋮---- -/// Returns true if the IP allowlist is configured (IP_ALLOWLIST env var is set). -pub fn ip_allowlist_active() -> bool { -IP_ALLOWLIST.is_some() -⋮---- -/// Middleware that rejects requests from IPs not in the allowlist. -/// Applied before auth so blocked IPs never reach authentication. -pub async fn check_ip_allowlist(request: Request, next: Next) -> Result { -// Extract client IP from X-Forwarded-For (if trusted) or connection info. -⋮---- -// Take the *rightmost* IP: a trusted reverse proxy appends the real client IP. -// The leftmost value is attacker-controlled and must not be trusted. -⋮---- -.get("x-forwarded-for") -⋮---- -.and_then(|s| { -s.rsplit(',') -.map(|p| p.trim()) -.find(|p| !p.is_empty()) -⋮---- -.and_then(|s| s.parse::().ok()) -⋮---- -// Fall back to ConnectInfo if available. -let client_ip = client_ip.or_else(|| { -⋮---- -.extensions() -⋮---- -.map(|ci| ci.0.ip()) -⋮---- -// If we have no IP at all (unlikely), deny by default when allowlist is active. -⋮---- -"IP address could not be determined".to_string(), -⋮---- -return Err((StatusCode::FORBIDDEN, Json(err)).into_response()); -⋮---- -if !is_ip_allowed(ip) { -⋮---- -"IP address not in allowlist".to_string(), -⋮---- -Ok(next.run(request).await) -⋮---- -mod ip_tests { -⋮---- -fn is_ip_allowed_no_allowlist() { -// When IP_ALLOWLIST is not set, all IPs are allowed. -// We cannot test this directly since LazyLock is static, but the function -// logic is: None => true. -assert!(is_ip_allowed("127.0.0.1".parse().unwrap()) || true); -⋮---- -fn xff_rightmost_prevents_spoofing() { -// Attacker sends X-Forwarded-For: 127.0.0.1; trusted proxy appends real IP. -// Must resolve to rightmost value (203.0.113.5), not the attacker-controlled leftmost. -⋮---- -.last() -.and_then(|s| s.parse().ok()) -.unwrap(); -assert_eq!(resolved, "203.0.113.5".parse::().unwrap()); -⋮---- -fn xff_single_ip_resolves() { -⋮---- -assert_eq!(resolved, "10.0.1.5".parse::().unwrap()); -⋮---- -mod auth_mode_tests { -⋮---- -fn parse_auth_mode_new_names() { -assert_eq!(AuthMode::from_env_str("oidc"), AuthMode::OidcOnly); -assert_eq!(AuthMode::from_env_str("oidc-only"), AuthMode::OidcOnly); -assert_eq!(AuthMode::from_env_str("oidc_only"), AuthMode::OidcOnly); -assert_eq!(AuthMode::from_env_str("keys"), AuthMode::KeysOnly); -assert_eq!(AuthMode::from_env_str("keys-only"), AuthMode::KeysOnly); -assert_eq!(AuthMode::from_env_str("keys_only"), AuthMode::KeysOnly); -assert_eq!(AuthMode::from_env_str("both"), AuthMode::Both); -⋮---- -fn parse_auth_mode_legacy_names() { -assert_eq!(AuthMode::from_env_str("jwt_only"), AuthMode::OidcOnly); -assert_eq!(AuthMode::from_env_str("jwt_or_keys"), AuthMode::Both); -assert_eq!(AuthMode::from_env_str("JWT_ONLY"), AuthMode::OidcOnly); -⋮---- -fn parse_auth_mode_unknown_defaults_to_both() { -assert_eq!(AuthMode::from_env_str("unknown"), AuthMode::Both); -assert_eq!(AuthMode::from_env_str(""), AuthMode::Both); -⋮---- -fn auth_mode_oidc_only() { -assert!(AuthMode::OidcOnly.allows_oidc()); -assert!(!AuthMode::OidcOnly.allows_key_auth()); -⋮---- -fn auth_mode_keys_only() { -assert!(AuthMode::KeysOnly.allows_key_auth()); -assert!(!AuthMode::KeysOnly.allows_oidc()); -⋮---- -fn auth_mode_both_allows_all() { -assert!(AuthMode::Both.allows_oidc()); -assert!(AuthMode::Both.allows_key_auth()); -⋮---- -fn auth_mode_from_env_defaults_to_both() { -// When AUTH_MODE is not set (or set to something unrecognized), -// from_env() returns Both for backward compatibility. -// Note: cannot safely manipulate env vars in parallel tests, -// so we test via from_env_str which from_env delegates to. -⋮---- -assert_eq!(mode, AuthMode::Both); - - - -use crate::metrics::Metrics; -⋮---- -use std::collections::HashMap; -⋮---- -use tokio::sync::Semaphore; -⋮---- -use crate::batch::anthropic_batch; -use super::passthrough::anthropic_passthrough; -use super::streaming::messages_stream; -use super::token_counting::count_tokens; -⋮---- -/// Custom JSON extractor that returns Anthropic-shaped error responses on -/// parse failure. Axum's built-in Json returns its own error format, which -/// would break clients expecting Anthropic error shapes. -pub(crate) struct AnthropicJson(pub T); -⋮---- -type Rejection = Response; -⋮---- -async fn from_request(req: axum::extract::Request, state: &S) -> Result { -⋮---- -Ok(Json(value)) => Ok(AnthropicJson(value)), -⋮---- -rejection.body_text(), -⋮---- -Err((StatusCode::BAD_REQUEST, Json(err)).into_response()) -⋮---- -/// Result of resolving a model name through the model router. -pub(crate) enum ResolvedModel { -/// Routed via model_list to a specific backend and actual model name. -⋮---- -/// The deployment Arc for recording in-flight/latency stats. -⋮---- -/// Model is known but all deployments are at their RPM limit. -⋮---- -/// No model router, or model not in router. Used legacy ModelMapping. -⋮---- -/// Per-backend state shared across request handlers. -/// -/// In single-backend mode, one `AppState` serves all routes. In multi-backend mode, -/// each backend gets its own `AppState` mounted under a prefix path (e.g., `/openai/v1/messages`). -⋮---- -pub struct AppState { -⋮---- -/// Runtime config (model mappings, log_bodies) read on every request. -/// Shared with admin server so config changes take effect immediately. -⋮---- -/// Shared admin state for request logging and live updates. None in tests. -⋮---- -/// Backend name for logging purposes. -⋮---- -/// Concurrency limiter. Uses try_acquire (fail-fast) instead of queueing -/// to prevent cascading latency under load. Requests exceeding the limit -/// get 429 immediately, matching Anthropic's rate limiting behavior. -⋮---- -/// Strip `stream_options` from streaming requests for local LLM compat. -⋮---- -/// Optional response cache for non-streaming requests. -⋮---- -/// Model-level router for LiteLLM model_list configs. None for TOML/env configs. -/// Wrapped in RwLock for dynamic model management via admin API. -⋮---- -/// All backend states, for cross-backend model routing. None unless model_router is set. -⋮---- -impl AppState { -/// Map a model name through the current runtime config for this backend. -pub(crate) fn map_model(&self, model: &str) -> String { -⋮---- -.read() -.unwrap_or_else(|e| e.into_inner()); -if let Some(mapping) = config.model_mappings.get(&self.backend_name) { -mapping.map_model(model) -⋮---- -model.to_string() -⋮---- -/// Resolve a model name through the model router (if set) or fall back to ModelMapping. -pub(crate) fn resolve_model(&self, model: &str) -> ResolvedModel { -⋮---- -let router = router_lock.read().unwrap_or_else(|e| e.into_inner()); -if let Some(routed) = router.route(model) { -⋮---- -backend_name: routed.backend_name.to_string(), -model: routed.actual_model.to_string(), -deployment: routed.deployment.clone(), -⋮---- -if router.has_model(model) { -⋮---- -ResolvedModel::Legacy(self.map_model(model)) -⋮---- -/// Resolve model and return (mapped_model, effective AppState, optional deployment). -/// If the model routes to a different backend, the returned state is cloned from -/// all_backends. Returns Err with a 429 response if all deployments are at limit. -/// The deployment Arc is returned so handlers can call record_start/record_finish. -⋮---- -pub(crate) fn resolve_model_and_state( -⋮---- -match self.resolve_model(model) { -⋮---- -.as_ref() -.and_then(|m| m.get(&backend_name)) -.cloned() -.unwrap_or_else(|| self.clone()); -Ok((mapped, effective, Some(deployment))) -⋮---- -"all deployments for this model are at their RPM limit".to_string(), -⋮---- -Err((StatusCode::TOO_MANY_REQUESTS, Json(err)).into_response()) -⋮---- -ResolvedModel::Legacy(mapped) => Ok((mapped, self.clone(), None)), -⋮---- -/// Whether request/response body logging is enabled. -pub(crate) fn log_bodies(&self) -> bool { -⋮---- -.unwrap_or_else(|e| e.into_inner()) -⋮---- -/// Global state for the multi-backend metrics endpoint. -⋮---- -struct GlobalState { -⋮---- -/// Build the axum router from a legacy single-backend Config. -pub fn app(config: Config) -> Router { -⋮---- -app_multi(multi) -⋮---- -/// Build the axum router from multi-backend configuration. -/// Creates nested sub-routers for each configured backend. -pub fn app_multi(config: MultiConfig) -> Router { -app_multi_with_shared(config, None, None) -⋮---- -/// Build the axum router with optional shared admin state and model router. -pub fn app_multi_with_shared( -⋮---- -// When no shared state (tests), build a standalone runtime config from the multi config. -⋮---- -s.runtime_config.clone() -⋮---- -model_mappings.insert(name.clone(), bc.model_mapping.clone()); -⋮---- -log_level: "info".to_string(), -⋮---- -// Build a shared cache instance for all backends. -⋮---- -// Build per-backend sub-routers. Keep a map of AppState so the default -// backend can reuse the same state (same semaphore, same reqwest client). -⋮---- -backend_metrics.insert(name.clone(), metrics.clone()); -⋮---- -runtime_config: runtime_config.clone(), -shared: shared.clone(), -backend_name: name.clone(), -⋮---- -cache: Some(response_cache.clone()), -model_router: model_router.clone(), -// all_backends is set after the loop (needs all states built first). -⋮---- -let sub = backend_router(state.clone(), mode); -backend_states.insert(name.clone(), (state, mode)); -⋮---- -// Nest under /{name}/ -router = router.nest(&format!("/{name}"), sub); -⋮---- -// If a model router is active, build the all_backends map so handlers can -// dispatch to a different backend when the router says so. -if model_router.is_some() { -⋮---- -.iter() -.map(|(k, (s, _))| (k.clone(), s.clone())) -.collect(), -⋮---- -// Patch each AppState in the map. Since we already built sub-routers with -// the old states (all_backends=None), this only affects the default backend -// and cross-backend routing lookups via effective_state(). The sub-router -// states don't need all_backends because they are only reached by prefix. -for (_, (state, _)) in backend_states.iter_mut() { -state.all_backends = Some(all_map.clone()); -⋮---- -// Default backend: also serve at un-prefixed /v1/messages for backward compat. -// Reuses the same AppState (shared semaphore, connection pool) as the named route. -if let Some((default_state, mode)) = backend_states.get(&config.default_backend) { -let default_sub = backend_router(default_state.clone(), *mode); -router = router.merge(default_sub); -⋮---- -// Metrics requires auth (prevents unauthenticated reconnaissance of -// backend names and traffic patterns). -⋮---- -.route( -⋮---- -get(|State(gs): State| async move { -⋮---- -for (name, m) in gs.backend_metrics.iter() { -let snap = m.snapshot(); -⋮---- -backends.insert( -name.clone(), -serde_json::to_value(&snap).unwrap_or_default(), -⋮---- -Json(serde_json::json!({ -⋮---- -.layer(axum::middleware::from_fn(super::middleware::validate_auth)); -⋮---- -// Health is public (no auth required). -⋮---- -.route("/health", get(health)) -.merge(metrics_route) -.merge(router) -.fallback(fallback_not_found) -.layer(axum::middleware::from_fn(super::middleware::add_request_id)); -⋮---- -// Apply IP allowlist middleware before auth if IP_ALLOWLIST is configured. -⋮---- -final_router = final_router.layer(axum::middleware::from_fn( -⋮---- -final_router.with_state(global_state) -⋮---- -/// Return Anthropic-shaped 404 for any unmatched route (PRD US-004). -async fn fallback_not_found() -> Response { -⋮---- -"Not found".to_string(), -⋮---- -(StatusCode::NOT_FOUND, Json(err)).into_response() -⋮---- -/// Which handler mode a backend uses. -⋮---- -enum HandlerMode { -/// Anthropic passthrough (no translation, forwards raw bytes). -⋮---- -/// Bedrock (SigV4 signing, event stream decoding, Anthropic format). -⋮---- -/// Gemini native generateContent (no OpenAI translation layer). -⋮---- -/// Translation (Anthropic -> OpenAI -> backend -> OpenAI -> Anthropic). -⋮---- -/// Build the sub-router for a single backend. -fn backend_router(state: AppState, mode: HandlerMode) -> Router { -// Routes common to all backend modes. -⋮---- -.route("/v1/models", get(models)) -.route("/v1/files", post(crate::batch::routes::upload_file)) -⋮---- -post(crate::batch::routes::create_batch).get(crate::batch::routes::list_batches), -⋮---- -get(crate::batch::routes::get_batch), -⋮---- -HandlerMode::Anthropic => common_routes.route("/v1/messages", post(anthropic_passthrough)), -HandlerMode::Bedrock => common_routes.route( -⋮---- -post(super::bedrock_passthrough::bedrock_passthrough), -⋮---- -HandlerMode::GeminiNative => common_routes.route( -⋮---- -post(super::gemini_native::gemini_native_handler), -⋮---- -.route("/v1/messages", post(messages)) -⋮---- -post(super::chat_completions::chat_completions), -⋮---- -.route("/v1/messages/count_tokens", post(count_tokens)) -⋮---- -post(anthropic_batch::create_anthropic_batch), -⋮---- -get(anthropic_batch::get_anthropic_batch), -⋮---- -get(anthropic_batch::get_anthropic_batch_results), -⋮---- -.route("/v1/embeddings", post(embeddings)) -⋮---- -post(super::audio::audio_transcriptions), -⋮---- -.route("/v1/audio/speech", post(super::audio::audio_speech)) -⋮---- -post(super::images::image_generations), -⋮---- -.route("/v1/rerank", post(rerank)) -.route("/v1/completions", post(completions)), -⋮---- -.layer(axum::middleware::from_fn(super::middleware::validate_auth)) -.layer(axum::middleware::from_fn( -⋮---- -.layer(DefaultBodyLimit::max(super::middleware::MAX_BODY_SIZE)) -.layer(axum::middleware::from_fn_with_state( -state.clone(), -⋮---- -.with_state(state) -⋮---- -/// Reject requests when the concurrency limit is reached (429), rather than -/// queueing them like Tower's ConcurrencyLimitLayer would. -/// The permit is stored in request extensions so streaming handlers can hold -/// it until the stream completes (not just until headers are sent). -async fn enforce_concurrency( -⋮---- -let Ok(permit) = state.concurrency.clone().try_acquire_owned() else { -⋮---- -"Proxy concurrency limit reached".to_string(), -⋮---- -return (StatusCode::TOO_MANY_REQUESTS, Json(err)).into_response(); -⋮---- -.extensions_mut() -.insert(ConcurrencyPermit(Arc::new(permit))); -next.run(request).await -⋮---- -/// Wrapper so OwnedSemaphorePermit can be stored in request extensions. -/// The field is never read directly; it exists as an RAII guard to hold -/// the permit until the struct is dropped. -⋮---- -pub(crate) struct ConcurrencyPermit( -⋮---- -/// Static Claude model entries, merged with model_list models at runtime. -⋮---- -vec![ -// Claude 4.x -⋮---- -// Claude 3.7 -⋮---- -// Claude 3.5 -⋮---- -// Claude 3 -⋮---- -/// GET /v1/models -- returns static Claude models merged with model_list entries. -async fn models(State(state): State) -> Json { -let mut data: Vec = STATIC_CLAUDE_MODELS.clone(); -⋮---- -// Merge models from the model router (LiteLLM model_list config). -⋮---- -.filter_map(|m| m["id"].as_str().map(|s| s.to_string())) -.collect(); -for model_name in router.known_models() { -if !static_ids.contains(model_name) { -data.push(serde_json::json!({ -⋮---- -async fn health() -> impl IntoResponse { -⋮---- -/// Convert a BackendError into an Anthropic error Response. -pub(super) fn backend_error_to_response(error: BackendError) -> Response { -if let Some((message, status)) = error.api_error_details() { -⋮---- -.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); -return (http_status, Json(anthropic_err)).into_response(); -⋮---- -// Transport or deserialization error -- log details server-side only, -// return a generic message to avoid leaking infrastructure details. -⋮---- -"An internal error occurred while communicating with the upstream service.".to_string(), -⋮---- -(StatusCode::INTERNAL_SERVER_ERROR, Json(err)).into_response() -⋮---- -/// Return the appropriate `x-anyllm-cache` header value. -pub(crate) fn cache_header_value(bypass: bool) -> axum::http::HeaderValue { -⋮---- -/// Store a serializable response in the cache if caching is enabled. -pub(crate) async fn try_cache_response( -⋮---- -if let Ok(resp_body) = serde_json::to_vec(response).map(bytes::Bytes::from) { -let ttl = cache_ttl.unwrap_or(c.default_ttl_secs); -c.put( -⋮---- -/// Inject degradation warnings as `x-anyllm-degradation` header if any features were dropped. -pub(crate) fn inject_degradation_header( -⋮---- -if let Some(val) = warnings.as_header_value() { -⋮---- -headers.insert("x-anyllm-degradation", hv); -⋮---- -/// Shared passthrough logic: extract content-type, forward to backend, relay response. -async fn passthrough_to_backend( -⋮---- -.get(axum::http::header::CONTENT_TYPE) -.and_then(|v| v.to_str().ok()) -.unwrap_or("application/json"); -⋮---- -.raw_passthrough(path, body, content_type) -⋮---- -let mut response = (status, resp_body).into_response(); -⋮---- -response.headers_mut().insert(k, v.clone()); -⋮---- -Err(e) => backend_error_to_response(e), -⋮---- -async fn embeddings( -⋮---- -passthrough_to_backend(&state, &headers, body, "/v1/embeddings").await -⋮---- -async fn rerank( -⋮---- -passthrough_to_backend(&state, &headers, body, "/v1/rerank").await -⋮---- -async fn completions( -⋮---- -passthrough_to_backend(&state, &headers, body, "/v1/completions").await -⋮---- -async fn messages( -⋮---- -// Hold concurrency permit for streaming: passed to the spawned task so -// the permit lives until the stream completes, not just until headers are sent. -let permit = permit.map(|axum::Extension(p)| p); -let vk_ctx = vk_ctx.map(|axum::Extension(c)| c); -⋮---- -.get("x-request-id") -⋮---- -.unwrap_or("unknown") -.to_string(), -⋮---- -model_requested: body.model.clone(), -⋮---- -state.metrics.record_request(); -⋮---- -// Enforce model allowlist policy for virtual keys. -⋮---- -format!("Model '{}' is not allowed for this API key.", body.model), -⋮---- -return (StatusCode::FORBIDDEN, Json(err)).into_response(); -⋮---- -if state.log_bodies() { -⋮---- -let warnings = compute_request_warnings(&body); -⋮---- -let is_streaming = body.stream == Some(true); -⋮---- -let (mapped_model, effective, deployment) = match state.resolve_model_and_state(&body.model) -⋮---- -d.record_start(); -⋮---- -// Logging deferred until stream completes (inside messages_stream tasks). -⋮---- -match messages_stream(effective, body, ctx, mapped_model, permit, vk_ctx.clone()).await { -⋮---- -// For streaming, record_finish is approximate (headers sent, not stream end). -⋮---- -d.record_finish(stream_start.elapsed().as_millis() as u64); -⋮---- -let mut response = sse.into_response(); -rate_limits.inject_anthropic_response_headers(response.headers_mut()); -inject_degradation_header(response.headers_mut(), &warnings); -response.headers_mut().insert( -⋮---- -// Pre-stream backend error: return proper HTTP status instead of 200 OK -return backend_error_to_response(e); -⋮---- -// Non-streaming: check cache before calling backend. -let body_value = serde_json::to_value(&body).unwrap_or_default(); -⋮---- -return (StatusCode::BAD_REQUEST, Json(err)).into_response(); -⋮---- -let bypass_cache = cache_ttl == Some(0); -⋮---- -Some(cache::cache_key_for_request( -⋮---- -// Check cache on non-bypass requests -⋮---- -if let Some(entry) = c.get(key).await { -⋮---- -.status(StatusCode::OK) -.header("content-type", "application/json") -.header("x-anyllm-cache", "hit") -.body(axum::body::Body::from(entry.response_body)) -.unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()); -⋮---- -// Resolve model routing (may switch to a different backend). -let (mapped_model, effective, deployment) = match state.resolve_model_and_state(&body.model) { -⋮---- -inject_gemini_thinking(&body, &effective.backend, &mut openai_req); -// Gemini/Vertex rejects standard JSON Schema keywords; sanitize tool schemas. -if matches!( -⋮---- -if let Some(tools) = openai_req.tools.take() { -openai_req.tools = Some( -⋮---- -.into_iter() -.map(|mut t| { -if let Some(params) = t.function.parameters.take() { -t.function.parameters = Some( -⋮---- -openai_req.model = mapped_model.clone(); -let mapped_model = openai_req.model.clone(); -let original_model = body.model.clone(); -⋮---- -match client.chat_completion(&openai_req).await { -⋮---- -d.record_finish(backend_start.elapsed().as_millis() as u64); -⋮---- -state.metrics.record_success(); -⋮---- -record_vk_tpm(&vk_ctx, anthropic_resp.usage.output_tokens); -⋮---- -log_request( -⋮---- -ctx.log_entry_with_attribution( -⋮---- -Some(mapped_model), -⋮---- -Some(( -⋮---- -Some(cost), -⋮---- -try_cache_response( -⋮---- -let cache_hv = cache_header_value(bypass_cache); -let mut response = (StatusCode::OK, Json(anthropic_resp)).into_response(); -⋮---- -response.headers_mut().insert("x-anyllm-cache", cache_hv); -⋮---- -state.metrics.record_error(); -let status = e.status_code(); -⋮---- -Some(e.to_string()), -⋮---- -backend_error_to_response(BackendError::from(e)) -⋮---- -responses_req.model = mapped_model.clone(); -let mapped_model = responses_req.model.clone(); -⋮---- -match client.responses(&responses_req).await { -⋮---- -// These backends are handled by separate handlers (passthrough / Bedrock / Gemini native). -// If we reach here, something is misconfigured. -⋮---- -"This backend does not use the translation handler".to_string(), -⋮---- -/// Captures per-request context shared across success/error log paths. -pub(crate) struct RequestCtx { -⋮---- -impl RequestCtx { -/// Build a log entry, filling common fields from the context. -pub(crate) fn log_entry( -⋮---- -request_id: self.request_id.clone(), -⋮---- -backend: backend_name.to_string(), -model_requested: Some(self.model_requested.clone()), -⋮---- -latency_ms: self.start.elapsed().as_millis() as u64, -input_tokens: tokens.map(|(i, _)| i), -output_tokens: tokens.map(|(_, o)| o), -⋮---- -/// Build a log entry with attribution (key_id from virtual key, cost from pricing). -⋮---- -pub(crate) fn log_entry_with_attribution( -⋮---- -let mut entry = self.log_entry( -⋮---- -entry.key_id = vk_ctx.as_ref().map(|ctx| ctx.key_id); -// Only store non-zero costs. -entry.cost_usd = cost_usd.filter(|&c| c > 0.0); -⋮---- -/// When routing through the Gemini OpenAI-compatible endpoint, inject Anthropic's -/// thinking config into the `google` extension field that Gemini expects. -pub(crate) fn inject_gemini_thinking( -⋮---- -if !matches!( -⋮---- -req.extra.insert( -"google".to_string(), -⋮---- -/// Record output tokens against the virtual key's TPM sliding window. -/// Called after the backend response is received and token count is known. -pub(crate) fn record_vk_tpm( -⋮---- -.record_tpm(crate::admin::keys::now_ms(), output_tokens); -⋮---- -/// Global webhook callback config, set once at startup. -⋮---- -/// Set the global webhook callback config (called once at startup). -pub fn set_callbacks(config: Arc) { -let _ = CALLBACKS.set(config); -⋮---- -/// Get a reference to the global webhook callback config, if set. -pub fn get_callbacks() -> Option<&'static Arc> { -CALLBACKS.get() -⋮---- -/// Log a completed request to the admin write buffer, broadcast to WebSocket clients, -/// and fire webhook callbacks if configured. -pub(crate) fn log_request(shared: &Option, entry: RequestLogEntry) { -if let Some(cb) = CALLBACKS.get() { -cb.notify(&entry); -⋮---- -.send(AdminEvent::RequestCompleted(entry.clone())); -let _ = shared.log_tx.try_send(entry); - - - -use std::sync::Arc; -⋮---- -async fn main() { -// ---- Phase 1: Collect env file overrides (before tracing init) ---- -let args: Vec = std::env::args().collect(); -⋮---- -.windows(2) -.find(|w| w[0] == "--env-file") -.map(|w| w[1].as_str()) -.or_else(|| { -if std::path::Path::new(".anyllm.env").exists() { -Some(".anyllm.env") -⋮---- -let env_file_vars = env_file_path.map(parse_env_file).unwrap_or_default(); -⋮---- -// ---- Phase 2: Apply env file vars (needed for RUST_LOG before tracing init) ---- -// SAFETY: single-threaded, before tokio spawns workers. -⋮---- -if !env_file_vars.is_empty() { -eprintln!( -⋮---- -// ---- Phase 3: Init tracing (needs RUST_LOG from env file) ---- -⋮---- -.with(filter) -.with(tracing_subscriber::fmt::layer().json()) -.with(otel_layer) -.init(); -⋮---- -// ---- Phase 4: Compute remaining env overrides and apply in one block ---- -⋮---- -// Apply alias overrides so config::MultiConfig::load() sees them. -// SAFETY: still single-threaded at this point (no spawns yet). -⋮---- -// Apply litellm master_key if PROXY_API_KEYS is still unset. -⋮---- -if std::env::var("PROXY_API_KEYS").is_err() { -// SAFETY: still single-threaded, no spawns yet. -⋮---- -// Wire up WEBHOOK_URLS and Langfuse env vars (if not already set from LiteLLM config). -if anyllm_proxy::server::routes::get_callbacks().is_none() { -⋮---- -.unwrap_or_default() -.split(',') -.map(|s| s.trim().to_string()) -.filter(|s| !s.is_empty()) -.collect(); -let mut named = vec![]; -⋮---- -named.push(anyllm_proxy::integrations::NamedIntegration::Langfuse(lf)); -⋮---- -// OIDC/JWT authentication (optional). When OIDC_ISSUER_URL is set, discover -// the OIDC configuration and load JWKS. Tokens that look like JWTs are -// validated against the JWKS before falling through to key-based auth. -⋮---- -let audience = std::env::var("OIDC_AUDIENCE").unwrap_or_else(|_| { -⋮---- -issuer_url.clone() -⋮---- -anyllm_proxy::server::middleware::set_oidc_config(config.clone()); -// Background task: refresh JWKS every 60 minutes. -⋮---- -interval.tick().await; // skip immediate tick -⋮---- -interval.tick().await; -if let Err(e) = config.refresh_jwks().await { -⋮---- -// Redis distributed rate limiting (optional, requires --features redis). -// When REDIS_URL is set, RPM/TPM checks are performed against Redis so -// multiple proxy instances share rate limit state. -⋮---- -// Admin web UI is opt-in: pass --webui or --admin to enable. -// DISABLE_ADMIN=1 overrides the flag (useful in container/scripted environments). -let flag_set = args.iter().any(|a| a == "--webui" || a == "--admin"); -let force_disabled = matches!( -⋮---- -// --- Admin setup (enabled only when --webui or --admin flag is passed) --- -// Returns Some((SharedState, admin Router, admin TcpListener)) when enabled. -⋮---- -.ok() -.and_then(|p| p.parse().ok()) -.unwrap_or(3001); -⋮---- -panic!("ADMIN_PORT ({admin_port}) must differ from LISTEN_PORT ({listen_port})"); -⋮---- -// SQLite: open or create the database file in the current directory. -let db_path = std::env::var("ADMIN_DB_PATH").unwrap_or_else(|_| "admin.db".into()); -⋮---- -rusqlite::Connection::open(&db_path).expect("failed to open SQLite database for admin"); -admin::db::init_db(&conn).expect("failed to initialize admin database schema"); -⋮---- -// Build initial RuntimeConfig from the loaded multi_config. -⋮---- -model_mappings.insert(name.clone(), bc.model_mapping.clone()); -⋮---- -let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); -⋮---- -// Apply config overrides from SQLite (survive restarts). -⋮---- -match key.as_str() { -⋮---- -// Apply the same allowlist enforced by the admin API to -// prevent a tampered SQLite database from enabling trace-level -// logging, which would expose API keys in HTTP headers. -⋮---- -let normalized = value.trim().to_lowercase(); -if ALLOWED_LOG_LEVELS.contains(&normalized.as_str()) { -⋮---- -k if k.ends_with(".big_model") => { -let backend = k.strip_suffix(".big_model").unwrap(); -if let Some(m) = runtime_config.model_mappings.get_mut(backend) { -m.big_model = value.clone(); -⋮---- -k if k.ends_with(".small_model") => { -let backend = k.strip_suffix(".small_model").unwrap(); -⋮---- -m.small_model = value.clone(); -⋮---- -if !overrides.is_empty() { -⋮---- -// Build the log_reload closure that captures the reload handle. -⋮---- -Ok(f) => handle.reload(f).is_ok(), -⋮---- -// Now wrap conn in Arc and start the write buffer. -// Uses std::sync::Mutex because rusqlite is synchronous; all access -// goes through spawn_blocking to avoid stalling the tokio executor. -⋮---- -let log_tx = admin::db::spawn_write_buffer(db.clone()); -⋮---- -// Load active virtual keys from SQLite into in-memory DashMap. -⋮---- -let conn_guard = db.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -virtual_keys.insert( -⋮---- -description: key_row.description.clone(), -expires_at: key_row.expires_at.as_deref().and_then(|s| { -⋮---- -.and_then(|e| i64::try_from(e).ok()) -⋮---- -.as_deref() -.and_then(admin::keys::BudgetDuration::parse), -period_start: key_row.period_start.clone(), -⋮---- -allowed_models: key_row.allowed_models.clone(), -⋮---- -// Make virtual keys and HMAC secret available to the auth middleware. -anyllm_proxy::server::middleware::set_virtual_keys(virtual_keys.clone()); -anyllm_proxy::server::middleware::set_hmac_secret(hmac_secret.clone()); -⋮---- -let virtual_keys_pruner = virtual_keys.clone(); -⋮---- -// Check and prune old rate limit states -for entry in virtual_keys_pruner.iter() { -let _ = entry.rate_state.check_rpm(0, now); -let _ = entry.rate_state.check_tpm(0, now); -⋮---- -db: db.clone(), -events_tx: events_tx.clone(), -runtime_config: runtime_config.clone(), -⋮---- -log_reload: Some(log_reload), -⋮---- -model_router: model_router.clone(), -⋮---- -// Admin token: use env var or generate random UUID written to a file. -let admin_token = std::env::var("ADMIN_TOKEN").unwrap_or_else(|_| { -let token = uuid::Uuid::new_v4().to_string(); -let token_path = resolve_admin_token_path(); -let token_path = token_path.to_string_lossy().to_string(); -// Write token to file with restrictive permissions instead of stderr, -// because stderr is captured by container log drivers in production. -if let Err(e) = write_token_file(&token_path, &token) { -// Do not print the token to stderr: container log drivers capture -// stderr and persist it in centralized logging systems. -panic!( -⋮---- -// Log the path, not the token itself. -⋮---- -// Spawn periodic tasks: log retention and metrics snapshot broadcast. -⋮---- -.and_then(|v| v.parse().ok()) -.unwrap_or(7); -⋮---- -let retention_db = shared.db.clone(); -⋮---- -// Periodic metrics snapshot broadcast (every 5 seconds) for WebSocket dashboard. -let snapshot_shared = shared.clone(); -⋮---- -// Skip computation if no WebSocket clients are listening. -if snapshot_shared.events_tx.receiver_count() == 0 { -⋮---- -for (name, m) in snapshot_shared.backend_metrics.iter() { -let snap = m.snapshot(); -⋮---- -backends.insert(name.clone(), snap); -⋮---- -let error_rate = aggregate.error_rate(); -// Count requests in the last 60 seconds for RPS. -⋮---- -let db = snapshot_shared.db.clone(); -⋮---- -.duration_since(std::time::UNIX_EPOCH) -⋮---- -.as_secs(); -let since = now_secs.saturating_sub(60); -⋮---- -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -admin::db::count_requests_since(&conn, since).unwrap_or(0) -⋮---- -.unwrap_or(0) as f64 -⋮---- -latency_p50_ms: None, // Computed on demand by REST endpoint -⋮---- -.send(admin::state::AdminEvent::MetricsSnapshot(snapshot)); -⋮---- -// Bind admin listener; spawned after the shutdown channel is created below. -let admin_app = admin::routes::admin_router(shared.clone(), admin_token); -let admin_addr = format!("127.0.0.1:{admin_port}"); -⋮---- -.unwrap_or_else(|e| panic!("failed to bind admin to {admin_addr}: {e}")); -⋮---- -Some((shared, admin_app, admin_listener)) -⋮---- -// Build proxy router with optional shared admin state. -⋮---- -admin_parts.as_ref().map(|(s, _, _)| s.clone()), -⋮---- -// --- Start servers --- -let proxy_addr = format!("0.0.0.0:{listen_port}"); -⋮---- -.unwrap_or_else(|e| panic!("failed to bind proxy to {proxy_addr}: {e}")); -⋮---- -// Warn if API keys are configured and listener is on a non-loopback address. -⋮---- -.local_addr() -.unwrap_or_else(|e| panic!("failed to get local address from listener: {e}")); -⋮---- -let has_proxy_keys = std::env::var("PROXY_API_KEYS").is_ok(); -⋮---- -.as_ref() -.map(|(shared, _, _)| !shared.virtual_keys.is_empty()) -.unwrap_or(false); -⋮---- -if (has_proxy_keys || has_virtual_keys) && !listen_addr.ip().is_loopback() { -⋮---- -// Single shutdown channel shared by proxy and (optionally) admin. -⋮---- -.with_graceful_shutdown(async move { -shutdown_rx1.changed().await.ok(); -⋮---- -.expect("proxy server error"); -⋮---- -let mut shutdown_rx2 = shutdown_tx.subscribe(); -Some(tokio::spawn(async move { -⋮---- -shutdown_rx2.changed().await.ok(); -⋮---- -.expect("admin server error"); -⋮---- -shutdown_signal().await; -let _ = shutdown_tx.send(true); -⋮---- -/// Parse a `.env`-format file and return `(key, value)` pairs to set. -/// -/// Rules: -/// - `KEY=VALUE` sets the variable. Surrounding whitespace is trimmed. -/// - Values may be optionally wrapped in `"double"` or `'single'` quotes. -/// - Lines starting with `#` (after trimming) are comments. -/// - Already-set environment variables are skipped; the real -/// environment always takes precedence over the file. -/// - `export KEY=VALUE` syntax is supported (the `export` prefix is stripped). -⋮---- -/// Returns pairs that should be applied via `set_var` in the consolidated block. -/// Compatible with Docker `--env-file` and standard dotenv tooling. -fn parse_env_file(path: &str) -> Vec<(String, String)> { -⋮---- -// Print directly; tracing isn't initialized yet. -eprintln!("anyllm_proxy: could not read env file '{path}': {e}"); -⋮---- -for (lineno, raw) in content.lines().enumerate() { -let line = raw.trim(); -if line.is_empty() || line.starts_with('#') { -⋮---- -// Strip optional `export ` prefix. -let line = line.strip_prefix("export ").map(str::trim).unwrap_or(line); -let Some((key, val)) = line.split_once('=') else { -⋮---- -let key = key.trim(); -if key.is_empty() { -⋮---- -// Strip optional surrounding quotes from the value. -let val = val.trim(); -let val = if (val.starts_with('"') && val.ends_with('"')) -|| (val.starts_with('\'') && val.ends_with('\'')) -⋮---- -&val[1..val.len() - 1] -⋮---- -// Only include if not already present so the real environment wins. -if std::env::var(key).is_err() { -pairs.push((key.to_string(), val.to_string())); -⋮---- -/// Resolve admin token file path from `ADMIN_TOKEN_PATH` env var, -/// falling back to `.admin_token` in the current directory. -fn resolve_admin_token_path() -> std::path::PathBuf { -⋮---- -/// Write the admin token to a file with mode 0600 (owner-only read/write). -/// On Unix, sets permissions atomically at creation to avoid a TOCTOU race -/// where the file is briefly world-readable before chmod. -fn write_token_file(path: &str, token: &str) -> std::io::Result<()> { -use std::io::Write; -⋮---- -use std::os::unix::fs::OpenOptionsExt; -⋮---- -.write(true) -.create(true) -.truncate(true) -.mode(0o600) -.open(path)? -⋮---- -file.write_all(token.as_bytes())?; -file.write_all(b"\n")?; -Ok(()) -⋮---- -async fn shutdown_signal() { -⋮---- -.expect("failed to install SIGTERM handler"); -⋮---- -ctrl_c.await.expect("failed to listen for Ctrl+C"); - - - -// Admin server routes. Served on a separate localhost-only listener. -⋮---- -use crate::admin::state::SharedState; -use crate::admin::ws::ws_handler; -⋮---- -use dashmap::DashMap; -⋮---- -/// Per-IP sliding window rate limiter for admin API endpoints. -/// Each entry is a VecDeque of millisecond timestamps within the last 60 seconds. -⋮---- -/// Maximum admin API requests per IP per 60-second window. -/// Default 10; can be overridden at runtime for tests via `set_admin_rpm`. -⋮---- -/// Override the admin rate limit (requests per minute per IP). -/// Intended for integration tests that need a higher limit. -pub fn set_admin_rpm(rpm: u32) { -ADMIN_RPM.store(rpm, Ordering::Relaxed); -⋮---- -/// Clear all rate limit state. Exposed for integration tests. -pub fn reset_admin_rate_limit() { -ADMIN_RATE_LIMIT.clear(); -⋮---- -/// Inner rate-limit check with an explicit rpm; avoids touching the global ADMIN_RPM in tests. -fn check_admin_rate_limit_with_rpm(ip: IpAddr, rpm: u32) -> bool { -⋮---- -let cutoff = now_ms.saturating_sub(60_000); -let mut window = ADMIN_RATE_LIMIT.entry(ip).or_default(); -// Evict timestamps older than 60 seconds. -while window.front().is_some_and(|&ts| ts < cutoff) { -window.pop_front(); -⋮---- -if window.len() >= rpm as usize { -⋮---- -window.push_back(now_ms); -⋮---- -/// Returns true if the request is within the rate limit, false if exceeded. -fn check_admin_rate_limit(ip: IpAddr) -> bool { -check_admin_rate_limit_with_rpm(ip, ADMIN_RPM.load(Ordering::Relaxed)) -⋮---- -/// Axum middleware that enforces per-IP rate limiting on admin API routes. -/// Returns 429 Too Many Requests when the limit is exceeded. -async fn admin_rate_limit_middleware( -⋮---- -// Extract client IP from ConnectInfo extension (set by into_make_service_with_connect_info). -⋮---- -.extensions() -⋮---- -.map(|ci| ci.0.ip()) -.unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); -⋮---- -if !check_admin_rate_limit(ip) { -⋮---- -return Err(StatusCode::TOO_MANY_REQUESTS); -⋮---- -Ok(next.run(req).await) -⋮---- -/// Reject model names containing path traversal sequences or suspicious characters. -/// Only alphanumerics plus `-_./: @` are allowed (covers known provider naming -/// conventions like `gpt-4o`, `us.meta.llama3-2-1b-instruct-v1:0`, -/// `accounts/fireworks/models/llama-v3p1-8b-instruct`). -fn is_safe_model_name(name: &str) -> bool { -!name.is_empty() -&& !name.contains("..") -&& !name.contains('?') -&& !name.contains('#') -⋮---- -.chars() -.all(|c| c.is_alphanumeric() || "-_./: @".contains(c)) -⋮---- -/// Check whether a host string (without port) is a localhost address. -fn is_localhost_host(host: &str) -> bool { -matches!(host, "127.0.0.1" | "localhost" | "[::1]" | "::1") -⋮---- -/// Reject cross-origin requests to the admin API. -/// Parses the Origin URL and checks the host component exactly -/// to prevent bypass via e.g. `http://127.0.0.1.attacker.com`. -/// -/// When no Origin header is present, validates the Host header instead -/// to guard against DNS rebinding attacks. -async fn reject_cross_origin( -⋮---- -if let Some(origin) = req.headers().get("origin") { -let origin_str = origin.to_str().map_err(|_| StatusCode::BAD_REQUEST)?; -⋮---- -Ok(url) => url.host_str().is_some_and(is_localhost_host), -⋮---- -return Err(StatusCode::FORBIDDEN); -⋮---- -// No Origin header: validate Host to prevent DNS rebinding attacks, -// where an attacker's domain resolves to localhost, causing the -// browser to send requests to our admin API. -⋮---- -.headers() -.get("host") -.and_then(|h| h.to_str().ok()) -.map(|h| { -// Strip optional port. Bracketed IPv6 like "[::1]:9090" -// must not be split naively on ':'. -let host_part = if h.starts_with('[') { -// "[::1]:9090" -> "[::1]", or "[::1]" if no port -h.split_once(']').map_or(h, |(bracket, _)| { -// Include the closing bracket for is_localhost_host -&h[..bracket.len() + 1] -⋮---- -// "localhost:9090" -> "localhost", but bare "::1" must -// not be split (contains colons but no port suffix). -// Only split if the part after the last colon is numeric. -match h.rsplit_once(':') { -Some((host, port)) if port.bytes().all(|b| b.is_ascii_digit()) => host, -⋮---- -is_localhost_host(host_part) -⋮---- -.unwrap_or(false); -⋮---- -/// Middleware that validates CSRF tokens for state-mutating HTTP methods. -⋮---- -/// Skips validation for GET, HEAD, OPTIONS. -/// For POST, PUT, DELETE: requires X-CSRF-Token header to match the csrf_token cookie. -/// Returns 403 with a descriptive error if the token is missing or mismatched. -/// Applied inside validate_admin_token so unauthenticated requests are rejected first. -pub async fn validate_csrf( -⋮---- -let method = req.method().clone(); -⋮---- -if matches!( -⋮---- -let headers = req.headers(); -⋮---- -.get("x-csrf-token") -.and_then(|v| v.to_str().ok()) -.unwrap_or(""); -⋮---- -.get("cookie") -⋮---- -.and_then(extract_csrf_cookie) -.unwrap_or_default(); -⋮---- -if !validate_csrf_tokens(header_token, &cookie_token) { -⋮---- -return (StatusCode::FORBIDDEN, axum::Json(body)).into_response(); -⋮---- -next.run(req).await -⋮---- -/// GET /admin/csrf-token -⋮---- -/// Returns a fresh CSRF token as JSON and sets it in a non-HttpOnly cookie. -/// The admin SPA reads the cookie in JS and includes it as `X-CSRF-Token` on -/// POST/PUT/DELETE requests (double-submit cookie pattern). -⋮---- -/// Security architecture note: -/// This route is intentionally public (no Bearer auth required). The SPA must -/// fetch a CSRF token to submit the login form itself, so requiring auth here -/// would be circular. Protection comes from two middleware layers applied to all -/// admin routes, including this one: -/// 1. `reject_cross_origin`: validates Origin/Host header; only requests -/// from localhost can reach any admin endpoint. -/// 2. `SameSite=Strict` on the cookie: browsers do not attach the cookie on -/// cross-site requests, preventing a cross-origin attacker from using a -/// CSRF token they fetched independently. -⋮---- -/// Together these make unauthenticated CSRF token fetching safe: an attacker who -/// can reach this endpoint is already on localhost and has other attack vectors. -/// If TLS is ever added to the admin server, also add `Secure` to Set-Cookie. -async fn get_csrf_token() -> axum::response::Response { -let token = generate_csrf_token(); -⋮---- -.status(StatusCode::OK) -.header("content-type", "application/json") -// SameSite=Strict prevents the cookie being sent on cross-site requests. -// Not httpOnly so the admin SPA JS can read and send it back as a header. -// Secure flag intentionally omitted: admin binds to 127.0.0.1 over plain HTTP, -// so setting Secure would prevent the browser from sending the cookie at all. -// If TLS is added to the admin server, Secure must be added here. -.header( -⋮---- -format!("csrf_token={token}; Path=/admin; SameSite=Strict; Max-Age=86400"), -⋮---- -.body(axum::body::Body::from( -serde_json::to_string(&body).unwrap(), -⋮---- -.unwrap() -.into_response() -⋮---- -/// Build the admin router. -/// Token is used for auth middleware on all routes except /admin/health. -pub fn admin_router(shared: SharedState, token: Arc) -> Router { -// Public routes (no auth). -// /admin/csrf-token is public so the SPA can fetch a token before and after login. -⋮---- -.route("/admin/health", get(health)) -.route("/admin/csrf-token", get(get_csrf_token)); -⋮---- -// Protected routes (require admin token + localhost origin check). -⋮---- -.route("/admin/api/config", get(get_config).put(put_config)) -.route("/admin/api/config/overrides", get(get_config_overrides)) -.route( -⋮---- -delete(delete_config_override), -⋮---- -.route("/admin/api/env", get(get_env)) -.route("/admin/api/metrics", get(get_metrics)) -.route("/admin/api/requests", get(get_requests)) -.route("/admin/api/requests/{id}", get(get_request_by_id)) -.route("/admin/api/backends", get(get_backends)) -.route("/admin/api/keys", post(create_key).get(list_keys)) -.route("/admin/api/keys/{id}", put(update_key).delete(revoke_key)) -⋮---- -get(super::spend::get_key_spend), -⋮---- -.route("/admin/api/models", get(list_models).post(add_model)) -.route("/admin/api/models/{name}", delete(remove_model)) -.route("/admin/api/audit", get(get_audit_log)) -.with_state(shared.clone()) -// Innermost: CSRF check runs after auth succeeds. -.layer(middleware::from_fn(validate_csrf)) -.layer(middleware::from_fn_with_state( -token.clone(), -⋮---- -.layer(middleware::from_fn(reject_cross_origin)) -.layer(middleware::from_fn(admin_rate_limit_middleware)); -⋮---- -// WebSocket: auth via first message since browsers can't set headers on WS. -// Origin check applied here too to prevent cross-site WebSocket hijacking. -let ws_state = (shared.clone(), token.clone()); -⋮---- -.route("/admin/ws", get(ws_handler)) -.with_state(ws_state) -.layer(middleware::from_fn(reject_cross_origin)); -⋮---- -// SPA serving (no auth required, token passed via query param in browser). -⋮---- -.route("/admin/", get(serve_spa)) -.route("/admin", get(serve_spa)); -⋮---- -// Merge all routes. -⋮---- -.merge(protected) -.merge(ws_route) -.merge(spa_route) -.layer(DefaultBodyLimit::max(1_048_576)) -⋮---- -async fn health() -> Json { -Json(serde_json::json!({"status": "ok"})) -⋮---- -/// Serve the embedded SPA HTML. -static SPA_HTML: &str = include_str!("../../admin-ui/index.html"); -⋮---- -async fn serve_spa() -> impl IntoResponse { -⋮---- -// Restrictive CSP: inline script/style needed because SPA is a single HTML file. -// frame-ancestors 'none' prevents clickjacking. -⋮---- -// -- Env endpoint -- -⋮---- -/// GET /admin/api/env -- effective environment variable values. -/// Secrets (API keys, tokens) are masked; plain config values are shown as-is. -async fn get_env() -> Json { -fn plain(key: &str) -> serde_json::Value { -⋮---- -Ok(v) if !v.is_empty() => serde_json::Value::String(v), -⋮---- -fn secret(key: &str) -> serde_json::Value { -⋮---- -Ok(v) if !v.is_empty() => { -⋮---- -Json(serde_json::json!({ -// Core proxy config -⋮---- -// OpenAI / compatible -⋮---- -// Vertex AI -⋮---- -// Gemini -⋮---- -// Azure OpenAI -⋮---- -// AWS Bedrock -⋮---- -// Google OAuth bearer token (full token — treat as secret) -⋮---- -// Auth -⋮---- -// TLS -⋮---- -// Network / security -⋮---- -// Admin -⋮---- -// -- Config endpoints -- -⋮---- -/// GET /admin/api/config -- effective config (env defaults + overrides). -async fn get_config(State(shared): State) -> Json { -// Clone config snapshot and drop the read guard before any .await points. -// std::sync::RwLockReadGuard is !Send, cannot be held across awaits. -⋮---- -.read() -.unwrap_or_else(|e| e.into_inner()); -⋮---- -backends.insert( -name.clone(), -⋮---- -(config.log_level.clone(), config.log_bodies, backends) -⋮---- -// Get overrides to mark which fields are overridden. -⋮---- -crate::admin::db::get_config_overrides(conn).unwrap_or_default() -⋮---- -let override_keys: Vec = overrides.iter().map(|(k, _, _)| k.clone()).collect(); -⋮---- -/// PUT /admin/api/config -- update config overrides. Partial JSON body. -async fn put_config( -⋮---- -// Collect the key-value pairs to persist, then do all SQLite I/O -// before touching in-memory state. This avoids holding the async -// MutexGuard across block_in_place. -⋮---- -if let Some(level) = body.get("log_level").and_then(|v| v.as_str()) { -// Allowlist: trace-level logging exposes HTTP headers (including API -// keys) in log output. Arbitrary filter directives could also be used -// to selectively leak data. Restrict to safe levels only. -⋮---- -let normalized = level.trim().to_lowercase(); -if !ALLOWED_LOG_LEVELS.contains(&normalized.as_str()) { -⋮---- -.into_response(); -⋮---- -db_writes.push(("log_level".to_string(), normalized)); -⋮---- -if let Some(val) = body.get("log_bodies").and_then(|v| v.as_bool()) { -⋮---- -db_writes.push(("log_bodies".to_string(), val.to_string())); -⋮---- -if let Some(backends) = body.get("backends").and_then(|v| v.as_object()) { -// Read current config to validate backend names exist -⋮---- -if config.model_mappings.contains_key(name) { -if let Some(big) = settings.get("big_model").and_then(|v| v.as_str()) { -if !is_safe_model_name(big) { -⋮---- -db_writes.push((format!("{name}.big_model"), big.to_string())); -⋮---- -if let Some(small) = settings.get("small_model").and_then(|v| v.as_str()) { -if !is_safe_model_name(small) { -⋮---- -db_writes.push((format!("{name}.small_model"), small.to_string())); -⋮---- -// Serialize config writes so concurrent requests cannot interleave -// Phase 1 (SQLite) and Phase 2 (in-memory), which would leave them -// inconsistent. -let _config_guard = shared.config_write_lock.lock().await; -⋮---- -// Phase 1: Persist to SQLite first. If the process crashes between -// phases, the database is the source of truth and config is restored -// on restart. Reversing the order would lose updates on crash. -⋮---- -let writes = db_writes.clone(); -⋮---- -crate::admin::db::set_config_override(conn, key, value).ok(); -⋮---- -// Phase 2: Apply to in-memory config (no async lock held) -⋮---- -.write() -⋮---- -// Audit log: capture old values before applying changes -⋮---- -let old_value = match key.as_str() { -"log_level" => config.log_level.clone(), -"log_bodies" => config.log_bodies.to_string(), -⋮---- -if let Some((backend, field)) = other.split_once('.') { -⋮---- -.get(backend) -.map(|m| match field { -"big_model" => m.big_model.clone(), -"small_model" => m.small_model.clone(), -_ => "".to_string(), -⋮---- -.unwrap_or_else(|| "".to_string()) -⋮---- -"".to_string() -⋮---- -match key.as_str() { -⋮---- -config.log_level = value.clone(); -⋮---- -if !reload(value) { -⋮---- -if let Some((backend, field)) = key.split_once('.') { -if let Some(mapping) = config.model_mappings.get_mut(backend) { -⋮---- -"big_model" => mapping.big_model = value.clone(), -"small_model" => mapping.small_model = value.clone(), -⋮---- -drop(_config_guard); -⋮---- -// Broadcast config changes. -⋮---- -.send(crate::admin::state::AdminEvent::ConfigChanged { -key: key.clone(), -value: value.clone(), -⋮---- -emit_audit( -⋮---- -action: "config_changed".into(), -target_type: "config".into(), -target_id: Some(key.clone()), -detail: Some(format!("value={value}")), -source_ip: Some(addr.ip().to_string()), -⋮---- -/// GET /admin/api/config/overrides -- only SQLite overrides. -async fn get_config_overrides(State(shared): State) -> Json { -⋮---- -.into_iter() -.map(|(k, v, updated_at)| { -⋮---- -.collect(); -⋮---- -Json(serde_json::json!({ "overrides": entries })) -⋮---- -/// DELETE /admin/api/config/overrides/:key -- remove a single override. -async fn delete_config_override( -⋮---- -let key_clone = key.clone(); -⋮---- -action: "config_deleted".into(), -⋮---- -(StatusCode::OK, Json(serde_json::json!({"deleted": key}))).into_response() -⋮---- -Json(serde_json::json!({"error": "override not found"})), -⋮---- -.into_response(), -⋮---- -Json(serde_json::json!({"error": "internal database error"})), -⋮---- -Json(serde_json::json!({"error": "internal error"})), -⋮---- -// -- Metrics endpoint -- -⋮---- -/// GET /admin/api/metrics -- current metrics snapshot. -async fn get_metrics(State(shared): State) -> Json { -⋮---- -for (name, m) in shared.backend_metrics.iter() { -let snap = m.snapshot(); -⋮---- -serde_json::to_value(&snap).unwrap_or_default(), -⋮---- -.unwrap_or((None, None, None)); -⋮---- -/// Compute p50, p95, p99 latency from the last 5 minutes of request log. -fn compute_latency_percentiles( -⋮---- -// Get latencies from recent requests, sorted. -let cutoff = crate::admin::db::now_iso8601(); // We want last 5 minutes -⋮---- -.prepare( -⋮---- -.ok(); -⋮---- -.as_mut() -.and_then(|s| { -s.query_map(rusqlite::params![cutoff], |row| { -row.get::<_, i64>(0).map(|v| v as u64) -⋮---- -.ok() -⋮---- -.map(|rows| rows.filter_map(|r| r.ok()).collect()) -⋮---- -if latencies.is_empty() { -⋮---- -let idx = ((pct / 100.0) * (latencies.len() as f64 - 1.0)).round() as usize; -latencies[idx.min(latencies.len() - 1)] -⋮---- -(Some(p(50.0)), Some(p(95.0)), Some(p(99.0))) -⋮---- -// -- Request log endpoints -- -⋮---- -struct RequestsQuery { -⋮---- -/// GET /admin/api/requests -- paginated request log. -async fn get_requests( -⋮---- -let limit = params.limit.unwrap_or(50).min(1000); -let offset = params.offset.unwrap_or(0); -⋮---- -backend.as_deref(), -since.as_deref(), -until.as_deref(), -status.as_deref(), -⋮---- -Some(Ok(entries)) => Json(serde_json::json!({ -⋮---- -None => Json(serde_json::json!({ -⋮---- -/// GET /admin/api/requests/:id -- single request detail. -async fn get_request_by_id( -⋮---- -(StatusCode::OK, Json(serde_json::to_value(entry).unwrap())).into_response() -⋮---- -Json(serde_json::json!({"error": "request not found"})), -⋮---- -// -- Backends endpoint -- -⋮---- -/// GET /admin/api/backends -- list configured backends with status. -async fn get_backends(State(shared): State) -> Json { -⋮---- -.get(name) -.map(|m| m.snapshot()) -⋮---- -backends.push(serde_json::json!({ -⋮---- -Json(serde_json::json!({ "backends": backends })) -⋮---- -// --- Virtual API Key Management --- -⋮---- -struct CreateKeyRequest { -⋮---- -/// POST /admin/api/keys -- create a new virtual API key. -async fn create_key( -⋮---- -let role_str = body.role.as_deref().unwrap_or("developer"); -⋮---- -let hash = key_hash_hex.clone(); -let prefix = key_prefix.clone(); -let desc = body.description.clone(); -let exp = body.expires_at.clone(); -⋮---- -let role_s = role_str.to_string(); -⋮---- -let budget_dur = body.budget_duration.clone(); -⋮---- -.as_ref() -.and_then(|v| serde_json::to_string(v).ok()); -⋮---- -description: desc.as_deref(), -expires_at: exp.as_deref(), -⋮---- -budget_duration: budget_dur.as_deref(), -⋮---- -shared.virtual_keys.insert( -⋮---- -description: body.description.clone(), -expires_at: body.expires_at.as_deref().and_then(|s| { -⋮---- -.and_then(|e| i64::try_from(e).ok()) -⋮---- -.as_deref() -.and_then(super::keys::BudgetDuration::parse), -period_start: Some(super::db::now_iso8601()), -⋮---- -allowed_models: body.allowed_models.clone(), -⋮---- -action: "key_created".into(), -target_type: "virtual_key".into(), -target_id: Some(id.to_string()), -detail: Some(format!( -⋮---- -Json(serde_json::json!({"error": "Failed to create key"})), -⋮---- -/// GET /admin/api/keys -- list all virtual keys. -async fn list_keys(State(shared): State) -> axum::response::Response { -⋮---- -.iter() -.map(|k| { -⋮---- -Json(serde_json::json!({ "keys": enriched })).into_response() -⋮---- -Json(serde_json::json!({"error": "Failed to list keys"})), -⋮---- -/// Request body for PUT /admin/api/keys/{id}. -/// All fields are optional: absent = clear (set to NULL); role is immutable after creation. -⋮---- -struct UpdateKeyRequest { -⋮---- -/// PUT /admin/api/keys/{id} -- update an existing virtual key (except role). -async fn update_key( -⋮---- -// Refresh the DashMap entry so in-flight auth sees updated limits. -⋮---- -shared.virtual_keys.entry(hash_bytes).and_modify(|meta| { -meta.description = body.description.clone(); -meta.expires_at = body.expires_at.as_deref().and_then(|s| { -⋮---- -if body.budget_duration.is_some() { -⋮---- -.and_then(super::keys::BudgetDuration::parse); -// Reset spend period to match db-layer reset. -⋮---- -meta.allowed_models = body.allowed_models.clone(); -⋮---- -action: "key_updated".into(), -⋮---- -detail: Some(format!("prefix={}", row.key_prefix)), -⋮---- -Json(serde_json::json!({"error": "Key not found or already revoked"})), -⋮---- -Json(serde_json::json!({"error": "Failed to update key"})), -⋮---- -/// DELETE /admin/api/keys/{id} -- revoke a virtual key. -async fn revoke_key( -⋮---- -shared.virtual_keys.remove(&hash_bytes); -⋮---- -action: "key_revoked".into(), -⋮---- -Json(serde_json::json!({"error": "Failed to revoke key"})), -⋮---- -// ---- Dynamic model management ---- -⋮---- -/// GET /admin/api/models -- list all routed model names and deployment counts. -async fn list_models(State(shared): State) -> impl IntoResponse { -⋮---- -let router = router_lock.read().unwrap_or_else(|e| e.into_inner()); -⋮---- -.list_models() -⋮---- -.map(|(name, count)| { -⋮---- -/// Request body for POST /admin/api/models. -⋮---- -struct AddModelRequest { -⋮---- -fn default_weight() -> u32 { -⋮---- -/// POST /admin/api/models -- add a deployment for a model name. -async fn add_model( -⋮---- -Json(serde_json::json!({"error": "no model router active"})), -⋮---- -body.backend_name.clone(), -body.actual_model.clone(), -⋮---- -let mut router = router_lock.write().unwrap_or_else(|e| e.into_inner()); -router.add_deployment(body.model_name.clone(), deployment); -⋮---- -action: "model_added".into(), -target_type: "model".into(), -target_id: Some(body.model_name.clone()), -⋮---- -/// DELETE /admin/api/models/{name} -- remove all deployments for a model. -async fn remove_model( -⋮---- -if router.remove_model(&name) { -⋮---- -action: "model_removed".into(), -⋮---- -target_id: Some(name.clone()), -⋮---- -Json(serde_json::json!({"status": "removed", "model_name": name})), -⋮---- -Json(serde_json::json!({"error": "model not found", "model_name": name})), -⋮---- -// --- Audit log --- -⋮---- -struct AuditQuery { -⋮---- -/// GET /admin/api/audit -- paginated audit log. -async fn get_audit_log( -⋮---- -action.as_deref(), -target_type.as_deref(), -⋮---- -/// Fire-and-forget audit log write. Failures are logged but never block the caller. -fn emit_audit(shared: &SharedState, entry: crate::admin::db::AuditEntry) { -let db = shared.db.clone(); -⋮---- -let conn = db.lock().unwrap_or_else(|e| e.into_inner()); -⋮---- -mod tests { -⋮---- -use axum::body::Body; -use axum::http::Request; -use tower::ServiceExt; -⋮---- -/// Build a minimal admin router for origin/host tests. -fn test_router() -> Router { -// Raise rate limit so parallel unit tests don't interfere. -set_admin_rpm(10_000); -⋮---- -let token = Arc::new("test-token".to_string()); -admin_router(shared, token) -⋮---- -async fn origin_localhost_allowed() { -let app = test_router(); -⋮---- -.header("origin", "http://localhost:9090") -.header("authorization", "Bearer test-token") -.body(Body::empty()) -.unwrap(); -let resp = app.oneshot(req).await.unwrap(); -assert_ne!(resp.status(), StatusCode::FORBIDDEN); -⋮---- -async fn origin_evil_rejected() { -⋮---- -.header("origin", "http://evil.com") -⋮---- -assert_eq!(resp.status(), StatusCode::FORBIDDEN); -⋮---- -async fn no_origin_localhost_host_allowed() { -⋮---- -.header("host", "localhost:9090") -⋮---- -async fn no_origin_127_host_allowed() { -⋮---- -.header("host", "127.0.0.1:9090") -⋮---- -async fn no_origin_evil_host_rejected() { -⋮---- -.header("host", "evil.com") -⋮---- -async fn no_origin_no_host_rejected() { -⋮---- -fn admin_rate_limit_enforced() { -// Use a unique IP and pass rpm directly to avoid mutating ADMIN_RPM, -// which would race with test_router() calling set_admin_rpm(10_000). -let ip: IpAddr = "198.51.100.1".parse().unwrap(); -ADMIN_RATE_LIMIT.remove(&ip); -⋮---- -assert!(check_admin_rate_limit_with_rpm(ip, 3)); -⋮---- -// 4th request in the same window should be rejected. -assert!(!check_admin_rate_limit_with_rpm(ip, 3)); -⋮---- -fn sliding_window_blocks_on_rpm_exceeded() { -// Use a unique IP to avoid test isolation issues. -let ip: IpAddr = "10.88.77.66".parse().unwrap(); -// With rpm=2, the first 2 requests must pass, the 3rd must fail. -assert!(check_admin_rate_limit_with_rpm(ip, 2)); -⋮---- -assert!(!check_admin_rate_limit_with_rpm(ip, 2), "3rd request must be blocked when rpm=2"); -⋮---- -/// POST to a protected admin route without CSRF token returns 403. -⋮---- -async fn post_without_csrf_returns_403() { -⋮---- -.body(Body::from(r#"{"description":"test"}"#)) -⋮---- -/// POST with matching CSRF header and cookie succeeds (auth passes, handler runs). -⋮---- -async fn post_with_valid_csrf_passes_middleware() { -⋮---- -let token = "a".repeat(64); -⋮---- -.header("x-csrf-token", &token) -.header("cookie", format!("csrf_token={token}")) -⋮---- -// 403 would mean CSRF rejected; any other status means CSRF passed. -⋮---- -/// DELETE without CSRF token returns 403. -⋮---- -async fn delete_without_csrf_returns_403() { -⋮---- -/// GET /admin/csrf-token returns 200 with JSON body and Set-Cookie header. -⋮---- -async fn get_csrf_token_sets_cookie() { -⋮---- -assert_eq!(resp.status(), StatusCode::OK); -⋮---- -.get("set-cookie") -⋮---- -assert!( -⋮---- -// Not httpOnly so JS can read it. -⋮---- -/// GET /admin/csrf-token returns JSON with csrf_token field. -⋮---- -async fn get_csrf_token_returns_json() { -⋮---- -let body_bytes = axum::body::to_bytes(resp.into_body(), 1 << 16) -⋮---- -let body: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap(); -let token = body["csrf_token"].as_str().unwrap(); -assert_eq!(token.len(), 64); -⋮---- -/// GET requests to protected routes do NOT require CSRF token. -⋮---- -async fn get_request_does_not_require_csrf() { -⋮---- -// CSRF should not reject GET; any non-403 means CSRF passed. -⋮---- -fn aws_access_key_id_uses_secret_pattern() { -// The secret() closure masks the value; this test verifies the masking logic. -let mask = |v: &str| if !v.is_empty() { "***REDACTED***".to_string() } else { "".to_string() }; -assert_eq!(mask("AKIAIOSFODNN7EXAMPLE"), "***REDACTED***"); -assert_eq!(mask(""), ""); -⋮---- -fn google_access_token_uses_secret_pattern() { -⋮---- -assert_eq!(mask("ya29.someoauthtoken"), "***REDACTED***"); - - -