diff --git a/crates/proxy/src/admin/state.rs b/crates/proxy/src/admin/state.rs index 0d20b26..52ce731 100644 --- a/crates/proxy/src/admin/state.rs +++ b/crates/proxy/src/admin/state.rs @@ -83,19 +83,25 @@ pub enum AdminEvent { ConfigChanged { key: String, value: String }, } -/// Data recorded for each proxied request. +/// Data recorded for each proxied request. Stored in SQLite and broadcast +/// to WebSocket clients for the live admin dashboard. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct RequestLogEntry { pub request_id: String, pub timestamp: String, pub backend: String, + /// Model name from the client's Anthropic request (before mapping). pub model_requested: Option, + /// Model name actually sent to the backend (after mapping). pub model_mapped: Option, pub status_code: u16, pub latency_ms: u64, pub input_tokens: Option, pub output_tokens: Option, + /// Whether the request used SSE streaming. Streaming requests only + /// track total count in metrics, not per-request success/error. pub is_streaming: bool, + /// Present only when the request failed; contains the error description. pub error_message: Option, } diff --git a/crates/proxy/src/backend/mod.rs b/crates/proxy/src/backend/mod.rs index 334c8ce..d6ee673 100644 --- a/crates/proxy/src/backend/mod.rs +++ b/crates/proxy/src/backend/mod.rs @@ -369,21 +369,34 @@ fn openai_duration_to_iso8601(s: &str) -> Option { fn convert_reset_duration(raw: &Option, field: &str) -> Option { raw.as_deref().map(|v| { openai_duration_to_iso8601(v).unwrap_or_else(|| { - tracing::warn!(value = v, field, "failed to parse reset duration, forwarding raw"); + tracing::warn!( + value = v, + field, + "failed to parse reset duration, forwarding raw" + ); v.to_string() }) }) } /// Rate limit headers extracted from backend responses. +/// Forwarded to clients as Anthropic-style `anthropic-ratelimit-*` headers. +/// See: #[derive(Debug, Default, Clone)] pub struct RateLimitHeaders { + /// Maximum requests allowed in the current window. pub requests_limit: Option, + /// Requests remaining before rate limiting kicks in. pub requests_remaining: Option, + /// ISO 8601 timestamp when the request limit resets. pub requests_reset: Option, + /// Maximum tokens allowed in the current window. pub tokens_limit: Option, + /// Tokens remaining before rate limiting kicks in. pub tokens_remaining: Option, + /// ISO 8601 timestamp when the token limit resets. pub tokens_reset: Option, + /// Seconds to wait before retrying (from `retry-after` header on 429s). pub retry_after: Option, } @@ -561,10 +574,7 @@ mod rate_limit_tests { #[test] fn parse_openai_duration_various_formats() { - assert_eq!( - parse_openai_duration("6ms"), - Some(Duration::from_millis(6)) - ); + assert_eq!(parse_openai_duration("6ms"), Some(Duration::from_millis(6))); assert_eq!( parse_openai_duration("1s"), Some(Duration::from_millis(1000)) diff --git a/crates/proxy/src/backend/openai_client.rs b/crates/proxy/src/backend/openai_client.rs index 22d9f2d..471a1f4 100644 --- a/crates/proxy/src/backend/openai_client.rs +++ b/crates/proxy/src/backend/openai_client.rs @@ -1,9 +1,8 @@ // reqwest client for calling OpenAI endpoints -// PLAN.md lines 649-650 use super::{build_http_client, RateLimitHeaders, RetryableError}; use crate::config::{BackendAuth, BackendKind, Config}; -use anthropic_openai_translate::openai; +use anyllm_translate::openai; use reqwest::Client; /// HTTP client for OpenAI-compatible Chat Completions APIs with retry logic. @@ -164,8 +163,11 @@ impl OpenAIClient { /// Errors from the OpenAI HTTP client. #[derive(Debug)] pub enum OpenAIClientError { + /// Transport-level failure (DNS, TLS, connection refused, timeout). Request(reqwest::Error), + /// Backend returned 2xx but the body was not valid ChatCompletionResponse JSON. Deserialization(reqwest::Error), + /// Backend returned a non-2xx status with a parseable OpenAI error body. ApiError { status: u16, error: openai::errors::ErrorResponse, diff --git a/crates/proxy/src/config/mod.rs b/crates/proxy/src/config/mod.rs index b24f51a..d8842a2 100644 --- a/crates/proxy/src/config/mod.rs +++ b/crates/proxy/src/config/mod.rs @@ -78,6 +78,8 @@ fn validate_gcp_identifier(name: &str, value: &str) { } impl Config { + /// Build configuration from environment variables. Panics on invalid values + /// (unknown backend, bad GCP identifiers) to fail fast at startup. pub fn from_env() -> Self { let backend_str = std::env::var("BACKEND").unwrap_or_else(|_| "openai".into()); let backend = match backend_str.to_ascii_lowercase().as_str() { @@ -237,10 +239,13 @@ pub struct ModelMapping { } impl ModelMapping { + /// Load model mapping from `BIG_MODEL` / `SMALL_MODEL` env vars with OpenAI defaults. pub fn from_env() -> Self { Self::from_env_with_defaults("gpt-4o", "gpt-4o-mini") } + /// Load model mapping from env vars, falling back to the provided defaults. + /// Each backend calls this with its own defaults (e.g., Gemini uses `gemini-2.5-pro`). pub fn from_env_with_defaults(big_default: &str, small_default: &str) -> Self { Self { big_model: std::env::var("BIG_MODEL").unwrap_or_else(|_| big_default.into()), @@ -290,13 +295,21 @@ pub fn resolve_env_value(value: &str) -> Result { /// Per-backend configuration. Each entry in `[backends.*]` deserializes into this. #[derive(Debug, Clone)] pub struct BackendConfig { + /// Which provider type this backend uses (OpenAI, Vertex, Gemini, Anthropic). pub kind: BackendKind, + /// API key for authentication. Resolved from env vars via `env:VAR_NAME` syntax. pub api_key: String, + /// Base URL of the backend API (e.g., `https://api.openai.com`). pub base_url: String, + /// Which OpenAI API format to use (Chat Completions or Responses). pub api_format: OpenAIApiFormat, + /// Anthropic-to-backend model name mapping. pub model_mapping: ModelMapping, + /// Optional mTLS and custom CA configuration. pub tls: TlsConfig, + /// How to authenticate to this backend (Bearer token or Google API key). pub backend_auth: BackendAuth, + /// Whether to log request/response bodies at debug level. pub log_bodies: bool, /// Strip `stream_options` from streaming requests. Needed for local LLMs /// (older Ollama, text-generation-webui, LM Studio) that reject unknown @@ -304,11 +317,15 @@ pub struct BackendConfig { pub omit_stream_options: bool, } -/// Top-level multi-backend configuration. +/// Top-level multi-backend configuration loaded from TOML. +/// Enables routing requests to different backends by route prefix. #[derive(Debug, Clone)] pub struct MultiConfig { + /// Port the proxy listens on (default: 3000). pub listen_port: u16, + /// Whether to log request/response bodies at debug level (global default). pub log_bodies: bool, + /// Backend name used when no route prefix matches. pub default_backend: String, /// Ordered map: key = route prefix (e.g. "openai"), value = backend config. pub backends: IndexMap, diff --git a/crates/proxy/src/metrics/mod.rs b/crates/proxy/src/metrics/mod.rs index 9148abb..14a0663 100644 --- a/crates/proxy/src/metrics/mod.rs +++ b/crates/proxy/src/metrics/mod.rs @@ -1,5 +1,4 @@ // Request metrics: count, latency, error rates -// PLAN.md lines 867-870 use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -19,24 +18,30 @@ struct MetricsInner { } impl Metrics { + /// Create a new zero-valued metrics counter. pub fn new() -> Self { Self::default() } // Relaxed ordering: these are independent counters with no cross-counter // invariants, so no synchronization is needed. Relaxed is fastest. + + /// Increment the total request counter. Called once per proxied request. pub fn record_request(&self) { self.inner.requests_total.fetch_add(1, Ordering::Relaxed); } + /// Increment the success counter (backend returned 2xx). pub fn record_success(&self) { self.inner.requests_success.fetch_add(1, Ordering::Relaxed); } + /// Increment the error counter (backend returned non-2xx or transport failure). pub fn record_error(&self) { self.inner.requests_error.fetch_add(1, Ordering::Relaxed); } + /// Take a point-in-time snapshot of all counters for the GET /metrics endpoint. pub fn snapshot(&self) -> MetricsSnapshot { MetricsSnapshot { requests_total: self.inner.requests_total.load(Ordering::Relaxed), @@ -46,10 +51,14 @@ impl Metrics { } } +/// Point-in-time snapshot of counters, serialized as JSON for GET /metrics. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct MetricsSnapshot { + /// Total proxied requests (success + error + in-flight). pub requests_total: u64, + /// Requests where the backend returned a 2xx status. pub requests_success: u64, + /// Requests that failed (non-2xx status or transport error). pub requests_error: u64, } diff --git a/crates/proxy/src/server/middleware.rs b/crates/proxy/src/server/middleware.rs index 4efd711..b984085 100644 --- a/crates/proxy/src/server/middleware.rs +++ b/crates/proxy/src/server/middleware.rs @@ -1,8 +1,7 @@ // Auth, logging, and request size limit middleware -// PLAN.md lines 890-893 -use anthropic_openai_translate::anthropic; -use anthropic_openai_translate::mapping::errors_map::create_anthropic_error; +use anyllm_translate::anthropic; +use anyllm_translate::mapping::errors_map::create_anthropic_error; use axum::{ body::Body, http::{HeaderMap, Request, StatusCode}, @@ -39,7 +38,9 @@ static ALLOWED_KEY_HASHES: LazyLock> = LazyLock::new(|| { ); } } - keys.iter().map(|k| Sha256::digest(k.as_bytes()).into()).collect() + keys.iter() + .map(|k| Sha256::digest(k.as_bytes()).into()) + .collect() }); /// Whether open-relay mode is explicitly enabled via PROXY_OPEN_RELAY=true. diff --git a/crates/proxy/src/server/sse.rs b/crates/proxy/src/server/sse.rs index c2a7b23..11fe150 100644 --- a/crates/proxy/src/server/sse.rs +++ b/crates/proxy/src/server/sse.rs @@ -1,7 +1,6 @@ // SSE responder helpers for Anthropic-format streaming -// PLAN.md lines 127-131 -use anthropic_openai_translate::anthropic::streaming::StreamEvent; +use anyllm_translate::anthropic::streaming::StreamEvent; use axum::response::sse::Event; /// Format a StreamEvent as an axum SSE Event with the correct Anthropic event type name. @@ -28,8 +27,8 @@ pub fn stream_event_to_sse(event: &StreamEvent) -> Result OpenAI message mapping -// PLAN.md lines 765-793, 964-977 use crate::anthropic; use crate::mapping::{streaming_map, tools_map, usage_map}; @@ -2066,7 +2065,10 @@ mod tests { fn o_series_model_gets_only_max_completion_tokens() { let req = make_request("o1-mini", Some("You are helpful.")); let oai = anthropic_to_openai_request(&req); - assert!(oai.max_tokens.is_none(), "o-series should not set max_tokens"); + assert!( + oai.max_tokens.is_none(), + "o-series should not set max_tokens" + ); assert_eq!(oai.max_completion_tokens, Some(1024)); // System role should be converted to Developer for o-series. assert_eq!(oai.messages[0].role, openai::ChatRole::Developer); @@ -2087,7 +2089,10 @@ mod tests { let mut req = make_request("o3-mini", None); req.temperature = Some(0.7); let oai = anthropic_to_openai_request(&req); - assert!(oai.temperature.is_none(), "o-series should strip temperature"); + assert!( + oai.temperature.is_none(), + "o-series should strip temperature" + ); } #[test] diff --git a/crates/translator/src/mapping/streaming_map.rs b/crates/translator/src/mapping/streaming_map.rs index 998f409..26bc08b 100644 --- a/crates/translator/src/mapping/streaming_map.rs +++ b/crates/translator/src/mapping/streaming_map.rs @@ -1,5 +1,4 @@ // Streaming state machine: OpenAI chunks -> Anthropic SSE events -// PLAN.md lines 123-151, 387-432, 796-807 use crate::anthropic; use crate::openai; diff --git a/crates/translator/src/mapping/tools_map.rs b/crates/translator/src/mapping/tools_map.rs index ec28112..e07ef56 100644 --- a/crates/translator/src/mapping/tools_map.rs +++ b/crates/translator/src/mapping/tools_map.rs @@ -1,5 +1,4 @@ // Tool definition and tool_choice mapping -// PLAN.md lines 770-776 use crate::anthropic; use crate::openai; diff --git a/crates/translator/src/mapping/usage_map.rs b/crates/translator/src/mapping/usage_map.rs index 8ea13ba..f38b82f 100644 --- a/crates/translator/src/mapping/usage_map.rs +++ b/crates/translator/src/mapping/usage_map.rs @@ -1,5 +1,4 @@ // Usage field mapping between Anthropic and OpenAI -// PLAN.md lines 786-792 use crate::anthropic; use crate::openai; diff --git a/crates/translator/src/middleware/client.rs b/crates/translator/src/middleware/client.rs index 4e0db29..a8cf50a 100644 --- a/crates/translator/src/middleware/client.rs +++ b/crates/translator/src/middleware/client.rs @@ -39,6 +39,8 @@ pub struct ForwardingClient { } impl ForwardingClient { + /// Create a client targeting `{backend_url}/v1/chat/completions`. + /// No retry logic; callers can add their own Tower retry layer. pub fn new(backend_url: &str, api_key: &str) -> Self { let base = backend_url.trim_end_matches('/'); Self { diff --git a/crates/translator/src/middleware/mod.rs b/crates/translator/src/middleware/mod.rs index 7a10070..2e34a23 100644 --- a/crates/translator/src/middleware/mod.rs +++ b/crates/translator/src/middleware/mod.rs @@ -1,12 +1,12 @@ //! Axum middleware for adding Anthropic Messages API compatibility to existing services. //! -//! Requires the `middleware` feature: `anthropic_openai_translate = { features = ["middleware"] }` +//! Requires the `middleware` feature: `anyllm_translate = { features = ["middleware"] }` //! //! # Usage //! //! ```rust,no_run -//! use anthropic_openai_translate::TranslationConfig; -//! use anthropic_openai_translate::middleware::{ +//! use anyllm_translate::TranslationConfig; +//! use anyllm_translate::middleware::{ //! AnthropicCompatConfig, AnthropicTranslationLayer, anthropic_compat_router, //! }; //! use axum::Router; @@ -66,6 +66,7 @@ pub struct AnthropicCompatConfig { } impl AnthropicCompatConfig { + /// Create a builder for configuring the middleware. pub fn builder() -> AnthropicCompatConfigBuilder { AnthropicCompatConfigBuilder { backend_url: String::new(), @@ -83,21 +84,25 @@ pub struct AnthropicCompatConfigBuilder { } impl AnthropicCompatConfigBuilder { + /// Set the base URL of the OpenAI-compatible backend (e.g., `https://api.openai.com`). pub fn backend_url(mut self, url: impl Into) -> Self { self.backend_url = url.into(); self } + /// Set the API key sent as a Bearer token to the backend. pub fn api_key(mut self, key: impl Into) -> Self { self.api_key = key.into(); self } + /// Set translation settings (model mapping, lossy behavior). pub fn translation(mut self, config: TranslationConfig) -> Self { self.translation = config; self } + /// Build the configuration. Does not validate; invalid URLs will fail at request time. pub fn build(self) -> AnthropicCompatConfig { AnthropicCompatConfig { backend_url: self.backend_url, @@ -152,6 +157,7 @@ pub struct AnthropicTranslationLayer { } impl AnthropicTranslationLayer { + /// Create a new layer that will intercept `POST /v1/messages` and translate. pub fn new(config: AnthropicCompatConfig) -> Self { Self { state: make_state(config), diff --git a/crates/translator/src/openai/chat_completions.rs b/crates/translator/src/openai/chat_completions.rs index be63402..92b230d 100644 --- a/crates/translator/src/openai/chat_completions.rs +++ b/crates/translator/src/openai/chat_completions.rs @@ -1,5 +1,4 @@ // OpenAI Chat Completions request/response types -// PLAN.md lines 78-87, 700-725 use serde::{Deserialize, Serialize}; diff --git a/crates/translator/src/openai/errors.rs b/crates/translator/src/openai/errors.rs index 386e855..d82f437 100644 --- a/crates/translator/src/openai/errors.rs +++ b/crates/translator/src/openai/errors.rs @@ -1,5 +1,4 @@ // OpenAI error types and rate limit headers -// PLAN.md lines 165-171 use serde::{Deserialize, Serialize}; diff --git a/crates/translator/src/openai/responses.rs b/crates/translator/src/openai/responses.rs index 91a4e0f..8256fbb 100644 --- a/crates/translator/src/openai/responses.rs +++ b/crates/translator/src/openai/responses.rs @@ -1,5 +1,4 @@ // OpenAI Responses API request/response types -// PLAN.md lines 89-94 use serde::{Deserialize, Serialize}; diff --git a/crates/translator/src/openai/streaming.rs b/crates/translator/src/openai/streaming.rs index 3a98735..28e3851 100644 --- a/crates/translator/src/openai/streaming.rs +++ b/crates/translator/src/openai/streaming.rs @@ -1,5 +1,4 @@ // OpenAI SSE streaming types (ChatCompletions chunks + Responses events) -// PLAN.md lines 138-146 use serde::{Deserialize, Serialize}; diff --git a/crates/translator/src/util/ids.rs b/crates/translator/src/util/ids.rs index cb224a0..c804650 100644 --- a/crates/translator/src/util/ids.rs +++ b/crates/translator/src/util/ids.rs @@ -1,5 +1,7 @@ // ID generation utilities for Anthropic-format identifiers. -// Uses UUID v4 (simple/no-hyphen format) with a domain prefix. +// Uses UUID v4 (simple/no-hyphen format) with a domain prefix to match the +// {prefix}_{hex} pattern that Anthropic SDKs and clients expect when parsing +// response IDs. See: https://docs.anthropic.com/en/api/messages /// Generate a message ID in Anthropic format (msg_ prefix + uuid v4 without hyphens). pub fn generate_message_id() -> String {