From e02e5b186e61f05482eeb2c630824b0dfe1dbdfa Mon Sep 17 00:00:00 2001 From: zyycn Date: Sun, 2 Aug 2026 01:00:42 +0800 Subject: [PATCH] refactor(store,api): split hotspot modules per boundary audit Split the four oversized files into mod.rs-style module directories without changing public ports or behavior: - gateway-store: observability/{model,mapping,admin_adapter,queries/*}, provider_accounts/{rows,repository,mapping,admin_adapter,core_adapter}, root {config,bundle,workers,admin_adapter,value} - gateway-api: admin/observability/{query,wire,routes,presenter} with mirrored contract tests and updated frozen architecture manifest Co-authored-by: Codex --- .../gateway-api/src/admin/observability.rs | 2455 ------------ .../src/admin/observability/mod.rs | 32 + .../src/admin/observability/presenter.rs | 1039 +++++ .../src/admin/observability/query.rs | 463 +++ .../src/admin/observability/routes.rs | 213 + .../src/admin/observability/wire.rs | 742 ++++ .../tests/admin/observability/mod.rs | 4 + .../tests/admin/observability/query.rs | 64 + .../response.rs} | 439 +-- .../crates/gateway-api/tests/architecture.rs | 10 +- .../crates/gateway-store/src/admin_adapter.rs | 296 ++ backend/crates/gateway-store/src/bundle.rs | 211 + backend/crates/gateway-store/src/config.rs | 141 + backend/crates/gateway-store/src/lib.rs | 1094 +----- .../src/postgres/observability.rs | 3475 ----------------- .../postgres/observability/admin_adapter.rs | 322 ++ .../src/postgres/observability/mapping.rs | 862 ++++ .../src/postgres/observability/mod.rs | 29 + .../src/postgres/observability/model.rs | 713 ++++ .../observability/queries/accounts.rs | 336 ++ .../observability/queries/dashboard.rs | 558 +++ .../src/postgres/observability/queries/mod.rs | 11 + .../src/postgres/observability/queries/ops.rs | 171 + .../postgres/observability/queries/usage.rs | 549 +++ .../src/postgres/provider_accounts.rs | 2619 ------------- .../provider_accounts/admin_adapter.rs | 663 ++++ .../provider_accounts/core_adapter.rs | 293 ++ .../src/postgres/provider_accounts/mapping.rs | 392 ++ .../src/postgres/provider_accounts/mod.rs | 69 + .../postgres/provider_accounts/repository.rs | 734 ++++ .../src/postgres/provider_accounts/rows.rs | 524 +++ backend/crates/gateway-store/src/value.rs | 262 ++ backend/crates/gateway-store/src/workers.rs | 195 + 33 files changed, 10093 insertions(+), 9887 deletions(-) delete mode 100644 backend/crates/gateway-api/src/admin/observability.rs create mode 100644 backend/crates/gateway-api/src/admin/observability/mod.rs create mode 100644 backend/crates/gateway-api/src/admin/observability/presenter.rs create mode 100644 backend/crates/gateway-api/src/admin/observability/query.rs create mode 100644 backend/crates/gateway-api/src/admin/observability/routes.rs create mode 100644 backend/crates/gateway-api/src/admin/observability/wire.rs create mode 100644 backend/crates/gateway-api/tests/admin/observability/mod.rs create mode 100644 backend/crates/gateway-api/tests/admin/observability/query.rs rename backend/crates/gateway-api/tests/admin/{observability.rs => observability/response.rs} (71%) create mode 100644 backend/crates/gateway-store/src/admin_adapter.rs create mode 100644 backend/crates/gateway-store/src/bundle.rs create mode 100644 backend/crates/gateway-store/src/config.rs delete mode 100644 backend/crates/gateway-store/src/postgres/observability.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/mapping.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/mod.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/model.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/queries/mod.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/queries/ops.rs create mode 100644 backend/crates/gateway-store/src/postgres/observability/queries/usage.rs delete mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts.rs create mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs create mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs create mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs create mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs create mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs create mode 100644 backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs create mode 100644 backend/crates/gateway-store/src/value.rs create mode 100644 backend/crates/gateway-store/src/workers.rs diff --git a/backend/crates/gateway-api/src/admin/observability.rs b/backend/crates/gateway-api/src/admin/observability.rs deleted file mode 100644 index 1ac8b4936..000000000 --- a/backend/crates/gateway-api/src/admin/observability.rs +++ /dev/null @@ -1,2455 +0,0 @@ -//! Dashboard、用量与诊断查询的 wire 映射和固定路由。 - -use std::{collections::BTreeMap, num::NonZeroU32}; - -use axum::{ - Router, - extract::{Query, State}, - http::StatusCode, - response::IntoResponse, - routing::get, -}; -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use chrono::{DateTime, Duration, Utc}; -use gateway_admin::model::{PageSize as DomainPageSize, observability as domain}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use super::presenter::{format_compact_number, format_number}; -use super::{ - AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta, - WireValidationError, wire::map_admin_service_error, -}; - -/// 观测列表默认页大小。 -pub const DEFAULT_PAGE_SIZE: u16 = 50; -/// 观测列表允许的最大页大小。 -pub const MAX_PAGE_SIZE: u16 = 100; -/// 游标编码在 HTTP 层解码前允许的最大长度。 -pub const MAX_CURSOR_BYTES: usize = 1024; - -/// Dashboard 查询参数。 -#[derive(Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DashboardQuery { - pub kind: Option, - pub start_time: Option, - pub end_time: Option, -} - -impl DashboardQuery { - /// 解析 dashboard 趋势类型。 - pub fn trend_kind(&self) -> Result { - TrendKind::parse(self.kind.as_deref()) - } -} - -/// 逻辑请求列表、汇总和洞察查询参数。 -#[derive(Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct UsageQuery { - pub page: Option, - pub page_size: Option, - pub cursor: Option, - pub kind: Option, - pub outcome: Option, - pub client_api_key_id: Option, - pub provider: Option, - pub request_id: Option, - pub account_id: Option, - pub route: Option, - pub model: Option, - pub status_code: Option, - pub transport: Option, - pub attempt_index: Option, - pub response_id: Option, - pub upstream_request_id: Option, - pub search: Option, - pub start_time: Option, - pub end_time: Option, -} - -impl UsageQuery { - /// 校验分页字段，不改变仓储层的分页 owner。 - pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> { - let page = self.page.unwrap_or(1); - if page == 0 { - return Err(WireValidationError::new("page")); - } - let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE); - if page_size == 0 || page_size > MAX_PAGE_SIZE { - return Err(WireValidationError::new("pageSize")); - } - Ok((page, page_size)) - } - - /// 校验游标的 wire 边界；编码和排序语义由应用层负责。 - pub fn validate_cursor(&self) -> Result<(), WireValidationError> { - validate_cursor(self.cursor.as_deref()) - } -} - -/// 详情查询参数。 -#[derive(Clone, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DetailQuery { - pub id: String, -} - -impl DetailQuery { - /// 校验详情 ID，错误不回显输入值。 - pub fn validate(&self) -> Result<(), WireValidationError> { - require_text(&self.id, "id") - } -} - -/// 诊断聚合查询参数。 -#[derive(Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DiagnosticsQuery { - pub dimension: Option, - pub start_time: Option, - pub end_time: Option, - pub provider: Option, - pub model: Option, - pub status_code: Option, - pub search: Option, -} - -impl DiagnosticsQuery { - /// 解析诊断维度。 - pub fn dimension(&self) -> Result { - DiagnosticDimension::parse(self.dimension.as_deref()) - } -} - -/// 运维错误查询参数。 -#[derive(Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct OpsQuery { - pub page: Option, - pub page_size: Option, - pub cursor: Option, - pub kind: Option, - pub client_api_key_id: Option, - pub provider: Option, - pub request_id: Option, - pub account_id: Option, - pub route: Option, - pub model: Option, - pub status_code: Option, - pub client_status_code: Option, - pub upstream_status_code: Option, - pub transport: Option, - pub attempt_index: Option, - pub failure_class: Option, - pub response_id: Option, - pub upstream_request_id: Option, - pub search: Option, - pub start_time: Option, - pub end_time: Option, -} - -impl OpsQuery { - /// 校验分页字段。 - pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> { - let page = self.page.unwrap_or(1); - if page == 0 { - return Err(WireValidationError::new("page")); - } - let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE); - if page_size == 0 || page_size > MAX_PAGE_SIZE { - return Err(WireValidationError::new("pageSize")); - } - Ok((page, page_size)) - } - - /// 校验游标的 wire 边界。 - pub fn validate_cursor(&self) -> Result<(), WireValidationError> { - validate_cursor(self.cursor.as_deref()) - } -} - -/// Dashboard 趋势类型。 -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] -#[serde(rename_all = "lowercase")] -pub enum TrendKind { - Usage, - Latency, - Errors, -} - -impl TrendKind { - /// 从 query 值解析趋势类型。 - pub fn parse(value: Option<&str>) -> Result { - match trimmed(value) { - None | Some("usage") => Ok(Self::Usage), - Some("latency") => Ok(Self::Latency), - Some("errors") => Ok(Self::Errors), - Some(_) => Err(WireValidationError::new("kind")), - } - } -} - -/// 诊断聚合维度。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DiagnosticDimension { - Model, - Account, - ApiKey, - Provider, - Transport, - Failure, - Status, -} - -impl DiagnosticDimension { - /// 从 query 值解析诊断维度。 - pub fn parse(value: Option<&str>) -> Result { - match trimmed(value) { - None | Some("model") => Ok(Self::Model), - Some("account") => Ok(Self::Account), - Some("apiKey" | "api_key") => Ok(Self::ApiKey), - Some("provider") => Ok(Self::Provider), - Some("transport") => Ok(Self::Transport), - Some("failureClass" | "failure_class") => Ok(Self::Failure), - Some("status") => Ok(Self::Status), - Some(_) => Err(WireValidationError::new("dimension")), - } - } - - /// 返回终态响应中的稳定维度名称。 - #[must_use] - pub const fn display_name(self) -> &'static str { - match self { - Self::Model => "model", - Self::Account => "account", - Self::ApiKey => "apiKey", - Self::Provider => "provider", - Self::Transport => "transport", - Self::Failure => "failureClass", - Self::Status => "status", - } - } -} - -/// 解析 RFC3339 时间；错误不回显原始值。 -pub fn parse_datetime(value: Option<&str>) -> Result>, WireValidationError> { - let Some(value) = trimmed(value) else { - return Ok(None); - }; - DateTime::parse_from_rfc3339(value) - .map(|value| Some(value.with_timezone(&Utc))) - .map_err(|_| WireValidationError::new("timeRange")) -} - -/// 解析 HTTP 状态码。 -pub fn parse_status(value: Option) -> Result, WireValidationError> { - value - .map(|value| { - u16::try_from(value) - .ok() - .filter(|value| (100..=599).contains(value)) - .ok_or_else(|| WireValidationError::new("statusCode")) - }) - .transpose() -} - -/// 解析尝试序号。 -pub fn parse_attempt_index(value: Option) -> Result, WireValidationError> { - value - .map(|value| { - u32::try_from(value) - .ok() - .filter(|value| *value > 0 && *value <= i32::MAX as u32) - .ok_or_else(|| WireValidationError::new("attemptIndex")) - }) - .transpose() -} - -fn require_text(value: &str, field: &'static str) -> Result<(), WireValidationError> { - if value.trim().is_empty() { - Err(WireValidationError::new(field)) - } else { - Ok(()) - } -} - -fn validate_cursor(value: Option<&str>) -> Result<(), WireValidationError> { - if value.is_some_and(|value| value.is_empty() || value.len() > MAX_CURSOR_BYTES) { - return Err(WireValidationError::new("cursor")); - } - Ok(()) -} - -fn trimmed(value: Option<&str>) -> Option<&str> { - value.map(str::trim).filter(|value| !value.is_empty()) -} - -fn domain_trend_kind(kind: TrendKind) -> domain::TrendKind { - match kind { - TrendKind::Usage => domain::TrendKind::Usage, - TrendKind::Latency => domain::TrendKind::Latency, - TrendKind::Errors => domain::TrendKind::Errors, - } -} - -fn domain_diagnostic_dimension(dimension: DiagnosticDimension) -> domain::DiagnosticDimension { - match dimension { - DiagnosticDimension::Model => domain::DiagnosticDimension::Model, - DiagnosticDimension::Account => domain::DiagnosticDimension::Account, - DiagnosticDimension::ApiKey => domain::DiagnosticDimension::ApiKey, - DiagnosticDimension::Provider => domain::DiagnosticDimension::Provider, - DiagnosticDimension::Transport => domain::DiagnosticDimension::Transport, - DiagnosticDimension::Failure => domain::DiagnosticDimension::Failure, - DiagnosticDimension::Status => domain::DiagnosticDimension::Status, - } -} - -fn request_outcome( - value: Option, -) -> Result, WireValidationError> { - non_empty(value) - .map(|value| { - domain::RequestOutcome::new(value).map_err(|_| WireValidationError::new("outcome")) - }) - .transpose() -} - -fn usage_range( - start: Option<&str>, - end: Option<&str>, -) -> Result { - let end = parse_datetime(end)?.unwrap_or_else(Utc::now); - let start = parse_datetime(start)?.unwrap_or(end - Duration::days(7)); - domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange")) -} - -fn dashboard_range( - start: Option<&str>, - end: Option<&str>, -) -> Result { - let end = parse_datetime(end)?.unwrap_or_else(Utc::now); - let start = - parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1)); - domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange")) -} - -fn dashboard_today_range( - start: Option<&str>, - end: Option<&str>, -) -> Result { - let end = parse_datetime(end)?.unwrap_or_else(Utc::now); - let start = parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end)); - domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange")) -} - -fn usage_filter(query: &UsageQuery) -> Result { - let outcome = query.outcome.clone().or_else(|| { - non_empty(query.kind.clone()).filter(|value| { - matches!( - value.as_str(), - "running" | "succeeded" | "failed" | "cancelled" | "incomplete" - ) - }) - }); - Ok(domain::UsageFilter { - client_api_key_ref: non_empty(query.client_api_key_id.clone()), - request_id: non_empty(query.request_id.clone()), - provider_account_ref: non_empty(query.account_id.clone()), - operation: non_empty(query.route.clone()), - provider_kind: non_empty(query.provider.clone()), - model: non_empty(query.model.clone()), - outcome: request_outcome(outcome)?, - status_code: parse_status(query.status_code)?, - transport: non_empty(query.transport.clone()), - attempt_index: parse_attempt_index(query.attempt_index)?, - response_id: non_empty(query.response_id.clone()), - upstream_request_id: non_empty(query.upstream_request_id.clone()), - search: non_empty(query.search.clone()), - }) -} - -fn usage_command( - query: &UsageQuery, -) -> Result<(domain::UsageQuery, u32, u16), WireValidationError> { - let (page, page_size) = query.validate_page()?; - query.validate_cursor()?; - let page_number = NonZeroU32::new(page) - .map(domain::PageNumber::new) - .ok_or_else(|| WireValidationError::new("page"))?; - let page_size_value = - DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?; - Ok(( - domain::UsageQuery { - range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?, - filter: usage_filter(query)?, - cursor: decode_observability_cursor(query.cursor.as_deref())?, - page: page_number, - page_size: page_size_value, - }, - page, - page_size, - )) -} - -fn ops_command(query: &OpsQuery) -> Result<(domain::OpsErrorQuery, u32, u16), WireValidationError> { - let (page, page_size) = query.validate_page()?; - query.validate_cursor()?; - let page_number = NonZeroU32::new(page) - .map(domain::PageNumber::new) - .ok_or_else(|| WireValidationError::new("page"))?; - let page_size_value = - DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?; - let status_code = parse_status( - query - .upstream_status_code - .or(query.client_status_code) - .or(query.status_code), - )?; - Ok(( - domain::OpsErrorQuery { - range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?, - filter: domain::OpsErrorFilter { - client_api_key_ref: non_empty(query.client_api_key_id.clone()), - request_id: non_empty(query.request_id.clone()), - provider_kind: non_empty(query.provider.clone()), - provider_account_ref: non_empty(query.account_id.clone()), - operation: non_empty(query.route.clone()).or_else(|| non_empty(query.kind.clone())), - model: non_empty(query.model.clone()), - transport: non_empty(query.transport.clone()), - attempt_index: parse_attempt_index(query.attempt_index)?, - response_id: non_empty(query.response_id.clone()), - upstream_request_id: non_empty(query.upstream_request_id.clone()), - failure_kind: non_empty(query.failure_class.clone()), - status_code, - search: non_empty(query.search.clone()), - }, - cursor: decode_observability_cursor(query.cursor.as_deref())?, - page: page_number, - page_size: page_size_value, - }, - page, - page_size, - )) -} - -fn decode_observability_cursor( - value: Option<&str>, -) -> Result, WireValidationError> { - value - .map(|encoded| { - if encoded.is_empty() || encoded.len() > MAX_CURSOR_BYTES { - return Err(WireValidationError::new("cursor")); - } - let bytes = URL_SAFE_NO_PAD - .decode(encoded) - .map_err(|_| WireValidationError::new("cursor"))?; - let wire: CursorWire = - serde_json::from_slice(&bytes).map_err(|_| WireValidationError::new("cursor"))?; - if wire.stable_id.trim().is_empty() { - return Err(WireValidationError::new("cursor")); - } - Ok(domain::ObservabilityCursor { - observed_at: wire.observed_at, - stable_id: wire.stable_id, - }) - }) - .transpose() -} - -fn encode_observability_cursor( - cursor: &domain::ObservabilityCursor, -) -> Result { - let bytes = serde_json::to_vec(&CursorWire { - observed_at: cursor.observed_at, - stable_id: cursor.stable_id.clone(), - }) - .map_err(|_| WireValidationError::new("cursor"))?; - Ok(URL_SAFE_NO_PAD.encode(bytes)) -} - -fn non_empty(value: Option) -> Option { - value - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) -} - -/// 观测列表游标的稳定 wire 形状。 -#[derive(Clone, Serialize, serde::Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct CursorWire { - pub observed_at: DateTime, - pub stable_id: String, -} - -/// 观测列表响应数据。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct PageData { - pub items: Vec, - pub page: PageMeta, - pub next_cursor: Option, -} - -/// Token 详情展示。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TokenDetailsView { - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub reasoning_tokens: Option, - pub image_input_tokens: Option, - pub image_output_tokens: Option, - pub total_tokens: Option, - pub input_tokens_display: String, - pub output_tokens_display: String, - pub cached_tokens_display: String, - pub cache_write_tokens_display: String, - pub reasoning_tokens_display: String, - pub image_input_tokens_display: String, - pub image_output_tokens_display: String, - pub total_tokens_display: String, -} - -/// 按货币展示的成本。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct CostView { - pub currency: String, - pub estimated_amount: String, -} - -/// 成本覆盖状态计数。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct CostCoverageView { - pub known: u64, - pub partial: u64, - pub unknown: u64, - pub not_billable: u64, -} - -/// Provider 受控价格规则生成的单次请求费用明细展示。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct BillingView { - pub input_amount_display: String, - pub output_amount_display: String, - pub cache_read_amount_display: String, - pub cache_write_amount_display: String, - pub standard_amount_display: String, - pub total_amount_display: String, - pub input_price_display: String, - pub output_price_display: String, - pub cache_read_price_display: String, - pub cache_write_price_display: String, - pub service_tier_display: String, - pub multiplier_display: String, -} - -/// 单条逻辑请求展示。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageRecordView { - pub id: String, - pub request_id: String, - pub client_api_key_id: Option, - pub kind: String, - pub provider: Option, - pub authentication_kind: Option, - pub account_id: Option, - pub account_email: Option, - pub account_name: Option, - pub route: String, - pub model: String, - pub requested_model: Option, - pub upstream_model: Option, - pub service_tier: Option, - pub status_code: Option, - pub transport: Option, - pub attempt_index: Option, - pub attempt_count: u64, - pub response_id: Option, - pub upstream_request_id: Option, - pub latency_ms: Option, - pub first_token_ms: Option, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub reasoning_tokens: Option, - pub image_input_tokens: Option, - pub image_output_tokens: Option, - pub message: String, - pub metadata: UsageRecordMetadataView, - pub created_at: DateTime, - pub created_at_display: String, - pub client_ip: Option, - pub user_agent: Option, - pub reasoning_effort: Option, - pub reasoning_preset: Option, - pub compact: Option, - pub request_kind: Option, - pub subagent_kind: Option, - pub token_details: TokenDetailsView, - pub billing: Option, - pub costs: Vec, - pub cost_coverage: CostCoverageView, - pub first_token_latency_ms: Option, - pub first_token_latency_ms_display: String, - pub latency_ms_display: String, - pub logical_outcome: String, -} - -/// 逻辑请求安全元数据；不承载原始请求体或上游错误正文。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageRecordMetadataView { - pub protocol: String, - pub logical_outcome: String, - pub attempt_count: u64, - pub requested_model: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub upstream_model: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub client_ip: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_agent: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub reasoning_preset: Option, - pub compact: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub request_kind: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub subagent_kind: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub transport: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub http_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub client_status_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub upstream_status_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub response_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub upstream_request_id: Option, - pub websocket_pool: Option, - pub image_generation_requested: bool, - pub image_generation_succeeded: Option, - pub latency_details: UsageLatencyDetailsView, - /// Provider 安全观测直接展开，避免 API 层理解任何 Provider 专属字段。 - #[serde(flatten)] - pub provider_metadata: BTreeMap, -} - -/// WebSocket 池决策的稳定形状；与 Provider 选择逻辑无关。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct WebSocketPoolMetadataView { - pub kind: String, -} - -/// 逻辑请求在上游和输出阶段测得的时延事实。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageLatencyDetailsView { - #[serde(skip_serializing_if = "Option::is_none")] - pub transport_decision_wait_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_connect_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub upstream_headers_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub first_event_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub first_reasoning_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub first_text_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub first_token_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub openai_processing_ms: Option, -} - -/// 单次上游尝试展示。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageAttemptView { - pub id: String, - pub attempt_index: u32, - pub trigger: String, - pub provider: String, - pub model: String, - pub transport: String, - pub send_state: String, - pub outcome: String, - pub downstream_committed: bool, - pub status_code: Option, - pub provider_error_code: Option, - pub failure_class: Option, - pub cost_estimate_status: String, - pub estimated_cost_amount: Option, - pub estimated_cost_currency: Option, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub total_tokens: Option, - pub first_token_ms: Option, - pub latency_ms: Option, - pub credential_name: Option, - pub account_id: Option, - pub account_name: Option, - pub account_email: Option, - pub authentication_kind: Option, - pub started_at: DateTime, - pub completed_at: Option>, -} - -/// 逻辑请求详情与其尝试列表。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageRecordDetailView { - #[serde(flatten)] - pub request: UsageRecordView, - pub attempts: Vec, - /// 尝试列表是否完整；best-effort 下恒为 false。 - pub attempts_complete: bool, -} - -/// Dashboard 趋势数据。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TrendData { - pub kind: TrendKind, - pub points: Vec, - pub summary: Vec, -} - -/// Dashboard 单个趋势桶。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TrendPointView { - pub time: String, - pub bucket: DateTime, - pub label: String, - pub requests: String, - pub requests_value: u64, - pub input_tokens: String, - pub input_tokens_value: u64, - pub output_tokens: String, - pub output_tokens_value: u64, - pub cached_tokens: String, - pub cached_tokens_value: u64, - pub cache_hit_rate_value: f64, - pub tokens_value: u64, - pub errors: String, - pub errors_value: u64, - pub latency: String, - pub latency_value: Option, - pub max_latency: String, - pub max_latency_value: Option, - pub min_latency: String, - pub min_latency_value: Option, - pub success_rate: String, - pub success_rate_value: Option, -} - -/// Dashboard 趋势摘要。 -#[derive(Debug, Serialize)] -pub struct TrendSummaryView { - pub label: String, - pub value: String, - pub ratio: Option, -} - -/// Dashboard 卡片集合。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardCardsView { - pub credentials: DashboardCredentialsCardView, - pub traffic: DashboardTrafficCardView, - pub tokens: DashboardTokensCardView, - pub cache: DashboardCacheCardView, -} - -/// Dashboard 上游凭据卡片。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardCredentialsCardView { - pub total: String, - pub total_value: u64, - pub available: String, - pub available_value: u64, - pub unavailable: String, - pub unavailable_value: u64, -} - -/// Dashboard 流量卡片。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardTrafficCardView { - pub today_requests: String, - pub today_requests_value: u64, - pub yesterday_requests_value: u64, - pub total_requests: String, -} - -/// Dashboard token 卡片。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardTokensCardView { - pub today_tokens: String, - pub today_tokens_value: u64, - pub yesterday_tokens_value: u64, - pub total_tokens: String, - pub total_billing_amount_usd: String, -} - -/// Dashboard 缓存卡片。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardCacheCardView { - pub today_hit_rate: String, - pub today_hit_rate_value: Option, - pub yesterday_hit_rate_value: Option, - pub total_hit_rate: String, - pub total_cached_tokens: String, - pub average_first_token_latency_ms: String, -} - -/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardAccountUsageView { - pub id: String, - pub provider: String, - pub authentication_kind: String, - pub email: String, - pub plan_type: Option, - pub tokens: String, - pub request_count: u64, - pub request_buckets: Vec, - pub quota_used_percent: Option, - pub last_used: String, -} - -/// Dashboard 账号单小时请求数。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardAccountRequestBucketView { - pub bucket_start: DateTime, - pub request_count: u64, -} - -/// Provider 账号池的持久事实汇总。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardPoolSummaryView { - pub total: u64, - pub active: u64, - pub expired: u64, - pub quota_exhausted: u64, - pub refreshing: Option, - pub disabled: u64, - pub banned: u64, -} - -/// 同 target 账号调度容量；Redis 未提供聚合事实时运行中槽位保持空值。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardCapacityInfoView { - pub max_concurrent_per_account: u64, - pub total_slots: u64, - pub used_slots: Option, - pub available_slots: Option, -} - -/// 逻辑请求指标展示。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct RequestMetricsView { - pub request_count: u64, - pub success_count: u64, - pub failure_count: u64, - pub cancelled_count: u64, - pub incomplete_count: u64, - pub caller_error_count: u64, - pub input_tokens: u64, - pub output_tokens: u64, - pub cached_tokens: u64, - pub cache_write_tokens: u64, - pub reasoning_tokens: u64, - pub total_tokens: u64, -} - -/// 上游尝试指标展示。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AttemptMetricsView { - pub attempt_count: u64, - pub success_count: u64, - pub failure_count: u64, - pub cancelled_count: u64, - pub incomplete_count: u64, - pub rate_limited_count: u64, - pub auth_failure_count: u64, - pub provider5xx_count: u64, - pub cost_coverage: CostCoverageView, - pub costs: Vec, -} - -/// 健康时间线单点。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct HealthTimelinePointView { - pub time: String, - pub status: String, - pub reliability_display: String, - pub success_requests: u64, - pub failed_requests: u64, - pub cancelled_requests: u64, - pub incomplete_requests: u64, - pub caller_error_requests: u64, -} - -/// Dashboard 健康时间线。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct HealthTimelineView { - pub title: String, - pub description: String, - pub reliability_display: String, - pub status: String, - pub success_requests: u64, - pub failed_requests: u64, - pub cancelled_requests: u64, - pub incomplete_requests: u64, - pub caller_error_requests: u64, - pub points: Vec, -} - -/// Dashboard 展示的实际 Provider 上游请求身份。 -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardWireProfileView { - pub provider: String, - pub product: String, - pub version: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub build: Option, - pub target: DashboardWireTargetView, - pub user_agent: String, - pub attributes: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub verified_at: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub release: Option, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardWireAttributeView { - pub label: String, - pub value: String, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardWireTargetView { - pub os_type: String, - pub os_version: String, - pub arch: String, - pub terminal: String, -} - -/// 发布检查与启动画像分离；未检查时使用明确的 `unchecked` 状态。 -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardDesktopReleaseView { - pub status: DashboardDesktopReleaseStatusView, - #[serde(skip_serializing_if = "Option::is_none")] - pub checked_at: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub latest_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub latest_build: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -/// Dashboard 发布检查的稳定 wire 状态。 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum DashboardDesktopReleaseStatusView { - Unchecked, - Aligned, - ReviewRequired, - CheckFailed, -} - -impl From for DashboardDesktopReleaseStatusView { - fn from(status: domain::DesktopReleaseStatus) -> Self { - match status { - domain::DesktopReleaseStatus::Unchecked => Self::Unchecked, - domain::DesktopReleaseStatus::Current => Self::Aligned, - domain::DesktopReleaseStatus::UpdateAvailable => Self::ReviewRequired, - domain::DesktopReleaseStatus::Failed => Self::CheckFailed, - } - } -} - -/// Dashboard 汇总响应数据。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DashboardDataView { - pub cards: DashboardCardsView, - pub trend: TrendData, - pub health_timeline: HealthTimelineView, - pub wire_profiles: Vec, - pub account_usage: Vec, - pub usage_records: Vec, - pub pool_summary: DashboardPoolSummaryView, - pub capacity_info: DashboardCapacityInfoView, - pub rotation_strategy: String, -} - -/// 用量汇总响应数据。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageSummaryView { - pub total_requests: String, - pub input_tokens: String, - pub output_tokens: String, - pub cached_tokens: String, - pub cache_write_tokens: String, - pub total_tokens: String, - pub average_latency_ms: String, - pub logical_requests: RequestMetricsView, - pub attempts: AttemptMetricsView, -} - -/// 洞察健康趋势点。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OverviewHealthPointView { - pub bucket: DateTime, - pub label: String, - pub success_requests: u64, - pub failed_requests: u64, - pub cancelled_requests: u64, - pub incomplete_requests: u64, - pub caller_error_requests: u64, - pub error_rate: f64, -} - -/// 洞察健康摘要。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OverviewHealthView { - pub total_requests: u64, - pub success_requests: u64, - pub failed_requests: u64, - pub cancelled_requests: u64, - pub incomplete_requests: u64, - pub caller_error_requests: u64, - pub success_rate: f64, - pub request_change_rate: Option, - pub success_rate_change: Option, - pub points: Vec, -} - -/// 洞察性能趋势点。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OverviewPerformancePointView { - pub bucket: DateTime, - pub label: String, - pub latency_p50_ms: Option, - pub latency_p95_ms: Option, - pub latency_p99_ms: Option, - pub first_token_p50_ms: Option, - pub first_token_p95_ms: Option, - pub first_token_p99_ms: Option, -} - -/// 洞察性能摘要。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OverviewPerformanceView { - pub latency_p50_ms: Option, - pub latency_p95_ms: Option, - pub latency_p99_ms: Option, - pub first_token_p50_ms: Option, - pub first_token_p95_ms: Option, - pub first_token_p99_ms: Option, - pub latency_coverage: f64, - pub first_token_coverage: f64, - pub points: Vec, -} - -/// 洞察成本趋势点。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OverviewCostPointView { - pub bucket: DateTime, - pub label: String, - pub input_tokens: u64, - pub output_tokens: u64, - pub cached_tokens: u64, - pub total_tokens: u64, - pub estimated_cost: Option, - pub standard_cost: Option, - pub cached_token_rate: f64, - pub cache_hit_request_rate: Option, -} - -/// 洞察成本摘要。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OverviewCostView { - pub estimated_cost: Option, - pub standard_cost: Option, - pub cost_per_request: Option, - pub tokens_per_request: f64, - pub cached_token_rate: f64, - pub cache_hit_request_rate: Option, - pub input_tokens: u64, - pub output_tokens: u64, - pub cached_tokens: u64, - pub total_tokens: u64, - pub points: Vec, - pub costs: Vec, - pub coverage: CostCoverageView, -} - -/// Provider 维度洞察摘要。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct ProviderOverviewView { - pub provider: String, - pub request_count: u64, - pub attempt_count: u64, - pub failure_count: u64, - pub total_tokens: u64, -} - -/// 用量洞察总响应。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UsageInsightsOverviewView { - pub granularity: String, - pub health: OverviewHealthView, - pub performance: OverviewPerformanceView, - pub cost: OverviewCostView, - pub attempts: AttemptMetricsView, - pub providers: Vec, -} - -/// 诊断聚合项目。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DiagnosticItemView { - pub key: String, - pub name: String, - pub request_count: u64, - pub success_count: u64, - pub error_count: u64, - pub error_rate: f64, - pub request_share: f64, - pub average_latency_ms: Option, - pub latency_p95_ms: Option, - pub estimated_cost: Option, - pub attempt_count: u64, - pub total_tokens: u64, -} - -/// 诊断聚合响应。 -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct DiagnosticsView { - pub dimension: String, - pub items: Vec, -} - -/// 运维错误项目。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OpsErrorView { - pub id: String, - pub request_id: Option, - pub client_api_key_id: Option, - pub kind: String, - pub provider: Option, - pub authentication_kind: Option, - pub account_id: Option, - pub route: String, - pub model: Option, - pub client_status_code: Option, - pub upstream_status_code: Option, - pub transport: Option, - pub attempt_index: Option, - pub failure_class: String, - pub response_id: Option, - pub upstream_request_id: Option, - pub latency_ms: Option, - pub message: String, - pub metadata: OpsErrorMetadataView, - pub created_at: DateTime, - pub created_at_display: String, -} - -/// 运维错误安全元数据；只保留可查询的标识，不回显秘密材料。 -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct OpsErrorMetadataView { - pub source: String, - pub component: String, - pub attempt_id: Option, - pub account_label: Option, -} - -pub fn router() -> Router -where - S: AdminSessionState + Clone + Send + Sync + 'static, -{ - Router::new() - .route("/api/admin/dashboard/summary", get(dashboard_summary::)) - .route("/api/admin/dashboard/trend", get(dashboard_trend::)) - .route("/api/admin/usage/records", get(usage_records::)) - .route( - "/api/admin/usage/records/detail", - get(usage_record_detail::), - ) - .route( - "/api/admin/usage/records/summary", - get(usage_records_summary::), - ) - .route( - "/api/admin/usage/insights/overview", - get(usage_insights_overview::), - ) - .route( - "/api/admin/usage/insights/diagnostics", - get(usage_insights_diagnostics::), - ) - .route("/api/admin/operations/errors", get(ops_errors::)) -} - -async fn dashboard_summary( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let kind = query.trend_kind().map_err(map_wire_error)?; - let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref()) - .map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .dashboard_summary(range, domain_trend_kind(kind)) - .await - .map_err(map_service_error)?; - Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(dashboard_view(result, kind)), - )) -} - -async fn dashboard_trend( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let kind = query.trend_kind().map_err(map_wire_error)?; - let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref()) - .map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .dashboard_trend(range, domain_trend_kind(kind)) - .await - .map_err(map_service_error)?; - Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(trend_view(result, kind)), - )) -} - -async fn usage_records( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let (command, page, page_size) = usage_command(&query).map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .usage_records(command) - .await - .map_err(map_service_error)?; - let data = usage_page_view(result, page, page_size) - .map_err(|_| AdminError::internal("Failed to encode observability cursor"))?; - Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data))) -} - -async fn usage_record_detail( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - query.validate().map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .usage_record_detail(query.id.trim()) - .await - .map_err(map_service_error)?; - Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(usage_detail_view(result)), - )) -} - -async fn usage_records_summary( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref()) - .map_err(map_wire_error)?; - let filter = usage_filter(&query).map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .usage_summary(range, filter) - .await - .map_err(map_service_error)?; - Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(usage_summary_view(result)), - )) -} - -async fn usage_insights_overview( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref()) - .map_err(map_wire_error)?; - let filter = usage_filter(&query).map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .usage_insights(range, filter) - .await - .map_err(map_service_error)?; - Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(usage_insights_view(result)), - )) -} - -async fn usage_insights_diagnostics( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let dimension = query.dimension().map_err(map_wire_error)?; - let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref()) - .map_err(map_wire_error)?; - let filter = domain::UsageFilter { - provider_kind: non_empty(query.provider), - model: non_empty(query.model), - status_code: parse_status(query.status_code).map_err(map_wire_error)?, - search: non_empty(query.search), - ..domain::UsageFilter::default() - }; - let result = state - .admin_services() - .observability() - .diagnostics(range, filter, domain_diagnostic_dimension(dimension)) - .await - .map_err(map_service_error)?; - Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(diagnostics_view(result, dimension)), - )) -} - -async fn ops_errors( - _auth: AdminAuth, - State(state): State, - Query(query): Query, -) -> Result -where - S: AdminSessionState + Send + Sync, -{ - let (command, page, page_size) = ops_command(&query).map_err(map_wire_error)?; - let result = state - .admin_services() - .observability() - .ops_errors(command) - .await - .map_err(map_service_error)?; - let data = ops_page_view(result, page, page_size) - .map_err(|_| AdminError::internal("Failed to encode observability cursor"))?; - Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data))) -} - -fn display_duration(value: Option) -> String { - let Some(value) = value.and_then(|value| i64::try_from(value).ok()) else { - return "—".to_owned(); - }; - if value < 1_000 { - format!("{value} ms") - } else if value < 60_000 { - format!("{:.2} s", value as f64 / 1_000.0) - } else if value < 3_600_000 { - format!("{:.1} min", value as f64 / 60_000.0) - } else { - format!("{:.1} h", value as f64 / 3_600_000.0) - } -} - -fn display_rate(value: f64) -> String { - if value.is_finite() { - format!("{:.1}%", value * 100.0) - } else { - "—".to_owned() - } -} - -fn china_datetime(value: &DateTime) -> String { - (*value + Duration::hours(8)) - .format("%Y-%m-%d %H:%M:%S") - .to_string() -} - -fn china_label(value: DateTime, format: &str) -> String { - (value + Duration::hours(8)).format(format).to_string() -} - -fn outcome_name(outcome: &domain::RequestOutcome) -> &str { - outcome.as_str() -} - -fn request_metrics_view(metrics: &domain::RequestMetrics) -> RequestMetricsView { - RequestMetricsView { - request_count: metrics.request_count, - success_count: metrics.success_count, - failure_count: metrics.failure_count, - cancelled_count: metrics.cancelled_count, - incomplete_count: metrics.incomplete_count, - caller_error_count: metrics.caller_error_count, - input_tokens: metrics.input_tokens, - output_tokens: metrics.output_tokens, - cached_tokens: metrics.cached_tokens, - cache_write_tokens: metrics.cache_write_tokens, - reasoning_tokens: metrics.reasoning_tokens, - total_tokens: metrics.total_tokens, - } -} - -fn cost_coverage_view(coverage: &domain::CostCoverage) -> CostCoverageView { - CostCoverageView { - known: coverage.known_count(), - partial: coverage.partial_count, - unknown: coverage.unavailable_count, - not_billable: coverage.not_billable_count, - } -} - -fn cost_views(costs: &[domain::CurrencyCost]) -> Vec { - costs - .iter() - .map(|cost| CostView { - currency: cost.currency.clone(), - estimated_amount: cost.amount.as_str().to_owned(), - }) - .collect() -} - -fn attempt_metrics_view(metrics: &domain::AttemptMetrics) -> AttemptMetricsView { - AttemptMetricsView { - attempt_count: metrics.attempt_count, - success_count: metrics.success_count, - failure_count: metrics.failure_count, - cancelled_count: metrics.cancelled_count, - incomplete_count: metrics.incomplete_count, - rate_limited_count: metrics.rate_limited_count, - auth_failure_count: metrics.auth_failure_count, - provider5xx_count: metrics.provider_5xx_count, - cost_coverage: cost_coverage_view(&metrics.cost_coverage), - costs: cost_views(&metrics.costs), - } -} - -fn token_details(record: &domain::UsageRecord) -> TokenDetailsView { - TokenDetailsView { - input_tokens: record.input_tokens, - output_tokens: record.output_tokens, - cached_tokens: record.cached_tokens, - cache_write_tokens: record.cache_write_tokens, - reasoning_tokens: record.reasoning_tokens, - image_input_tokens: record.image_input_tokens, - image_output_tokens: record.image_output_tokens, - total_tokens: record.total_tokens, - input_tokens_display: record - .input_tokens - .map_or_else(|| "-".to_owned(), format_number), - output_tokens_display: record - .output_tokens - .map_or_else(|| "-".to_owned(), format_number), - cached_tokens_display: record - .cached_tokens - .map_or_else(|| "-".to_owned(), format_compact_number), - cache_write_tokens_display: record - .cache_write_tokens - .map_or_else(|| "-".to_owned(), format_compact_number), - reasoning_tokens_display: record - .reasoning_tokens - .map_or_else(|| "-".to_owned(), format_number), - image_input_tokens_display: record - .image_input_tokens - .map_or_else(|| "-".to_owned(), format_number), - image_output_tokens_display: record - .image_output_tokens - .map_or_else(|| "-".to_owned(), format_number), - total_tokens_display: record - .total_tokens - .map_or_else(|| "-".to_owned(), format_number), - } -} - -fn format_decimal_currency(amount: &str, currency: &str) -> String { - if currency == "USD" { - let value = amount.parse::().map_or(0.0, |value| value); - let precision = if value != 0.0 && value.abs() < 0.01 { - 4 - } else { - 2 - }; - format!("${value:.precision$}") - } else { - format!("{currency} {amount}") - } -} - -fn format_money(cost: &domain::CurrencyCost) -> String { - format_decimal_currency(cost.amount.as_str(), &cost.currency) -} - -fn format_token_price(cost: &domain::CurrencyCost) -> String { - if cost.currency != "USD" { - return format!("{} {} / 1M Token", cost.currency, cost.amount.as_str()); - } - format!("${} / 1M Token", cost.amount.as_str()) -} - -fn format_service_tier(service_tier: Option<&str>) -> String { - match service_tier { - Some("priority" | "fast") => "Fast".to_owned(), - Some("flex") => "Flex".to_owned(), - Some("default") | None => "Default".to_owned(), - Some(other) => other.to_owned(), - } -} - -fn billing_view(billing: Option<&domain::UsageBilling>) -> Option { - match billing? { - domain::UsageBilling::Total { total, .. } => Some(BillingView { - input_amount_display: "—".to_owned(), - output_amount_display: "—".to_owned(), - cache_read_amount_display: "—".to_owned(), - cache_write_amount_display: "—".to_owned(), - standard_amount_display: "—".to_owned(), - total_amount_display: format_money(total), - input_price_display: "—".to_owned(), - output_price_display: "—".to_owned(), - cache_read_price_display: "—".to_owned(), - cache_write_price_display: "—".to_owned(), - service_tier_display: "—".to_owned(), - multiplier_display: "—".to_owned(), - }), - domain::UsageBilling::Calculated(value) => Some(BillingView { - input_amount_display: format_money(&value.input_amount), - output_amount_display: format_money(&value.output_amount), - cache_read_amount_display: format_money(&value.cache_read_amount), - cache_write_amount_display: format_money(&value.cache_write_amount), - standard_amount_display: format_money(&value.standard_amount), - total_amount_display: format_money(&value.total_amount), - input_price_display: format_token_price(&value.input_price_per_million), - output_price_display: format_token_price(&value.output_price_per_million), - cache_read_price_display: format_token_price(&value.cache_read_price_per_million), - cache_write_price_display: format_token_price(&value.cache_write_price_per_million), - service_tier_display: format_service_tier(value.service_tier.as_deref()), - multiplier_display: format!("{:.2}x", f64::from(value.multiplier_percent) / 100.0), - }), - } -} - -fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView { - let tokens = token_details(&record); - let billing = billing_view(record.billing.as_ref()); - let costs = record - .cost_amount - .as_ref() - .zip(record.cost_currency.as_ref()) - .map(|(amount, currency)| { - vec![CostView { - currency: currency.clone(), - estimated_amount: amount.as_str().to_owned(), - }] - }) - .unwrap_or_default(); - let status_code = record - .client_status_code - .or(record.upstream_status_code) - .map(i64::from); - let outcome = outcome_name(&record.outcome).to_owned(); - let message = record - .error_message - .clone() - .unwrap_or_else(|| outcome.clone()); - let first_token_display = display_duration(record.first_token_ms); - let latency_display = display_duration(record.latency_ms); - let cost_coverage = match record.cost_source.as_str() { - "provider_reported" | "calculated" => CostCoverageView { - known: 1, - partial: 0, - unknown: 0, - not_billable: 0, - }, - _ => CostCoverageView { - known: 0, - partial: 0, - unknown: 1, - not_billable: 0, - }, - }; - let model = record - .upstream_model_id - .clone() - .unwrap_or_else(|| record.requested_model_id.clone()); - let transport = record - .upstream_transport - .clone() - .or_else(|| Some(record.client_transport.clone())); - let metadata = UsageRecordMetadataView { - protocol: record.protocol.clone(), - logical_outcome: outcome.clone(), - attempt_count: u64::from(record.attempt_count), - requested_model: record.requested_model_id.clone(), - upstream_model: record.upstream_model_id.clone(), - client_ip: record.client_ip.clone(), - user_agent: record.user_agent.clone(), - reasoning_effort: record.reasoning_effort.clone(), - reasoning_preset: record.reasoning_preset.clone(), - compact: record.compact, - request_kind: record.request_kind.clone(), - subagent_kind: record.subagent_kind.clone(), - transport: transport.clone(), - http_version: record.http_version.clone(), - client_status_code: record.client_status_code.map(i64::from), - upstream_status_code: record.upstream_status_code.map(i64::from), - response_id: record.client_response_id.clone(), - upstream_request_id: record.upstream_request_id.clone(), - websocket_pool: record - .websocket_pool - .clone() - .map(|kind| WebSocketPoolMetadataView { kind }), - image_generation_requested: record.image_generation_requested, - image_generation_succeeded: record.image_generation_succeeded, - latency_details: UsageLatencyDetailsView { - transport_decision_wait_ms: record.transport_decision_wait_ms, - ws_connect_ms: record.connect_ms, - upstream_headers_ms: record.headers_ms, - first_event_ms: record.first_event_ms, - first_reasoning_ms: record.first_reasoning_ms, - first_text_ms: record.first_text_ms, - first_token_ms: record.first_token_ms, - openai_processing_ms: record.provider_processing_ms, - }, - provider_metadata: provider_metadata_fields(record.provider_metadata_json.as_deref()), - }; - UsageRecordView { - id: record.id.clone(), - request_id: record.id, - client_api_key_id: Some(record.client_api_key_ref), - kind: record.operation, - provider: record.provider_kind, - authentication_kind: record.provider_account_authentication_kind, - account_id: record.provider_account_ref, - account_email: record.provider_account_email, - account_name: record.provider_account_name, - route: record.endpoint, - model, - requested_model: Some(record.requested_model_id), - upstream_model: record.upstream_model_id, - service_tier: record.service_tier, - status_code, - transport, - attempt_index: None, - attempt_count: u64::from(record.attempt_count), - response_id: record.client_response_id, - upstream_request_id: record.upstream_request_id, - latency_ms: record.latency_ms, - first_token_ms: record.first_token_ms, - input_tokens: record.input_tokens, - output_tokens: record.output_tokens, - cached_tokens: record.cached_tokens, - cache_write_tokens: record.cache_write_tokens, - reasoning_tokens: record.reasoning_tokens, - image_input_tokens: record.image_input_tokens, - image_output_tokens: record.image_output_tokens, - message, - metadata, - created_at: record.started_at, - created_at_display: china_datetime(&record.started_at), - client_ip: record.client_ip, - user_agent: record.user_agent, - reasoning_effort: record.reasoning_effort, - reasoning_preset: record.reasoning_preset, - compact: Some(record.compact), - request_kind: record.request_kind, - subagent_kind: record.subagent_kind, - token_details: tokens, - billing, - costs, - cost_coverage, - first_token_latency_ms: record.first_token_ms, - first_token_latency_ms_display: first_token_display, - latency_ms_display: latency_display, - logical_outcome: outcome, - } -} - -fn provider_metadata_fields(value: Option<&str>) -> BTreeMap { - const CORE_METADATA_FIELDS: &[&str] = &[ - "protocol", - "logicalOutcome", - "attemptCount", - "requestedModel", - "upstreamModel", - "clientIp", - "userAgent", - "reasoningEffort", - "reasoningPreset", - "compact", - "requestKind", - "subagentKind", - "transport", - "httpVersion", - "clientStatusCode", - "upstreamStatusCode", - "responseId", - "upstreamRequestId", - "websocketPool", - "imageGenerationRequested", - "imageGenerationSucceeded", - "latencyDetails", - ]; - let Some(Ok(Value::Object(fields))) = value.map(serde_json::from_str::) else { - return BTreeMap::new(); - }; - fields - .into_iter() - .filter(|(field, _)| !CORE_METADATA_FIELDS.contains(&field.as_str())) - .collect() -} - -fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptView { - let occurred_at = attempt.occurred_at; - let credential_name = attempt - .provider_account_name - .as_ref() - .or(attempt.provider_account_ref.as_ref()) - .cloned(); - UsageAttemptView { - id: attempt.id, - attempt_index: attempt.attempt_index, - trigger: attempt.source, - provider: attempt - .provider_kind - .unwrap_or_else(|| "unknown".to_owned()), - model: attempt - .upstream_model_id - .unwrap_or_else(|| "unknown".to_owned()), - transport: attempt - .upstream_transport - .unwrap_or_else(|| "unknown".to_owned()), - send_state: attempt - .upstream_send_state - .unwrap_or_else(|| "unknown".to_owned()), - outcome: outcome_name(&attempt.outcome).to_owned(), - downstream_committed: attempt.downstream_committed, - status_code: attempt.status_code, - provider_error_code: attempt.provider_error_code, - failure_class: attempt.failure_kind, - cost_estimate_status: attempt - .cost_source - .clone() - .unwrap_or_else(|| "unavailable".to_owned()), - estimated_cost_amount: attempt.cost_amount.map(|amount| amount.as_str().to_owned()), - estimated_cost_currency: attempt.cost_currency, - input_tokens: attempt.input_tokens, - output_tokens: attempt.output_tokens, - cached_tokens: attempt.cached_tokens, - total_tokens: attempt.total_tokens, - first_token_ms: None, - latency_ms: attempt.latency_ms, - credential_name, - account_id: attempt.provider_account_ref, - account_name: attempt.provider_account_name, - account_email: attempt.provider_account_email, - authentication_kind: attempt.provider_account_authentication_kind, - started_at: occurred_at, - completed_at: Some(occurred_at), - } -} - -fn usage_detail_view(detail: domain::UsageDetail) -> UsageRecordDetailView { - UsageRecordDetailView { - request: usage_record_view(detail.request), - attempts: detail - .attempts - .into_iter() - .map(usage_attempt_view) - .collect(), - attempts_complete: false, - } -} - -fn page_meta(page: u32, page_size: u16, total: u64) -> PageMeta { - let page_size_u64 = u64::from(page_size); - let total_pages = total.saturating_add(page_size_u64 - 1) / page_size_u64; - PageMeta::new( - page, - u32::from(page_size), - total, - u32::try_from(total_pages).unwrap_or(u32::MAX), - ) -} - -fn usage_page_view( - page: domain::UsagePage, - page_number: u32, - page_size: u16, -) -> Result, WireValidationError> { - Ok(PageData { - items: page.items.into_iter().map(usage_record_view).collect(), - page: page_meta(page_number, page_size, page.total), - next_cursor: page - .next_cursor - .as_ref() - .map(encode_observability_cursor) - .transpose()?, - }) -} - -fn ops_error_view(error: domain::OpsError) -> OpsErrorView { - let account_label = error - .provider_account_email - .as_ref() - .or(error.provider_account_name.as_ref()) - .or(error.provider_account_ref.as_ref()) - .cloned(); - OpsErrorView { - id: error.event_id, - request_id: error.request_id, - client_api_key_id: error.client_api_key_ref, - kind: error.component.clone(), - provider: error.provider_kind, - authentication_kind: error.provider_account_authentication_kind, - account_id: error.provider_account_ref, - route: error.endpoint.unwrap_or_else(|| "—".to_owned()), - model: error.upstream_model_id, - client_status_code: error.client_status_code.map(i64::from), - upstream_status_code: error.upstream_status_code.map(i64::from), - transport: error.upstream_transport, - attempt_index: error.attempt_index, - failure_class: error.failure_kind, - response_id: error.client_response_id, - upstream_request_id: error.upstream_request_id, - latency_ms: error.latency_ms, - message: error.message, - metadata: OpsErrorMetadataView { - source: error.source, - component: error.component, - attempt_id: None, - account_label, - }, - created_at: error.occurred_at, - created_at_display: china_datetime(&error.occurred_at), - } -} - -fn ops_page_view( - page: domain::OpsErrorPage, - page_number: u32, - page_size: u16, -) -> Result, WireValidationError> { - Ok(PageData { - items: page.items.into_iter().map(ops_error_view).collect(), - page: page_meta(page_number, page_size, page.total), - next_cursor: page - .next_cursor - .as_ref() - .map(encode_observability_cursor) - .transpose()?, - }) -} - -fn trend_point_view(point: domain::TrendPoint) -> TrendPointView { - let local_time = china_label(point.bucket_start, "%H:%M"); - let label = china_label(point.bucket_start, "%m-%d %H:%M"); - let success_rate_value = point.success_rate.map(|value| value * 100.0); - TrendPointView { - time: local_time, - bucket: point.bucket_start, - label, - requests: format_compact_number(point.metrics.request_count), - requests_value: point.metrics.request_count, - input_tokens: format_compact_number(point.metrics.input_tokens), - input_tokens_value: point.metrics.input_tokens, - output_tokens: format_compact_number(point.metrics.output_tokens), - output_tokens_value: point.metrics.output_tokens, - cached_tokens: format_compact_number(point.metrics.cached_tokens), - cached_tokens_value: point.metrics.cached_tokens, - cache_hit_rate_value: point.cached_token_rate, - tokens_value: point.metrics.total_tokens, - errors: format_compact_number(point.service_failure_count), - errors_value: point.service_failure_count, - latency: display_duration(point.average_latency_ms), - latency_value: point.average_latency_ms, - max_latency: display_duration(point.metrics.max_latency_ms), - max_latency_value: point.metrics.max_latency_ms, - min_latency: display_duration(point.metrics.min_latency_ms), - min_latency_value: point.metrics.min_latency_ms, - success_rate: success_rate_value - .map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")), - success_rate_value, - } -} - -fn trend_summary_view(kind: TrendKind, summary: &domain::TrendSummary) -> Vec { - match kind { - TrendKind::Usage => vec![ - TrendSummaryView { - label: "输入".to_owned(), - value: format_compact_number(summary.input_tokens), - ratio: None, - }, - TrendSummaryView { - label: "输出".to_owned(), - value: format_compact_number(summary.output_tokens), - ratio: None, - }, - TrendSummaryView { - label: "缓存".to_owned(), - value: format_compact_number(summary.cached_tokens), - ratio: None, - }, - ], - TrendKind::Latency => vec![ - TrendSummaryView { - label: "平均".to_owned(), - value: display_duration(summary.average_latency_ms), - ratio: None, - }, - TrendSummaryView { - label: "最高".to_owned(), - value: display_duration(summary.max_latency_ms), - ratio: None, - }, - TrendSummaryView { - label: "最低".to_owned(), - value: display_duration(summary.min_latency_ms), - ratio: None, - }, - ], - TrendKind::Errors => vec![ - TrendSummaryView { - label: "错误数".to_owned(), - value: format_compact_number(summary.service_failure_count), - ratio: None, - }, - TrendSummaryView { - label: "成功率".to_owned(), - value: "—".to_owned(), - ratio: summary - .success_rate - .map(|value| format!("{:.1}%", value * 100.0)), - }, - TrendSummaryView { - label: "总请求".to_owned(), - value: format_compact_number(summary.request_count), - ratio: None, - }, - ], - } -} - -fn trend_view(trend: domain::Trend, kind: TrendKind) -> TrendData { - TrendData { - kind, - summary: trend_summary_view(kind, &trend.summary), - points: trend.points.into_iter().map(trend_point_view).collect(), - } -} - -fn health_status_name(status: domain::HealthStatus) -> &'static str { - match status { - domain::HealthStatus::Future => "future", - domain::HealthStatus::NoData => "no_data", - domain::HealthStatus::Unavailable => "unavailable", - domain::HealthStatus::LowSample => "low_sample", - domain::HealthStatus::Unstable => "unstable", - domain::HealthStatus::Stable => "stable", - } -} - -fn reliability_display(value: Option) -> String { - value.map_or_else(|| "-".to_owned(), |value| format!("{value:.1}%")) -} - -fn health_timeline_view(timeline: domain::HealthTimeline) -> HealthTimelineView { - HealthTimelineView { - title: "请求健康时间线".to_owned(), - description: "有效请求可用性".to_owned(), - reliability_display: reliability_display(timeline.reliability_percent), - status: health_status_name(timeline.status).to_owned(), - success_requests: timeline.success_requests, - failed_requests: timeline.failed_requests, - cancelled_requests: timeline.cancelled_requests, - incomplete_requests: timeline.incomplete_requests, - caller_error_requests: timeline.caller_error_requests, - points: timeline - .points - .into_iter() - .enumerate() - .map(|(index, point)| { - let elapsed_minutes = i64::try_from(index).unwrap_or(i64::MAX).saturating_mul(15); - HealthTimelinePointView { - time: format!("{:02}:{:02}", elapsed_minutes / 60, elapsed_minutes % 60), - status: health_status_name(point.status).to_owned(), - reliability_display: reliability_display(point.reliability_percent), - success_requests: point.success_requests, - failed_requests: point.failed_requests, - cancelled_requests: point.cancelled_requests, - incomplete_requests: point.incomplete_requests, - caller_error_requests: point.caller_error_requests, - } - }) - .collect(), - } -} - -fn wire_profile_view(profile: domain::DashboardWireProfile) -> DashboardWireProfileView { - DashboardWireProfileView { - provider: profile.provider, - product: profile.product, - version: profile.version, - build: profile.build, - target: DashboardWireTargetView { - os_type: profile.target.os_type, - os_version: profile.target.os_version, - arch: profile.target.arch, - terminal: profile.target.terminal, - }, - user_agent: profile.user_agent, - attributes: profile - .attributes - .into_iter() - .map(|attribute| DashboardWireAttributeView { - label: attribute.label, - value: attribute.value, - }) - .collect(), - verified_at: profile.verified_at, - release: profile.release.map(|release| DashboardDesktopReleaseView { - status: release.status.into(), - checked_at: release.checked_at, - latest_version: release.latest_version, - latest_build: release.latest_build, - error: release.error, - }), - } -} - -fn relative_time(value: Option>, now: DateTime) -> String { - let Some(value) = value else { - return "从未使用".to_owned(); - }; - let elapsed = now.signed_duration_since(value); - if elapsed.num_seconds() < 0 { - return china_datetime(&value); - } - if elapsed.num_seconds() < 60 { - return "刚刚".to_owned(); - } - if elapsed.num_minutes() < 60 { - return format!("{} 分钟前", elapsed.num_minutes()); - } - if elapsed.num_hours() < 24 { - return format!("{} 小时前", elapsed.num_hours()); - } - format!("{} 天前", elapsed.num_days()) -} - -fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> DashboardDataView { - let domain::DashboardResult { - observation, - today, - yesterday, - total_billing_usd, - total_cached_token_rate, - average_first_token_latency_ms, - trend, - health_timeline, - wire_profiles, - capacity, - rotation_strategy, - } = result; - let domain::DashboardObservation { - range, - requests, - provider_accounts, - account_usage, - recent_requests, - .. - } = observation; - let mut account_usage_views = Vec::with_capacity(account_usage.len()); - for credential in account_usage { - account_usage_views.push(DashboardAccountUsageView { - id: credential.account_id.clone(), - provider: credential.provider_kind.clone(), - authentication_kind: credential.authentication_kind.clone(), - email: credential - .email - .clone() - .unwrap_or_else(|| credential.name.clone()), - plan_type: credential.plan_type.clone(), - tokens: credential - .total_tokens - .map_or_else(|| "—".to_owned(), format_compact_number), - request_count: credential.request_count, - request_buckets: credential - .request_buckets - .iter() - .map(|bucket| DashboardAccountRequestBucketView { - bucket_start: bucket.bucket_start, - request_count: bucket.request_count, - }) - .collect(), - quota_used_percent: credential.quota_used_percent, - last_used: relative_time(credential.last_used_at, range.end), - }); - } - DashboardDataView { - cards: DashboardCardsView { - credentials: DashboardCredentialsCardView { - total: format_compact_number(provider_accounts.total), - total_value: provider_accounts.total, - available: format_compact_number(provider_accounts.active), - available_value: provider_accounts.active, - unavailable: format_compact_number(provider_accounts.unavailable), - unavailable_value: provider_accounts.unavailable, - }, - traffic: DashboardTrafficCardView { - today_requests: format_compact_number(today.request_count), - today_requests_value: today.request_count, - yesterday_requests_value: yesterday.request_count, - total_requests: format_compact_number(requests.request_count), - }, - tokens: DashboardTokensCardView { - today_tokens: format_compact_number(today.total_tokens), - today_tokens_value: today.total_tokens, - yesterday_tokens_value: yesterday.total_tokens, - total_tokens: format_compact_number(requests.total_tokens), - total_billing_amount_usd: total_billing_usd - .as_ref() - .map_or_else(|| "—".to_owned(), |amount| format!("${}", amount.as_str())), - }, - cache: DashboardCacheCardView { - today_hit_rate: display_rate(today.cached_token_rate), - today_hit_rate_value: today.observed_cached_token_rate, - yesterday_hit_rate_value: yesterday.observed_cached_token_rate, - total_hit_rate: display_rate(total_cached_token_rate), - total_cached_tokens: format_compact_number(requests.cached_tokens), - average_first_token_latency_ms: display_duration(average_first_token_latency_ms), - }, - }, - trend: trend_view(trend, kind), - health_timeline: health_timeline_view(health_timeline), - wire_profiles: wire_profiles.into_iter().map(wire_profile_view).collect(), - account_usage: account_usage_views, - usage_records: recent_requests.into_iter().map(usage_record_view).collect(), - pool_summary: DashboardPoolSummaryView { - total: provider_accounts.total, - active: provider_accounts.active, - expired: provider_accounts.expired, - quota_exhausted: provider_accounts.quota_exhausted, - refreshing: provider_accounts.refreshing, - disabled: provider_accounts.disabled, - banned: provider_accounts.banned, - }, - capacity_info: DashboardCapacityInfoView { - max_concurrent_per_account: capacity.max_concurrent_per_account, - total_slots: capacity.total_slots, - used_slots: capacity.used_slots, - available_slots: capacity.available_slots, - }, - rotation_strategy: rotation_strategy.as_str().to_owned(), - } -} - -fn usage_summary_view(summary: domain::UsageSummary) -> UsageSummaryView { - let overview = summary.overview; - UsageSummaryView { - total_requests: format_compact_number(overview.requests.request_count), - input_tokens: format_compact_number(overview.requests.input_tokens), - output_tokens: format_compact_number(overview.requests.output_tokens), - cached_tokens: format_compact_number(overview.requests.cached_tokens), - cache_write_tokens: format_compact_number(overview.requests.cache_write_tokens), - total_tokens: format_compact_number(overview.requests.total_tokens), - average_latency_ms: display_duration(summary.average_latency_ms), - logical_requests: request_metrics_view(&overview.requests), - attempts: attempt_metrics_view(&overview.attempts), - } -} - -fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsightsOverviewView { - let health_points = insights - .health - .points - .iter() - .map(|point| OverviewHealthPointView { - bucket: point.bucket_start, - label: china_label(point.bucket_start, "%m-%d %H:%M"), - success_requests: point.success_requests, - failed_requests: point.failed_requests, - cancelled_requests: point.cancelled_requests, - incomplete_requests: point.incomplete_requests, - caller_error_requests: point.caller_error_requests, - error_rate: point.error_rate, - }) - .collect(); - let performance_points = insights - .performance - .points - .iter() - .map(|point| OverviewPerformancePointView { - bucket: point.bucket_start, - label: china_label(point.bucket_start, "%m-%d %H:%M"), - latency_p50_ms: point.latency_percentiles.p50_ms.map(|value| value.as_f64()), - latency_p95_ms: point.latency_percentiles.p95_ms.map(|value| value.as_f64()), - latency_p99_ms: point.latency_percentiles.p99_ms.map(|value| value.as_f64()), - first_token_p50_ms: point - .first_token_latency_percentiles - .p50_ms - .map(|value| value.as_f64()), - first_token_p95_ms: point - .first_token_latency_percentiles - .p95_ms - .map(|value| value.as_f64()), - first_token_p99_ms: point - .first_token_latency_percentiles - .p99_ms - .map(|value| value.as_f64()), - }) - .collect(); - let cost_points = insights - .cost - .points - .iter() - .map(|point| OverviewCostPointView { - bucket: point.bucket_start, - label: china_label(point.bucket_start, "%m-%d %H:%M"), - input_tokens: point.input_tokens, - output_tokens: point.output_tokens, - cached_tokens: point.cached_tokens, - total_tokens: point.total_tokens, - estimated_cost: point.estimated_cost.as_ref().map(ToString::to_string), - standard_cost: point.standard_cost.as_ref().map(ToString::to_string), - cached_token_rate: point.cached_token_rate, - cache_hit_request_rate: point.cache_hit_request_rate, - }) - .collect(); - UsageInsightsOverviewView { - granularity: granularity_name(insights.granularity).to_owned(), - health: OverviewHealthView { - total_requests: insights.health.total_requests, - success_requests: insights.health.success_requests, - failed_requests: insights.health.failed_requests, - cancelled_requests: insights.health.cancelled_requests, - incomplete_requests: insights.health.incomplete_requests, - caller_error_requests: insights.health.caller_error_requests, - success_rate: insights.health.success_rate, - request_change_rate: None, - success_rate_change: None, - points: health_points, - }, - performance: OverviewPerformanceView { - latency_p50_ms: insights - .performance - .latency_percentiles - .p50_ms - .map(|value| value.as_f64()), - latency_p95_ms: insights - .performance - .latency_percentiles - .p95_ms - .map(|value| value.as_f64()), - latency_p99_ms: insights - .performance - .latency_percentiles - .p99_ms - .map(|value| value.as_f64()), - first_token_p50_ms: insights - .performance - .first_token_latency_percentiles - .p50_ms - .map(|value| value.as_f64()), - first_token_p95_ms: insights - .performance - .first_token_latency_percentiles - .p95_ms - .map(|value| value.as_f64()), - first_token_p99_ms: insights - .performance - .first_token_latency_percentiles - .p99_ms - .map(|value| value.as_f64()), - latency_coverage: insights.performance.latency_coverage, - first_token_coverage: insights.performance.first_token_coverage, - points: performance_points, - }, - cost: OverviewCostView { - estimated_cost: insights - .cost - .estimated_cost - .as_ref() - .map(ToString::to_string), - standard_cost: insights - .cost - .standard_cost - .as_ref() - .map(ToString::to_string), - cost_per_request: insights - .cost - .cost_per_request - .as_ref() - .map(ToString::to_string), - tokens_per_request: insights.cost.tokens_per_request, - cached_token_rate: insights.cost.cached_token_rate, - cache_hit_request_rate: insights.cost.cache_hit_request_rate, - input_tokens: insights.cost.input_tokens, - output_tokens: insights.cost.output_tokens, - cached_tokens: insights.cost.cached_tokens, - total_tokens: insights.cost.total_tokens, - points: cost_points, - costs: cost_views(&insights.cost.costs), - coverage: cost_coverage_view(&insights.cost.coverage), - }, - attempts: attempt_metrics_view(&insights.attempts), - providers: insights - .providers - .into_iter() - .map(|provider| ProviderOverviewView { - provider: provider.provider_kind, - request_count: provider.request_count, - attempt_count: provider.attempt_count, - failure_count: provider.failure_count, - total_tokens: provider.total_tokens, - }) - .collect(), - } -} - -fn granularity_name(granularity: domain::Granularity) -> &'static str { - match granularity { - domain::Granularity::FifteenMinutes => "15m", - domain::Granularity::Hour => "1h", - domain::Granularity::Day => "1d", - } -} - -fn diagnostics_view( - result: domain::DiagnosticsResult, - dimension: DiagnosticDimension, -) -> DiagnosticsView { - DiagnosticsView { - dimension: dimension.display_name().to_owned(), - items: result - .items - .into_iter() - .map(|item| DiagnosticItemView { - key: item.key, - name: if item.name == "__none__" { - "未知".to_owned() - } else { - item.name - }, - request_count: item.request_count, - success_count: item.success_count, - error_count: item.error_count, - error_rate: item.error_rate, - request_share: item.request_share, - average_latency_ms: item.average_latency_ms, - latency_p95_ms: item.latency_p95_ms, - estimated_cost: item.estimated_cost.as_ref().map(ToString::to_string), - attempt_count: item.attempt_count, - total_tokens: item.total_tokens, - }) - .collect(), - } -} - -fn map_wire_error(error: WireValidationError) -> AdminError { - let message = match error.field() { - "timeRange" => "Invalid time range", - "statusCode" => "Status code must be between 100 and 599", - "attemptIndex" => "Attempt index is out of range", - "kind" => "Invalid dashboard trend kind", - "dimension" => "Invalid diagnostics dimension", - "outcome" => "Invalid Observability query", - "id" => "Usage record ID is required", - "page" | "pageSize" => "Invalid Observability query", - "cursor" => "Invalid observability cursor", - _ => "Invalid observability query", - }; - AdminError::bad_request(message) -} - -fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError { - map_admin_service_error(error, "Observability repository unavailable") -} diff --git a/backend/crates/gateway-api/src/admin/observability/mod.rs b/backend/crates/gateway-api/src/admin/observability/mod.rs new file mode 100644 index 000000000..c1cfceec5 --- /dev/null +++ b/backend/crates/gateway-api/src/admin/observability/mod.rs @@ -0,0 +1,32 @@ +//! Dashboard、用量与诊断查询的 wire 映射和固定路由。 + +use std::{collections::BTreeMap, num::NonZeroU32}; + +use axum::{ + Router, + extract::{Query, State}, + http::StatusCode, + response::IntoResponse, + routing::get, +}; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use chrono::{DateTime, Duration, Utc}; +use gateway_admin::model::{PageSize as DomainPageSize, observability as domain}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::presenter::{format_compact_number, format_number}; +use super::{ + AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta, + WireValidationError, wire::map_admin_service_error, +}; + +mod presenter; +mod query; +mod routes; +mod wire; + +pub(crate) use presenter::*; +pub use query::*; +pub use routes::*; +pub use wire::*; diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs new file mode 100644 index 000000000..ebf35ef1f --- /dev/null +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -0,0 +1,1039 @@ +//! 过渡期展示格式化；目标迁移到 Vue presenter。 + +use super::*; + +pub(crate) fn display_duration(value: Option) -> String { + let Some(value) = value.and_then(|value| i64::try_from(value).ok()) else { + return "—".to_owned(); + }; + if value < 1_000 { + format!("{value} ms") + } else if value < 60_000 { + format!("{:.2} s", value as f64 / 1_000.0) + } else if value < 3_600_000 { + format!("{:.1} min", value as f64 / 60_000.0) + } else { + format!("{:.1} h", value as f64 / 3_600_000.0) + } +} + +pub(crate) fn display_rate(value: f64) -> String { + if value.is_finite() { + format!("{:.1}%", value * 100.0) + } else { + "—".to_owned() + } +} + +pub(crate) fn china_datetime(value: &DateTime) -> String { + (*value + Duration::hours(8)) + .format("%Y-%m-%d %H:%M:%S") + .to_string() +} + +pub(crate) fn china_label(value: DateTime, format: &str) -> String { + (value + Duration::hours(8)).format(format).to_string() +} + +pub(crate) fn outcome_name(outcome: &domain::RequestOutcome) -> &str { + outcome.as_str() +} + +pub(crate) fn request_metrics_view(metrics: &domain::RequestMetrics) -> RequestMetricsView { + RequestMetricsView { + request_count: metrics.request_count, + success_count: metrics.success_count, + failure_count: metrics.failure_count, + cancelled_count: metrics.cancelled_count, + incomplete_count: metrics.incomplete_count, + caller_error_count: metrics.caller_error_count, + input_tokens: metrics.input_tokens, + output_tokens: metrics.output_tokens, + cached_tokens: metrics.cached_tokens, + cache_write_tokens: metrics.cache_write_tokens, + reasoning_tokens: metrics.reasoning_tokens, + total_tokens: metrics.total_tokens, + } +} + +pub(crate) fn cost_coverage_view(coverage: &domain::CostCoverage) -> CostCoverageView { + CostCoverageView { + known: coverage.known_count(), + partial: coverage.partial_count, + unknown: coverage.unavailable_count, + not_billable: coverage.not_billable_count, + } +} + +pub(crate) fn cost_views(costs: &[domain::CurrencyCost]) -> Vec { + costs + .iter() + .map(|cost| CostView { + currency: cost.currency.clone(), + estimated_amount: cost.amount.as_str().to_owned(), + }) + .collect() +} + +pub(crate) fn attempt_metrics_view(metrics: &domain::AttemptMetrics) -> AttemptMetricsView { + AttemptMetricsView { + attempt_count: metrics.attempt_count, + success_count: metrics.success_count, + failure_count: metrics.failure_count, + cancelled_count: metrics.cancelled_count, + incomplete_count: metrics.incomplete_count, + rate_limited_count: metrics.rate_limited_count, + auth_failure_count: metrics.auth_failure_count, + provider5xx_count: metrics.provider_5xx_count, + cost_coverage: cost_coverage_view(&metrics.cost_coverage), + costs: cost_views(&metrics.costs), + } +} + +pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView { + TokenDetailsView { + input_tokens: record.input_tokens, + output_tokens: record.output_tokens, + cached_tokens: record.cached_tokens, + cache_write_tokens: record.cache_write_tokens, + reasoning_tokens: record.reasoning_tokens, + image_input_tokens: record.image_input_tokens, + image_output_tokens: record.image_output_tokens, + total_tokens: record.total_tokens, + input_tokens_display: record + .input_tokens + .map_or_else(|| "-".to_owned(), format_number), + output_tokens_display: record + .output_tokens + .map_or_else(|| "-".to_owned(), format_number), + cached_tokens_display: record + .cached_tokens + .map_or_else(|| "-".to_owned(), format_compact_number), + cache_write_tokens_display: record + .cache_write_tokens + .map_or_else(|| "-".to_owned(), format_compact_number), + reasoning_tokens_display: record + .reasoning_tokens + .map_or_else(|| "-".to_owned(), format_number), + image_input_tokens_display: record + .image_input_tokens + .map_or_else(|| "-".to_owned(), format_number), + image_output_tokens_display: record + .image_output_tokens + .map_or_else(|| "-".to_owned(), format_number), + total_tokens_display: record + .total_tokens + .map_or_else(|| "-".to_owned(), format_number), + } +} + +pub(crate) fn format_decimal_currency(amount: &str, currency: &str) -> String { + if currency == "USD" { + let value = amount.parse::().map_or(0.0, |value| value); + let precision = if value != 0.0 && value.abs() < 0.01 { + 4 + } else { + 2 + }; + format!("${value:.precision$}") + } else { + format!("{currency} {amount}") + } +} + +pub(crate) fn format_money(cost: &domain::CurrencyCost) -> String { + format_decimal_currency(cost.amount.as_str(), &cost.currency) +} + +pub(crate) fn format_token_price(cost: &domain::CurrencyCost) -> String { + if cost.currency != "USD" { + return format!("{} {} / 1M Token", cost.currency, cost.amount.as_str()); + } + format!("${} / 1M Token", cost.amount.as_str()) +} + +pub(crate) fn format_service_tier(service_tier: Option<&str>) -> String { + match service_tier { + Some("priority" | "fast") => "Fast".to_owned(), + Some("flex") => "Flex".to_owned(), + Some("default") | None => "Default".to_owned(), + Some(other) => other.to_owned(), + } +} + +pub(crate) fn billing_view(billing: Option<&domain::UsageBilling>) -> Option { + match billing? { + domain::UsageBilling::Total { total, .. } => Some(BillingView { + input_amount_display: "—".to_owned(), + output_amount_display: "—".to_owned(), + cache_read_amount_display: "—".to_owned(), + cache_write_amount_display: "—".to_owned(), + standard_amount_display: "—".to_owned(), + total_amount_display: format_money(total), + input_price_display: "—".to_owned(), + output_price_display: "—".to_owned(), + cache_read_price_display: "—".to_owned(), + cache_write_price_display: "—".to_owned(), + service_tier_display: "—".to_owned(), + multiplier_display: "—".to_owned(), + }), + domain::UsageBilling::Calculated(value) => Some(BillingView { + input_amount_display: format_money(&value.input_amount), + output_amount_display: format_money(&value.output_amount), + cache_read_amount_display: format_money(&value.cache_read_amount), + cache_write_amount_display: format_money(&value.cache_write_amount), + standard_amount_display: format_money(&value.standard_amount), + total_amount_display: format_money(&value.total_amount), + input_price_display: format_token_price(&value.input_price_per_million), + output_price_display: format_token_price(&value.output_price_per_million), + cache_read_price_display: format_token_price(&value.cache_read_price_per_million), + cache_write_price_display: format_token_price(&value.cache_write_price_per_million), + service_tier_display: format_service_tier(value.service_tier.as_deref()), + multiplier_display: format!("{:.2}x", f64::from(value.multiplier_percent) / 100.0), + }), + } +} + +pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView { + let tokens = token_details(&record); + let billing = billing_view(record.billing.as_ref()); + let costs = record + .cost_amount + .as_ref() + .zip(record.cost_currency.as_ref()) + .map(|(amount, currency)| { + vec![CostView { + currency: currency.clone(), + estimated_amount: amount.as_str().to_owned(), + }] + }) + .unwrap_or_default(); + let status_code = record + .client_status_code + .or(record.upstream_status_code) + .map(i64::from); + let outcome = outcome_name(&record.outcome).to_owned(); + let message = record + .error_message + .clone() + .unwrap_or_else(|| outcome.clone()); + let first_token_display = display_duration(record.first_token_ms); + let latency_display = display_duration(record.latency_ms); + let cost_coverage = match record.cost_source.as_str() { + "provider_reported" | "calculated" => CostCoverageView { + known: 1, + partial: 0, + unknown: 0, + not_billable: 0, + }, + _ => CostCoverageView { + known: 0, + partial: 0, + unknown: 1, + not_billable: 0, + }, + }; + let model = record + .upstream_model_id + .clone() + .unwrap_or_else(|| record.requested_model_id.clone()); + let transport = record + .upstream_transport + .clone() + .or_else(|| Some(record.client_transport.clone())); + let metadata = UsageRecordMetadataView { + protocol: record.protocol.clone(), + logical_outcome: outcome.clone(), + attempt_count: u64::from(record.attempt_count), + requested_model: record.requested_model_id.clone(), + upstream_model: record.upstream_model_id.clone(), + client_ip: record.client_ip.clone(), + user_agent: record.user_agent.clone(), + reasoning_effort: record.reasoning_effort.clone(), + reasoning_preset: record.reasoning_preset.clone(), + compact: record.compact, + request_kind: record.request_kind.clone(), + subagent_kind: record.subagent_kind.clone(), + transport: transport.clone(), + http_version: record.http_version.clone(), + client_status_code: record.client_status_code.map(i64::from), + upstream_status_code: record.upstream_status_code.map(i64::from), + response_id: record.client_response_id.clone(), + upstream_request_id: record.upstream_request_id.clone(), + websocket_pool: record + .websocket_pool + .clone() + .map(|kind| WebSocketPoolMetadataView { kind }), + image_generation_requested: record.image_generation_requested, + image_generation_succeeded: record.image_generation_succeeded, + latency_details: UsageLatencyDetailsView { + transport_decision_wait_ms: record.transport_decision_wait_ms, + ws_connect_ms: record.connect_ms, + upstream_headers_ms: record.headers_ms, + first_event_ms: record.first_event_ms, + first_reasoning_ms: record.first_reasoning_ms, + first_text_ms: record.first_text_ms, + first_token_ms: record.first_token_ms, + openai_processing_ms: record.provider_processing_ms, + }, + provider_metadata: provider_metadata_fields(record.provider_metadata_json.as_deref()), + }; + UsageRecordView { + id: record.id.clone(), + request_id: record.id, + client_api_key_id: Some(record.client_api_key_ref), + kind: record.operation, + provider: record.provider_kind, + authentication_kind: record.provider_account_authentication_kind, + account_id: record.provider_account_ref, + account_email: record.provider_account_email, + account_name: record.provider_account_name, + route: record.endpoint, + model, + requested_model: Some(record.requested_model_id), + upstream_model: record.upstream_model_id, + service_tier: record.service_tier, + status_code, + transport, + attempt_index: None, + attempt_count: u64::from(record.attempt_count), + response_id: record.client_response_id, + upstream_request_id: record.upstream_request_id, + latency_ms: record.latency_ms, + first_token_ms: record.first_token_ms, + input_tokens: record.input_tokens, + output_tokens: record.output_tokens, + cached_tokens: record.cached_tokens, + cache_write_tokens: record.cache_write_tokens, + reasoning_tokens: record.reasoning_tokens, + image_input_tokens: record.image_input_tokens, + image_output_tokens: record.image_output_tokens, + message, + metadata, + created_at: record.started_at, + created_at_display: china_datetime(&record.started_at), + client_ip: record.client_ip, + user_agent: record.user_agent, + reasoning_effort: record.reasoning_effort, + reasoning_preset: record.reasoning_preset, + compact: Some(record.compact), + request_kind: record.request_kind, + subagent_kind: record.subagent_kind, + token_details: tokens, + billing, + costs, + cost_coverage, + first_token_latency_ms: record.first_token_ms, + first_token_latency_ms_display: first_token_display, + latency_ms_display: latency_display, + logical_outcome: outcome, + } +} + +pub(crate) fn provider_metadata_fields(value: Option<&str>) -> BTreeMap { + const CORE_METADATA_FIELDS: &[&str] = &[ + "protocol", + "logicalOutcome", + "attemptCount", + "requestedModel", + "upstreamModel", + "clientIp", + "userAgent", + "reasoningEffort", + "reasoningPreset", + "compact", + "requestKind", + "subagentKind", + "transport", + "httpVersion", + "clientStatusCode", + "upstreamStatusCode", + "responseId", + "upstreamRequestId", + "websocketPool", + "imageGenerationRequested", + "imageGenerationSucceeded", + "latencyDetails", + ]; + let Some(Ok(Value::Object(fields))) = value.map(serde_json::from_str::) else { + return BTreeMap::new(); + }; + fields + .into_iter() + .filter(|(field, _)| !CORE_METADATA_FIELDS.contains(&field.as_str())) + .collect() +} + +pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptView { + let occurred_at = attempt.occurred_at; + let credential_name = attempt + .provider_account_name + .as_ref() + .or(attempt.provider_account_ref.as_ref()) + .cloned(); + UsageAttemptView { + id: attempt.id, + attempt_index: attempt.attempt_index, + trigger: attempt.source, + provider: attempt + .provider_kind + .unwrap_or_else(|| "unknown".to_owned()), + model: attempt + .upstream_model_id + .unwrap_or_else(|| "unknown".to_owned()), + transport: attempt + .upstream_transport + .unwrap_or_else(|| "unknown".to_owned()), + send_state: attempt + .upstream_send_state + .unwrap_or_else(|| "unknown".to_owned()), + outcome: outcome_name(&attempt.outcome).to_owned(), + downstream_committed: attempt.downstream_committed, + status_code: attempt.status_code, + provider_error_code: attempt.provider_error_code, + failure_class: attempt.failure_kind, + cost_estimate_status: attempt + .cost_source + .clone() + .unwrap_or_else(|| "unavailable".to_owned()), + estimated_cost_amount: attempt.cost_amount.map(|amount| amount.as_str().to_owned()), + estimated_cost_currency: attempt.cost_currency, + input_tokens: attempt.input_tokens, + output_tokens: attempt.output_tokens, + cached_tokens: attempt.cached_tokens, + total_tokens: attempt.total_tokens, + first_token_ms: None, + latency_ms: attempt.latency_ms, + credential_name, + account_id: attempt.provider_account_ref, + account_name: attempt.provider_account_name, + account_email: attempt.provider_account_email, + authentication_kind: attempt.provider_account_authentication_kind, + started_at: occurred_at, + completed_at: Some(occurred_at), + } +} + +pub(crate) fn usage_detail_view(detail: domain::UsageDetail) -> UsageRecordDetailView { + UsageRecordDetailView { + request: usage_record_view(detail.request), + attempts: detail + .attempts + .into_iter() + .map(usage_attempt_view) + .collect(), + attempts_complete: false, + } +} + +pub(crate) fn page_meta(page: u32, page_size: u16, total: u64) -> PageMeta { + let page_size_u64 = u64::from(page_size); + let total_pages = total.saturating_add(page_size_u64 - 1) / page_size_u64; + PageMeta::new( + page, + u32::from(page_size), + total, + u32::try_from(total_pages).unwrap_or(u32::MAX), + ) +} + +pub(crate) fn usage_page_view( + page: domain::UsagePage, + page_number: u32, + page_size: u16, +) -> Result, WireValidationError> { + Ok(PageData { + items: page.items.into_iter().map(usage_record_view).collect(), + page: page_meta(page_number, page_size, page.total), + next_cursor: page + .next_cursor + .as_ref() + .map(encode_observability_cursor) + .transpose()?, + }) +} + +pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView { + let account_label = error + .provider_account_email + .as_ref() + .or(error.provider_account_name.as_ref()) + .or(error.provider_account_ref.as_ref()) + .cloned(); + OpsErrorView { + id: error.event_id, + request_id: error.request_id, + client_api_key_id: error.client_api_key_ref, + kind: error.component.clone(), + provider: error.provider_kind, + authentication_kind: error.provider_account_authentication_kind, + account_id: error.provider_account_ref, + route: error.endpoint.unwrap_or_else(|| "—".to_owned()), + model: error.upstream_model_id, + client_status_code: error.client_status_code.map(i64::from), + upstream_status_code: error.upstream_status_code.map(i64::from), + transport: error.upstream_transport, + attempt_index: error.attempt_index, + failure_class: error.failure_kind, + response_id: error.client_response_id, + upstream_request_id: error.upstream_request_id, + latency_ms: error.latency_ms, + message: error.message, + metadata: OpsErrorMetadataView { + source: error.source, + component: error.component, + attempt_id: None, + account_label, + }, + created_at: error.occurred_at, + created_at_display: china_datetime(&error.occurred_at), + } +} + +pub(crate) fn ops_page_view( + page: domain::OpsErrorPage, + page_number: u32, + page_size: u16, +) -> Result, WireValidationError> { + Ok(PageData { + items: page.items.into_iter().map(ops_error_view).collect(), + page: page_meta(page_number, page_size, page.total), + next_cursor: page + .next_cursor + .as_ref() + .map(encode_observability_cursor) + .transpose()?, + }) +} + +pub(crate) fn trend_point_view(point: domain::TrendPoint) -> TrendPointView { + let local_time = china_label(point.bucket_start, "%H:%M"); + let label = china_label(point.bucket_start, "%m-%d %H:%M"); + let success_rate_value = point.success_rate.map(|value| value * 100.0); + TrendPointView { + time: local_time, + bucket: point.bucket_start, + label, + requests: format_compact_number(point.metrics.request_count), + requests_value: point.metrics.request_count, + input_tokens: format_compact_number(point.metrics.input_tokens), + input_tokens_value: point.metrics.input_tokens, + output_tokens: format_compact_number(point.metrics.output_tokens), + output_tokens_value: point.metrics.output_tokens, + cached_tokens: format_compact_number(point.metrics.cached_tokens), + cached_tokens_value: point.metrics.cached_tokens, + cache_hit_rate_value: point.cached_token_rate, + tokens_value: point.metrics.total_tokens, + errors: format_compact_number(point.service_failure_count), + errors_value: point.service_failure_count, + latency: display_duration(point.average_latency_ms), + latency_value: point.average_latency_ms, + max_latency: display_duration(point.metrics.max_latency_ms), + max_latency_value: point.metrics.max_latency_ms, + min_latency: display_duration(point.metrics.min_latency_ms), + min_latency_value: point.metrics.min_latency_ms, + success_rate: success_rate_value + .map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")), + success_rate_value, + } +} + +pub(crate) fn trend_summary_view( + kind: TrendKind, + summary: &domain::TrendSummary, +) -> Vec { + match kind { + TrendKind::Usage => vec![ + TrendSummaryView { + label: "输入".to_owned(), + value: format_compact_number(summary.input_tokens), + ratio: None, + }, + TrendSummaryView { + label: "输出".to_owned(), + value: format_compact_number(summary.output_tokens), + ratio: None, + }, + TrendSummaryView { + label: "缓存".to_owned(), + value: format_compact_number(summary.cached_tokens), + ratio: None, + }, + ], + TrendKind::Latency => vec![ + TrendSummaryView { + label: "平均".to_owned(), + value: display_duration(summary.average_latency_ms), + ratio: None, + }, + TrendSummaryView { + label: "最高".to_owned(), + value: display_duration(summary.max_latency_ms), + ratio: None, + }, + TrendSummaryView { + label: "最低".to_owned(), + value: display_duration(summary.min_latency_ms), + ratio: None, + }, + ], + TrendKind::Errors => vec![ + TrendSummaryView { + label: "错误数".to_owned(), + value: format_compact_number(summary.service_failure_count), + ratio: None, + }, + TrendSummaryView { + label: "成功率".to_owned(), + value: "—".to_owned(), + ratio: summary + .success_rate + .map(|value| format!("{:.1}%", value * 100.0)), + }, + TrendSummaryView { + label: "总请求".to_owned(), + value: format_compact_number(summary.request_count), + ratio: None, + }, + ], + } +} + +pub(crate) fn trend_view(trend: domain::Trend, kind: TrendKind) -> TrendData { + TrendData { + kind, + summary: trend_summary_view(kind, &trend.summary), + points: trend.points.into_iter().map(trend_point_view).collect(), + } +} + +pub(crate) fn health_status_name(status: domain::HealthStatus) -> &'static str { + match status { + domain::HealthStatus::Future => "future", + domain::HealthStatus::NoData => "no_data", + domain::HealthStatus::Unavailable => "unavailable", + domain::HealthStatus::LowSample => "low_sample", + domain::HealthStatus::Unstable => "unstable", + domain::HealthStatus::Stable => "stable", + } +} + +pub(crate) fn reliability_display(value: Option) -> String { + value.map_or_else(|| "-".to_owned(), |value| format!("{value:.1}%")) +} + +pub(crate) fn health_timeline_view(timeline: domain::HealthTimeline) -> HealthTimelineView { + HealthTimelineView { + title: "请求健康时间线".to_owned(), + description: "有效请求可用性".to_owned(), + reliability_display: reliability_display(timeline.reliability_percent), + status: health_status_name(timeline.status).to_owned(), + success_requests: timeline.success_requests, + failed_requests: timeline.failed_requests, + cancelled_requests: timeline.cancelled_requests, + incomplete_requests: timeline.incomplete_requests, + caller_error_requests: timeline.caller_error_requests, + points: timeline + .points + .into_iter() + .enumerate() + .map(|(index, point)| { + let elapsed_minutes = i64::try_from(index).unwrap_or(i64::MAX).saturating_mul(15); + HealthTimelinePointView { + time: format!("{:02}:{:02}", elapsed_minutes / 60, elapsed_minutes % 60), + status: health_status_name(point.status).to_owned(), + reliability_display: reliability_display(point.reliability_percent), + success_requests: point.success_requests, + failed_requests: point.failed_requests, + cancelled_requests: point.cancelled_requests, + incomplete_requests: point.incomplete_requests, + caller_error_requests: point.caller_error_requests, + } + }) + .collect(), + } +} + +pub(crate) fn wire_profile_view(profile: domain::DashboardWireProfile) -> DashboardWireProfileView { + DashboardWireProfileView { + provider: profile.provider, + product: profile.product, + version: profile.version, + build: profile.build, + target: DashboardWireTargetView { + os_type: profile.target.os_type, + os_version: profile.target.os_version, + arch: profile.target.arch, + terminal: profile.target.terminal, + }, + user_agent: profile.user_agent, + attributes: profile + .attributes + .into_iter() + .map(|attribute| DashboardWireAttributeView { + label: attribute.label, + value: attribute.value, + }) + .collect(), + verified_at: profile.verified_at, + release: profile.release.map(|release| DashboardDesktopReleaseView { + status: release.status.into(), + checked_at: release.checked_at, + latest_version: release.latest_version, + latest_build: release.latest_build, + error: release.error, + }), + } +} + +pub(crate) fn relative_time(value: Option>, now: DateTime) -> String { + let Some(value) = value else { + return "从未使用".to_owned(); + }; + let elapsed = now.signed_duration_since(value); + if elapsed.num_seconds() < 0 { + return china_datetime(&value); + } + if elapsed.num_seconds() < 60 { + return "刚刚".to_owned(); + } + if elapsed.num_minutes() < 60 { + return format!("{} 分钟前", elapsed.num_minutes()); + } + if elapsed.num_hours() < 24 { + return format!("{} 小时前", elapsed.num_hours()); + } + format!("{} 天前", elapsed.num_days()) +} + +pub(crate) fn dashboard_view( + result: domain::DashboardResult, + kind: TrendKind, +) -> DashboardDataView { + let domain::DashboardResult { + observation, + today, + yesterday, + total_billing_usd, + total_cached_token_rate, + average_first_token_latency_ms, + trend, + health_timeline, + wire_profiles, + capacity, + rotation_strategy, + } = result; + let domain::DashboardObservation { + range, + requests, + provider_accounts, + account_usage, + recent_requests, + .. + } = observation; + let mut account_usage_views = Vec::with_capacity(account_usage.len()); + for credential in account_usage { + account_usage_views.push(DashboardAccountUsageView { + id: credential.account_id.clone(), + provider: credential.provider_kind.clone(), + authentication_kind: credential.authentication_kind.clone(), + email: credential + .email + .clone() + .unwrap_or_else(|| credential.name.clone()), + plan_type: credential.plan_type.clone(), + tokens: credential + .total_tokens + .map_or_else(|| "—".to_owned(), format_compact_number), + request_count: credential.request_count, + request_buckets: credential + .request_buckets + .iter() + .map(|bucket| DashboardAccountRequestBucketView { + bucket_start: bucket.bucket_start, + request_count: bucket.request_count, + }) + .collect(), + quota_used_percent: credential.quota_used_percent, + last_used: relative_time(credential.last_used_at, range.end), + }); + } + DashboardDataView { + cards: DashboardCardsView { + credentials: DashboardCredentialsCardView { + total: format_compact_number(provider_accounts.total), + total_value: provider_accounts.total, + available: format_compact_number(provider_accounts.active), + available_value: provider_accounts.active, + unavailable: format_compact_number(provider_accounts.unavailable), + unavailable_value: provider_accounts.unavailable, + }, + traffic: DashboardTrafficCardView { + today_requests: format_compact_number(today.request_count), + today_requests_value: today.request_count, + yesterday_requests_value: yesterday.request_count, + total_requests: format_compact_number(requests.request_count), + }, + tokens: DashboardTokensCardView { + today_tokens: format_compact_number(today.total_tokens), + today_tokens_value: today.total_tokens, + yesterday_tokens_value: yesterday.total_tokens, + total_tokens: format_compact_number(requests.total_tokens), + total_billing_amount_usd: total_billing_usd + .as_ref() + .map_or_else(|| "—".to_owned(), |amount| format!("${}", amount.as_str())), + }, + cache: DashboardCacheCardView { + today_hit_rate: display_rate(today.cached_token_rate), + today_hit_rate_value: today.observed_cached_token_rate, + yesterday_hit_rate_value: yesterday.observed_cached_token_rate, + total_hit_rate: display_rate(total_cached_token_rate), + total_cached_tokens: format_compact_number(requests.cached_tokens), + average_first_token_latency_ms: display_duration(average_first_token_latency_ms), + }, + }, + trend: trend_view(trend, kind), + health_timeline: health_timeline_view(health_timeline), + wire_profiles: wire_profiles.into_iter().map(wire_profile_view).collect(), + account_usage: account_usage_views, + usage_records: recent_requests.into_iter().map(usage_record_view).collect(), + pool_summary: DashboardPoolSummaryView { + total: provider_accounts.total, + active: provider_accounts.active, + expired: provider_accounts.expired, + quota_exhausted: provider_accounts.quota_exhausted, + refreshing: provider_accounts.refreshing, + disabled: provider_accounts.disabled, + banned: provider_accounts.banned, + }, + capacity_info: DashboardCapacityInfoView { + max_concurrent_per_account: capacity.max_concurrent_per_account, + total_slots: capacity.total_slots, + used_slots: capacity.used_slots, + available_slots: capacity.available_slots, + }, + rotation_strategy: rotation_strategy.as_str().to_owned(), + } +} + +pub(crate) fn usage_summary_view(summary: domain::UsageSummary) -> UsageSummaryView { + let overview = summary.overview; + UsageSummaryView { + total_requests: format_compact_number(overview.requests.request_count), + input_tokens: format_compact_number(overview.requests.input_tokens), + output_tokens: format_compact_number(overview.requests.output_tokens), + cached_tokens: format_compact_number(overview.requests.cached_tokens), + cache_write_tokens: format_compact_number(overview.requests.cache_write_tokens), + total_tokens: format_compact_number(overview.requests.total_tokens), + average_latency_ms: display_duration(summary.average_latency_ms), + logical_requests: request_metrics_view(&overview.requests), + attempts: attempt_metrics_view(&overview.attempts), + } +} + +pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsightsOverviewView { + let health_points = insights + .health + .points + .iter() + .map(|point| OverviewHealthPointView { + bucket: point.bucket_start, + label: china_label(point.bucket_start, "%m-%d %H:%M"), + success_requests: point.success_requests, + failed_requests: point.failed_requests, + cancelled_requests: point.cancelled_requests, + incomplete_requests: point.incomplete_requests, + caller_error_requests: point.caller_error_requests, + error_rate: point.error_rate, + }) + .collect(); + let performance_points = insights + .performance + .points + .iter() + .map(|point| OverviewPerformancePointView { + bucket: point.bucket_start, + label: china_label(point.bucket_start, "%m-%d %H:%M"), + latency_p50_ms: point.latency_percentiles.p50_ms.map(|value| value.as_f64()), + latency_p95_ms: point.latency_percentiles.p95_ms.map(|value| value.as_f64()), + latency_p99_ms: point.latency_percentiles.p99_ms.map(|value| value.as_f64()), + first_token_p50_ms: point + .first_token_latency_percentiles + .p50_ms + .map(|value| value.as_f64()), + first_token_p95_ms: point + .first_token_latency_percentiles + .p95_ms + .map(|value| value.as_f64()), + first_token_p99_ms: point + .first_token_latency_percentiles + .p99_ms + .map(|value| value.as_f64()), + }) + .collect(); + let cost_points = insights + .cost + .points + .iter() + .map(|point| OverviewCostPointView { + bucket: point.bucket_start, + label: china_label(point.bucket_start, "%m-%d %H:%M"), + input_tokens: point.input_tokens, + output_tokens: point.output_tokens, + cached_tokens: point.cached_tokens, + total_tokens: point.total_tokens, + estimated_cost: point.estimated_cost.as_ref().map(ToString::to_string), + standard_cost: point.standard_cost.as_ref().map(ToString::to_string), + cached_token_rate: point.cached_token_rate, + cache_hit_request_rate: point.cache_hit_request_rate, + }) + .collect(); + UsageInsightsOverviewView { + granularity: granularity_name(insights.granularity).to_owned(), + health: OverviewHealthView { + total_requests: insights.health.total_requests, + success_requests: insights.health.success_requests, + failed_requests: insights.health.failed_requests, + cancelled_requests: insights.health.cancelled_requests, + incomplete_requests: insights.health.incomplete_requests, + caller_error_requests: insights.health.caller_error_requests, + success_rate: insights.health.success_rate, + request_change_rate: None, + success_rate_change: None, + points: health_points, + }, + performance: OverviewPerformanceView { + latency_p50_ms: insights + .performance + .latency_percentiles + .p50_ms + .map(|value| value.as_f64()), + latency_p95_ms: insights + .performance + .latency_percentiles + .p95_ms + .map(|value| value.as_f64()), + latency_p99_ms: insights + .performance + .latency_percentiles + .p99_ms + .map(|value| value.as_f64()), + first_token_p50_ms: insights + .performance + .first_token_latency_percentiles + .p50_ms + .map(|value| value.as_f64()), + first_token_p95_ms: insights + .performance + .first_token_latency_percentiles + .p95_ms + .map(|value| value.as_f64()), + first_token_p99_ms: insights + .performance + .first_token_latency_percentiles + .p99_ms + .map(|value| value.as_f64()), + latency_coverage: insights.performance.latency_coverage, + first_token_coverage: insights.performance.first_token_coverage, + points: performance_points, + }, + cost: OverviewCostView { + estimated_cost: insights + .cost + .estimated_cost + .as_ref() + .map(ToString::to_string), + standard_cost: insights + .cost + .standard_cost + .as_ref() + .map(ToString::to_string), + cost_per_request: insights + .cost + .cost_per_request + .as_ref() + .map(ToString::to_string), + tokens_per_request: insights.cost.tokens_per_request, + cached_token_rate: insights.cost.cached_token_rate, + cache_hit_request_rate: insights.cost.cache_hit_request_rate, + input_tokens: insights.cost.input_tokens, + output_tokens: insights.cost.output_tokens, + cached_tokens: insights.cost.cached_tokens, + total_tokens: insights.cost.total_tokens, + points: cost_points, + costs: cost_views(&insights.cost.costs), + coverage: cost_coverage_view(&insights.cost.coverage), + }, + attempts: attempt_metrics_view(&insights.attempts), + providers: insights + .providers + .into_iter() + .map(|provider| ProviderOverviewView { + provider: provider.provider_kind, + request_count: provider.request_count, + attempt_count: provider.attempt_count, + failure_count: provider.failure_count, + total_tokens: provider.total_tokens, + }) + .collect(), + } +} + +pub(crate) fn granularity_name(granularity: domain::Granularity) -> &'static str { + match granularity { + domain::Granularity::FifteenMinutes => "15m", + domain::Granularity::Hour => "1h", + domain::Granularity::Day => "1d", + } +} + +pub(crate) fn diagnostics_view( + result: domain::DiagnosticsResult, + dimension: DiagnosticDimension, +) -> DiagnosticsView { + DiagnosticsView { + dimension: dimension.display_name().to_owned(), + items: result + .items + .into_iter() + .map(|item| DiagnosticItemView { + key: item.key, + name: if item.name == "__none__" { + "未知".to_owned() + } else { + item.name + }, + request_count: item.request_count, + success_count: item.success_count, + error_count: item.error_count, + error_rate: item.error_rate, + request_share: item.request_share, + average_latency_ms: item.average_latency_ms, + latency_p95_ms: item.latency_p95_ms, + estimated_cost: item.estimated_cost.as_ref().map(ToString::to_string), + attempt_count: item.attempt_count, + total_tokens: item.total_tokens, + }) + .collect(), + } +} + +pub(crate) fn map_wire_error(error: WireValidationError) -> AdminError { + let message = match error.field() { + "timeRange" => "Invalid time range", + "statusCode" => "Status code must be between 100 and 599", + "attemptIndex" => "Attempt index is out of range", + "kind" => "Invalid dashboard trend kind", + "dimension" => "Invalid diagnostics dimension", + "outcome" => "Invalid Observability query", + "id" => "Usage record ID is required", + "page" | "pageSize" => "Invalid Observability query", + "cursor" => "Invalid observability cursor", + _ => "Invalid observability query", + }; + AdminError::bad_request(message) +} + +pub(crate) fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError { + map_admin_service_error(error, "Observability repository unavailable") +} diff --git a/backend/crates/gateway-api/src/admin/observability/query.rs b/backend/crates/gateway-api/src/admin/observability/query.rs new file mode 100644 index 000000000..b5d7abf77 --- /dev/null +++ b/backend/crates/gateway-api/src/admin/observability/query.rs @@ -0,0 +1,463 @@ +//! 查询 wire 类型、校验与领域命令映射。 + +use super::*; + +use base64::Engine as _; + +/// 观测列表默认页大小。 +pub const DEFAULT_PAGE_SIZE: u16 = 50; +/// 观测列表允许的最大页大小。 +pub const MAX_PAGE_SIZE: u16 = 100; +/// 游标编码在 HTTP 层解码前允许的最大长度。 +pub const MAX_CURSOR_BYTES: usize = 1024; + +/// Dashboard 查询参数。 +#[derive(Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DashboardQuery { + pub kind: Option, + pub start_time: Option, + pub end_time: Option, +} + +impl DashboardQuery { + /// 解析 dashboard 趋势类型。 + pub fn trend_kind(&self) -> Result { + TrendKind::parse(self.kind.as_deref()) + } +} + +/// 逻辑请求列表、汇总和洞察查询参数。 +#[derive(Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UsageQuery { + pub page: Option, + pub page_size: Option, + pub cursor: Option, + pub kind: Option, + pub outcome: Option, + pub client_api_key_id: Option, + pub provider: Option, + pub request_id: Option, + pub account_id: Option, + pub route: Option, + pub model: Option, + pub status_code: Option, + pub transport: Option, + pub attempt_index: Option, + pub response_id: Option, + pub upstream_request_id: Option, + pub search: Option, + pub start_time: Option, + pub end_time: Option, +} + +impl UsageQuery { + /// 校验分页字段，不改变仓储层的分页 owner。 + pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> { + let page = self.page.unwrap_or(1); + if page == 0 { + return Err(WireValidationError::new("page")); + } + let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE); + if page_size == 0 || page_size > MAX_PAGE_SIZE { + return Err(WireValidationError::new("pageSize")); + } + Ok((page, page_size)) + } + + /// 校验游标的 wire 边界；编码和排序语义由应用层负责。 + pub fn validate_cursor(&self) -> Result<(), WireValidationError> { + validate_cursor(self.cursor.as_deref()) + } +} + +/// 详情查询参数。 +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DetailQuery { + pub id: String, +} + +impl DetailQuery { + /// 校验详情 ID，错误不回显输入值。 + pub fn validate(&self) -> Result<(), WireValidationError> { + require_text(&self.id, "id") + } +} + +/// 诊断聚合查询参数。 +#[derive(Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DiagnosticsQuery { + pub dimension: Option, + pub start_time: Option, + pub end_time: Option, + pub provider: Option, + pub model: Option, + pub status_code: Option, + pub search: Option, +} + +impl DiagnosticsQuery { + /// 解析诊断维度。 + pub fn dimension(&self) -> Result { + DiagnosticDimension::parse(self.dimension.as_deref()) + } +} + +/// 运维错误查询参数。 +#[derive(Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OpsQuery { + pub page: Option, + pub page_size: Option, + pub cursor: Option, + pub kind: Option, + pub client_api_key_id: Option, + pub provider: Option, + pub request_id: Option, + pub account_id: Option, + pub route: Option, + pub model: Option, + pub status_code: Option, + pub client_status_code: Option, + pub upstream_status_code: Option, + pub transport: Option, + pub attempt_index: Option, + pub failure_class: Option, + pub response_id: Option, + pub upstream_request_id: Option, + pub search: Option, + pub start_time: Option, + pub end_time: Option, +} + +impl OpsQuery { + /// 校验分页字段。 + pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> { + let page = self.page.unwrap_or(1); + if page == 0 { + return Err(WireValidationError::new("page")); + } + let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE); + if page_size == 0 || page_size > MAX_PAGE_SIZE { + return Err(WireValidationError::new("pageSize")); + } + Ok((page, page_size)) + } + + /// 校验游标的 wire 边界。 + pub fn validate_cursor(&self) -> Result<(), WireValidationError> { + validate_cursor(self.cursor.as_deref()) + } +} + +/// Dashboard 趋势类型。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "lowercase")] +pub enum TrendKind { + Usage, + Latency, + Errors, +} + +impl TrendKind { + /// 从 query 值解析趋势类型。 + pub fn parse(value: Option<&str>) -> Result { + match trimmed(value) { + None | Some("usage") => Ok(Self::Usage), + Some("latency") => Ok(Self::Latency), + Some("errors") => Ok(Self::Errors), + Some(_) => Err(WireValidationError::new("kind")), + } + } +} + +/// 诊断聚合维度。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiagnosticDimension { + Model, + Account, + ApiKey, + Provider, + Transport, + Failure, + Status, +} + +impl DiagnosticDimension { + /// 从 query 值解析诊断维度。 + pub fn parse(value: Option<&str>) -> Result { + match trimmed(value) { + None | Some("model") => Ok(Self::Model), + Some("account") => Ok(Self::Account), + Some("apiKey" | "api_key") => Ok(Self::ApiKey), + Some("provider") => Ok(Self::Provider), + Some("transport") => Ok(Self::Transport), + Some("failureClass" | "failure_class") => Ok(Self::Failure), + Some("status") => Ok(Self::Status), + Some(_) => Err(WireValidationError::new("dimension")), + } + } + + /// 返回终态响应中的稳定维度名称。 + #[must_use] + pub const fn display_name(self) -> &'static str { + match self { + Self::Model => "model", + Self::Account => "account", + Self::ApiKey => "apiKey", + Self::Provider => "provider", + Self::Transport => "transport", + Self::Failure => "failureClass", + Self::Status => "status", + } + } +} + +/// 解析 RFC3339 时间；错误不回显原始值。 +pub fn parse_datetime(value: Option<&str>) -> Result>, WireValidationError> { + let Some(value) = trimmed(value) else { + return Ok(None); + }; + DateTime::parse_from_rfc3339(value) + .map(|value| Some(value.with_timezone(&Utc))) + .map_err(|_| WireValidationError::new("timeRange")) +} + +/// 解析 HTTP 状态码。 +pub fn parse_status(value: Option) -> Result, WireValidationError> { + value + .map(|value| { + u16::try_from(value) + .ok() + .filter(|value| (100..=599).contains(value)) + .ok_or_else(|| WireValidationError::new("statusCode")) + }) + .transpose() +} + +/// 解析尝试序号。 +pub fn parse_attempt_index(value: Option) -> Result, WireValidationError> { + value + .map(|value| { + u32::try_from(value) + .ok() + .filter(|value| *value > 0 && *value <= i32::MAX as u32) + .ok_or_else(|| WireValidationError::new("attemptIndex")) + }) + .transpose() +} + +pub(crate) fn require_text(value: &str, field: &'static str) -> Result<(), WireValidationError> { + if value.trim().is_empty() { + Err(WireValidationError::new(field)) + } else { + Ok(()) + } +} + +pub(crate) fn validate_cursor(value: Option<&str>) -> Result<(), WireValidationError> { + if value.is_some_and(|value| value.is_empty() || value.len() > MAX_CURSOR_BYTES) { + return Err(WireValidationError::new("cursor")); + } + Ok(()) +} + +pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +pub(crate) fn domain_trend_kind(kind: TrendKind) -> domain::TrendKind { + match kind { + TrendKind::Usage => domain::TrendKind::Usage, + TrendKind::Latency => domain::TrendKind::Latency, + TrendKind::Errors => domain::TrendKind::Errors, + } +} + +pub(crate) fn domain_diagnostic_dimension( + dimension: DiagnosticDimension, +) -> domain::DiagnosticDimension { + match dimension { + DiagnosticDimension::Model => domain::DiagnosticDimension::Model, + DiagnosticDimension::Account => domain::DiagnosticDimension::Account, + DiagnosticDimension::ApiKey => domain::DiagnosticDimension::ApiKey, + DiagnosticDimension::Provider => domain::DiagnosticDimension::Provider, + DiagnosticDimension::Transport => domain::DiagnosticDimension::Transport, + DiagnosticDimension::Failure => domain::DiagnosticDimension::Failure, + DiagnosticDimension::Status => domain::DiagnosticDimension::Status, + } +} + +pub(crate) fn request_outcome( + value: Option, +) -> Result, WireValidationError> { + non_empty(value) + .map(|value| { + domain::RequestOutcome::new(value).map_err(|_| WireValidationError::new("outcome")) + }) + .transpose() +} + +pub(crate) fn usage_range( + start: Option<&str>, + end: Option<&str>, +) -> Result { + let end = parse_datetime(end)?.unwrap_or_else(Utc::now); + let start = parse_datetime(start)?.unwrap_or(end - Duration::days(7)); + domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange")) +} + +pub(crate) fn dashboard_range( + start: Option<&str>, + end: Option<&str>, +) -> Result { + let end = parse_datetime(end)?.unwrap_or_else(Utc::now); + let start = + parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1)); + domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange")) +} + +pub(crate) fn dashboard_today_range( + start: Option<&str>, + end: Option<&str>, +) -> Result { + let end = parse_datetime(end)?.unwrap_or_else(Utc::now); + let start = parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end)); + domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange")) +} + +pub(crate) fn usage_filter(query: &UsageQuery) -> Result { + let outcome = query.outcome.clone().or_else(|| { + non_empty(query.kind.clone()).filter(|value| { + matches!( + value.as_str(), + "running" | "succeeded" | "failed" | "cancelled" | "incomplete" + ) + }) + }); + Ok(domain::UsageFilter { + client_api_key_ref: non_empty(query.client_api_key_id.clone()), + request_id: non_empty(query.request_id.clone()), + provider_account_ref: non_empty(query.account_id.clone()), + operation: non_empty(query.route.clone()), + provider_kind: non_empty(query.provider.clone()), + model: non_empty(query.model.clone()), + outcome: request_outcome(outcome)?, + status_code: parse_status(query.status_code)?, + transport: non_empty(query.transport.clone()), + attempt_index: parse_attempt_index(query.attempt_index)?, + response_id: non_empty(query.response_id.clone()), + upstream_request_id: non_empty(query.upstream_request_id.clone()), + search: non_empty(query.search.clone()), + }) +} + +pub(crate) fn usage_command( + query: &UsageQuery, +) -> Result<(domain::UsageQuery, u32, u16), WireValidationError> { + let (page, page_size) = query.validate_page()?; + query.validate_cursor()?; + let page_number = NonZeroU32::new(page) + .map(domain::PageNumber::new) + .ok_or_else(|| WireValidationError::new("page"))?; + let page_size_value = + DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?; + Ok(( + domain::UsageQuery { + range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?, + filter: usage_filter(query)?, + cursor: decode_observability_cursor(query.cursor.as_deref())?, + page: page_number, + page_size: page_size_value, + }, + page, + page_size, + )) +} + +pub(crate) fn ops_command( + query: &OpsQuery, +) -> Result<(domain::OpsErrorQuery, u32, u16), WireValidationError> { + let (page, page_size) = query.validate_page()?; + query.validate_cursor()?; + let page_number = NonZeroU32::new(page) + .map(domain::PageNumber::new) + .ok_or_else(|| WireValidationError::new("page"))?; + let page_size_value = + DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?; + let status_code = parse_status( + query + .upstream_status_code + .or(query.client_status_code) + .or(query.status_code), + )?; + Ok(( + domain::OpsErrorQuery { + range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?, + filter: domain::OpsErrorFilter { + client_api_key_ref: non_empty(query.client_api_key_id.clone()), + request_id: non_empty(query.request_id.clone()), + provider_kind: non_empty(query.provider.clone()), + provider_account_ref: non_empty(query.account_id.clone()), + operation: non_empty(query.route.clone()).or_else(|| non_empty(query.kind.clone())), + model: non_empty(query.model.clone()), + transport: non_empty(query.transport.clone()), + attempt_index: parse_attempt_index(query.attempt_index)?, + response_id: non_empty(query.response_id.clone()), + upstream_request_id: non_empty(query.upstream_request_id.clone()), + failure_kind: non_empty(query.failure_class.clone()), + status_code, + search: non_empty(query.search.clone()), + }, + cursor: decode_observability_cursor(query.cursor.as_deref())?, + page: page_number, + page_size: page_size_value, + }, + page, + page_size, + )) +} + +pub(crate) fn decode_observability_cursor( + value: Option<&str>, +) -> Result, WireValidationError> { + value + .map(|encoded| { + if encoded.is_empty() || encoded.len() > MAX_CURSOR_BYTES { + return Err(WireValidationError::new("cursor")); + } + let bytes = URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|_| WireValidationError::new("cursor"))?; + let wire: CursorWire = + serde_json::from_slice(&bytes).map_err(|_| WireValidationError::new("cursor"))?; + if wire.stable_id.trim().is_empty() { + return Err(WireValidationError::new("cursor")); + } + Ok(domain::ObservabilityCursor { + observed_at: wire.observed_at, + stable_id: wire.stable_id, + }) + }) + .transpose() +} + +pub(crate) fn encode_observability_cursor( + cursor: &domain::ObservabilityCursor, +) -> Result { + let bytes = serde_json::to_vec(&CursorWire { + observed_at: cursor.observed_at, + stable_id: cursor.stable_id.clone(), + }) + .map_err(|_| WireValidationError::new("cursor"))?; + Ok(URL_SAFE_NO_PAD.encode(bytes)) +} + +pub(crate) fn non_empty(value: Option) -> Option { + value + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} diff --git a/backend/crates/gateway-api/src/admin/observability/routes.rs b/backend/crates/gateway-api/src/admin/observability/routes.rs new file mode 100644 index 000000000..c324d2ae3 --- /dev/null +++ b/backend/crates/gateway-api/src/admin/observability/routes.rs @@ -0,0 +1,213 @@ +//! 固定 `/api/admin` 观测路由与 handler。 + +use super::*; + +pub fn router() -> Router +where + S: AdminSessionState + Clone + Send + Sync + 'static, +{ + Router::new() + .route("/api/admin/dashboard/summary", get(dashboard_summary::)) + .route("/api/admin/dashboard/trend", get(dashboard_trend::)) + .route("/api/admin/usage/records", get(usage_records::)) + .route( + "/api/admin/usage/records/detail", + get(usage_record_detail::), + ) + .route( + "/api/admin/usage/records/summary", + get(usage_records_summary::), + ) + .route( + "/api/admin/usage/insights/overview", + get(usage_insights_overview::), + ) + .route( + "/api/admin/usage/insights/diagnostics", + get(usage_insights_diagnostics::), + ) + .route("/api/admin/operations/errors", get(ops_errors::)) +} + +pub(crate) async fn dashboard_summary( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let kind = query.trend_kind().map_err(map_wire_error)?; + let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref()) + .map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .dashboard_summary(range, domain_trend_kind(kind)) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(dashboard_view(result, kind)), + )) +} + +pub(crate) async fn dashboard_trend( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let kind = query.trend_kind().map_err(map_wire_error)?; + let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref()) + .map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .dashboard_trend(range, domain_trend_kind(kind)) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(trend_view(result, kind)), + )) +} + +pub(crate) async fn usage_records( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let (command, page, page_size) = usage_command(&query).map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .usage_records(command) + .await + .map_err(map_service_error)?; + let data = usage_page_view(result, page, page_size) + .map_err(|_| AdminError::internal("Failed to encode observability cursor"))?; + Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data))) +} + +pub(crate) async fn usage_record_detail( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + query.validate().map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .usage_record_detail(query.id.trim()) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(usage_detail_view(result)), + )) +} + +pub(crate) async fn usage_records_summary( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref()) + .map_err(map_wire_error)?; + let filter = usage_filter(&query).map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .usage_summary(range, filter) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(usage_summary_view(result)), + )) +} + +pub(crate) async fn usage_insights_overview( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref()) + .map_err(map_wire_error)?; + let filter = usage_filter(&query).map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .usage_insights(range, filter) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(usage_insights_view(result)), + )) +} + +pub(crate) async fn usage_insights_diagnostics( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let dimension = query.dimension().map_err(map_wire_error)?; + let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref()) + .map_err(map_wire_error)?; + let filter = domain::UsageFilter { + provider_kind: non_empty(query.provider), + model: non_empty(query.model), + status_code: parse_status(query.status_code).map_err(map_wire_error)?, + search: non_empty(query.search), + ..domain::UsageFilter::default() + }; + let result = state + .admin_services() + .observability() + .diagnostics(range, filter, domain_diagnostic_dimension(dimension)) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(diagnostics_view(result, dimension)), + )) +} + +pub(crate) async fn ops_errors( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let (command, page, page_size) = ops_command(&query).map_err(map_wire_error)?; + let result = state + .admin_services() + .observability() + .ops_errors(command) + .await + .map_err(map_service_error)?; + let data = ops_page_view(result, page, page_size) + .map_err(|_| AdminError::internal("Failed to encode observability cursor"))?; + Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data))) +} diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs new file mode 100644 index 000000000..459b44b9c --- /dev/null +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -0,0 +1,742 @@ +//! 响应 DTO 与固定 wire 形状。 + +use super::*; + +/// 观测列表游标的稳定 wire 形状。 +#[derive(Clone, Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CursorWire { + pub observed_at: DateTime, + pub stable_id: String, +} + +/// 观测列表响应数据。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PageData { + pub items: Vec, + pub page: PageMeta, + pub next_cursor: Option, +} + +/// Token 详情展示。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TokenDetailsView { + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub reasoning_tokens: Option, + pub image_input_tokens: Option, + pub image_output_tokens: Option, + pub total_tokens: Option, + pub input_tokens_display: String, + pub output_tokens_display: String, + pub cached_tokens_display: String, + pub cache_write_tokens_display: String, + pub reasoning_tokens_display: String, + pub image_input_tokens_display: String, + pub image_output_tokens_display: String, + pub total_tokens_display: String, +} + +/// 按货币展示的成本。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CostView { + pub currency: String, + pub estimated_amount: String, +} + +/// 成本覆盖状态计数。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CostCoverageView { + pub known: u64, + pub partial: u64, + pub unknown: u64, + pub not_billable: u64, +} + +/// Provider 受控价格规则生成的单次请求费用明细展示。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BillingView { + pub input_amount_display: String, + pub output_amount_display: String, + pub cache_read_amount_display: String, + pub cache_write_amount_display: String, + pub standard_amount_display: String, + pub total_amount_display: String, + pub input_price_display: String, + pub output_price_display: String, + pub cache_read_price_display: String, + pub cache_write_price_display: String, + pub service_tier_display: String, + pub multiplier_display: String, +} + +/// 单条逻辑请求展示。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageRecordView { + pub id: String, + pub request_id: String, + pub client_api_key_id: Option, + pub kind: String, + pub provider: Option, + pub authentication_kind: Option, + pub account_id: Option, + pub account_email: Option, + pub account_name: Option, + pub route: String, + pub model: String, + pub requested_model: Option, + pub upstream_model: Option, + pub service_tier: Option, + pub status_code: Option, + pub transport: Option, + pub attempt_index: Option, + pub attempt_count: u64, + pub response_id: Option, + pub upstream_request_id: Option, + pub latency_ms: Option, + pub first_token_ms: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub reasoning_tokens: Option, + pub image_input_tokens: Option, + pub image_output_tokens: Option, + pub message: String, + pub metadata: UsageRecordMetadataView, + pub created_at: DateTime, + pub created_at_display: String, + pub client_ip: Option, + pub user_agent: Option, + pub reasoning_effort: Option, + pub reasoning_preset: Option, + pub compact: Option, + pub request_kind: Option, + pub subagent_kind: Option, + pub token_details: TokenDetailsView, + pub billing: Option, + pub costs: Vec, + pub cost_coverage: CostCoverageView, + pub first_token_latency_ms: Option, + pub first_token_latency_ms_display: String, + pub latency_ms_display: String, + pub logical_outcome: String, +} + +/// 逻辑请求安全元数据；不承载原始请求体或上游错误正文。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageRecordMetadataView { + pub protocol: String, + pub logical_outcome: String, + pub attempt_count: u64, + pub requested_model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub upstream_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub client_ip: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_agent: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_preset: Option, + pub compact: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub request_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub subagent_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub transport: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub http_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub client_status_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub upstream_status_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub upstream_request_id: Option, + pub websocket_pool: Option, + pub image_generation_requested: bool, + pub image_generation_succeeded: Option, + pub latency_details: UsageLatencyDetailsView, + /// Provider 安全观测直接展开，避免 API 层理解任何 Provider 专属字段。 + #[serde(flatten)] + pub provider_metadata: BTreeMap, +} + +/// WebSocket 池决策的稳定形状；与 Provider 选择逻辑无关。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebSocketPoolMetadataView { + pub kind: String, +} + +/// 逻辑请求在上游和输出阶段测得的时延事实。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageLatencyDetailsView { + #[serde(skip_serializing_if = "Option::is_none")] + pub transport_decision_wait_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_connect_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub upstream_headers_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_event_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_reasoning_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_text_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_token_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub openai_processing_ms: Option, +} + +/// 单次上游尝试展示。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageAttemptView { + pub id: String, + pub attempt_index: u32, + pub trigger: String, + pub provider: String, + pub model: String, + pub transport: String, + pub send_state: String, + pub outcome: String, + pub downstream_committed: bool, + pub status_code: Option, + pub provider_error_code: Option, + pub failure_class: Option, + pub cost_estimate_status: String, + pub estimated_cost_amount: Option, + pub estimated_cost_currency: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub total_tokens: Option, + pub first_token_ms: Option, + pub latency_ms: Option, + pub credential_name: Option, + pub account_id: Option, + pub account_name: Option, + pub account_email: Option, + pub authentication_kind: Option, + pub started_at: DateTime, + pub completed_at: Option>, +} + +/// 逻辑请求详情与其尝试列表。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageRecordDetailView { + #[serde(flatten)] + pub request: UsageRecordView, + pub attempts: Vec, + /// 尝试列表是否完整；best-effort 下恒为 false。 + pub attempts_complete: bool, +} + +/// Dashboard 趋势数据。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrendData { + pub kind: TrendKind, + pub points: Vec, + pub summary: Vec, +} + +/// Dashboard 单个趋势桶。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrendPointView { + pub time: String, + pub bucket: DateTime, + pub label: String, + pub requests: String, + pub requests_value: u64, + pub input_tokens: String, + pub input_tokens_value: u64, + pub output_tokens: String, + pub output_tokens_value: u64, + pub cached_tokens: String, + pub cached_tokens_value: u64, + pub cache_hit_rate_value: f64, + pub tokens_value: u64, + pub errors: String, + pub errors_value: u64, + pub latency: String, + pub latency_value: Option, + pub max_latency: String, + pub max_latency_value: Option, + pub min_latency: String, + pub min_latency_value: Option, + pub success_rate: String, + pub success_rate_value: Option, +} + +/// Dashboard 趋势摘要。 +#[derive(Debug, Serialize)] +pub struct TrendSummaryView { + pub label: String, + pub value: String, + pub ratio: Option, +} + +/// Dashboard 卡片集合。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardCardsView { + pub credentials: DashboardCredentialsCardView, + pub traffic: DashboardTrafficCardView, + pub tokens: DashboardTokensCardView, + pub cache: DashboardCacheCardView, +} + +/// Dashboard 上游凭据卡片。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardCredentialsCardView { + pub total: String, + pub total_value: u64, + pub available: String, + pub available_value: u64, + pub unavailable: String, + pub unavailable_value: u64, +} + +/// Dashboard 流量卡片。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardTrafficCardView { + pub today_requests: String, + pub today_requests_value: u64, + pub yesterday_requests_value: u64, + pub total_requests: String, +} + +/// Dashboard token 卡片。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardTokensCardView { + pub today_tokens: String, + pub today_tokens_value: u64, + pub yesterday_tokens_value: u64, + pub total_tokens: String, + pub total_billing_amount_usd: String, +} + +/// Dashboard 缓存卡片。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardCacheCardView { + pub today_hit_rate: String, + pub today_hit_rate_value: Option, + pub yesterday_hit_rate_value: Option, + pub total_hit_rate: String, + pub total_cached_tokens: String, + pub average_first_token_latency_ms: String, +} + +/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardAccountUsageView { + pub id: String, + pub provider: String, + pub authentication_kind: String, + pub email: String, + pub plan_type: Option, + pub tokens: String, + pub request_count: u64, + pub request_buckets: Vec, + pub quota_used_percent: Option, + pub last_used: String, +} + +/// Dashboard 账号单小时请求数。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardAccountRequestBucketView { + pub bucket_start: DateTime, + pub request_count: u64, +} + +/// Provider 账号池的持久事实汇总。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardPoolSummaryView { + pub total: u64, + pub active: u64, + pub expired: u64, + pub quota_exhausted: u64, + pub refreshing: Option, + pub disabled: u64, + pub banned: u64, +} + +/// 同 target 账号调度容量；Redis 未提供聚合事实时运行中槽位保持空值。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardCapacityInfoView { + pub max_concurrent_per_account: u64, + pub total_slots: u64, + pub used_slots: Option, + pub available_slots: Option, +} + +/// 逻辑请求指标展示。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestMetricsView { + pub request_count: u64, + pub success_count: u64, + pub failure_count: u64, + pub cancelled_count: u64, + pub incomplete_count: u64, + pub caller_error_count: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, + pub cache_write_tokens: u64, + pub reasoning_tokens: u64, + pub total_tokens: u64, +} + +/// 上游尝试指标展示。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AttemptMetricsView { + pub attempt_count: u64, + pub success_count: u64, + pub failure_count: u64, + pub cancelled_count: u64, + pub incomplete_count: u64, + pub rate_limited_count: u64, + pub auth_failure_count: u64, + pub provider5xx_count: u64, + pub cost_coverage: CostCoverageView, + pub costs: Vec, +} + +/// 健康时间线单点。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HealthTimelinePointView { + pub time: String, + pub status: String, + pub reliability_display: String, + pub success_requests: u64, + pub failed_requests: u64, + pub cancelled_requests: u64, + pub incomplete_requests: u64, + pub caller_error_requests: u64, +} + +/// Dashboard 健康时间线。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HealthTimelineView { + pub title: String, + pub description: String, + pub reliability_display: String, + pub status: String, + pub success_requests: u64, + pub failed_requests: u64, + pub cancelled_requests: u64, + pub incomplete_requests: u64, + pub caller_error_requests: u64, + pub points: Vec, +} + +/// Dashboard 展示的实际 Provider 上游请求身份。 +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardWireProfileView { + pub provider: String, + pub product: String, + pub version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub build: Option, + pub target: DashboardWireTargetView, + pub user_agent: String, + pub attributes: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub verified_at: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub release: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardWireAttributeView { + pub label: String, + pub value: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardWireTargetView { + pub os_type: String, + pub os_version: String, + pub arch: String, + pub terminal: String, +} + +/// 发布检查与启动画像分离；未检查时使用明确的 `unchecked` 状态。 +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardDesktopReleaseView { + pub status: DashboardDesktopReleaseStatusView, + #[serde(skip_serializing_if = "Option::is_none")] + pub checked_at: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub latest_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub latest_build: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Dashboard 发布检查的稳定 wire 状态。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DashboardDesktopReleaseStatusView { + Unchecked, + Aligned, + ReviewRequired, + CheckFailed, +} + +impl From for DashboardDesktopReleaseStatusView { + fn from(status: domain::DesktopReleaseStatus) -> Self { + match status { + domain::DesktopReleaseStatus::Unchecked => Self::Unchecked, + domain::DesktopReleaseStatus::Current => Self::Aligned, + domain::DesktopReleaseStatus::UpdateAvailable => Self::ReviewRequired, + domain::DesktopReleaseStatus::Failed => Self::CheckFailed, + } + } +} + +/// Dashboard 汇总响应数据。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardDataView { + pub cards: DashboardCardsView, + pub trend: TrendData, + pub health_timeline: HealthTimelineView, + pub wire_profiles: Vec, + pub account_usage: Vec, + pub usage_records: Vec, + pub pool_summary: DashboardPoolSummaryView, + pub capacity_info: DashboardCapacityInfoView, + pub rotation_strategy: String, +} + +/// 用量汇总响应数据。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageSummaryView { + pub total_requests: String, + pub input_tokens: String, + pub output_tokens: String, + pub cached_tokens: String, + pub cache_write_tokens: String, + pub total_tokens: String, + pub average_latency_ms: String, + pub logical_requests: RequestMetricsView, + pub attempts: AttemptMetricsView, +} + +/// 洞察健康趋势点。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OverviewHealthPointView { + pub bucket: DateTime, + pub label: String, + pub success_requests: u64, + pub failed_requests: u64, + pub cancelled_requests: u64, + pub incomplete_requests: u64, + pub caller_error_requests: u64, + pub error_rate: f64, +} + +/// 洞察健康摘要。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OverviewHealthView { + pub total_requests: u64, + pub success_requests: u64, + pub failed_requests: u64, + pub cancelled_requests: u64, + pub incomplete_requests: u64, + pub caller_error_requests: u64, + pub success_rate: f64, + pub request_change_rate: Option, + pub success_rate_change: Option, + pub points: Vec, +} + +/// 洞察性能趋势点。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OverviewPerformancePointView { + pub bucket: DateTime, + pub label: String, + pub latency_p50_ms: Option, + pub latency_p95_ms: Option, + pub latency_p99_ms: Option, + pub first_token_p50_ms: Option, + pub first_token_p95_ms: Option, + pub first_token_p99_ms: Option, +} + +/// 洞察性能摘要。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OverviewPerformanceView { + pub latency_p50_ms: Option, + pub latency_p95_ms: Option, + pub latency_p99_ms: Option, + pub first_token_p50_ms: Option, + pub first_token_p95_ms: Option, + pub first_token_p99_ms: Option, + pub latency_coverage: f64, + pub first_token_coverage: f64, + pub points: Vec, +} + +/// 洞察成本趋势点。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OverviewCostPointView { + pub bucket: DateTime, + pub label: String, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, + pub total_tokens: u64, + pub estimated_cost: Option, + pub standard_cost: Option, + pub cached_token_rate: f64, + pub cache_hit_request_rate: Option, +} + +/// 洞察成本摘要。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OverviewCostView { + pub estimated_cost: Option, + pub standard_cost: Option, + pub cost_per_request: Option, + pub tokens_per_request: f64, + pub cached_token_rate: f64, + pub cache_hit_request_rate: Option, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, + pub total_tokens: u64, + pub points: Vec, + pub costs: Vec, + pub coverage: CostCoverageView, +} + +/// Provider 维度洞察摘要。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProviderOverviewView { + pub provider: String, + pub request_count: u64, + pub attempt_count: u64, + pub failure_count: u64, + pub total_tokens: u64, +} + +/// 用量洞察总响应。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UsageInsightsOverviewView { + pub granularity: String, + pub health: OverviewHealthView, + pub performance: OverviewPerformanceView, + pub cost: OverviewCostView, + pub attempts: AttemptMetricsView, + pub providers: Vec, +} + +/// 诊断聚合项目。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DiagnosticItemView { + pub key: String, + pub name: String, + pub request_count: u64, + pub success_count: u64, + pub error_count: u64, + pub error_rate: f64, + pub request_share: f64, + pub average_latency_ms: Option, + pub latency_p95_ms: Option, + pub estimated_cost: Option, + pub attempt_count: u64, + pub total_tokens: u64, +} + +/// 诊断聚合响应。 +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DiagnosticsView { + pub dimension: String, + pub items: Vec, +} + +/// 运维错误项目。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OpsErrorView { + pub id: String, + pub request_id: Option, + pub client_api_key_id: Option, + pub kind: String, + pub provider: Option, + pub authentication_kind: Option, + pub account_id: Option, + pub route: String, + pub model: Option, + pub client_status_code: Option, + pub upstream_status_code: Option, + pub transport: Option, + pub attempt_index: Option, + pub failure_class: String, + pub response_id: Option, + pub upstream_request_id: Option, + pub latency_ms: Option, + pub message: String, + pub metadata: OpsErrorMetadataView, + pub created_at: DateTime, + pub created_at_display: String, +} + +/// 运维错误安全元数据；只保留可查询的标识，不回显秘密材料。 +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OpsErrorMetadataView { + pub source: String, + pub component: String, + pub attempt_id: Option, + pub account_label: Option, +} diff --git a/backend/crates/gateway-api/tests/admin/observability/mod.rs b/backend/crates/gateway-api/tests/admin/observability/mod.rs new file mode 100644 index 000000000..0ff97a871 --- /dev/null +++ b/backend/crates/gateway-api/tests/admin/observability/mod.rs @@ -0,0 +1,4 @@ +//! Admin 观测 wire/路由的固定合同测试。 + +mod query; +mod response; diff --git a/backend/crates/gateway-api/tests/admin/observability/query.rs b/backend/crates/gateway-api/tests/admin/observability/query.rs new file mode 100644 index 000000000..374594040 --- /dev/null +++ b/backend/crates/gateway-api/tests/admin/observability/query.rs @@ -0,0 +1,64 @@ +//! 查询 wire 校验的固定合同测试。 + +use gateway_api::admin::observability::{ + DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery, + parse_attempt_index, parse_datetime, parse_status, +}; +use serde_json::json; + +#[test] +fn dashboard_query_should_parse_terminal_trend_kinds() { + let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap(); + assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors); +} + +#[test] +fn dashboard_query_should_reject_unknown_trend_kind() { + let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap(); + assert_eq!(query.trend_kind().unwrap_err().field(), "kind"); +} + +#[test] +fn usage_query_should_bound_page_size_and_cursor() { + let query: UsageQuery = serde_json::from_value(json!({ + "page": 2, + "pageSize": 100, + "cursor": "opaque" + })) + .unwrap(); + assert_eq!(query.validate_page().unwrap(), (2, 100)); + assert!(query.validate_cursor().is_ok()); +} + +#[test] +fn usage_query_should_reject_zero_page() { + let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap(); + assert_eq!(query.validate_page().unwrap_err().field(), "page"); +} + +#[test] +fn ops_query_should_reject_page_size_above_terminal_limit() { + let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap(); + assert_eq!(query.validate_page().unwrap_err().field(), "pageSize"); +} + +#[test] +fn diagnostics_query_should_keep_wire_dimension_name() { + let query: DiagnosticsQuery = + serde_json::from_value(json!({"dimension": "failure_class"})).unwrap(); + assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure); + assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass"); +} + +#[test] +fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() { + assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode"); + assert_eq!( + parse_attempt_index(Some(0)).unwrap_err().field(), + "attemptIndex" + ); + assert_eq!( + parse_datetime(Some("not-a-time")).unwrap_err().field(), + "timeRange" + ); +} diff --git a/backend/crates/gateway-api/tests/admin/observability.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs similarity index 71% rename from backend/crates/gateway-api/tests/admin/observability.rs rename to backend/crates/gateway-api/tests/admin/observability/response.rs index 2fe9a0f32..ce9eb1b80 100644 --- a/backend/crates/gateway-api/tests/admin/observability.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -1,261 +1,196 @@ -mod query { - use gateway_api::admin::observability::{ - DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery, - parse_attempt_index, parse_datetime, parse_status, +//! 响应 DTO 与固定 wire 形状的合同测试。 + +use chrono::{TimeZone, Utc}; +use gateway_admin::model::observability::DesktopReleaseStatus; +use gateway_api::admin::PageMeta; +use gateway_api::admin::observability::{ + BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView, + DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView, + DashboardWireProfileView, DashboardWireTargetView, PageData, TokenDetailsView, TrendData, + TrendKind, TrendPointView, TrendSummaryView, +}; +use serde_json::json; + +#[test] +fn usage_page_should_keep_terminal_camel_case_shape() { + let data = PageData { + items: vec![json!({"id": "request_1"})], + page: PageMeta::new(1, 50, 1, 1), + next_cursor: Some("cursor".to_owned()), }; - use serde_json::json; - - #[test] - fn dashboard_query_should_parse_terminal_trend_kinds() { - let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap(); - assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors); - } - - #[test] - fn dashboard_query_should_reject_unknown_trend_kind() { - let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap(); - assert_eq!(query.trend_kind().unwrap_err().field(), "kind"); - } - - #[test] - fn usage_query_should_bound_page_size_and_cursor() { - let query: UsageQuery = serde_json::from_value(json!({ - "page": 2, - "pageSize": 100, - "cursor": "opaque" - })) - .unwrap(); - assert_eq!(query.validate_page().unwrap(), (2, 100)); - assert!(query.validate_cursor().is_ok()); - } - - #[test] - fn usage_query_should_reject_zero_page() { - let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap(); - assert_eq!(query.validate_page().unwrap_err().field(), "page"); - } - - #[test] - fn ops_query_should_reject_page_size_above_terminal_limit() { - let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap(); - assert_eq!(query.validate_page().unwrap_err().field(), "pageSize"); - } - - #[test] - fn diagnostics_query_should_keep_wire_dimension_name() { - let query: DiagnosticsQuery = - serde_json::from_value(json!({"dimension": "failure_class"})).unwrap(); - assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure); - assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass"); - } - - #[test] - fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() { - assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode"); - assert_eq!( - parse_attempt_index(Some(0)).unwrap_err().field(), - "attemptIndex" - ); - assert_eq!( - parse_datetime(Some("not-a-time")).unwrap_err().field(), - "timeRange" - ); - } + let value = serde_json::to_value(data).unwrap(); + assert_eq!(value["page"]["pageSize"], 50); + assert_eq!(value["nextCursor"], "cursor"); } -mod response { - use chrono::{TimeZone, Utc}; - use gateway_admin::model::observability::DesktopReleaseStatus; - use gateway_api::admin::PageMeta; - use gateway_api::admin::observability::{ - BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView, - DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView, - DashboardWireProfileView, DashboardWireTargetView, PageData, TokenDetailsView, TrendData, - TrendKind, TrendPointView, TrendSummaryView, - }; - use serde_json::json; +#[test] +fn dashboard_wire_profiles_should_keep_provider_specific_attributes() { + let value = serde_json::to_value(DashboardWireProfileView { + provider: "xai".to_owned(), + product: "Grok Build".to_owned(), + version: "0.2.106".to_owned(), + build: None, + target: DashboardWireTargetView { + os_type: "linux".to_owned(), + os_version: "—".to_owned(), + arch: "x86_64".to_owned(), + terminal: "headless".to_owned(), + }, + user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(), + attributes: vec![DashboardWireAttributeView { + label: "客户端标识".to_owned(), + value: "grok-shell".to_owned(), + }], + verified_at: None, + release: None, + }) + .expect("dashboard profile"); - #[test] - fn usage_page_should_keep_terminal_camel_case_shape() { - let data = PageData { - items: vec![json!({"id": "request_1"})], - page: PageMeta::new(1, 50, 1, 1), - next_cursor: Some("cursor".to_owned()), - }; - let value = serde_json::to_value(data).unwrap(); - assert_eq!(value["page"]["pageSize"], 50); - assert_eq!(value["nextCursor"], "cursor"); - } + assert_eq!(value["provider"], "xai"); + assert_eq!(value["version"], "0.2.106"); + assert_eq!(value["attributes"][0]["label"], "客户端标识"); + assert!(value.get("release").is_none()); + assert!(value.get("verifiedAt").is_none()); +} - #[test] - fn dashboard_wire_profiles_should_keep_provider_specific_attributes() { - let value = serde_json::to_value(DashboardWireProfileView { - provider: "xai".to_owned(), - product: "Grok Build".to_owned(), - version: "0.2.106".to_owned(), - build: None, - target: DashboardWireTargetView { - os_type: "linux".to_owned(), - os_version: "—".to_owned(), - arch: "x86_64".to_owned(), - terminal: "headless".to_owned(), - }, - user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(), - attributes: vec![DashboardWireAttributeView { - label: "客户端标识".to_owned(), - value: "grok-shell".to_owned(), - }], - verified_at: None, - release: None, - }) - .expect("dashboard profile"); - - assert_eq!(value["provider"], "xai"); - assert_eq!(value["version"], "0.2.106"); - assert_eq!(value["attributes"][0]["label"], "客户端标识"); - assert!(value.get("release").is_none()); - assert!(value.get("verifiedAt").is_none()); - } - - #[test] - fn dashboard_account_usage_should_keep_daily_request_timeline() { - let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap(); - let value = serde_json::to_value(DashboardAccountUsageView { - id: "account_1".to_owned(), - provider: "xai".to_owned(), - authentication_kind: "oauth".to_owned(), - email: "account@example.com".to_owned(), - plan_type: Some("free".to_owned()), - tokens: "—".to_owned(), +#[test] +fn dashboard_account_usage_should_keep_daily_request_timeline() { + let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap(); + let value = serde_json::to_value(DashboardAccountUsageView { + id: "account_1".to_owned(), + provider: "xai".to_owned(), + authentication_kind: "oauth".to_owned(), + email: "account@example.com".to_owned(), + plan_type: Some("free".to_owned()), + tokens: "—".to_owned(), + request_count: 3, + request_buckets: vec![DashboardAccountRequestBucketView { + bucket_start, request_count: 3, - request_buckets: vec![DashboardAccountRequestBucketView { - bucket_start, - request_count: 3, - }], - quota_used_percent: None, - last_used: "刚刚".to_owned(), - }) - .expect("dashboard account usage"); + }], + quota_used_percent: None, + last_used: "刚刚".to_owned(), + }) + .expect("dashboard account usage"); - assert_eq!(value["requestCount"], 3); - assert_eq!( - value["requestBuckets"][0]["bucketStart"], - "1970-01-01T00:00:00Z" - ); - assert_eq!(value["requestBuckets"][0]["requestCount"], 3); - } + assert_eq!(value["requestCount"], 3); + assert_eq!( + value["requestBuckets"][0]["bucketStart"], + "1970-01-01T00:00:00Z" + ); + assert_eq!(value["requestBuckets"][0]["requestCount"], 3); +} - #[test] - fn trend_wire_should_serialize_kind_and_values_without_store_types() { - let now = Utc.timestamp_opt(0, 0).single().unwrap(); - let data = TrendData { - kind: TrendKind::Usage, - points: vec![TrendPointView { - time: "08:00".to_owned(), - bucket: now, - label: "01-01 08:00".to_owned(), - requests: "1".to_owned(), - requests_value: 1, - input_tokens: "2".to_owned(), - input_tokens_value: 2, - output_tokens: "3".to_owned(), - output_tokens_value: 3, - cached_tokens: "0".to_owned(), - cached_tokens_value: 0, - cache_hit_rate_value: 0.0, - tokens_value: 5, - errors: "0".to_owned(), - errors_value: 0, - latency: "1 ms".to_owned(), - latency_value: Some(1), - max_latency: "1 ms".to_owned(), - max_latency_value: Some(1), - min_latency: "1 ms".to_owned(), - min_latency_value: Some(1), - success_rate: "100.0%".to_owned(), - success_rate_value: Some(100.0), - }], - summary: vec![TrendSummaryView { - label: "输入".to_owned(), - value: "2".to_owned(), - ratio: None, - }], - }; - let value = serde_json::to_value(data).unwrap(); - assert_eq!(value["kind"], "usage"); - assert_eq!(value["points"][0]["requestsValue"], 1); - } +#[test] +fn trend_wire_should_serialize_kind_and_values_without_store_types() { + let now = Utc.timestamp_opt(0, 0).single().unwrap(); + let data = TrendData { + kind: TrendKind::Usage, + points: vec![TrendPointView { + time: "08:00".to_owned(), + bucket: now, + label: "01-01 08:00".to_owned(), + requests: "1".to_owned(), + requests_value: 1, + input_tokens: "2".to_owned(), + input_tokens_value: 2, + output_tokens: "3".to_owned(), + output_tokens_value: 3, + cached_tokens: "0".to_owned(), + cached_tokens_value: 0, + cache_hit_rate_value: 0.0, + tokens_value: 5, + errors: "0".to_owned(), + errors_value: 0, + latency: "1 ms".to_owned(), + latency_value: Some(1), + max_latency: "1 ms".to_owned(), + max_latency_value: Some(1), + min_latency: "1 ms".to_owned(), + min_latency_value: Some(1), + success_rate: "100.0%".to_owned(), + success_rate_value: Some(100.0), + }], + summary: vec![TrendSummaryView { + label: "输入".to_owned(), + value: "2".to_owned(), + ratio: None, + }], + }; + let value = serde_json::to_value(data).unwrap(); + assert_eq!(value["kind"], "usage"); + assert_eq!(value["points"][0]["requestsValue"], 1); +} - #[test] - fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() { - let coverage = CostCoverageView { - known: 1, - partial: 0, - unknown: 0, - not_billable: 0, - }; - let token_details = TokenDetailsView { - input_tokens: Some(1), - output_tokens: Some(2), - cached_tokens: None, - cache_write_tokens: None, - reasoning_tokens: None, - image_input_tokens: None, - image_output_tokens: None, - total_tokens: Some(3), - input_tokens_display: "1".to_owned(), - output_tokens_display: "2".to_owned(), - cached_tokens_display: "-".to_owned(), - cache_write_tokens_display: "-".to_owned(), - reasoning_tokens_display: "-".to_owned(), - image_input_tokens_display: "-".to_owned(), - image_output_tokens_display: "-".to_owned(), - total_tokens_display: "3".to_owned(), - }; - let cursor = CursorWire { - observed_at: Utc.timestamp_opt(0, 0).single().unwrap(), - stable_id: "request_1".to_owned(), - }; - let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap(); - assert!(value.to_string().contains("known")); - assert!(!value.to_string().contains("secret")); - } +#[test] +fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() { + let coverage = CostCoverageView { + known: 1, + partial: 0, + unknown: 0, + not_billable: 0, + }; + let token_details = TokenDetailsView { + input_tokens: Some(1), + output_tokens: Some(2), + cached_tokens: None, + cache_write_tokens: None, + reasoning_tokens: None, + image_input_tokens: None, + image_output_tokens: None, + total_tokens: Some(3), + input_tokens_display: "1".to_owned(), + output_tokens_display: "2".to_owned(), + cached_tokens_display: "-".to_owned(), + cache_write_tokens_display: "-".to_owned(), + reasoning_tokens_display: "-".to_owned(), + image_input_tokens_display: "-".to_owned(), + image_output_tokens_display: "-".to_owned(), + total_tokens_display: "3".to_owned(), + }; + let cursor = CursorWire { + observed_at: Utc.timestamp_opt(0, 0).single().unwrap(), + stable_id: "request_1".to_owned(), + }; + let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap(); + assert!(value.to_string().contains("known")); + assert!(!value.to_string().contains("secret")); +} - #[test] - fn billing_view_should_preserve_the_original_detail_contract() { - let value = serde_json::to_value(BillingView { - input_amount_display: "$0.03".to_owned(), - output_amount_display: "$0.00".to_owned(), - cache_read_amount_display: "$0.14".to_owned(), - cache_write_amount_display: "$0.00".to_owned(), - standard_amount_display: "$0.17".to_owned(), - total_amount_display: "$0.17".to_owned(), - input_price_display: "$10.0000 / 1M Token".to_owned(), - output_price_display: "$60.0000 / 1M Token".to_owned(), - cache_read_price_display: "$1.0000 / 1M Token".to_owned(), - cache_write_price_display: "$12.5000 / 1M Token".to_owned(), - service_tier_display: "Fast".to_owned(), - multiplier_display: "1.00x".to_owned(), - }) - .expect("billing view"); +#[test] +fn billing_view_should_preserve_the_original_detail_contract() { + let value = serde_json::to_value(BillingView { + input_amount_display: "$0.03".to_owned(), + output_amount_display: "$0.00".to_owned(), + cache_read_amount_display: "$0.14".to_owned(), + cache_write_amount_display: "$0.00".to_owned(), + standard_amount_display: "$0.17".to_owned(), + total_amount_display: "$0.17".to_owned(), + input_price_display: "$10.0000 / 1M Token".to_owned(), + output_price_display: "$60.0000 / 1M Token".to_owned(), + cache_read_price_display: "$1.0000 / 1M Token".to_owned(), + cache_write_price_display: "$12.5000 / 1M Token".to_owned(), + service_tier_display: "Fast".to_owned(), + multiplier_display: "1.00x".to_owned(), + }) + .expect("billing view"); - assert_eq!(value["inputAmountDisplay"], "$0.03"); - assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token"); - assert_eq!(value["serviceTierDisplay"], "Fast"); - assert_eq!(value["multiplierDisplay"], "1.00x"); - } + assert_eq!(value["inputAmountDisplay"], "$0.03"); + assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token"); + assert_eq!(value["serviceTierDisplay"], "Fast"); + assert_eq!(value["multiplierDisplay"], "1.00x"); +} - #[test] - fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() { - for (domain, expected) in [ - (DesktopReleaseStatus::Unchecked, "unchecked"), - (DesktopReleaseStatus::Current, "aligned"), - (DesktopReleaseStatus::UpdateAvailable, "review_required"), - (DesktopReleaseStatus::Failed, "check_failed"), - ] { - let status = DashboardDesktopReleaseStatusView::from(domain); - assert_eq!(serde_json::to_value(status).unwrap(), expected); - } +#[test] +fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() { + for (domain, expected) in [ + (DesktopReleaseStatus::Unchecked, "unchecked"), + (DesktopReleaseStatus::Current, "aligned"), + (DesktopReleaseStatus::UpdateAvailable, "review_required"), + (DesktopReleaseStatus::Failed, "check_failed"), + ] { + let status = DashboardDesktopReleaseStatusView::from(domain); + assert_eq!(serde_json::to_value(status).unwrap(), expected); } } @@ -272,7 +207,7 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli use gateway_api::admin::observability; use tower::ServiceExt as _; - use super::{AdminTestFixture, AdminTestState}; + use crate::admin::{AdminTestFixture, AdminTestState}; let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); @@ -344,7 +279,7 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() { use gateway_api::admin::observability; use tower::ServiceExt as _; - use super::{AdminTestFixture, AdminTestState}; + use crate::admin::{AdminTestFixture, AdminTestState}; let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); @@ -451,7 +386,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() { use gateway_api::admin::observability; use tower::ServiceExt as _; - use super::{AdminTestFixture, AdminTestState}; + use crate::admin::{AdminTestFixture, AdminTestState}; let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); @@ -536,7 +471,7 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() { use gateway_api::admin::observability; use tower::ServiceExt as _; - use super::{AdminTestFixture, AdminTestState}; + use crate::admin::{AdminTestFixture, AdminTestState}; let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); @@ -678,7 +613,7 @@ async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() { use gateway_api::admin::observability; use tower::ServiceExt as _; - use super::{AdminTestFixture, AdminTestState}; + use crate::admin::{AdminTestFixture, AdminTestState}; let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); @@ -714,7 +649,7 @@ async fn usage_route_should_expose_image_and_websocket_facts() { use gateway_api::admin::observability; use tower::ServiceExt as _; - use super::{AdminTestFixture, AdminTestState}; + use crate::admin::{AdminTestFixture, AdminTestState}; let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); diff --git a/backend/crates/gateway-api/tests/architecture.rs b/backend/crates/gateway-api/tests/architecture.rs index 39a54ae0b..07c25307c 100644 --- a/backend/crates/gateway-api/tests/architecture.rs +++ b/backend/crates/gateway-api/tests/architecture.rs @@ -38,7 +38,11 @@ fn source_tree_should_match_frozen_machine_manifest() { "src/admin/backups.rs", "src/admin/client_keys.rs", "src/admin/mod.rs", - "src/admin/observability.rs", + "src/admin/observability/mod.rs", + "src/admin/observability/presenter.rs", + "src/admin/observability/query.rs", + "src/admin/observability/routes.rs", + "src/admin/observability/wire.rs", "src/admin/presenter.rs", "src/admin/settings.rs", "src/admin/system.rs", @@ -73,7 +77,9 @@ fn test_tree_should_match_frozen_rust_mirror() { "tests/admin/auth.rs", "tests/admin/client_keys.rs", "tests/admin/mod.rs", - "tests/admin/observability.rs", + "tests/admin/observability/mod.rs", + "tests/admin/observability/query.rs", + "tests/admin/observability/response.rs", "tests/admin/settings.rs", "tests/admin/system.rs", "tests/admin/wire.rs", diff --git a/backend/crates/gateway-store/src/admin_adapter.rs b/backend/crates/gateway-store/src/admin_adapter.rs new file mode 100644 index 000000000..2b76466e0 --- /dev/null +++ b/backend/crates/gateway-store/src/admin_adapter.rs @@ -0,0 +1,296 @@ +//! Admin 认证与设置 adapter。 + +use super::*; + +pub(crate) struct AdminAuthStoreAdapter { + pub(crate) security: postgres::PgAdminSecurityAuditRepository, + pub(crate) settings: postgres::PgRuntimeSettingsRepository, + pub(crate) state: redis::RedisAdminAuthStateRepository, +} + +pub(crate) struct AdminSettingsStoreAdapter { + pub(crate) control_plane: postgres::PgControlPlaneRepository, +} + +#[async_trait::async_trait] +impl SettingsStore for AdminSettingsStoreAdapter { + async fn load_runtime_settings(&self) -> AdminStoreResult { + let snapshot = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane) + .await + .map_err(|error| admin_store_error("runtime settings", error))?; + admin_runtime_settings(snapshot.settings) + } + + async fn admin_api_key_exists(&self) -> AdminStoreResult { + postgres::ControlPlaneRepository::load_control_plane(&self.control_plane) + .await + .map(|snapshot| snapshot.settings.admin_api_key.is_some()) + .map_err(|error| admin_store_error("admin API key", error)) + } + + async fn replace_runtime_settings( + &self, + command: ReplaceRuntimeSettings, + context: &MutationContext, + ) -> AdminStoreResult { + let current = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane) + .await + .map_err(|error| admin_store_error("runtime settings", error))?; + let replacement = postgres::ControlPlaneReplacement { + settings: postgres::RuntimeSettingsUpdate { + admin_api_key: current.settings.admin_api_key, + refresh_margin_seconds: command.refresh_margin_seconds, + refresh_concurrency: command.refresh_concurrency, + max_concurrent_per_account: command.max_concurrent_per_account, + request_interval_ms: command.request_interval_ms, + rotation_strategy: command.rotation_strategy.as_str().to_owned(), + model_mappings: store_model_mappings(command.model_mappings), + usage_retention_days: command.usage_retention_days, + ops_event_retention_days: command.ops_event_retention_days, + audit_retention_days: command.audit_retention_days, + }, + audit: mutation_audit( + context, + "settings.replace", + "runtime_settings", + "1", + vec![ + "model_mappings_json".to_owned(), + "refresh_margin_seconds".to_owned(), + "refresh_concurrency".to_owned(), + "max_concurrent_per_account".to_owned(), + "request_interval_ms".to_owned(), + "rotation_strategy".to_owned(), + "retention".to_owned(), + ], + ), + }; + let snapshot = postgres::ControlPlaneRepository::replace_control_plane( + &self.control_plane, + replacement, + ) + .await + .map_err(|error| admin_store_error("runtime settings", error))?; + admin_runtime_settings(snapshot.settings) + } + + async fn replace_admin_api_key( + &self, + key: AdminApiKey, + context: &MutationContext, + ) -> AdminStoreResult { + self.replace_admin_api_key_value(Some(key.expose_for_auth().to_owned()), context) + .await + } + + async fn delete_admin_api_key( + &self, + context: &MutationContext, + ) -> AdminStoreResult { + self.replace_admin_api_key_value(None, context).await + } +} + +impl AdminSettingsStoreAdapter { + async fn replace_admin_api_key_value( + &self, + admin_api_key: Option, + context: &MutationContext, + ) -> AdminStoreResult { + let exists = admin_api_key.is_some(); + let revision = postgres::ControlPlaneRepository::replace_admin_api_key( + &self.control_plane, + admin_api_key, + mutation_audit( + context, + if exists { + "admin_api_key.replace" + } else { + "admin_api_key.delete" + }, + "runtime_settings", + "1", + vec!["admin_api_key".to_owned()], + ), + ) + .await + .map_err(|error| admin_store_error("admin API key", error))?; + Ok(AdminApiKeyMutation { + config_revision: admin_revision(revision)?, + exists, + }) + } +} + +pub(crate) fn admin_runtime_settings( + settings: postgres::RuntimeSettings, +) -> AdminStoreResult { + let rotation_strategy = AdminRotationStrategy::parse(settings.rotation_strategy.as_str()) + .ok_or_else(|| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + "runtime settings", + "rotation strategy is invalid", + ) + })?; + let model_mappings = settings + .model_mappings + .into_iter() + .map(|(public, upstream)| { + let public = gateway_core::routing::PublicModelId::new(public).map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + "runtime settings", + "public model mapping is invalid", + ) + })?; + let upstream = gateway_core::routing::UpstreamModelId::new(upstream).map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + "runtime settings", + "upstream model mapping is invalid", + ) + })?; + Ok((public, upstream)) + }) + .collect::>()?; + Ok(AdminRuntimeSettings { + config_revision: admin_revision(settings.config_revision)?, + model_mappings, + refresh_margin_seconds: settings.refresh_margin_seconds, + refresh_concurrency: settings.refresh_concurrency, + max_concurrent_per_account: settings.max_concurrent_per_account, + request_interval_ms: settings.request_interval_ms, + rotation_strategy, + usage_retention_days: settings.usage_retention_days, + ops_event_retention_days: settings.ops_event_retention_days, + audit_retention_days: settings.audit_retention_days, + updated_at: settings.updated_at, + }) +} + +pub(crate) fn store_model_mappings( + mappings: ModelMappings, +) -> std::collections::BTreeMap { + mappings + .into_iter() + .map(|(public, upstream)| (public.as_str().to_owned(), upstream.as_str().to_owned())) + .collect() +} + +#[async_trait::async_trait] +impl AuthStore for AdminAuthStoreAdapter { + async fn load_password_hash(&self, admin_user_id: &str) -> AdminStoreResult> { + postgres::AdminSecurityAuditRepository::password_hash(&self.security, admin_user_id) + .await + .map_err(|error| admin_store_error("admin authentication", error)) + } + + async fn create_password_hash_if_absent( + &self, + admin_user_id: &str, + password_hash: &str, + ) -> AdminStoreResult { + postgres::AdminSecurityAuditRepository::create_password_hash_if_absent( + &self.security, + admin_user_id, + password_hash, + ) + .await + .map_err(|error| admin_store_error("admin authentication", error)) + } + + async fn load_admin_api_key(&self) -> AdminStoreResult> { + postgres::RuntimeSettingsRepository::load_runtime_settings(&self.settings) + .await + .map(|settings| settings.admin_api_key.map(AdminApiKey::new)) + .map_err(|error| admin_store_error("admin API key", error)) + } + + async fn load_session(&self, session_id: &str) -> AdminStoreResult> { + redis::AdminAuthStateRepository::load_admin_session(&self.state, session_id) + .await + .map(|session| { + session.map(|record| AdminSession { + admin_user_id: record.admin_user_id, + expires_at: record.expires_at, + }) + }) + .map_err(|error| admin_store_error("admin session", error)) + } + + async fn store_session( + &self, + session_id: &str, + session: &AdminSession, + ) -> AdminStoreResult<()> { + redis::AdminAuthStateRepository::store_admin_session( + &self.state, + session_id, + &redis::AdminSessionRecord { + admin_user_id: session.admin_user_id.clone(), + expires_at: session.expires_at, + }, + ) + .await + .map_err(|error| admin_store_error("admin session", error)) + } + + async fn delete_session(&self, session_id: &str) -> AdminStoreResult> { + redis::AdminAuthStateRepository::delete_admin_session(&self.state, session_id) + .await + .map(|session| { + session.map(|record| AdminSession { + admin_user_id: record.admin_user_id, + expires_at: record.expires_at, + }) + }) + .map_err(|error| admin_store_error("admin session", error)) + } + + async fn append_audit_event(&self, event: AdminAuditModel) -> AdminStoreResult<()> { + let config_revision = event + .config_revision + .map(|revision| i64::try_from(revision.get())) + .transpose() + .map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + "admin audit", + "config revision is outside the supported range", + ) + })?; + let actor_kind = match event.actor_kind { + gateway_admin::model::auth::AuditActorKind::AdminSession => { + postgres::AdminAuditActorKind::AdminSession + } + gateway_admin::model::auth::AuditActorKind::AdminApiKey => { + postgres::AdminAuditActorKind::AdminApiKey + } + gateway_admin::model::auth::AuditActorKind::System => { + postgres::AdminAuditActorKind::System + } + gateway_admin::model::auth::AuditActorKind::Anonymous => { + postgres::AdminAuditActorKind::Anonymous + } + }; + postgres::AdminSecurityAuditRepository::append_admin_audit_event( + &self.security, + postgres::AdminAuditEvent { + id: event.id, + actor_kind, + actor_admin_user_id: event.actor_admin_user_id, + actor_ref: event.actor_ref, + admin_request_id: event.request_id, + action: event.action, + entity_kind: event.entity_kind, + entity_ref: event.entity_ref, + config_revision, + changed_fields: event.changed_fields, + created_at: event.occurred_at, + }, + ) + .await + .map_err(|error| admin_store_error("admin audit", error)) + } +} diff --git a/backend/crates/gateway-store/src/bundle.rs b/backend/crates/gateway-store/src/bundle.rs new file mode 100644 index 000000000..8ba6c8e87 --- /dev/null +++ b/backend/crates/gateway-store/src/bundle.rs @@ -0,0 +1,211 @@ +//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。 + +use super::*; + +/// 已完成连接、迁移与 cooldown hydration 的 Store 能力集合。 +pub struct StoreBundle { + admin_ports: AdminStorePorts, + core_ports: CoreStorePorts, + provider_ports: ProviderStorePorts, + worker_leader_lease: Arc, + health_probes: Vec>, + worker_contributions: Vec, +} + +impl StoreBundle { + #[must_use] + pub fn admin_ports(&self) -> AdminStorePorts { + self.admin_ports.clone() + } + + #[must_use] + pub fn core_ports(&self) -> CoreStorePorts { + self.core_ports.clone() + } + + #[must_use] + pub fn provider_ports(&self) -> ProviderStorePorts { + self.provider_ports.clone() + } + + #[must_use] + pub fn worker_leader_lease(&self) -> Arc { + Arc::clone(&self.worker_leader_lease) + } + + #[must_use] + pub fn health_probes(&self) -> Vec> { + self.health_probes.clone() + } + + pub fn take_worker_contributions(&mut self) -> Vec { + std::mem::take(&mut self.worker_contributions) + } +} + +/// 在返回 Bundle 前完成全部 Store 启动屏障。 +pub async fn initialize(mut config: StoreConfig) -> StoreResult { + const REDIS_NAMESPACE: &str = "codex-proxy-rs"; + + config.resolve_and_validate(std::path::Path::new("."))?; + let pool = postgres::connect_and_migrate(&config.database_url()?, config.pool).await?; + let redis_client = ::redis::Client::open(config.redis_url()?) + .map_err(|_| redis_unavailable("create Redis client"))?; + let redis_connection = redis_client + .get_connection_manager() + .await + .map_err(|_| redis_unavailable("connect Redis manager"))?; + + let provider_accounts = Arc::new(postgres::PgProviderAccountRepository::new(pool.clone())); + let cooldowns = Arc::new(redis::RedisCredentialCooldownRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?); + let account_store = Arc::new(redis::CooldownCachingProviderAccountStore::new( + provider_accounts, + cooldowns.clone(), + )); + let hydrated = account_store.hydrate(std::time::SystemTime::now()).await; + tracing::info!(hydrated, "账号 cooldown Redis 热缓存重建完成"); + + let credential_leases = + redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?; + let provider_leases = Arc::new(redis::RedisProviderLeaseCoordinator::new( + credential_leases.clone(), + )); + let provider_session_affinity = Arc::new(redis::RedisProviderSessionAffinityRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?); + let credential_state = Arc::new(redis::RedisCredentialStateRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?); + let runtime_policy = Arc::new(postgres::PgRuntimeSettingsRepository::new(pool.clone())); + let oauth_pending = Arc::new(redis::RedisOAuthPendingFlowRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?); + + let admin_ports = AdminStorePorts::new( + Arc::new(postgres::PgAdminAccountStore::new(pool.clone())), + Arc::new(AdminAuthStoreAdapter { + security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()), + settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()), + state: redis::RedisAdminAuthStateRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?, + }), + Arc::new(postgres::PgAdminClientKeyStore::new(pool.clone())), + Arc::new(postgres::PgAdminObservabilityStore::new( + pool.clone(), + Some(credential_leases.clone()), + )), + Arc::new(AdminSettingsStoreAdapter { + control_plane: postgres::PgControlPlaneRepository::new(pool.clone()), + }), + backup_ports(pool.clone(), &config)?, + ); + + let execution_repository = Arc::new(postgres::PgExecutionStore::new(pool.clone())); + let (execution, execution_writer) = + postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository)); + let execution = Arc::new(execution); + let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone())); + let admissions: Arc = Arc::new( + redis::RedisClientAdmissionRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?, + ); + let circuits: Arc = + Arc::new(redis::RedisProviderCircuitRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + gateway_core::engine::execution::ProviderCircuitPolicy::default(), + )?); + // Continuation affinity 是下一轮请求的路由事实，Core 必须直接等待 Redis 确认。 + let continuation: Arc = + Arc::new(redis::RedisNativeContinuationRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?); + let (admissions, admission_release_writer) = + redis::BufferedClientAdmissionPort::new(admissions); + let (circuits, circuit_feedback_writer) = redis::BufferedProviderCircuitPort::new(circuits); + let core_ports = CoreStorePorts::new( + execution, + ( + Arc::new(admissions), + Arc::new(postgres::PgClientAdmissionRecoveryRepository::new( + pool.clone(), + )), + ), + Arc::new(circuits), + continuation, + ( + Arc::new(postgres::PgRuntimeSnapshotRepository::new(pool.clone())), + Arc::new(redis::RedisRuntimeChangeRepository::new( + redis_client, + REDIS_NAMESPACE, + )?), + ), + Arc::new(postgres::PgClientApiKeyUsageSink::new(pool.clone())), + ); + + let provider_ports = ProviderStorePorts::new( + account_store, + provider_leases, + provider_session_affinity, + Arc::new(redis::RedisProviderSessionExclusionRepository::new( + redis_connection.clone(), + REDIS_NAMESPACE, + )?), + credential_state.clone(), + credential_state, + cooldowns, + runtime_policy, + oauth_pending, + ); + let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new( + credential_leases, + )); + let health_probes: Vec> = vec![ + Arc::new(PostgresHealthProbe { pool: pool.clone() }), + Arc::new(RedisHealthProbe { + connection: redis_connection, + }), + ]; + let worker_contributions = store_worker_contributions( + execution_repository, + execution_writer, + admission_release_writer, + circuit_feedback_writer, + retention, + )?; + Ok(StoreBundle { + admin_ports, + core_ports, + provider_ports, + worker_leader_lease, + health_probes, + worker_contributions, + }) +} + +/// 构造备份控制面的仓储、导出器与对象存储适配器。 +pub(crate) fn backup_ports( + pool: sqlx::PgPool, + config: &StoreConfig, +) -> StoreResult { + let staging = Arc::new(backup::staging::StagingArea::open( + std::path::PathBuf::from(".runtime/data/backup-staging"), + backup::staging::DEFAULT_MAX_ARCHIVE_BYTES, + )?); + let repository = Arc::new(postgres::PgBackupRepository::new(pool)); + let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new( + staging, + config.database.url.clone(), + config.database.password.clone(), + )); + let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new()); + Ok(BackupStorePorts::new(repository, dump, object_store)) +} diff --git a/backend/crates/gateway-store/src/config.rs b/backend/crates/gateway-store/src/config.rs new file mode 100644 index 000000000..91769b3c4 --- /dev/null +++ b/backend/crates/gateway-store/src/config.rs @@ -0,0 +1,141 @@ +//! Store 启动配置、环境变量解析与校验。 + +use super::*; + +pub(crate) const DATABASE_URL_ENV: &str = "CPR_DATABASE_URL"; +pub(crate) const REDIS_URL_ENV: &str = "CPR_REDIS_URL"; +pub(crate) const DATABASE_PASSWORD_ENV: &str = "CPR_DATABASE_PASSWORD"; +pub(crate) const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD"; + +/// Store 自己拥有并校验的启动配置。 +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoreConfig { + pub(crate) database: StoreConnectionConfig, + pub(crate) redis: StoreConnectionConfig, + #[serde(default)] + pub(crate) pool: StorePoolConfig, +} + +/// PostgreSQL 连接池预算；acquire 超时决定池耗尽时快速失败而非排队积压。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct StorePoolConfig { + pub max_connections: u32, + pub acquire_timeout_seconds: u64, +} + +impl Default for StorePoolConfig { + fn default() -> Self { + Self { + max_connections: 10, + acquire_timeout_seconds: 5, + } + } +} + +impl StorePoolConfig { + fn validate(&self) -> StoreResult<()> { + if self.max_connections == 0 || self.acquire_timeout_seconds == 0 { + return Err(StoreError::InvalidData { + entity: "store config", + message: "pool limits must be positive".to_owned(), + }); + } + Ok(()) + } +} + +impl StoreConfig { + pub fn resolve_and_validate(&mut self, _source_dir: &std::path::Path) -> StoreResult<()> { + if let Some(url) = optional_environment_value(DATABASE_URL_ENV)? { + self.database.url = url; + } + if let Some(url) = optional_environment_value(REDIS_URL_ENV)? { + self.redis.url = url; + } + if let Some(password) = optional_environment_value(DATABASE_PASSWORD_ENV)? { + self.database.password = password; + } + if let Some(password) = optional_environment_value(REDIS_PASSWORD_ENV)? { + self.redis.password = password; + } + self.database.validate("database")?; + self.redis.validate("redis")?; + self.pool.validate()?; + Ok(()) + } + + pub(crate) fn database_url(&self) -> StoreResult { + self.database.connection_url("database") + } + + pub(crate) fn redis_url(&self) -> StoreResult { + self.redis.connection_url("redis") + } +} + +pub(crate) fn optional_environment_value(name: &'static str) -> StoreResult> { + match std::env::var(name) { + Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData { + entity: "store config", + message: format!("environment variable {name} is empty"), + }), + Ok(value) => Ok(Some(value)), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData { + entity: "store config", + message: format!("environment variable {name} is not Unicode"), + }), + } +} + +impl fmt::Debug for StoreConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StoreConfig") + .field("database", &"[REDACTED]") + .field("redis", &"[REDACTED]") + .finish() + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StoreConnectionConfig { + pub(crate) url: String, + pub(crate) password: String, +} + +impl StoreConnectionConfig { + fn validate(&self, field: &'static str) -> StoreResult<()> { + require_nonempty("store config", field, &self.url)?; + if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(StoreError::InvalidData { + entity: "store config", + message: format!("{field}.password must be exactly 48 hexadecimal characters"), + }); + } + self.connection_url(field).map(|_| ()) + } + + fn connection_url(&self, field: &'static str) -> StoreResult { + let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData { + entity: "store config", + message: format!("{field}.url is invalid"), + })?; + if url.password().is_some() { + return Err(StoreError::InvalidData { + entity: "store config", + message: format!("{field}.url must not contain a password"), + }); + } + url.set_password(Some(&self.password)) + .map_err(|()| StoreError::InvalidData { + entity: "store config", + message: format!("{field}.url cannot carry credentials"), + })?; + Ok(url.to_string()) + } +} diff --git a/backend/crates/gateway-store/src/lib.rs b/backend/crates/gateway-store/src/lib.rs index c069237bf..fab901ebd 100644 --- a/backend/crates/gateway-store/src/lib.rs +++ b/backend/crates/gateway-store/src/lib.rs @@ -29,1092 +29,18 @@ use gateway_core::task::{ use serde::Deserialize; use serde_json::{Map, Value}; -const DATABASE_URL_ENV: &str = "CPR_DATABASE_URL"; -const REDIS_URL_ENV: &str = "CPR_REDIS_URL"; -const DATABASE_PASSWORD_ENV: &str = "CPR_DATABASE_PASSWORD"; -const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD"; +mod admin_adapter; +mod bundle; +mod config; +mod value; +mod workers; pub mod backup; pub mod postgres; pub mod redis; -/// 发生错误的基础设施边界。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum StoreBackend { - PostgreSql, - Redis, -} - -/// 上层状态机需要区分的稳定冲突类型。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ConflictKind { - StaleRevision, - AlreadyFinalized, - DownstreamAlreadyCommitted, - RequestNotRunning, - InvalidTransition, - LeaseLost, - FencingTokenStale, -} - -/// Store adapter 的稳定错误边界。 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum StoreError { - #[error("{backend:?} store is unavailable: {message}")] - Unavailable { - backend: StoreBackend, - message: String, - }, - #[error("{entity} {id} was not found")] - NotFound { entity: &'static str, id: String }, - #[error("store conflict for {entity} {id}: {kind:?}")] - Conflict { - entity: &'static str, - id: String, - kind: ConflictKind, - }, - #[error("invalid persisted {entity}: {message}")] - InvalidData { - entity: &'static str, - message: String, - }, -} - -pub type StoreResult = Result; - -/// Store 自己拥有并校验的启动配置。 -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StoreConfig { - database: StoreConnectionConfig, - redis: StoreConnectionConfig, - #[serde(default)] - pool: StorePoolConfig, -} - -/// PostgreSQL 连接池预算；acquire 超时决定池耗尽时快速失败而非排队积压。 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(deny_unknown_fields, default)] -pub struct StorePoolConfig { - pub max_connections: u32, - pub acquire_timeout_seconds: u64, -} - -impl Default for StorePoolConfig { - fn default() -> Self { - Self { - max_connections: 10, - acquire_timeout_seconds: 5, - } - } -} - -impl StorePoolConfig { - fn validate(&self) -> StoreResult<()> { - if self.max_connections == 0 || self.acquire_timeout_seconds == 0 { - return Err(StoreError::InvalidData { - entity: "store config", - message: "pool limits must be positive".to_owned(), - }); - } - Ok(()) - } -} - -impl StoreConfig { - pub fn resolve_and_validate(&mut self, _source_dir: &std::path::Path) -> StoreResult<()> { - if let Some(url) = optional_environment_value(DATABASE_URL_ENV)? { - self.database.url = url; - } - if let Some(url) = optional_environment_value(REDIS_URL_ENV)? { - self.redis.url = url; - } - if let Some(password) = optional_environment_value(DATABASE_PASSWORD_ENV)? { - self.database.password = password; - } - if let Some(password) = optional_environment_value(REDIS_PASSWORD_ENV)? { - self.redis.password = password; - } - self.database.validate("database")?; - self.redis.validate("redis")?; - self.pool.validate()?; - Ok(()) - } - - fn database_url(&self) -> StoreResult { - self.database.connection_url("database") - } - - fn redis_url(&self) -> StoreResult { - self.redis.connection_url("redis") - } -} - -fn optional_environment_value(name: &'static str) -> StoreResult> { - match std::env::var(name) { - Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData { - entity: "store config", - message: format!("environment variable {name} is empty"), - }), - Ok(value) => Ok(Some(value)), - Err(std::env::VarError::NotPresent) => Ok(None), - Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData { - entity: "store config", - message: format!("environment variable {name} is not Unicode"), - }), - } -} - -impl fmt::Debug for StoreConfig { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("StoreConfig") - .field("database", &"[REDACTED]") - .field("redis", &"[REDACTED]") - .finish() - } -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -struct StoreConnectionConfig { - url: String, - password: String, -} - -impl StoreConnectionConfig { - fn validate(&self, field: &'static str) -> StoreResult<()> { - require_nonempty("store config", field, &self.url)?; - if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit()) - { - return Err(StoreError::InvalidData { - entity: "store config", - message: format!("{field}.password must be exactly 48 hexadecimal characters"), - }); - } - self.connection_url(field).map(|_| ()) - } - - fn connection_url(&self, field: &'static str) -> StoreResult { - let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData { - entity: "store config", - message: format!("{field}.url is invalid"), - })?; - if url.password().is_some() { - return Err(StoreError::InvalidData { - entity: "store config", - message: format!("{field}.url must not contain a password"), - }); - } - url.set_password(Some(&self.password)) - .map_err(|()| StoreError::InvalidData { - entity: "store config", - message: format!("{field}.url cannot carry credentials"), - })?; - Ok(url.to_string()) - } -} - -/// 已完成连接、迁移与 cooldown hydration 的 Store 能力集合。 -pub struct StoreBundle { - admin_ports: AdminStorePorts, - core_ports: CoreStorePorts, - provider_ports: ProviderStorePorts, - worker_leader_lease: Arc, - health_probes: Vec>, - worker_contributions: Vec, -} - -impl StoreBundle { - #[must_use] - pub fn admin_ports(&self) -> AdminStorePorts { - self.admin_ports.clone() - } - - #[must_use] - pub fn core_ports(&self) -> CoreStorePorts { - self.core_ports.clone() - } - - #[must_use] - pub fn provider_ports(&self) -> ProviderStorePorts { - self.provider_ports.clone() - } - - #[must_use] - pub fn worker_leader_lease(&self) -> Arc { - Arc::clone(&self.worker_leader_lease) - } - - #[must_use] - pub fn health_probes(&self) -> Vec> { - self.health_probes.clone() - } - - pub fn take_worker_contributions(&mut self) -> Vec { - std::mem::take(&mut self.worker_contributions) - } -} - -/// 在返回 Bundle 前完成全部 Store 启动屏障。 -pub async fn initialize(mut config: StoreConfig) -> StoreResult { - const REDIS_NAMESPACE: &str = "codex-proxy-rs"; - - config.resolve_and_validate(std::path::Path::new("."))?; - let pool = postgres::connect_and_migrate(&config.database_url()?, config.pool).await?; - let redis_client = ::redis::Client::open(config.redis_url()?) - .map_err(|_| redis_unavailable("create Redis client"))?; - let redis_connection = redis_client - .get_connection_manager() - .await - .map_err(|_| redis_unavailable("connect Redis manager"))?; - - let provider_accounts = Arc::new(postgres::PgProviderAccountRepository::new(pool.clone())); - let cooldowns = Arc::new(redis::RedisCredentialCooldownRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?); - let account_store = Arc::new(redis::CooldownCachingProviderAccountStore::new( - provider_accounts, - cooldowns.clone(), - )); - let hydrated = account_store.hydrate(std::time::SystemTime::now()).await; - tracing::info!(hydrated, "账号 cooldown Redis 热缓存重建完成"); - - let credential_leases = - redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?; - let provider_leases = Arc::new(redis::RedisProviderLeaseCoordinator::new( - credential_leases.clone(), - )); - let provider_session_affinity = Arc::new(redis::RedisProviderSessionAffinityRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?); - let credential_state = Arc::new(redis::RedisCredentialStateRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?); - let runtime_policy = Arc::new(postgres::PgRuntimeSettingsRepository::new(pool.clone())); - let oauth_pending = Arc::new(redis::RedisOAuthPendingFlowRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?); - - let admin_ports = AdminStorePorts::new( - Arc::new(postgres::PgAdminAccountStore::new(pool.clone())), - Arc::new(AdminAuthStoreAdapter { - security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()), - settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()), - state: redis::RedisAdminAuthStateRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?, - }), - Arc::new(postgres::PgAdminClientKeyStore::new(pool.clone())), - Arc::new(postgres::PgAdminObservabilityStore::new( - pool.clone(), - Some(credential_leases.clone()), - )), - Arc::new(AdminSettingsStoreAdapter { - control_plane: postgres::PgControlPlaneRepository::new(pool.clone()), - }), - backup_ports(pool.clone(), &config)?, - ); - - let execution_repository = Arc::new(postgres::PgExecutionStore::new(pool.clone())); - let (execution, execution_writer) = - postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository)); - let execution = Arc::new(execution); - let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone())); - let admissions: Arc = Arc::new( - redis::RedisClientAdmissionRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?, - ); - let circuits: Arc = - Arc::new(redis::RedisProviderCircuitRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - gateway_core::engine::execution::ProviderCircuitPolicy::default(), - )?); - // Continuation affinity 是下一轮请求的路由事实，Core 必须直接等待 Redis 确认。 - let continuation: Arc = - Arc::new(redis::RedisNativeContinuationRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?); - let (admissions, admission_release_writer) = - redis::BufferedClientAdmissionPort::new(admissions); - let (circuits, circuit_feedback_writer) = redis::BufferedProviderCircuitPort::new(circuits); - let core_ports = CoreStorePorts::new( - execution, - ( - Arc::new(admissions), - Arc::new(postgres::PgClientAdmissionRecoveryRepository::new( - pool.clone(), - )), - ), - Arc::new(circuits), - continuation, - ( - Arc::new(postgres::PgRuntimeSnapshotRepository::new(pool.clone())), - Arc::new(redis::RedisRuntimeChangeRepository::new( - redis_client, - REDIS_NAMESPACE, - )?), - ), - Arc::new(postgres::PgClientApiKeyUsageSink::new(pool.clone())), - ); - - let provider_ports = ProviderStorePorts::new( - account_store, - provider_leases, - provider_session_affinity, - Arc::new(redis::RedisProviderSessionExclusionRepository::new( - redis_connection.clone(), - REDIS_NAMESPACE, - )?), - credential_state.clone(), - credential_state, - cooldowns, - runtime_policy, - oauth_pending, - ); - let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new( - credential_leases, - )); - let health_probes: Vec> = vec![ - Arc::new(PostgresHealthProbe { pool: pool.clone() }), - Arc::new(RedisHealthProbe { - connection: redis_connection, - }), - ]; - let worker_contributions = store_worker_contributions( - execution_repository, - execution_writer, - admission_release_writer, - circuit_feedback_writer, - retention, - )?; - Ok(StoreBundle { - admin_ports, - core_ports, - provider_ports, - worker_leader_lease, - health_probes, - worker_contributions, - }) -} - -/// 构造备份控制面的仓储、导出器与对象存储适配器。 -fn backup_ports(pool: sqlx::PgPool, config: &StoreConfig) -> StoreResult { - let staging = Arc::new(backup::staging::StagingArea::open( - std::path::PathBuf::from(".runtime/data/backup-staging"), - backup::staging::DEFAULT_MAX_ARCHIVE_BYTES, - )?); - let repository = Arc::new(postgres::PgBackupRepository::new(pool)); - let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new( - staging, - config.database.url.clone(), - config.database.password.clone(), - )); - let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new()); - Ok(BackupStorePorts::new(repository, dump, object_store)) -} - -fn store_worker_contributions( - execution: Arc, - execution_writer: postgres::ExecutionObservationWriter, - admission_release_writer: redis::ClientAdmissionReleaseWriter, - circuit_feedback_writer: redis::ProviderCircuitFeedbackWriter, - retention: Arc, -) -> StoreResult> { - let stale_id = WorkerId::try_new(WorkerKind::StaleModelRequestRecovery, "postgres") - .map_err(worker_definition_error)?; - let retention_id = - WorkerId::try_new(WorkerKind::Retention, "postgres").map_err(worker_definition_error)?; - let ops_flush_id = - WorkerId::try_new(WorkerKind::OpsFlush, "postgres").map_err(worker_definition_error)?; - let admission_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_admission") - .map_err(worker_definition_error)?; - let circuit_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_circuit") - .map_err(worker_definition_error)?; - let ops_flush_restart = - DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60)) - .map_err(worker_definition_error)?; - Ok(vec![ - WorkerContribution::Registration(scheduled_worker( - stale_id, - Duration::from_secs(30), - Box::new(StaleModelRequestRecoveryTask { execution }), - )?), - WorkerContribution::Registration(scheduled_worker( - retention_id, - Duration::from_secs(60 * 60), - Box::new(RetentionTask { retention }), - )?), - WorkerContribution::Registration( - WorkerRegistration::try_new( - ops_flush_id, - WorkerRunnable::Daemon { - restart: ops_flush_restart, - task: Box::new(execution_writer), - }, - ) - .map_err(worker_definition_error)?, - ), - WorkerContribution::Registration( - WorkerRegistration::try_new( - admission_flush_id, - WorkerRunnable::Daemon { - restart: ops_flush_restart, - task: Box::new(admission_release_writer), - }, - ) - .map_err(worker_definition_error)?, - ), - WorkerContribution::Registration( - WorkerRegistration::try_new( - circuit_flush_id, - WorkerRunnable::Daemon { - restart: ops_flush_restart, - task: Box::new(circuit_feedback_writer), - }, - ) - .map_err(worker_definition_error)?, - ), - ]) -} - -fn scheduled_worker( - id: WorkerId, - interval: Duration, - task: Box, -) -> StoreResult { - let schedule = WorkerSchedule::try_new( - interval, - Duration::from_secs(1), - Duration::from_secs(60), - Duration::from_secs(15 * 60), - Duration::from_secs(5 * 60), - ) - .map_err(worker_definition_error)?; - let lease = WorkerLeaseRequest::try_new(id.clone(), schedule.leader_lease_ttl()) - .map_err(worker_definition_error)?; - WorkerRegistration::try_new( - id, - WorkerRunnable::Scheduled { - schedule, - lease: Some(lease), - task, - }, - ) - .map_err(worker_definition_error) -} - -fn worker_definition_error(error: gateway_core::task::WorkerDefinitionError) -> StoreError { - StoreError::InvalidData { - entity: "store worker plan", - message: error.to_string(), - } -} - -struct StaleModelRequestRecoveryTask { - execution: Arc, -} - -impl ScheduledTask for StaleModelRequestRecoveryTask { - fn run_cycle( - &self, - _context: WorkerCycleContext, - ) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> { - Box::pin(async move { - gateway_core::engine::ExecutionStore::recover_expired( - self.execution.as_ref(), - SystemTime::now(), - ) - .await - .map(|_| ()) - .map_err(|_| WorkerTaskError::safe("stale request recovery failed")) - }) - } -} - -struct RetentionTask { - retention: Arc, -} - -impl ScheduledTask for RetentionTask { - fn run_cycle( - &self, - _context: WorkerCycleContext, - ) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> { - Box::pin(async move { - let settings = - postgres::RetentionRepository::load_retention_settings(self.retention.as_ref()) - .await - .map_err(|_| WorkerTaskError::safe("retention settings read failed"))?; - postgres::RetentionRepository::apply_retention( - self.retention.as_ref(), - chrono::Utc::now(), - settings, - ) - .await - .map(|_| ()) - .map_err(|_| WorkerTaskError::safe("retention cleanup failed")) - }) - } -} - -struct PostgresHealthProbe { - pool: sqlx::PgPool, -} - -impl HealthProbe for PostgresHealthProbe { - fn name(&self) -> &'static str { - "postgres" - } - - fn check(&self) -> futures::future::BoxFuture<'_, HealthState> { - Box::pin(async move { - match sqlx::query_scalar::<_, i32>("select 1") - .fetch_one(&self.pool) - .await - { - Ok(1) => HealthState::Healthy, - Ok(_) => HealthState::Unhealthy("PostgreSQL health result is invalid".to_owned()), - Err(_) => HealthState::Unhealthy("PostgreSQL is unavailable".to_owned()), - } - }) - } -} - -struct RedisHealthProbe { - connection: ::redis::aio::ConnectionManager, -} - -struct AdminAuthStoreAdapter { - security: postgres::PgAdminSecurityAuditRepository, - settings: postgres::PgRuntimeSettingsRepository, - state: redis::RedisAdminAuthStateRepository, -} - -struct AdminSettingsStoreAdapter { - control_plane: postgres::PgControlPlaneRepository, -} - -#[async_trait::async_trait] -impl SettingsStore for AdminSettingsStoreAdapter { - async fn load_runtime_settings(&self) -> AdminStoreResult { - let snapshot = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane) - .await - .map_err(|error| admin_store_error("runtime settings", error))?; - admin_runtime_settings(snapshot.settings) - } - - async fn admin_api_key_exists(&self) -> AdminStoreResult { - postgres::ControlPlaneRepository::load_control_plane(&self.control_plane) - .await - .map(|snapshot| snapshot.settings.admin_api_key.is_some()) - .map_err(|error| admin_store_error("admin API key", error)) - } - - async fn replace_runtime_settings( - &self, - command: ReplaceRuntimeSettings, - context: &MutationContext, - ) -> AdminStoreResult { - let current = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane) - .await - .map_err(|error| admin_store_error("runtime settings", error))?; - let replacement = postgres::ControlPlaneReplacement { - settings: postgres::RuntimeSettingsUpdate { - admin_api_key: current.settings.admin_api_key, - refresh_margin_seconds: command.refresh_margin_seconds, - refresh_concurrency: command.refresh_concurrency, - max_concurrent_per_account: command.max_concurrent_per_account, - request_interval_ms: command.request_interval_ms, - rotation_strategy: command.rotation_strategy.as_str().to_owned(), - model_mappings: store_model_mappings(command.model_mappings), - usage_retention_days: command.usage_retention_days, - ops_event_retention_days: command.ops_event_retention_days, - audit_retention_days: command.audit_retention_days, - }, - audit: mutation_audit( - context, - "settings.replace", - "runtime_settings", - "1", - vec![ - "model_mappings_json".to_owned(), - "refresh_margin_seconds".to_owned(), - "refresh_concurrency".to_owned(), - "max_concurrent_per_account".to_owned(), - "request_interval_ms".to_owned(), - "rotation_strategy".to_owned(), - "retention".to_owned(), - ], - ), - }; - let snapshot = postgres::ControlPlaneRepository::replace_control_plane( - &self.control_plane, - replacement, - ) - .await - .map_err(|error| admin_store_error("runtime settings", error))?; - admin_runtime_settings(snapshot.settings) - } - - async fn replace_admin_api_key( - &self, - key: AdminApiKey, - context: &MutationContext, - ) -> AdminStoreResult { - self.replace_admin_api_key_value(Some(key.expose_for_auth().to_owned()), context) - .await - } - - async fn delete_admin_api_key( - &self, - context: &MutationContext, - ) -> AdminStoreResult { - self.replace_admin_api_key_value(None, context).await - } -} - -impl AdminSettingsStoreAdapter { - async fn replace_admin_api_key_value( - &self, - admin_api_key: Option, - context: &MutationContext, - ) -> AdminStoreResult { - let exists = admin_api_key.is_some(); - let revision = postgres::ControlPlaneRepository::replace_admin_api_key( - &self.control_plane, - admin_api_key, - mutation_audit( - context, - if exists { - "admin_api_key.replace" - } else { - "admin_api_key.delete" - }, - "runtime_settings", - "1", - vec!["admin_api_key".to_owned()], - ), - ) - .await - .map_err(|error| admin_store_error("admin API key", error))?; - Ok(AdminApiKeyMutation { - config_revision: admin_revision(revision)?, - exists, - }) - } -} - -fn admin_runtime_settings( - settings: postgres::RuntimeSettings, -) -> AdminStoreResult { - let rotation_strategy = AdminRotationStrategy::parse(settings.rotation_strategy.as_str()) - .ok_or_else(|| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - "runtime settings", - "rotation strategy is invalid", - ) - })?; - let model_mappings = settings - .model_mappings - .into_iter() - .map(|(public, upstream)| { - let public = gateway_core::routing::PublicModelId::new(public).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - "runtime settings", - "public model mapping is invalid", - ) - })?; - let upstream = gateway_core::routing::UpstreamModelId::new(upstream).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - "runtime settings", - "upstream model mapping is invalid", - ) - })?; - Ok((public, upstream)) - }) - .collect::>()?; - Ok(AdminRuntimeSettings { - config_revision: admin_revision(settings.config_revision)?, - model_mappings, - refresh_margin_seconds: settings.refresh_margin_seconds, - refresh_concurrency: settings.refresh_concurrency, - max_concurrent_per_account: settings.max_concurrent_per_account, - request_interval_ms: settings.request_interval_ms, - rotation_strategy, - usage_retention_days: settings.usage_retention_days, - ops_event_retention_days: settings.ops_event_retention_days, - audit_retention_days: settings.audit_retention_days, - updated_at: settings.updated_at, - }) -} - -fn store_model_mappings(mappings: ModelMappings) -> std::collections::BTreeMap { - mappings - .into_iter() - .map(|(public, upstream)| (public.as_str().to_owned(), upstream.as_str().to_owned())) - .collect() -} - -pub(crate) fn store_revision(revision: AdminRevision) -> AdminStoreResult { - Revision::new(revision.get()).map_err(|error| admin_store_error("config revision", error)) -} - -pub(crate) fn admin_revision(revision: Revision) -> AdminStoreResult { - AdminRevision::new(revision.get()).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - "config revision", - "config revision is invalid", - ) - }) -} - -pub(crate) fn mutation_audit( - context: &MutationContext, - action: &str, - entity_kind: &str, - entity_ref: &str, - changed_fields: Vec, -) -> postgres::AdminAuditEvent { - let (actor_kind, actor_admin_user_id, actor_ref) = match &context.actor { - MutationActor::AdminSession { admin_user_id } => ( - postgres::AdminAuditActorKind::AdminSession, - Some(admin_user_id.clone()), - admin_user_id.clone(), - ), - MutationActor::AdminApiKey => ( - postgres::AdminAuditActorKind::AdminApiKey, - None, - "admin_api_key".to_owned(), - ), - MutationActor::System => ( - postgres::AdminAuditActorKind::System, - None, - "system".to_owned(), - ), - }; - postgres::AdminAuditEvent { - id: format!("audit_{}", uuid::Uuid::now_v7().simple()), - actor_kind, - actor_admin_user_id, - actor_ref, - admin_request_id: Some(context.request_id.clone()), - action: action.to_owned(), - entity_kind: entity_kind.to_owned(), - entity_ref: entity_ref.to_owned(), - config_revision: None, - changed_fields, - created_at: chrono::Utc::now(), - } -} - -#[async_trait::async_trait] -impl AuthStore for AdminAuthStoreAdapter { - async fn load_password_hash(&self, admin_user_id: &str) -> AdminStoreResult> { - postgres::AdminSecurityAuditRepository::password_hash(&self.security, admin_user_id) - .await - .map_err(|error| admin_store_error("admin authentication", error)) - } - - async fn create_password_hash_if_absent( - &self, - admin_user_id: &str, - password_hash: &str, - ) -> AdminStoreResult { - postgres::AdminSecurityAuditRepository::create_password_hash_if_absent( - &self.security, - admin_user_id, - password_hash, - ) - .await - .map_err(|error| admin_store_error("admin authentication", error)) - } - - async fn load_admin_api_key(&self) -> AdminStoreResult> { - postgres::RuntimeSettingsRepository::load_runtime_settings(&self.settings) - .await - .map(|settings| settings.admin_api_key.map(AdminApiKey::new)) - .map_err(|error| admin_store_error("admin API key", error)) - } - - async fn load_session(&self, session_id: &str) -> AdminStoreResult> { - redis::AdminAuthStateRepository::load_admin_session(&self.state, session_id) - .await - .map(|session| { - session.map(|record| AdminSession { - admin_user_id: record.admin_user_id, - expires_at: record.expires_at, - }) - }) - .map_err(|error| admin_store_error("admin session", error)) - } - - async fn store_session( - &self, - session_id: &str, - session: &AdminSession, - ) -> AdminStoreResult<()> { - redis::AdminAuthStateRepository::store_admin_session( - &self.state, - session_id, - &redis::AdminSessionRecord { - admin_user_id: session.admin_user_id.clone(), - expires_at: session.expires_at, - }, - ) - .await - .map_err(|error| admin_store_error("admin session", error)) - } - - async fn delete_session(&self, session_id: &str) -> AdminStoreResult> { - redis::AdminAuthStateRepository::delete_admin_session(&self.state, session_id) - .await - .map(|session| { - session.map(|record| AdminSession { - admin_user_id: record.admin_user_id, - expires_at: record.expires_at, - }) - }) - .map_err(|error| admin_store_error("admin session", error)) - } - - async fn append_audit_event(&self, event: AdminAuditModel) -> AdminStoreResult<()> { - let config_revision = event - .config_revision - .map(|revision| i64::try_from(revision.get())) - .transpose() - .map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - "admin audit", - "config revision is outside the supported range", - ) - })?; - let actor_kind = match event.actor_kind { - gateway_admin::model::auth::AuditActorKind::AdminSession => { - postgres::AdminAuditActorKind::AdminSession - } - gateway_admin::model::auth::AuditActorKind::AdminApiKey => { - postgres::AdminAuditActorKind::AdminApiKey - } - gateway_admin::model::auth::AuditActorKind::System => { - postgres::AdminAuditActorKind::System - } - gateway_admin::model::auth::AuditActorKind::Anonymous => { - postgres::AdminAuditActorKind::Anonymous - } - }; - postgres::AdminSecurityAuditRepository::append_admin_audit_event( - &self.security, - postgres::AdminAuditEvent { - id: event.id, - actor_kind, - actor_admin_user_id: event.actor_admin_user_id, - actor_ref: event.actor_ref, - admin_request_id: event.request_id, - action: event.action, - entity_kind: event.entity_kind, - entity_ref: event.entity_ref, - config_revision, - changed_fields: event.changed_fields, - created_at: event.occurred_at, - }, - ) - .await - .map_err(|error| admin_store_error("admin audit", error)) - } -} - -pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> AdminStoreError { - let kind = match error { - StoreError::NotFound { .. } => AdminStoreErrorKind::NotFound, - StoreError::Conflict { - kind: ConflictKind::StaleRevision, - .. - } => AdminStoreErrorKind::StaleRevision, - StoreError::Conflict { .. } => AdminStoreErrorKind::Conflict, - StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid, - StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable, - }; - AdminStoreError::new(kind, resource, "store operation failed") -} - -impl HealthProbe for RedisHealthProbe { - fn name(&self) -> &'static str { - "redis" - } - - fn check(&self) -> futures::future::BoxFuture<'_, HealthState> { - Box::pin(async move { - let mut connection = self.connection.clone(); - match ::redis::cmd("PING") - .query_async::(&mut connection) - .await - { - Ok(response) if response == "PONG" => HealthState::Healthy, - Ok(_) => HealthState::Unhealthy("Redis health result is invalid".to_owned()), - Err(_) => HealthState::Unhealthy("Redis is unavailable".to_owned()), - } - }) - } -} - -/// 正整数 revision 或 fencing token。 -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Revision(NonZeroU64); - -impl Revision { - pub fn new(value: u64) -> StoreResult { - NonZeroU64::new(value) - .map(Self) - .ok_or_else(|| StoreError::InvalidData { - entity: "revision", - message: "must be greater than zero".to_owned(), - }) - } - - #[must_use] - pub const fn get(self) -> u64 { - self.0.get() - } -} - -/// `numeric(20,10)` 可无损表达的非负金额。 -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct DecimalAmount(String); - -impl DecimalAmount { - #[must_use] - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for DecimalAmount { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(&self.0) - } -} - -impl FromStr for DecimalAmount { - type Err = StoreError; - - fn from_str(input: &str) -> Result { - let input = input.trim(); - let mut parts = input.split('.'); - let whole = parts.next().unwrap_or_default(); - let fraction = parts.next(); - let valid = !whole.is_empty() - && whole.len() <= 10 - && whole.bytes().all(|byte| byte.is_ascii_digit()) - && parts.next().is_none() - && fraction.is_none_or(|value| { - !value.is_empty() - && value.len() <= 10 - && value.bytes().all(|byte| byte.is_ascii_digit()) - }); - if !valid { - return Err(StoreError::InvalidData { - entity: "decimal amount", - message: "expected a non-negative numeric(20,10) value".to_owned(), - }); - } - - let whole = whole.trim_start_matches('0'); - let whole = if whole.is_empty() { "0" } else { whole }; - let fraction = fraction.unwrap_or_default().trim_end_matches('0'); - let canonical = if fraction.is_empty() { - whole.to_owned() - } else { - format!("{whole}.{fraction}") - }; - Ok(Self(canonical)) - } -} - -/// Provider-owned JSON object。Store 只验证 object 与大小，不解释内部 key。 -#[derive(Clone, PartialEq)] -pub struct JsonObject(Map); - -impl JsonObject { - pub fn try_from_value( - entity: &'static str, - value: Value, - max_serialized_bytes: usize, - ) -> StoreResult { - let serialized_bytes = serde_json::to_vec(&value) - .map_err(|error| StoreError::InvalidData { - entity, - message: error.to_string(), - })? - .len(); - let Value::Object(fields) = value else { - return Err(StoreError::InvalidData { - entity, - message: "top-level JSON value must be an object".to_owned(), - }); - }; - if serialized_bytes > max_serialized_bytes { - return Err(StoreError::InvalidData { - entity, - message: format!("serialized JSON exceeds {max_serialized_bytes} bytes"), - }); - } - Ok(Self(fields)) - } - - #[must_use] - pub fn as_value(&self) -> Value { - Value::Object(self.0.clone()) - } - - #[must_use] - pub fn fields(&self) -> &Map { - &self.0 - } -} - -impl fmt::Debug for JsonObject { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("JsonObject([REDACTED])") - } -} - -pub(crate) fn require_nonempty( - entity: &'static str, - field: &'static str, - value: &str, -) -> StoreResult<()> { - if value.trim().is_empty() { - Err(StoreError::InvalidData { - entity, - message: format!("{field} must not be empty"), - }) - } else { - Ok(()) - } -} - -pub(crate) fn postgres_unavailable(operation: &'static str) -> StoreError { - StoreError::Unavailable { - backend: StoreBackend::PostgreSql, - message: operation.to_owned(), - } -} - -pub(crate) fn redis_unavailable(operation: &'static str) -> StoreError { - StoreError::Unavailable { - backend: StoreBackend::Redis, - message: operation.to_owned(), - } -} +pub(crate) use admin_adapter::*; +pub use bundle::*; +pub use config::*; +pub use value::*; +pub(crate) use workers::*; diff --git a/backend/crates/gateway-store/src/postgres/observability.rs b/backend/crates/gateway-store/src/postgres/observability.rs deleted file mode 100644 index 9e560a64e..000000000 --- a/backend/crates/gateway-store/src/postgres/observability.rs +++ /dev/null @@ -1,3475 +0,0 @@ -//! 从 `model_requests`、`ops_events` 与账号公共投影读取观测事实。 - -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - str::FromStr, -}; - -use async_trait::async_trait; -use chrono::{DateTime, TimeDelta, Utc}; -use gateway_admin::{ - model::observability as admin_observability, - ports::store::{AdminStoreResult, ObservabilityStore as AdminObservabilityStore}, -}; -use sqlx::{PgPool, Postgres, QueryBuilder, Row}; - -use crate::{ - DecimalAmount, StoreError, StoreResult, admin_store_error, postgres_unavailable, - redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository}, - require_nonempty, -}; - -const MAX_PAGE_SIZE: u16 = 100; -const MAX_FILTER_BYTES: usize = 256; -const MAX_SEARCH_BYTES: usize = 512; -const MAX_ACCOUNT_IDS: usize = 200; -/// 概览卡只展示最近使用的四个账号；完整账号用量由账号管理页单独查询。 -const DASHBOARD_ACCOUNT_LIMIT: u16 = 4; -const DIAGNOSTIC_LIMIT: i64 = 100; -const ACCOUNT_USAGE_TIMELINE_HOURS: i64 = 24; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ObservabilityRange { - pub start: DateTime, - pub end: DateTime, -} - -impl ObservabilityRange { - pub fn new(start: DateTime, end: DateTime) -> StoreResult { - if end.signed_duration_since(start) <= TimeDelta::zero() { - return Err(invalid("time range must be positive")); - } - Ok(Self { start, end }) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ObservabilityPageSize(u16); - -impl ObservabilityPageSize { - pub fn new(value: u16) -> StoreResult { - if value == 0 || value > MAX_PAGE_SIZE { - return Err(invalid("page size must be between 1 and 100")); - } - Ok(Self(value)) - } - - #[must_use] - pub const fn get(self) -> u16 { - self.0 - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ObservabilityPageNumber(u32); - -impl ObservabilityPageNumber { - pub fn new(value: u32) -> StoreResult { - if value == 0 { - return Err(invalid("page number must be positive")); - } - Ok(Self(value)) - } - - #[must_use] - pub const fn get(self) -> u32 { - self.0 - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ObservabilityCursor { - pub observed_at: DateTime, - pub stable_id: String, -} - -impl ObservabilityCursor { - pub fn new(observed_at: DateTime, stable_id: impl Into) -> StoreResult { - let stable_id = stable_id.into(); - validate_text(&stable_id, MAX_FILTER_BYTES, "cursor ID")?; - Ok(Self { - observed_at, - stable_id, - }) - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct UsageRecordFilter { - pub client_api_key_ref: Option, - pub request_id: Option, - pub provider_account_ref: Option, - pub operation: Option, - pub provider_kind: Option, - pub model: Option, - pub outcome: Option, - pub status_code: Option, - pub transport: Option, - pub attempt_index: Option, - pub response_id: Option, - pub upstream_request_id: Option, - pub search: Option, -} - -impl UsageRecordFilter { - pub fn validate(&self) -> StoreResult<()> { - for (value, field) in [ - (self.client_api_key_ref.as_deref(), "client API key filter"), - (self.request_id.as_deref(), "request ID filter"), - ( - self.provider_account_ref.as_deref(), - "provider account filter", - ), - (self.operation.as_deref(), "operation filter"), - (self.provider_kind.as_deref(), "provider filter"), - (self.model.as_deref(), "model filter"), - (self.outcome.as_deref(), "outcome filter"), - (self.transport.as_deref(), "transport filter"), - ( - self.upstream_request_id.as_deref(), - "upstream request ID filter", - ), - ] { - validate_optional_text(value, MAX_FILTER_BYTES, field)?; - } - validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?; - if self - .status_code - .is_some_and(|status| !(100..=599).contains(&status)) - { - return Err(invalid("status code filter must be between 100 and 599")); - } - if self - .attempt_index - .is_some_and(|index| index == 0 || i32::try_from(index).is_err()) - { - return Err(invalid("attempt index filter is out of range")); - } - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UsageRecordQuery { - pub range: ObservabilityRange, - pub filter: UsageRecordFilter, - pub cursor: Option, - pub page: ObservabilityPageNumber, - pub page_size: ObservabilityPageSize, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct OpsErrorFilter { - pub client_api_key_ref: Option, - pub request_id: Option, - pub provider_account_ref: Option, - pub provider_kind: Option, - pub operation: Option, - pub model: Option, - pub transport: Option, - pub attempt_index: Option, - pub response_id: Option, - pub upstream_request_id: Option, - pub failure_kind: Option, - pub status_code: Option, - pub search: Option, -} - -impl OpsErrorFilter { - pub fn validate(&self) -> StoreResult<()> { - for (value, field) in [ - (self.client_api_key_ref.as_deref(), "client API key filter"), - (self.request_id.as_deref(), "request ID filter"), - ( - self.provider_account_ref.as_deref(), - "provider account filter", - ), - (self.provider_kind.as_deref(), "provider filter"), - (self.operation.as_deref(), "operation filter"), - (self.model.as_deref(), "model filter"), - (self.transport.as_deref(), "transport filter"), - ( - self.upstream_request_id.as_deref(), - "upstream request ID filter", - ), - (self.failure_kind.as_deref(), "failure kind filter"), - ] { - validate_optional_text(value, MAX_FILTER_BYTES, field)?; - } - validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?; - if self - .status_code - .is_some_and(|status| !(100..=599).contains(&status)) - { - return Err(invalid("status code filter must be between 100 and 599")); - } - if self - .attempt_index - .is_some_and(|index| index == 0 || i32::try_from(index).is_err()) - { - return Err(invalid("attempt index filter is out of range")); - } - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct OpsErrorQuery { - pub range: ObservabilityRange, - pub filter: OpsErrorFilter, - pub cursor: Option, - pub page: ObservabilityPageNumber, - pub page_size: ObservabilityPageSize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DiagnosticDimension { - Provider, - Model, - Account, - ApiKey, - Transport, - Failure, - Status, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CurrencyCostTotal { - pub currency: String, - pub amount: DecimalAmount, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct CostCoverage { - pub provider_reported_count: u64, - pub calculated_count: u64, - pub unavailable_count: u64, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct RequestMetrics { - pub request_count: u64, - pub success_count: u64, - pub failure_count: u64, - pub cancelled_count: u64, - pub incomplete_count: u64, - pub caller_error_count: u64, - pub input_tokens: u64, - pub output_tokens: u64, - pub cached_tokens: u64, - pub cache_write_tokens: u64, - pub reasoning_tokens: u64, - pub total_tokens: u64, - pub first_token_latency_sum: u64, - pub first_token_latency_count: u64, - pub latency_sum: u64, - pub latency_count: u64, - pub max_latency_ms: Option, - pub min_latency_ms: Option, - /// 分母：`input_tokens is not null`，即上游确实报告过 input token 事实的请求。 - pub cache_eligible_request_count: u64, - /// 分子：分母集合中 `cached_tokens > 0` 的请求。 - pub cache_hit_request_count: u64, - pub latency_percentiles: LatencyPercentiles, - pub first_token_latency_percentiles: LatencyPercentiles, -} - -impl RequestMetrics { - /// 请求级 cache hit rate；没有 input token 事实时返回 `None`。 - #[must_use] - pub fn cache_hit_request_rate(&self) -> Option { - (self.cache_eligible_request_count > 0) - .then(|| self.cache_hit_request_count as f64 / self.cache_eligible_request_count as f64) - } -} - -/// PostgreSQL `percentile_cont` 的非负、有限毫秒值；bits 保留插值小数且可安全比较。 -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct PercentileMilliseconds(u64); - -impl PercentileMilliseconds { - fn new(value: f64) -> StoreResult { - if !value.is_finite() || value < 0.0 { - return Err(postgres_unavailable("decode latency percentile")); - } - Ok(Self(value.to_bits())) - } - - #[must_use] - pub const fn as_f64(self) -> f64 { - f64::from_bits(self.0) - } -} - -impl std::fmt::Debug for PercentileMilliseconds { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.as_f64().fmt(formatter) - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct LatencyPercentiles { - pub p50_ms: Option, - pub p95_ms: Option, - pub p99_ms: Option, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AttemptMetrics { - pub attempt_count: u64, - pub success_count: u64, - pub failure_count: u64, - pub cancelled_count: u64, - pub incomplete_count: u64, - pub rate_limited_count: u64, - pub auth_failure_count: u64, - pub provider_5xx_count: u64, - pub cost_coverage: CostCoverage, - pub costs: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ObservationGranularity { - FifteenMinutes, - Hour, - Day, -} - -impl ObservationGranularity { - #[must_use] - pub const fn seconds(self) -> i64 { - match self { - Self::FifteenMinutes => 15 * 60, - Self::Hour => 60 * 60, - Self::Day => 24 * 60 * 60, - } - } - - const fn sql_interval(self) -> &'static str { - match self { - Self::FifteenMinutes => "15 minutes", - Self::Hour => "1 hour", - Self::Day => "1 day", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RequestMetricPoint { - pub bucket_start: DateTime, - pub granularity: ObservationGranularity, - pub metrics: RequestMetrics, - pub cost_coverage: CostCoverage, - pub costs: Vec, -} - -/// 已完整交付且由 Provider 计算费用的请求事实。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CalculatedUsageBillingFact { - pub bucket_start: DateTime, - pub provider_kind: String, - pub upstream_model_id: String, - pub service_tier: Option, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub total: CurrencyCostTotal, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ProviderAccountMetrics { - pub total: u64, - pub enabled: u64, - pub unavailable: u64, - pub active: u64, - pub expired: u64, - pub quota_exhausted: u64, - pub disabled: u64, - pub banned: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountRequestBucket { - pub bucket_start: DateTime, - pub request_count: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountUsageObservation { - pub account_id: String, - pub provider_kind: String, - pub authentication_kind: String, - pub name: String, - pub email: Option, - pub plan_type: Option, - pub enabled: bool, - pub availability: String, - pub request_count: u64, - pub success_count: u64, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub reasoning_tokens: Option, - pub image_input_tokens: Option, - pub image_output_tokens: Option, - pub image_request_count: u64, - pub image_request_failed_count: u64, - pub total_tokens: Option, - pub cost_coverage: CostCoverage, - pub costs: Vec, - pub last_used_at: Option>, - pub request_buckets: Vec, - pub models: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountModelUsageObservation { - pub model: String, - pub request_count: u64, - pub success_count: u64, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub reasoning_tokens: Option, - pub image_input_tokens: Option, - pub image_output_tokens: Option, - pub image_request_count: u64, - pub image_request_failed_count: u64, - pub total_tokens: Option, - pub cost_coverage: CostCoverage, - pub costs: Vec, - pub last_used_at: DateTime, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountUsageQuery { - pub range: ObservabilityRange, - pub account_ids: Option>, - pub limit: u16, - include_hourly_request_buckets: bool, -} - -impl ProviderAccountUsageQuery { - pub fn for_accounts(range: ObservabilityRange, account_ids: Vec) -> StoreResult { - if account_ids.is_empty() || account_ids.len() > MAX_ACCOUNT_IDS { - return Err(invalid( - "account usage query requires between 1 and 200 IDs", - )); - } - validate_account_ids(&account_ids)?; - Ok(Self { - range, - limit: u16::try_from(account_ids.len()) - .map_err(|_| invalid("account usage query is too large"))?, - account_ids: Some(account_ids), - include_hourly_request_buckets: false, - }) - } - - pub fn recent(range: ObservabilityRange, limit: u16) -> StoreResult { - if limit == 0 || usize::from(limit) > MAX_ACCOUNT_IDS { - return Err(invalid("account usage limit must be between 1 and 200")); - } - Ok(Self { - range, - account_ids: None, - limit, - include_hourly_request_buckets: false, - }) - } - - pub fn with_hourly_request_buckets(mut self) -> StoreResult { - if self.range.end.signed_duration_since(self.range.start) - > TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS) - { - return Err(invalid("account request timeline cannot exceed 24 hours")); - } - self.include_hourly_request_buckets = true; - Ok(self) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DashboardObservation { - pub range: ObservabilityRange, - pub requests: RequestMetrics, - pub attempts: AttemptMetrics, - pub provider_accounts: ProviderAccountMetrics, - pub trend: Vec, - pub account_usage: Vec, - pub recent_requests: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UsageRecord { - pub id: String, - pub client_api_key_ref: String, - pub config_revision: u64, - pub protocol: String, - pub operation: String, - pub endpoint: String, - pub client_transport: String, - pub requested_model_id: String, - pub provider_kind: Option, - pub provider_account_ref: Option, - pub provider_account_name: Option, - pub provider_account_email: Option, - pub provider_account_authentication_kind: Option, - pub upstream_model_id: Option, - pub upstream_transport: Option, - pub http_version: Option, - pub websocket_pool: Option, - pub service_tier: Option, - pub provider_metadata_json: Option, - pub attempt_count: u32, - pub upstream_send_state: String, - pub downstream_committed_at: Option>, - pub outcome: String, - pub client_status_code: Option, - pub upstream_status_code: Option, - pub client_response_id: Option, - pub upstream_request_id: Option, - pub upstream_response_id: Option, - pub error_kind: Option, - pub provider_error_code: Option, - pub error_message: Option, - pub retry_after_ms: Option, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub reasoning_tokens: Option, - pub image_input_tokens: Option, - pub image_output_tokens: Option, - pub total_tokens: Option, - pub cost_source: String, - pub cost_amount: Option, - pub cost_currency: Option, - pub transport_decision_wait_ms: Option, - pub connect_ms: Option, - pub headers_ms: Option, - pub first_event_ms: Option, - pub first_reasoning_ms: Option, - pub first_text_ms: Option, - pub first_token_ms: Option, - pub provider_processing_ms: Option, - pub latency_ms: Option, - pub client_ip: Option, - pub user_agent: Option, - pub reasoning_effort: Option, - pub reasoning_preset: Option, - pub request_kind: Option, - pub subagent_kind: Option, - pub compact: bool, - pub image_generation_requested: bool, - pub image_generation_succeeded: Option, - pub started_at: DateTime, - pub deadline_at: DateTime, - pub completed_at: Option>, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UsageRecordPage { - pub items: Vec, - pub total: u64, - pub next_cursor: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UsageAttemptObservation { - pub source: String, - pub id: String, - pub attempt_index: u32, - pub component: String, - pub operation: String, - pub provider_kind: Option, - pub provider_account_ref: Option, - pub provider_account_name: Option, - pub provider_account_email: Option, - pub provider_account_authentication_kind: Option, - pub upstream_model_id: Option, - pub upstream_transport: Option, - pub upstream_send_state: Option, - pub outcome: String, - pub downstream_committed: bool, - pub status_code: Option, - pub provider_error_code: Option, - pub failure_kind: Option, - pub retry_after_ms: Option, - pub upstream_request_id: Option, - pub latency_ms: Option, - pub message: Option, - pub input_tokens: Option, - pub output_tokens: Option, - pub cached_tokens: Option, - pub cache_write_tokens: Option, - pub reasoning_tokens: Option, - pub total_tokens: Option, - pub cost_source: Option, - pub cost_amount: Option, - pub cost_currency: Option, - pub occurred_at: DateTime, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UsageRecordDetail { - pub request: UsageRecord, - pub attempts: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderObservation { - pub provider_kind: String, - pub request_count: u64, - pub attempt_count: u64, - pub failure_count: u64, - pub total_tokens: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UsageOverview { - pub range: ObservabilityRange, - pub requests: RequestMetrics, - pub attempts: AttemptMetrics, - pub providers: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DiagnosticObservation { - pub key: String, - pub name: String, - pub request_count: u64, - pub success_count: u64, - pub failure_count: u64, - pub attempt_count: u64, - pub total_tokens: u64, - pub average_latency_ms: Option, - pub latency_p95_ms: Option, - pub cost_coverage: CostCoverage, - pub costs: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct OpsErrorRecord { - pub source: String, - pub event_id: String, - pub request_id: Option, - pub attempt_index: Option, - pub client_api_key_ref: Option, - pub component: String, - pub operation: String, - pub endpoint: Option, - pub provider_kind: Option, - pub provider_account_ref: Option, - pub provider_account_name: Option, - pub provider_account_email: Option, - pub provider_account_authentication_kind: Option, - pub upstream_model_id: Option, - pub upstream_transport: Option, - pub failure_kind: String, - pub client_status_code: Option, - pub upstream_status_code: Option, - pub provider_error_code: Option, - pub client_response_id: Option, - pub upstream_request_id: Option, - pub latency_ms: Option, - pub message: String, - pub occurrence_count: u32, - pub occurred_at: DateTime, - pub stable_sort_id: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct OpsErrorPage { - pub items: Vec, - pub total: u64, - pub next_cursor: Option, -} - -#[async_trait] -pub trait ObservabilityRepository: Send + Sync { - async fn dashboard_summary( - &self, - range: ObservabilityRange, - ) -> StoreResult; - async fn dashboard_trend( - &self, - range: ObservabilityRange, - ) -> StoreResult>; - async fn usage_trend( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - ) -> StoreResult>; - async fn usage_calculated_billing_facts( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - ) -> StoreResult>; - async fn provider_account_usage( - &self, - query: ProviderAccountUsageQuery, - ) -> StoreResult>; - async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult; - async fn usage_record_detail(&self, request_id: &str) -> StoreResult; - async fn usage_summary( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - ) -> StoreResult; - async fn usage_diagnostics( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - dimension: DiagnosticDimension, - ) -> StoreResult>; - async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult; -} - -#[derive(Clone)] -pub struct PgObservabilityRepository { - pool: PgPool, -} - -impl PgObservabilityRepository { - #[must_use] - pub const fn new(pool: PgPool) -> Self { - Self { pool } - } -} - -/// `gateway-admin` 观测端口的 PostgreSQL adapter。 -/// -/// SQL 查询与内部投影继续由 [`PgObservabilityRepository`] 唯一拥有；本类型只负责 -/// Admin UTC 领域模型与持久化投影之间的无格式化转换。 -#[derive(Clone)] -pub struct PgAdminObservabilityStore { - repository: PgObservabilityRepository, - runtime_signals: Option, -} - -impl PgAdminObservabilityStore { - #[must_use] - pub fn new(pool: PgPool, runtime_signals: Option) -> Self { - Self { - repository: PgObservabilityRepository::new(pool), - runtime_signals, - } - } -} - -#[async_trait] -impl ObservabilityRepository for PgObservabilityRepository { - async fn dashboard_summary( - &self, - range: ObservabilityRange, - ) -> StoreResult { - let filter = UsageRecordFilter::default(); - let account_usage_range = ObservabilityRange::new( - range.end - TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS), - range.end, - )?; - let account_usage_query = - ProviderAccountUsageQuery::recent(account_usage_range, DASHBOARD_ACCOUNT_LIMIT)? - .with_hourly_request_buckets()?; - let recent_query = UsageRecordQuery { - range, - filter: UsageRecordFilter { - outcome: Some("succeeded".to_owned()), - ..UsageRecordFilter::default() - }, - cursor: None, - page: ObservabilityPageNumber::new(1)?, - page_size: ObservabilityPageSize::new(10)?, - }; - // 分两批并发：页面延迟从六路之和降为两批各自最慢者之和，同时单次 - // 渲染最多占用 3 个池连接，不与数据面写路径争抢整个连接池。 - let (requests, attempts, provider_accounts) = futures::try_join!( - request_metrics(&self.pool, range, &filter), - attempt_metrics(&self.pool, range, &filter), - provider_account_metrics(&self.pool, range.end), - )?; - let (trend, account_usage, recent_requests) = futures::try_join!( - request_metric_series(&self.pool, range, &filter), - provider_account_usage(&self.pool, account_usage_query), - async { Ok(list_usage_records(&self.pool, recent_query).await?.items) }, - )?; - Ok(DashboardObservation { - range, - requests, - attempts, - provider_accounts, - trend, - account_usage, - recent_requests, - }) - } - - async fn dashboard_trend( - &self, - range: ObservabilityRange, - ) -> StoreResult> { - request_metric_series(&self.pool, range, &UsageRecordFilter::default()).await - } - - async fn usage_trend( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - ) -> StoreResult> { - request_metric_series(&self.pool, range, &filter).await - } - - async fn usage_calculated_billing_facts( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - ) -> StoreResult> { - calculated_usage_billing_facts(&self.pool, range, &filter).await - } - - async fn provider_account_usage( - &self, - query: ProviderAccountUsageQuery, - ) -> StoreResult> { - provider_account_usage(&self.pool, query).await - } - - async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult { - list_usage_records(&self.pool, query).await - } - - async fn usage_record_detail(&self, request_id: &str) -> StoreResult { - usage_record_detail(&self.pool, request_id).await - } - - async fn usage_summary( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - ) -> StoreResult { - filter.validate()?; - let requests = request_metrics(&self.pool, range, &filter).await?; - let attempts = attempt_metrics(&self.pool, range, &filter).await?; - let providers = provider_observations(&self.pool, range, &filter).await?; - Ok(UsageOverview { - range, - requests, - attempts, - providers, - }) - } - - async fn usage_diagnostics( - &self, - range: ObservabilityRange, - filter: UsageRecordFilter, - dimension: DiagnosticDimension, - ) -> StoreResult> { - usage_diagnostics(&self.pool, range, &filter, dimension).await - } - - async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult { - list_ops_errors(&self.pool, query).await - } -} - -#[async_trait] -impl AdminObservabilityStore for PgAdminObservabilityStore { - async fn dashboard_summary( - &self, - range: admin_observability::TimeRange, - ) -> AdminStoreResult { - let observation = self - .repository - .dashboard_summary(store_range(range)?) - .await - .map_err(observability_error)?; - admin_dashboard_observation(observation) - } - - async fn dashboard_runtime_slots( - &self, - observed_at: DateTime, - ) -> AdminStoreResult> { - let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at) - .await - .map_err(observability_error)?; - let active_accounts = u64::try_from(active_account_ids.len()) - .map_err(|_| observability_error(invalid("active account count overflows u64")))?; - if active_account_ids.is_empty() { - return Ok(Some(admin_observability::DashboardRuntimeSlots { - active_accounts, - used_slots: Some(0), - })); - } - let Some(runtime_signals) = &self.runtime_signals else { - return Ok(None); - }; - let signals = match runtime_signals - .credential_runtime_signals(&active_account_ids) - .await - { - Ok(signals) => signals, - Err(_) => { - return Ok(Some(admin_observability::DashboardRuntimeSlots { - active_accounts, - used_slots: None, - })); - } - }; - let used_slots = signals.into_iter().fold(0_u64, |total, signal| { - total.saturating_add(u64::from(signal.in_flight)) - }); - Ok(Some(admin_observability::DashboardRuntimeSlots { - active_accounts, - used_slots: Some(used_slots), - })) - } - - async fn dashboard_trend( - &self, - range: admin_observability::TimeRange, - ) -> AdminStoreResult> { - self.repository - .dashboard_trend(store_range(range)?) - .await - .map_err(observability_error)? - .into_iter() - .map(admin_request_metric_point) - .collect() - } - - async fn usage_trend( - &self, - range: admin_observability::TimeRange, - filter: admin_observability::UsageFilter, - ) -> AdminStoreResult> { - self.repository - .usage_trend(store_range(range)?, store_usage_filter(filter)) - .await - .map_err(observability_error)? - .into_iter() - .map(admin_request_metric_point) - .collect() - } - - async fn usage_calculated_billing_facts( - &self, - range: admin_observability::TimeRange, - filter: admin_observability::UsageFilter, - ) -> AdminStoreResult> { - self.repository - .usage_calculated_billing_facts(store_range(range)?, store_usage_filter(filter)) - .await - .map_err(observability_error)? - .into_iter() - .map(admin_calculated_usage_billing_fact) - .collect() - } - - async fn list_usage_records( - &self, - query: admin_observability::UsageQuery, - ) -> AdminStoreResult { - let page = self - .repository - .list_usage_records(store_usage_query(query)?) - .await - .map_err(observability_error)?; - admin_usage_page(page) - } - - async fn usage_record_detail( - &self, - request_id: &str, - ) -> AdminStoreResult { - let detail = self - .repository - .usage_record_detail(request_id) - .await - .map_err(observability_error)?; - admin_usage_detail(detail) - } - - async fn usage_summary( - &self, - range: admin_observability::TimeRange, - filter: admin_observability::UsageFilter, - ) -> AdminStoreResult { - let overview = self - .repository - .usage_summary(store_range(range)?, store_usage_filter(filter)) - .await - .map_err(observability_error)?; - admin_usage_overview(overview) - } - - async fn usage_diagnostics( - &self, - range: admin_observability::TimeRange, - filter: admin_observability::UsageFilter, - dimension: admin_observability::DiagnosticDimension, - ) -> AdminStoreResult> { - self.repository - .usage_diagnostics( - store_range(range)?, - store_usage_filter(filter), - store_diagnostic_dimension(dimension), - ) - .await - .map_err(observability_error)? - .into_iter() - .map(admin_diagnostic_observation) - .collect() - } - - async fn list_ops_errors( - &self, - query: admin_observability::OpsErrorQuery, - ) -> AdminStoreResult { - let page = self - .repository - .list_ops_errors(store_ops_error_query(query)?) - .await - .map_err(observability_error)?; - admin_ops_error_page(page) - } -} - -fn store_range(range: admin_observability::TimeRange) -> AdminStoreResult { - ObservabilityRange::new(range.start, range.end).map_err(observability_error) -} - -fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter { - UsageRecordFilter { - client_api_key_ref: filter.client_api_key_ref, - request_id: filter.request_id, - provider_account_ref: filter.provider_account_ref, - operation: filter.operation, - provider_kind: filter.provider_kind, - model: filter.model, - outcome: filter.outcome.map(store_request_outcome), - status_code: filter.status_code, - transport: filter.transport, - attempt_index: filter.attempt_index, - response_id: filter.response_id, - upstream_request_id: filter.upstream_request_id, - search: filter.search, - } -} - -fn store_request_outcome(outcome: admin_observability::RequestOutcome) -> String { - outcome.as_str().to_owned() -} - -fn store_cursor( - cursor: admin_observability::ObservabilityCursor, -) -> AdminStoreResult { - ObservabilityCursor::new(cursor.observed_at, cursor.stable_id).map_err(observability_error) -} - -fn store_usage_query(query: admin_observability::UsageQuery) -> AdminStoreResult { - Ok(UsageRecordQuery { - range: store_range(query.range)?, - filter: store_usage_filter(query.filter), - cursor: query.cursor.map(store_cursor).transpose()?, - page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?, - page_size: ObservabilityPageSize::new(query.page_size.get()) - .map_err(observability_error)?, - }) -} - -fn store_ops_error_filter(filter: admin_observability::OpsErrorFilter) -> OpsErrorFilter { - OpsErrorFilter { - client_api_key_ref: filter.client_api_key_ref, - request_id: filter.request_id, - provider_account_ref: filter.provider_account_ref, - provider_kind: filter.provider_kind, - operation: filter.operation, - model: filter.model, - transport: filter.transport, - attempt_index: filter.attempt_index, - response_id: filter.response_id, - upstream_request_id: filter.upstream_request_id, - failure_kind: filter.failure_kind, - status_code: filter.status_code, - search: filter.search, - } -} - -fn store_ops_error_query( - query: admin_observability::OpsErrorQuery, -) -> AdminStoreResult { - Ok(OpsErrorQuery { - range: store_range(query.range)?, - filter: store_ops_error_filter(query.filter), - cursor: query.cursor.map(store_cursor).transpose()?, - page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?, - page_size: ObservabilityPageSize::new(query.page_size.get()) - .map_err(observability_error)?, - }) -} - -const fn store_diagnostic_dimension( - dimension: admin_observability::DiagnosticDimension, -) -> DiagnosticDimension { - match dimension { - admin_observability::DiagnosticDimension::Provider => DiagnosticDimension::Provider, - admin_observability::DiagnosticDimension::Model => DiagnosticDimension::Model, - admin_observability::DiagnosticDimension::Account => DiagnosticDimension::Account, - admin_observability::DiagnosticDimension::ApiKey => DiagnosticDimension::ApiKey, - admin_observability::DiagnosticDimension::Transport => DiagnosticDimension::Transport, - admin_observability::DiagnosticDimension::Failure => DiagnosticDimension::Failure, - admin_observability::DiagnosticDimension::Status => DiagnosticDimension::Status, - } -} - -fn admin_dashboard_observation( - observation: DashboardObservation, -) -> AdminStoreResult { - let DashboardObservation { - range, - requests, - attempts, - provider_accounts, - trend, - account_usage, - recent_requests, - } = observation; - Ok(admin_observability::DashboardObservation { - range: admin_range(range), - requests: admin_request_metrics(requests)?, - attempts: admin_attempt_metrics(attempts)?, - provider_accounts: admin_account_pool_metrics(provider_accounts), - trend: trend - .into_iter() - .map(admin_request_metric_point) - .collect::>()?, - account_usage: account_usage - .into_iter() - .map(admin_dashboard_account_usage) - .collect::>()?, - recent_requests: recent_requests - .into_iter() - .map(admin_usage_record) - .collect::>()?, - }) -} - -const fn admin_range(range: ObservabilityRange) -> admin_observability::TimeRange { - admin_observability::TimeRange { - start: range.start, - end: range.end, - } -} - -fn admin_request_metrics( - metrics: RequestMetrics, -) -> AdminStoreResult { - Ok(admin_observability::RequestMetrics { - request_count: metrics.request_count, - success_count: metrics.success_count, - failure_count: metrics.failure_count, - cancelled_count: metrics.cancelled_count, - incomplete_count: metrics.incomplete_count, - caller_error_count: metrics.caller_error_count, - input_tokens: metrics.input_tokens, - output_tokens: metrics.output_tokens, - cached_tokens: metrics.cached_tokens, - cache_write_tokens: metrics.cache_write_tokens, - reasoning_tokens: metrics.reasoning_tokens, - total_tokens: metrics.total_tokens, - first_token_latency_sum_ms: metrics.first_token_latency_sum, - first_token_latency_count: metrics.first_token_latency_count, - latency_sum_ms: metrics.latency_sum, - latency_count: metrics.latency_count, - min_latency_ms: metrics.min_latency_ms, - max_latency_ms: metrics.max_latency_ms, - latency_percentiles: admin_latency_percentiles(metrics.latency_percentiles)?, - first_token_latency_percentiles: admin_latency_percentiles( - metrics.first_token_latency_percentiles, - )?, - cache_eligible_request_count: metrics.cache_eligible_request_count, - cache_hit_request_count: metrics.cache_hit_request_count, - }) -} - -fn admin_latency_percentiles( - percentiles: LatencyPercentiles, -) -> AdminStoreResult { - Ok(admin_observability::LatencyPercentiles { - p50_ms: percentiles - .p50_ms - .map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64())) - .transpose() - .map_err(|_| invalid_admin_percentile())?, - p95_ms: percentiles - .p95_ms - .map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64())) - .transpose() - .map_err(|_| invalid_admin_percentile())?, - p99_ms: percentiles - .p99_ms - .map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64())) - .transpose() - .map_err(|_| invalid_admin_percentile())?, - }) -} - -fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError { - observability_error(StoreError::InvalidData { - entity: "observability latency percentile", - message: "latency percentile is outside the admin contract".to_owned(), - }) -} - -fn admin_attempt_metrics( - metrics: AttemptMetrics, -) -> AdminStoreResult { - Ok(admin_observability::AttemptMetrics { - attempt_count: metrics.attempt_count, - success_count: metrics.success_count, - failure_count: metrics.failure_count, - cancelled_count: metrics.cancelled_count, - incomplete_count: metrics.incomplete_count, - rate_limited_count: metrics.rate_limited_count, - auth_failure_count: metrics.auth_failure_count, - provider_5xx_count: metrics.provider_5xx_count, - cost_coverage: admin_cost_coverage(metrics.cost_coverage), - costs: admin_currency_costs(metrics.costs)?, - }) -} - -const fn admin_cost_coverage(coverage: CostCoverage) -> admin_observability::CostCoverage { - admin_observability::CostCoverage { - provider_reported_count: coverage.provider_reported_count, - calculated_count: coverage.calculated_count, - partial_count: 0, - unavailable_count: coverage.unavailable_count, - not_billable_count: 0, - } -} - -fn admin_currency_costs( - costs: Vec, -) -> AdminStoreResult> { - costs - .into_iter() - .map(|cost| { - Ok(admin_observability::CurrencyCost { - currency: cost.currency, - amount: admin_decimal_amount(cost.amount)?, - }) - }) - .collect() -} - -fn admin_decimal_amount( - amount: DecimalAmount, -) -> AdminStoreResult { - admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| { - observability_error(StoreError::InvalidData { - entity: "observability decimal amount", - message: "amount is outside the admin numeric contract".to_owned(), - }) - }) -} - -fn admin_optional_decimal_amount( - amount: Option, -) -> AdminStoreResult> { - amount.map(admin_decimal_amount).transpose() -} - -const fn admin_account_pool_metrics( - metrics: ProviderAccountMetrics, -) -> admin_observability::AccountPoolMetrics { - admin_observability::AccountPoolMetrics { - total: metrics.total, - enabled: metrics.enabled, - unavailable: metrics.unavailable, - active: metrics.active, - expired: metrics.expired, - quota_exhausted: metrics.quota_exhausted, - refreshing: None, - disabled: metrics.disabled, - banned: metrics.banned, - } -} - -fn admin_dashboard_account_usage( - usage: ProviderAccountUsageObservation, -) -> AdminStoreResult { - Ok(admin_observability::DashboardAccountUsage { - account_id: usage.account_id, - provider_kind: usage.provider_kind, - authentication_kind: usage.authentication_kind, - name: usage.name, - email: usage.email, - plan_type: usage.plan_type, - enabled: usage.enabled, - availability: usage.availability, - request_count: usage.request_count, - success_count: usage.success_count, - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - cached_tokens: usage.cached_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - image_input_tokens: usage.image_input_tokens, - image_output_tokens: usage.image_output_tokens, - image_request_count: usage.image_request_count, - image_request_failed_count: usage.image_request_failed_count, - total_tokens: usage.total_tokens, - cost_coverage: admin_cost_coverage(usage.cost_coverage), - costs: admin_currency_costs(usage.costs)?, - last_used_at: usage.last_used_at, - request_buckets: usage - .request_buckets - .into_iter() - .map( - |bucket| admin_observability::DashboardAccountRequestBucket { - bucket_start: bucket.bucket_start, - request_count: bucket.request_count, - }, - ) - .collect(), - quota_used_percent: None, - models: usage - .models - .into_iter() - .map(admin_dashboard_account_model_usage) - .collect::>()?, - }) -} - -fn admin_dashboard_account_model_usage( - usage: ProviderAccountModelUsageObservation, -) -> AdminStoreResult { - Ok(admin_observability::DashboardAccountModelUsage { - model: usage.model, - request_count: usage.request_count, - success_count: usage.success_count, - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - cached_tokens: usage.cached_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - image_input_tokens: usage.image_input_tokens, - image_output_tokens: usage.image_output_tokens, - image_request_count: usage.image_request_count, - image_request_failed_count: usage.image_request_failed_count, - total_tokens: usage.total_tokens, - cost_coverage: admin_cost_coverage(usage.cost_coverage), - costs: admin_currency_costs(usage.costs)?, - last_used_at: usage.last_used_at, - }) -} - -fn admin_request_metric_point( - point: RequestMetricPoint, -) -> AdminStoreResult { - Ok(admin_observability::RequestMetricPoint { - bucket_start: point.bucket_start, - granularity: admin_granularity(point.granularity), - metrics: admin_request_metrics(point.metrics)?, - cost_coverage: admin_cost_coverage(point.cost_coverage), - costs: admin_currency_costs(point.costs)?, - }) -} - -fn admin_calculated_usage_billing_fact( - fact: CalculatedUsageBillingFact, -) -> AdminStoreResult { - Ok(admin_observability::UsageCalculatedBillingFact { - bucket_start: fact.bucket_start, - provider_kind: fact.provider_kind, - upstream_model_id: fact.upstream_model_id, - service_tier: fact.service_tier, - input_tokens: fact.input_tokens, - output_tokens: fact.output_tokens, - cached_tokens: fact.cached_tokens, - cache_write_tokens: fact.cache_write_tokens, - total: admin_observability::CurrencyCost { - currency: fact.total.currency, - amount: admin_decimal_amount(fact.total.amount)?, - }, - }) -} - -const fn admin_granularity( - granularity: ObservationGranularity, -) -> admin_observability::Granularity { - match granularity { - ObservationGranularity::FifteenMinutes => admin_observability::Granularity::FifteenMinutes, - ObservationGranularity::Hour => admin_observability::Granularity::Hour, - ObservationGranularity::Day => admin_observability::Granularity::Day, - } -} - -fn admin_usage_page(page: UsageRecordPage) -> AdminStoreResult { - Ok(admin_observability::UsagePage { - items: page - .items - .into_iter() - .map(admin_usage_record) - .collect::>()?, - total: page.total, - next_cursor: page.next_cursor.map(admin_cursor), - }) -} - -fn admin_cursor(cursor: ObservabilityCursor) -> admin_observability::ObservabilityCursor { - admin_observability::ObservabilityCursor { - observed_at: cursor.observed_at, - stable_id: cursor.stable_id, - } -} - -fn admin_usage_record(record: UsageRecord) -> AdminStoreResult { - let billing = match (&record.cost_amount, &record.cost_currency) { - (Some(amount), Some(currency)) => Some(admin_observability::UsageBilling::Total { - source: record.cost_source.clone(), - total: admin_observability::CurrencyCost { - currency: currency.clone(), - amount: admin_decimal_amount(amount.clone())?, - }, - }), - (None, None) => None, - _ => { - return Err(observability_error(StoreError::InvalidData { - entity: "observability request billing", - message: "cost amount and currency must be present together".to_owned(), - })); - } - }; - Ok(admin_observability::UsageRecord { - id: record.id, - client_api_key_ref: record.client_api_key_ref, - config_revision: record.config_revision, - protocol: record.protocol, - operation: record.operation, - endpoint: record.endpoint, - client_transport: record.client_transport, - requested_model_id: record.requested_model_id, - provider_kind: record.provider_kind, - provider_account_ref: record.provider_account_ref, - provider_account_name: record.provider_account_name, - provider_account_email: record.provider_account_email, - provider_account_authentication_kind: record.provider_account_authentication_kind, - upstream_model_id: record.upstream_model_id, - upstream_transport: record.upstream_transport, - http_version: record.http_version, - websocket_pool: record.websocket_pool, - service_tier: record.service_tier, - provider_metadata_json: record.provider_metadata_json, - attempt_count: record.attempt_count, - upstream_send_state: record.upstream_send_state, - downstream_committed_at: record.downstream_committed_at, - outcome: admin_request_outcome(&record.outcome)?, - client_status_code: record.client_status_code, - upstream_status_code: record.upstream_status_code, - client_response_id: record.client_response_id, - upstream_request_id: record.upstream_request_id, - upstream_response_id: record.upstream_response_id, - error_kind: record.error_kind, - provider_error_code: record.provider_error_code, - error_message: record.error_message, - retry_after_ms: record.retry_after_ms, - input_tokens: record.input_tokens, - output_tokens: record.output_tokens, - cached_tokens: record.cached_tokens, - cache_write_tokens: record.cache_write_tokens, - reasoning_tokens: record.reasoning_tokens, - image_input_tokens: record.image_input_tokens, - image_output_tokens: record.image_output_tokens, - total_tokens: record.total_tokens, - cost_source: record.cost_source, - cost_amount: admin_optional_decimal_amount(record.cost_amount)?, - cost_currency: record.cost_currency, - billing, - transport_decision_wait_ms: record.transport_decision_wait_ms, - connect_ms: record.connect_ms, - headers_ms: record.headers_ms, - first_event_ms: record.first_event_ms, - first_reasoning_ms: record.first_reasoning_ms, - first_text_ms: record.first_text_ms, - first_token_ms: record.first_token_ms, - provider_processing_ms: record.provider_processing_ms, - latency_ms: record.latency_ms, - client_ip: record.client_ip, - user_agent: record.user_agent, - reasoning_effort: record.reasoning_effort, - reasoning_preset: record.reasoning_preset, - request_kind: record.request_kind, - subagent_kind: record.subagent_kind, - compact: record.compact, - image_generation_requested: record.image_generation_requested, - image_generation_succeeded: record.image_generation_succeeded, - started_at: record.started_at, - deadline_at: record.deadline_at, - completed_at: record.completed_at, - }) -} - -fn admin_request_outcome(outcome: &str) -> AdminStoreResult { - admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| { - observability_error(StoreError::InvalidData { - entity: "observability request outcome", - message: "invalid request outcome".to_owned(), - }) - }) -} - -fn admin_usage_detail( - detail: UsageRecordDetail, -) -> AdminStoreResult { - Ok(admin_observability::UsageDetail { - request: admin_usage_record(detail.request)?, - attempts: detail - .attempts - .into_iter() - .map(admin_usage_attempt) - .collect::>()?, - }) -} - -fn admin_usage_attempt( - attempt: UsageAttemptObservation, -) -> AdminStoreResult { - Ok(admin_observability::UsageAttempt { - source: attempt.source, - id: attempt.id, - attempt_index: attempt.attempt_index, - component: attempt.component, - operation: attempt.operation, - provider_kind: attempt.provider_kind, - provider_account_ref: attempt.provider_account_ref, - provider_account_name: attempt.provider_account_name, - provider_account_email: attempt.provider_account_email, - provider_account_authentication_kind: attempt.provider_account_authentication_kind, - upstream_model_id: attempt.upstream_model_id, - upstream_transport: attempt.upstream_transport, - upstream_send_state: attempt.upstream_send_state, - outcome: admin_request_outcome(&attempt.outcome)?, - downstream_committed: attempt.downstream_committed, - status_code: attempt.status_code, - provider_error_code: attempt.provider_error_code, - failure_kind: attempt.failure_kind, - retry_after_ms: attempt.retry_after_ms, - upstream_request_id: attempt.upstream_request_id, - latency_ms: attempt.latency_ms, - message: attempt.message, - input_tokens: attempt.input_tokens, - output_tokens: attempt.output_tokens, - cached_tokens: attempt.cached_tokens, - cache_write_tokens: attempt.cache_write_tokens, - reasoning_tokens: attempt.reasoning_tokens, - total_tokens: attempt.total_tokens, - cost_source: attempt.cost_source, - cost_amount: admin_optional_decimal_amount(attempt.cost_amount)?, - cost_currency: attempt.cost_currency, - occurred_at: attempt.occurred_at, - }) -} - -fn admin_usage_overview( - overview: UsageOverview, -) -> AdminStoreResult { - Ok(admin_observability::UsageOverview { - range: admin_range(overview.range), - requests: admin_request_metrics(overview.requests)?, - attempts: admin_attempt_metrics(overview.attempts)?, - providers: overview - .providers - .into_iter() - .map(admin_provider_observation) - .collect(), - }) -} - -fn admin_provider_observation( - observation: ProviderObservation, -) -> admin_observability::ProviderObservation { - admin_observability::ProviderObservation { - provider_kind: observation.provider_kind, - request_count: observation.request_count, - attempt_count: observation.attempt_count, - failure_count: observation.failure_count, - total_tokens: observation.total_tokens, - } -} - -fn admin_diagnostic_observation( - observation: DiagnosticObservation, -) -> AdminStoreResult { - Ok(admin_observability::DiagnosticObservation { - key: observation.key, - name: observation.name, - request_count: observation.request_count, - success_count: observation.success_count, - failure_count: observation.failure_count, - attempt_count: observation.attempt_count, - total_tokens: observation.total_tokens, - average_latency_ms: observation.average_latency_ms, - latency_p95_ms: observation.latency_p95_ms, - cost_coverage: admin_cost_coverage(observation.cost_coverage), - costs: admin_currency_costs(observation.costs)?, - }) -} - -fn admin_ops_error_page(page: OpsErrorPage) -> AdminStoreResult { - Ok(admin_observability::OpsErrorPage { - items: page.items.into_iter().map(admin_ops_error).collect(), - total: page.total, - next_cursor: page.next_cursor.map(admin_cursor), - }) -} - -fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::OpsError { - admin_observability::OpsError { - source: error.source, - event_id: error.event_id, - request_id: error.request_id, - attempt_index: error.attempt_index, - client_api_key_ref: error.client_api_key_ref, - component: error.component, - operation: error.operation, - endpoint: error.endpoint, - provider_kind: error.provider_kind, - provider_account_ref: error.provider_account_ref, - provider_account_name: error.provider_account_name, - provider_account_email: error.provider_account_email, - provider_account_authentication_kind: error.provider_account_authentication_kind, - upstream_model_id: error.upstream_model_id, - upstream_transport: error.upstream_transport, - failure_kind: error.failure_kind, - client_status_code: error.client_status_code, - upstream_status_code: error.upstream_status_code, - provider_error_code: error.provider_error_code, - client_response_id: error.client_response_id, - upstream_request_id: error.upstream_request_id, - latency_ms: error.latency_ms, - message: error.message, - occurrence_count: error.occurrence_count, - occurred_at: error.occurred_at, - stable_sort_id: error.stable_sort_id, - } -} - -fn observability_error(error: StoreError) -> gateway_admin::ports::store::AdminStoreError { - admin_store_error("observability", error) -} - -async fn request_metrics( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult { - filter.validate()?; - // 结果计数覆盖范围内全部请求（成功率/失败率的分母分子）；用量、缓存、 - // 延迟与成本聚合仅统计用量事实（已完整交付客户端的成功响应）。 - let fact = completed_usage_fact_predicate("mr"); - let mut query = QueryBuilder::::new(format!( - "select count(*)::bigint as request_count, - count(*) filter (where outcome = 'succeeded')::bigint as success_count, - count(*) filter (where outcome = 'failed')::bigint as failure_count, - count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count, - count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count, - count(*) filter (where client_status_code between 400 and 499)::bigint - as caller_error_count, - coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens, - coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens, - coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens, - coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint - as cache_write_tokens, - coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint - as reasoning_tokens, - coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens, - coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint - as first_token_latency_sum, - count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count, - coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum, - count(latency_ms) filter (where {fact})::bigint as latency_count, - max(latency_ms) filter (where {fact})::bigint as max_latency_ms, - min(latency_ms) filter (where {fact})::bigint as min_latency_ms, - count(*) filter (where {fact} and input_tokens is not null)::bigint - as cache_eligible_request_count, - count(*) filter (where {fact} and input_tokens is not null - and cached_tokens > 0)::bigint - as cache_hit_request_count, - percentile_cont(0.50) within group (order by latency_ms) - filter (where {fact}) as latency_p50_ms, - percentile_cont(0.95) within group (order by latency_ms) - filter (where {fact}) as latency_p95_ms, - percentile_cont(0.99) within group (order by latency_ms) - filter (where {fact}) as latency_p99_ms, - percentile_cont(0.50) within group (order by first_token_ms) - filter (where {fact}) as first_token_p50_ms, - percentile_cont(0.95) within group (order by first_token_ms) - filter (where {fact}) as first_token_p95_ms, - percentile_cont(0.99) within group (order by first_token_ms) - filter (where {fact}) as first_token_p99_ms - from model_requests mr where mr.started_at >= " - )); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - push_usage_filter(&mut query, filter, "mr"); - let row = query - .build() - .fetch_one(pool) - .await - .map_err(|_| postgres_unavailable("load request metrics"))?; - request_metrics_from_row(&row) -} - -async fn request_metric_series( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult> { - filter.validate()?; - let granularity = granularity_for(range); - // 与 request_metrics 同一契约：结果计数覆盖全部请求，用量/延迟/成本 - // 聚合仅统计用量事实。 - let fact = completed_usage_fact_predicate("mr"); - let mut query = QueryBuilder::::new("select date_bin("); - query.push_bind(granularity.sql_interval()); - query.push(format!( - "::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start, - count(*)::bigint as request_count, - count(*) filter (where outcome = 'succeeded')::bigint as success_count, - count(*) filter (where outcome = 'failed')::bigint as failure_count, - count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count, - count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count, - count(*) filter (where client_status_code between 400 and 499)::bigint - as caller_error_count, - coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens, - coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens, - coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens, - coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint - as cache_write_tokens, - coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint - as reasoning_tokens, - coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens, - coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint - as first_token_latency_sum, - count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count, - coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum, - count(latency_ms) filter (where {fact})::bigint as latency_count, - max(latency_ms) filter (where {fact})::bigint as max_latency_ms, - min(latency_ms) filter (where {fact})::bigint as min_latency_ms, - count(*) filter (where {fact} and input_tokens is not null)::bigint - as cache_eligible_request_count, - count(*) filter (where {fact} and input_tokens is not null - and cached_tokens > 0)::bigint - as cache_hit_request_count, - percentile_cont(0.50) within group (order by latency_ms) - filter (where {fact}) as latency_p50_ms, - percentile_cont(0.95) within group (order by latency_ms) - filter (where {fact}) as latency_p95_ms, - percentile_cont(0.99) within group (order by latency_ms) - filter (where {fact}) as latency_p99_ms, - percentile_cont(0.50) within group (order by first_token_ms) - filter (where {fact}) as first_token_p50_ms, - percentile_cont(0.95) within group (order by first_token_ms) - filter (where {fact}) as first_token_p95_ms, - percentile_cont(0.99) within group (order by first_token_ms) - filter (where {fact}) as first_token_p99_ms, - count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(*) filter (where {fact} and cost_source = 'calculated')::bigint - as calculated_count, - count(*) filter (where {fact} and cost_source = 'unavailable')::bigint - as unavailable_count - from model_requests mr where mr.started_at >= " - )); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - push_usage_filter(&mut query, filter, "mr"); - query.push(" group by bucket_start order by bucket_start"); - let rows = query - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load request metric series"))?; - - let mut points = BTreeMap::new(); - for row in rows { - let bucket_start = get(&row, "bucket_start")?; - points.insert( - bucket_start, - RequestMetricPoint { - bucket_start, - granularity, - metrics: request_metrics_from_row(&row)?, - cost_coverage: coverage_from_row(&row)?, - costs: Vec::new(), - }, - ); - } - let bucket_costs = request_costs_by_bucket(pool, range, filter, granularity).await?; - for (bucket, costs) in bucket_costs { - if let Some(point) = points.get_mut(&bucket) { - point.costs = costs; - } - } - fill_metric_gaps(range, granularity, points) -} - -async fn calculated_usage_billing_facts( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult> { - filter.validate()?; - let granularity = granularity_for(range); - let mut query = QueryBuilder::::new("select date_bin("); - query.push_bind(granularity.sql_interval()); - query.push( - "::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start, - mr.provider_kind, mr.upstream_model_id, mr.service_tier, - mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens, - mr.cost_currency, mr.cost_amount::text as amount - from model_requests mr where mr.started_at >= ", - ); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - query.push( - " and mr.cost_source = 'calculated' - and mr.cost_amount is not null and mr.cost_currency is not null - and nullif(mr.provider_kind, '') is not null - and nullif(mr.upstream_model_id, '') is not null", - ); - push_completed_usage_fact_filter(&mut query, "mr"); - push_usage_filter(&mut query, filter, "mr"); - query.push(" order by bucket_start, mr.id"); - query - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load calculated usage billing facts"))? - .iter() - .map(calculated_usage_billing_fact_from_row) - .collect() -} - -async fn request_costs_by_bucket( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, - granularity: ObservationGranularity, -) -> StoreResult, Vec>> { - let mut query = QueryBuilder::::new("select date_bin("); - query.push_bind(granularity.sql_interval()); - query.push( - "::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start, - mr.cost_currency, sum(mr.cost_amount)::text as amount - from model_requests mr - where mr.started_at >= ", - ); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - query.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); - push_completed_usage_fact_filter(&mut query, "mr"); - push_usage_filter(&mut query, filter, "mr"); - query.push(" group by bucket_start, mr.cost_currency order by bucket_start, mr.cost_currency"); - let rows = query - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load request costs by bucket"))?; - let mut result = BTreeMap::, Vec>::new(); - for row in rows { - result - .entry(get(&row, "bucket_start")?) - .or_default() - .push(cost_from_row(&row)?); - } - Ok(result) -} - -async fn attempt_metrics( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult { - filter.validate()?; - let mut query = QueryBuilder::::new( - "with selected_requests as ( - select mr.* from model_requests mr where mr.started_at >= ", - ); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - push_usage_filter(&mut query, filter, "mr"); - query.push( - "), usage_facts as ( - select * from selected_requests - where outcome = 'succeeded' - and downstream_committed_at is not null - and client_status_code between 200 and 399 - ), failures as ( - select coalesce(oe.occurrence_count, 1)::bigint as occurrences, - oe.failure_kind, oe.status_code - from ops_events oe - join selected_requests sr on sr.id = oe.model_request_id - union all - select 1::bigint, coalesce(sr.error_kind, 'failed'), - coalesce(sr.upstream_status_code, sr.client_status_code) - from selected_requests sr - where sr.outcome = 'failed' and sr.attempt_count > 0 - ) - select coalesce((select sum(attempt_count) from selected_requests), 0)::bigint - as attempt_count, - coalesce((select count(*) from selected_requests - where outcome = 'succeeded' and attempt_count > 0), 0)::bigint - as success_count, - coalesce((select sum(occurrences) from failures), 0)::bigint as failure_count, - coalesce((select count(*) from selected_requests - where outcome = 'cancelled' and attempt_count > 0), 0)::bigint - as cancelled_count, - coalesce((select count(*) from selected_requests - where outcome = 'incomplete' and attempt_count > 0), 0)::bigint - as incomplete_count, - coalesce((select sum(occurrences) from failures - where failure_kind in ('rate_limited', 'quota_exhausted') - or status_code = 429), 0)::bigint as rate_limited_count, - coalesce((select sum(occurrences) from failures - where failure_kind in ('authentication', 'authorization', - 'invalid_credential') - or status_code in (401, 403)), 0)::bigint as auth_failure_count, - coalesce((select sum(occurrences) from failures - where status_code between 500 and 599), 0)::bigint - as provider_5xx_count, - coalesce((select count(*) from usage_facts - where cost_source = 'provider_reported'), 0)::bigint - as provider_reported_count, - coalesce((select count(*) from usage_facts - where cost_source = 'calculated'), 0)::bigint - as calculated_count, - coalesce((select count(*) from usage_facts - where cost_source = 'unavailable'), 0)::bigint - as unavailable_count", - ); - let row = query - .build() - .fetch_one(pool) - .await - .map_err(|_| postgres_unavailable("load attempt metrics"))?; - Ok(AttemptMetrics { - attempt_count: unsigned(&row, "attempt_count")?, - success_count: unsigned(&row, "success_count")?, - failure_count: unsigned(&row, "failure_count")?, - cancelled_count: unsigned(&row, "cancelled_count")?, - incomplete_count: unsigned(&row, "incomplete_count")?, - rate_limited_count: unsigned(&row, "rate_limited_count")?, - auth_failure_count: unsigned(&row, "auth_failure_count")?, - provider_5xx_count: unsigned(&row, "provider_5xx_count")?, - cost_coverage: coverage_from_row(&row)?, - costs: request_costs(pool, range, filter).await?, - }) -} - -async fn request_costs( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult> { - let mut query = QueryBuilder::::new( - "select mr.cost_currency, sum(mr.cost_amount)::text as amount - from model_requests mr where mr.started_at >= ", - ); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - query.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); - push_completed_usage_fact_filter(&mut query, "mr"); - push_usage_filter(&mut query, filter, "mr"); - query.push(" group by mr.cost_currency order by mr.cost_currency"); - query - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load request costs"))? - .iter() - .map(cost_from_row) - .collect() -} - -async fn provider_account_metrics( - pool: &PgPool, - observed_at: DateTime, -) -> StoreResult { - let row = sqlx::query( - "with normalized as ( - select enabled, - case - when not enabled then 'disabled' - when availability = 'banned' then 'banned' - when availability = 'quota_exhausted' - or ( - availability = 'cooldown' - and availability_reason = 'usage_limit_exhausted' - and (cooldown_until is null or cooldown_until > $1) - ) - then 'quota_exhausted' - when availability in ('expired', 'invalid') - or (access_token_expires_at is not null and access_token_expires_at <= $1) - then 'expired' - else 'active' - end as status - from provider_accounts - ) - select count(*)::bigint as total, - count(*) filter (where enabled)::bigint as enabled, - count(*) filter (where status <> 'active')::bigint as unavailable, - count(*) filter (where status = 'active')::bigint as active, - count(*) filter (where status = 'expired')::bigint as expired, - count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted, - count(*) filter (where status = 'disabled')::bigint as disabled, - count(*) filter (where status = 'banned')::bigint as banned - from normalized", - ) - .bind(observed_at) - .fetch_one(pool) - .await - .map_err(|_| postgres_unavailable("load provider account metrics"))?; - Ok(ProviderAccountMetrics { - total: unsigned(&row, "total")?, - enabled: unsigned(&row, "enabled")?, - unavailable: unsigned(&row, "unavailable")?, - active: unsigned(&row, "active")?, - expired: unsigned(&row, "expired")?, - quota_exhausted: unsigned(&row, "quota_exhausted")?, - disabled: unsigned(&row, "disabled")?, - banned: unsigned(&row, "banned")?, - }) -} - -async fn active_provider_account_ids( - pool: &PgPool, - observed_at: DateTime, -) -> StoreResult> { - sqlx::query( - "with normalized as ( - select id, - case - when not enabled then 'disabled' - when availability = 'banned' then 'banned' - when availability = 'quota_exhausted' - or ( - availability = 'cooldown' - and availability_reason = 'usage_limit_exhausted' - and (cooldown_until is null or cooldown_until > $1) - ) - then 'quota_exhausted' - when availability in ('expired', 'invalid') - or (access_token_expires_at is not null and access_token_expires_at <= $1) - then 'expired' - else 'active' - end as status - from provider_accounts - ) - select id - from normalized - where status = 'active' - order by id", - ) - .bind(observed_at) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load active provider account IDs"))? - .iter() - .map(|row| get(row, "id")) - .collect() -} - -async fn provider_observations( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult> { - let mut query = QueryBuilder::::new( - "select coalesce(mr.provider_kind, 'unrouted') as provider_kind, - count(*)::bigint as request_count, - coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count, - count(*) filter (where mr.outcome = 'failed')::bigint as failure_count, - coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens - from model_requests mr where mr.started_at >= ", - ); - query.push_bind(range.start); - query.push(" and mr.started_at < "); - query.push_bind(range.end); - push_completed_usage_fact_filter(&mut query, "mr"); - push_usage_filter(&mut query, filter, "mr"); - query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc"); - let rows = query - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load provider observations"))?; - rows.iter() - .map(|row| { - Ok(ProviderObservation { - provider_kind: get(row, "provider_kind")?, - request_count: unsigned(row, "request_count")?, - attempt_count: unsigned(row, "attempt_count")?, - failure_count: unsigned(row, "failure_count")?, - total_tokens: unsigned(row, "total_tokens")?, - }) - }) - .collect() -} - -fn request_metrics_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { - Ok(RequestMetrics { - request_count: unsigned(row, "request_count")?, - success_count: unsigned(row, "success_count")?, - failure_count: unsigned(row, "failure_count")?, - cancelled_count: unsigned(row, "cancelled_count")?, - incomplete_count: unsigned(row, "incomplete_count")?, - caller_error_count: unsigned(row, "caller_error_count")?, - input_tokens: unsigned(row, "input_tokens")?, - output_tokens: unsigned(row, "output_tokens")?, - cached_tokens: unsigned(row, "cached_tokens")?, - cache_write_tokens: unsigned(row, "cache_write_tokens")?, - reasoning_tokens: unsigned(row, "reasoning_tokens")?, - total_tokens: unsigned(row, "total_tokens")?, - first_token_latency_sum: unsigned(row, "first_token_latency_sum")?, - first_token_latency_count: unsigned(row, "first_token_latency_count")?, - latency_sum: unsigned(row, "latency_sum")?, - latency_count: unsigned(row, "latency_count")?, - max_latency_ms: optional_unsigned(row, "max_latency_ms")?, - min_latency_ms: optional_unsigned(row, "min_latency_ms")?, - cache_eligible_request_count: unsigned(row, "cache_eligible_request_count")?, - cache_hit_request_count: unsigned(row, "cache_hit_request_count")?, - latency_percentiles: LatencyPercentiles { - p50_ms: optional_percentile(row, "latency_p50_ms")?, - p95_ms: optional_percentile(row, "latency_p95_ms")?, - p99_ms: optional_percentile(row, "latency_p99_ms")?, - }, - first_token_latency_percentiles: LatencyPercentiles { - p50_ms: optional_percentile(row, "first_token_p50_ms")?, - p95_ms: optional_percentile(row, "first_token_p95_ms")?, - p99_ms: optional_percentile(row, "first_token_p99_ms")?, - }, - }) -} - -fn optional_percentile( - row: &sqlx::postgres::PgRow, - column: &str, -) -> StoreResult> { - row.try_get::, _>(column) - .map_err(|_| postgres_unavailable("decode latency percentile"))? - .map(PercentileMilliseconds::new) - .transpose() -} - -fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { - Ok(CostCoverage { - provider_reported_count: unsigned(row, "provider_reported_count")?, - calculated_count: unsigned(row, "calculated_count")?, - unavailable_count: unsigned(row, "unavailable_count")?, - }) -} - -fn granularity_for(range: ObservabilityRange) -> ObservationGranularity { - let seconds = range.end.signed_duration_since(range.start).num_seconds(); - if seconds <= 2 * 24 * 60 * 60 { - ObservationGranularity::FifteenMinutes - } else if seconds <= 31 * 24 * 60 * 60 { - ObservationGranularity::Hour - } else { - ObservationGranularity::Day - } -} - -fn fill_metric_gaps( - range: ObservabilityRange, - granularity: ObservationGranularity, - mut points: BTreeMap, RequestMetricPoint>, -) -> StoreResult> { - let seconds = granularity.seconds(); - let start_epoch = range.start.timestamp().div_euclid(seconds) * seconds; - let mut bucket = DateTime::from_timestamp(start_epoch, 0) - .ok_or_else(|| invalid("metric range start is outside supported timestamps"))?; - let step = TimeDelta::seconds(seconds); - let mut result = Vec::new(); - while bucket < range.end { - result.push(points.remove(&bucket).unwrap_or(RequestMetricPoint { - bucket_start: bucket, - granularity, - metrics: RequestMetrics::default(), - cost_coverage: CostCoverage::default(), - costs: Vec::new(), - })); - bucket = bucket - .checked_add_signed(step) - .ok_or_else(|| invalid("metric range exceeds supported timestamps"))?; - } - Ok(result) -} - -fn push_usage_filter(query: &mut QueryBuilder, filter: &UsageRecordFilter, alias: &str) { - if let Some(value) = &filter.client_api_key_ref { - query.push(format!(" and {alias}.client_api_key_ref = ")); - query.push_bind(value.clone()); - } - if let Some(value) = &filter.request_id { - query.push(format!(" and {alias}.id = ")); - query.push_bind(value.clone()); - } - if let Some(value) = &filter.provider_account_ref { - query.push(format!(" and {alias}.provider_account_ref = ")); - query.push_bind(value.clone()); - } - if let Some(value) = &filter.operation { - query.push(format!(" and {alias}.operation = ")); - query.push_bind(value.clone()); - } - if let Some(value) = &filter.provider_kind { - query.push(format!(" and {alias}.provider_kind = ")); - query.push_bind(value.clone()); - } - if let Some(value) = &filter.model { - query.push(format!(" and ({alias}.requested_model_id = ")); - query.push_bind(value.clone()); - query.push(format!(" or {alias}.upstream_model_id = ")); - query.push_bind(value.clone()); - query.push(")"); - } - if let Some(value) = &filter.outcome { - query.push(format!(" and {alias}.outcome = ")); - query.push_bind(value.clone()); - } - if let Some(value) = filter.status_code { - query.push(format!(" and ({alias}.client_status_code = ")); - query.push_bind(i32::from(value)); - query.push(format!(" or {alias}.upstream_status_code = ")); - query.push_bind(i32::from(value)); - query.push(")"); - } - if let Some(value) = &filter.transport { - query.push(format!(" and ({alias}.client_transport = ")); - query.push_bind(value.clone()); - query.push(format!(" or {alias}.upstream_transport = ")); - query.push_bind(value.clone()); - query.push(")"); - } - if let Some(value) = filter.attempt_index { - query.push(format!(" and ({alias}.attempt_count = ")); - query.push_bind(i32::try_from(value).unwrap_or(i32::MAX)); - query.push(format!( - " or exists (select 1 from ops_events attempt_event - where attempt_event.model_request_id = {alias}.id - and attempt_event.attempt_index = " - )); - query.push_bind(i32::try_from(value).unwrap_or(i32::MAX)); - query.push("))"); - } - if let Some(value) = &filter.response_id { - query.push(format!(" and ({alias}.client_response_id = ")); - query.push_bind(value.as_bytes().to_vec()); - query.push(format!(" or {alias}.upstream_response_id = ")); - query.push_bind(value.as_bytes().to_vec()); - query.push(")"); - } - if let Some(value) = &filter.upstream_request_id { - query.push(format!(" and {alias}.upstream_request_id = ")); - query.push_bind(value.clone()); - } - if let Some(value) = &filter.search { - query.push(format!( - " and lower(concat_ws(' ', {alias}.id, {alias}.client_api_key_ref, - {alias}.requested_model_id, {alias}.provider_kind, - {alias}.provider_account_ref, {alias}.upstream_model_id, - encode({alias}.client_response_id, 'escape'), {alias}.upstream_request_id, - {alias}.error_kind, {alias}.provider_error_code, - {alias}.error_message)) like " - )); - query.push_bind(format!("%{}%", value.to_lowercase())); - } -} - -/// 只把已完整交付给客户端的成功响应投影为使用事实。 -/// -/// `model_requests` 同时承担执行审计：包括上游已发送、但尚未收到首个事件就断开的 -/// WebSocket 请求。这些失败仍必须留给 Ops Errors 和调度健康度分析，不能混入用量、 -/// 成本、账号使用次数或请求明细。 -fn completed_usage_fact_predicate(alias: &str) -> String { - format!( - "{alias}.outcome = 'succeeded' and {alias}.downstream_committed_at is not null and {alias}.client_status_code between 200 and 399" - ) -} - -fn push_completed_usage_fact_filter(query: &mut QueryBuilder, alias: &str) { - query.push(format!(" and {}", completed_usage_fact_predicate(alias))); -} - -async fn provider_account_usage( - pool: &PgPool, - query: ProviderAccountUsageQuery, -) -> StoreResult> { - if let Some(account_ids) = &query.account_ids { - validate_account_ids(account_ids)?; - } - if query.limit == 0 || usize::from(query.limit) > MAX_ACCOUNT_IDS { - return Err(invalid("account usage limit must be between 1 and 200")); - } - - let mut statement = QueryBuilder::::new( - "select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, - pa.plan_type, pa.enabled, pa.availability, - count(mr.id)::bigint as request_count, - count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count, - sum(mr.input_tokens)::bigint as input_tokens, - sum(mr.output_tokens)::bigint as output_tokens, - sum(mr.cached_tokens)::bigint as cached_tokens, - sum(mr.cache_write_tokens)::bigint as cache_write_tokens, - sum(mr.reasoning_tokens)::bigint as reasoning_tokens, - sum(mr.image_input_tokens)::bigint as image_input_tokens, - sum(mr.image_output_tokens)::bigint as image_output_tokens, - count(mr.id) filter (where mr.image_generation_succeeded is true)::bigint - as image_request_count, - count(mr.id) filter (where mr.image_generation_succeeded is false)::bigint - as image_request_failed_count, - sum(mr.total_tokens)::bigint as total_tokens, - count(mr.id) filter (where mr.cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(mr.id) filter (where mr.cost_source = 'calculated')::bigint - as calculated_count, - count(mr.id) filter (where mr.cost_source = 'unavailable')::bigint - as unavailable_count, - max(mr.started_at) as last_used_at - from provider_accounts pa - left join model_requests mr - on mr.provider_account_ref = pa.id and mr.started_at >= ", - ); - statement.push_bind(query.range.start); - statement.push(" and mr.started_at < "); - statement.push_bind(query.range.end); - push_completed_usage_fact_filter(&mut statement, "mr"); - if let Some(account_ids) = &query.account_ids { - statement.push(" where pa.id = any("); - statement.push_bind(account_ids.clone()); - statement.push("::text[])"); - } - statement.push( - " group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, - pa.plan_type, pa.enabled, pa.availability - order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ", - ); - statement.push_bind(i64::from(query.limit)); - let rows = statement - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load provider account usage"))?; - - let mut observations = rows - .iter() - .map(provider_account_from_row) - .collect::>>()?; - if observations.is_empty() { - return Ok(observations); - } - let account_ids = observations - .iter() - .map(|item| item.account_id.clone()) - .collect::>(); - let mut request_buckets = if query.include_hourly_request_buckets { - provider_account_request_buckets(pool, query.range, &account_ids).await? - } else { - HashMap::new() - }; - let account_costs = account_costs(pool, query.range, &account_ids).await?; - let model_rows = account_model_rows(pool, query.range, &account_ids).await?; - let model_costs = account_model_costs(pool, query.range, &account_ids).await?; - let account_positions = observations - .iter() - .enumerate() - .map(|(index, item)| (item.account_id.clone(), index)) - .collect::>(); - for observation in &mut observations { - observation.costs = account_costs - .get(&observation.account_id) - .cloned() - .unwrap_or_default(); - observation.request_buckets = request_buckets - .remove(&observation.account_id) - .unwrap_or_default(); - } - for row in model_rows { - let account_id: String = get(&row, "provider_account_ref")?; - let Some(position) = account_positions.get(&account_id).copied() else { - continue; - }; - let model: String = get(&row, "model")?; - observations[position] - .models - .push(ProviderAccountModelUsageObservation { - costs: model_costs - .get(&(account_id, model.clone())) - .cloned() - .unwrap_or_default(), - model, - request_count: unsigned(&row, "request_count")?, - success_count: unsigned(&row, "success_count")?, - input_tokens: optional_unsigned(&row, "input_tokens")?, - output_tokens: optional_unsigned(&row, "output_tokens")?, - cached_tokens: optional_unsigned(&row, "cached_tokens")?, - cache_write_tokens: optional_unsigned(&row, "cache_write_tokens")?, - reasoning_tokens: optional_unsigned(&row, "reasoning_tokens")?, - image_input_tokens: optional_unsigned(&row, "image_input_tokens")?, - image_output_tokens: optional_unsigned(&row, "image_output_tokens")?, - image_request_count: unsigned(&row, "image_request_count")?, - image_request_failed_count: unsigned(&row, "image_request_failed_count")?, - total_tokens: optional_unsigned(&row, "total_tokens")?, - cost_coverage: coverage_from_row(&row)?, - last_used_at: get(&row, "last_used_at")?, - }); - } - Ok(observations) -} - -async fn provider_account_request_buckets( - pool: &PgPool, - range: ObservabilityRange, - account_ids: &[String], -) -> StoreResult>> { - let rows = sqlx::query( - "select provider_account_ref, - floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index, - count(*)::bigint as request_count - from model_requests mr - where mr.provider_account_ref = any($2::text[]) - and mr.started_at >= $1 and mr.started_at < $3 - and mr.outcome = 'succeeded' - and mr.downstream_committed_at is not null - and mr.client_status_code between 200 and 399 - group by mr.provider_account_ref, bucket_index - order by mr.provider_account_ref, bucket_index", - ) - .bind(range.start) - .bind(account_ids) - .bind(range.end) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load provider account request timeline"))?; - - let mut observed = HashMap::>::new(); - for row in rows { - observed - .entry(get(&row, "provider_account_ref")?) - .or_default() - .insert( - unsigned(&row, "bucket_index")?, - unsigned(&row, "request_count")?, - ); - } - - let step = TimeDelta::hours(1); - let mut timelines = HashMap::with_capacity(account_ids.len()); - for account_id in account_ids { - let mut account_observed = observed.remove(account_id).unwrap_or_default(); - let mut bucket_start = range.start; - let mut bucket_index = 0_u64; - let mut buckets = Vec::new(); - while bucket_start < range.end { - buckets.push(ProviderAccountRequestBucket { - bucket_start, - request_count: account_observed.remove(&bucket_index).unwrap_or_default(), - }); - bucket_start = bucket_start - .checked_add_signed(step) - .ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?; - bucket_index = bucket_index - .checked_add(1) - .ok_or_else(|| invalid("account request timeline exceeds supported buckets"))?; - } - timelines.insert(account_id.clone(), buckets); - } - Ok(timelines) -} - -async fn account_costs( - pool: &PgPool, - range: ObservabilityRange, - account_ids: &[String], -) -> StoreResult>> { - let rows = sqlx::query( - "select mr.provider_account_ref, mr.cost_currency, sum(mr.cost_amount)::text as amount - from model_requests mr - where mr.provider_account_ref = any($1::text[]) - and mr.started_at >= $2 and mr.started_at < $3 - and mr.outcome = 'succeeded' - and mr.downstream_committed_at is not null - and mr.client_status_code between 200 and 399 - and mr.cost_amount is not null and mr.cost_currency is not null - group by mr.provider_account_ref, mr.cost_currency - order by mr.provider_account_ref, mr.cost_currency", - ) - .bind(account_ids) - .bind(range.start) - .bind(range.end) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load provider account costs"))?; - let mut costs = HashMap::>::new(); - for row in rows { - costs - .entry(get(&row, "provider_account_ref")?) - .or_default() - .push(cost_from_row(&row)?); - } - Ok(costs) -} - -async fn account_model_rows( - pool: &PgPool, - range: ObservabilityRange, - account_ids: &[String], -) -> StoreResult> { - sqlx::query( - "select mr.provider_account_ref, - coalesce(mr.upstream_model_id, mr.requested_model_id) as model, - count(*)::bigint as request_count, - count(*) filter (where mr.outcome = 'succeeded')::bigint as success_count, - sum(mr.input_tokens)::bigint as input_tokens, - sum(mr.output_tokens)::bigint as output_tokens, - sum(mr.cached_tokens)::bigint as cached_tokens, - sum(mr.cache_write_tokens)::bigint as cache_write_tokens, - sum(mr.reasoning_tokens)::bigint as reasoning_tokens, - sum(mr.image_input_tokens)::bigint as image_input_tokens, - sum(mr.image_output_tokens)::bigint as image_output_tokens, - count(*) filter (where mr.image_generation_succeeded is true)::bigint - as image_request_count, - count(*) filter (where mr.image_generation_succeeded is false)::bigint - as image_request_failed_count, - sum(mr.total_tokens)::bigint as total_tokens, - count(*) filter (where mr.cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(*) filter (where mr.cost_source = 'calculated')::bigint - as calculated_count, - count(*) filter (where mr.cost_source = 'unavailable')::bigint - as unavailable_count, - max(mr.started_at) as last_used_at - from model_requests mr - where mr.provider_account_ref = any($1::text[]) - and mr.started_at >= $2 and mr.started_at < $3 - and mr.outcome = 'succeeded' - and mr.downstream_committed_at is not null - and mr.client_status_code between 200 and 399 - group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id) - order by mr.provider_account_ref, request_count desc, model", - ) - .bind(account_ids) - .bind(range.start) - .bind(range.end) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load provider account model usage")) -} - -async fn account_model_costs( - pool: &PgPool, - range: ObservabilityRange, - account_ids: &[String], -) -> StoreResult>> { - let rows = sqlx::query( - "select mr.provider_account_ref, - coalesce(mr.upstream_model_id, mr.requested_model_id) as model, - mr.cost_currency, sum(mr.cost_amount)::text as amount - from model_requests mr - where mr.provider_account_ref = any($1::text[]) - and mr.started_at >= $2 and mr.started_at < $3 - and mr.outcome = 'succeeded' - and mr.downstream_committed_at is not null - and mr.client_status_code between 200 and 399 - and mr.cost_amount is not null and mr.cost_currency is not null - group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id), - mr.cost_currency - order by mr.provider_account_ref, model, mr.cost_currency", - ) - .bind(account_ids) - .bind(range.start) - .bind(range.end) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load provider account model costs"))?; - let mut costs = HashMap::<(String, String), Vec>::new(); - for row in rows { - costs - .entry((get(&row, "provider_account_ref")?, get(&row, "model")?)) - .or_default() - .push(cost_from_row(&row)?); - } - Ok(costs) -} - -fn provider_account_from_row( - row: &sqlx::postgres::PgRow, -) -> StoreResult { - Ok(ProviderAccountUsageObservation { - account_id: get(row, "id")?, - provider_kind: get(row, "provider_kind")?, - authentication_kind: get(row, "authentication_kind")?, - name: get(row, "name")?, - email: get(row, "email")?, - plan_type: get(row, "plan_type")?, - enabled: get(row, "enabled")?, - availability: get(row, "availability")?, - request_count: unsigned(row, "request_count")?, - success_count: unsigned(row, "success_count")?, - input_tokens: optional_unsigned(row, "input_tokens")?, - output_tokens: optional_unsigned(row, "output_tokens")?, - cached_tokens: optional_unsigned(row, "cached_tokens")?, - cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?, - reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?, - image_input_tokens: optional_unsigned(row, "image_input_tokens")?, - image_output_tokens: optional_unsigned(row, "image_output_tokens")?, - image_request_count: unsigned(row, "image_request_count")?, - image_request_failed_count: unsigned(row, "image_request_failed_count")?, - total_tokens: optional_unsigned(row, "total_tokens")?, - cost_coverage: coverage_from_row(row)?, - costs: Vec::new(), - last_used_at: get(row, "last_used_at")?, - request_buckets: Vec::new(), - models: Vec::new(), - }) -} - -const USAGE_RECORD_SELECT: &str = - "select mr.id, mr.client_api_key_ref, mr.config_revision, mr.protocol, mr.operation, - mr.endpoint, mr.client_transport, mr.requested_model_id, - mr.provider_kind, mr.provider_account_ref, - mr.provider_account_name_snapshot as provider_account_name, - mr.provider_account_email_snapshot as provider_account_email, - mr.provider_account_authentication_kind_snapshot - as provider_account_authentication_kind, - mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool, - mr.service_tier, mr.provider_observation_json, - mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at, - mr.outcome, mr.client_status_code, mr.upstream_status_code, - mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id, - mr.error_kind, mr.provider_error_code, mr.error_message, mr.retry_after_ms, - mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens, - mr.reasoning_tokens, mr.image_input_tokens, mr.image_output_tokens, - mr.total_tokens, mr.cost_source, mr.cost_amount::text, - mr.cost_currency, mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms, - mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms, - mr.provider_processing_ms, mr.latency_ms, mr.client_ip::text as client_ip, - mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind, - mr.subagent_kind, mr.compact, mr.image_generation_requested, - mr.image_generation_succeeded, mr.started_at, mr.deadline_at, mr.completed_at - from model_requests mr"; - -async fn list_usage_records( - pool: &PgPool, - query: UsageRecordQuery, -) -> StoreResult { - query.filter.validate()?; - let total = count_usage_records(pool, query.range, &query.filter).await?; - let mut statement = QueryBuilder::::new(USAGE_RECORD_SELECT); - statement.push(" where mr.started_at >= "); - statement.push_bind(query.range.start); - statement.push(" and mr.started_at < "); - statement.push_bind(query.range.end); - push_completed_usage_fact_filter(&mut statement, "mr"); - push_usage_filter(&mut statement, &query.filter, "mr"); - if let Some(cursor) = &query.cursor { - statement.push(" and (mr.started_at, mr.id) < ("); - statement.push_bind(cursor.observed_at); - statement.push(", "); - statement.push_bind(cursor.stable_id.clone()); - statement.push(")"); - } - statement.push(" order by mr.started_at desc, mr.id desc limit "); - statement.push_bind(i64::from(query.page_size.get()) + 1); - if query.cursor.is_none() { - let offset = u64::from(query.page.get() - 1) - .checked_mul(u64::from(query.page_size.get())) - .and_then(|value| i64::try_from(value).ok()) - .ok_or_else(|| invalid("usage page offset is too large"))?; - statement.push(" offset "); - statement.push_bind(offset); - } - let rows = statement - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("list usage records"))?; - let mut items = rows - .iter() - .map(usage_record_from_row) - .collect::>>()?; - let has_more = items.len() > usize::from(query.page_size.get()); - if has_more { - items.pop(); - } - let next_cursor = if has_more { - items - .last() - .map(|item| ObservabilityCursor::new(item.started_at, item.id.clone())) - .transpose()? - } else { - None - }; - Ok(UsageRecordPage { - items, - total, - next_cursor, - }) -} - -async fn count_usage_records( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, -) -> StoreResult { - let mut statement = QueryBuilder::::new( - "select count(*)::bigint from model_requests mr where mr.started_at >= ", - ); - statement.push_bind(range.start); - statement.push(" and mr.started_at < "); - statement.push_bind(range.end); - push_completed_usage_fact_filter(&mut statement, "mr"); - push_usage_filter(&mut statement, filter, "mr"); - let total = statement - .build_query_scalar::() - .fetch_one(pool) - .await - .map_err(|_| postgres_unavailable("count usage records"))?; - to_u64(total) -} - -async fn usage_record_detail(pool: &PgPool, request_id: &str) -> StoreResult { - require_nonempty("model request", "id", request_id)?; - validate_text(request_id, MAX_FILTER_BYTES, "request ID")?; - let mut statement = QueryBuilder::::new(USAGE_RECORD_SELECT); - statement.push(" where mr.id = "); - statement.push_bind(request_id.to_owned()); - push_completed_usage_fact_filter(&mut statement, "mr"); - let row = statement - .build() - .fetch_optional(pool) - .await - .map_err(|_| postgres_unavailable("load usage record detail"))? - .ok_or_else(|| StoreError::NotFound { - entity: "model request", - id: request_id.to_owned(), - })?; - let request = usage_record_from_row(&row)?; - let rows = sqlx::query( - "select id, attempt_index, component, operation, - provider_kind, provider_account_ref, - provider_account_name_snapshot as provider_account_name, - provider_account_email_snapshot as provider_account_email, - provider_account_authentication_kind_snapshot - as provider_account_authentication_kind, - upstream_model_id, - failure_kind, status_code, provider_error_code, retry_after_ms, - upstream_request_id, latency_ms, message, created_at - from ops_events where model_request_id = $1 - order by attempt_index, created_at, id", - ) - .bind(request_id) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load usage attempt observations"))?; - let mut attempts = rows - .iter() - .map(intermediate_attempt_from_row) - .collect::>>()?; - if request.attempt_count > 0 { - attempts.push(final_attempt_from_request(&request)); - } - Ok(UsageRecordDetail { request, attempts }) -} - -fn intermediate_attempt_from_row( - row: &sqlx::postgres::PgRow, -) -> StoreResult { - Ok(UsageAttemptObservation { - source: "ops_event".to_owned(), - id: get(row, "id")?, - attempt_index: to_u32(get(row, "attempt_index")?)?, - component: get(row, "component")?, - operation: get(row, "operation")?, - provider_kind: get(row, "provider_kind")?, - provider_account_ref: get(row, "provider_account_ref")?, - provider_account_name: get(row, "provider_account_name")?, - provider_account_email: get(row, "provider_account_email")?, - provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?, - upstream_model_id: get(row, "upstream_model_id")?, - upstream_transport: None, - upstream_send_state: None, - outcome: "failed".to_owned(), - downstream_committed: false, - status_code: optional_status(row, "status_code")?, - provider_error_code: get(row, "provider_error_code")?, - failure_kind: get(row, "failure_kind")?, - retry_after_ms: optional_unsigned(row, "retry_after_ms")?, - upstream_request_id: get(row, "upstream_request_id")?, - latency_ms: optional_unsigned(row, "latency_ms")?, - message: get(row, "message")?, - input_tokens: None, - output_tokens: None, - cached_tokens: None, - cache_write_tokens: None, - reasoning_tokens: None, - total_tokens: None, - cost_source: None, - cost_amount: None, - cost_currency: None, - occurred_at: get(row, "created_at")?, - }) -} - -fn final_attempt_from_request(request: &UsageRecord) -> UsageAttemptObservation { - UsageAttemptObservation { - source: "model_request".to_owned(), - id: format!("{}:final", request.id), - attempt_index: request.attempt_count, - component: "model_request".to_owned(), - operation: request.operation.clone(), - provider_kind: request.provider_kind.clone(), - provider_account_ref: request.provider_account_ref.clone(), - provider_account_name: request.provider_account_name.clone(), - provider_account_email: request.provider_account_email.clone(), - provider_account_authentication_kind: request.provider_account_authentication_kind.clone(), - upstream_model_id: request.upstream_model_id.clone(), - upstream_transport: request.upstream_transport.clone(), - upstream_send_state: Some(request.upstream_send_state.clone()), - outcome: request.outcome.clone(), - downstream_committed: request.downstream_committed_at.is_some(), - status_code: request.upstream_status_code.or(request.client_status_code), - provider_error_code: request.provider_error_code.clone(), - failure_kind: request.error_kind.clone(), - retry_after_ms: request.retry_after_ms, - upstream_request_id: request.upstream_request_id.clone(), - latency_ms: request.latency_ms, - message: request.error_message.clone(), - input_tokens: request.input_tokens, - output_tokens: request.output_tokens, - cached_tokens: request.cached_tokens, - cache_write_tokens: request.cache_write_tokens, - reasoning_tokens: request.reasoning_tokens, - total_tokens: request.total_tokens, - cost_source: Some(request.cost_source.clone()), - cost_amount: request.cost_amount.clone(), - cost_currency: request.cost_currency.clone(), - occurred_at: request.completed_at.unwrap_or(request.started_at), - } -} - -async fn usage_diagnostics( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, - dimension: DiagnosticDimension, -) -> StoreResult> { - filter.validate()?; - let dimension_sql = diagnostic_dimension_sql(dimension); - let mut statement = QueryBuilder::::new("select "); - statement.push(dimension_sql); - statement.push( - " as dimension_name, - count(*)::bigint as request_count, - count(*) filter (where mr.outcome = 'succeeded')::bigint as success_count, - count(*) filter (where mr.outcome = 'failed')::bigint as failure_count, - coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count, - coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens, - round(avg(mr.latency_ms))::bigint as average_latency_ms, - round(percentile_cont(0.95) within group (order by mr.latency_ms))::bigint - as latency_p95_ms, - count(*) filter (where mr.cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(*) filter (where mr.cost_source = 'calculated')::bigint - as calculated_count, - count(*) filter (where mr.cost_source = 'unavailable')::bigint - as unavailable_count - from model_requests mr where mr.started_at >= ", - ); - statement.push_bind(range.start); - statement.push(" and mr.started_at < "); - statement.push_bind(range.end); - push_diagnostic_fact_filter(&mut statement, dimension); - push_usage_filter(&mut statement, filter, "mr"); - statement.push(" group by dimension_name order by request_count desc, dimension_name limit "); - statement.push_bind(DIAGNOSTIC_LIMIT); - let rows = statement - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load usage diagnostics"))?; - let mut observations = rows - .iter() - .map(|row| { - Ok(DiagnosticObservation { - key: get(row, "dimension_name")?, - name: get(row, "dimension_name")?, - request_count: unsigned(row, "request_count")?, - success_count: unsigned(row, "success_count")?, - failure_count: unsigned(row, "failure_count")?, - attempt_count: unsigned(row, "attempt_count")?, - total_tokens: unsigned(row, "total_tokens")?, - average_latency_ms: optional_unsigned(row, "average_latency_ms")?, - latency_p95_ms: optional_unsigned(row, "latency_p95_ms")?, - cost_coverage: coverage_from_row(row)?, - costs: Vec::new(), - }) - }) - .collect::>>()?; - let mut display_names = match dimension { - DiagnosticDimension::Account => { - diagnostic_account_display_names( - pool, - &observations - .iter() - .map(|item| item.key.clone()) - .collect::>(), - ) - .await? - } - DiagnosticDimension::ApiKey => { - diagnostic_api_key_display_names( - pool, - &observations - .iter() - .map(|item| item.key.clone()) - .collect::>(), - ) - .await? - } - _ => HashMap::new(), - }; - for observation in &mut observations { - if let Some(name) = display_names.remove(&observation.key) { - observation.name = name; - } - } - let positions = observations - .iter() - .enumerate() - .map(|(index, item)| (item.key.clone(), index)) - .collect::>(); - let costs = diagnostic_costs(pool, range, filter, dimension_sql, dimension).await?; - for (key, values) in costs { - if let Some(position) = positions.get(&key).copied() { - observations[position].costs = values; - } - } - Ok(observations) -} - -async fn diagnostic_account_display_names( - pool: &PgPool, - account_ids: &[String], -) -> StoreResult> { - if account_ids.is_empty() { - return Ok(HashMap::new()); - } - let rows = sqlx::query( - "select provider_account_ref, - coalesce( - (array_agg(provider_account_email_snapshot order by started_at desc) - filter (where provider_account_email_snapshot is not null))[1], - (array_agg(provider_account_name_snapshot order by started_at desc) - filter (where provider_account_name_snapshot is not null))[1], - provider_account_ref - ) as display_name - from model_requests - where provider_account_ref = any($1::text[]) - group by provider_account_ref", - ) - .bind(account_ids) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load diagnostic account display names"))?; - let mut display_names = HashMap::with_capacity(rows.len()); - for row in rows { - display_names.insert( - get(&row, "provider_account_ref")?, - get(&row, "display_name")?, - ); - } - Ok(display_names) -} - -async fn diagnostic_api_key_display_names( - pool: &PgPool, - key_ids: &[String], -) -> StoreResult> { - if key_ids.is_empty() { - return Ok(HashMap::new()); - } - let rows = sqlx::query("select id, name from client_api_keys where id = any($1::text[])") - .bind(key_ids) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load diagnostic api key display names"))?; - let mut display_names = HashMap::with_capacity(rows.len()); - for row in rows { - display_names.insert(get(&row, "id")?, get(&row, "name")?); - } - Ok(display_names) -} - -async fn diagnostic_costs( - pool: &PgPool, - range: ObservabilityRange, - filter: &UsageRecordFilter, - dimension_sql: &'static str, - dimension: DiagnosticDimension, -) -> StoreResult>> { - let mut statement = QueryBuilder::::new("select "); - statement.push(dimension_sql); - statement.push( - " as dimension_name, mr.cost_currency, sum(mr.cost_amount)::text as amount - from model_requests mr where mr.started_at >= ", - ); - statement.push_bind(range.start); - statement.push(" and mr.started_at < "); - statement.push_bind(range.end); - statement.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); - push_diagnostic_fact_filter(&mut statement, dimension); - push_usage_filter(&mut statement, filter, "mr"); - statement.push( - " group by dimension_name, mr.cost_currency order by dimension_name, mr.cost_currency", - ); - let rows = statement - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load diagnostic costs"))?; - let mut result = HashMap::>::new(); - for row in rows { - result - .entry(get(&row, "dimension_name")?) - .or_default() - .push(cost_from_row(&row)?); - } - Ok(result) -} - -fn diagnostic_dimension_sql(dimension: DiagnosticDimension) -> &'static str { - match dimension { - DiagnosticDimension::Provider => "coalesce(mr.provider_kind, 'unrouted')", - DiagnosticDimension::Model => { - "coalesce(mr.upstream_model_id, mr.requested_model_id, 'unknown')" - } - DiagnosticDimension::Account => "coalesce(mr.provider_account_ref, 'unrouted')", - DiagnosticDimension::ApiKey => "mr.client_api_key_ref", - DiagnosticDimension::Transport => { - "coalesce(mr.upstream_transport, mr.client_transport, 'unknown')" - } - DiagnosticDimension::Failure => "coalesce(mr.error_kind, 'none')", - DiagnosticDimension::Status => { - "coalesce(mr.upstream_status_code, mr.client_status_code)::text" - } - } -} - -fn push_diagnostic_fact_filter( - statement: &mut QueryBuilder, - dimension: DiagnosticDimension, -) { - if dimension == DiagnosticDimension::Failure { - statement.push(" and mr.error_kind is not null"); - } else { - push_completed_usage_fact_filter(statement, "mr"); - } -} - -const OPS_ERRORS_CTE: &str = "with errors as ( - select 'model_request'::text as source, - mr.id as event_id, mr.id as request_id, - nullif(mr.attempt_count, 0) as attempt_index, - mr.client_api_key_ref, 'model_request'::text as component, mr.operation, - mr.endpoint, - mr.provider_kind, - mr.provider_account_ref, - mr.provider_account_name_snapshot as provider_account_name, - mr.provider_account_email_snapshot as provider_account_email, - mr.provider_account_authentication_kind_snapshot - as provider_account_authentication_kind, - mr.upstream_model_id, mr.upstream_transport, - coalesce(mr.error_kind, 'failed') as failure_kind, - mr.client_status_code, mr.upstream_status_code, - mr.provider_error_code, mr.client_response_id, mr.upstream_request_id, - mr.latency_ms, coalesce(mr.error_message, mr.error_kind, 'request failed') as message, - 1::integer as occurrence_count, - coalesce(mr.completed_at, mr.started_at) as occurred_at, - 'model_request:' || mr.id as stable_sort_id - from model_requests mr where mr.outcome = 'failed' - union all - select 'ops_event'::text, oe.id, oe.model_request_id, oe.attempt_index, - mr.client_api_key_ref, oe.component, oe.operation, - null::text as endpoint, - oe.provider_kind, oe.provider_account_ref, - oe.provider_account_name_snapshot as provider_account_name, - oe.provider_account_email_snapshot as provider_account_email, - oe.provider_account_authentication_kind_snapshot - as provider_account_authentication_kind, - oe.upstream_model_id, null::text, oe.failure_kind, - null::integer as client_status_code, oe.status_code as upstream_status_code, - oe.provider_error_code, mr.client_response_id, oe.upstream_request_id, - oe.latency_ms, oe.message, oe.occurrence_count, oe.created_at, - 'ops_event:' || oe.id - from ops_events oe left join model_requests mr on mr.id = oe.model_request_id - )"; - -async fn list_ops_errors(pool: &PgPool, query: OpsErrorQuery) -> StoreResult { - query.filter.validate()?; - let total = count_ops_errors(pool, query.range, &query.filter).await?; - let mut statement = QueryBuilder::::new(OPS_ERRORS_CTE); - statement.push( - " select source, event_id, request_id, attempt_index, client_api_key_ref, - component, operation, endpoint, provider_kind, provider_account_ref, - provider_account_name, provider_account_email, - provider_account_authentication_kind, - upstream_model_id, upstream_transport, - failure_kind, client_status_code, upstream_status_code, - provider_error_code, client_response_id, - upstream_request_id, latency_ms, message, occurrence_count, occurred_at, - stable_sort_id - from errors e where e.occurred_at >= ", - ); - statement.push_bind(query.range.start); - statement.push(" and e.occurred_at < "); - statement.push_bind(query.range.end); - push_ops_filter(&mut statement, &query.filter); - if let Some(cursor) = &query.cursor { - statement.push(" and (e.occurred_at, e.stable_sort_id) < ("); - statement.push_bind(cursor.observed_at); - statement.push(", "); - statement.push_bind(cursor.stable_id.clone()); - statement.push(")"); - } - statement.push(" order by e.occurred_at desc, e.stable_sort_id desc limit "); - statement.push_bind(i64::from(query.page_size.get()) + 1); - if query.cursor.is_none() { - let offset = u64::from(query.page.get() - 1) - .checked_mul(u64::from(query.page_size.get())) - .and_then(|value| i64::try_from(value).ok()) - .ok_or_else(|| invalid("ops error page offset is too large"))?; - statement.push(" offset "); - statement.push_bind(offset); - } - let rows = statement - .build() - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("list ops errors"))?; - let mut items = rows - .iter() - .map(ops_error_from_row) - .collect::>>()?; - let has_more = items.len() > usize::from(query.page_size.get()); - if has_more { - items.pop(); - } - let next_cursor = if has_more { - items - .last() - .map(|item| ObservabilityCursor::new(item.occurred_at, item.stable_sort_id.clone())) - .transpose()? - } else { - None - }; - Ok(OpsErrorPage { - items, - total, - next_cursor, - }) -} - -async fn count_ops_errors( - pool: &PgPool, - range: ObservabilityRange, - filter: &OpsErrorFilter, -) -> StoreResult { - let mut statement = QueryBuilder::::new(OPS_ERRORS_CTE); - statement.push(" select count(*)::bigint from errors e where e.occurred_at >= "); - statement.push_bind(range.start); - statement.push(" and e.occurred_at < "); - statement.push_bind(range.end); - push_ops_filter(&mut statement, filter); - let total = statement - .build_query_scalar::() - .fetch_one(pool) - .await - .map_err(|_| postgres_unavailable("count ops errors"))?; - to_u64(total) -} - -fn push_ops_filter(statement: &mut QueryBuilder, filter: &OpsErrorFilter) { - for (column, value) in [ - ("client_api_key_ref", &filter.client_api_key_ref), - ("request_id", &filter.request_id), - ("provider_account_ref", &filter.provider_account_ref), - ("provider_kind", &filter.provider_kind), - ("operation", &filter.operation), - ("upstream_transport", &filter.transport), - ("upstream_request_id", &filter.upstream_request_id), - ("failure_kind", &filter.failure_kind), - ] { - if let Some(value) = value { - statement.push(format!(" and e.{column} = ")); - statement.push_bind(value.clone()); - } - } - if let Some(value) = &filter.response_id { - statement.push(" and e.client_response_id = "); - statement.push_bind(value.as_bytes().to_vec()); - } - if let Some(model) = &filter.model { - statement.push(" and e.upstream_model_id = "); - statement.push_bind(model.clone()); - } - if let Some(index) = filter.attempt_index { - statement.push(" and e.attempt_index = "); - statement.push_bind(i32::try_from(index).unwrap_or(i32::MAX)); - } - if let Some(status) = filter.status_code { - statement.push(" and e.upstream_status_code = "); - statement.push_bind(i32::from(status)); - } - if let Some(search) = &filter.search { - statement.push( - " and lower(concat_ws(' ', e.event_id, e.request_id, e.client_api_key_ref, - e.component, e.operation, e.provider_kind, e.provider_account_ref, - e.upstream_model_id, e.failure_kind, e.provider_error_code, - e.upstream_request_id, e.message)) like ", - ); - statement.push_bind(format!("%{}%", search.to_lowercase())); - } -} - -fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { - Ok(UsageRecord { - id: get(row, "id")?, - client_api_key_ref: get(row, "client_api_key_ref")?, - config_revision: unsigned(row, "config_revision")?, - protocol: get(row, "protocol")?, - operation: get(row, "operation")?, - endpoint: get(row, "endpoint")?, - client_transport: get(row, "client_transport")?, - requested_model_id: get(row, "requested_model_id")?, - provider_kind: get(row, "provider_kind")?, - provider_account_ref: get(row, "provider_account_ref")?, - provider_account_name: get(row, "provider_account_name")?, - provider_account_email: get(row, "provider_account_email")?, - provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?, - upstream_model_id: get(row, "upstream_model_id")?, - upstream_transport: get(row, "upstream_transport")?, - http_version: get(row, "http_version")?, - websocket_pool: get(row, "websocket_pool")?, - service_tier: get(row, "service_tier")?, - provider_metadata_json: get::>(row, "provider_observation_json")? - .map(|value| serde_json::to_string(&value)) - .transpose() - .map_err(|_| postgres_unavailable("encode provider observation"))?, - attempt_count: to_u32(get(row, "attempt_count")?)?, - upstream_send_state: get(row, "upstream_send_state")?, - downstream_committed_at: get(row, "downstream_committed_at")?, - outcome: get(row, "outcome")?, - client_status_code: optional_status(row, "client_status_code")?, - upstream_status_code: optional_status(row, "upstream_status_code")?, - client_response_id: opaque_response_id(row, "client_response_id")?, - upstream_request_id: get(row, "upstream_request_id")?, - upstream_response_id: opaque_response_id(row, "upstream_response_id")?, - error_kind: get(row, "error_kind")?, - provider_error_code: get(row, "provider_error_code")?, - error_message: get(row, "error_message")?, - retry_after_ms: optional_unsigned(row, "retry_after_ms")?, - input_tokens: optional_unsigned(row, "input_tokens")?, - output_tokens: optional_unsigned(row, "output_tokens")?, - cached_tokens: optional_unsigned(row, "cached_tokens")?, - cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?, - reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?, - image_input_tokens: optional_unsigned(row, "image_input_tokens")?, - image_output_tokens: optional_unsigned(row, "image_output_tokens")?, - total_tokens: optional_unsigned(row, "total_tokens")?, - cost_source: get(row, "cost_source")?, - cost_amount: optional_decimal(row, "cost_amount")?, - cost_currency: get(row, "cost_currency")?, - transport_decision_wait_ms: optional_unsigned(row, "transport_decision_wait_ms")?, - connect_ms: optional_unsigned(row, "connect_ms")?, - headers_ms: optional_unsigned(row, "headers_ms")?, - first_event_ms: optional_unsigned(row, "first_event_ms")?, - first_reasoning_ms: optional_unsigned(row, "first_reasoning_ms")?, - first_text_ms: optional_unsigned(row, "first_text_ms")?, - first_token_ms: optional_unsigned(row, "first_token_ms")?, - provider_processing_ms: optional_unsigned(row, "provider_processing_ms")?, - latency_ms: optional_unsigned(row, "latency_ms")?, - client_ip: get(row, "client_ip")?, - user_agent: get(row, "user_agent")?, - reasoning_effort: get(row, "reasoning_effort")?, - reasoning_preset: get(row, "reasoning_preset")?, - request_kind: get(row, "request_kind")?, - subagent_kind: get(row, "subagent_kind")?, - compact: get(row, "compact")?, - image_generation_requested: get(row, "image_generation_requested")?, - image_generation_succeeded: get(row, "image_generation_succeeded")?, - started_at: get(row, "started_at")?, - deadline_at: get(row, "deadline_at")?, - completed_at: get(row, "completed_at")?, - }) -} - -fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { - Ok(OpsErrorRecord { - source: get(row, "source")?, - event_id: get(row, "event_id")?, - request_id: get(row, "request_id")?, - attempt_index: get::>(row, "attempt_index")? - .map(to_u32) - .transpose()?, - client_api_key_ref: get(row, "client_api_key_ref")?, - component: get(row, "component")?, - operation: get(row, "operation")?, - endpoint: get(row, "endpoint")?, - provider_kind: get(row, "provider_kind")?, - provider_account_ref: get(row, "provider_account_ref")?, - provider_account_name: get(row, "provider_account_name")?, - provider_account_email: get(row, "provider_account_email")?, - provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?, - upstream_model_id: get(row, "upstream_model_id")?, - upstream_transport: get(row, "upstream_transport")?, - failure_kind: get(row, "failure_kind")?, - client_status_code: optional_status(row, "client_status_code")?, - upstream_status_code: optional_status(row, "upstream_status_code")?, - provider_error_code: get(row, "provider_error_code")?, - client_response_id: opaque_response_id(row, "client_response_id")?, - upstream_request_id: get(row, "upstream_request_id")?, - latency_ms: optional_unsigned(row, "latency_ms")?, - message: get(row, "message")?, - occurrence_count: to_u32(get(row, "occurrence_count")?)?, - occurred_at: get(row, "occurred_at")?, - stable_sort_id: get(row, "stable_sort_id")?, - }) -} - -fn cost_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { - Ok(CurrencyCostTotal { - currency: get(row, "cost_currency")?, - amount: DecimalAmount::from_str(&get::(row, "amount")?)?, - }) -} - -fn calculated_usage_billing_fact_from_row( - row: &sqlx::postgres::PgRow, -) -> StoreResult { - Ok(CalculatedUsageBillingFact { - bucket_start: get(row, "bucket_start")?, - provider_kind: get(row, "provider_kind")?, - upstream_model_id: get(row, "upstream_model_id")?, - service_tier: get(row, "service_tier")?, - input_tokens: optional_unsigned(row, "input_tokens")?, - output_tokens: optional_unsigned(row, "output_tokens")?, - cached_tokens: optional_unsigned(row, "cached_tokens")?, - cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?, - total: cost_from_row(row)?, - }) -} - -fn optional_decimal( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> StoreResult> { - get::>(row, column)? - .map(|value| DecimalAmount::from_str(&value)) - .transpose() -} - -fn opaque_response_id( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> StoreResult> { - get::>>(row, column)? - .map(String::from_utf8) - .transpose() - .map_err(|_| invalid(column)) -} - -fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> { - let mut unique = BTreeSet::new(); - for account_id in account_ids { - validate_text(account_id, MAX_FILTER_BYTES, "provider account ID")?; - if !unique.insert(account_id.as_str()) { - return Err(invalid("account usage query contains duplicate IDs")); - } - } - Ok(()) -} - -fn validate_optional_text( - value: Option<&str>, - max_bytes: usize, - field: &'static str, -) -> StoreResult<()> { - value.map_or(Ok(()), |value| validate_text(value, max_bytes, field)) -} - -fn validate_text(value: &str, max_bytes: usize, field: &'static str) -> StoreResult<()> { - if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) { - return Err(invalid(&format!("{field} is invalid"))); - } - Ok(()) -} - -fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -> StoreResult -where - T: sqlx::Decode<'r, Postgres> + sqlx::Type, -{ - row.try_get(column).map_err(|_| invalid(column)) -} - -fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult { - to_u64(get(row, column)?) -} - -fn optional_unsigned( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> StoreResult> { - get::>(row, column)?.map(to_u64).transpose() -} - -fn optional_status(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult> { - get::>(row, column)? - .map(|value| { - u16::try_from(value) - .ok() - .filter(|value| (100..=599).contains(value)) - .ok_or_else(|| invalid("status code is outside its supported range")) - }) - .transpose() -} - -fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("numeric observation is negative")) -} - -fn to_u32(value: i32) -> StoreResult { - u32::try_from(value).map_err(|_| invalid("integer observation is negative")) -} - -fn invalid(message: &str) -> StoreError { - StoreError::InvalidData { - entity: "observability", - message: message.to_owned(), - } -} diff --git a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs new file mode 100644 index 000000000..449d43d5c --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs @@ -0,0 +1,322 @@ +//! Pg 观测 adapter：实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。 + +use super::*; + +use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository}; + +#[derive(Clone)] +pub struct PgObservabilityRepository { + pool: PgPool, +} + +impl PgObservabilityRepository { + #[must_use] + pub const fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +/// `gateway-admin` 观测端口的 PostgreSQL adapter。 +/// +/// SQL 查询与内部投影继续由 [`PgObservabilityRepository`] 唯一拥有；本类型只负责 +/// Admin UTC 领域模型与持久化投影之间的无格式化转换。 +#[derive(Clone)] +pub struct PgAdminObservabilityStore { + repository: PgObservabilityRepository, + runtime_signals: Option, +} + +impl PgAdminObservabilityStore { + #[must_use] + pub fn new(pool: PgPool, runtime_signals: Option) -> Self { + Self { + repository: PgObservabilityRepository::new(pool), + runtime_signals, + } + } +} + +#[async_trait] +impl ObservabilityRepository for PgObservabilityRepository { + async fn dashboard_summary( + &self, + range: ObservabilityRange, + ) -> StoreResult { + let filter = UsageRecordFilter::default(); + let account_usage_range = ObservabilityRange::new( + range.end - TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS), + range.end, + )?; + let account_usage_query = + ProviderAccountUsageQuery::recent(account_usage_range, DASHBOARD_ACCOUNT_LIMIT)? + .with_hourly_request_buckets()?; + let recent_query = UsageRecordQuery { + range, + filter: UsageRecordFilter { + outcome: Some("succeeded".to_owned()), + ..UsageRecordFilter::default() + }, + cursor: None, + page: ObservabilityPageNumber::new(1)?, + page_size: ObservabilityPageSize::new(10)?, + }; + // 分两批并发：页面延迟从六路之和降为两批各自最慢者之和，同时单次 + // 渲染最多占用 3 个池连接，不与数据面写路径争抢整个连接池。 + let (requests, attempts, provider_accounts) = futures::try_join!( + request_metrics(&self.pool, range, &filter), + attempt_metrics(&self.pool, range, &filter), + provider_account_metrics(&self.pool, range.end), + )?; + let (trend, account_usage, recent_requests) = futures::try_join!( + request_metric_series(&self.pool, range, &filter), + provider_account_usage(&self.pool, account_usage_query), + async { Ok(list_usage_records(&self.pool, recent_query).await?.items) }, + )?; + Ok(DashboardObservation { + range, + requests, + attempts, + provider_accounts, + trend, + account_usage, + recent_requests, + }) + } + + async fn dashboard_trend( + &self, + range: ObservabilityRange, + ) -> StoreResult> { + request_metric_series(&self.pool, range, &UsageRecordFilter::default()).await + } + + async fn usage_trend( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + ) -> StoreResult> { + request_metric_series(&self.pool, range, &filter).await + } + + async fn usage_calculated_billing_facts( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + ) -> StoreResult> { + calculated_usage_billing_facts(&self.pool, range, &filter).await + } + + async fn provider_account_usage( + &self, + query: ProviderAccountUsageQuery, + ) -> StoreResult> { + provider_account_usage(&self.pool, query).await + } + + async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult { + list_usage_records(&self.pool, query).await + } + + async fn usage_record_detail(&self, request_id: &str) -> StoreResult { + usage_record_detail(&self.pool, request_id).await + } + + async fn usage_summary( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + ) -> StoreResult { + filter.validate()?; + let requests = request_metrics(&self.pool, range, &filter).await?; + let attempts = attempt_metrics(&self.pool, range, &filter).await?; + let providers = provider_observations(&self.pool, range, &filter).await?; + Ok(UsageOverview { + range, + requests, + attempts, + providers, + }) + } + + async fn usage_diagnostics( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + dimension: DiagnosticDimension, + ) -> StoreResult> { + usage_diagnostics(&self.pool, range, &filter, dimension).await + } + + async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult { + list_ops_errors(&self.pool, query).await + } +} + +#[async_trait] +impl AdminObservabilityStore for PgAdminObservabilityStore { + async fn dashboard_summary( + &self, + range: admin_observability::TimeRange, + ) -> AdminStoreResult { + let observation = self + .repository + .dashboard_summary(store_range(range)?) + .await + .map_err(observability_error)?; + admin_dashboard_observation(observation) + } + + async fn dashboard_runtime_slots( + &self, + observed_at: DateTime, + ) -> AdminStoreResult> { + let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at) + .await + .map_err(observability_error)?; + let active_accounts = u64::try_from(active_account_ids.len()) + .map_err(|_| observability_error(invalid("active account count overflows u64")))?; + if active_account_ids.is_empty() { + return Ok(Some(admin_observability::DashboardRuntimeSlots { + active_accounts, + used_slots: Some(0), + })); + } + let Some(runtime_signals) = &self.runtime_signals else { + return Ok(None); + }; + let signals = match runtime_signals + .credential_runtime_signals(&active_account_ids) + .await + { + Ok(signals) => signals, + Err(_) => { + return Ok(Some(admin_observability::DashboardRuntimeSlots { + active_accounts, + used_slots: None, + })); + } + }; + let used_slots = signals.into_iter().fold(0_u64, |total, signal| { + total.saturating_add(u64::from(signal.in_flight)) + }); + Ok(Some(admin_observability::DashboardRuntimeSlots { + active_accounts, + used_slots: Some(used_slots), + })) + } + + async fn dashboard_trend( + &self, + range: admin_observability::TimeRange, + ) -> AdminStoreResult> { + self.repository + .dashboard_trend(store_range(range)?) + .await + .map_err(observability_error)? + .into_iter() + .map(admin_request_metric_point) + .collect() + } + + async fn usage_trend( + &self, + range: admin_observability::TimeRange, + filter: admin_observability::UsageFilter, + ) -> AdminStoreResult> { + self.repository + .usage_trend(store_range(range)?, store_usage_filter(filter)) + .await + .map_err(observability_error)? + .into_iter() + .map(admin_request_metric_point) + .collect() + } + + async fn usage_calculated_billing_facts( + &self, + range: admin_observability::TimeRange, + filter: admin_observability::UsageFilter, + ) -> AdminStoreResult> { + self.repository + .usage_calculated_billing_facts(store_range(range)?, store_usage_filter(filter)) + .await + .map_err(observability_error)? + .into_iter() + .map(admin_calculated_usage_billing_fact) + .collect() + } + + async fn list_usage_records( + &self, + query: admin_observability::UsageQuery, + ) -> AdminStoreResult { + let page = self + .repository + .list_usage_records(store_usage_query(query)?) + .await + .map_err(observability_error)?; + admin_usage_page(page) + } + + async fn usage_record_detail( + &self, + request_id: &str, + ) -> AdminStoreResult { + let detail = self + .repository + .usage_record_detail(request_id) + .await + .map_err(observability_error)?; + admin_usage_detail(detail) + } + + async fn usage_summary( + &self, + range: admin_observability::TimeRange, + filter: admin_observability::UsageFilter, + ) -> AdminStoreResult { + let overview = self + .repository + .usage_summary(store_range(range)?, store_usage_filter(filter)) + .await + .map_err(observability_error)?; + admin_usage_overview(overview) + } + + async fn usage_diagnostics( + &self, + range: admin_observability::TimeRange, + filter: admin_observability::UsageFilter, + dimension: admin_observability::DiagnosticDimension, + ) -> AdminStoreResult> { + self.repository + .usage_diagnostics( + store_range(range)?, + store_usage_filter(filter), + store_diagnostic_dimension(dimension), + ) + .await + .map_err(observability_error)? + .into_iter() + .map(admin_diagnostic_observation) + .collect() + } + + async fn list_ops_errors( + &self, + query: admin_observability::OpsErrorQuery, + ) -> AdminStoreResult { + let page = self + .repository + .list_ops_errors(store_ops_error_query(query)?) + .await + .map_err(observability_error)?; + admin_ops_error_page(page) + } +} + +pub(crate) fn observability_error( + error: StoreError, +) -> gateway_admin::ports::store::AdminStoreError { + admin_store_error("observability", error) +} diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs new file mode 100644 index 000000000..cf690069e --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -0,0 +1,862 @@ +//! Store 与 Admin 领域之间的无格式化转换及 row 解析。 + +use super::*; + +pub(crate) fn store_range( + range: admin_observability::TimeRange, +) -> AdminStoreResult { + ObservabilityRange::new(range.start, range.end).map_err(observability_error) +} + +pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter { + UsageRecordFilter { + client_api_key_ref: filter.client_api_key_ref, + request_id: filter.request_id, + provider_account_ref: filter.provider_account_ref, + operation: filter.operation, + provider_kind: filter.provider_kind, + model: filter.model, + outcome: filter.outcome.map(store_request_outcome), + status_code: filter.status_code, + transport: filter.transport, + attempt_index: filter.attempt_index, + response_id: filter.response_id, + upstream_request_id: filter.upstream_request_id, + search: filter.search, + } +} + +pub(crate) fn store_request_outcome(outcome: admin_observability::RequestOutcome) -> String { + outcome.as_str().to_owned() +} + +pub(crate) fn store_cursor( + cursor: admin_observability::ObservabilityCursor, +) -> AdminStoreResult { + ObservabilityCursor::new(cursor.observed_at, cursor.stable_id).map_err(observability_error) +} + +pub(crate) fn store_usage_query( + query: admin_observability::UsageQuery, +) -> AdminStoreResult { + Ok(UsageRecordQuery { + range: store_range(query.range)?, + filter: store_usage_filter(query.filter), + cursor: query.cursor.map(store_cursor).transpose()?, + page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?, + page_size: ObservabilityPageSize::new(query.page_size.get()) + .map_err(observability_error)?, + }) +} + +pub(crate) fn store_ops_error_filter( + filter: admin_observability::OpsErrorFilter, +) -> OpsErrorFilter { + OpsErrorFilter { + client_api_key_ref: filter.client_api_key_ref, + request_id: filter.request_id, + provider_account_ref: filter.provider_account_ref, + provider_kind: filter.provider_kind, + operation: filter.operation, + model: filter.model, + transport: filter.transport, + attempt_index: filter.attempt_index, + response_id: filter.response_id, + upstream_request_id: filter.upstream_request_id, + failure_kind: filter.failure_kind, + status_code: filter.status_code, + search: filter.search, + } +} + +pub(crate) fn store_ops_error_query( + query: admin_observability::OpsErrorQuery, +) -> AdminStoreResult { + Ok(OpsErrorQuery { + range: store_range(query.range)?, + filter: store_ops_error_filter(query.filter), + cursor: query.cursor.map(store_cursor).transpose()?, + page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?, + page_size: ObservabilityPageSize::new(query.page_size.get()) + .map_err(observability_error)?, + }) +} + +pub(crate) const fn store_diagnostic_dimension( + dimension: admin_observability::DiagnosticDimension, +) -> DiagnosticDimension { + match dimension { + admin_observability::DiagnosticDimension::Provider => DiagnosticDimension::Provider, + admin_observability::DiagnosticDimension::Model => DiagnosticDimension::Model, + admin_observability::DiagnosticDimension::Account => DiagnosticDimension::Account, + admin_observability::DiagnosticDimension::ApiKey => DiagnosticDimension::ApiKey, + admin_observability::DiagnosticDimension::Transport => DiagnosticDimension::Transport, + admin_observability::DiagnosticDimension::Failure => DiagnosticDimension::Failure, + admin_observability::DiagnosticDimension::Status => DiagnosticDimension::Status, + } +} + +pub(crate) fn admin_dashboard_observation( + observation: DashboardObservation, +) -> AdminStoreResult { + let DashboardObservation { + range, + requests, + attempts, + provider_accounts, + trend, + account_usage, + recent_requests, + } = observation; + Ok(admin_observability::DashboardObservation { + range: admin_range(range), + requests: admin_request_metrics(requests)?, + attempts: admin_attempt_metrics(attempts)?, + provider_accounts: admin_account_pool_metrics(provider_accounts), + trend: trend + .into_iter() + .map(admin_request_metric_point) + .collect::>()?, + account_usage: account_usage + .into_iter() + .map(admin_dashboard_account_usage) + .collect::>()?, + recent_requests: recent_requests + .into_iter() + .map(admin_usage_record) + .collect::>()?, + }) +} + +pub(crate) const fn admin_range(range: ObservabilityRange) -> admin_observability::TimeRange { + admin_observability::TimeRange { + start: range.start, + end: range.end, + } +} + +pub(crate) fn admin_request_metrics( + metrics: RequestMetrics, +) -> AdminStoreResult { + Ok(admin_observability::RequestMetrics { + request_count: metrics.request_count, + success_count: metrics.success_count, + failure_count: metrics.failure_count, + cancelled_count: metrics.cancelled_count, + incomplete_count: metrics.incomplete_count, + caller_error_count: metrics.caller_error_count, + input_tokens: metrics.input_tokens, + output_tokens: metrics.output_tokens, + cached_tokens: metrics.cached_tokens, + cache_write_tokens: metrics.cache_write_tokens, + reasoning_tokens: metrics.reasoning_tokens, + total_tokens: metrics.total_tokens, + first_token_latency_sum_ms: metrics.first_token_latency_sum, + first_token_latency_count: metrics.first_token_latency_count, + latency_sum_ms: metrics.latency_sum, + latency_count: metrics.latency_count, + min_latency_ms: metrics.min_latency_ms, + max_latency_ms: metrics.max_latency_ms, + latency_percentiles: admin_latency_percentiles(metrics.latency_percentiles)?, + first_token_latency_percentiles: admin_latency_percentiles( + metrics.first_token_latency_percentiles, + )?, + cache_eligible_request_count: metrics.cache_eligible_request_count, + cache_hit_request_count: metrics.cache_hit_request_count, + }) +} + +pub(crate) fn admin_latency_percentiles( + percentiles: LatencyPercentiles, +) -> AdminStoreResult { + Ok(admin_observability::LatencyPercentiles { + p50_ms: percentiles + .p50_ms + .map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64())) + .transpose() + .map_err(|_| invalid_admin_percentile())?, + p95_ms: percentiles + .p95_ms + .map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64())) + .transpose() + .map_err(|_| invalid_admin_percentile())?, + p99_ms: percentiles + .p99_ms + .map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64())) + .transpose() + .map_err(|_| invalid_admin_percentile())?, + }) +} + +pub(crate) fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError { + observability_error(StoreError::InvalidData { + entity: "observability latency percentile", + message: "latency percentile is outside the admin contract".to_owned(), + }) +} + +pub(crate) fn admin_attempt_metrics( + metrics: AttemptMetrics, +) -> AdminStoreResult { + Ok(admin_observability::AttemptMetrics { + attempt_count: metrics.attempt_count, + success_count: metrics.success_count, + failure_count: metrics.failure_count, + cancelled_count: metrics.cancelled_count, + incomplete_count: metrics.incomplete_count, + rate_limited_count: metrics.rate_limited_count, + auth_failure_count: metrics.auth_failure_count, + provider_5xx_count: metrics.provider_5xx_count, + cost_coverage: admin_cost_coverage(metrics.cost_coverage), + costs: admin_currency_costs(metrics.costs)?, + }) +} + +pub(crate) const fn admin_cost_coverage( + coverage: CostCoverage, +) -> admin_observability::CostCoverage { + admin_observability::CostCoverage { + provider_reported_count: coverage.provider_reported_count, + calculated_count: coverage.calculated_count, + partial_count: 0, + unavailable_count: coverage.unavailable_count, + not_billable_count: 0, + } +} + +pub(crate) fn admin_currency_costs( + costs: Vec, +) -> AdminStoreResult> { + costs + .into_iter() + .map(|cost| { + Ok(admin_observability::CurrencyCost { + currency: cost.currency, + amount: admin_decimal_amount(cost.amount)?, + }) + }) + .collect() +} + +pub(crate) fn admin_decimal_amount( + amount: DecimalAmount, +) -> AdminStoreResult { + admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| { + observability_error(StoreError::InvalidData { + entity: "observability decimal amount", + message: "amount is outside the admin numeric contract".to_owned(), + }) + }) +} + +pub(crate) fn admin_optional_decimal_amount( + amount: Option, +) -> AdminStoreResult> { + amount.map(admin_decimal_amount).transpose() +} + +pub(crate) const fn admin_account_pool_metrics( + metrics: ProviderAccountMetrics, +) -> admin_observability::AccountPoolMetrics { + admin_observability::AccountPoolMetrics { + total: metrics.total, + enabled: metrics.enabled, + unavailable: metrics.unavailable, + active: metrics.active, + expired: metrics.expired, + quota_exhausted: metrics.quota_exhausted, + refreshing: None, + disabled: metrics.disabled, + banned: metrics.banned, + } +} + +pub(crate) fn admin_dashboard_account_usage( + usage: ProviderAccountUsageObservation, +) -> AdminStoreResult { + Ok(admin_observability::DashboardAccountUsage { + account_id: usage.account_id, + provider_kind: usage.provider_kind, + authentication_kind: usage.authentication_kind, + name: usage.name, + email: usage.email, + plan_type: usage.plan_type, + enabled: usage.enabled, + availability: usage.availability, + request_count: usage.request_count, + success_count: usage.success_count, + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cached_tokens: usage.cached_tokens, + cache_write_tokens: usage.cache_write_tokens, + reasoning_tokens: usage.reasoning_tokens, + image_input_tokens: usage.image_input_tokens, + image_output_tokens: usage.image_output_tokens, + image_request_count: usage.image_request_count, + image_request_failed_count: usage.image_request_failed_count, + total_tokens: usage.total_tokens, + cost_coverage: admin_cost_coverage(usage.cost_coverage), + costs: admin_currency_costs(usage.costs)?, + last_used_at: usage.last_used_at, + request_buckets: usage + .request_buckets + .into_iter() + .map( + |bucket| admin_observability::DashboardAccountRequestBucket { + bucket_start: bucket.bucket_start, + request_count: bucket.request_count, + }, + ) + .collect(), + quota_used_percent: None, + models: usage + .models + .into_iter() + .map(admin_dashboard_account_model_usage) + .collect::>()?, + }) +} + +pub(crate) fn admin_dashboard_account_model_usage( + usage: ProviderAccountModelUsageObservation, +) -> AdminStoreResult { + Ok(admin_observability::DashboardAccountModelUsage { + model: usage.model, + request_count: usage.request_count, + success_count: usage.success_count, + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cached_tokens: usage.cached_tokens, + cache_write_tokens: usage.cache_write_tokens, + reasoning_tokens: usage.reasoning_tokens, + image_input_tokens: usage.image_input_tokens, + image_output_tokens: usage.image_output_tokens, + image_request_count: usage.image_request_count, + image_request_failed_count: usage.image_request_failed_count, + total_tokens: usage.total_tokens, + cost_coverage: admin_cost_coverage(usage.cost_coverage), + costs: admin_currency_costs(usage.costs)?, + last_used_at: usage.last_used_at, + }) +} + +pub(crate) fn admin_request_metric_point( + point: RequestMetricPoint, +) -> AdminStoreResult { + Ok(admin_observability::RequestMetricPoint { + bucket_start: point.bucket_start, + granularity: admin_granularity(point.granularity), + metrics: admin_request_metrics(point.metrics)?, + cost_coverage: admin_cost_coverage(point.cost_coverage), + costs: admin_currency_costs(point.costs)?, + }) +} + +pub(crate) fn admin_calculated_usage_billing_fact( + fact: CalculatedUsageBillingFact, +) -> AdminStoreResult { + Ok(admin_observability::UsageCalculatedBillingFact { + bucket_start: fact.bucket_start, + provider_kind: fact.provider_kind, + upstream_model_id: fact.upstream_model_id, + service_tier: fact.service_tier, + input_tokens: fact.input_tokens, + output_tokens: fact.output_tokens, + cached_tokens: fact.cached_tokens, + cache_write_tokens: fact.cache_write_tokens, + total: admin_observability::CurrencyCost { + currency: fact.total.currency, + amount: admin_decimal_amount(fact.total.amount)?, + }, + }) +} + +pub(crate) const fn admin_granularity( + granularity: ObservationGranularity, +) -> admin_observability::Granularity { + match granularity { + ObservationGranularity::FifteenMinutes => admin_observability::Granularity::FifteenMinutes, + ObservationGranularity::Hour => admin_observability::Granularity::Hour, + ObservationGranularity::Day => admin_observability::Granularity::Day, + } +} + +pub(crate) fn admin_usage_page( + page: UsageRecordPage, +) -> AdminStoreResult { + Ok(admin_observability::UsagePage { + items: page + .items + .into_iter() + .map(admin_usage_record) + .collect::>()?, + total: page.total, + next_cursor: page.next_cursor.map(admin_cursor), + }) +} + +pub(crate) fn admin_cursor( + cursor: ObservabilityCursor, +) -> admin_observability::ObservabilityCursor { + admin_observability::ObservabilityCursor { + observed_at: cursor.observed_at, + stable_id: cursor.stable_id, + } +} + +pub(crate) fn admin_usage_record( + record: UsageRecord, +) -> AdminStoreResult { + let billing = match (&record.cost_amount, &record.cost_currency) { + (Some(amount), Some(currency)) => Some(admin_observability::UsageBilling::Total { + source: record.cost_source.clone(), + total: admin_observability::CurrencyCost { + currency: currency.clone(), + amount: admin_decimal_amount(amount.clone())?, + }, + }), + (None, None) => None, + _ => { + return Err(observability_error(StoreError::InvalidData { + entity: "observability request billing", + message: "cost amount and currency must be present together".to_owned(), + })); + } + }; + Ok(admin_observability::UsageRecord { + id: record.id, + client_api_key_ref: record.client_api_key_ref, + config_revision: record.config_revision, + protocol: record.protocol, + operation: record.operation, + endpoint: record.endpoint, + client_transport: record.client_transport, + requested_model_id: record.requested_model_id, + provider_kind: record.provider_kind, + provider_account_ref: record.provider_account_ref, + provider_account_name: record.provider_account_name, + provider_account_email: record.provider_account_email, + provider_account_authentication_kind: record.provider_account_authentication_kind, + upstream_model_id: record.upstream_model_id, + upstream_transport: record.upstream_transport, + http_version: record.http_version, + websocket_pool: record.websocket_pool, + service_tier: record.service_tier, + provider_metadata_json: record.provider_metadata_json, + attempt_count: record.attempt_count, + upstream_send_state: record.upstream_send_state, + downstream_committed_at: record.downstream_committed_at, + outcome: admin_request_outcome(&record.outcome)?, + client_status_code: record.client_status_code, + upstream_status_code: record.upstream_status_code, + client_response_id: record.client_response_id, + upstream_request_id: record.upstream_request_id, + upstream_response_id: record.upstream_response_id, + error_kind: record.error_kind, + provider_error_code: record.provider_error_code, + error_message: record.error_message, + retry_after_ms: record.retry_after_ms, + input_tokens: record.input_tokens, + output_tokens: record.output_tokens, + cached_tokens: record.cached_tokens, + cache_write_tokens: record.cache_write_tokens, + reasoning_tokens: record.reasoning_tokens, + image_input_tokens: record.image_input_tokens, + image_output_tokens: record.image_output_tokens, + total_tokens: record.total_tokens, + cost_source: record.cost_source, + cost_amount: admin_optional_decimal_amount(record.cost_amount)?, + cost_currency: record.cost_currency, + billing, + transport_decision_wait_ms: record.transport_decision_wait_ms, + connect_ms: record.connect_ms, + headers_ms: record.headers_ms, + first_event_ms: record.first_event_ms, + first_reasoning_ms: record.first_reasoning_ms, + first_text_ms: record.first_text_ms, + first_token_ms: record.first_token_ms, + provider_processing_ms: record.provider_processing_ms, + latency_ms: record.latency_ms, + client_ip: record.client_ip, + user_agent: record.user_agent, + reasoning_effort: record.reasoning_effort, + reasoning_preset: record.reasoning_preset, + request_kind: record.request_kind, + subagent_kind: record.subagent_kind, + compact: record.compact, + image_generation_requested: record.image_generation_requested, + image_generation_succeeded: record.image_generation_succeeded, + started_at: record.started_at, + deadline_at: record.deadline_at, + completed_at: record.completed_at, + }) +} + +pub(crate) fn admin_request_outcome( + outcome: &str, +) -> AdminStoreResult { + admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| { + observability_error(StoreError::InvalidData { + entity: "observability request outcome", + message: "invalid request outcome".to_owned(), + }) + }) +} + +pub(crate) fn admin_usage_detail( + detail: UsageRecordDetail, +) -> AdminStoreResult { + Ok(admin_observability::UsageDetail { + request: admin_usage_record(detail.request)?, + attempts: detail + .attempts + .into_iter() + .map(admin_usage_attempt) + .collect::>()?, + }) +} + +pub(crate) fn admin_usage_attempt( + attempt: UsageAttemptObservation, +) -> AdminStoreResult { + Ok(admin_observability::UsageAttempt { + source: attempt.source, + id: attempt.id, + attempt_index: attempt.attempt_index, + component: attempt.component, + operation: attempt.operation, + provider_kind: attempt.provider_kind, + provider_account_ref: attempt.provider_account_ref, + provider_account_name: attempt.provider_account_name, + provider_account_email: attempt.provider_account_email, + provider_account_authentication_kind: attempt.provider_account_authentication_kind, + upstream_model_id: attempt.upstream_model_id, + upstream_transport: attempt.upstream_transport, + upstream_send_state: attempt.upstream_send_state, + outcome: admin_request_outcome(&attempt.outcome)?, + downstream_committed: attempt.downstream_committed, + status_code: attempt.status_code, + provider_error_code: attempt.provider_error_code, + failure_kind: attempt.failure_kind, + retry_after_ms: attempt.retry_after_ms, + upstream_request_id: attempt.upstream_request_id, + latency_ms: attempt.latency_ms, + message: attempt.message, + input_tokens: attempt.input_tokens, + output_tokens: attempt.output_tokens, + cached_tokens: attempt.cached_tokens, + cache_write_tokens: attempt.cache_write_tokens, + reasoning_tokens: attempt.reasoning_tokens, + total_tokens: attempt.total_tokens, + cost_source: attempt.cost_source, + cost_amount: admin_optional_decimal_amount(attempt.cost_amount)?, + cost_currency: attempt.cost_currency, + occurred_at: attempt.occurred_at, + }) +} + +pub(crate) fn admin_usage_overview( + overview: UsageOverview, +) -> AdminStoreResult { + Ok(admin_observability::UsageOverview { + range: admin_range(overview.range), + requests: admin_request_metrics(overview.requests)?, + attempts: admin_attempt_metrics(overview.attempts)?, + providers: overview + .providers + .into_iter() + .map(admin_provider_observation) + .collect(), + }) +} + +pub(crate) fn admin_provider_observation( + observation: ProviderObservation, +) -> admin_observability::ProviderObservation { + admin_observability::ProviderObservation { + provider_kind: observation.provider_kind, + request_count: observation.request_count, + attempt_count: observation.attempt_count, + failure_count: observation.failure_count, + total_tokens: observation.total_tokens, + } +} + +pub(crate) fn admin_diagnostic_observation( + observation: DiagnosticObservation, +) -> AdminStoreResult { + Ok(admin_observability::DiagnosticObservation { + key: observation.key, + name: observation.name, + request_count: observation.request_count, + success_count: observation.success_count, + failure_count: observation.failure_count, + attempt_count: observation.attempt_count, + total_tokens: observation.total_tokens, + average_latency_ms: observation.average_latency_ms, + latency_p95_ms: observation.latency_p95_ms, + cost_coverage: admin_cost_coverage(observation.cost_coverage), + costs: admin_currency_costs(observation.costs)?, + }) +} + +pub(crate) fn admin_ops_error_page( + page: OpsErrorPage, +) -> AdminStoreResult { + Ok(admin_observability::OpsErrorPage { + items: page.items.into_iter().map(admin_ops_error).collect(), + total: page.total, + next_cursor: page.next_cursor.map(admin_cursor), + }) +} + +pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::OpsError { + admin_observability::OpsError { + source: error.source, + event_id: error.event_id, + request_id: error.request_id, + attempt_index: error.attempt_index, + client_api_key_ref: error.client_api_key_ref, + component: error.component, + operation: error.operation, + endpoint: error.endpoint, + provider_kind: error.provider_kind, + provider_account_ref: error.provider_account_ref, + provider_account_name: error.provider_account_name, + provider_account_email: error.provider_account_email, + provider_account_authentication_kind: error.provider_account_authentication_kind, + upstream_model_id: error.upstream_model_id, + upstream_transport: error.upstream_transport, + failure_kind: error.failure_kind, + client_status_code: error.client_status_code, + upstream_status_code: error.upstream_status_code, + provider_error_code: error.provider_error_code, + client_response_id: error.client_response_id, + upstream_request_id: error.upstream_request_id, + latency_ms: error.latency_ms, + message: error.message, + occurrence_count: error.occurrence_count, + occurred_at: error.occurred_at, + stable_sort_id: error.stable_sort_id, + } +} + +pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { + Ok(UsageRecord { + id: get(row, "id")?, + client_api_key_ref: get(row, "client_api_key_ref")?, + config_revision: unsigned(row, "config_revision")?, + protocol: get(row, "protocol")?, + operation: get(row, "operation")?, + endpoint: get(row, "endpoint")?, + client_transport: get(row, "client_transport")?, + requested_model_id: get(row, "requested_model_id")?, + provider_kind: get(row, "provider_kind")?, + provider_account_ref: get(row, "provider_account_ref")?, + provider_account_name: get(row, "provider_account_name")?, + provider_account_email: get(row, "provider_account_email")?, + provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?, + upstream_model_id: get(row, "upstream_model_id")?, + upstream_transport: get(row, "upstream_transport")?, + http_version: get(row, "http_version")?, + websocket_pool: get(row, "websocket_pool")?, + service_tier: get(row, "service_tier")?, + provider_metadata_json: get::>(row, "provider_observation_json")? + .map(|value| serde_json::to_string(&value)) + .transpose() + .map_err(|_| postgres_unavailable("encode provider observation"))?, + attempt_count: to_u32(get(row, "attempt_count")?)?, + upstream_send_state: get(row, "upstream_send_state")?, + downstream_committed_at: get(row, "downstream_committed_at")?, + outcome: get(row, "outcome")?, + client_status_code: optional_status(row, "client_status_code")?, + upstream_status_code: optional_status(row, "upstream_status_code")?, + client_response_id: opaque_response_id(row, "client_response_id")?, + upstream_request_id: get(row, "upstream_request_id")?, + upstream_response_id: opaque_response_id(row, "upstream_response_id")?, + error_kind: get(row, "error_kind")?, + provider_error_code: get(row, "provider_error_code")?, + error_message: get(row, "error_message")?, + retry_after_ms: optional_unsigned(row, "retry_after_ms")?, + input_tokens: optional_unsigned(row, "input_tokens")?, + output_tokens: optional_unsigned(row, "output_tokens")?, + cached_tokens: optional_unsigned(row, "cached_tokens")?, + cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?, + reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?, + image_input_tokens: optional_unsigned(row, "image_input_tokens")?, + image_output_tokens: optional_unsigned(row, "image_output_tokens")?, + total_tokens: optional_unsigned(row, "total_tokens")?, + cost_source: get(row, "cost_source")?, + cost_amount: optional_decimal(row, "cost_amount")?, + cost_currency: get(row, "cost_currency")?, + transport_decision_wait_ms: optional_unsigned(row, "transport_decision_wait_ms")?, + connect_ms: optional_unsigned(row, "connect_ms")?, + headers_ms: optional_unsigned(row, "headers_ms")?, + first_event_ms: optional_unsigned(row, "first_event_ms")?, + first_reasoning_ms: optional_unsigned(row, "first_reasoning_ms")?, + first_text_ms: optional_unsigned(row, "first_text_ms")?, + first_token_ms: optional_unsigned(row, "first_token_ms")?, + provider_processing_ms: optional_unsigned(row, "provider_processing_ms")?, + latency_ms: optional_unsigned(row, "latency_ms")?, + client_ip: get(row, "client_ip")?, + user_agent: get(row, "user_agent")?, + reasoning_effort: get(row, "reasoning_effort")?, + reasoning_preset: get(row, "reasoning_preset")?, + request_kind: get(row, "request_kind")?, + subagent_kind: get(row, "subagent_kind")?, + compact: get(row, "compact")?, + image_generation_requested: get(row, "image_generation_requested")?, + image_generation_succeeded: get(row, "image_generation_succeeded")?, + started_at: get(row, "started_at")?, + deadline_at: get(row, "deadline_at")?, + completed_at: get(row, "completed_at")?, + }) +} + +pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { + Ok(OpsErrorRecord { + source: get(row, "source")?, + event_id: get(row, "event_id")?, + request_id: get(row, "request_id")?, + attempt_index: get::>(row, "attempt_index")? + .map(to_u32) + .transpose()?, + client_api_key_ref: get(row, "client_api_key_ref")?, + component: get(row, "component")?, + operation: get(row, "operation")?, + endpoint: get(row, "endpoint")?, + provider_kind: get(row, "provider_kind")?, + provider_account_ref: get(row, "provider_account_ref")?, + provider_account_name: get(row, "provider_account_name")?, + provider_account_email: get(row, "provider_account_email")?, + provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?, + upstream_model_id: get(row, "upstream_model_id")?, + upstream_transport: get(row, "upstream_transport")?, + failure_kind: get(row, "failure_kind")?, + client_status_code: optional_status(row, "client_status_code")?, + upstream_status_code: optional_status(row, "upstream_status_code")?, + provider_error_code: get(row, "provider_error_code")?, + client_response_id: opaque_response_id(row, "client_response_id")?, + upstream_request_id: get(row, "upstream_request_id")?, + latency_ms: optional_unsigned(row, "latency_ms")?, + message: get(row, "message")?, + occurrence_count: to_u32(get(row, "occurrence_count")?)?, + occurred_at: get(row, "occurred_at")?, + stable_sort_id: get(row, "stable_sort_id")?, + }) +} + +pub(crate) fn cost_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { + Ok(CurrencyCostTotal { + currency: get(row, "cost_currency")?, + amount: DecimalAmount::from_str(&get::(row, "amount")?)?, + }) +} + +pub(crate) fn calculated_usage_billing_fact_from_row( + row: &sqlx::postgres::PgRow, +) -> StoreResult { + Ok(CalculatedUsageBillingFact { + bucket_start: get(row, "bucket_start")?, + provider_kind: get(row, "provider_kind")?, + upstream_model_id: get(row, "upstream_model_id")?, + service_tier: get(row, "service_tier")?, + input_tokens: optional_unsigned(row, "input_tokens")?, + output_tokens: optional_unsigned(row, "output_tokens")?, + cached_tokens: optional_unsigned(row, "cached_tokens")?, + cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?, + total: cost_from_row(row)?, + }) +} + +pub(crate) fn optional_decimal( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> StoreResult> { + get::>(row, column)? + .map(|value| DecimalAmount::from_str(&value)) + .transpose() +} + +pub(crate) fn opaque_response_id( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> StoreResult> { + get::>>(row, column)? + .map(String::from_utf8) + .transpose() + .map_err(|_| invalid(column)) +} + +pub(crate) fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> { + let mut unique = BTreeSet::new(); + for account_id in account_ids { + validate_text(account_id, MAX_FILTER_BYTES, "provider account ID")?; + if !unique.insert(account_id.as_str()) { + return Err(invalid("account usage query contains duplicate IDs")); + } + } + Ok(()) +} + +pub(crate) fn validate_optional_text( + value: Option<&str>, + max_bytes: usize, + field: &'static str, +) -> StoreResult<()> { + value.map_or(Ok(()), |value| validate_text(value, max_bytes, field)) +} + +pub(crate) fn validate_text(value: &str, max_bytes: usize, field: &'static str) -> StoreResult<()> { + if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) { + return Err(invalid(&format!("{field} is invalid"))); + } + Ok(()) +} + +pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -> StoreResult +where + T: sqlx::Decode<'r, Postgres> + sqlx::Type, +{ + row.try_get(column).map_err(|_| invalid(column)) +} + +pub(crate) fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult { + to_u64(get(row, column)?) +} + +pub(crate) fn optional_unsigned( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> StoreResult> { + get::>(row, column)?.map(to_u64).transpose() +} + +pub(crate) fn optional_status( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> StoreResult> { + get::>(row, column)? + .map(|value| { + u16::try_from(value) + .ok() + .filter(|value| (100..=599).contains(value)) + .ok_or_else(|| invalid("status code is outside its supported range")) + }) + .transpose() +} + +pub(crate) fn to_u64(value: i64) -> StoreResult { + u64::try_from(value).map_err(|_| invalid("numeric observation is negative")) +} + +pub(crate) fn to_u32(value: i32) -> StoreResult { + u32::try_from(value).map_err(|_| invalid("integer observation is negative")) +} + +pub(crate) fn invalid(message: &str) -> StoreError { + StoreError::InvalidData { + entity: "observability", + message: message.to_owned(), + } +} diff --git a/backend/crates/gateway-store/src/postgres/observability/mod.rs b/backend/crates/gateway-store/src/postgres/observability/mod.rs new file mode 100644 index 000000000..72eb21dd1 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/mod.rs @@ -0,0 +1,29 @@ +//! 从 `model_requests`、`ops_events` 与账号公共投影读取观测事实。 + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + str::FromStr, +}; + +use async_trait::async_trait; +use chrono::{DateTime, TimeDelta, Utc}; +use gateway_admin::{ + model::observability as admin_observability, + ports::store::{AdminStoreResult, ObservabilityStore as AdminObservabilityStore}, +}; +use sqlx::{PgPool, Postgres, QueryBuilder, Row}; + +use crate::{ + DecimalAmount, StoreError, StoreResult, admin_store_error, postgres_unavailable, + require_nonempty, +}; + +mod admin_adapter; +mod mapping; +mod model; +mod queries; + +pub use admin_adapter::*; +pub(crate) use mapping::*; +pub use model::*; +pub(crate) use queries::*; diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs new file mode 100644 index 000000000..92a7e6219 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -0,0 +1,713 @@ +//! 查询模型、校验与观测端口契约。 + +use super::*; + +pub(crate) const MAX_PAGE_SIZE: u16 = 100; +pub(crate) const MAX_FILTER_BYTES: usize = 256; +pub(crate) const MAX_SEARCH_BYTES: usize = 512; +pub(crate) const MAX_ACCOUNT_IDS: usize = 200; +/// 概览卡只展示最近使用的四个账号；完整账号用量由账号管理页单独查询。 +pub(crate) const DASHBOARD_ACCOUNT_LIMIT: u16 = 4; +pub(crate) const DIAGNOSTIC_LIMIT: i64 = 100; +pub(crate) const ACCOUNT_USAGE_TIMELINE_HOURS: i64 = 24; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObservabilityRange { + pub start: DateTime, + pub end: DateTime, +} + +impl ObservabilityRange { + pub fn new(start: DateTime, end: DateTime) -> StoreResult { + if end.signed_duration_since(start) <= TimeDelta::zero() { + return Err(invalid("time range must be positive")); + } + Ok(Self { start, end }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObservabilityPageSize(u16); + +impl ObservabilityPageSize { + pub fn new(value: u16) -> StoreResult { + if value == 0 || value > MAX_PAGE_SIZE { + return Err(invalid("page size must be between 1 and 100")); + } + Ok(Self(value)) + } + + #[must_use] + pub const fn get(self) -> u16 { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ObservabilityPageNumber(u32); + +impl ObservabilityPageNumber { + pub fn new(value: u32) -> StoreResult { + if value == 0 { + return Err(invalid("page number must be positive")); + } + Ok(Self(value)) + } + + #[must_use] + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservabilityCursor { + pub observed_at: DateTime, + pub stable_id: String, +} + +impl ObservabilityCursor { + pub fn new(observed_at: DateTime, stable_id: impl Into) -> StoreResult { + let stable_id = stable_id.into(); + validate_text(&stable_id, MAX_FILTER_BYTES, "cursor ID")?; + Ok(Self { + observed_at, + stable_id, + }) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct UsageRecordFilter { + pub client_api_key_ref: Option, + pub request_id: Option, + pub provider_account_ref: Option, + pub operation: Option, + pub provider_kind: Option, + pub model: Option, + pub outcome: Option, + pub status_code: Option, + pub transport: Option, + pub attempt_index: Option, + pub response_id: Option, + pub upstream_request_id: Option, + pub search: Option, +} + +impl UsageRecordFilter { + pub fn validate(&self) -> StoreResult<()> { + for (value, field) in [ + (self.client_api_key_ref.as_deref(), "client API key filter"), + (self.request_id.as_deref(), "request ID filter"), + ( + self.provider_account_ref.as_deref(), + "provider account filter", + ), + (self.operation.as_deref(), "operation filter"), + (self.provider_kind.as_deref(), "provider filter"), + (self.model.as_deref(), "model filter"), + (self.outcome.as_deref(), "outcome filter"), + (self.transport.as_deref(), "transport filter"), + ( + self.upstream_request_id.as_deref(), + "upstream request ID filter", + ), + ] { + validate_optional_text(value, MAX_FILTER_BYTES, field)?; + } + validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?; + if self + .status_code + .is_some_and(|status| !(100..=599).contains(&status)) + { + return Err(invalid("status code filter must be between 100 and 599")); + } + if self + .attempt_index + .is_some_and(|index| index == 0 || i32::try_from(index).is_err()) + { + return Err(invalid("attempt index filter is out of range")); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UsageRecordQuery { + pub range: ObservabilityRange, + pub filter: UsageRecordFilter, + pub cursor: Option, + pub page: ObservabilityPageNumber, + pub page_size: ObservabilityPageSize, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OpsErrorFilter { + pub client_api_key_ref: Option, + pub request_id: Option, + pub provider_account_ref: Option, + pub provider_kind: Option, + pub operation: Option, + pub model: Option, + pub transport: Option, + pub attempt_index: Option, + pub response_id: Option, + pub upstream_request_id: Option, + pub failure_kind: Option, + pub status_code: Option, + pub search: Option, +} + +impl OpsErrorFilter { + pub fn validate(&self) -> StoreResult<()> { + for (value, field) in [ + (self.client_api_key_ref.as_deref(), "client API key filter"), + (self.request_id.as_deref(), "request ID filter"), + ( + self.provider_account_ref.as_deref(), + "provider account filter", + ), + (self.provider_kind.as_deref(), "provider filter"), + (self.operation.as_deref(), "operation filter"), + (self.model.as_deref(), "model filter"), + (self.transport.as_deref(), "transport filter"), + ( + self.upstream_request_id.as_deref(), + "upstream request ID filter", + ), + (self.failure_kind.as_deref(), "failure kind filter"), + ] { + validate_optional_text(value, MAX_FILTER_BYTES, field)?; + } + validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?; + if self + .status_code + .is_some_and(|status| !(100..=599).contains(&status)) + { + return Err(invalid("status code filter must be between 100 and 599")); + } + if self + .attempt_index + .is_some_and(|index| index == 0 || i32::try_from(index).is_err()) + { + return Err(invalid("attempt index filter is out of range")); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpsErrorQuery { + pub range: ObservabilityRange, + pub filter: OpsErrorFilter, + pub cursor: Option, + pub page: ObservabilityPageNumber, + pub page_size: ObservabilityPageSize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiagnosticDimension { + Provider, + Model, + Account, + ApiKey, + Transport, + Failure, + Status, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CurrencyCostTotal { + pub currency: String, + pub amount: DecimalAmount, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CostCoverage { + pub provider_reported_count: u64, + pub calculated_count: u64, + pub unavailable_count: u64, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RequestMetrics { + pub request_count: u64, + pub success_count: u64, + pub failure_count: u64, + pub cancelled_count: u64, + pub incomplete_count: u64, + pub caller_error_count: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, + pub cache_write_tokens: u64, + pub reasoning_tokens: u64, + pub total_tokens: u64, + pub first_token_latency_sum: u64, + pub first_token_latency_count: u64, + pub latency_sum: u64, + pub latency_count: u64, + pub max_latency_ms: Option, + pub min_latency_ms: Option, + /// 分母：`input_tokens is not null`，即上游确实报告过 input token 事实的请求。 + pub cache_eligible_request_count: u64, + /// 分子：分母集合中 `cached_tokens > 0` 的请求。 + pub cache_hit_request_count: u64, + pub latency_percentiles: LatencyPercentiles, + pub first_token_latency_percentiles: LatencyPercentiles, +} + +impl RequestMetrics { + /// 请求级 cache hit rate；没有 input token 事实时返回 `None`。 + #[must_use] + pub fn cache_hit_request_rate(&self) -> Option { + (self.cache_eligible_request_count > 0) + .then(|| self.cache_hit_request_count as f64 / self.cache_eligible_request_count as f64) + } +} + +/// PostgreSQL `percentile_cont` 的非负、有限毫秒值；bits 保留插值小数且可安全比较。 +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct PercentileMilliseconds(u64); + +impl PercentileMilliseconds { + pub(crate) fn new(value: f64) -> StoreResult { + if !value.is_finite() || value < 0.0 { + return Err(postgres_unavailable("decode latency percentile")); + } + Ok(Self(value.to_bits())) + } + + #[must_use] + pub const fn as_f64(self) -> f64 { + f64::from_bits(self.0) + } +} + +impl std::fmt::Debug for PercentileMilliseconds { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_f64().fmt(formatter) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct LatencyPercentiles { + pub p50_ms: Option, + pub p95_ms: Option, + pub p99_ms: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AttemptMetrics { + pub attempt_count: u64, + pub success_count: u64, + pub failure_count: u64, + pub cancelled_count: u64, + pub incomplete_count: u64, + pub rate_limited_count: u64, + pub auth_failure_count: u64, + pub provider_5xx_count: u64, + pub cost_coverage: CostCoverage, + pub costs: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObservationGranularity { + FifteenMinutes, + Hour, + Day, +} + +impl ObservationGranularity { + #[must_use] + pub const fn seconds(self) -> i64 { + match self { + Self::FifteenMinutes => 15 * 60, + Self::Hour => 60 * 60, + Self::Day => 24 * 60 * 60, + } + } + + pub(crate) const fn sql_interval(self) -> &'static str { + match self { + Self::FifteenMinutes => "15 minutes", + Self::Hour => "1 hour", + Self::Day => "1 day", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestMetricPoint { + pub bucket_start: DateTime, + pub granularity: ObservationGranularity, + pub metrics: RequestMetrics, + pub cost_coverage: CostCoverage, + pub costs: Vec, +} + +/// 已完整交付且由 Provider 计算费用的请求事实。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CalculatedUsageBillingFact { + pub bucket_start: DateTime, + pub provider_kind: String, + pub upstream_model_id: String, + pub service_tier: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub total: CurrencyCostTotal, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProviderAccountMetrics { + pub total: u64, + pub enabled: u64, + pub unavailable: u64, + pub active: u64, + pub expired: u64, + pub quota_exhausted: u64, + pub disabled: u64, + pub banned: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountRequestBucket { + pub bucket_start: DateTime, + pub request_count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountUsageObservation { + pub account_id: String, + pub provider_kind: String, + pub authentication_kind: String, + pub name: String, + pub email: Option, + pub plan_type: Option, + pub enabled: bool, + pub availability: String, + pub request_count: u64, + pub success_count: u64, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub reasoning_tokens: Option, + pub image_input_tokens: Option, + pub image_output_tokens: Option, + pub image_request_count: u64, + pub image_request_failed_count: u64, + pub total_tokens: Option, + pub cost_coverage: CostCoverage, + pub costs: Vec, + pub last_used_at: Option>, + pub request_buckets: Vec, + pub models: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountModelUsageObservation { + pub model: String, + pub request_count: u64, + pub success_count: u64, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub reasoning_tokens: Option, + pub image_input_tokens: Option, + pub image_output_tokens: Option, + pub image_request_count: u64, + pub image_request_failed_count: u64, + pub total_tokens: Option, + pub cost_coverage: CostCoverage, + pub costs: Vec, + pub last_used_at: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountUsageQuery { + pub range: ObservabilityRange, + pub account_ids: Option>, + pub limit: u16, + pub(crate) include_hourly_request_buckets: bool, +} + +impl ProviderAccountUsageQuery { + pub fn for_accounts(range: ObservabilityRange, account_ids: Vec) -> StoreResult { + if account_ids.is_empty() || account_ids.len() > MAX_ACCOUNT_IDS { + return Err(invalid( + "account usage query requires between 1 and 200 IDs", + )); + } + validate_account_ids(&account_ids)?; + Ok(Self { + range, + limit: u16::try_from(account_ids.len()) + .map_err(|_| invalid("account usage query is too large"))?, + account_ids: Some(account_ids), + include_hourly_request_buckets: false, + }) + } + + pub fn recent(range: ObservabilityRange, limit: u16) -> StoreResult { + if limit == 0 || usize::from(limit) > MAX_ACCOUNT_IDS { + return Err(invalid("account usage limit must be between 1 and 200")); + } + Ok(Self { + range, + account_ids: None, + limit, + include_hourly_request_buckets: false, + }) + } + + pub fn with_hourly_request_buckets(mut self) -> StoreResult { + if self.range.end.signed_duration_since(self.range.start) + > TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS) + { + return Err(invalid("account request timeline cannot exceed 24 hours")); + } + self.include_hourly_request_buckets = true; + Ok(self) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DashboardObservation { + pub range: ObservabilityRange, + pub requests: RequestMetrics, + pub attempts: AttemptMetrics, + pub provider_accounts: ProviderAccountMetrics, + pub trend: Vec, + pub account_usage: Vec, + pub recent_requests: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UsageRecord { + pub id: String, + pub client_api_key_ref: String, + pub config_revision: u64, + pub protocol: String, + pub operation: String, + pub endpoint: String, + pub client_transport: String, + pub requested_model_id: String, + pub provider_kind: Option, + pub provider_account_ref: Option, + pub provider_account_name: Option, + pub provider_account_email: Option, + pub provider_account_authentication_kind: Option, + pub upstream_model_id: Option, + pub upstream_transport: Option, + pub http_version: Option, + pub websocket_pool: Option, + pub service_tier: Option, + pub provider_metadata_json: Option, + pub attempt_count: u32, + pub upstream_send_state: String, + pub downstream_committed_at: Option>, + pub outcome: String, + pub client_status_code: Option, + pub upstream_status_code: Option, + pub client_response_id: Option, + pub upstream_request_id: Option, + pub upstream_response_id: Option, + pub error_kind: Option, + pub provider_error_code: Option, + pub error_message: Option, + pub retry_after_ms: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub reasoning_tokens: Option, + pub image_input_tokens: Option, + pub image_output_tokens: Option, + pub total_tokens: Option, + pub cost_source: String, + pub cost_amount: Option, + pub cost_currency: Option, + pub transport_decision_wait_ms: Option, + pub connect_ms: Option, + pub headers_ms: Option, + pub first_event_ms: Option, + pub first_reasoning_ms: Option, + pub first_text_ms: Option, + pub first_token_ms: Option, + pub provider_processing_ms: Option, + pub latency_ms: Option, + pub client_ip: Option, + pub user_agent: Option, + pub reasoning_effort: Option, + pub reasoning_preset: Option, + pub request_kind: Option, + pub subagent_kind: Option, + pub compact: bool, + pub image_generation_requested: bool, + pub image_generation_succeeded: Option, + pub started_at: DateTime, + pub deadline_at: DateTime, + pub completed_at: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UsageRecordPage { + pub items: Vec, + pub total: u64, + pub next_cursor: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UsageAttemptObservation { + pub source: String, + pub id: String, + pub attempt_index: u32, + pub component: String, + pub operation: String, + pub provider_kind: Option, + pub provider_account_ref: Option, + pub provider_account_name: Option, + pub provider_account_email: Option, + pub provider_account_authentication_kind: Option, + pub upstream_model_id: Option, + pub upstream_transport: Option, + pub upstream_send_state: Option, + pub outcome: String, + pub downstream_committed: bool, + pub status_code: Option, + pub provider_error_code: Option, + pub failure_kind: Option, + pub retry_after_ms: Option, + pub upstream_request_id: Option, + pub latency_ms: Option, + pub message: Option, + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_tokens: Option, + pub cache_write_tokens: Option, + pub reasoning_tokens: Option, + pub total_tokens: Option, + pub cost_source: Option, + pub cost_amount: Option, + pub cost_currency: Option, + pub occurred_at: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UsageRecordDetail { + pub request: UsageRecord, + pub attempts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderObservation { + pub provider_kind: String, + pub request_count: u64, + pub attempt_count: u64, + pub failure_count: u64, + pub total_tokens: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UsageOverview { + pub range: ObservabilityRange, + pub requests: RequestMetrics, + pub attempts: AttemptMetrics, + pub providers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiagnosticObservation { + pub key: String, + pub name: String, + pub request_count: u64, + pub success_count: u64, + pub failure_count: u64, + pub attempt_count: u64, + pub total_tokens: u64, + pub average_latency_ms: Option, + pub latency_p95_ms: Option, + pub cost_coverage: CostCoverage, + pub costs: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpsErrorRecord { + pub source: String, + pub event_id: String, + pub request_id: Option, + pub attempt_index: Option, + pub client_api_key_ref: Option, + pub component: String, + pub operation: String, + pub endpoint: Option, + pub provider_kind: Option, + pub provider_account_ref: Option, + pub provider_account_name: Option, + pub provider_account_email: Option, + pub provider_account_authentication_kind: Option, + pub upstream_model_id: Option, + pub upstream_transport: Option, + pub failure_kind: String, + pub client_status_code: Option, + pub upstream_status_code: Option, + pub provider_error_code: Option, + pub client_response_id: Option, + pub upstream_request_id: Option, + pub latency_ms: Option, + pub message: String, + pub occurrence_count: u32, + pub occurred_at: DateTime, + pub stable_sort_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OpsErrorPage { + pub items: Vec, + pub total: u64, + pub next_cursor: Option, +} + +#[async_trait] +pub trait ObservabilityRepository: Send + Sync { + async fn dashboard_summary( + &self, + range: ObservabilityRange, + ) -> StoreResult; + async fn dashboard_trend( + &self, + range: ObservabilityRange, + ) -> StoreResult>; + async fn usage_trend( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + ) -> StoreResult>; + async fn usage_calculated_billing_facts( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + ) -> StoreResult>; + async fn provider_account_usage( + &self, + query: ProviderAccountUsageQuery, + ) -> StoreResult>; + async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult; + async fn usage_record_detail(&self, request_id: &str) -> StoreResult; + async fn usage_summary( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + ) -> StoreResult; + async fn usage_diagnostics( + &self, + range: ObservabilityRange, + filter: UsageRecordFilter, + dimension: DiagnosticDimension, + ) -> StoreResult>; + async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult; +} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs new file mode 100644 index 000000000..43036f8e8 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs @@ -0,0 +1,336 @@ +//! 账号用量查询族。 + +use super::super::*; + +pub(crate) async fn provider_account_usage( + pool: &PgPool, + query: ProviderAccountUsageQuery, +) -> StoreResult> { + if let Some(account_ids) = &query.account_ids { + validate_account_ids(account_ids)?; + } + if query.limit == 0 || usize::from(query.limit) > MAX_ACCOUNT_IDS { + return Err(invalid("account usage limit must be between 1 and 200")); + } + + let mut statement = QueryBuilder::::new( + "select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, + pa.plan_type, pa.enabled, pa.availability, + count(mr.id)::bigint as request_count, + count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count, + sum(mr.input_tokens)::bigint as input_tokens, + sum(mr.output_tokens)::bigint as output_tokens, + sum(mr.cached_tokens)::bigint as cached_tokens, + sum(mr.cache_write_tokens)::bigint as cache_write_tokens, + sum(mr.reasoning_tokens)::bigint as reasoning_tokens, + sum(mr.image_input_tokens)::bigint as image_input_tokens, + sum(mr.image_output_tokens)::bigint as image_output_tokens, + count(mr.id) filter (where mr.image_generation_succeeded is true)::bigint + as image_request_count, + count(mr.id) filter (where mr.image_generation_succeeded is false)::bigint + as image_request_failed_count, + sum(mr.total_tokens)::bigint as total_tokens, + count(mr.id) filter (where mr.cost_source = 'provider_reported')::bigint + as provider_reported_count, + count(mr.id) filter (where mr.cost_source = 'calculated')::bigint + as calculated_count, + count(mr.id) filter (where mr.cost_source = 'unavailable')::bigint + as unavailable_count, + max(mr.started_at) as last_used_at + from provider_accounts pa + left join model_requests mr + on mr.provider_account_ref = pa.id and mr.started_at >= ", + ); + statement.push_bind(query.range.start); + statement.push(" and mr.started_at < "); + statement.push_bind(query.range.end); + push_completed_usage_fact_filter(&mut statement, "mr"); + if let Some(account_ids) = &query.account_ids { + statement.push(" where pa.id = any("); + statement.push_bind(account_ids.clone()); + statement.push("::text[])"); + } + statement.push( + " group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, + pa.plan_type, pa.enabled, pa.availability + order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ", + ); + statement.push_bind(i64::from(query.limit)); + let rows = statement + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load provider account usage"))?; + + let mut observations = rows + .iter() + .map(provider_account_from_row) + .collect::>>()?; + if observations.is_empty() { + return Ok(observations); + } + let account_ids = observations + .iter() + .map(|item| item.account_id.clone()) + .collect::>(); + let mut request_buckets = if query.include_hourly_request_buckets { + provider_account_request_buckets(pool, query.range, &account_ids).await? + } else { + HashMap::new() + }; + let account_costs = account_costs(pool, query.range, &account_ids).await?; + let model_rows = account_model_rows(pool, query.range, &account_ids).await?; + let model_costs = account_model_costs(pool, query.range, &account_ids).await?; + let account_positions = observations + .iter() + .enumerate() + .map(|(index, item)| (item.account_id.clone(), index)) + .collect::>(); + for observation in &mut observations { + observation.costs = account_costs + .get(&observation.account_id) + .cloned() + .unwrap_or_default(); + observation.request_buckets = request_buckets + .remove(&observation.account_id) + .unwrap_or_default(); + } + for row in model_rows { + let account_id: String = get(&row, "provider_account_ref")?; + let Some(position) = account_positions.get(&account_id).copied() else { + continue; + }; + let model: String = get(&row, "model")?; + observations[position] + .models + .push(ProviderAccountModelUsageObservation { + costs: model_costs + .get(&(account_id, model.clone())) + .cloned() + .unwrap_or_default(), + model, + request_count: unsigned(&row, "request_count")?, + success_count: unsigned(&row, "success_count")?, + input_tokens: optional_unsigned(&row, "input_tokens")?, + output_tokens: optional_unsigned(&row, "output_tokens")?, + cached_tokens: optional_unsigned(&row, "cached_tokens")?, + cache_write_tokens: optional_unsigned(&row, "cache_write_tokens")?, + reasoning_tokens: optional_unsigned(&row, "reasoning_tokens")?, + image_input_tokens: optional_unsigned(&row, "image_input_tokens")?, + image_output_tokens: optional_unsigned(&row, "image_output_tokens")?, + image_request_count: unsigned(&row, "image_request_count")?, + image_request_failed_count: unsigned(&row, "image_request_failed_count")?, + total_tokens: optional_unsigned(&row, "total_tokens")?, + cost_coverage: coverage_from_row(&row)?, + last_used_at: get(&row, "last_used_at")?, + }); + } + Ok(observations) +} + +pub(crate) async fn provider_account_request_buckets( + pool: &PgPool, + range: ObservabilityRange, + account_ids: &[String], +) -> StoreResult>> { + let rows = sqlx::query( + "select provider_account_ref, + floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index, + count(*)::bigint as request_count + from model_requests mr + where mr.provider_account_ref = any($2::text[]) + and mr.started_at >= $1 and mr.started_at < $3 + and mr.outcome = 'succeeded' + and mr.downstream_committed_at is not null + and mr.client_status_code between 200 and 399 + group by mr.provider_account_ref, bucket_index + order by mr.provider_account_ref, bucket_index", + ) + .bind(range.start) + .bind(account_ids) + .bind(range.end) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load provider account request timeline"))?; + + let mut observed = HashMap::>::new(); + for row in rows { + observed + .entry(get(&row, "provider_account_ref")?) + .or_default() + .insert( + unsigned(&row, "bucket_index")?, + unsigned(&row, "request_count")?, + ); + } + + let step = TimeDelta::hours(1); + let mut timelines = HashMap::with_capacity(account_ids.len()); + for account_id in account_ids { + let mut account_observed = observed.remove(account_id).unwrap_or_default(); + let mut bucket_start = range.start; + let mut bucket_index = 0_u64; + let mut buckets = Vec::new(); + while bucket_start < range.end { + buckets.push(ProviderAccountRequestBucket { + bucket_start, + request_count: account_observed.remove(&bucket_index).unwrap_or_default(), + }); + bucket_start = bucket_start + .checked_add_signed(step) + .ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?; + bucket_index = bucket_index + .checked_add(1) + .ok_or_else(|| invalid("account request timeline exceeds supported buckets"))?; + } + timelines.insert(account_id.clone(), buckets); + } + Ok(timelines) +} + +pub(crate) async fn account_costs( + pool: &PgPool, + range: ObservabilityRange, + account_ids: &[String], +) -> StoreResult>> { + let rows = sqlx::query( + "select mr.provider_account_ref, mr.cost_currency, sum(mr.cost_amount)::text as amount + from model_requests mr + where mr.provider_account_ref = any($1::text[]) + and mr.started_at >= $2 and mr.started_at < $3 + and mr.outcome = 'succeeded' + and mr.downstream_committed_at is not null + and mr.client_status_code between 200 and 399 + and mr.cost_amount is not null and mr.cost_currency is not null + group by mr.provider_account_ref, mr.cost_currency + order by mr.provider_account_ref, mr.cost_currency", + ) + .bind(account_ids) + .bind(range.start) + .bind(range.end) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load provider account costs"))?; + let mut costs = HashMap::>::new(); + for row in rows { + costs + .entry(get(&row, "provider_account_ref")?) + .or_default() + .push(cost_from_row(&row)?); + } + Ok(costs) +} + +pub(crate) async fn account_model_rows( + pool: &PgPool, + range: ObservabilityRange, + account_ids: &[String], +) -> StoreResult> { + sqlx::query( + "select mr.provider_account_ref, + coalesce(mr.upstream_model_id, mr.requested_model_id) as model, + count(*)::bigint as request_count, + count(*) filter (where mr.outcome = 'succeeded')::bigint as success_count, + sum(mr.input_tokens)::bigint as input_tokens, + sum(mr.output_tokens)::bigint as output_tokens, + sum(mr.cached_tokens)::bigint as cached_tokens, + sum(mr.cache_write_tokens)::bigint as cache_write_tokens, + sum(mr.reasoning_tokens)::bigint as reasoning_tokens, + sum(mr.image_input_tokens)::bigint as image_input_tokens, + sum(mr.image_output_tokens)::bigint as image_output_tokens, + count(*) filter (where mr.image_generation_succeeded is true)::bigint + as image_request_count, + count(*) filter (where mr.image_generation_succeeded is false)::bigint + as image_request_failed_count, + sum(mr.total_tokens)::bigint as total_tokens, + count(*) filter (where mr.cost_source = 'provider_reported')::bigint + as provider_reported_count, + count(*) filter (where mr.cost_source = 'calculated')::bigint + as calculated_count, + count(*) filter (where mr.cost_source = 'unavailable')::bigint + as unavailable_count, + max(mr.started_at) as last_used_at + from model_requests mr + where mr.provider_account_ref = any($1::text[]) + and mr.started_at >= $2 and mr.started_at < $3 + and mr.outcome = 'succeeded' + and mr.downstream_committed_at is not null + and mr.client_status_code between 200 and 399 + group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id) + order by mr.provider_account_ref, request_count desc, model", + ) + .bind(account_ids) + .bind(range.start) + .bind(range.end) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load provider account model usage")) +} + +pub(crate) async fn account_model_costs( + pool: &PgPool, + range: ObservabilityRange, + account_ids: &[String], +) -> StoreResult>> { + let rows = sqlx::query( + "select mr.provider_account_ref, + coalesce(mr.upstream_model_id, mr.requested_model_id) as model, + mr.cost_currency, sum(mr.cost_amount)::text as amount + from model_requests mr + where mr.provider_account_ref = any($1::text[]) + and mr.started_at >= $2 and mr.started_at < $3 + and mr.outcome = 'succeeded' + and mr.downstream_committed_at is not null + and mr.client_status_code between 200 and 399 + and mr.cost_amount is not null and mr.cost_currency is not null + group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id), + mr.cost_currency + order by mr.provider_account_ref, model, mr.cost_currency", + ) + .bind(account_ids) + .bind(range.start) + .bind(range.end) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load provider account model costs"))?; + let mut costs = HashMap::<(String, String), Vec>::new(); + for row in rows { + costs + .entry((get(&row, "provider_account_ref")?, get(&row, "model")?)) + .or_default() + .push(cost_from_row(&row)?); + } + Ok(costs) +} + +pub(crate) fn provider_account_from_row( + row: &sqlx::postgres::PgRow, +) -> StoreResult { + Ok(ProviderAccountUsageObservation { + account_id: get(row, "id")?, + provider_kind: get(row, "provider_kind")?, + authentication_kind: get(row, "authentication_kind")?, + name: get(row, "name")?, + email: get(row, "email")?, + plan_type: get(row, "plan_type")?, + enabled: get(row, "enabled")?, + availability: get(row, "availability")?, + request_count: unsigned(row, "request_count")?, + success_count: unsigned(row, "success_count")?, + input_tokens: optional_unsigned(row, "input_tokens")?, + output_tokens: optional_unsigned(row, "output_tokens")?, + cached_tokens: optional_unsigned(row, "cached_tokens")?, + cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?, + reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?, + image_input_tokens: optional_unsigned(row, "image_input_tokens")?, + image_output_tokens: optional_unsigned(row, "image_output_tokens")?, + image_request_count: unsigned(row, "image_request_count")?, + image_request_failed_count: unsigned(row, "image_request_failed_count")?, + total_tokens: optional_unsigned(row, "total_tokens")?, + cost_coverage: coverage_from_row(row)?, + costs: Vec::new(), + last_used_at: get(row, "last_used_at")?, + request_buckets: Vec::new(), + models: Vec::new(), + }) +} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs new file mode 100644 index 000000000..95da964cd --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs @@ -0,0 +1,558 @@ +//! Dashboard 与趋势查询族。 + +use super::super::*; + +pub(crate) async fn request_metrics( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult { + filter.validate()?; + // 结果计数覆盖范围内全部请求（成功率/失败率的分母分子）；用量、缓存、 + // 延迟与成本聚合仅统计用量事实（已完整交付客户端的成功响应）。 + let fact = completed_usage_fact_predicate("mr"); + let mut query = QueryBuilder::::new(format!( + "select count(*)::bigint as request_count, + count(*) filter (where outcome = 'succeeded')::bigint as success_count, + count(*) filter (where outcome = 'failed')::bigint as failure_count, + count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count, + count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count, + count(*) filter (where client_status_code between 400 and 499)::bigint + as caller_error_count, + coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens, + coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens, + coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens, + coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint + as cache_write_tokens, + coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint + as reasoning_tokens, + coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens, + coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint + as first_token_latency_sum, + count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count, + coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum, + count(latency_ms) filter (where {fact})::bigint as latency_count, + max(latency_ms) filter (where {fact})::bigint as max_latency_ms, + min(latency_ms) filter (where {fact})::bigint as min_latency_ms, + count(*) filter (where {fact} and input_tokens is not null)::bigint + as cache_eligible_request_count, + count(*) filter (where {fact} and input_tokens is not null + and cached_tokens > 0)::bigint + as cache_hit_request_count, + percentile_cont(0.50) within group (order by latency_ms) + filter (where {fact}) as latency_p50_ms, + percentile_cont(0.95) within group (order by latency_ms) + filter (where {fact}) as latency_p95_ms, + percentile_cont(0.99) within group (order by latency_ms) + filter (where {fact}) as latency_p99_ms, + percentile_cont(0.50) within group (order by first_token_ms) + filter (where {fact}) as first_token_p50_ms, + percentile_cont(0.95) within group (order by first_token_ms) + filter (where {fact}) as first_token_p95_ms, + percentile_cont(0.99) within group (order by first_token_ms) + filter (where {fact}) as first_token_p99_ms + from model_requests mr where mr.started_at >= " + )); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + push_usage_filter(&mut query, filter, "mr"); + let row = query + .build() + .fetch_one(pool) + .await + .map_err(|_| postgres_unavailable("load request metrics"))?; + request_metrics_from_row(&row) +} + +pub(crate) async fn request_metric_series( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult> { + filter.validate()?; + let granularity = granularity_for(range); + // 与 request_metrics 同一契约：结果计数覆盖全部请求，用量/延迟/成本 + // 聚合仅统计用量事实。 + let fact = completed_usage_fact_predicate("mr"); + let mut query = QueryBuilder::::new("select date_bin("); + query.push_bind(granularity.sql_interval()); + query.push(format!( + "::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start, + count(*)::bigint as request_count, + count(*) filter (where outcome = 'succeeded')::bigint as success_count, + count(*) filter (where outcome = 'failed')::bigint as failure_count, + count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count, + count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count, + count(*) filter (where client_status_code between 400 and 499)::bigint + as caller_error_count, + coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens, + coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens, + coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens, + coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint + as cache_write_tokens, + coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint + as reasoning_tokens, + coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens, + coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint + as first_token_latency_sum, + count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count, + coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum, + count(latency_ms) filter (where {fact})::bigint as latency_count, + max(latency_ms) filter (where {fact})::bigint as max_latency_ms, + min(latency_ms) filter (where {fact})::bigint as min_latency_ms, + count(*) filter (where {fact} and input_tokens is not null)::bigint + as cache_eligible_request_count, + count(*) filter (where {fact} and input_tokens is not null + and cached_tokens > 0)::bigint + as cache_hit_request_count, + percentile_cont(0.50) within group (order by latency_ms) + filter (where {fact}) as latency_p50_ms, + percentile_cont(0.95) within group (order by latency_ms) + filter (where {fact}) as latency_p95_ms, + percentile_cont(0.99) within group (order by latency_ms) + filter (where {fact}) as latency_p99_ms, + percentile_cont(0.50) within group (order by first_token_ms) + filter (where {fact}) as first_token_p50_ms, + percentile_cont(0.95) within group (order by first_token_ms) + filter (where {fact}) as first_token_p95_ms, + percentile_cont(0.99) within group (order by first_token_ms) + filter (where {fact}) as first_token_p99_ms, + count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint + as provider_reported_count, + count(*) filter (where {fact} and cost_source = 'calculated')::bigint + as calculated_count, + count(*) filter (where {fact} and cost_source = 'unavailable')::bigint + as unavailable_count + from model_requests mr where mr.started_at >= " + )); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + push_usage_filter(&mut query, filter, "mr"); + query.push(" group by bucket_start order by bucket_start"); + let rows = query + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load request metric series"))?; + + let mut points = BTreeMap::new(); + for row in rows { + let bucket_start = get(&row, "bucket_start")?; + points.insert( + bucket_start, + RequestMetricPoint { + bucket_start, + granularity, + metrics: request_metrics_from_row(&row)?, + cost_coverage: coverage_from_row(&row)?, + costs: Vec::new(), + }, + ); + } + let bucket_costs = request_costs_by_bucket(pool, range, filter, granularity).await?; + for (bucket, costs) in bucket_costs { + if let Some(point) = points.get_mut(&bucket) { + point.costs = costs; + } + } + fill_metric_gaps(range, granularity, points) +} + +pub(crate) async fn calculated_usage_billing_facts( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult> { + filter.validate()?; + let granularity = granularity_for(range); + let mut query = QueryBuilder::::new("select date_bin("); + query.push_bind(granularity.sql_interval()); + query.push( + "::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start, + mr.provider_kind, mr.upstream_model_id, mr.service_tier, + mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens, + mr.cost_currency, mr.cost_amount::text as amount + from model_requests mr where mr.started_at >= ", + ); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + query.push( + " and mr.cost_source = 'calculated' + and mr.cost_amount is not null and mr.cost_currency is not null + and nullif(mr.provider_kind, '') is not null + and nullif(mr.upstream_model_id, '') is not null", + ); + push_completed_usage_fact_filter(&mut query, "mr"); + push_usage_filter(&mut query, filter, "mr"); + query.push(" order by bucket_start, mr.id"); + query + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load calculated usage billing facts"))? + .iter() + .map(calculated_usage_billing_fact_from_row) + .collect() +} + +pub(crate) async fn request_costs_by_bucket( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, + granularity: ObservationGranularity, +) -> StoreResult, Vec>> { + let mut query = QueryBuilder::::new("select date_bin("); + query.push_bind(granularity.sql_interval()); + query.push( + "::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start, + mr.cost_currency, sum(mr.cost_amount)::text as amount + from model_requests mr + where mr.started_at >= ", + ); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + query.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); + push_completed_usage_fact_filter(&mut query, "mr"); + push_usage_filter(&mut query, filter, "mr"); + query.push(" group by bucket_start, mr.cost_currency order by bucket_start, mr.cost_currency"); + let rows = query + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load request costs by bucket"))?; + let mut result = BTreeMap::, Vec>::new(); + for row in rows { + result + .entry(get(&row, "bucket_start")?) + .or_default() + .push(cost_from_row(&row)?); + } + Ok(result) +} + +pub(crate) async fn attempt_metrics( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult { + filter.validate()?; + let mut query = QueryBuilder::::new( + "with selected_requests as ( + select mr.* from model_requests mr where mr.started_at >= ", + ); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + push_usage_filter(&mut query, filter, "mr"); + query.push( + "), usage_facts as ( + select * from selected_requests + where outcome = 'succeeded' + and downstream_committed_at is not null + and client_status_code between 200 and 399 + ), failures as ( + select coalesce(oe.occurrence_count, 1)::bigint as occurrences, + oe.failure_kind, oe.status_code + from ops_events oe + join selected_requests sr on sr.id = oe.model_request_id + union all + select 1::bigint, coalesce(sr.error_kind, 'failed'), + coalesce(sr.upstream_status_code, sr.client_status_code) + from selected_requests sr + where sr.outcome = 'failed' and sr.attempt_count > 0 + ) + select coalesce((select sum(attempt_count) from selected_requests), 0)::bigint + as attempt_count, + coalesce((select count(*) from selected_requests + where outcome = 'succeeded' and attempt_count > 0), 0)::bigint + as success_count, + coalesce((select sum(occurrences) from failures), 0)::bigint as failure_count, + coalesce((select count(*) from selected_requests + where outcome = 'cancelled' and attempt_count > 0), 0)::bigint + as cancelled_count, + coalesce((select count(*) from selected_requests + where outcome = 'incomplete' and attempt_count > 0), 0)::bigint + as incomplete_count, + coalesce((select sum(occurrences) from failures + where failure_kind in ('rate_limited', 'quota_exhausted') + or status_code = 429), 0)::bigint as rate_limited_count, + coalesce((select sum(occurrences) from failures + where failure_kind in ('authentication', 'authorization', + 'invalid_credential') + or status_code in (401, 403)), 0)::bigint as auth_failure_count, + coalesce((select sum(occurrences) from failures + where status_code between 500 and 599), 0)::bigint + as provider_5xx_count, + coalesce((select count(*) from usage_facts + where cost_source = 'provider_reported'), 0)::bigint + as provider_reported_count, + coalesce((select count(*) from usage_facts + where cost_source = 'calculated'), 0)::bigint + as calculated_count, + coalesce((select count(*) from usage_facts + where cost_source = 'unavailable'), 0)::bigint + as unavailable_count", + ); + let row = query + .build() + .fetch_one(pool) + .await + .map_err(|_| postgres_unavailable("load attempt metrics"))?; + Ok(AttemptMetrics { + attempt_count: unsigned(&row, "attempt_count")?, + success_count: unsigned(&row, "success_count")?, + failure_count: unsigned(&row, "failure_count")?, + cancelled_count: unsigned(&row, "cancelled_count")?, + incomplete_count: unsigned(&row, "incomplete_count")?, + rate_limited_count: unsigned(&row, "rate_limited_count")?, + auth_failure_count: unsigned(&row, "auth_failure_count")?, + provider_5xx_count: unsigned(&row, "provider_5xx_count")?, + cost_coverage: coverage_from_row(&row)?, + costs: request_costs(pool, range, filter).await?, + }) +} + +pub(crate) async fn request_costs( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult> { + let mut query = QueryBuilder::::new( + "select mr.cost_currency, sum(mr.cost_amount)::text as amount + from model_requests mr where mr.started_at >= ", + ); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + query.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); + push_completed_usage_fact_filter(&mut query, "mr"); + push_usage_filter(&mut query, filter, "mr"); + query.push(" group by mr.cost_currency order by mr.cost_currency"); + query + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load request costs"))? + .iter() + .map(cost_from_row) + .collect() +} + +pub(crate) async fn provider_account_metrics( + pool: &PgPool, + observed_at: DateTime, +) -> StoreResult { + let row = sqlx::query( + "with normalized as ( + select enabled, + case + when not enabled then 'disabled' + when availability = 'banned' then 'banned' + when availability = 'quota_exhausted' + or ( + availability = 'cooldown' + and availability_reason = 'usage_limit_exhausted' + and (cooldown_until is null or cooldown_until > $1) + ) + then 'quota_exhausted' + when availability in ('expired', 'invalid') + or (access_token_expires_at is not null and access_token_expires_at <= $1) + then 'expired' + else 'active' + end as status + from provider_accounts + ) + select count(*)::bigint as total, + count(*) filter (where enabled)::bigint as enabled, + count(*) filter (where status <> 'active')::bigint as unavailable, + count(*) filter (where status = 'active')::bigint as active, + count(*) filter (where status = 'expired')::bigint as expired, + count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted, + count(*) filter (where status = 'disabled')::bigint as disabled, + count(*) filter (where status = 'banned')::bigint as banned + from normalized", + ) + .bind(observed_at) + .fetch_one(pool) + .await + .map_err(|_| postgres_unavailable("load provider account metrics"))?; + Ok(ProviderAccountMetrics { + total: unsigned(&row, "total")?, + enabled: unsigned(&row, "enabled")?, + unavailable: unsigned(&row, "unavailable")?, + active: unsigned(&row, "active")?, + expired: unsigned(&row, "expired")?, + quota_exhausted: unsigned(&row, "quota_exhausted")?, + disabled: unsigned(&row, "disabled")?, + banned: unsigned(&row, "banned")?, + }) +} + +pub(crate) async fn active_provider_account_ids( + pool: &PgPool, + observed_at: DateTime, +) -> StoreResult> { + sqlx::query( + "with normalized as ( + select id, + case + when not enabled then 'disabled' + when availability = 'banned' then 'banned' + when availability = 'quota_exhausted' + or ( + availability = 'cooldown' + and availability_reason = 'usage_limit_exhausted' + and (cooldown_until is null or cooldown_until > $1) + ) + then 'quota_exhausted' + when availability in ('expired', 'invalid') + or (access_token_expires_at is not null and access_token_expires_at <= $1) + then 'expired' + else 'active' + end as status + from provider_accounts + ) + select id + from normalized + where status = 'active' + order by id", + ) + .bind(observed_at) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load active provider account IDs"))? + .iter() + .map(|row| get(row, "id")) + .collect() +} + +pub(crate) async fn provider_observations( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult> { + let mut query = QueryBuilder::::new( + "select coalesce(mr.provider_kind, 'unrouted') as provider_kind, + count(*)::bigint as request_count, + coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count, + count(*) filter (where mr.outcome = 'failed')::bigint as failure_count, + coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens + from model_requests mr where mr.started_at >= ", + ); + query.push_bind(range.start); + query.push(" and mr.started_at < "); + query.push_bind(range.end); + push_completed_usage_fact_filter(&mut query, "mr"); + push_usage_filter(&mut query, filter, "mr"); + query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc"); + let rows = query + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load provider observations"))?; + rows.iter() + .map(|row| { + Ok(ProviderObservation { + provider_kind: get(row, "provider_kind")?, + request_count: unsigned(row, "request_count")?, + attempt_count: unsigned(row, "attempt_count")?, + failure_count: unsigned(row, "failure_count")?, + total_tokens: unsigned(row, "total_tokens")?, + }) + }) + .collect() +} + +pub(crate) fn request_metrics_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { + Ok(RequestMetrics { + request_count: unsigned(row, "request_count")?, + success_count: unsigned(row, "success_count")?, + failure_count: unsigned(row, "failure_count")?, + cancelled_count: unsigned(row, "cancelled_count")?, + incomplete_count: unsigned(row, "incomplete_count")?, + caller_error_count: unsigned(row, "caller_error_count")?, + input_tokens: unsigned(row, "input_tokens")?, + output_tokens: unsigned(row, "output_tokens")?, + cached_tokens: unsigned(row, "cached_tokens")?, + cache_write_tokens: unsigned(row, "cache_write_tokens")?, + reasoning_tokens: unsigned(row, "reasoning_tokens")?, + total_tokens: unsigned(row, "total_tokens")?, + first_token_latency_sum: unsigned(row, "first_token_latency_sum")?, + first_token_latency_count: unsigned(row, "first_token_latency_count")?, + latency_sum: unsigned(row, "latency_sum")?, + latency_count: unsigned(row, "latency_count")?, + max_latency_ms: optional_unsigned(row, "max_latency_ms")?, + min_latency_ms: optional_unsigned(row, "min_latency_ms")?, + cache_eligible_request_count: unsigned(row, "cache_eligible_request_count")?, + cache_hit_request_count: unsigned(row, "cache_hit_request_count")?, + latency_percentiles: LatencyPercentiles { + p50_ms: optional_percentile(row, "latency_p50_ms")?, + p95_ms: optional_percentile(row, "latency_p95_ms")?, + p99_ms: optional_percentile(row, "latency_p99_ms")?, + }, + first_token_latency_percentiles: LatencyPercentiles { + p50_ms: optional_percentile(row, "first_token_p50_ms")?, + p95_ms: optional_percentile(row, "first_token_p95_ms")?, + p99_ms: optional_percentile(row, "first_token_p99_ms")?, + }, + }) +} + +pub(crate) fn optional_percentile( + row: &sqlx::postgres::PgRow, + column: &str, +) -> StoreResult> { + row.try_get::, _>(column) + .map_err(|_| postgres_unavailable("decode latency percentile"))? + .map(PercentileMilliseconds::new) + .transpose() +} + +pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { + Ok(CostCoverage { + provider_reported_count: unsigned(row, "provider_reported_count")?, + calculated_count: unsigned(row, "calculated_count")?, + unavailable_count: unsigned(row, "unavailable_count")?, + }) +} + +pub(crate) fn granularity_for(range: ObservabilityRange) -> ObservationGranularity { + let seconds = range.end.signed_duration_since(range.start).num_seconds(); + if seconds <= 2 * 24 * 60 * 60 { + ObservationGranularity::FifteenMinutes + } else if seconds <= 31 * 24 * 60 * 60 { + ObservationGranularity::Hour + } else { + ObservationGranularity::Day + } +} + +pub(crate) fn fill_metric_gaps( + range: ObservabilityRange, + granularity: ObservationGranularity, + mut points: BTreeMap, RequestMetricPoint>, +) -> StoreResult> { + let seconds = granularity.seconds(); + let start_epoch = range.start.timestamp().div_euclid(seconds) * seconds; + let mut bucket = DateTime::from_timestamp(start_epoch, 0) + .ok_or_else(|| invalid("metric range start is outside supported timestamps"))?; + let step = TimeDelta::seconds(seconds); + let mut result = Vec::new(); + while bucket < range.end { + result.push(points.remove(&bucket).unwrap_or(RequestMetricPoint { + bucket_start: bucket, + granularity, + metrics: RequestMetrics::default(), + cost_coverage: CostCoverage::default(), + costs: Vec::new(), + })); + bucket = bucket + .checked_add_signed(step) + .ok_or_else(|| invalid("metric range exceeds supported timestamps"))?; + } + Ok(result) +} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/mod.rs b/backend/crates/gateway-store/src/postgres/observability/queries/mod.rs new file mode 100644 index 000000000..b78061658 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/queries/mod.rs @@ -0,0 +1,11 @@ +//! 观测查询族：dashboard / usage / ops / accounts。 + +mod accounts; +mod dashboard; +mod ops; +mod usage; + +pub(crate) use accounts::*; +pub(crate) use dashboard::*; +pub(crate) use ops::*; +pub(crate) use usage::*; diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs b/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs new file mode 100644 index 000000000..32405df28 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs @@ -0,0 +1,171 @@ +//! Ops 错误查询族。 + +use super::super::*; + +pub(crate) const OPS_ERRORS_CTE: &str = "with errors as ( + select 'model_request'::text as source, + mr.id as event_id, mr.id as request_id, + nullif(mr.attempt_count, 0) as attempt_index, + mr.client_api_key_ref, 'model_request'::text as component, mr.operation, + mr.endpoint, + mr.provider_kind, + mr.provider_account_ref, + mr.provider_account_name_snapshot as provider_account_name, + mr.provider_account_email_snapshot as provider_account_email, + mr.provider_account_authentication_kind_snapshot + as provider_account_authentication_kind, + mr.upstream_model_id, mr.upstream_transport, + coalesce(mr.error_kind, 'failed') as failure_kind, + mr.client_status_code, mr.upstream_status_code, + mr.provider_error_code, mr.client_response_id, mr.upstream_request_id, + mr.latency_ms, coalesce(mr.error_message, mr.error_kind, 'request failed') as message, + 1::integer as occurrence_count, + coalesce(mr.completed_at, mr.started_at) as occurred_at, + 'model_request:' || mr.id as stable_sort_id + from model_requests mr where mr.outcome = 'failed' + union all + select 'ops_event'::text, oe.id, oe.model_request_id, oe.attempt_index, + mr.client_api_key_ref, oe.component, oe.operation, + null::text as endpoint, + oe.provider_kind, oe.provider_account_ref, + oe.provider_account_name_snapshot as provider_account_name, + oe.provider_account_email_snapshot as provider_account_email, + oe.provider_account_authentication_kind_snapshot + as provider_account_authentication_kind, + oe.upstream_model_id, null::text, oe.failure_kind, + null::integer as client_status_code, oe.status_code as upstream_status_code, + oe.provider_error_code, mr.client_response_id, oe.upstream_request_id, + oe.latency_ms, oe.message, oe.occurrence_count, oe.created_at, + 'ops_event:' || oe.id + from ops_events oe left join model_requests mr on mr.id = oe.model_request_id + )"; + +pub(crate) async fn list_ops_errors( + pool: &PgPool, + query: OpsErrorQuery, +) -> StoreResult { + query.filter.validate()?; + let total = count_ops_errors(pool, query.range, &query.filter).await?; + let mut statement = QueryBuilder::::new(OPS_ERRORS_CTE); + statement.push( + " select source, event_id, request_id, attempt_index, client_api_key_ref, + component, operation, endpoint, provider_kind, provider_account_ref, + provider_account_name, provider_account_email, + provider_account_authentication_kind, + upstream_model_id, upstream_transport, + failure_kind, client_status_code, upstream_status_code, + provider_error_code, client_response_id, + upstream_request_id, latency_ms, message, occurrence_count, occurred_at, + stable_sort_id + from errors e where e.occurred_at >= ", + ); + statement.push_bind(query.range.start); + statement.push(" and e.occurred_at < "); + statement.push_bind(query.range.end); + push_ops_filter(&mut statement, &query.filter); + if let Some(cursor) = &query.cursor { + statement.push(" and (e.occurred_at, e.stable_sort_id) < ("); + statement.push_bind(cursor.observed_at); + statement.push(", "); + statement.push_bind(cursor.stable_id.clone()); + statement.push(")"); + } + statement.push(" order by e.occurred_at desc, e.stable_sort_id desc limit "); + statement.push_bind(i64::from(query.page_size.get()) + 1); + if query.cursor.is_none() { + let offset = u64::from(query.page.get() - 1) + .checked_mul(u64::from(query.page_size.get())) + .and_then(|value| i64::try_from(value).ok()) + .ok_or_else(|| invalid("ops error page offset is too large"))?; + statement.push(" offset "); + statement.push_bind(offset); + } + let rows = statement + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("list ops errors"))?; + let mut items = rows + .iter() + .map(ops_error_from_row) + .collect::>>()?; + let has_more = items.len() > usize::from(query.page_size.get()); + if has_more { + items.pop(); + } + let next_cursor = if has_more { + items + .last() + .map(|item| ObservabilityCursor::new(item.occurred_at, item.stable_sort_id.clone())) + .transpose()? + } else { + None + }; + Ok(OpsErrorPage { + items, + total, + next_cursor, + }) +} + +pub(crate) async fn count_ops_errors( + pool: &PgPool, + range: ObservabilityRange, + filter: &OpsErrorFilter, +) -> StoreResult { + let mut statement = QueryBuilder::::new(OPS_ERRORS_CTE); + statement.push(" select count(*)::bigint from errors e where e.occurred_at >= "); + statement.push_bind(range.start); + statement.push(" and e.occurred_at < "); + statement.push_bind(range.end); + push_ops_filter(&mut statement, filter); + let total = statement + .build_query_scalar::() + .fetch_one(pool) + .await + .map_err(|_| postgres_unavailable("count ops errors"))?; + to_u64(total) +} + +pub(crate) fn push_ops_filter(statement: &mut QueryBuilder, filter: &OpsErrorFilter) { + for (column, value) in [ + ("client_api_key_ref", &filter.client_api_key_ref), + ("request_id", &filter.request_id), + ("provider_account_ref", &filter.provider_account_ref), + ("provider_kind", &filter.provider_kind), + ("operation", &filter.operation), + ("upstream_transport", &filter.transport), + ("upstream_request_id", &filter.upstream_request_id), + ("failure_kind", &filter.failure_kind), + ] { + if let Some(value) = value { + statement.push(format!(" and e.{column} = ")); + statement.push_bind(value.clone()); + } + } + if let Some(value) = &filter.response_id { + statement.push(" and e.client_response_id = "); + statement.push_bind(value.as_bytes().to_vec()); + } + if let Some(model) = &filter.model { + statement.push(" and e.upstream_model_id = "); + statement.push_bind(model.clone()); + } + if let Some(index) = filter.attempt_index { + statement.push(" and e.attempt_index = "); + statement.push_bind(i32::try_from(index).unwrap_or(i32::MAX)); + } + if let Some(status) = filter.status_code { + statement.push(" and e.upstream_status_code = "); + statement.push_bind(i32::from(status)); + } + if let Some(search) = &filter.search { + statement.push( + " and lower(concat_ws(' ', e.event_id, e.request_id, e.client_api_key_ref, + e.component, e.operation, e.provider_kind, e.provider_account_ref, + e.upstream_model_id, e.failure_kind, e.provider_error_code, + e.upstream_request_id, e.message)) like ", + ); + statement.push_bind(format!("%{}%", search.to_lowercase())); + } +} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs new file mode 100644 index 000000000..7157bcf73 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -0,0 +1,549 @@ +//! Usage 明细、诊断与过滤查询族。 + +use super::super::*; + +pub(crate) fn push_usage_filter( + query: &mut QueryBuilder, + filter: &UsageRecordFilter, + alias: &str, +) { + if let Some(value) = &filter.client_api_key_ref { + query.push(format!(" and {alias}.client_api_key_ref = ")); + query.push_bind(value.clone()); + } + if let Some(value) = &filter.request_id { + query.push(format!(" and {alias}.id = ")); + query.push_bind(value.clone()); + } + if let Some(value) = &filter.provider_account_ref { + query.push(format!(" and {alias}.provider_account_ref = ")); + query.push_bind(value.clone()); + } + if let Some(value) = &filter.operation { + query.push(format!(" and {alias}.operation = ")); + query.push_bind(value.clone()); + } + if let Some(value) = &filter.provider_kind { + query.push(format!(" and {alias}.provider_kind = ")); + query.push_bind(value.clone()); + } + if let Some(value) = &filter.model { + query.push(format!(" and ({alias}.requested_model_id = ")); + query.push_bind(value.clone()); + query.push(format!(" or {alias}.upstream_model_id = ")); + query.push_bind(value.clone()); + query.push(")"); + } + if let Some(value) = &filter.outcome { + query.push(format!(" and {alias}.outcome = ")); + query.push_bind(value.clone()); + } + if let Some(value) = filter.status_code { + query.push(format!(" and ({alias}.client_status_code = ")); + query.push_bind(i32::from(value)); + query.push(format!(" or {alias}.upstream_status_code = ")); + query.push_bind(i32::from(value)); + query.push(")"); + } + if let Some(value) = &filter.transport { + query.push(format!(" and ({alias}.client_transport = ")); + query.push_bind(value.clone()); + query.push(format!(" or {alias}.upstream_transport = ")); + query.push_bind(value.clone()); + query.push(")"); + } + if let Some(value) = filter.attempt_index { + query.push(format!(" and ({alias}.attempt_count = ")); + query.push_bind(i32::try_from(value).unwrap_or(i32::MAX)); + query.push(format!( + " or exists (select 1 from ops_events attempt_event + where attempt_event.model_request_id = {alias}.id + and attempt_event.attempt_index = " + )); + query.push_bind(i32::try_from(value).unwrap_or(i32::MAX)); + query.push("))"); + } + if let Some(value) = &filter.response_id { + query.push(format!(" and ({alias}.client_response_id = ")); + query.push_bind(value.as_bytes().to_vec()); + query.push(format!(" or {alias}.upstream_response_id = ")); + query.push_bind(value.as_bytes().to_vec()); + query.push(")"); + } + if let Some(value) = &filter.upstream_request_id { + query.push(format!(" and {alias}.upstream_request_id = ")); + query.push_bind(value.clone()); + } + if let Some(value) = &filter.search { + query.push(format!( + " and lower(concat_ws(' ', {alias}.id, {alias}.client_api_key_ref, + {alias}.requested_model_id, {alias}.provider_kind, + {alias}.provider_account_ref, {alias}.upstream_model_id, + encode({alias}.client_response_id, 'escape'), {alias}.upstream_request_id, + {alias}.error_kind, {alias}.provider_error_code, + {alias}.error_message)) like " + )); + query.push_bind(format!("%{}%", value.to_lowercase())); + } +} + +/// 只把已完整交付给客户端的成功响应投影为使用事实。 +/// +/// `model_requests` 同时承担执行审计：包括上游已发送、但尚未收到首个事件就断开的 +/// WebSocket 请求。这些失败仍必须留给 Ops Errors 和调度健康度分析，不能混入用量、 +/// 成本、账号使用次数或请求明细。 +pub(crate) fn completed_usage_fact_predicate(alias: &str) -> String { + format!( + "{alias}.outcome = 'succeeded' and {alias}.downstream_committed_at is not null and {alias}.client_status_code between 200 and 399" + ) +} + +pub(crate) fn push_completed_usage_fact_filter(query: &mut QueryBuilder, alias: &str) { + query.push(format!(" and {}", completed_usage_fact_predicate(alias))); +} + +pub(crate) const USAGE_RECORD_SELECT: &str = + "select mr.id, mr.client_api_key_ref, mr.config_revision, mr.protocol, mr.operation, + mr.endpoint, mr.client_transport, mr.requested_model_id, + mr.provider_kind, mr.provider_account_ref, + mr.provider_account_name_snapshot as provider_account_name, + mr.provider_account_email_snapshot as provider_account_email, + mr.provider_account_authentication_kind_snapshot + as provider_account_authentication_kind, + mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool, + mr.service_tier, mr.provider_observation_json, + mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at, + mr.outcome, mr.client_status_code, mr.upstream_status_code, + mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id, + mr.error_kind, mr.provider_error_code, mr.error_message, mr.retry_after_ms, + mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens, + mr.reasoning_tokens, mr.image_input_tokens, mr.image_output_tokens, + mr.total_tokens, mr.cost_source, mr.cost_amount::text, + mr.cost_currency, mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms, + mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms, + mr.provider_processing_ms, mr.latency_ms, mr.client_ip::text as client_ip, + mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind, + mr.subagent_kind, mr.compact, mr.image_generation_requested, + mr.image_generation_succeeded, mr.started_at, mr.deadline_at, mr.completed_at + from model_requests mr"; + +pub(crate) async fn list_usage_records( + pool: &PgPool, + query: UsageRecordQuery, +) -> StoreResult { + query.filter.validate()?; + let total = count_usage_records(pool, query.range, &query.filter).await?; + let mut statement = QueryBuilder::::new(USAGE_RECORD_SELECT); + statement.push(" where mr.started_at >= "); + statement.push_bind(query.range.start); + statement.push(" and mr.started_at < "); + statement.push_bind(query.range.end); + push_completed_usage_fact_filter(&mut statement, "mr"); + push_usage_filter(&mut statement, &query.filter, "mr"); + if let Some(cursor) = &query.cursor { + statement.push(" and (mr.started_at, mr.id) < ("); + statement.push_bind(cursor.observed_at); + statement.push(", "); + statement.push_bind(cursor.stable_id.clone()); + statement.push(")"); + } + statement.push(" order by mr.started_at desc, mr.id desc limit "); + statement.push_bind(i64::from(query.page_size.get()) + 1); + if query.cursor.is_none() { + let offset = u64::from(query.page.get() - 1) + .checked_mul(u64::from(query.page_size.get())) + .and_then(|value| i64::try_from(value).ok()) + .ok_or_else(|| invalid("usage page offset is too large"))?; + statement.push(" offset "); + statement.push_bind(offset); + } + let rows = statement + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("list usage records"))?; + let mut items = rows + .iter() + .map(usage_record_from_row) + .collect::>>()?; + let has_more = items.len() > usize::from(query.page_size.get()); + if has_more { + items.pop(); + } + let next_cursor = if has_more { + items + .last() + .map(|item| ObservabilityCursor::new(item.started_at, item.id.clone())) + .transpose()? + } else { + None + }; + Ok(UsageRecordPage { + items, + total, + next_cursor, + }) +} + +pub(crate) async fn count_usage_records( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, +) -> StoreResult { + let mut statement = QueryBuilder::::new( + "select count(*)::bigint from model_requests mr where mr.started_at >= ", + ); + statement.push_bind(range.start); + statement.push(" and mr.started_at < "); + statement.push_bind(range.end); + push_completed_usage_fact_filter(&mut statement, "mr"); + push_usage_filter(&mut statement, filter, "mr"); + let total = statement + .build_query_scalar::() + .fetch_one(pool) + .await + .map_err(|_| postgres_unavailable("count usage records"))?; + to_u64(total) +} + +pub(crate) async fn usage_record_detail( + pool: &PgPool, + request_id: &str, +) -> StoreResult { + require_nonempty("model request", "id", request_id)?; + validate_text(request_id, MAX_FILTER_BYTES, "request ID")?; + let mut statement = QueryBuilder::::new(USAGE_RECORD_SELECT); + statement.push(" where mr.id = "); + statement.push_bind(request_id.to_owned()); + push_completed_usage_fact_filter(&mut statement, "mr"); + let row = statement + .build() + .fetch_optional(pool) + .await + .map_err(|_| postgres_unavailable("load usage record detail"))? + .ok_or_else(|| StoreError::NotFound { + entity: "model request", + id: request_id.to_owned(), + })?; + let request = usage_record_from_row(&row)?; + let rows = sqlx::query( + "select id, attempt_index, component, operation, + provider_kind, provider_account_ref, + provider_account_name_snapshot as provider_account_name, + provider_account_email_snapshot as provider_account_email, + provider_account_authentication_kind_snapshot + as provider_account_authentication_kind, + upstream_model_id, + failure_kind, status_code, provider_error_code, retry_after_ms, + upstream_request_id, latency_ms, message, created_at + from ops_events where model_request_id = $1 + order by attempt_index, created_at, id", + ) + .bind(request_id) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load usage attempt observations"))?; + let mut attempts = rows + .iter() + .map(intermediate_attempt_from_row) + .collect::>>()?; + if request.attempt_count > 0 { + attempts.push(final_attempt_from_request(&request)); + } + Ok(UsageRecordDetail { request, attempts }) +} + +pub(crate) fn intermediate_attempt_from_row( + row: &sqlx::postgres::PgRow, +) -> StoreResult { + Ok(UsageAttemptObservation { + source: "ops_event".to_owned(), + id: get(row, "id")?, + attempt_index: to_u32(get(row, "attempt_index")?)?, + component: get(row, "component")?, + operation: get(row, "operation")?, + provider_kind: get(row, "provider_kind")?, + provider_account_ref: get(row, "provider_account_ref")?, + provider_account_name: get(row, "provider_account_name")?, + provider_account_email: get(row, "provider_account_email")?, + provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?, + upstream_model_id: get(row, "upstream_model_id")?, + upstream_transport: None, + upstream_send_state: None, + outcome: "failed".to_owned(), + downstream_committed: false, + status_code: optional_status(row, "status_code")?, + provider_error_code: get(row, "provider_error_code")?, + failure_kind: get(row, "failure_kind")?, + retry_after_ms: optional_unsigned(row, "retry_after_ms")?, + upstream_request_id: get(row, "upstream_request_id")?, + latency_ms: optional_unsigned(row, "latency_ms")?, + message: get(row, "message")?, + input_tokens: None, + output_tokens: None, + cached_tokens: None, + cache_write_tokens: None, + reasoning_tokens: None, + total_tokens: None, + cost_source: None, + cost_amount: None, + cost_currency: None, + occurred_at: get(row, "created_at")?, + }) +} + +pub(crate) fn final_attempt_from_request(request: &UsageRecord) -> UsageAttemptObservation { + UsageAttemptObservation { + source: "model_request".to_owned(), + id: format!("{}:final", request.id), + attempt_index: request.attempt_count, + component: "model_request".to_owned(), + operation: request.operation.clone(), + provider_kind: request.provider_kind.clone(), + provider_account_ref: request.provider_account_ref.clone(), + provider_account_name: request.provider_account_name.clone(), + provider_account_email: request.provider_account_email.clone(), + provider_account_authentication_kind: request.provider_account_authentication_kind.clone(), + upstream_model_id: request.upstream_model_id.clone(), + upstream_transport: request.upstream_transport.clone(), + upstream_send_state: Some(request.upstream_send_state.clone()), + outcome: request.outcome.clone(), + downstream_committed: request.downstream_committed_at.is_some(), + status_code: request.upstream_status_code.or(request.client_status_code), + provider_error_code: request.provider_error_code.clone(), + failure_kind: request.error_kind.clone(), + retry_after_ms: request.retry_after_ms, + upstream_request_id: request.upstream_request_id.clone(), + latency_ms: request.latency_ms, + message: request.error_message.clone(), + input_tokens: request.input_tokens, + output_tokens: request.output_tokens, + cached_tokens: request.cached_tokens, + cache_write_tokens: request.cache_write_tokens, + reasoning_tokens: request.reasoning_tokens, + total_tokens: request.total_tokens, + cost_source: Some(request.cost_source.clone()), + cost_amount: request.cost_amount.clone(), + cost_currency: request.cost_currency.clone(), + occurred_at: request.completed_at.unwrap_or(request.started_at), + } +} + +pub(crate) async fn usage_diagnostics( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, + dimension: DiagnosticDimension, +) -> StoreResult> { + filter.validate()?; + let dimension_sql = diagnostic_dimension_sql(dimension); + let mut statement = QueryBuilder::::new("select "); + statement.push(dimension_sql); + statement.push( + " as dimension_name, + count(*)::bigint as request_count, + count(*) filter (where mr.outcome = 'succeeded')::bigint as success_count, + count(*) filter (where mr.outcome = 'failed')::bigint as failure_count, + coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count, + coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens, + round(avg(mr.latency_ms))::bigint as average_latency_ms, + round(percentile_cont(0.95) within group (order by mr.latency_ms))::bigint + as latency_p95_ms, + count(*) filter (where mr.cost_source = 'provider_reported')::bigint + as provider_reported_count, + count(*) filter (where mr.cost_source = 'calculated')::bigint + as calculated_count, + count(*) filter (where mr.cost_source = 'unavailable')::bigint + as unavailable_count + from model_requests mr where mr.started_at >= ", + ); + statement.push_bind(range.start); + statement.push(" and mr.started_at < "); + statement.push_bind(range.end); + push_diagnostic_fact_filter(&mut statement, dimension); + push_usage_filter(&mut statement, filter, "mr"); + statement.push(" group by dimension_name order by request_count desc, dimension_name limit "); + statement.push_bind(DIAGNOSTIC_LIMIT); + let rows = statement + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load usage diagnostics"))?; + let mut observations = rows + .iter() + .map(|row| { + Ok(DiagnosticObservation { + key: get(row, "dimension_name")?, + name: get(row, "dimension_name")?, + request_count: unsigned(row, "request_count")?, + success_count: unsigned(row, "success_count")?, + failure_count: unsigned(row, "failure_count")?, + attempt_count: unsigned(row, "attempt_count")?, + total_tokens: unsigned(row, "total_tokens")?, + average_latency_ms: optional_unsigned(row, "average_latency_ms")?, + latency_p95_ms: optional_unsigned(row, "latency_p95_ms")?, + cost_coverage: coverage_from_row(row)?, + costs: Vec::new(), + }) + }) + .collect::>>()?; + let mut display_names = match dimension { + DiagnosticDimension::Account => { + diagnostic_account_display_names( + pool, + &observations + .iter() + .map(|item| item.key.clone()) + .collect::>(), + ) + .await? + } + DiagnosticDimension::ApiKey => { + diagnostic_api_key_display_names( + pool, + &observations + .iter() + .map(|item| item.key.clone()) + .collect::>(), + ) + .await? + } + _ => HashMap::new(), + }; + for observation in &mut observations { + if let Some(name) = display_names.remove(&observation.key) { + observation.name = name; + } + } + let positions = observations + .iter() + .enumerate() + .map(|(index, item)| (item.key.clone(), index)) + .collect::>(); + let costs = diagnostic_costs(pool, range, filter, dimension_sql, dimension).await?; + for (key, values) in costs { + if let Some(position) = positions.get(&key).copied() { + observations[position].costs = values; + } + } + Ok(observations) +} + +pub(crate) async fn diagnostic_account_display_names( + pool: &PgPool, + account_ids: &[String], +) -> StoreResult> { + if account_ids.is_empty() { + return Ok(HashMap::new()); + } + let rows = sqlx::query( + "select provider_account_ref, + coalesce( + (array_agg(provider_account_email_snapshot order by started_at desc) + filter (where provider_account_email_snapshot is not null))[1], + (array_agg(provider_account_name_snapshot order by started_at desc) + filter (where provider_account_name_snapshot is not null))[1], + provider_account_ref + ) as display_name + from model_requests + where provider_account_ref = any($1::text[]) + group by provider_account_ref", + ) + .bind(account_ids) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load diagnostic account display names"))?; + let mut display_names = HashMap::with_capacity(rows.len()); + for row in rows { + display_names.insert( + get(&row, "provider_account_ref")?, + get(&row, "display_name")?, + ); + } + Ok(display_names) +} + +pub(crate) async fn diagnostic_api_key_display_names( + pool: &PgPool, + key_ids: &[String], +) -> StoreResult> { + if key_ids.is_empty() { + return Ok(HashMap::new()); + } + let rows = sqlx::query("select id, name from client_api_keys where id = any($1::text[])") + .bind(key_ids) + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load diagnostic api key display names"))?; + let mut display_names = HashMap::with_capacity(rows.len()); + for row in rows { + display_names.insert(get(&row, "id")?, get(&row, "name")?); + } + Ok(display_names) +} + +pub(crate) async fn diagnostic_costs( + pool: &PgPool, + range: ObservabilityRange, + filter: &UsageRecordFilter, + dimension_sql: &'static str, + dimension: DiagnosticDimension, +) -> StoreResult>> { + let mut statement = QueryBuilder::::new("select "); + statement.push(dimension_sql); + statement.push( + " as dimension_name, mr.cost_currency, sum(mr.cost_amount)::text as amount + from model_requests mr where mr.started_at >= ", + ); + statement.push_bind(range.start); + statement.push(" and mr.started_at < "); + statement.push_bind(range.end); + statement.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); + push_diagnostic_fact_filter(&mut statement, dimension); + push_usage_filter(&mut statement, filter, "mr"); + statement.push( + " group by dimension_name, mr.cost_currency order by dimension_name, mr.cost_currency", + ); + let rows = statement + .build() + .fetch_all(pool) + .await + .map_err(|_| postgres_unavailable("load diagnostic costs"))?; + let mut result = HashMap::>::new(); + for row in rows { + result + .entry(get(&row, "dimension_name")?) + .or_default() + .push(cost_from_row(&row)?); + } + Ok(result) +} + +pub(crate) fn diagnostic_dimension_sql(dimension: DiagnosticDimension) -> &'static str { + match dimension { + DiagnosticDimension::Provider => "coalesce(mr.provider_kind, 'unrouted')", + DiagnosticDimension::Model => { + "coalesce(mr.upstream_model_id, mr.requested_model_id, 'unknown')" + } + DiagnosticDimension::Account => "coalesce(mr.provider_account_ref, 'unrouted')", + DiagnosticDimension::ApiKey => "mr.client_api_key_ref", + DiagnosticDimension::Transport => { + "coalesce(mr.upstream_transport, mr.client_transport, 'unknown')" + } + DiagnosticDimension::Failure => "coalesce(mr.error_kind, 'none')", + DiagnosticDimension::Status => { + "coalesce(mr.upstream_status_code, mr.client_status_code)::text" + } + } +} + +pub(crate) fn push_diagnostic_fact_filter( + statement: &mut QueryBuilder, + dimension: DiagnosticDimension, +) { + if dimension == DiagnosticDimension::Failure { + statement.push(" and mr.error_kind is not null"); + } else { + push_completed_usage_fact_filter(statement, "mr"); + } +} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts.rs b/backend/crates/gateway-store/src/postgres/provider_accounts.rs deleted file mode 100644 index a275572c2..000000000 --- a/backend/crates/gateway-store/src/postgres/provider_accounts.rs +++ /dev/null @@ -1,2619 +0,0 @@ -//! 明文 `provider_accounts` 与凭证 revision CAS 的唯一 PostgreSQL owner。 - -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt, - str::FromStr, -}; - -use async_trait::async_trait; -use chrono::{DateTime, TimeDelta, Utc}; -use gateway_admin::{ - model::{ - MutationContext, Revision as AdminRevision, - accounts::{ - AccountCost, AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage, - AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort, - AccountSortField as AdminAccountSortField, AccountStatus as AdminAccountStatus, - AccountSummary, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, - DeleteAccounts, SetAccountEnabled, SortDirection as AdminSortDirection, - }, - observability::{ - CostCoverage as AdminCostCoverage, DecimalAmount as AdminDecimalAmount, TimeRange, - }, - provider_credentials::{ - AuthorizationCommit, AuthorizationCredentialCommit, CredentialCursor, - CredentialDetails, CredentialImportCommit, CredentialImportResult, CredentialListQuery, - CredentialListWindow, CredentialMutationResult, CredentialPage, - CredentialRotationCommit, PreparedCredentialCreate, PreparedCredentialImport, - PreparedCredentialRotationFacts, ProviderDocument, ProviderExportCredentialInput, - }, - }, - ports::store::{AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStoreResult}, -}; -use sqlx::{PgPool, Postgres, Row, Transaction}; - -use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision as CoreCredentialRevision, LoadedCredential, - NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential, - ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId, - ProviderAccountIdentity, ProviderAccountStore, - ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome, -}; -use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind}; -use gateway_core::routing::ProviderKind; - -use crate::{ - ConflictKind, JsonObject, Revision, StoreError, StoreResult, admin_revision, admin_store_error, - mutation_audit, postgres_unavailable, require_nonempty, store_revision, -}; - -use super::{ - AdminAuditEvent, ControlPlaneRepository, CurrencyCostTotal, ObservabilityRange, - ObservabilityRepository, PgControlPlaneRepository, PgObservabilityRepository, - ProviderAccountModelUsageObservation, ProviderAccountUsageObservation, - ProviderAccountUsageQuery, append_admin_audit_event_in_transaction, - bump_config_revision_in_transaction, -}; - -const ENTITY: &str = "provider account"; -const CREDENTIALS_MAX_BYTES: usize = 256 * 1024; -const QUOTA_MAX_BYTES: usize = 128 * 1024; -const MAX_ADMIN_IMPORT_BATCH: usize = 200; -const ADMIN_USAGE_CHUNK_SIZE: usize = 200; - -const ACCOUNT_USAGE_BY_WINDOWS_SQL: &str = "with requested_windows as ( - select * - from unnest($1::text[], $2::text[], $3::timestamptz[], $4::timestamptz[]) - as requested(account_id, window_key, window_start, window_end) - ) - select requested.account_id, - requested.window_key, - count(mr.id)::bigint as request_count, - count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count, - sum(mr.input_tokens)::bigint as input_tokens, - sum(mr.output_tokens)::bigint as output_tokens, - sum(mr.cached_tokens)::bigint as cached_tokens, - sum(mr.cache_write_tokens)::bigint as cache_write_tokens, - sum(mr.reasoning_tokens)::bigint as reasoning_tokens, - sum(mr.image_input_tokens)::bigint as image_input_tokens, - sum(mr.image_output_tokens)::bigint as image_output_tokens, - count(mr.id) filter (where mr.image_generation_succeeded is true)::bigint - as image_request_count, - count(mr.id) filter (where mr.image_generation_succeeded is false)::bigint - as image_request_failed_count, - coalesce(sum(coalesce( - mr.total_tokens, - coalesce(mr.input_tokens, 0) + coalesce(mr.output_tokens, 0) - )), 0)::bigint as total_tokens, - count(mr.id) filter (where mr.cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(mr.id) filter (where mr.cost_source = 'calculated')::bigint - as calculated_count, - count(mr.id) filter (where mr.cost_source = 'unavailable')::bigint - as unavailable_count, - max(mr.started_at) as last_used_at - from requested_windows requested - left join model_requests mr - on mr.provider_account_ref = requested.account_id - and mr.started_at >= requested.window_start - and mr.started_at < requested.window_end - and mr.outcome = 'succeeded' - and mr.downstream_committed_at is not null - and mr.client_status_code between 200 and 399 - group by requested.account_id, requested.window_key - order by requested.account_id, requested.window_key"; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountAdminScope { - pub provider_kind: String, -} - -impl ProviderAccountAdminScope { - pub fn validate(&self) -> StoreResult<()> { - require_nonempty(ENTITY, "provider_kind", &self.provider_kind) - } - - fn contains(&self, account: &NewProviderAccount) -> bool { - account.provider_kind == self.provider_kind - } -} - -/// 解析数据库 availability 列；字符串映射本体唯一归属 gateway-core。 -fn parse_availability(value: &str) -> StoreResult { - AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value")) -} - -fn validate_availability_cooldown( - availability: AccountAvailability, - has_cooldown_until: bool, -) -> StoreResult<()> { - if availability == AccountAvailability::Cooldown && !has_cooldown_until { - return Err(invalid("cooldown availability requires cooldown_until")); - } - if has_cooldown_until - && !matches!( - availability, - AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted - ) - { - return Err(invalid( - "cooldown_until is only allowed for cooldown or quota_exhausted availability", - )); - } - Ok(()) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountSummary { - pub id: String, - pub provider_kind: String, - pub name: String, - pub email: Option, - pub upstream_user_id: String, - pub upstream_account_id: Option, - pub plan_type: Option, - pub authentication_kind: String, - pub credential_revision: Revision, - pub has_refresh_token: bool, - pub access_token_expires_at: Option>, - pub next_refresh_at: Option>, - pub enabled: bool, - pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, - pub availability_observed_at: DateTime, - pub quota_observed_at: Option>, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Clone, PartialEq)] -pub struct ProviderAccountRecord { - pub summary: ProviderAccountSummary, - pub provider_credentials_json: JsonObject, - pub provider_quota_json: Option, -} - -impl fmt::Debug for ProviderAccountRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ProviderAccountRecord") - .field("summary", &self.summary) - .field("provider_credentials_json", &"[REDACTED]") - .field( - "provider_quota_json", - &self.provider_quota_json.as_ref().map(|_| "[PROVIDER JSON]"), - ) - .finish() - } -} - -#[derive(Clone)] -pub struct NewProviderAccount { - pub id: String, - pub provider_kind: String, - pub name: String, - pub email: Option, - pub upstream_user_id: String, - pub upstream_account_id: Option, - pub plan_type: Option, - pub authentication_kind: String, - pub provider_credentials_json: JsonObject, - pub has_refresh_token: bool, - pub access_token_expires_at: Option>, - pub next_refresh_at: Option>, - pub enabled: bool, - pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, - pub availability_observed_at: DateTime, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UpdateProviderAccount { - pub id: String, - pub name: String, - pub email: Option, - pub plan_type: Option, -} - -impl fmt::Debug for NewProviderAccount { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("NewProviderAccount") - .field("id", &self.id) - .field("provider_kind", &self.provider_kind) - .field("name", &self.name) - .field("enabled", &self.enabled) - .field("availability", &self.availability) - .field("cooldown_until", &self.cooldown_until) - .field("provider_credentials_json", &"[REDACTED]") - .finish_non_exhaustive() - } -} - -impl NewProviderAccount { - pub fn validate(&self) -> StoreResult<()> { - require_nonempty(ENTITY, "id", &self.id)?; - require_nonempty(ENTITY, "provider_kind", &self.provider_kind)?; - require_nonempty(ENTITY, "name", &self.name)?; - require_nonempty(ENTITY, "upstream_user_id", &self.upstream_user_id)?; - require_nonempty(ENTITY, "authentication_kind", &self.authentication_kind)?; - if !self.has_refresh_token && self.next_refresh_at.is_some() { - return Err(invalid("next_refresh_at requires a refresh token")); - } - validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; - validate_object_size( - "provider_credentials_json", - &self.provider_credentials_json, - CREDENTIALS_MAX_BYTES, - ) - } -} - -#[derive(Clone)] -pub struct ImportProviderAccounts { - pub scope: ProviderAccountAdminScope, - pub accounts: Vec, - pub audit: AdminAuditEvent, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountAdminImport { - pub config_revision: Revision, - pub account_ids: Vec, -} - -impl fmt::Debug for ImportProviderAccounts { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ImportProviderAccounts") - .field("scope", &self.scope) - .field( - "account_ids", - &self - .accounts - .iter() - .map(|account| account.id.as_str()) - .collect::>(), - ) - .field("audit", &self.audit) - .finish() - } -} - -impl ImportProviderAccounts { - pub fn validate(&self) -> StoreResult<()> { - self.scope.validate()?; - if self.accounts.is_empty() || self.accounts.len() > MAX_ADMIN_IMPORT_BATCH { - return Err(invalid( - "admin import batch must contain between 1 and 200 accounts", - )); - } - let mut ids = BTreeSet::new(); - for account in &self.accounts { - account.validate()?; - if !self.scope.contains(account) { - return Err(invalid( - "imported account is outside the Provider admin scope", - )); - } - if !ids.insert(account.id.as_str()) { - return Err(invalid("admin import batch contains duplicate account IDs")); - } - } - Ok(()) - } -} - -#[derive(Clone)] -pub struct RotateProviderAccount { - pub scope: ProviderAccountAdminScope, - pub profile: UpdateProviderAccount, - pub replacement_identity: Option, - pub credential: ProviderCredentialUpdate, - pub audit: AdminAuditEvent, -} - -impl fmt::Debug for RotateProviderAccount { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RotateProviderAccount") - .field("scope", &self.scope) - .field("profile", &self.profile) - .field("replacement_identity", &self.replacement_identity) - .field("credential", &self.credential) - .field("audit", &self.audit) - .finish() - } -} - -#[derive(Debug, Clone)] -pub struct SetProviderAccountEnabled { - pub scope: ProviderAccountAdminScope, - pub account_id: String, - pub enabled: bool, - pub audit: AdminAuditEvent, -} - -#[derive(Debug, Clone)] -pub struct DeleteProviderAccounts { - pub scope: ProviderAccountAdminScope, - pub account_ids: Vec, - pub audit: AdminAuditEvent, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ProviderAccountAdminRotation { - pub config_revision: Revision, - pub credential_revision: Revision, -} - -#[derive(Clone)] -pub struct ProviderCredentialUpdate { - pub account_id: String, - pub expected_revision: Revision, - pub provider_credentials_json: JsonObject, - pub has_refresh_token: bool, - pub access_token_expires_at: Option>, - pub next_refresh_at: Option>, -} - -impl fmt::Debug for ProviderCredentialUpdate { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ProviderCredentialUpdate") - .field("account_id", &self.account_id) - .field("expected_revision", &self.expected_revision) - .field("provider_credentials_json", &"[REDACTED]") - .finish_non_exhaustive() - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ProviderAccountObservation { - pub account_id: String, - pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, - pub provider_quota_json: Option, - pub availability_observed_at: DateTime, - pub quota_observed_at: Option>, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProviderAccountStateUpdate { - pub account_id: String, - pub expected_revision: Revision, - pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, - pub availability_observed_at: DateTime, -} - -impl ProviderAccountStateUpdate { - pub fn validate(&self) -> StoreResult<()> { - require_nonempty(ENTITY, "account_id", &self.account_id)?; - validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; - Ok(()) - } -} - -impl ProviderAccountObservation { - pub fn validate(&self) -> StoreResult<()> { - require_nonempty(ENTITY, "account_id", &self.account_id)?; - validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; - if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() { - return Err(invalid("quota JSON and quota_observed_at must agree")); - } - if let Some(quota) = &self.provider_quota_json { - validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?; - } - Ok(()) - } -} - -#[async_trait] -pub trait ProviderAccountRepository: Send + Sync { - async fn load_provider_account(&self, id: &str) -> StoreResult>; - async fn list_provider_accounts( - &self, - provider_kind: Option<&str>, - include_disabled: bool, - ) -> StoreResult>; - async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()>; - async fn update_provider_account(&self, account: UpdateProviderAccount) -> StoreResult; - async fn compare_and_swap_credentials( - &self, - update: ProviderCredentialUpdate, - ) -> StoreResult; - async fn apply_provider_account_state( - &self, - update: ProviderAccountStateUpdate, - ) -> StoreResult; - async fn set_provider_account_enabled(&self, id: &str, enabled: bool) -> StoreResult; - async fn compare_and_swap_provider_quota( - &self, - account_id: &str, - expected_revision: Revision, - quota: Option, - observed_at: Option>, - ) -> StoreResult; - async fn delete_provider_account(&self, id: &str) -> StoreResult; -} - -#[async_trait] -pub trait ProviderAccountAdminRepository: Send + Sync { - async fn export_provider_accounts( - &self, - scope: ProviderAccountAdminScope, - account_ids: Vec, - ) -> StoreResult>; - - async fn import_provider_accounts( - &self, - command: ImportProviderAccounts, - ) -> StoreResult; - - async fn rotate_provider_account( - &self, - command: RotateProviderAccount, - ) -> StoreResult; - - async fn set_provider_account_enabled_admin( - &self, - command: SetProviderAccountEnabled, - ) -> StoreResult; - - async fn delete_provider_accounts_admin( - &self, - command: DeleteProviderAccounts, - ) -> StoreResult; -} - -#[derive(Clone)] -pub struct PgProviderAccountRepository { - pool: PgPool, -} - -impl PgProviderAccountRepository { - #[must_use] - pub const fn new(pool: PgPool) -> Self { - Self { pool } - } -} - -#[async_trait] -impl ProviderAccountRepository for PgProviderAccountRepository { - async fn load_provider_account(&self, id: &str) -> StoreResult> { - require_nonempty(ENTITY, "id", id)?; - let row = sqlx::query(ACCOUNT_SELECT) - .bind(id) - .fetch_optional(&self.pool) - .await - .map_err(|_| postgres_unavailable("load provider account"))?; - row.map(account_record_from_row).transpose() - } - - async fn list_provider_accounts( - &self, - provider_kind: Option<&str>, - include_disabled: bool, - ) -> StoreResult> { - let rows = sqlx::query( - "select id, provider_kind, name, email, upstream_user_id, - upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token, - access_token_expires_at, next_refresh_at, enabled, availability, - availability_reason, cooldown_until, availability_observed_at, - quota_observed_at, created_at, updated_at - from provider_accounts - where ($1::text is null or provider_kind = $1) and ($2 or enabled) - order by provider_kind, name, id", - ) - .bind(provider_kind) - .bind(include_disabled) - .fetch_all(&self.pool) - .await - .map_err(|_| postgres_unavailable("list provider accounts"))?; - rows.into_iter().map(account_summary_from_row).collect() - } - - async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> { - account.validate()?; - sqlx::query( - "insert into provider_accounts ( - id, provider_kind, name, email, upstream_user_id, - upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, next_refresh_at, enabled, - availability, availability_reason, cooldown_until, provider_quota_json, - availability_observed_at, quota_observed_at, created_at, updated_at - ) values ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, - $14, null, $15, null, $16, null, now(), greatest(now(), $16) - )", - ) - .bind(account.id) - .bind(account.provider_kind) - .bind(account.name) - .bind(account.email) - .bind(account.upstream_user_id) - .bind(account.upstream_account_id) - .bind(account.plan_type) - .bind(account.authentication_kind) - .bind(account.provider_credentials_json.as_value()) - .bind(account.has_refresh_token) - .bind(account.access_token_expires_at) - .bind(account.next_refresh_at) - .bind(account.enabled) - .bind(account.availability.as_str()) - .bind(account.cooldown_until) - .bind(account.availability_observed_at) - .execute(&self.pool) - .await - .map_err(|_| postgres_unavailable("insert provider account"))?; - Ok(()) - } - - async fn update_provider_account(&self, account: UpdateProviderAccount) -> StoreResult { - require_nonempty(ENTITY, "id", &account.id)?; - require_nonempty(ENTITY, "name", &account.name)?; - let result = sqlx::query( - "update provider_accounts - set name = $2, email = $3, plan_type = $4, updated_at = now() - where id = $1", - ) - .bind(account.id) - .bind(account.name) - .bind(account.email) - .bind(account.plan_type) - .execute(&self.pool) - .await - .map_err(|_| postgres_unavailable("update provider account"))?; - Ok(result.rows_affected() == 1) - } - - async fn compare_and_swap_credentials( - &self, - update: ProviderCredentialUpdate, - ) -> StoreResult { - require_nonempty(ENTITY, "account_id", &update.account_id)?; - validate_object_size( - "provider_credentials_json", - &update.provider_credentials_json, - CREDENTIALS_MAX_BYTES, - )?; - if !update.has_refresh_token && update.next_refresh_at.is_some() { - return Err(invalid("next_refresh_at requires a refresh token")); - } - let next = sqlx::query_scalar::<_, i64>( - "update provider_accounts - set provider_credentials_json = $3, - credential_revision = credential_revision + 1, - has_refresh_token = $4, - access_token_expires_at = $5, - next_refresh_at = $6, - updated_at = now() - where id = $1 and credential_revision = $2 - returning credential_revision", - ) - .bind(&update.account_id) - .bind(to_i64(update.expected_revision.get())?) - .bind(update.provider_credentials_json.as_value()) - .bind(update.has_refresh_token) - .bind(update.access_token_expires_at) - .bind(update.next_refresh_at) - .fetch_optional(&self.pool) - .await - .map_err(|_| postgres_unavailable("compare and swap provider credentials"))? - .ok_or(StoreError::Conflict { - entity: ENTITY, - id: update.account_id, - kind: ConflictKind::StaleRevision, - })?; - Revision::new(to_u64(next)?) - } - - async fn apply_provider_account_state( - &self, - update: ProviderAccountStateUpdate, - ) -> StoreResult { - update.validate()?; - let result = sqlx::query( - "update provider_accounts - set availability = case when enabled then $3 else availability end, - availability_reason = case when enabled then $4 else availability_reason end, - cooldown_until = case when enabled then $5 else cooldown_until end, - availability_observed_at = case - when enabled then $6 - else availability_observed_at - end, - updated_at = case when enabled then greatest(now(), $6) else updated_at end - where id = $1 and credential_revision = $2 - and (availability_observed_at is null or availability_observed_at <= $6)", - ) - .bind(update.account_id) - .bind(to_i64(update.expected_revision.get())?) - .bind(update.availability.as_str()) - .bind(update.availability_reason) - .bind(update.cooldown_until) - .bind(update.availability_observed_at) - .execute(&self.pool) - .await - .map_err(|_| postgres_unavailable("apply provider account state"))?; - Ok(result.rows_affected() == 1) - } - - async fn set_provider_account_enabled(&self, id: &str, enabled: bool) -> StoreResult { - require_nonempty(ENTITY, "id", id)?; - let result = sqlx::query( - "update provider_accounts set enabled = $2, updated_at = now() where id = $1", - ) - .bind(id) - .bind(enabled) - .execute(&self.pool) - .await - .map_err(|_| postgres_unavailable("set provider account enabled state"))?; - Ok(result.rows_affected() == 1) - } - - async fn compare_and_swap_provider_quota( - &self, - account_id: &str, - expected_revision: Revision, - quota: Option, - observed_at: Option>, - ) -> StoreResult { - require_nonempty(ENTITY, "account_id", account_id)?; - if quota.is_some() != observed_at.is_some() { - return Err(invalid("quota JSON and observed_at must agree")); - } - if let Some(quota) = quota.as_ref() { - validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?; - } - let result = sqlx::query( - "update provider_accounts - set provider_quota_json = $3, quota_observed_at = $4, - updated_at = greatest(now(), coalesce($4, now())) - where id = $1 and credential_revision = $2 - and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)", - ) - .bind(account_id) - .bind(to_i64(expected_revision.get())?) - .bind(quota.map(|value| value.as_value())) - .bind(observed_at) - .execute(&self.pool) - .await - .map_err(|_| postgres_unavailable("compare and swap provider quota"))?; - Ok(result.rows_affected() == 1) - } - - async fn delete_provider_account(&self, id: &str) -> StoreResult { - require_nonempty(ENTITY, "id", id)?; - let result = sqlx::query("delete from provider_accounts where id = $1 and not enabled") - .bind(id) - .execute(&self.pool) - .await - .map_err(|_| postgres_unavailable("delete disabled provider account"))?; - Ok(result.rows_affected() == 1) - } -} - -#[async_trait] -impl ProviderAccountAdminRepository for PgProviderAccountRepository { - async fn export_provider_accounts( - &self, - scope: ProviderAccountAdminScope, - account_ids: Vec, - ) -> StoreResult> { - scope.validate()?; - validate_admin_account_ids(&account_ids)?; - let rows = sqlx::query(ACCOUNT_SELECT_BY_IDS) - .bind(&account_ids) - .bind(&scope.provider_kind) - .fetch_all(&self.pool) - .await - .map_err(|_| postgres_unavailable("export provider accounts"))?; - let records = rows - .into_iter() - .map(account_record_from_row) - .collect::>>()?; - if records.len() != account_ids.len() { - return Err(invalid( - "one or more exported accounts are missing or outside the Provider scope", - )); - } - let by_id = records - .into_iter() - .map(|record| (record.summary.id.clone(), record)) - .collect::>(); - account_ids - .into_iter() - .map(|id| { - by_id.get(&id).cloned().ok_or_else(|| { - invalid("one or more exported accounts are missing after loading") - }) - }) - .collect() - } - - async fn import_provider_accounts( - &self, - command: ImportProviderAccounts, - ) -> StoreResult { - command.validate()?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin import"))?; - let result = async { - let revision = bump_config_revision_in_transaction(&mut transaction).await?; - let mut account_ids = Vec::with_capacity(command.accounts.len()); - for account in &command.accounts { - account_ids - .push(upsert_provider_account_in_transaction(&mut transaction, account).await?); - } - append_admin_audit_event_in_transaction(&mut transaction, command.audit, revision) - .await?; - Ok(ProviderAccountAdminImport { - config_revision: revision, - account_ids, - }) - } - .await; - finish_admin_transaction(transaction, result, "provider account admin import").await - } - - async fn rotate_provider_account( - &self, - command: RotateProviderAccount, - ) -> StoreResult { - command.scope.validate()?; - require_nonempty(ENTITY, "account_id", &command.profile.id)?; - require_nonempty(ENTITY, "name", &command.profile.name)?; - if let Some(identity) = &command.replacement_identity { - require_nonempty(ENTITY, "upstream_user_id", identity.upstream_user_id())?; - if let Some(account_id) = identity.upstream_account_id() { - require_nonempty(ENTITY, "upstream_account_id", account_id)?; - } - } - if command.profile.id != command.credential.account_id { - return Err(invalid("rotated profile and credential account IDs differ")); - } - validate_credential_update(&command.credential)?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin rotation"))?; - let result = async { - let config_revision = bump_config_revision_in_transaction(&mut transaction).await?; - let credential_revision = rotate_provider_account_in_transaction( - &mut transaction, - &command.scope, - &command.profile, - command.replacement_identity.as_ref(), - &command.credential, - ) - .await?; - append_admin_audit_event_in_transaction( - &mut transaction, - command.audit, - config_revision, - ) - .await?; - Ok(ProviderAccountAdminRotation { - config_revision, - credential_revision, - }) - } - .await; - finish_admin_transaction(transaction, result, "provider account admin rotation").await - } - - async fn set_provider_account_enabled_admin( - &self, - command: SetProviderAccountEnabled, - ) -> StoreResult { - command.scope.validate()?; - require_nonempty(ENTITY, "account_id", &command.account_id)?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin state change"))?; - let result = async { - let revision = bump_config_revision_in_transaction(&mut transaction).await?; - set_provider_account_enabled_in_transaction( - &mut transaction, - &command.scope, - &command.account_id, - command.enabled, - ) - .await?; - append_admin_audit_event_in_transaction(&mut transaction, command.audit, revision) - .await?; - Ok(revision) - } - .await; - finish_admin_transaction(transaction, result, "provider account admin state change").await - } - - async fn delete_provider_accounts_admin( - &self, - command: DeleteProviderAccounts, - ) -> StoreResult { - command.scope.validate()?; - validate_admin_account_ids(&command.account_ids)?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin deletion"))?; - let result = async { - let revision = bump_config_revision_in_transaction(&mut transaction).await?; - delete_provider_accounts_in_transaction( - &mut transaction, - &command.scope, - &command.account_ids, - ) - .await?; - append_admin_audit_event_in_transaction(&mut transaction, command.audit, revision) - .await?; - Ok(revision) - } - .await; - finish_admin_transaction(transaction, result, "provider account admin deletion").await - } -} - -/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。 -/// -/// 三个 PostgreSQL adapter 都保持私有，调用方只能取得 [`AccountStore`] 暴露的领域能力。 -#[derive(Clone)] -pub struct PgAdminAccountStore { - pool: PgPool, - accounts: PgProviderAccountRepository, - observability: PgObservabilityRepository, - control_plane: PgControlPlaneRepository, -} - -impl PgAdminAccountStore { - #[must_use] - pub fn new(pool: PgPool) -> Self { - Self { - pool: pool.clone(), - accounts: PgProviderAccountRepository::new(pool.clone()), - observability: PgObservabilityRepository::new(pool.clone()), - control_plane: PgControlPlaneRepository::new(pool), - } - } - - async fn usage_observations( - &self, - range: ObservabilityRange, - account_ids: &[String], - ) -> AdminStoreResult> { - if account_ids.is_empty() { - return Ok(Vec::new()); - } - let mut observations = Vec::with_capacity(account_ids.len()); - for account_ids in account_ids.chunks(ADMIN_USAGE_CHUNK_SIZE) { - let query = ProviderAccountUsageQuery::for_accounts(range, account_ids.to_vec()) - .and_then(|query| { - if range.end.signed_duration_since(range.start) <= TimeDelta::hours(24) { - query.with_hourly_request_buckets() - } else { - Ok(query) - } - }) - .map_err(|error| admin_store_error(ENTITY, error))?; - observations.extend( - self.observability - .provider_account_usage(query) - .await - .map_err(|error| admin_store_error(ENTITY, error))?, - ); - } - Ok(observations) - } - - async fn usage_by_windows( - &self, - windows: &[AccountUsageWindowQuery], - ) -> AdminStoreResult> { - if windows.is_empty() { - return Ok(Vec::new()); - } - let account_ids = windows - .iter() - .map(|window| window.account_id.clone()) - .collect::>() - .into_iter() - .collect::>(); - validate_admin_account_ids(&account_ids) - .map_err(|error| admin_store_error(ENTITY, error))?; - for window in windows { - require_nonempty(ENTITY, "quota window key", &window.key) - .map_err(|error| admin_store_error(ENTITY, error))?; - ObservabilityRange::new(window.range.start, window.range.end) - .map_err(|error| admin_store_error(ENTITY, error))?; - } - let keys = windows - .iter() - .map(|window| window.key.clone()) - .collect::>(); - let starts = windows - .iter() - .map(|window| window.range.start) - .collect::>(); - let ends = windows - .iter() - .map(|window| window.range.end) - .collect::>(); - let account_ids = windows - .iter() - .map(|window| window.account_id.clone()) - .collect::>(); - sqlx::query(ACCOUNT_USAGE_BY_WINDOWS_SQL) - .bind(account_ids) - .bind(keys) - .bind(starts) - .bind(ends) - .fetch_all(&self.pool) - .await - .map_err(|_| { - admin_store_error( - ENTITY, - postgres_unavailable("load provider account quota window usage"), - ) - })? - .iter() - .map(admin_account_usage_window) - .collect() - } - - async fn required_scope( - &self, - account_id: &str, - ) -> AdminStoreResult { - let record = self - .accounts - .load_provider_account(account_id) - .await - .map_err(|error| admin_store_error(ENTITY, error))? - .ok_or_else(|| { - admin_store_error( - ENTITY, - StoreError::NotFound { - entity: ENTITY, - id: account_id.to_owned(), - }, - ) - })?; - Ok(ProviderAccountAdminScope { - provider_kind: record.summary.provider_kind, - }) - } - - async fn commit_prepared_import( - &self, - prepared: PreparedCredentialImport, - context: &MutationContext, - action: &str, - ) -> AdminStoreResult { - let provider_kind = prepared.provider_kind.as_str().to_owned(); - let accounts = prepared - .credentials - .into_iter() - .map(prepared_account) - .collect::>>() - .map_err(|error| admin_store_error(ENTITY, error))?; - let imported = self - .accounts - .import_provider_accounts(ImportProviderAccounts { - scope: ProviderAccountAdminScope { - provider_kind: provider_kind.clone(), - }, - accounts, - audit: mutation_audit( - context, - action, - "provider_account", - &provider_kind, - vec!["credentials".to_owned()], - ), - }) - .await - .map_err(|error| admin_store_error(ENTITY, error))?; - Ok(CredentialImportResult { - config_revision: admin_revision(imported.config_revision)?, - credential_ids: imported - .account_ids - .into_iter() - .map(CoreProviderAccountId::new) - .collect::, _>>() - .map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Unavailable, - ENTITY, - "provider account import returned an invalid account ID", - ) - })?, - }) - } - - async fn commit_prepared_rotation( - &self, - prepared: PreparedCredentialRotationFacts, - context: &MutationContext, - action: &str, - ) -> AdminStoreResult { - let account_id = prepared.account_id.clone(); - let scope = ProviderAccountAdminScope { - provider_kind: prepared.provider_kind.as_str().to_owned(), - }; - let rotation = self - .accounts - .rotate_provider_account(RotateProviderAccount { - scope, - profile: UpdateProviderAccount { - id: account_id.as_str().to_owned(), - name: prepared.name, - email: prepared.email, - plan_type: prepared.plan_type, - }, - replacement_identity: prepared.replacement_identity, - credential: ProviderCredentialUpdate { - account_id: account_id.as_str().to_owned(), - expected_revision: store_revision(prepared.expected_credential_revision)?, - provider_credentials_json: provider_document_json(prepared.provider_material) - .map_err(|error| admin_store_error(ENTITY, error))?, - has_refresh_token: prepared.has_refresh_token, - access_token_expires_at: prepared.access_token_expires_at, - next_refresh_at: prepared.next_refresh_at, - }, - audit: mutation_audit( - context, - action, - "provider_account", - account_id.as_str(), - vec!["credentials".to_owned()], - ), - }) - .await - .map_err(|error| admin_store_error(ENTITY, error))?; - Ok(CredentialMutationResult { - config_revision: admin_revision(rotation.config_revision)?, - account_id, - credential_revision: Some(admin_revision(rotation.credential_revision)?), - }) - } -} - -#[async_trait] -impl AccountStore for PgAdminAccountStore { - async fn list_accounts(&self, query: AdminAccountListQuery) -> AdminStoreResult { - if query.page == 0 { - return Err(AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "page number must be positive", - )); - } - let (control_plane, accounts) = futures::try_join!( - self.control_plane.load_control_plane(), - self.accounts.list_provider_accounts(None, true), - ) - .map_err(|error| admin_store_error(ENTITY, error))?; - let now = Utc::now(); - let summary = admin_account_summary(&accounts, now); - let mut items = accounts - .into_iter() - .filter_map(|account| { - let status = admin_account_status(&account, now); - account_matches_admin_query(&account, status, &query).then_some( - AdminAccountListItem { - account, - status, - usage: None, - }, - ) - }) - .collect::>(); - - if query.sort.is_some_and(|sort| { - matches!( - sort.field, - AdminAccountSortField::Usage | AdminAccountSortField::LastUsedAt - ) - }) { - let range = retained_usage_range(control_plane.settings.usage_retention_days, now)?; - let account_ids = items - .iter() - .map(|item| item.account.id.clone()) - .collect::>(); - let mut usage_by_account = self - .usage_observations(range, &account_ids) - .await? - .into_iter() - .map(|usage| (usage.account_id.clone(), usage)) - .collect::>(); - for item in &mut items { - item.usage = usage_by_account.remove(&item.account.id); - } - } - sort_admin_account_items(&mut items, query.sort); - - let total = u64::try_from(items.len()).unwrap_or(u64::MAX); - let page_size = usize::from(query.page_size.get()); - let offset = u64::from(query.page - 1).saturating_mul(u64::from(query.page_size.get())); - let offset = usize::try_from(offset).unwrap_or(usize::MAX); - let items = items - .into_iter() - .skip(offset) - .take(page_size) - .map(|item| admin_account_record(item.account)) - .collect::>>()?; - Ok(AccountPage { - config_revision: admin_revision(control_plane.settings.config_revision)?, - items, - total, - summary, - }) - } - - async fn load_account(&self, account_id: &str) -> AdminStoreResult> { - self.accounts - .load_provider_account(account_id) - .await - .map_err(|error| admin_store_error(ENTITY, error))? - .map(|record| admin_account_record(record.summary)) - .transpose() - } - - async fn load_account_usage( - &self, - range: TimeRange, - account_ids: &[String], - ) -> AdminStoreResult> { - let range = ObservabilityRange::new(range.start, range.end) - .map_err(|error| admin_store_error(ENTITY, error))?; - self.usage_observations(range, account_ids) - .await? - .into_iter() - .map(admin_account_usage) - .collect() - } - - async fn load_account_usage_by_windows( - &self, - windows: &[AccountUsageWindowQuery], - ) -> AdminStoreResult> { - self.usage_by_windows(windows).await - } - - async fn list_credentials( - &self, - provider_kind: &ProviderKind, - query: CredentialListQuery, - ) -> AdminStoreResult { - let (control_plane, accounts) = futures::try_join!( - self.control_plane.load_control_plane(), - self.accounts - .list_provider_accounts(Some(provider_kind.as_str()), true), - ) - .map_err(|error| admin_store_error(ENTITY, error))?; - let mut accounts = accounts - .into_iter() - .filter(|account| account.provider_kind == provider_kind.as_str()) - .filter(|account| { - query - .availability - .as_ref() - .is_none_or(|expected| expected.matches(account.availability)) - }) - .filter(|account| { - query - .enabled - .is_none_or(|enabled| account.enabled == enabled) - }) - .filter(|account| { - let CredentialListWindow::Page { - cursor: Some(cursor), - .. - } = &query.window - else { - return true; - }; - account.created_at > cursor.created_at - || (account.created_at == cursor.created_at - && account.id.as_str() > cursor.account_id.as_str()) - }) - .collect::>(); - if matches!(&query.window, CredentialListWindow::Page { .. }) { - accounts.sort_by(|left, right| { - left.created_at - .cmp(&right.created_at) - .then_with(|| left.id.cmp(&right.id)) - }); - } - let next_cursor = match query.window { - CredentialListWindow::All => None, - CredentialListWindow::Page { page_size, .. } => { - let page_size = usize::from(page_size.get()); - let has_more = accounts.len() > page_size; - accounts.truncate(page_size); - has_more - .then(|| accounts.last()) - .flatten() - .map(|account| { - Ok(CredentialCursor { - created_at: account.created_at, - account_id: CoreProviderAccountId::new(account.id.clone()).map_err( - |_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "persisted Provider account ID is invalid", - ) - }, - )?, - }) - }) - .transpose()? - } - }; - Ok(CredentialPage { - config_revision: admin_revision(control_plane.settings.config_revision)?, - items: accounts - .into_iter() - .map(admin_account_record) - .collect::>>()?, - next_cursor, - }) - } - - async fn credential_details( - &self, - provider_kind: &ProviderKind, - account_id: &CoreProviderAccountId, - ) -> AdminStoreResult> { - let (control_plane, account) = futures::try_join!( - self.control_plane.load_control_plane(), - self.accounts.load_provider_account(account_id.as_str()), - ) - .map_err(|error| admin_store_error(ENTITY, error))?; - account - .filter(|record| record.summary.provider_kind == provider_kind.as_str()) - .map(|record| { - Ok(CredentialDetails { - config_revision: admin_revision(control_plane.settings.config_revision)?, - credential: admin_account_record(record.summary)?, - }) - }) - .transpose() - } - - async fn load_credentials_for_export( - &self, - provider_kind: &ProviderKind, - account_ids: &[CoreProviderAccountId], - ) -> AdminStoreResult> { - let ids = account_ids - .iter() - .map(|id| id.as_str().to_owned()) - .collect::>(); - validate_admin_account_ids(&ids).map_err(|error| admin_store_error(ENTITY, error))?; - let mut credentials = Vec::with_capacity(account_ids.len()); - for account_id in account_ids { - let record = self - .accounts - .load_provider_account(account_id.as_str()) - .await - .map_err(|error| admin_store_error(ENTITY, error))? - .ok_or_else(|| { - AdminStoreError::new( - AdminStoreErrorKind::NotFound, - ENTITY, - "one or more exported credentials do not exist", - ) - })?; - if record.summary.provider_kind != provider_kind.as_str() { - return Err(AdminStoreError::new( - AdminStoreErrorKind::NotFound, - ENTITY, - "one or more exported credentials belong to another Provider", - )); - } - credentials.push(ProviderExportCredentialInput { - account: admin_account_record(record.summary)?, - provider_material: ProviderDocument::new(OpaqueProviderData::new( - record.provider_credentials_json.fields().clone(), - )), - }); - } - Ok(credentials) - } - - async fn commit_credential_import( - &self, - command: CredentialImportCommit, - context: &MutationContext, - ) -> AdminStoreResult { - self.commit_prepared_import(command.prepared, context, "import_document") - .await - } - - async fn commit_authorization( - &self, - command: AuthorizationCommit, - context: &MutationContext, - ) -> AdminStoreResult { - match command.credential { - AuthorizationCredentialCommit::Create(credential) => { - let account_id = credential.account_id.clone(); - let result = self - .commit_prepared_import( - PreparedCredentialImport { - provider_kind: credential.provider_kind.clone(), - credentials: vec![credential], - }, - context, - "authorize", - ) - .await?; - let details = self - .accounts - .load_provider_account(account_id.as_str()) - .await - .map_err(|error| admin_store_error(ENTITY, error))? - .ok_or_else(|| { - AdminStoreError::new( - AdminStoreErrorKind::Unavailable, - ENTITY, - "authorized credential was not visible after commit", - ) - })?; - Ok(CredentialMutationResult { - config_revision: result.config_revision, - account_id, - credential_revision: Some(admin_revision(details.summary.credential_revision)?), - }) - } - AuthorizationCredentialCommit::Reauthorize(prepared) => { - self.commit_prepared_rotation(prepared, context, "reauthorize") - .await - } - } - } - - async fn commit_credential_rotation( - &self, - command: CredentialRotationCommit, - context: &MutationContext, - ) -> AdminStoreResult { - self.commit_prepared_rotation(command.prepared, context, "rotate_credential") - .await - } - - async fn commit_credential_refresh( - &self, - command: CredentialRotationCommit, - context: &MutationContext, - ) -> AdminStoreResult { - self.commit_prepared_rotation(command.prepared, context, "refresh_credential") - .await - } - - async fn set_account_enabled( - &self, - command: SetAccountEnabled, - context: &MutationContext, - ) -> AdminStoreResult { - let scope = self.required_scope(&command.account_id).await?; - let action = if command.enabled { "enable" } else { "disable" }; - self.accounts - .set_provider_account_enabled_admin(SetProviderAccountEnabled { - scope, - account_id: command.account_id.clone(), - enabled: command.enabled, - audit: mutation_audit( - context, - action, - "provider_account", - &command.account_id, - vec!["enabled".to_owned()], - ), - }) - .await - .map_err(|error| admin_store_error(ENTITY, error)) - .and_then(admin_revision) - } - - async fn delete_accounts( - &self, - command: DeleteAccounts, - context: &MutationContext, - ) -> AdminStoreResult { - let first_account_id = command.account_ids.first().ok_or_else(|| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "account deletion requires at least one account ID", - ) - })?; - let scope = self.required_scope(first_account_id).await?; - let audit_target = if command.account_ids.len() == 1 { - first_account_id.clone() - } else { - "provider_accounts".to_owned() - }; - self.accounts - .delete_provider_accounts_admin(DeleteProviderAccounts { - scope, - account_ids: command.account_ids, - audit: mutation_audit( - context, - "delete", - "provider_account", - &audit_target, - Vec::new(), - ), - }) - .await - .map_err(|error| admin_store_error(ENTITY, error)) - .and_then(admin_revision) - } - - async fn record_credential_export( - &self, - account_ids: &[CoreProviderAccountId], - context: &MutationContext, - ) -> AdminStoreResult<()> { - let ids = account_ids - .iter() - .map(|id| id.as_str().to_owned()) - .collect::>(); - validate_admin_account_ids(&ids).map_err(|error| admin_store_error(ENTITY, error))?; - for account_id in &ids { - if self - .accounts - .load_provider_account(account_id) - .await - .map_err(|error| admin_store_error(ENTITY, error))? - .is_none() - { - return Err(AdminStoreError::new( - AdminStoreErrorKind::NotFound, - ENTITY, - "one or more exported credentials do not exist", - )); - } - } - let control_plane = self - .control_plane - .load_control_plane() - .await - .map_err(|error| admin_store_error(ENTITY, error))?; - let revision = control_plane.settings.config_revision; - let mut transaction = self.accounts.pool.begin().await.map_err(|_| { - admin_store_error( - ENTITY, - postgres_unavailable("begin credential export audit"), - ) - })?; - let result = async { - for account_id in &ids { - append_admin_audit_event_in_transaction( - &mut transaction, - mutation_audit( - context, - "export_credentials", - "provider_account", - account_id, - Vec::new(), - ), - revision, - ) - .await?; - } - Ok(()) - } - .await; - finish_admin_transaction(transaction, result, "credential export audit") - .await - .map_err(|error| admin_store_error(ENTITY, error)) - } -} - -struct AdminAccountListItem { - account: ProviderAccountSummary, - status: AdminAccountStatus, - usage: Option, -} - -fn retained_usage_range( - retention_days: u32, - now: DateTime, -) -> AdminStoreResult { - let duration = TimeDelta::try_days(i64::from(retention_days)).ok_or_else(|| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "usage retention is outside the supported range", - ) - })?; - let start = now.checked_sub_signed(duration).ok_or_else(|| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "usage retention start is outside the supported range", - ) - })?; - ObservabilityRange::new(start, now).map_err(|error| admin_store_error(ENTITY, error)) -} - -fn account_matches_admin_query( - account: &ProviderAccountSummary, - status: AdminAccountStatus, - query: &AdminAccountListQuery, -) -> bool { - let provider_matches = query - .provider_kind - .as_ref() - .is_none_or(|provider| account.provider_kind == provider.as_str()); - let search_matches = query.search.as_ref().is_none_or(|search| { - let search = search.to_lowercase(); - [ - Some(account.id.as_str()), - Some(account.name.as_str()), - account.email.as_deref(), - account.upstream_account_id.as_deref(), - Some(account.upstream_user_id.as_str()), - ] - .into_iter() - .flatten() - .any(|value| value.to_lowercase().contains(&search)) - }); - let status_matches = query.status.is_none_or(|expected| expected == status); - provider_matches && search_matches && status_matches -} - -fn admin_account_status( - account: &ProviderAccountSummary, - now: DateTime, -) -> AdminAccountStatus { - if !account.enabled { - AdminAccountStatus::Disabled - } else if account.availability == AccountAvailability::Banned { - AdminAccountStatus::Banned - } else if account.availability == AccountAvailability::QuotaExhausted - || (account.availability == AccountAvailability::Cooldown - && account.availability_reason.as_deref() == Some("usage_limit_exhausted") - && account.cooldown_until.is_none_or(|until| until > now)) - { - AdminAccountStatus::QuotaExhausted - } else if account.availability == AccountAvailability::Expired - || account.availability == AccountAvailability::Invalid - || account - .access_token_expires_at - .is_some_and(|expires_at| expires_at <= now) - { - AdminAccountStatus::Expired - } else { - AdminAccountStatus::Active - } -} - -fn admin_account_summary( - accounts: &[ProviderAccountSummary], - now: DateTime, -) -> AccountSummary { - let mut summary = AccountSummary { - total: u64::try_from(accounts.len()).unwrap_or(u64::MAX), - active: 0, - quota_exhausted: 0, - unavailable: 0, - }; - for account in accounts { - match admin_account_status(account, now) { - AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1), - AdminAccountStatus::QuotaExhausted => { - summary.quota_exhausted = summary.quota_exhausted.saturating_add(1); - } - AdminAccountStatus::Expired - | AdminAccountStatus::Disabled - | AdminAccountStatus::Banned => { - summary.unavailable = summary.unavailable.saturating_add(1); - } - } - } - summary -} - -fn sort_admin_account_items(items: &mut [AdminAccountListItem], sort: Option) { - let Some(sort) = sort else { - items.sort_by(|left, right| left.account.id.cmp(&right.account.id)); - return; - }; - items.sort_by(|left, right| { - let ordering = match sort.field { - AdminAccountSortField::Email => left.account.email.cmp(&right.account.email), - AdminAccountSortField::Status => { - admin_account_status_name(left.status).cmp(admin_account_status_name(right.status)) - } - AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type), - AdminAccountSortField::Usage => left - .usage - .as_ref() - .and_then(|usage| usage.total_tokens) - .cmp(&right.usage.as_ref().and_then(|usage| usage.total_tokens)), - AdminAccountSortField::LastUsedAt => left - .usage - .as_ref() - .and_then(|usage| usage.last_used_at.as_ref()) - .cmp( - &right - .usage - .as_ref() - .and_then(|usage| usage.last_used_at.as_ref()), - ), - AdminAccountSortField::ExpiresAt => left - .account - .access_token_expires_at - .cmp(&right.account.access_token_expires_at), - } - .then_with(|| left.account.id.cmp(&right.account.id)); - match sort.direction { - AdminSortDirection::Asc => ordering, - AdminSortDirection::Desc => ordering.reverse(), - } - }); -} - -const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str { - match status { - AdminAccountStatus::Active => "active", - AdminAccountStatus::Expired => "expired", - AdminAccountStatus::QuotaExhausted => "quota_exhausted", - AdminAccountStatus::Disabled => "disabled", - AdminAccountStatus::Banned => "banned", - } -} - -fn admin_account_record(summary: ProviderAccountSummary) -> AdminStoreResult { - Ok(AccountRecord { - id: summary.id, - provider_kind: ProviderKind::new(summary.provider_kind).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "persisted Provider kind is invalid", - ) - })?, - name: summary.name, - email: summary.email, - upstream_user_id: summary.upstream_user_id, - upstream_account_id: summary.upstream_account_id, - plan_type: summary.plan_type, - authentication_kind: summary.authentication_kind, - credential_revision: admin_revision(summary.credential_revision)?, - has_refresh_token: summary.has_refresh_token, - access_token_expires_at: summary.access_token_expires_at, - next_refresh_at: summary.next_refresh_at, - enabled: summary.enabled, - availability: summary.availability, - availability_reason: summary.availability_reason, - cooldown_until: summary.cooldown_until, - availability_observed_at: summary.availability_observed_at, - quota_observed_at: summary.quota_observed_at, - created_at: summary.created_at, - updated_at: summary.updated_at, - }) -} - -fn prepared_account(credential: PreparedCredentialCreate) -> StoreResult { - Ok(NewProviderAccount { - id: credential.account_id.as_str().to_owned(), - provider_kind: credential.provider_kind.as_str().to_owned(), - name: credential.name, - email: credential.email, - upstream_user_id: credential.upstream_user_id, - upstream_account_id: credential.upstream_account_id, - plan_type: credential.plan_type, - authentication_kind: credential.authentication_kind, - provider_credentials_json: provider_document_json(credential.provider_material)?, - has_refresh_token: credential.has_refresh_token, - access_token_expires_at: credential.access_token_expires_at, - next_refresh_at: credential.next_refresh_at, - enabled: credential.enabled, - availability: credential.availability, - availability_reason: credential.availability_reason, - cooldown_until: credential.cooldown_until, - availability_observed_at: credential.availability_observed_at, - }) -} - -fn provider_document_json(document: ProviderDocument) -> StoreResult { - JsonObject::try_from_value( - ENTITY, - serde_json::Value::Object(document.into_provider_data().into_inner()), - CREDENTIALS_MAX_BYTES, - ) -} - -fn admin_account_usage(usage: ProviderAccountUsageObservation) -> AdminStoreResult { - Ok(AccountUsage { - account_id: usage.account_id, - request_count: usage.request_count, - success_count: usage.success_count, - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - cached_tokens: usage.cached_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - image_input_tokens: usage.image_input_tokens, - image_output_tokens: usage.image_output_tokens, - image_request_count: usage.image_request_count, - image_request_failed_count: usage.image_request_failed_count, - total_tokens: usage.total_tokens, - cost_coverage: admin_account_cost_coverage(usage.cost_coverage), - costs: admin_account_costs(usage.costs)?, - last_used_at: usage.last_used_at, - request_buckets: usage - .request_buckets - .into_iter() - .map(|bucket| AccountRequestBucket { - bucket_start: bucket.bucket_start, - request_count: bucket.request_count, - }) - .collect(), - models: usage - .models - .into_iter() - .map(admin_account_model_usage) - .collect::>>()?, - }) -} - -fn admin_account_usage_window( - row: &sqlx::postgres::PgRow, -) -> AdminStoreResult { - let provider_reported_count = window_usage_count(row, "provider_reported_count")?; - let calculated_count = window_usage_count(row, "calculated_count")?; - let unavailable_count = window_usage_count(row, "unavailable_count")?; - Ok(AccountUsageWindowResult { - account_id: window_usage_value(row, "account_id")?, - key: window_usage_value(row, "window_key")?, - usage: AccountUsage { - account_id: window_usage_value(row, "account_id")?, - request_count: window_usage_count(row, "request_count")?, - success_count: window_usage_count(row, "success_count")?, - input_tokens: optional_window_usage_count(row, "input_tokens")?, - output_tokens: optional_window_usage_count(row, "output_tokens")?, - cached_tokens: optional_window_usage_count(row, "cached_tokens")?, - cache_write_tokens: optional_window_usage_count(row, "cache_write_tokens")?, - reasoning_tokens: optional_window_usage_count(row, "reasoning_tokens")?, - image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?, - image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?, - image_request_count: window_usage_count(row, "image_request_count")?, - image_request_failed_count: window_usage_count(row, "image_request_failed_count")?, - total_tokens: optional_window_usage_count(row, "total_tokens")?, - cost_coverage: AdminCostCoverage { - provider_reported_count, - calculated_count, - partial_count: 0, - unavailable_count, - not_billable_count: 0, - }, - costs: Vec::new(), - last_used_at: window_usage_value(row, "last_used_at")?, - request_buckets: Vec::new(), - models: Vec::new(), - }, - }) -} - -fn window_usage_count(row: &sqlx::postgres::PgRow, column: &'static str) -> AdminStoreResult { - let value = window_usage_value::(row, column)?; - u64::try_from(value).map_err(|_| invalid_window_usage(column)) -} - -fn optional_window_usage_count( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> AdminStoreResult> { - window_usage_value::>(row, column)? - .map(|value| u64::try_from(value).map_err(|_| invalid_window_usage(column))) - .transpose() -} - -fn window_usage_value(row: &sqlx::postgres::PgRow, column: &'static str) -> AdminStoreResult -where - for<'row> T: sqlx::Decode<'row, Postgres> + sqlx::Type, -{ - row.try_get(column) - .map_err(|_| invalid_window_usage(column)) -} - -fn invalid_window_usage(column: &'static str) -> AdminStoreError { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - format!("invalid quota window usage column: {column}"), - ) -} - -fn admin_account_model_usage( - usage: ProviderAccountModelUsageObservation, -) -> AdminStoreResult { - Ok(AccountModelUsage { - model: usage.model, - request_count: usage.request_count, - success_count: usage.success_count, - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - cached_tokens: usage.cached_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - image_input_tokens: usage.image_input_tokens, - image_output_tokens: usage.image_output_tokens, - image_request_count: usage.image_request_count, - image_request_failed_count: usage.image_request_failed_count, - total_tokens: usage.total_tokens, - cost_coverage: admin_account_cost_coverage(usage.cost_coverage), - costs: admin_account_costs(usage.costs)?, - last_used_at: usage.last_used_at, - }) -} - -const fn admin_account_cost_coverage(coverage: super::CostCoverage) -> AdminCostCoverage { - AdminCostCoverage { - provider_reported_count: coverage.provider_reported_count, - calculated_count: coverage.calculated_count, - partial_count: 0, - unavailable_count: coverage.unavailable_count, - not_billable_count: 0, - } -} - -fn admin_account_costs(costs: Vec) -> AdminStoreResult> { - costs - .into_iter() - .map(|cost| { - let amount = AdminDecimalAmount::from_str(cost.amount.as_str()).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "persisted account cost is invalid", - ) - })?; - Ok(AccountCost { - currency: cost.currency, - amount, - }) - }) - .collect() -} - -async fn upsert_provider_account_in_transaction( - transaction: &mut Transaction<'_, Postgres>, - account: &NewProviderAccount, -) -> StoreResult { - account.validate()?; - let imported_id = sqlx::query_scalar::<_, String>( - "insert into provider_accounts ( - id, provider_kind, name, email, upstream_user_id, - upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, next_refresh_at, enabled, - availability, availability_reason, cooldown_until, provider_quota_json, - availability_observed_at, quota_observed_at, created_at, updated_at - ) values ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, - $14, $15, $16, null, $17, null, now(), greatest(now(), $17) - ) - on conflict ( - provider_kind, - upstream_user_id, - (coalesce(upstream_account_id, '')) - ) do update set - name = excluded.name, - email = excluded.email, - plan_type = excluded.plan_type, - authentication_kind = excluded.authentication_kind, - provider_credentials_json = excluded.provider_credentials_json, - credential_revision = provider_accounts.credential_revision + 1, - has_refresh_token = excluded.has_refresh_token, - access_token_expires_at = excluded.access_token_expires_at, - next_refresh_at = excluded.next_refresh_at, - enabled = excluded.enabled, - availability = excluded.availability, - availability_reason = excluded.availability_reason, - cooldown_until = excluded.cooldown_until, - provider_quota_json = null, - availability_observed_at = excluded.availability_observed_at, - quota_observed_at = null, - updated_at = greatest(now(), excluded.availability_observed_at) - returning id", - ) - .bind(&account.id) - .bind(&account.provider_kind) - .bind(&account.name) - .bind(&account.email) - .bind(&account.upstream_user_id) - .bind(&account.upstream_account_id) - .bind(&account.plan_type) - .bind(&account.authentication_kind) - .bind(account.provider_credentials_json.as_value()) - .bind(account.has_refresh_token) - .bind(account.access_token_expires_at) - .bind(account.next_refresh_at) - .bind(account.enabled) - .bind(account.availability.as_str()) - .bind(&account.availability_reason) - .bind(account.cooldown_until) - .bind(account.availability_observed_at) - .fetch_optional(&mut **transaction) - .await - .map_err(|error| { - if error - .as_database_error() - .is_some_and(sqlx::error::DatabaseError::is_unique_violation) - { - StoreError::Conflict { - entity: ENTITY, - id: account.id.clone(), - kind: ConflictKind::InvalidTransition, - } - } else { - postgres_unavailable("upsert provider account in admin transaction") - } - })? - .ok_or_else(|| StoreError::Conflict { - entity: ENTITY, - id: account.id.clone(), - kind: ConflictKind::InvalidTransition, - })?; - Ok(imported_id) -} - -async fn rotate_provider_account_in_transaction( - transaction: &mut Transaction<'_, Postgres>, - scope: &ProviderAccountAdminScope, - profile: &UpdateProviderAccount, - replacement_identity: Option<&ProviderAccountIdentity>, - update: &ProviderCredentialUpdate, -) -> StoreResult { - let replace_identity = replacement_identity.is_some(); - let upstream_user_id = replacement_identity.map(ProviderAccountIdentity::upstream_user_id); - let upstream_account_id = - replacement_identity.and_then(ProviderAccountIdentity::upstream_account_id); - let next = sqlx::query_scalar::<_, i64>( - "update provider_accounts - set name = $4, - email = $5, - plan_type = $6, - provider_credentials_json = $7, - credential_revision = credential_revision + 1, - has_refresh_token = $8, - access_token_expires_at = $9, - next_refresh_at = $10, - upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end, - upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end, - availability = case - when not enabled then availability - when availability <> 'quota_exhausted' then 'ready' - else availability - end, - availability_reason = case - when not enabled then availability_reason - when availability <> 'quota_exhausted' then null - else availability_reason - end, - cooldown_until = case - when not enabled then cooldown_until - when availability <> 'quota_exhausted' then null - else cooldown_until - end, - availability_observed_at = case - when not enabled then availability_observed_at - when availability <> 'quota_exhausted' then now() - else availability_observed_at - end, - updated_at = now() - where id = $1 and provider_kind = $2 - and credential_revision = $3 - returning credential_revision", - ) - .bind(&update.account_id) - .bind(&scope.provider_kind) - .bind(to_i64(update.expected_revision.get())?) - .bind(&profile.name) - .bind(&profile.email) - .bind(&profile.plan_type) - .bind(update.provider_credentials_json.as_value()) - .bind(update.has_refresh_token) - .bind(update.access_token_expires_at) - .bind(update.next_refresh_at) - .bind(replace_identity) - .bind(upstream_user_id) - .bind(upstream_account_id) - .fetch_optional(&mut **transaction) - .await - .map_err(|error| { - if error - .as_database_error() - .is_some_and(sqlx::error::DatabaseError::is_unique_violation) - { - StoreError::Conflict { - entity: ENTITY, - id: update.account_id.clone(), - kind: ConflictKind::InvalidTransition, - } - } else { - postgres_unavailable("rotate provider account in admin transaction") - } - })? - .ok_or_else(|| StoreError::Conflict { - entity: ENTITY, - id: update.account_id.clone(), - kind: ConflictKind::StaleRevision, - })?; - Revision::new(to_u64(next)?) -} - -async fn set_provider_account_enabled_in_transaction( - transaction: &mut Transaction<'_, Postgres>, - scope: &ProviderAccountAdminScope, - account_id: &str, - enabled: bool, -) -> StoreResult<()> { - let result = sqlx::query( - "update provider_accounts set enabled = $3, updated_at = now() - where id = $1 and provider_kind = $2", - ) - .bind(account_id) - .bind(&scope.provider_kind) - .bind(enabled) - .execute(&mut **transaction) - .await - .map_err(|_| postgres_unavailable("set provider account state in admin transaction"))?; - require_admin_account_changed(result.rows_affected(), account_id) -} - -async fn delete_provider_accounts_in_transaction( - transaction: &mut Transaction<'_, Postgres>, - scope: &ProviderAccountAdminScope, - account_ids: &[String], -) -> StoreResult<()> { - let deleted = sqlx::query_scalar::<_, String>( - "delete from provider_accounts - where id = any($1::text[]) and provider_kind = $2 - returning id", - ) - .bind(account_ids) - .bind(&scope.provider_kind) - .fetch_all(&mut **transaction) - .await - .map_err(|_| postgres_unavailable("delete provider account in admin transaction"))?; - let deleted = deleted.into_iter().collect::>(); - let expected = account_ids.iter().cloned().collect::>(); - if deleted == expected { - Ok(()) - } else { - Err(invalid( - "all deleted accounts must exist and match Provider scope", - )) - } -} - -fn validate_credential_update(update: &ProviderCredentialUpdate) -> StoreResult<()> { - require_nonempty(ENTITY, "account_id", &update.account_id)?; - validate_object_size( - "provider_credentials_json", - &update.provider_credentials_json, - CREDENTIALS_MAX_BYTES, - )?; - if !update.has_refresh_token && update.next_refresh_at.is_some() { - return Err(invalid("next_refresh_at requires a refresh token")); - } - Ok(()) -} - -fn require_admin_account_changed(rows_affected: u64, account_id: &str) -> StoreResult<()> { - if rows_affected == 1 { - Ok(()) - } else { - Err(StoreError::NotFound { - entity: ENTITY, - id: account_id.to_owned(), - }) - } -} - -fn validate_admin_account_ids(account_ids: &[String]) -> StoreResult<()> { - if account_ids.is_empty() || account_ids.len() > MAX_ADMIN_IMPORT_BATCH { - return Err(invalid( - "admin account selection must contain between 1 and 200 IDs", - )); - } - let mut unique = BTreeSet::new(); - for account_id in account_ids { - require_nonempty(ENTITY, "account_id", account_id)?; - if !unique.insert(account_id.as_str()) { - return Err(invalid("admin account selection contains duplicate IDs")); - } - } - Ok(()) -} - -async fn finish_admin_transaction( - transaction: Transaction<'_, Postgres>, - result: StoreResult, - operation: &'static str, -) -> StoreResult { - match result { - Ok(value) => { - transaction - .commit() - .await - .map_err(|_| postgres_unavailable(operation))?; - Ok(value) - } - Err(error) => { - transaction - .rollback() - .await - .map_err(|_| postgres_unavailable(operation))?; - Err(error) - } - } -} - -#[async_trait] -impl ProviderAccountStore for PgProviderAccountRepository { - async fn create_account(&self, account: CoreNewProviderAccount) -> Result<(), CoreStoreError> { - if account.account.revision().get() != 1 { - return Err(CoreStoreError::new(CoreStoreErrorKind::InvalidData)); - } - let credential = JsonObject::try_from_value( - "provider_credentials_json", - serde_json::Value::Object(account.credential.into_inner()), - CREDENTIALS_MAX_BYTES, - ) - .map_err(core_store_error)?; - self.insert_provider_account(NewProviderAccount { - id: account.account.id().as_str().to_owned(), - provider_kind: account.account.provider().as_str().to_owned(), - name: account.account.name().to_owned(), - email: account.account.email().map(str::to_owned), - upstream_user_id: account.account.upstream_user_id().to_owned(), - upstream_account_id: account.account.upstream_account_id().map(str::to_owned), - plan_type: account.account.plan_type().map(str::to_owned), - authentication_kind: account.account.authentication_kind().to_owned(), - provider_credentials_json: credential, - has_refresh_token: account.account.has_refresh_token(), - access_token_expires_at: account - .account - .access_token_expires_at() - .map(DateTime::::from), - next_refresh_at: account.account.next_refresh_at().map(DateTime::::from), - enabled: account.account.enabled(), - availability: account.account.availability(), - availability_reason: None, - cooldown_until: account.account.cooldown_until().map(DateTime::::from), - availability_observed_at: Utc::now(), - }) - .await - .map_err(core_store_error) - } - - async fn get_account( - &self, - account: &CoreProviderAccountId, - ) -> Result, CoreStoreError> { - self.load_provider_account(account.as_str()) - .await - .map_err(core_store_error)? - .map(|record| core_account_from_summary(record.summary)) - .transpose() - } - - async fn list_accounts(&self) -> Result, CoreStoreError> { - self.list_provider_accounts(None, true) - .await - .map_err(core_store_error)? - .into_iter() - .map(core_account_from_summary) - .collect() - } - - async fn list_for_provider( - &self, - provider: &ProviderKind, - ) -> Result, CoreStoreError> { - self.list_provider_accounts(Some(provider.as_str()), false) - .await - .map_err(core_store_error)? - .into_iter() - .map(core_account_from_summary) - .collect() - } - - async fn load_credential( - &self, - account: &CoreProviderAccountId, - expected_revision: CoreCredentialRevision, - ) -> Result { - let loaded = self.load_current_credential(account).await?; - if loaded.account.revision().get() != expected_revision.get() { - return Err(CoreStoreError::new(CoreStoreErrorKind::Conflict)); - } - Ok(loaded) - } - - async fn load_current_credential( - &self, - account: &CoreProviderAccountId, - ) -> Result { - let record = self - .load_provider_account(account.as_str()) - .await - .map_err(core_store_error)? - .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - Ok(LoadedCredential { - account: core_account_from_summary(record.summary)?, - credential: PlaintextCredential::new(record.provider_credentials_json.fields().clone()), - }) - } - - async fn compare_and_swap_credential( - &self, - update: CredentialCasUpdate, - ) -> Result { - let ( - account_id, - expected_revision, - profile, - credential, - has_refresh_token, - access_token_expires_at, - next_refresh_at, - ) = update.into_parts(); - if profile.account_id != account_id { - return Err(CoreStoreError::new(CoreStoreErrorKind::InvalidData)); - } - let account_id = account_id.as_str().to_owned(); - let credentials = JsonObject::try_from_value( - "provider_credentials_json", - serde_json::Value::Object(credential.into_inner()), - CREDENTIALS_MAX_BYTES, - ) - .map_err(core_store_error)?; - let next = sqlx::query_scalar::<_, i64>( - "update provider_accounts - set name = $3, email = $4, plan_type = $5, - provider_credentials_json = $6, - credential_revision = credential_revision + 1, - has_refresh_token = $7, access_token_expires_at = $8, - next_refresh_at = $9, updated_at = now() - where id = $1 and credential_revision = $2 - returning credential_revision", - ) - .bind(&account_id) - .bind(to_i64(expected_revision.get()).map_err(core_store_error)?) - .bind(profile.name) - .bind(profile.email) - .bind(profile.plan_type) - .bind(credentials.as_value()) - .bind(has_refresh_token) - .bind(access_token_expires_at.map(DateTime::::from)) - .bind(next_refresh_at.map(DateTime::::from)) - .fetch_optional(&self.pool) - .await - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; - match next { - Some(next) => Ok(CredentialCasOutcome::Updated( - CoreCredentialRevision::new(to_u64(next).map_err(core_store_error)?) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, - )), - None => Ok(CredentialCasOutcome::Conflict), - } - } - - async fn get_quotas( - &self, - accounts: &[CoreProviderAccountId], - ) -> Result, CoreStoreError> { - if accounts.is_empty() { - return Ok(Vec::new()); - } - let account_ids = accounts - .iter() - .map(|account| account.as_str().to_owned()) - .collect::>(); - let rows = sqlx::query( - "select id, credential_revision, provider_quota_json, quota_observed_at \ - from provider_accounts \ - where id = any($1) and provider_quota_json is not null", - ) - .bind(account_ids) - .fetch_all(&self.pool) - .await - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; - rows.into_iter() - .map(|row| { - let account_id = row - .try_get::("id") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let revision = row - .try_get::("credential_revision") - .ok() - .and_then(|value| u64::try_from(value).ok()) - .and_then(|value| CoreCredentialRevision::new(value).ok()) - .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let quota = row - .try_get::("provider_quota_json") - .ok() - .and_then(|value| value.as_object().cloned()) - .map(OpaqueProviderData::new) - .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let observed_at = row - .try_get::, _>("quota_observed_at") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - Ok(QuotaObservation { - account_id: CoreProviderAccountId::new(account_id) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, - expected_revision: revision, - quota: Some(quota), - observed_at: Some(observed_at.into()), - }) - }) - .collect() - } - - async fn compare_and_swap_quota( - &self, - observation: QuotaObservation, - ) -> Result { - let quota = observation - .quota - .map(|value| { - JsonObject::try_from_value( - "provider_quota_json", - serde_json::Value::Object(value.into_inner()), - QUOTA_MAX_BYTES, - ) - }) - .transpose() - .map_err(core_store_error)?; - let updated = self - .compare_and_swap_provider_quota( - observation.account_id.as_str(), - Revision::new(observation.expected_revision.get()).map_err(core_store_error)?, - quota, - observation.observed_at.map(DateTime::::from), - ) - .await - .map_err(core_store_error)?; - Ok(if updated { - QuotaWriteOutcome::Updated - } else { - QuotaWriteOutcome::Conflict - }) - } - - async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), CoreStoreError> { - let updated = self - .apply_provider_account_state(ProviderAccountStateUpdate { - account_id: change.account_id.as_str().to_owned(), - expected_revision: Revision::new(change.expected_revision.get()) - .map_err(core_store_error)?, - availability: change.availability, - availability_reason: change.reason, - cooldown_until: change.cooldown_until.map(DateTime::::from), - availability_observed_at: DateTime::::from(change.observed_at), - }) - .await - .map_err(core_store_error)?; - if updated { - Ok(()) - } else { - Err(CoreStoreError::new(CoreStoreErrorKind::Conflict)) - } - } - - async fn update_account( - &self, - update: CoreProviderAccountUpdate, - ) -> Result<(), CoreStoreError> { - let updated = self - .update_provider_account(UpdateProviderAccount { - id: update.account_id.as_str().to_owned(), - name: update.name, - email: update.email, - plan_type: update.plan_type, - }) - .await - .map_err(core_store_error)?; - require_core_update(updated) - } - - async fn set_enabled( - &self, - account: &CoreProviderAccountId, - enabled: bool, - ) -> Result<(), CoreStoreError> { - let updated = self - .set_provider_account_enabled(account.as_str(), enabled) - .await - .map_err(core_store_error)?; - require_core_update(updated) - } - - async fn delete_account(&self, account: &CoreProviderAccountId) -> Result<(), CoreStoreError> { - let deleted = self - .delete_provider_account(account.as_str()) - .await - .map_err(core_store_error)?; - require_core_update(deleted) - } -} - -const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id, - upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - availability_reason, cooldown_until, provider_quota_json, - availability_observed_at, quota_observed_at, created_at, updated_at - from provider_accounts where id = $1"; - -const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id, - upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - availability_reason, cooldown_until, provider_quota_json, - availability_observed_at, quota_observed_at, created_at, updated_at - from provider_accounts - where id = any($1::text[]) and provider_kind = $2 - order by id"; - -fn account_record_from_row(row: sqlx::postgres::PgRow) -> StoreResult { - let credentials = JsonObject::try_from_value( - "provider_credentials_json", - row.try_get("provider_credentials_json") - .map_err(|_| invalid("invalid credentials JSON"))?, - CREDENTIALS_MAX_BYTES, - )?; - let quota = row - .try_get::, _>("provider_quota_json") - .map_err(|_| invalid("invalid quota JSON"))? - .map(|value| JsonObject::try_from_value("provider_quota_json", value, QUOTA_MAX_BYTES)) - .transpose()?; - Ok(ProviderAccountRecord { - summary: account_summary_from_row(row)?, - provider_credentials_json: credentials, - provider_quota_json: quota, - }) -} - -fn core_account_from_summary( - summary: ProviderAccountSummary, -) -> Result { - let id = CoreProviderAccountId::new(summary.id) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let provider = ProviderKind::new(summary.provider_kind) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let revision = CoreCredentialRevision::new(summary.credential_revision.get()) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - Ok(CoreProviderAccount::new( - id, - provider, - summary.name, - summary.upstream_user_id, - summary.authentication_kind, - revision, - summary.access_token_expires_at.map(Into::into), - ) - .with_profile( - summary.email, - summary.upstream_account_id, - summary.plan_type, - ) - .with_runtime_state( - summary.enabled, - summary.availability, - summary.cooldown_until.map(Into::into), - ) - .with_refresh_schedule( - summary.has_refresh_token, - summary.next_refresh_at.map(Into::into), - )) -} - -fn core_store_error(error: StoreError) -> CoreStoreError { - let kind = match error { - StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable, - StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict, - StoreError::NotFound { .. } | StoreError::InvalidData { .. } => { - CoreStoreErrorKind::InvalidData - } - }; - CoreStoreError::new(kind) -} - -fn require_core_update(updated: bool) -> Result<(), CoreStoreError> { - if updated { - Ok(()) - } else { - Err(CoreStoreError::new(CoreStoreErrorKind::InvalidState)) - } -} - -fn account_summary_from_row(row: sqlx::postgres::PgRow) -> StoreResult { - let revision = row - .try_get::("credential_revision") - .map_err(|_| invalid("invalid credential revision"))?; - let availability = row - .try_get::("availability") - .map_err(|_| invalid("invalid availability"))?; - Ok(ProviderAccountSummary { - id: get(&row, "id")?, - provider_kind: get(&row, "provider_kind")?, - name: get(&row, "name")?, - email: get(&row, "email")?, - upstream_user_id: get(&row, "upstream_user_id")?, - upstream_account_id: get(&row, "upstream_account_id")?, - plan_type: get(&row, "plan_type")?, - authentication_kind: get(&row, "authentication_kind")?, - credential_revision: Revision::new(to_u64(revision)?)?, - has_refresh_token: get(&row, "has_refresh_token")?, - access_token_expires_at: get(&row, "access_token_expires_at")?, - next_refresh_at: get(&row, "next_refresh_at")?, - enabled: get(&row, "enabled")?, - availability: parse_availability(&availability)?, - availability_reason: get(&row, "availability_reason")?, - cooldown_until: get(&row, "cooldown_until")?, - availability_observed_at: get(&row, "availability_observed_at")?, - quota_observed_at: get(&row, "quota_observed_at")?, - created_at: get(&row, "created_at")?, - updated_at: get(&row, "updated_at")?, - }) -} - -fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -> StoreResult -where - T: sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type, -{ - row.try_get(column).map_err(|_| invalid(column)) -} - -fn validate_object_size(field: &'static str, object: &JsonObject, max: usize) -> StoreResult<()> { - let size = serde_json::to_vec(&object.as_value()) - .map_err(|error| StoreError::InvalidData { - entity: ENTITY, - message: error.to_string(), - })? - .len(); - if size > max { - Err(invalid(field)) - } else { - Ok(()) - } -} - -fn to_i64(value: u64) -> StoreResult { - i64::try_from(value).map_err(|_| invalid("revision is too large")) -} - -fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("revision must be positive")) -} - -fn invalid(message: &str) -> StoreError { - StoreError::InvalidData { - entity: ENTITY, - message: message.to_owned(), - } -} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs new file mode 100644 index 000000000..b3aad1e18 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -0,0 +1,663 @@ +//! `gateway-admin` 账号端口的 PostgreSQL adapter。 + +use super::*; + +/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。 +/// +/// 三个 PostgreSQL adapter 都保持私有，调用方只能取得 [`AccountStore`] 暴露的领域能力。 +#[derive(Clone)] +pub struct PgAdminAccountStore { + pool: PgPool, + accounts: PgProviderAccountRepository, + observability: PgObservabilityRepository, + control_plane: PgControlPlaneRepository, +} + +impl PgAdminAccountStore { + #[must_use] + pub fn new(pool: PgPool) -> Self { + Self { + pool: pool.clone(), + accounts: PgProviderAccountRepository::new(pool.clone()), + observability: PgObservabilityRepository::new(pool.clone()), + control_plane: PgControlPlaneRepository::new(pool), + } + } + + async fn usage_observations( + &self, + range: ObservabilityRange, + account_ids: &[String], + ) -> AdminStoreResult> { + if account_ids.is_empty() { + return Ok(Vec::new()); + } + let mut observations = Vec::with_capacity(account_ids.len()); + for account_ids in account_ids.chunks(ADMIN_USAGE_CHUNK_SIZE) { + let query = ProviderAccountUsageQuery::for_accounts(range, account_ids.to_vec()) + .and_then(|query| { + if range.end.signed_duration_since(range.start) <= TimeDelta::hours(24) { + query.with_hourly_request_buckets() + } else { + Ok(query) + } + }) + .map_err(|error| admin_store_error(ENTITY, error))?; + observations.extend( + self.observability + .provider_account_usage(query) + .await + .map_err(|error| admin_store_error(ENTITY, error))?, + ); + } + Ok(observations) + } + + async fn usage_by_windows( + &self, + windows: &[AccountUsageWindowQuery], + ) -> AdminStoreResult> { + if windows.is_empty() { + return Ok(Vec::new()); + } + let account_ids = windows + .iter() + .map(|window| window.account_id.clone()) + .collect::>() + .into_iter() + .collect::>(); + validate_admin_account_ids(&account_ids) + .map_err(|error| admin_store_error(ENTITY, error))?; + for window in windows { + require_nonempty(ENTITY, "quota window key", &window.key) + .map_err(|error| admin_store_error(ENTITY, error))?; + ObservabilityRange::new(window.range.start, window.range.end) + .map_err(|error| admin_store_error(ENTITY, error))?; + } + let keys = windows + .iter() + .map(|window| window.key.clone()) + .collect::>(); + let starts = windows + .iter() + .map(|window| window.range.start) + .collect::>(); + let ends = windows + .iter() + .map(|window| window.range.end) + .collect::>(); + let account_ids = windows + .iter() + .map(|window| window.account_id.clone()) + .collect::>(); + sqlx::query(ACCOUNT_USAGE_BY_WINDOWS_SQL) + .bind(account_ids) + .bind(keys) + .bind(starts) + .bind(ends) + .fetch_all(&self.pool) + .await + .map_err(|_| { + admin_store_error( + ENTITY, + postgres_unavailable("load provider account quota window usage"), + ) + })? + .iter() + .map(admin_account_usage_window) + .collect() + } + + async fn required_scope( + &self, + account_id: &str, + ) -> AdminStoreResult { + let record = self + .accounts + .load_provider_account(account_id) + .await + .map_err(|error| admin_store_error(ENTITY, error))? + .ok_or_else(|| { + admin_store_error( + ENTITY, + StoreError::NotFound { + entity: ENTITY, + id: account_id.to_owned(), + }, + ) + })?; + Ok(ProviderAccountAdminScope { + provider_kind: record.summary.provider_kind, + }) + } + + async fn commit_prepared_import( + &self, + prepared: PreparedCredentialImport, + context: &MutationContext, + action: &str, + ) -> AdminStoreResult { + let provider_kind = prepared.provider_kind.as_str().to_owned(); + let accounts = prepared + .credentials + .into_iter() + .map(prepared_account) + .collect::>>() + .map_err(|error| admin_store_error(ENTITY, error))?; + let imported = self + .accounts + .import_provider_accounts(ImportProviderAccounts { + scope: ProviderAccountAdminScope { + provider_kind: provider_kind.clone(), + }, + accounts, + audit: mutation_audit( + context, + action, + "provider_account", + &provider_kind, + vec!["credentials".to_owned()], + ), + }) + .await + .map_err(|error| admin_store_error(ENTITY, error))?; + Ok(CredentialImportResult { + config_revision: admin_revision(imported.config_revision)?, + credential_ids: imported + .account_ids + .into_iter() + .map(CoreProviderAccountId::new) + .collect::, _>>() + .map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Unavailable, + ENTITY, + "provider account import returned an invalid account ID", + ) + })?, + }) + } + + async fn commit_prepared_rotation( + &self, + prepared: PreparedCredentialRotationFacts, + context: &MutationContext, + action: &str, + ) -> AdminStoreResult { + let account_id = prepared.account_id.clone(); + let scope = ProviderAccountAdminScope { + provider_kind: prepared.provider_kind.as_str().to_owned(), + }; + let rotation = self + .accounts + .rotate_provider_account(RotateProviderAccount { + scope, + profile: UpdateProviderAccount { + id: account_id.as_str().to_owned(), + name: prepared.name, + email: prepared.email, + plan_type: prepared.plan_type, + }, + replacement_identity: prepared.replacement_identity, + credential: ProviderCredentialUpdate { + account_id: account_id.as_str().to_owned(), + expected_revision: store_revision(prepared.expected_credential_revision)?, + provider_credentials_json: provider_document_json(prepared.provider_material) + .map_err(|error| admin_store_error(ENTITY, error))?, + has_refresh_token: prepared.has_refresh_token, + access_token_expires_at: prepared.access_token_expires_at, + next_refresh_at: prepared.next_refresh_at, + }, + audit: mutation_audit( + context, + action, + "provider_account", + account_id.as_str(), + vec!["credentials".to_owned()], + ), + }) + .await + .map_err(|error| admin_store_error(ENTITY, error))?; + Ok(CredentialMutationResult { + config_revision: admin_revision(rotation.config_revision)?, + account_id, + credential_revision: Some(admin_revision(rotation.credential_revision)?), + }) + } +} + +#[async_trait] +impl AccountStore for PgAdminAccountStore { + async fn list_accounts(&self, query: AdminAccountListQuery) -> AdminStoreResult { + if query.page == 0 { + return Err(AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "page number must be positive", + )); + } + let (control_plane, accounts) = futures::try_join!( + self.control_plane.load_control_plane(), + self.accounts.list_provider_accounts(None, true), + ) + .map_err(|error| admin_store_error(ENTITY, error))?; + let now = Utc::now(); + let summary = admin_account_summary(&accounts, now); + let mut items = accounts + .into_iter() + .filter_map(|account| { + let status = admin_account_status(&account, now); + account_matches_admin_query(&account, status, &query).then_some( + AdminAccountListItem { + account, + status, + usage: None, + }, + ) + }) + .collect::>(); + + if query.sort.is_some_and(|sort| { + matches!( + sort.field, + AdminAccountSortField::Usage | AdminAccountSortField::LastUsedAt + ) + }) { + let range = retained_usage_range(control_plane.settings.usage_retention_days, now)?; + let account_ids = items + .iter() + .map(|item| item.account.id.clone()) + .collect::>(); + let mut usage_by_account = self + .usage_observations(range, &account_ids) + .await? + .into_iter() + .map(|usage| (usage.account_id.clone(), usage)) + .collect::>(); + for item in &mut items { + item.usage = usage_by_account.remove(&item.account.id); + } + } + sort_admin_account_items(&mut items, query.sort); + + let total = u64::try_from(items.len()).unwrap_or(u64::MAX); + let page_size = usize::from(query.page_size.get()); + let offset = u64::from(query.page - 1).saturating_mul(u64::from(query.page_size.get())); + let offset = usize::try_from(offset).unwrap_or(usize::MAX); + let items = items + .into_iter() + .skip(offset) + .take(page_size) + .map(|item| admin_account_record(item.account)) + .collect::>>()?; + Ok(AccountPage { + config_revision: admin_revision(control_plane.settings.config_revision)?, + items, + total, + summary, + }) + } + + async fn load_account(&self, account_id: &str) -> AdminStoreResult> { + self.accounts + .load_provider_account(account_id) + .await + .map_err(|error| admin_store_error(ENTITY, error))? + .map(|record| admin_account_record(record.summary)) + .transpose() + } + + async fn load_account_usage( + &self, + range: TimeRange, + account_ids: &[String], + ) -> AdminStoreResult> { + let range = ObservabilityRange::new(range.start, range.end) + .map_err(|error| admin_store_error(ENTITY, error))?; + self.usage_observations(range, account_ids) + .await? + .into_iter() + .map(admin_account_usage) + .collect() + } + + async fn load_account_usage_by_windows( + &self, + windows: &[AccountUsageWindowQuery], + ) -> AdminStoreResult> { + self.usage_by_windows(windows).await + } + + async fn list_credentials( + &self, + provider_kind: &ProviderKind, + query: CredentialListQuery, + ) -> AdminStoreResult { + let (control_plane, accounts) = futures::try_join!( + self.control_plane.load_control_plane(), + self.accounts + .list_provider_accounts(Some(provider_kind.as_str()), true), + ) + .map_err(|error| admin_store_error(ENTITY, error))?; + let mut accounts = accounts + .into_iter() + .filter(|account| account.provider_kind == provider_kind.as_str()) + .filter(|account| { + query + .availability + .as_ref() + .is_none_or(|expected| expected.matches(account.availability)) + }) + .filter(|account| { + query + .enabled + .is_none_or(|enabled| account.enabled == enabled) + }) + .filter(|account| { + let CredentialListWindow::Page { + cursor: Some(cursor), + .. + } = &query.window + else { + return true; + }; + account.created_at > cursor.created_at + || (account.created_at == cursor.created_at + && account.id.as_str() > cursor.account_id.as_str()) + }) + .collect::>(); + if matches!(&query.window, CredentialListWindow::Page { .. }) { + accounts.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + } + let next_cursor = match query.window { + CredentialListWindow::All => None, + CredentialListWindow::Page { page_size, .. } => { + let page_size = usize::from(page_size.get()); + let has_more = accounts.len() > page_size; + accounts.truncate(page_size); + has_more + .then(|| accounts.last()) + .flatten() + .map(|account| { + Ok(CredentialCursor { + created_at: account.created_at, + account_id: CoreProviderAccountId::new(account.id.clone()).map_err( + |_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "persisted Provider account ID is invalid", + ) + }, + )?, + }) + }) + .transpose()? + } + }; + Ok(CredentialPage { + config_revision: admin_revision(control_plane.settings.config_revision)?, + items: accounts + .into_iter() + .map(admin_account_record) + .collect::>>()?, + next_cursor, + }) + } + + async fn credential_details( + &self, + provider_kind: &ProviderKind, + account_id: &CoreProviderAccountId, + ) -> AdminStoreResult> { + let (control_plane, account) = futures::try_join!( + self.control_plane.load_control_plane(), + self.accounts.load_provider_account(account_id.as_str()), + ) + .map_err(|error| admin_store_error(ENTITY, error))?; + account + .filter(|record| record.summary.provider_kind == provider_kind.as_str()) + .map(|record| { + Ok(CredentialDetails { + config_revision: admin_revision(control_plane.settings.config_revision)?, + credential: admin_account_record(record.summary)?, + }) + }) + .transpose() + } + + async fn load_credentials_for_export( + &self, + provider_kind: &ProviderKind, + account_ids: &[CoreProviderAccountId], + ) -> AdminStoreResult> { + let ids = account_ids + .iter() + .map(|id| id.as_str().to_owned()) + .collect::>(); + validate_admin_account_ids(&ids).map_err(|error| admin_store_error(ENTITY, error))?; + let mut credentials = Vec::with_capacity(account_ids.len()); + for account_id in account_ids { + let record = self + .accounts + .load_provider_account(account_id.as_str()) + .await + .map_err(|error| admin_store_error(ENTITY, error))? + .ok_or_else(|| { + AdminStoreError::new( + AdminStoreErrorKind::NotFound, + ENTITY, + "one or more exported credentials do not exist", + ) + })?; + if record.summary.provider_kind != provider_kind.as_str() { + return Err(AdminStoreError::new( + AdminStoreErrorKind::NotFound, + ENTITY, + "one or more exported credentials belong to another Provider", + )); + } + credentials.push(ProviderExportCredentialInput { + account: admin_account_record(record.summary)?, + provider_material: ProviderDocument::new(OpaqueProviderData::new( + record.provider_credentials_json.fields().clone(), + )), + }); + } + Ok(credentials) + } + + async fn commit_credential_import( + &self, + command: CredentialImportCommit, + context: &MutationContext, + ) -> AdminStoreResult { + self.commit_prepared_import(command.prepared, context, "import_document") + .await + } + + async fn commit_authorization( + &self, + command: AuthorizationCommit, + context: &MutationContext, + ) -> AdminStoreResult { + match command.credential { + AuthorizationCredentialCommit::Create(credential) => { + let account_id = credential.account_id.clone(); + let result = self + .commit_prepared_import( + PreparedCredentialImport { + provider_kind: credential.provider_kind.clone(), + credentials: vec![credential], + }, + context, + "authorize", + ) + .await?; + let details = self + .accounts + .load_provider_account(account_id.as_str()) + .await + .map_err(|error| admin_store_error(ENTITY, error))? + .ok_or_else(|| { + AdminStoreError::new( + AdminStoreErrorKind::Unavailable, + ENTITY, + "authorized credential was not visible after commit", + ) + })?; + Ok(CredentialMutationResult { + config_revision: result.config_revision, + account_id, + credential_revision: Some(admin_revision(details.summary.credential_revision)?), + }) + } + AuthorizationCredentialCommit::Reauthorize(prepared) => { + self.commit_prepared_rotation(prepared, context, "reauthorize") + .await + } + } + } + + async fn commit_credential_rotation( + &self, + command: CredentialRotationCommit, + context: &MutationContext, + ) -> AdminStoreResult { + self.commit_prepared_rotation(command.prepared, context, "rotate_credential") + .await + } + + async fn commit_credential_refresh( + &self, + command: CredentialRotationCommit, + context: &MutationContext, + ) -> AdminStoreResult { + self.commit_prepared_rotation(command.prepared, context, "refresh_credential") + .await + } + + async fn set_account_enabled( + &self, + command: SetAccountEnabled, + context: &MutationContext, + ) -> AdminStoreResult { + let scope = self.required_scope(&command.account_id).await?; + let action = if command.enabled { "enable" } else { "disable" }; + self.accounts + .set_provider_account_enabled_admin(SetProviderAccountEnabled { + scope, + account_id: command.account_id.clone(), + enabled: command.enabled, + audit: mutation_audit( + context, + action, + "provider_account", + &command.account_id, + vec!["enabled".to_owned()], + ), + }) + .await + .map_err(|error| admin_store_error(ENTITY, error)) + .and_then(admin_revision) + } + + async fn delete_accounts( + &self, + command: DeleteAccounts, + context: &MutationContext, + ) -> AdminStoreResult { + let first_account_id = command.account_ids.first().ok_or_else(|| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "account deletion requires at least one account ID", + ) + })?; + let scope = self.required_scope(first_account_id).await?; + let audit_target = if command.account_ids.len() == 1 { + first_account_id.clone() + } else { + "provider_accounts".to_owned() + }; + self.accounts + .delete_provider_accounts_admin(DeleteProviderAccounts { + scope, + account_ids: command.account_ids, + audit: mutation_audit( + context, + "delete", + "provider_account", + &audit_target, + Vec::new(), + ), + }) + .await + .map_err(|error| admin_store_error(ENTITY, error)) + .and_then(admin_revision) + } + + async fn record_credential_export( + &self, + account_ids: &[CoreProviderAccountId], + context: &MutationContext, + ) -> AdminStoreResult<()> { + let ids = account_ids + .iter() + .map(|id| id.as_str().to_owned()) + .collect::>(); + validate_admin_account_ids(&ids).map_err(|error| admin_store_error(ENTITY, error))?; + for account_id in &ids { + if self + .accounts + .load_provider_account(account_id) + .await + .map_err(|error| admin_store_error(ENTITY, error))? + .is_none() + { + return Err(AdminStoreError::new( + AdminStoreErrorKind::NotFound, + ENTITY, + "one or more exported credentials do not exist", + )); + } + } + let control_plane = self + .control_plane + .load_control_plane() + .await + .map_err(|error| admin_store_error(ENTITY, error))?; + let revision = control_plane.settings.config_revision; + let mut transaction = self.accounts.pool.begin().await.map_err(|_| { + admin_store_error( + ENTITY, + postgres_unavailable("begin credential export audit"), + ) + })?; + let result = async { + for account_id in &ids { + append_admin_audit_event_in_transaction( + &mut transaction, + mutation_audit( + context, + "export_credentials", + "provider_account", + account_id, + Vec::new(), + ), + revision, + ) + .await?; + } + Ok(()) + } + .await; + finish_admin_transaction(transaction, result, "credential export audit") + .await + .map_err(|error| admin_store_error(ENTITY, error)) + } +} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs new file mode 100644 index 000000000..5787d767c --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs @@ -0,0 +1,293 @@ +//! Core `ProviderAccountStore` 端口适配与 core 投影映射。 + +use super::*; + +#[async_trait] +impl ProviderAccountStore for PgProviderAccountRepository { + async fn create_account(&self, account: CoreNewProviderAccount) -> Result<(), CoreStoreError> { + if account.account.revision().get() != 1 { + return Err(CoreStoreError::new(CoreStoreErrorKind::InvalidData)); + } + let credential = JsonObject::try_from_value( + "provider_credentials_json", + serde_json::Value::Object(account.credential.into_inner()), + CREDENTIALS_MAX_BYTES, + ) + .map_err(core_store_error)?; + self.insert_provider_account(NewProviderAccount { + id: account.account.id().as_str().to_owned(), + provider_kind: account.account.provider().as_str().to_owned(), + name: account.account.name().to_owned(), + email: account.account.email().map(str::to_owned), + upstream_user_id: account.account.upstream_user_id().to_owned(), + upstream_account_id: account.account.upstream_account_id().map(str::to_owned), + plan_type: account.account.plan_type().map(str::to_owned), + authentication_kind: account.account.authentication_kind().to_owned(), + provider_credentials_json: credential, + has_refresh_token: account.account.has_refresh_token(), + access_token_expires_at: account + .account + .access_token_expires_at() + .map(DateTime::::from), + next_refresh_at: account.account.next_refresh_at().map(DateTime::::from), + enabled: account.account.enabled(), + availability: account.account.availability(), + availability_reason: None, + cooldown_until: account.account.cooldown_until().map(DateTime::::from), + availability_observed_at: Utc::now(), + }) + .await + .map_err(core_store_error) + } + + async fn get_account( + &self, + account: &CoreProviderAccountId, + ) -> Result, CoreStoreError> { + self.load_provider_account(account.as_str()) + .await + .map_err(core_store_error)? + .map(|record| core_account_from_summary(record.summary)) + .transpose() + } + + async fn list_accounts(&self) -> Result, CoreStoreError> { + self.list_provider_accounts(None, true) + .await + .map_err(core_store_error)? + .into_iter() + .map(core_account_from_summary) + .collect() + } + + async fn list_for_provider( + &self, + provider: &ProviderKind, + ) -> Result, CoreStoreError> { + self.list_provider_accounts(Some(provider.as_str()), false) + .await + .map_err(core_store_error)? + .into_iter() + .map(core_account_from_summary) + .collect() + } + + async fn load_credential( + &self, + account: &CoreProviderAccountId, + expected_revision: CoreCredentialRevision, + ) -> Result { + let loaded = self.load_current_credential(account).await?; + if loaded.account.revision().get() != expected_revision.get() { + return Err(CoreStoreError::new(CoreStoreErrorKind::Conflict)); + } + Ok(loaded) + } + + async fn load_current_credential( + &self, + account: &CoreProviderAccountId, + ) -> Result { + let record = self + .load_provider_account(account.as_str()) + .await + .map_err(core_store_error)? + .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + Ok(LoadedCredential { + account: core_account_from_summary(record.summary)?, + credential: PlaintextCredential::new(record.provider_credentials_json.fields().clone()), + }) + } + + async fn compare_and_swap_credential( + &self, + update: CredentialCasUpdate, + ) -> Result { + let ( + account_id, + expected_revision, + profile, + credential, + has_refresh_token, + access_token_expires_at, + next_refresh_at, + ) = update.into_parts(); + if profile.account_id != account_id { + return Err(CoreStoreError::new(CoreStoreErrorKind::InvalidData)); + } + let account_id = account_id.as_str().to_owned(); + let credentials = JsonObject::try_from_value( + "provider_credentials_json", + serde_json::Value::Object(credential.into_inner()), + CREDENTIALS_MAX_BYTES, + ) + .map_err(core_store_error)?; + let next = sqlx::query_scalar::<_, i64>( + "update provider_accounts + set name = $3, email = $4, plan_type = $5, + provider_credentials_json = $6, + credential_revision = credential_revision + 1, + has_refresh_token = $7, access_token_expires_at = $8, + next_refresh_at = $9, updated_at = now() + where id = $1 and credential_revision = $2 + returning credential_revision", + ) + .bind(&account_id) + .bind(to_i64(expected_revision.get()).map_err(core_store_error)?) + .bind(profile.name) + .bind(profile.email) + .bind(profile.plan_type) + .bind(credentials.as_value()) + .bind(has_refresh_token) + .bind(access_token_expires_at.map(DateTime::::from)) + .bind(next_refresh_at.map(DateTime::::from)) + .fetch_optional(&self.pool) + .await + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; + match next { + Some(next) => Ok(CredentialCasOutcome::Updated( + CoreCredentialRevision::new(to_u64(next).map_err(core_store_error)?) + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + )), + None => Ok(CredentialCasOutcome::Conflict), + } + } + + async fn get_quotas( + &self, + accounts: &[CoreProviderAccountId], + ) -> Result, CoreStoreError> { + if accounts.is_empty() { + return Ok(Vec::new()); + } + let account_ids = accounts + .iter() + .map(|account| account.as_str().to_owned()) + .collect::>(); + let rows = sqlx::query( + "select id, credential_revision, provider_quota_json, quota_observed_at \ + from provider_accounts \ + where id = any($1) and provider_quota_json is not null", + ) + .bind(account_ids) + .fetch_all(&self.pool) + .await + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; + rows.into_iter() + .map(|row| { + let account_id = row + .try_get::("id") + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let revision = row + .try_get::("credential_revision") + .ok() + .and_then(|value| u64::try_from(value).ok()) + .and_then(|value| CoreCredentialRevision::new(value).ok()) + .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let quota = row + .try_get::("provider_quota_json") + .ok() + .and_then(|value| value.as_object().cloned()) + .map(OpaqueProviderData::new) + .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let observed_at = row + .try_get::, _>("quota_observed_at") + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + Ok(QuotaObservation { + account_id: CoreProviderAccountId::new(account_id) + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + expected_revision: revision, + quota: Some(quota), + observed_at: Some(observed_at.into()), + }) + }) + .collect() + } + + async fn compare_and_swap_quota( + &self, + observation: QuotaObservation, + ) -> Result { + let quota = observation + .quota + .map(|value| { + JsonObject::try_from_value( + "provider_quota_json", + serde_json::Value::Object(value.into_inner()), + QUOTA_MAX_BYTES, + ) + }) + .transpose() + .map_err(core_store_error)?; + let updated = self + .compare_and_swap_provider_quota( + observation.account_id.as_str(), + Revision::new(observation.expected_revision.get()).map_err(core_store_error)?, + quota, + observation.observed_at.map(DateTime::::from), + ) + .await + .map_err(core_store_error)?; + Ok(if updated { + QuotaWriteOutcome::Updated + } else { + QuotaWriteOutcome::Conflict + }) + } + + async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), CoreStoreError> { + let updated = self + .apply_provider_account_state(ProviderAccountStateUpdate { + account_id: change.account_id.as_str().to_owned(), + expected_revision: Revision::new(change.expected_revision.get()) + .map_err(core_store_error)?, + availability: change.availability, + availability_reason: change.reason, + cooldown_until: change.cooldown_until.map(DateTime::::from), + availability_observed_at: DateTime::::from(change.observed_at), + }) + .await + .map_err(core_store_error)?; + if updated { + Ok(()) + } else { + Err(CoreStoreError::new(CoreStoreErrorKind::Conflict)) + } + } + + async fn update_account( + &self, + update: CoreProviderAccountUpdate, + ) -> Result<(), CoreStoreError> { + let updated = self + .update_provider_account(UpdateProviderAccount { + id: update.account_id.as_str().to_owned(), + name: update.name, + email: update.email, + plan_type: update.plan_type, + }) + .await + .map_err(core_store_error)?; + require_core_update(updated) + } + + async fn set_enabled( + &self, + account: &CoreProviderAccountId, + enabled: bool, + ) -> Result<(), CoreStoreError> { + let updated = self + .set_provider_account_enabled(account.as_str(), enabled) + .await + .map_err(core_store_error)?; + require_core_update(updated) + } + + async fn delete_account(&self, account: &CoreProviderAccountId) -> Result<(), CoreStoreError> { + let deleted = self + .delete_provider_account(account.as_str()) + .await + .map_err(core_store_error)?; + require_core_update(deleted) + } +} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs new file mode 100644 index 000000000..3d4a0363a --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs @@ -0,0 +1,392 @@ +//! Admin 账号列表、用量与成本的领域映射。 + +use super::*; + +pub(crate) struct AdminAccountListItem { + pub(crate) account: ProviderAccountSummary, + pub(crate) status: AdminAccountStatus, + pub(crate) usage: Option, +} + +pub(crate) fn retained_usage_range( + retention_days: u32, + now: DateTime, +) -> AdminStoreResult { + let duration = TimeDelta::try_days(i64::from(retention_days)).ok_or_else(|| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "usage retention is outside the supported range", + ) + })?; + let start = now.checked_sub_signed(duration).ok_or_else(|| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "usage retention start is outside the supported range", + ) + })?; + ObservabilityRange::new(start, now).map_err(|error| admin_store_error(ENTITY, error)) +} + +pub(crate) fn account_matches_admin_query( + account: &ProviderAccountSummary, + status: AdminAccountStatus, + query: &AdminAccountListQuery, +) -> bool { + let provider_matches = query + .provider_kind + .as_ref() + .is_none_or(|provider| account.provider_kind == provider.as_str()); + let search_matches = query.search.as_ref().is_none_or(|search| { + let search = search.to_lowercase(); + [ + Some(account.id.as_str()), + Some(account.name.as_str()), + account.email.as_deref(), + account.upstream_account_id.as_deref(), + Some(account.upstream_user_id.as_str()), + ] + .into_iter() + .flatten() + .any(|value| value.to_lowercase().contains(&search)) + }); + let status_matches = query.status.is_none_or(|expected| expected == status); + provider_matches && search_matches && status_matches +} + +pub(crate) fn admin_account_status( + account: &ProviderAccountSummary, + now: DateTime, +) -> AdminAccountStatus { + if !account.enabled { + AdminAccountStatus::Disabled + } else if account.availability == AccountAvailability::Banned { + AdminAccountStatus::Banned + } else if account.availability == AccountAvailability::QuotaExhausted + || (account.availability == AccountAvailability::Cooldown + && account.availability_reason.as_deref() == Some("usage_limit_exhausted") + && account.cooldown_until.is_none_or(|until| until > now)) + { + AdminAccountStatus::QuotaExhausted + } else if account.availability == AccountAvailability::Expired + || account.availability == AccountAvailability::Invalid + || account + .access_token_expires_at + .is_some_and(|expires_at| expires_at <= now) + { + AdminAccountStatus::Expired + } else { + AdminAccountStatus::Active + } +} + +pub(crate) fn admin_account_summary( + accounts: &[ProviderAccountSummary], + now: DateTime, +) -> AccountSummary { + let mut summary = AccountSummary { + total: u64::try_from(accounts.len()).unwrap_or(u64::MAX), + active: 0, + quota_exhausted: 0, + unavailable: 0, + }; + for account in accounts { + match admin_account_status(account, now) { + AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1), + AdminAccountStatus::QuotaExhausted => { + summary.quota_exhausted = summary.quota_exhausted.saturating_add(1); + } + AdminAccountStatus::Expired + | AdminAccountStatus::Disabled + | AdminAccountStatus::Banned => { + summary.unavailable = summary.unavailable.saturating_add(1); + } + } + } + summary +} + +pub(crate) fn sort_admin_account_items( + items: &mut [AdminAccountListItem], + sort: Option, +) { + let Some(sort) = sort else { + items.sort_by(|left, right| left.account.id.cmp(&right.account.id)); + return; + }; + items.sort_by(|left, right| { + let ordering = match sort.field { + AdminAccountSortField::Email => left.account.email.cmp(&right.account.email), + AdminAccountSortField::Status => { + admin_account_status_name(left.status).cmp(admin_account_status_name(right.status)) + } + AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type), + AdminAccountSortField::Usage => left + .usage + .as_ref() + .and_then(|usage| usage.total_tokens) + .cmp(&right.usage.as_ref().and_then(|usage| usage.total_tokens)), + AdminAccountSortField::LastUsedAt => left + .usage + .as_ref() + .and_then(|usage| usage.last_used_at.as_ref()) + .cmp( + &right + .usage + .as_ref() + .and_then(|usage| usage.last_used_at.as_ref()), + ), + AdminAccountSortField::ExpiresAt => left + .account + .access_token_expires_at + .cmp(&right.account.access_token_expires_at), + } + .then_with(|| left.account.id.cmp(&right.account.id)); + match sort.direction { + AdminSortDirection::Asc => ordering, + AdminSortDirection::Desc => ordering.reverse(), + } + }); +} + +pub(crate) const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str { + match status { + AdminAccountStatus::Active => "active", + AdminAccountStatus::Expired => "expired", + AdminAccountStatus::QuotaExhausted => "quota_exhausted", + AdminAccountStatus::Disabled => "disabled", + AdminAccountStatus::Banned => "banned", + } +} + +pub(crate) fn admin_account_record( + summary: ProviderAccountSummary, +) -> AdminStoreResult { + Ok(AccountRecord { + id: summary.id, + provider_kind: ProviderKind::new(summary.provider_kind).map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "persisted Provider kind is invalid", + ) + })?, + name: summary.name, + email: summary.email, + upstream_user_id: summary.upstream_user_id, + upstream_account_id: summary.upstream_account_id, + plan_type: summary.plan_type, + authentication_kind: summary.authentication_kind, + credential_revision: admin_revision(summary.credential_revision)?, + has_refresh_token: summary.has_refresh_token, + access_token_expires_at: summary.access_token_expires_at, + next_refresh_at: summary.next_refresh_at, + enabled: summary.enabled, + availability: summary.availability, + availability_reason: summary.availability_reason, + cooldown_until: summary.cooldown_until, + availability_observed_at: summary.availability_observed_at, + quota_observed_at: summary.quota_observed_at, + created_at: summary.created_at, + updated_at: summary.updated_at, + }) +} + +pub(crate) fn prepared_account( + credential: PreparedCredentialCreate, +) -> StoreResult { + Ok(NewProviderAccount { + id: credential.account_id.as_str().to_owned(), + provider_kind: credential.provider_kind.as_str().to_owned(), + name: credential.name, + email: credential.email, + upstream_user_id: credential.upstream_user_id, + upstream_account_id: credential.upstream_account_id, + plan_type: credential.plan_type, + authentication_kind: credential.authentication_kind, + provider_credentials_json: provider_document_json(credential.provider_material)?, + has_refresh_token: credential.has_refresh_token, + access_token_expires_at: credential.access_token_expires_at, + next_refresh_at: credential.next_refresh_at, + enabled: credential.enabled, + availability: credential.availability, + availability_reason: credential.availability_reason, + cooldown_until: credential.cooldown_until, + availability_observed_at: credential.availability_observed_at, + }) +} + +pub(crate) fn provider_document_json(document: ProviderDocument) -> StoreResult { + JsonObject::try_from_value( + ENTITY, + serde_json::Value::Object(document.into_provider_data().into_inner()), + CREDENTIALS_MAX_BYTES, + ) +} + +pub(crate) fn admin_account_usage( + usage: ProviderAccountUsageObservation, +) -> AdminStoreResult { + Ok(AccountUsage { + account_id: usage.account_id, + request_count: usage.request_count, + success_count: usage.success_count, + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cached_tokens: usage.cached_tokens, + cache_write_tokens: usage.cache_write_tokens, + reasoning_tokens: usage.reasoning_tokens, + image_input_tokens: usage.image_input_tokens, + image_output_tokens: usage.image_output_tokens, + image_request_count: usage.image_request_count, + image_request_failed_count: usage.image_request_failed_count, + total_tokens: usage.total_tokens, + cost_coverage: admin_account_cost_coverage(usage.cost_coverage), + costs: admin_account_costs(usage.costs)?, + last_used_at: usage.last_used_at, + request_buckets: usage + .request_buckets + .into_iter() + .map(|bucket| AccountRequestBucket { + bucket_start: bucket.bucket_start, + request_count: bucket.request_count, + }) + .collect(), + models: usage + .models + .into_iter() + .map(admin_account_model_usage) + .collect::>>()?, + }) +} + +pub(crate) fn admin_account_usage_window( + row: &sqlx::postgres::PgRow, +) -> AdminStoreResult { + let provider_reported_count = window_usage_count(row, "provider_reported_count")?; + let calculated_count = window_usage_count(row, "calculated_count")?; + let unavailable_count = window_usage_count(row, "unavailable_count")?; + Ok(AccountUsageWindowResult { + account_id: window_usage_value(row, "account_id")?, + key: window_usage_value(row, "window_key")?, + usage: AccountUsage { + account_id: window_usage_value(row, "account_id")?, + request_count: window_usage_count(row, "request_count")?, + success_count: window_usage_count(row, "success_count")?, + input_tokens: optional_window_usage_count(row, "input_tokens")?, + output_tokens: optional_window_usage_count(row, "output_tokens")?, + cached_tokens: optional_window_usage_count(row, "cached_tokens")?, + cache_write_tokens: optional_window_usage_count(row, "cache_write_tokens")?, + reasoning_tokens: optional_window_usage_count(row, "reasoning_tokens")?, + image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?, + image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?, + image_request_count: window_usage_count(row, "image_request_count")?, + image_request_failed_count: window_usage_count(row, "image_request_failed_count")?, + total_tokens: optional_window_usage_count(row, "total_tokens")?, + cost_coverage: AdminCostCoverage { + provider_reported_count, + calculated_count, + partial_count: 0, + unavailable_count, + not_billable_count: 0, + }, + costs: Vec::new(), + last_used_at: window_usage_value(row, "last_used_at")?, + request_buckets: Vec::new(), + models: Vec::new(), + }, + }) +} + +pub(crate) fn window_usage_count( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> AdminStoreResult { + let value = window_usage_value::(row, column)?; + u64::try_from(value).map_err(|_| invalid_window_usage(column)) +} + +pub(crate) fn optional_window_usage_count( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> AdminStoreResult> { + window_usage_value::>(row, column)? + .map(|value| u64::try_from(value).map_err(|_| invalid_window_usage(column))) + .transpose() +} + +pub(crate) fn window_usage_value( + row: &sqlx::postgres::PgRow, + column: &'static str, +) -> AdminStoreResult +where + for<'row> T: sqlx::Decode<'row, Postgres> + sqlx::Type, +{ + row.try_get(column) + .map_err(|_| invalid_window_usage(column)) +} + +pub(crate) fn invalid_window_usage(column: &'static str) -> AdminStoreError { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + format!("invalid quota window usage column: {column}"), + ) +} + +pub(crate) fn admin_account_model_usage( + usage: ProviderAccountModelUsageObservation, +) -> AdminStoreResult { + Ok(AccountModelUsage { + model: usage.model, + request_count: usage.request_count, + success_count: usage.success_count, + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + cached_tokens: usage.cached_tokens, + cache_write_tokens: usage.cache_write_tokens, + reasoning_tokens: usage.reasoning_tokens, + image_input_tokens: usage.image_input_tokens, + image_output_tokens: usage.image_output_tokens, + image_request_count: usage.image_request_count, + image_request_failed_count: usage.image_request_failed_count, + total_tokens: usage.total_tokens, + cost_coverage: admin_account_cost_coverage(usage.cost_coverage), + costs: admin_account_costs(usage.costs)?, + last_used_at: usage.last_used_at, + }) +} + +pub(crate) const fn admin_account_cost_coverage( + coverage: crate::postgres::CostCoverage, +) -> AdminCostCoverage { + AdminCostCoverage { + provider_reported_count: coverage.provider_reported_count, + calculated_count: coverage.calculated_count, + partial_count: 0, + unavailable_count: coverage.unavailable_count, + not_billable_count: 0, + } +} + +pub(crate) fn admin_account_costs( + costs: Vec, +) -> AdminStoreResult> { + costs + .into_iter() + .map(|cost| { + let amount = AdminDecimalAmount::from_str(cost.amount.as_str()).map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + ENTITY, + "persisted account cost is invalid", + ) + })?; + Ok(AccountCost { + currency: cost.currency, + amount, + }) + }) + .collect() +} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs new file mode 100644 index 000000000..2c45dfb6f --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs @@ -0,0 +1,69 @@ +//! 明文 `provider_accounts` 与凭证 revision CAS 的唯一 PostgreSQL owner。 + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, + str::FromStr, +}; + +use async_trait::async_trait; +use chrono::{DateTime, TimeDelta, Utc}; +use gateway_admin::{ + model::{ + MutationContext, Revision as AdminRevision, + accounts::{ + AccountCost, AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage, + AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort, + AccountSortField as AdminAccountSortField, AccountStatus as AdminAccountStatus, + AccountSummary, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, + DeleteAccounts, SetAccountEnabled, SortDirection as AdminSortDirection, + }, + observability::{ + CostCoverage as AdminCostCoverage, DecimalAmount as AdminDecimalAmount, TimeRange, + }, + provider_credentials::{ + AuthorizationCommit, AuthorizationCredentialCommit, CredentialCursor, + CredentialDetails, CredentialImportCommit, CredentialImportResult, CredentialListQuery, + CredentialListWindow, CredentialMutationResult, CredentialPage, + CredentialRotationCommit, PreparedCredentialCreate, PreparedCredentialImport, + PreparedCredentialRotationFacts, ProviderDocument, ProviderExportCredentialInput, + }, + }, + ports::store::{AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStoreResult}, +}; +use sqlx::{PgPool, Postgres, Row, Transaction}; + +use gateway_core::engine::credential::{ + AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, + CredentialRevision as CoreCredentialRevision, LoadedCredential, + NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential, + ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId, + ProviderAccountIdentity, ProviderAccountStore, + ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome, +}; +use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind}; +use gateway_core::routing::ProviderKind; + +use crate::{ + ConflictKind, JsonObject, Revision, StoreError, StoreResult, admin_revision, admin_store_error, + mutation_audit, postgres_unavailable, require_nonempty, store_revision, +}; + +use super::{ + AdminAuditEvent, ControlPlaneRepository, CurrencyCostTotal, ObservabilityRange, + ObservabilityRepository, PgControlPlaneRepository, PgObservabilityRepository, + ProviderAccountModelUsageObservation, ProviderAccountUsageObservation, + ProviderAccountUsageQuery, append_admin_audit_event_in_transaction, + bump_config_revision_in_transaction, +}; + +mod admin_adapter; +mod core_adapter; +mod mapping; +mod repository; +mod rows; + +pub use admin_adapter::*; +pub(crate) use mapping::*; +pub use repository::*; +pub use rows::*; diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs new file mode 100644 index 000000000..48b0bf4a7 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs @@ -0,0 +1,734 @@ +//! Pg 账号 repository：Core/Admin 端口实现与 admin 事务。 + +use super::*; + +#[async_trait] +pub trait ProviderAccountRepository: Send + Sync { + async fn load_provider_account(&self, id: &str) -> StoreResult>; + async fn list_provider_accounts( + &self, + provider_kind: Option<&str>, + include_disabled: bool, + ) -> StoreResult>; + async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()>; + async fn update_provider_account(&self, account: UpdateProviderAccount) -> StoreResult; + async fn compare_and_swap_credentials( + &self, + update: ProviderCredentialUpdate, + ) -> StoreResult; + async fn apply_provider_account_state( + &self, + update: ProviderAccountStateUpdate, + ) -> StoreResult; + async fn set_provider_account_enabled(&self, id: &str, enabled: bool) -> StoreResult; + async fn compare_and_swap_provider_quota( + &self, + account_id: &str, + expected_revision: Revision, + quota: Option, + observed_at: Option>, + ) -> StoreResult; + async fn delete_provider_account(&self, id: &str) -> StoreResult; +} + +#[async_trait] +pub trait ProviderAccountAdminRepository: Send + Sync { + async fn export_provider_accounts( + &self, + scope: ProviderAccountAdminScope, + account_ids: Vec, + ) -> StoreResult>; + + async fn import_provider_accounts( + &self, + command: ImportProviderAccounts, + ) -> StoreResult; + + async fn rotate_provider_account( + &self, + command: RotateProviderAccount, + ) -> StoreResult; + + async fn set_provider_account_enabled_admin( + &self, + command: SetProviderAccountEnabled, + ) -> StoreResult; + + async fn delete_provider_accounts_admin( + &self, + command: DeleteProviderAccounts, + ) -> StoreResult; +} + +#[derive(Clone)] +pub struct PgProviderAccountRepository { + pub(crate) pool: PgPool, +} + +impl PgProviderAccountRepository { + #[must_use] + pub const fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl ProviderAccountRepository for PgProviderAccountRepository { + async fn load_provider_account(&self, id: &str) -> StoreResult> { + require_nonempty(ENTITY, "id", id)?; + let row = sqlx::query(ACCOUNT_SELECT) + .bind(id) + .fetch_optional(&self.pool) + .await + .map_err(|_| postgres_unavailable("load provider account"))?; + row.map(account_record_from_row).transpose() + } + + async fn list_provider_accounts( + &self, + provider_kind: Option<&str>, + include_disabled: bool, + ) -> StoreResult> { + let rows = sqlx::query( + "select id, provider_kind, name, email, upstream_user_id, + upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token, + access_token_expires_at, next_refresh_at, enabled, availability, + availability_reason, cooldown_until, availability_observed_at, + quota_observed_at, created_at, updated_at + from provider_accounts + where ($1::text is null or provider_kind = $1) and ($2 or enabled) + order by provider_kind, name, id", + ) + .bind(provider_kind) + .bind(include_disabled) + .fetch_all(&self.pool) + .await + .map_err(|_| postgres_unavailable("list provider accounts"))?; + rows.into_iter().map(account_summary_from_row).collect() + } + + async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> { + account.validate()?; + sqlx::query( + "insert into provider_accounts ( + id, provider_kind, name, email, upstream_user_id, + upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, + has_refresh_token, access_token_expires_at, next_refresh_at, enabled, + availability, availability_reason, cooldown_until, provider_quota_json, + availability_observed_at, quota_observed_at, created_at, updated_at + ) values ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, + $14, null, $15, null, $16, null, now(), greatest(now(), $16) + )", + ) + .bind(account.id) + .bind(account.provider_kind) + .bind(account.name) + .bind(account.email) + .bind(account.upstream_user_id) + .bind(account.upstream_account_id) + .bind(account.plan_type) + .bind(account.authentication_kind) + .bind(account.provider_credentials_json.as_value()) + .bind(account.has_refresh_token) + .bind(account.access_token_expires_at) + .bind(account.next_refresh_at) + .bind(account.enabled) + .bind(account.availability.as_str()) + .bind(account.cooldown_until) + .bind(account.availability_observed_at) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("insert provider account"))?; + Ok(()) + } + + async fn update_provider_account(&self, account: UpdateProviderAccount) -> StoreResult { + require_nonempty(ENTITY, "id", &account.id)?; + require_nonempty(ENTITY, "name", &account.name)?; + let result = sqlx::query( + "update provider_accounts + set name = $2, email = $3, plan_type = $4, updated_at = now() + where id = $1", + ) + .bind(account.id) + .bind(account.name) + .bind(account.email) + .bind(account.plan_type) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("update provider account"))?; + Ok(result.rows_affected() == 1) + } + + async fn compare_and_swap_credentials( + &self, + update: ProviderCredentialUpdate, + ) -> StoreResult { + require_nonempty(ENTITY, "account_id", &update.account_id)?; + validate_object_size( + "provider_credentials_json", + &update.provider_credentials_json, + CREDENTIALS_MAX_BYTES, + )?; + if !update.has_refresh_token && update.next_refresh_at.is_some() { + return Err(invalid("next_refresh_at requires a refresh token")); + } + let next = sqlx::query_scalar::<_, i64>( + "update provider_accounts + set provider_credentials_json = $3, + credential_revision = credential_revision + 1, + has_refresh_token = $4, + access_token_expires_at = $5, + next_refresh_at = $6, + updated_at = now() + where id = $1 and credential_revision = $2 + returning credential_revision", + ) + .bind(&update.account_id) + .bind(to_i64(update.expected_revision.get())?) + .bind(update.provider_credentials_json.as_value()) + .bind(update.has_refresh_token) + .bind(update.access_token_expires_at) + .bind(update.next_refresh_at) + .fetch_optional(&self.pool) + .await + .map_err(|_| postgres_unavailable("compare and swap provider credentials"))? + .ok_or(StoreError::Conflict { + entity: ENTITY, + id: update.account_id, + kind: ConflictKind::StaleRevision, + })?; + Revision::new(to_u64(next)?) + } + + async fn apply_provider_account_state( + &self, + update: ProviderAccountStateUpdate, + ) -> StoreResult { + update.validate()?; + let result = sqlx::query( + "update provider_accounts + set availability = case when enabled then $3 else availability end, + availability_reason = case when enabled then $4 else availability_reason end, + cooldown_until = case when enabled then $5 else cooldown_until end, + availability_observed_at = case + when enabled then $6 + else availability_observed_at + end, + updated_at = case when enabled then greatest(now(), $6) else updated_at end + where id = $1 and credential_revision = $2 + and (availability_observed_at is null or availability_observed_at <= $6)", + ) + .bind(update.account_id) + .bind(to_i64(update.expected_revision.get())?) + .bind(update.availability.as_str()) + .bind(update.availability_reason) + .bind(update.cooldown_until) + .bind(update.availability_observed_at) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("apply provider account state"))?; + Ok(result.rows_affected() == 1) + } + + async fn set_provider_account_enabled(&self, id: &str, enabled: bool) -> StoreResult { + require_nonempty(ENTITY, "id", id)?; + let result = sqlx::query( + "update provider_accounts set enabled = $2, updated_at = now() where id = $1", + ) + .bind(id) + .bind(enabled) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("set provider account enabled state"))?; + Ok(result.rows_affected() == 1) + } + + async fn compare_and_swap_provider_quota( + &self, + account_id: &str, + expected_revision: Revision, + quota: Option, + observed_at: Option>, + ) -> StoreResult { + require_nonempty(ENTITY, "account_id", account_id)?; + if quota.is_some() != observed_at.is_some() { + return Err(invalid("quota JSON and observed_at must agree")); + } + if let Some(quota) = quota.as_ref() { + validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?; + } + let result = sqlx::query( + "update provider_accounts + set provider_quota_json = $3, quota_observed_at = $4, + updated_at = greatest(now(), coalesce($4, now())) + where id = $1 and credential_revision = $2 + and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)", + ) + .bind(account_id) + .bind(to_i64(expected_revision.get())?) + .bind(quota.map(|value| value.as_value())) + .bind(observed_at) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("compare and swap provider quota"))?; + Ok(result.rows_affected() == 1) + } + + async fn delete_provider_account(&self, id: &str) -> StoreResult { + require_nonempty(ENTITY, "id", id)?; + let result = sqlx::query("delete from provider_accounts where id = $1 and not enabled") + .bind(id) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("delete disabled provider account"))?; + Ok(result.rows_affected() == 1) + } +} + +#[async_trait] +impl ProviderAccountAdminRepository for PgProviderAccountRepository { + async fn export_provider_accounts( + &self, + scope: ProviderAccountAdminScope, + account_ids: Vec, + ) -> StoreResult> { + scope.validate()?; + validate_admin_account_ids(&account_ids)?; + let rows = sqlx::query(ACCOUNT_SELECT_BY_IDS) + .bind(&account_ids) + .bind(&scope.provider_kind) + .fetch_all(&self.pool) + .await + .map_err(|_| postgres_unavailable("export provider accounts"))?; + let records = rows + .into_iter() + .map(account_record_from_row) + .collect::>>()?; + if records.len() != account_ids.len() { + return Err(invalid( + "one or more exported accounts are missing or outside the Provider scope", + )); + } + let by_id = records + .into_iter() + .map(|record| (record.summary.id.clone(), record)) + .collect::>(); + account_ids + .into_iter() + .map(|id| { + by_id.get(&id).cloned().ok_or_else(|| { + invalid("one or more exported accounts are missing after loading") + }) + }) + .collect() + } + + async fn import_provider_accounts( + &self, + command: ImportProviderAccounts, + ) -> StoreResult { + command.validate()?; + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| postgres_unavailable("begin provider account admin import"))?; + let result = async { + let revision = bump_config_revision_in_transaction(&mut transaction).await?; + let mut account_ids = Vec::with_capacity(command.accounts.len()); + for account in &command.accounts { + account_ids + .push(upsert_provider_account_in_transaction(&mut transaction, account).await?); + } + append_admin_audit_event_in_transaction(&mut transaction, command.audit, revision) + .await?; + Ok(ProviderAccountAdminImport { + config_revision: revision, + account_ids, + }) + } + .await; + finish_admin_transaction(transaction, result, "provider account admin import").await + } + + async fn rotate_provider_account( + &self, + command: RotateProviderAccount, + ) -> StoreResult { + command.scope.validate()?; + require_nonempty(ENTITY, "account_id", &command.profile.id)?; + require_nonempty(ENTITY, "name", &command.profile.name)?; + if let Some(identity) = &command.replacement_identity { + require_nonempty(ENTITY, "upstream_user_id", identity.upstream_user_id())?; + if let Some(account_id) = identity.upstream_account_id() { + require_nonempty(ENTITY, "upstream_account_id", account_id)?; + } + } + if command.profile.id != command.credential.account_id { + return Err(invalid("rotated profile and credential account IDs differ")); + } + validate_credential_update(&command.credential)?; + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| postgres_unavailable("begin provider account admin rotation"))?; + let result = async { + let config_revision = bump_config_revision_in_transaction(&mut transaction).await?; + let credential_revision = rotate_provider_account_in_transaction( + &mut transaction, + &command.scope, + &command.profile, + command.replacement_identity.as_ref(), + &command.credential, + ) + .await?; + append_admin_audit_event_in_transaction( + &mut transaction, + command.audit, + config_revision, + ) + .await?; + Ok(ProviderAccountAdminRotation { + config_revision, + credential_revision, + }) + } + .await; + finish_admin_transaction(transaction, result, "provider account admin rotation").await + } + + async fn set_provider_account_enabled_admin( + &self, + command: SetProviderAccountEnabled, + ) -> StoreResult { + command.scope.validate()?; + require_nonempty(ENTITY, "account_id", &command.account_id)?; + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| postgres_unavailable("begin provider account admin state change"))?; + let result = async { + let revision = bump_config_revision_in_transaction(&mut transaction).await?; + set_provider_account_enabled_in_transaction( + &mut transaction, + &command.scope, + &command.account_id, + command.enabled, + ) + .await?; + append_admin_audit_event_in_transaction(&mut transaction, command.audit, revision) + .await?; + Ok(revision) + } + .await; + finish_admin_transaction(transaction, result, "provider account admin state change").await + } + + async fn delete_provider_accounts_admin( + &self, + command: DeleteProviderAccounts, + ) -> StoreResult { + command.scope.validate()?; + validate_admin_account_ids(&command.account_ids)?; + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| postgres_unavailable("begin provider account admin deletion"))?; + let result = async { + let revision = bump_config_revision_in_transaction(&mut transaction).await?; + delete_provider_accounts_in_transaction( + &mut transaction, + &command.scope, + &command.account_ids, + ) + .await?; + append_admin_audit_event_in_transaction(&mut transaction, command.audit, revision) + .await?; + Ok(revision) + } + .await; + finish_admin_transaction(transaction, result, "provider account admin deletion").await + } +} + +pub(crate) async fn upsert_provider_account_in_transaction( + transaction: &mut Transaction<'_, Postgres>, + account: &NewProviderAccount, +) -> StoreResult { + account.validate()?; + let imported_id = sqlx::query_scalar::<_, String>( + "insert into provider_accounts ( + id, provider_kind, name, email, upstream_user_id, + upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, + has_refresh_token, access_token_expires_at, next_refresh_at, enabled, + availability, availability_reason, cooldown_until, provider_quota_json, + availability_observed_at, quota_observed_at, created_at, updated_at + ) values ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, + $14, $15, $16, null, $17, null, now(), greatest(now(), $17) + ) + on conflict ( + provider_kind, + upstream_user_id, + (coalesce(upstream_account_id, '')) + ) do update set + name = excluded.name, + email = excluded.email, + plan_type = excluded.plan_type, + authentication_kind = excluded.authentication_kind, + provider_credentials_json = excluded.provider_credentials_json, + credential_revision = provider_accounts.credential_revision + 1, + has_refresh_token = excluded.has_refresh_token, + access_token_expires_at = excluded.access_token_expires_at, + next_refresh_at = excluded.next_refresh_at, + enabled = excluded.enabled, + availability = excluded.availability, + availability_reason = excluded.availability_reason, + cooldown_until = excluded.cooldown_until, + provider_quota_json = null, + availability_observed_at = excluded.availability_observed_at, + quota_observed_at = null, + updated_at = greatest(now(), excluded.availability_observed_at) + returning id", + ) + .bind(&account.id) + .bind(&account.provider_kind) + .bind(&account.name) + .bind(&account.email) + .bind(&account.upstream_user_id) + .bind(&account.upstream_account_id) + .bind(&account.plan_type) + .bind(&account.authentication_kind) + .bind(account.provider_credentials_json.as_value()) + .bind(account.has_refresh_token) + .bind(account.access_token_expires_at) + .bind(account.next_refresh_at) + .bind(account.enabled) + .bind(account.availability.as_str()) + .bind(&account.availability_reason) + .bind(account.cooldown_until) + .bind(account.availability_observed_at) + .fetch_optional(&mut **transaction) + .await + .map_err(|error| { + if error + .as_database_error() + .is_some_and(sqlx::error::DatabaseError::is_unique_violation) + { + StoreError::Conflict { + entity: ENTITY, + id: account.id.clone(), + kind: ConflictKind::InvalidTransition, + } + } else { + postgres_unavailable("upsert provider account in admin transaction") + } + })? + .ok_or_else(|| StoreError::Conflict { + entity: ENTITY, + id: account.id.clone(), + kind: ConflictKind::InvalidTransition, + })?; + Ok(imported_id) +} + +pub(crate) async fn rotate_provider_account_in_transaction( + transaction: &mut Transaction<'_, Postgres>, + scope: &ProviderAccountAdminScope, + profile: &UpdateProviderAccount, + replacement_identity: Option<&ProviderAccountIdentity>, + update: &ProviderCredentialUpdate, +) -> StoreResult { + let replace_identity = replacement_identity.is_some(); + let upstream_user_id = replacement_identity.map(ProviderAccountIdentity::upstream_user_id); + let upstream_account_id = + replacement_identity.and_then(ProviderAccountIdentity::upstream_account_id); + let next = sqlx::query_scalar::<_, i64>( + "update provider_accounts + set name = $4, + email = $5, + plan_type = $6, + provider_credentials_json = $7, + credential_revision = credential_revision + 1, + has_refresh_token = $8, + access_token_expires_at = $9, + next_refresh_at = $10, + upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end, + upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end, + availability = case + when not enabled then availability + when availability <> 'quota_exhausted' then 'ready' + else availability + end, + availability_reason = case + when not enabled then availability_reason + when availability <> 'quota_exhausted' then null + else availability_reason + end, + cooldown_until = case + when not enabled then cooldown_until + when availability <> 'quota_exhausted' then null + else cooldown_until + end, + availability_observed_at = case + when not enabled then availability_observed_at + when availability <> 'quota_exhausted' then now() + else availability_observed_at + end, + updated_at = now() + where id = $1 and provider_kind = $2 + and credential_revision = $3 + returning credential_revision", + ) + .bind(&update.account_id) + .bind(&scope.provider_kind) + .bind(to_i64(update.expected_revision.get())?) + .bind(&profile.name) + .bind(&profile.email) + .bind(&profile.plan_type) + .bind(update.provider_credentials_json.as_value()) + .bind(update.has_refresh_token) + .bind(update.access_token_expires_at) + .bind(update.next_refresh_at) + .bind(replace_identity) + .bind(upstream_user_id) + .bind(upstream_account_id) + .fetch_optional(&mut **transaction) + .await + .map_err(|error| { + if error + .as_database_error() + .is_some_and(sqlx::error::DatabaseError::is_unique_violation) + { + StoreError::Conflict { + entity: ENTITY, + id: update.account_id.clone(), + kind: ConflictKind::InvalidTransition, + } + } else { + postgres_unavailable("rotate provider account in admin transaction") + } + })? + .ok_or_else(|| StoreError::Conflict { + entity: ENTITY, + id: update.account_id.clone(), + kind: ConflictKind::StaleRevision, + })?; + Revision::new(to_u64(next)?) +} + +pub(crate) async fn set_provider_account_enabled_in_transaction( + transaction: &mut Transaction<'_, Postgres>, + scope: &ProviderAccountAdminScope, + account_id: &str, + enabled: bool, +) -> StoreResult<()> { + let result = sqlx::query( + "update provider_accounts set enabled = $3, updated_at = now() + where id = $1 and provider_kind = $2", + ) + .bind(account_id) + .bind(&scope.provider_kind) + .bind(enabled) + .execute(&mut **transaction) + .await + .map_err(|_| postgres_unavailable("set provider account state in admin transaction"))?; + require_admin_account_changed(result.rows_affected(), account_id) +} + +pub(crate) async fn delete_provider_accounts_in_transaction( + transaction: &mut Transaction<'_, Postgres>, + scope: &ProviderAccountAdminScope, + account_ids: &[String], +) -> StoreResult<()> { + let deleted = sqlx::query_scalar::<_, String>( + "delete from provider_accounts + where id = any($1::text[]) and provider_kind = $2 + returning id", + ) + .bind(account_ids) + .bind(&scope.provider_kind) + .fetch_all(&mut **transaction) + .await + .map_err(|_| postgres_unavailable("delete provider account in admin transaction"))?; + let deleted = deleted.into_iter().collect::>(); + let expected = account_ids.iter().cloned().collect::>(); + if deleted == expected { + Ok(()) + } else { + Err(invalid( + "all deleted accounts must exist and match Provider scope", + )) + } +} + +pub(crate) fn validate_credential_update(update: &ProviderCredentialUpdate) -> StoreResult<()> { + require_nonempty(ENTITY, "account_id", &update.account_id)?; + validate_object_size( + "provider_credentials_json", + &update.provider_credentials_json, + CREDENTIALS_MAX_BYTES, + )?; + if !update.has_refresh_token && update.next_refresh_at.is_some() { + return Err(invalid("next_refresh_at requires a refresh token")); + } + Ok(()) +} + +pub(crate) fn require_admin_account_changed( + rows_affected: u64, + account_id: &str, +) -> StoreResult<()> { + if rows_affected == 1 { + Ok(()) + } else { + Err(StoreError::NotFound { + entity: ENTITY, + id: account_id.to_owned(), + }) + } +} + +pub(crate) fn validate_admin_account_ids(account_ids: &[String]) -> StoreResult<()> { + if account_ids.is_empty() || account_ids.len() > MAX_ADMIN_IMPORT_BATCH { + return Err(invalid( + "admin account selection must contain between 1 and 200 IDs", + )); + } + let mut unique = BTreeSet::new(); + for account_id in account_ids { + require_nonempty(ENTITY, "account_id", account_id)?; + if !unique.insert(account_id.as_str()) { + return Err(invalid("admin account selection contains duplicate IDs")); + } + } + Ok(()) +} + +pub(crate) async fn finish_admin_transaction( + transaction: Transaction<'_, Postgres>, + result: StoreResult, + operation: &'static str, +) -> StoreResult { + match result { + Ok(value) => { + transaction + .commit() + .await + .map_err(|_| postgres_unavailable(operation))?; + Ok(value) + } + Err(error) => { + transaction + .rollback() + .await + .map_err(|_| postgres_unavailable(operation))?; + Err(error) + } + } +} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs new file mode 100644 index 000000000..47ec13f9c --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -0,0 +1,524 @@ +//! `provider_accounts` row 模型、解码与校验。 + +use super::*; + +pub(crate) const ENTITY: &str = "provider account"; +pub(crate) const CREDENTIALS_MAX_BYTES: usize = 256 * 1024; +pub(crate) const QUOTA_MAX_BYTES: usize = 128 * 1024; +pub(crate) const MAX_ADMIN_IMPORT_BATCH: usize = 200; +pub(crate) const ADMIN_USAGE_CHUNK_SIZE: usize = 200; + +pub(crate) const ACCOUNT_USAGE_BY_WINDOWS_SQL: &str = "with requested_windows as ( + select * + from unnest($1::text[], $2::text[], $3::timestamptz[], $4::timestamptz[]) + as requested(account_id, window_key, window_start, window_end) + ) + select requested.account_id, + requested.window_key, + count(mr.id)::bigint as request_count, + count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count, + sum(mr.input_tokens)::bigint as input_tokens, + sum(mr.output_tokens)::bigint as output_tokens, + sum(mr.cached_tokens)::bigint as cached_tokens, + sum(mr.cache_write_tokens)::bigint as cache_write_tokens, + sum(mr.reasoning_tokens)::bigint as reasoning_tokens, + sum(mr.image_input_tokens)::bigint as image_input_tokens, + sum(mr.image_output_tokens)::bigint as image_output_tokens, + count(mr.id) filter (where mr.image_generation_succeeded is true)::bigint + as image_request_count, + count(mr.id) filter (where mr.image_generation_succeeded is false)::bigint + as image_request_failed_count, + coalesce(sum(coalesce( + mr.total_tokens, + coalesce(mr.input_tokens, 0) + coalesce(mr.output_tokens, 0) + )), 0)::bigint as total_tokens, + count(mr.id) filter (where mr.cost_source = 'provider_reported')::bigint + as provider_reported_count, + count(mr.id) filter (where mr.cost_source = 'calculated')::bigint + as calculated_count, + count(mr.id) filter (where mr.cost_source = 'unavailable')::bigint + as unavailable_count, + max(mr.started_at) as last_used_at + from requested_windows requested + left join model_requests mr + on mr.provider_account_ref = requested.account_id + and mr.started_at >= requested.window_start + and mr.started_at < requested.window_end + and mr.outcome = 'succeeded' + and mr.downstream_committed_at is not null + and mr.client_status_code between 200 and 399 + group by requested.account_id, requested.window_key + order by requested.account_id, requested.window_key"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountAdminScope { + pub provider_kind: String, +} + +impl ProviderAccountAdminScope { + pub fn validate(&self) -> StoreResult<()> { + require_nonempty(ENTITY, "provider_kind", &self.provider_kind) + } + + fn contains(&self, account: &NewProviderAccount) -> bool { + account.provider_kind == self.provider_kind + } +} + +/// 解析数据库 availability 列；字符串映射本体唯一归属 gateway-core。 +pub(crate) fn parse_availability(value: &str) -> StoreResult { + AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value")) +} + +pub(crate) fn validate_availability_cooldown( + availability: AccountAvailability, + has_cooldown_until: bool, +) -> StoreResult<()> { + if availability == AccountAvailability::Cooldown && !has_cooldown_until { + return Err(invalid("cooldown availability requires cooldown_until")); + } + if has_cooldown_until + && !matches!( + availability, + AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted + ) + { + return Err(invalid( + "cooldown_until is only allowed for cooldown or quota_exhausted availability", + )); + } + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountSummary { + pub id: String, + pub provider_kind: String, + pub name: String, + pub email: Option, + pub upstream_user_id: String, + pub upstream_account_id: Option, + pub plan_type: Option, + pub authentication_kind: String, + pub credential_revision: Revision, + pub has_refresh_token: bool, + pub access_token_expires_at: Option>, + pub next_refresh_at: Option>, + pub enabled: bool, + pub availability: AccountAvailability, + pub availability_reason: Option, + pub cooldown_until: Option>, + pub availability_observed_at: DateTime, + pub quota_observed_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Clone, PartialEq)] +pub struct ProviderAccountRecord { + pub summary: ProviderAccountSummary, + pub provider_credentials_json: JsonObject, + pub provider_quota_json: Option, +} + +impl fmt::Debug for ProviderAccountRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderAccountRecord") + .field("summary", &self.summary) + .field("provider_credentials_json", &"[REDACTED]") + .field( + "provider_quota_json", + &self.provider_quota_json.as_ref().map(|_| "[PROVIDER JSON]"), + ) + .finish() + } +} + +#[derive(Clone)] +pub struct NewProviderAccount { + pub id: String, + pub provider_kind: String, + pub name: String, + pub email: Option, + pub upstream_user_id: String, + pub upstream_account_id: Option, + pub plan_type: Option, + pub authentication_kind: String, + pub provider_credentials_json: JsonObject, + pub has_refresh_token: bool, + pub access_token_expires_at: Option>, + pub next_refresh_at: Option>, + pub enabled: bool, + pub availability: AccountAvailability, + pub availability_reason: Option, + pub cooldown_until: Option>, + pub availability_observed_at: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateProviderAccount { + pub id: String, + pub name: String, + pub email: Option, + pub plan_type: Option, +} + +impl fmt::Debug for NewProviderAccount { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NewProviderAccount") + .field("id", &self.id) + .field("provider_kind", &self.provider_kind) + .field("name", &self.name) + .field("enabled", &self.enabled) + .field("availability", &self.availability) + .field("cooldown_until", &self.cooldown_until) + .field("provider_credentials_json", &"[REDACTED]") + .finish_non_exhaustive() + } +} + +impl NewProviderAccount { + pub fn validate(&self) -> StoreResult<()> { + require_nonempty(ENTITY, "id", &self.id)?; + require_nonempty(ENTITY, "provider_kind", &self.provider_kind)?; + require_nonempty(ENTITY, "name", &self.name)?; + require_nonempty(ENTITY, "upstream_user_id", &self.upstream_user_id)?; + require_nonempty(ENTITY, "authentication_kind", &self.authentication_kind)?; + if !self.has_refresh_token && self.next_refresh_at.is_some() { + return Err(invalid("next_refresh_at requires a refresh token")); + } + validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; + validate_object_size( + "provider_credentials_json", + &self.provider_credentials_json, + CREDENTIALS_MAX_BYTES, + ) + } +} + +#[derive(Clone)] +pub struct ImportProviderAccounts { + pub scope: ProviderAccountAdminScope, + pub accounts: Vec, + pub audit: AdminAuditEvent, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountAdminImport { + pub config_revision: Revision, + pub account_ids: Vec, +} + +impl fmt::Debug for ImportProviderAccounts { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ImportProviderAccounts") + .field("scope", &self.scope) + .field( + "account_ids", + &self + .accounts + .iter() + .map(|account| account.id.as_str()) + .collect::>(), + ) + .field("audit", &self.audit) + .finish() + } +} + +impl ImportProviderAccounts { + pub fn validate(&self) -> StoreResult<()> { + self.scope.validate()?; + if self.accounts.is_empty() || self.accounts.len() > MAX_ADMIN_IMPORT_BATCH { + return Err(invalid( + "admin import batch must contain between 1 and 200 accounts", + )); + } + let mut ids = BTreeSet::new(); + for account in &self.accounts { + account.validate()?; + if !self.scope.contains(account) { + return Err(invalid( + "imported account is outside the Provider admin scope", + )); + } + if !ids.insert(account.id.as_str()) { + return Err(invalid("admin import batch contains duplicate account IDs")); + } + } + Ok(()) + } +} + +#[derive(Clone)] +pub struct RotateProviderAccount { + pub scope: ProviderAccountAdminScope, + pub profile: UpdateProviderAccount, + pub replacement_identity: Option, + pub credential: ProviderCredentialUpdate, + pub audit: AdminAuditEvent, +} + +impl fmt::Debug for RotateProviderAccount { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RotateProviderAccount") + .field("scope", &self.scope) + .field("profile", &self.profile) + .field("replacement_identity", &self.replacement_identity) + .field("credential", &self.credential) + .field("audit", &self.audit) + .finish() + } +} + +#[derive(Debug, Clone)] +pub struct SetProviderAccountEnabled { + pub scope: ProviderAccountAdminScope, + pub account_id: String, + pub enabled: bool, + pub audit: AdminAuditEvent, +} + +#[derive(Debug, Clone)] +pub struct DeleteProviderAccounts { + pub scope: ProviderAccountAdminScope, + pub account_ids: Vec, + pub audit: AdminAuditEvent, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderAccountAdminRotation { + pub config_revision: Revision, + pub credential_revision: Revision, +} + +#[derive(Clone)] +pub struct ProviderCredentialUpdate { + pub account_id: String, + pub expected_revision: Revision, + pub provider_credentials_json: JsonObject, + pub has_refresh_token: bool, + pub access_token_expires_at: Option>, + pub next_refresh_at: Option>, +} + +impl fmt::Debug for ProviderCredentialUpdate { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderCredentialUpdate") + .field("account_id", &self.account_id) + .field("expected_revision", &self.expected_revision) + .field("provider_credentials_json", &"[REDACTED]") + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ProviderAccountObservation { + pub account_id: String, + pub availability: AccountAvailability, + pub availability_reason: Option, + pub cooldown_until: Option>, + pub provider_quota_json: Option, + pub availability_observed_at: DateTime, + pub quota_observed_at: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderAccountStateUpdate { + pub account_id: String, + pub expected_revision: Revision, + pub availability: AccountAvailability, + pub availability_reason: Option, + pub cooldown_until: Option>, + pub availability_observed_at: DateTime, +} + +impl ProviderAccountStateUpdate { + pub fn validate(&self) -> StoreResult<()> { + require_nonempty(ENTITY, "account_id", &self.account_id)?; + validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; + Ok(()) + } +} + +impl ProviderAccountObservation { + pub fn validate(&self) -> StoreResult<()> { + require_nonempty(ENTITY, "account_id", &self.account_id)?; + validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; + if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() { + return Err(invalid("quota JSON and quota_observed_at must agree")); + } + if let Some(quota) = &self.provider_quota_json { + validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?; + } + Ok(()) + } +} + +pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id, + upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, + has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, + availability_reason, cooldown_until, provider_quota_json, + availability_observed_at, quota_observed_at, created_at, updated_at + from provider_accounts where id = $1"; + +pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id, + upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, + has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, + availability_reason, cooldown_until, provider_quota_json, + availability_observed_at, quota_observed_at, created_at, updated_at + from provider_accounts + where id = any($1::text[]) and provider_kind = $2 + order by id"; + +pub(crate) fn account_record_from_row( + row: sqlx::postgres::PgRow, +) -> StoreResult { + let credentials = JsonObject::try_from_value( + "provider_credentials_json", + row.try_get("provider_credentials_json") + .map_err(|_| invalid("invalid credentials JSON"))?, + CREDENTIALS_MAX_BYTES, + )?; + let quota = row + .try_get::, _>("provider_quota_json") + .map_err(|_| invalid("invalid quota JSON"))? + .map(|value| JsonObject::try_from_value("provider_quota_json", value, QUOTA_MAX_BYTES)) + .transpose()?; + Ok(ProviderAccountRecord { + summary: account_summary_from_row(row)?, + provider_credentials_json: credentials, + provider_quota_json: quota, + }) +} + +pub(crate) fn core_account_from_summary( + summary: ProviderAccountSummary, +) -> Result { + let id = CoreProviderAccountId::new(summary.id) + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let provider = ProviderKind::new(summary.provider_kind) + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let revision = CoreCredentialRevision::new(summary.credential_revision.get()) + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + Ok(CoreProviderAccount::new( + id, + provider, + summary.name, + summary.upstream_user_id, + summary.authentication_kind, + revision, + summary.access_token_expires_at.map(Into::into), + ) + .with_profile( + summary.email, + summary.upstream_account_id, + summary.plan_type, + ) + .with_runtime_state( + summary.enabled, + summary.availability, + summary.cooldown_until.map(Into::into), + ) + .with_refresh_schedule( + summary.has_refresh_token, + summary.next_refresh_at.map(Into::into), + )) +} + +pub(crate) fn core_store_error(error: StoreError) -> CoreStoreError { + let kind = match error { + StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable, + StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict, + StoreError::NotFound { .. } | StoreError::InvalidData { .. } => { + CoreStoreErrorKind::InvalidData + } + }; + CoreStoreError::new(kind) +} + +pub(crate) fn require_core_update(updated: bool) -> Result<(), CoreStoreError> { + if updated { + Ok(()) + } else { + Err(CoreStoreError::new(CoreStoreErrorKind::InvalidState)) + } +} + +pub(crate) fn account_summary_from_row( + row: sqlx::postgres::PgRow, +) -> StoreResult { + let revision = row + .try_get::("credential_revision") + .map_err(|_| invalid("invalid credential revision"))?; + let availability = row + .try_get::("availability") + .map_err(|_| invalid("invalid availability"))?; + Ok(ProviderAccountSummary { + id: get(&row, "id")?, + provider_kind: get(&row, "provider_kind")?, + name: get(&row, "name")?, + email: get(&row, "email")?, + upstream_user_id: get(&row, "upstream_user_id")?, + upstream_account_id: get(&row, "upstream_account_id")?, + plan_type: get(&row, "plan_type")?, + authentication_kind: get(&row, "authentication_kind")?, + credential_revision: Revision::new(to_u64(revision)?)?, + has_refresh_token: get(&row, "has_refresh_token")?, + access_token_expires_at: get(&row, "access_token_expires_at")?, + next_refresh_at: get(&row, "next_refresh_at")?, + enabled: get(&row, "enabled")?, + availability: parse_availability(&availability)?, + availability_reason: get(&row, "availability_reason")?, + cooldown_until: get(&row, "cooldown_until")?, + availability_observed_at: get(&row, "availability_observed_at")?, + quota_observed_at: get(&row, "quota_observed_at")?, + created_at: get(&row, "created_at")?, + updated_at: get(&row, "updated_at")?, + }) +} + +pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -> StoreResult +where + T: sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type, +{ + row.try_get(column).map_err(|_| invalid(column)) +} + +pub(crate) fn validate_object_size( + field: &'static str, + object: &JsonObject, + max: usize, +) -> StoreResult<()> { + let size = serde_json::to_vec(&object.as_value()) + .map_err(|error| StoreError::InvalidData { + entity: ENTITY, + message: error.to_string(), + })? + .len(); + if size > max { + Err(invalid(field)) + } else { + Ok(()) + } +} + +pub(crate) fn to_i64(value: u64) -> StoreResult { + i64::try_from(value).map_err(|_| invalid("revision is too large")) +} + +pub(crate) fn to_u64(value: i64) -> StoreResult { + u64::try_from(value).map_err(|_| invalid("revision must be positive")) +} + +pub(crate) fn invalid(message: &str) -> StoreError { + StoreError::InvalidData { + entity: ENTITY, + message: message.to_owned(), + } +} diff --git a/backend/crates/gateway-store/src/value.rs b/backend/crates/gateway-store/src/value.rs new file mode 100644 index 000000000..43ba0442e --- /dev/null +++ b/backend/crates/gateway-store/src/value.rs @@ -0,0 +1,262 @@ +//! Store 值类型、错误与跨层映射。 + +use super::*; + +/// 发生错误的基础设施边界。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoreBackend { + PostgreSql, + Redis, +} + +/// 上层状态机需要区分的稳定冲突类型。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConflictKind { + StaleRevision, + AlreadyFinalized, + DownstreamAlreadyCommitted, + RequestNotRunning, + InvalidTransition, + LeaseLost, + FencingTokenStale, +} + +/// Store adapter 的稳定错误边界。 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum StoreError { + #[error("{backend:?} store is unavailable: {message}")] + Unavailable { + backend: StoreBackend, + message: String, + }, + #[error("{entity} {id} was not found")] + NotFound { entity: &'static str, id: String }, + #[error("store conflict for {entity} {id}: {kind:?}")] + Conflict { + entity: &'static str, + id: String, + kind: ConflictKind, + }, + #[error("invalid persisted {entity}: {message}")] + InvalidData { + entity: &'static str, + message: String, + }, +} + +pub type StoreResult = Result; +pub(crate) fn store_revision(revision: AdminRevision) -> AdminStoreResult { + Revision::new(revision.get()).map_err(|error| admin_store_error("config revision", error)) +} + +pub(crate) fn admin_revision(revision: Revision) -> AdminStoreResult { + AdminRevision::new(revision.get()).map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + "config revision", + "config revision is invalid", + ) + }) +} + +pub(crate) fn mutation_audit( + context: &MutationContext, + action: &str, + entity_kind: &str, + entity_ref: &str, + changed_fields: Vec, +) -> postgres::AdminAuditEvent { + let (actor_kind, actor_admin_user_id, actor_ref) = match &context.actor { + MutationActor::AdminSession { admin_user_id } => ( + postgres::AdminAuditActorKind::AdminSession, + Some(admin_user_id.clone()), + admin_user_id.clone(), + ), + MutationActor::AdminApiKey => ( + postgres::AdminAuditActorKind::AdminApiKey, + None, + "admin_api_key".to_owned(), + ), + MutationActor::System => ( + postgres::AdminAuditActorKind::System, + None, + "system".to_owned(), + ), + }; + postgres::AdminAuditEvent { + id: format!("audit_{}", uuid::Uuid::now_v7().simple()), + actor_kind, + actor_admin_user_id, + actor_ref, + admin_request_id: Some(context.request_id.clone()), + action: action.to_owned(), + entity_kind: entity_kind.to_owned(), + entity_ref: entity_ref.to_owned(), + config_revision: None, + changed_fields, + created_at: chrono::Utc::now(), + } +} + +pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> AdminStoreError { + let kind = match error { + StoreError::NotFound { .. } => AdminStoreErrorKind::NotFound, + StoreError::Conflict { + kind: ConflictKind::StaleRevision, + .. + } => AdminStoreErrorKind::StaleRevision, + StoreError::Conflict { .. } => AdminStoreErrorKind::Conflict, + StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid, + StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable, + }; + AdminStoreError::new(kind, resource, "store operation failed") +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Revision(NonZeroU64); + +impl Revision { + pub fn new(value: u64) -> StoreResult { + NonZeroU64::new(value) + .map(Self) + .ok_or_else(|| StoreError::InvalidData { + entity: "revision", + message: "must be greater than zero".to_owned(), + }) + } + + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } +} + +/// `numeric(20,10)` 可无损表达的非负金额。 +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DecimalAmount(String); + +impl DecimalAmount { + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for DecimalAmount { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl FromStr for DecimalAmount { + type Err = StoreError; + + fn from_str(input: &str) -> Result { + let input = input.trim(); + let mut parts = input.split('.'); + let whole = parts.next().unwrap_or_default(); + let fraction = parts.next(); + let valid = !whole.is_empty() + && whole.len() <= 10 + && whole.bytes().all(|byte| byte.is_ascii_digit()) + && parts.next().is_none() + && fraction.is_none_or(|value| { + !value.is_empty() + && value.len() <= 10 + && value.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !valid { + return Err(StoreError::InvalidData { + entity: "decimal amount", + message: "expected a non-negative numeric(20,10) value".to_owned(), + }); + } + + let whole = whole.trim_start_matches('0'); + let whole = if whole.is_empty() { "0" } else { whole }; + let fraction = fraction.unwrap_or_default().trim_end_matches('0'); + let canonical = if fraction.is_empty() { + whole.to_owned() + } else { + format!("{whole}.{fraction}") + }; + Ok(Self(canonical)) + } +} + +/// Provider-owned JSON object。Store 只验证 object 与大小，不解释内部 key。 +#[derive(Clone, PartialEq)] +pub struct JsonObject(Map); + +impl JsonObject { + pub fn try_from_value( + entity: &'static str, + value: Value, + max_serialized_bytes: usize, + ) -> StoreResult { + let serialized_bytes = serde_json::to_vec(&value) + .map_err(|error| StoreError::InvalidData { + entity, + message: error.to_string(), + })? + .len(); + let Value::Object(fields) = value else { + return Err(StoreError::InvalidData { + entity, + message: "top-level JSON value must be an object".to_owned(), + }); + }; + if serialized_bytes > max_serialized_bytes { + return Err(StoreError::InvalidData { + entity, + message: format!("serialized JSON exceeds {max_serialized_bytes} bytes"), + }); + } + Ok(Self(fields)) + } + + #[must_use] + pub fn as_value(&self) -> Value { + Value::Object(self.0.clone()) + } + + #[must_use] + pub fn fields(&self) -> &Map { + &self.0 + } +} + +impl fmt::Debug for JsonObject { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JsonObject([REDACTED])") + } +} + +pub(crate) fn require_nonempty( + entity: &'static str, + field: &'static str, + value: &str, +) -> StoreResult<()> { + if value.trim().is_empty() { + Err(StoreError::InvalidData { + entity, + message: format!("{field} must not be empty"), + }) + } else { + Ok(()) + } +} + +pub(crate) fn postgres_unavailable(operation: &'static str) -> StoreError { + StoreError::Unavailable { + backend: StoreBackend::PostgreSql, + message: operation.to_owned(), + } +} + +pub(crate) fn redis_unavailable(operation: &'static str) -> StoreError { + StoreError::Unavailable { + backend: StoreBackend::Redis, + message: operation.to_owned(), + } +} diff --git a/backend/crates/gateway-store/src/workers.rs b/backend/crates/gateway-store/src/workers.rs new file mode 100644 index 000000000..ac74768ee --- /dev/null +++ b/backend/crates/gateway-store/src/workers.rs @@ -0,0 +1,195 @@ +//! Worker 贡献、调度定义与健康探针。 + +use super::*; + +pub(crate) fn store_worker_contributions( + execution: Arc, + execution_writer: postgres::ExecutionObservationWriter, + admission_release_writer: redis::ClientAdmissionReleaseWriter, + circuit_feedback_writer: redis::ProviderCircuitFeedbackWriter, + retention: Arc, +) -> StoreResult> { + let stale_id = WorkerId::try_new(WorkerKind::StaleModelRequestRecovery, "postgres") + .map_err(worker_definition_error)?; + let retention_id = + WorkerId::try_new(WorkerKind::Retention, "postgres").map_err(worker_definition_error)?; + let ops_flush_id = + WorkerId::try_new(WorkerKind::OpsFlush, "postgres").map_err(worker_definition_error)?; + let admission_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_admission") + .map_err(worker_definition_error)?; + let circuit_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_circuit") + .map_err(worker_definition_error)?; + let ops_flush_restart = + DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60)) + .map_err(worker_definition_error)?; + Ok(vec![ + WorkerContribution::Registration(scheduled_worker( + stale_id, + Duration::from_secs(30), + Box::new(StaleModelRequestRecoveryTask { execution }), + )?), + WorkerContribution::Registration(scheduled_worker( + retention_id, + Duration::from_secs(60 * 60), + Box::new(RetentionTask { retention }), + )?), + WorkerContribution::Registration( + WorkerRegistration::try_new( + ops_flush_id, + WorkerRunnable::Daemon { + restart: ops_flush_restart, + task: Box::new(execution_writer), + }, + ) + .map_err(worker_definition_error)?, + ), + WorkerContribution::Registration( + WorkerRegistration::try_new( + admission_flush_id, + WorkerRunnable::Daemon { + restart: ops_flush_restart, + task: Box::new(admission_release_writer), + }, + ) + .map_err(worker_definition_error)?, + ), + WorkerContribution::Registration( + WorkerRegistration::try_new( + circuit_flush_id, + WorkerRunnable::Daemon { + restart: ops_flush_restart, + task: Box::new(circuit_feedback_writer), + }, + ) + .map_err(worker_definition_error)?, + ), + ]) +} + +pub(crate) fn scheduled_worker( + id: WorkerId, + interval: Duration, + task: Box, +) -> StoreResult { + let schedule = WorkerSchedule::try_new( + interval, + Duration::from_secs(1), + Duration::from_secs(60), + Duration::from_secs(15 * 60), + Duration::from_secs(5 * 60), + ) + .map_err(worker_definition_error)?; + let lease = WorkerLeaseRequest::try_new(id.clone(), schedule.leader_lease_ttl()) + .map_err(worker_definition_error)?; + WorkerRegistration::try_new( + id, + WorkerRunnable::Scheduled { + schedule, + lease: Some(lease), + task, + }, + ) + .map_err(worker_definition_error) +} + +pub(crate) fn worker_definition_error( + error: gateway_core::task::WorkerDefinitionError, +) -> StoreError { + StoreError::InvalidData { + entity: "store worker plan", + message: error.to_string(), + } +} + +pub(crate) struct StaleModelRequestRecoveryTask { + execution: Arc, +} + +impl ScheduledTask for StaleModelRequestRecoveryTask { + fn run_cycle( + &self, + _context: WorkerCycleContext, + ) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> { + Box::pin(async move { + gateway_core::engine::ExecutionStore::recover_expired( + self.execution.as_ref(), + SystemTime::now(), + ) + .await + .map(|_| ()) + .map_err(|_| WorkerTaskError::safe("stale request recovery failed")) + }) + } +} + +pub(crate) struct RetentionTask { + retention: Arc, +} + +impl ScheduledTask for RetentionTask { + fn run_cycle( + &self, + _context: WorkerCycleContext, + ) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> { + Box::pin(async move { + let settings = + postgres::RetentionRepository::load_retention_settings(self.retention.as_ref()) + .await + .map_err(|_| WorkerTaskError::safe("retention settings read failed"))?; + postgres::RetentionRepository::apply_retention( + self.retention.as_ref(), + chrono::Utc::now(), + settings, + ) + .await + .map(|_| ()) + .map_err(|_| WorkerTaskError::safe("retention cleanup failed")) + }) + } +} + +pub(crate) struct PostgresHealthProbe { + pub(crate) pool: sqlx::PgPool, +} + +impl HealthProbe for PostgresHealthProbe { + fn name(&self) -> &'static str { + "postgres" + } + + fn check(&self) -> futures::future::BoxFuture<'_, HealthState> { + Box::pin(async move { + match sqlx::query_scalar::<_, i32>("select 1") + .fetch_one(&self.pool) + .await + { + Ok(1) => HealthState::Healthy, + Ok(_) => HealthState::Unhealthy("PostgreSQL health result is invalid".to_owned()), + Err(_) => HealthState::Unhealthy("PostgreSQL is unavailable".to_owned()), + } + }) + } +} + +pub(crate) struct RedisHealthProbe { + pub(crate) connection: ::redis::aio::ConnectionManager, +} +impl HealthProbe for RedisHealthProbe { + fn name(&self) -> &'static str { + "redis" + } + + fn check(&self) -> futures::future::BoxFuture<'_, HealthState> { + Box::pin(async move { + let mut connection = self.connection.clone(); + match ::redis::cmd("PING") + .query_async::(&mut connection) + .await + { + Ok(response) if response == "PONG" => HealthState::Healthy, + Ok(_) => HealthState::Unhealthy("Redis health result is invalid".to_owned()), + Err(_) => HealthState::Unhealthy("Redis is unavailable".to_owned()), + } + }) + } +}