mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-12 00:06:23 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
00bab8d390 | ||
|
|
f6e364ca33 | ||
|
|
4cd98889a7 | ||
|
|
c9b613659c | ||
|
|
01b4029998 | ||
|
|
f583e50e88 |
@@ -649,6 +649,9 @@ pub struct UsageAttempt {
|
||||
pub operation: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub upstream_send_state: Option<String>,
|
||||
@@ -709,6 +712,7 @@ pub struct UsageSummary {
|
||||
/// 单个诊断维度值的聚合结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DiagnosticObservation {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -716,6 +720,7 @@ pub struct DiagnosticObservation {
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
}
|
||||
@@ -730,8 +735,12 @@ pub struct OpsError {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
@@ -1025,6 +1034,7 @@ pub struct UsageInsights {
|
||||
/// 已完成分母计算的诊断项。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsItem {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -1032,6 +1042,7 @@ pub struct DiagnosticsItem {
|
||||
pub error_rate: f64,
|
||||
pub request_share: f64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
|
||||
@@ -11,14 +11,8 @@ use super::Revision;
|
||||
/// 客户端模型到上游模型的全局精确映射。
|
||||
pub type ModelMappings = BTreeMap<PublicModelId, UpstreamModelId>;
|
||||
|
||||
/// 账号调度策略。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RotationStrategy {
|
||||
Smart,
|
||||
QuotaResetPriority,
|
||||
RoundRobin,
|
||||
Sticky,
|
||||
}
|
||||
/// 账号调度策略;由 Core 拥有稳定值与 wire 映射。
|
||||
pub use gateway_core::engine::credential::RotationStrategy;
|
||||
|
||||
/// 完整运行设置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
||||
@@ -247,6 +247,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
items: items
|
||||
.into_iter()
|
||||
.map(|item| DiagnosticsItem {
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
request_count: item.request_count,
|
||||
success_count: item.request_count.saturating_sub(item.failure_count),
|
||||
@@ -254,6 +255,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
error_rate: rate_or_zero(item.failure_count, item.request_count),
|
||||
request_share: rate_or_zero(item.request_count, total_requests),
|
||||
average_latency_ms: item.average_latency_ms,
|
||||
latency_p95_ms: item.latency_p95_ms,
|
||||
estimated_cost: usd_cost(&item.costs),
|
||||
attempt_count: item.attempt_count,
|
||||
total_tokens: item.total_tokens,
|
||||
|
||||
@@ -695,6 +695,7 @@ fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) ->
|
||||
|
||||
fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
DiagnosticObservation {
|
||||
key: name.to_owned(),
|
||||
name: name.to_owned(),
|
||||
request_count,
|
||||
success_count: request_count,
|
||||
@@ -702,6 +703,7 @@ fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
attempt_count: request_count,
|
||||
total_tokens: request_count.saturating_mul(100),
|
||||
average_latency_ms: Some(100),
|
||||
latency_p95_ms: Some(200),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
}
|
||||
|
||||
@@ -583,6 +583,7 @@ pub struct UsageRecordView {
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub route: String,
|
||||
pub model: String,
|
||||
pub requested_model: Option<String>,
|
||||
@@ -723,7 +724,10 @@ pub struct UsageAttemptView {
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub credential_name: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
@@ -767,8 +771,6 @@ pub struct TrendPointView {
|
||||
pub errors_value: u64,
|
||||
pub latency: String,
|
||||
pub latency_value: Option<u64>,
|
||||
pub first_token_latency: String,
|
||||
pub first_token_latency_value: Option<u64>,
|
||||
pub max_latency: String,
|
||||
pub max_latency_value: Option<u64>,
|
||||
pub min_latency: String,
|
||||
@@ -840,21 +842,6 @@ pub struct DashboardCacheCardView {
|
||||
pub average_first_token_latency_ms: String,
|
||||
}
|
||||
|
||||
/// Dashboard 凭据用量摘要。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCredentialUsageView {
|
||||
pub id: String,
|
||||
pub display_name: String,
|
||||
pub plan_type: Option<String>,
|
||||
pub tokens: String,
|
||||
pub tokens_value: Option<u64>,
|
||||
pub last_used: String,
|
||||
pub provider: String,
|
||||
pub availability: String,
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -1012,18 +999,6 @@ pub struct DashboardDesktopReleaseView {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latest_build: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub published_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub minimum_system_version: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub hardware_requirements: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub download_url: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub download_size: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub signature_present: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
@@ -1057,14 +1032,10 @@ pub struct DashboardDataView {
|
||||
pub health_timeline: HealthTimelineView,
|
||||
pub wire_profiles: Vec<DashboardWireProfileView>,
|
||||
pub account_usage: Vec<DashboardAccountUsageView>,
|
||||
pub credential_usage: Vec<DashboardCredentialUsageView>,
|
||||
pub usage_records: Vec<UsageRecordView>,
|
||||
pub pool_summary: DashboardPoolSummaryView,
|
||||
pub capacity_info: DashboardCapacityInfoView,
|
||||
pub rotation_strategy: String,
|
||||
pub logical_requests: RequestMetricsView,
|
||||
pub attempts: AttemptMetricsView,
|
||||
pub costs: Vec<CostView>,
|
||||
}
|
||||
|
||||
/// 用量汇总响应数据。
|
||||
@@ -1203,6 +1174,7 @@ pub struct UsageInsightsOverviewView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DiagnosticItemView {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -1210,6 +1182,7 @@ pub struct DiagnosticItemView {
|
||||
pub error_rate: f64,
|
||||
pub request_share: f64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub estimated_cost: Option<String>,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
@@ -1232,6 +1205,7 @@ pub struct OpsErrorView {
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub kind: String,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
@@ -1791,6 +1765,7 @@ fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView {
|
||||
authentication_kind: record.provider_account_authentication_kind,
|
||||
account_id: record.provider_account_ref,
|
||||
account_email: record.provider_account_email,
|
||||
account_name: record.provider_account_name,
|
||||
route: record.endpoint,
|
||||
model,
|
||||
requested_model: Some(record.requested_model_id),
|
||||
@@ -1869,6 +1844,11 @@ fn provider_metadata_fields(value: Option<&str>) -> BTreeMap<String, Value> {
|
||||
|
||||
fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptView {
|
||||
let occurred_at = attempt.occurred_at;
|
||||
let credential_name = attempt
|
||||
.provider_account_name
|
||||
.as_ref()
|
||||
.or(attempt.provider_account_ref.as_ref())
|
||||
.cloned();
|
||||
UsageAttemptView {
|
||||
id: attempt.id,
|
||||
attempt_index: attempt.attempt_index,
|
||||
@@ -1902,8 +1882,11 @@ fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptView {
|
||||
total_tokens: attempt.total_tokens,
|
||||
first_token_ms: None,
|
||||
latency_ms: attempt.latency_ms,
|
||||
credential_name: attempt.provider_account_ref,
|
||||
account_email: None,
|
||||
credential_name,
|
||||
account_id: attempt.provider_account_ref,
|
||||
account_name: attempt.provider_account_name,
|
||||
account_email: attempt.provider_account_email,
|
||||
authentication_kind: attempt.provider_account_authentication_kind,
|
||||
started_at: occurred_at,
|
||||
completed_at: Some(occurred_at),
|
||||
}
|
||||
@@ -1948,14 +1931,21 @@ fn usage_page_view(
|
||||
}
|
||||
|
||||
fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
let account_label = error
|
||||
.provider_account_email
|
||||
.as_ref()
|
||||
.or(error.provider_account_name.as_ref())
|
||||
.or(error.provider_account_ref.as_ref())
|
||||
.cloned();
|
||||
OpsErrorView {
|
||||
id: error.event_id,
|
||||
request_id: error.request_id,
|
||||
client_api_key_id: error.client_api_key_ref,
|
||||
kind: error.operation.clone(),
|
||||
kind: error.component.clone(),
|
||||
provider: error.provider_kind,
|
||||
authentication_kind: error.provider_account_authentication_kind,
|
||||
account_id: error.provider_account_ref,
|
||||
route: error.operation,
|
||||
route: error.endpoint.unwrap_or_else(|| "—".to_owned()),
|
||||
model: error.upstream_model_id,
|
||||
client_status_code: error.client_status_code.map(i64::from),
|
||||
upstream_status_code: error.upstream_status_code.map(i64::from),
|
||||
@@ -1970,7 +1960,7 @@ fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
source: error.source,
|
||||
component: error.component,
|
||||
attempt_id: None,
|
||||
account_label: None,
|
||||
account_label,
|
||||
},
|
||||
created_at: error.occurred_at,
|
||||
created_at_display: china_datetime(&error.occurred_at),
|
||||
@@ -2015,8 +2005,6 @@ fn trend_point_view(point: domain::TrendPoint) -> TrendPointView {
|
||||
errors_value: point.service_failure_count,
|
||||
latency: display_duration(point.average_latency_ms),
|
||||
latency_value: point.average_latency_ms,
|
||||
first_token_latency: display_duration(point.average_first_token_latency_ms),
|
||||
first_token_latency_value: point.average_first_token_latency_ms,
|
||||
max_latency: display_duration(point.metrics.max_latency_ms),
|
||||
max_latency_value: point.metrics.max_latency_ms,
|
||||
min_latency: display_duration(point.metrics.min_latency_ms),
|
||||
@@ -2167,12 +2155,6 @@ fn wire_profile_view(profile: domain::DashboardWireProfile) -> DashboardWireProf
|
||||
checked_at: release.checked_at,
|
||||
latest_version: release.latest_version,
|
||||
latest_build: release.latest_build,
|
||||
published_at: release.published_at,
|
||||
minimum_system_version: release.minimum_system_version,
|
||||
hardware_requirements: release.hardware_requirements,
|
||||
download_url: release.download_url,
|
||||
download_size: release.download_size,
|
||||
signature_present: release.signature_present,
|
||||
error: release.error,
|
||||
}),
|
||||
}
|
||||
@@ -2198,19 +2180,6 @@ fn relative_time(value: Option<DateTime<Utc>>, now: DateTime<Utc>) -> String {
|
||||
format!("{} 天前", elapsed.num_days())
|
||||
}
|
||||
|
||||
fn rotation_strategy_name(
|
||||
strategy: gateway_admin::model::settings::RotationStrategy,
|
||||
) -> &'static str {
|
||||
match strategy {
|
||||
gateway_admin::model::settings::RotationStrategy::Smart => "smart",
|
||||
gateway_admin::model::settings::RotationStrategy::QuotaResetPriority => {
|
||||
"quota_reset_priority"
|
||||
}
|
||||
gateway_admin::model::settings::RotationStrategy::RoundRobin => "round_robin",
|
||||
gateway_admin::model::settings::RotationStrategy::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> DashboardDataView {
|
||||
let domain::DashboardResult {
|
||||
observation,
|
||||
@@ -2228,14 +2197,12 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
|
||||
let domain::DashboardObservation {
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
provider_accounts,
|
||||
trend: _,
|
||||
account_usage,
|
||||
recent_requests,
|
||||
..
|
||||
} = observation;
|
||||
let mut account_usage_views = Vec::with_capacity(account_usage.len());
|
||||
let mut credential_usage_views = Vec::with_capacity(account_usage.len());
|
||||
for credential in account_usage {
|
||||
account_usage_views.push(DashboardAccountUsageView {
|
||||
id: credential.account_id.clone(),
|
||||
@@ -2261,21 +2228,6 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
|
||||
quota_used_percent: credential.quota_used_percent,
|
||||
last_used: relative_time(credential.last_used_at, range.end),
|
||||
});
|
||||
credential_usage_views.push(DashboardCredentialUsageView {
|
||||
id: credential.account_id,
|
||||
display_name: credential.email.unwrap_or(credential.name),
|
||||
plan_type: credential.plan_type,
|
||||
tokens: credential
|
||||
.total_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
tokens_value: credential.total_tokens,
|
||||
last_used: credential
|
||||
.last_used_at
|
||||
.map_or_else(|| "-".to_owned(), |value| china_datetime(&value)),
|
||||
provider: credential.provider_kind,
|
||||
availability: credential.availability,
|
||||
request_count: credential.request_count,
|
||||
});
|
||||
}
|
||||
DashboardDataView {
|
||||
cards: DashboardCardsView {
|
||||
@@ -2315,7 +2267,6 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
|
||||
health_timeline: health_timeline_view(health_timeline),
|
||||
wire_profiles: wire_profiles.into_iter().map(wire_profile_view).collect(),
|
||||
account_usage: account_usage_views,
|
||||
credential_usage: credential_usage_views,
|
||||
usage_records: recent_requests.into_iter().map(usage_record_view).collect(),
|
||||
pool_summary: DashboardPoolSummaryView {
|
||||
total: provider_accounts.total,
|
||||
@@ -2332,10 +2283,7 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
|
||||
used_slots: capacity.used_slots,
|
||||
available_slots: capacity.available_slots,
|
||||
},
|
||||
rotation_strategy: rotation_strategy_name(rotation_strategy).to_owned(),
|
||||
logical_requests: request_metrics_view(&requests),
|
||||
attempts: attempt_metrics_view(&attempts),
|
||||
costs: cost_views(&attempts.costs),
|
||||
rotation_strategy: rotation_strategy.as_str().to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2520,6 +2468,7 @@ fn diagnostics_view(
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|item| DiagnosticItemView {
|
||||
key: item.key,
|
||||
name: if item.name == "__none__" {
|
||||
"未知".to_owned()
|
||||
} else {
|
||||
@@ -2531,6 +2480,7 @@ fn diagnostics_view(
|
||||
error_rate: item.error_rate,
|
||||
request_share: item.request_share,
|
||||
average_latency_ms: item.average_latency_ms,
|
||||
latency_p95_ms: item.latency_p95_ms,
|
||||
estimated_cost: item.estimated_cost.as_ref().map(ToString::to_string),
|
||||
attempt_count: item.attempt_count,
|
||||
total_tokens: item.total_tokens,
|
||||
|
||||
@@ -72,10 +72,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
if i64::try_from(self.request_interval_ms).is_err() {
|
||||
return Err(WireValidationError::new("requestIntervalMs"));
|
||||
}
|
||||
if !matches!(
|
||||
self.rotation_strategy.as_str(),
|
||||
"smart" | "quota_reset_priority" | "round_robin" | "sticky"
|
||||
) {
|
||||
if RotationStrategy::parse(&self.rotation_strategy).is_none() {
|
||||
return Err(WireValidationError::new("rotationStrategy"));
|
||||
}
|
||||
Ok(())
|
||||
@@ -91,7 +88,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
max_concurrent_per_account: u32::try_from(self.max_concurrent_per_account)
|
||||
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
|
||||
request_interval_ms: self.request_interval_ms,
|
||||
rotation_strategy: parse_rotation_strategy(&self.rotation_strategy)
|
||||
rotation_strategy: RotationStrategy::parse(&self.rotation_strategy)
|
||||
.ok_or_else(|| WireValidationError::new("rotationStrategy"))?,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsUsageRetentionOverflow"))?,
|
||||
@@ -111,7 +108,7 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
rotation_strategy: rotation_strategy_name(settings.rotation_strategy).to_owned(),
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
@@ -319,25 +316,6 @@ fn wire_model_mappings(mappings: DomainModelMappings) -> ModelMappings {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_rotation_strategy(value: &str) -> Option<RotationStrategy> {
|
||||
match value {
|
||||
"smart" => Some(RotationStrategy::Smart),
|
||||
"quota_reset_priority" => Some(RotationStrategy::QuotaResetPriority),
|
||||
"round_robin" => Some(RotationStrategy::RoundRobin),
|
||||
"sticky" => Some(RotationStrategy::Sticky),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const fn rotation_strategy_name(strategy: RotationStrategy) -> &'static str {
|
||||
match strategy {
|
||||
RotationStrategy::Smart => "smart",
|
||||
RotationStrategy::QuotaResetPriority => "quota_reset_priority",
|
||||
RotationStrategy::RoundRobin => "round_robin",
|
||||
RotationStrategy::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= max_len
|
||||
|
||||
@@ -9,7 +9,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::{HeaderName, HeaderValue, Method, StatusCode};
|
||||
use axum::http::{HeaderName, HeaderValue, Method, Request, StatusCode};
|
||||
use axum::routing::get;
|
||||
use gateway_admin::AdminServices;
|
||||
use gateway_core::engine::execution::ExecutionService;
|
||||
@@ -166,10 +166,32 @@ pub fn initialize(
|
||||
Duration::from_secs(seconds),
|
||||
));
|
||||
}
|
||||
// Trace 必须在 SetRequestId 内侧,span 才能捕获本服务生成的 request_id;
|
||||
// 默认 DEBUG span 会被 info 日志过滤器丢弃,这里显式用 info_span。
|
||||
let trace_layer = TraceLayer::new_for_http().make_span_with({
|
||||
let request_id_header = request_id_header.clone();
|
||||
move |request: &Request<_>| {
|
||||
let request_id = request
|
||||
.headers()
|
||||
.get(&request_id_header)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
tracing::info_span!(
|
||||
"request",
|
||||
request_id = %request_id,
|
||||
method = %request.method().as_str(),
|
||||
uri = %request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(axum::http::uri::PathAndQuery::as_str)
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
});
|
||||
let router = router
|
||||
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
|
||||
.layer(trace_layer)
|
||||
.layer(SetRequestIdLayer::new(request_id_header, MakeRequestUuid))
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state);
|
||||
Ok(ApiBundle { router })
|
||||
}
|
||||
|
||||
@@ -23,9 +23,9 @@ use gateway_admin::{
|
||||
NewClientKey, SetClientKeyEnabled, UpdateClientKey,
|
||||
},
|
||||
observability::{
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsErrorPage,
|
||||
OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter, UsageOverview,
|
||||
UsagePage, UsageQuery, UsageRecord,
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsError,
|
||||
OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageRecord,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
|
||||
@@ -79,6 +79,9 @@ pub(super) struct AdminTestFixture {
|
||||
pub auth: Arc<MemoryAuthStore>,
|
||||
pub settings: Arc<MemorySettingsStore>,
|
||||
pub usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
pub usage_detail: Arc<Mutex<Option<UsageDetail>>>,
|
||||
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
}
|
||||
|
||||
@@ -93,9 +96,15 @@ impl AdminTestFixture {
|
||||
let settings = Arc::new(MemorySettingsStore::new(api_key));
|
||||
let client_keys = Arc::new(MemoryClientKeyStore);
|
||||
let usage_records = Arc::new(Mutex::new(Vec::new()));
|
||||
let usage_detail = Arc::new(Mutex::new(None));
|
||||
let diagnostics = Arc::new(Mutex::new(Vec::new()));
|
||||
let ops_errors = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
usage_detail: Arc::clone(&usage_detail),
|
||||
diagnostics: Arc::clone(&diagnostics),
|
||||
ops_errors: Arc::clone(&ops_errors),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
@@ -128,6 +137,9 @@ impl AdminTestFixture {
|
||||
auth,
|
||||
settings,
|
||||
usage_records,
|
||||
usage_detail,
|
||||
diagnostics,
|
||||
ops_errors,
|
||||
dashboard_observation,
|
||||
}
|
||||
}
|
||||
@@ -396,6 +408,9 @@ impl ClientKeyStore for MemoryClientKeyStore {
|
||||
|
||||
struct UnusedStore {
|
||||
usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
usage_detail: Arc<Mutex<Option<UsageDetail>>>,
|
||||
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
}
|
||||
|
||||
@@ -546,7 +561,11 @@ impl ObservabilityStore for UnusedStore {
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
||||
Err(unavailable("usage detail"))
|
||||
self.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("usage detail"))
|
||||
}
|
||||
|
||||
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
|
||||
@@ -559,11 +578,16 @@ impl ObservabilityStore for UnusedStore {
|
||||
_: UsageFilter,
|
||||
_: DiagnosticDimension,
|
||||
) -> AdminStoreResult<Vec<DiagnosticObservation>> {
|
||||
Err(unavailable("usage diagnostics"))
|
||||
Ok(self.diagnostics.lock().expect("diagnostics").clone())
|
||||
}
|
||||
|
||||
async fn list_ops_errors(&self, _: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
Err(unavailable("ops errors"))
|
||||
let items = self.ops_errors.lock().expect("ops errors").clone();
|
||||
Ok(OpsErrorPage {
|
||||
total: items.len() as u64,
|
||||
items,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -167,8 +167,6 @@ mod response {
|
||||
errors_value: 0,
|
||||
latency: "1 ms".to_owned(),
|
||||
latency_value: Some(1),
|
||||
first_token_latency: "1 ms".to_owned(),
|
||||
first_token_latency_value: Some(1),
|
||||
max_latency: "1 ms".to_owned(),
|
||||
max_latency_value: Some(1),
|
||||
min_latency: "1 ms".to_owned(),
|
||||
@@ -334,6 +332,342 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{UsageAttempt, UsageDetail};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
fixture
|
||||
.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
"req_detail",
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
"alpha@example.invalid",
|
||||
"oauth",
|
||||
now,
|
||||
),
|
||||
attempts: vec![UsageAttempt {
|
||||
source: "ops_event".to_owned(),
|
||||
id: "ops_detail".to_owned(),
|
||||
attempt_index: 1,
|
||||
component: "routing".to_owned(),
|
||||
operation: "fallback".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_snap_b".to_owned()),
|
||||
provider_account_name: Some("Snapshot Beta".to_owned()),
|
||||
provider_account_email: Some("beta@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-b".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
upstream_send_state: Some("sent".to_owned()),
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Failed,
|
||||
downstream_committed: false,
|
||||
status_code: Some(429),
|
||||
provider_error_code: Some("rate_limit".to_owned()),
|
||||
failure_kind: Some("rate_limited".to_owned()),
|
||||
retry_after_ms: Some(1_000),
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(120),
|
||||
message: Some("limited".to_owned()),
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
total_tokens: None,
|
||||
cost_source: Some("unavailable".to_owned()),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
occurred_at: now,
|
||||
}],
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records/detail?id=req_detail")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_detail_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("usage detail request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage detail response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage detail body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage detail JSON");
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"accountId": value["data"]["accountId"],
|
||||
"accountName": value["data"]["accountName"],
|
||||
"accountEmail": value["data"]["accountEmail"],
|
||||
"authenticationKind": value["data"]["authenticationKind"],
|
||||
"attemptCredentialName": value["data"]["attempts"][0]["credentialName"],
|
||||
"attemptAccountId": value["data"]["attempts"][0]["accountId"],
|
||||
"attemptAccountName": value["data"]["attempts"][0]["accountName"],
|
||||
"attemptAccountEmail": value["data"]["attempts"][0]["accountEmail"],
|
||||
"attemptAuthenticationKind": value["data"]["attempts"][0]["authenticationKind"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"accountId": "acct_snap_a",
|
||||
"accountName": "Snapshot Alpha",
|
||||
"accountEmail": "alpha@example.invalid",
|
||||
"authenticationKind": "oauth",
|
||||
"attemptCredentialName": "Snapshot Beta",
|
||||
"attemptAccountId": "acct_snap_b",
|
||||
"attemptAccountName": "Snapshot Beta",
|
||||
"attemptAccountEmail": "beta@example.invalid",
|
||||
"attemptAuthenticationKind": "api_key",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::OpsError;
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.ops_errors
|
||||
.lock()
|
||||
.expect("ops errors")
|
||||
.push(OpsError {
|
||||
source: "model_request".to_owned(),
|
||||
event_id: "err_snapshot".to_owned(),
|
||||
request_id: Some("req_err".to_owned()),
|
||||
attempt_index: Some(1),
|
||||
client_api_key_ref: Some("key_err".to_owned()),
|
||||
component: "model_request".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_err".to_owned()),
|
||||
provider_account_name: None,
|
||||
provider_account_email: Some("err@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-err".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
failure_kind: "upstream_error".to_owned(),
|
||||
client_status_code: Some(502),
|
||||
upstream_status_code: Some(502),
|
||||
provider_error_code: Some("upstream".to_owned()),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
occurrence_count: 1,
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/operations/errors")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_ops_errors_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("ops errors request"),
|
||||
)
|
||||
.await
|
||||
.expect("ops errors response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("ops errors body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("ops errors JSON");
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"provider": value["data"]["items"][0]["provider"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"kind": value["data"]["items"][0]["kind"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountLabel": value["data"]["items"][0]["metadata"]["accountLabel"],
|
||||
"clientStatusCode": value["data"]["items"][0]["clientStatusCode"],
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"provider": "openai",
|
||||
"authenticationKind": "api_key",
|
||||
"kind": "model_request",
|
||||
"accountId": "acct_err",
|
||||
"accountLabel": "err@example.invalid",
|
||||
"clientStatusCode": 502,
|
||||
"route": "/v1/responses",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_admin::model::observability::{CostCoverage, DiagnosticObservation};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.diagnostics
|
||||
.lock()
|
||||
.expect("diagnostics")
|
||||
.push(DiagnosticObservation {
|
||||
key: "acct_diag".to_owned(),
|
||||
name: "diag@example.invalid".to_owned(),
|
||||
request_count: 2,
|
||||
success_count: 2,
|
||||
failure_count: 0,
|
||||
attempt_count: 2,
|
||||
total_tokens: 200,
|
||||
average_latency_ms: Some(100),
|
||||
latency_p95_ms: Some(3800),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/insights/diagnostics?dimension=account")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_diagnostics_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("diagnostics request"),
|
||||
)
|
||||
.await
|
||||
.expect("diagnostics response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["items"][0]["key"],
|
||||
&value["data"]["items"][0]["name"],
|
||||
&value["data"]["items"][0]["latencyP95Ms"],
|
||||
&value["data"]["dimension"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("acct_diag"),
|
||||
&serde_json::json!("diag@example.invalid"),
|
||||
&serde_json::json!(3800),
|
||||
&serde_json::json!("account"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
fn usage_record_with_account(
|
||||
id: &str,
|
||||
account_id: &str,
|
||||
name: &str,
|
||||
email: &str,
|
||||
authentication_kind: &str,
|
||||
started_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> gateway_admin::model::observability::UsageRecord {
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
|
||||
UsageRecord {
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_detail".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "coding".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some(account_id.to_owned()),
|
||||
provider_account_name: Some(name.to_owned()),
|
||||
provider_account_email: Some(email.to_owned()),
|
||||
provider_account_authentication_kind: Some(authentication_kind.to_owned()),
|
||||
upstream_model_id: Some("upstream-model".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: Some(started_at),
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
transport_decision_wait_ms: None,
|
||||
connect_ms: None,
|
||||
headers_ms: None,
|
||||
first_event_ms: None,
|
||||
first_reasoning_ms: None,
|
||||
first_text_ms: None,
|
||||
first_token_ms: None,
|
||||
provider_processing_ms: None,
|
||||
latency_ms: None,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
image_generation_requested: false,
|
||||
image_generation_succeeded: None,
|
||||
started_at,
|
||||
deadline_at: started_at + chrono::Duration::seconds(30),
|
||||
completed_at: Some(started_at),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
use axum::{
|
||||
@@ -401,9 +735,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "grok-4.5".to_owned(),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_ref: None,
|
||||
provider_account_name: None,
|
||||
provider_account_email: None,
|
||||
provider_account_ref: Some("acct_snapshot".to_owned()),
|
||||
provider_account_name: Some("Snapshot Alpha".to_owned()),
|
||||
provider_account_email: Some("alpha@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("oauth".to_owned()),
|
||||
upstream_model_id: Some("grok-4.5".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
@@ -515,6 +849,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountName": value["data"]["items"][0]["accountName"],
|
||||
"accountEmail": value["data"]["items"][0]["accountEmail"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
@@ -542,6 +879,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"serviceTier": "priority",
|
||||
"accountId": "acct_snapshot",
|
||||
"accountName": "Snapshot Alpha",
|
||||
"accountEmail": "alpha@example.invalid",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
|
||||
@@ -190,9 +190,26 @@ impl fmt::Debug for ContinuationBinding {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable")]
|
||||
pub struct NativeContinuationStoreError;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable: {detail}")]
|
||||
pub struct NativeContinuationStoreError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
impl NativeContinuationStoreError {
|
||||
/// 构造带可排查细节的亲和存储错误;`detail` 不应包含 response handle。
|
||||
#[must_use]
|
||||
pub fn new(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn detail(&self) -> &str {
|
||||
&self.detail
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失的 previous-response 亲和存储端口。
|
||||
///
|
||||
|
||||
@@ -698,6 +698,17 @@ impl RotationStrategy {
|
||||
Self::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"smart" => Some(Self::Smart),
|
||||
"quota_reset_priority" => Some(Self::QuotaResetPriority),
|
||||
"round_robin" => Some(Self::RoundRobin),
|
||||
"sticky" => Some(Self::Sticky),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `runtime_settings` 冻结到一次请求计划的账号调度策略。
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::net::IpAddr;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
@@ -39,7 +40,7 @@ use crate::routing::{
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
const REDIS_COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClientTransport {
|
||||
@@ -173,6 +174,22 @@ pub enum ProviderCircuitDecision {
|
||||
#[error("provider circuit store is unavailable")]
|
||||
pub struct ProviderCircuitError;
|
||||
|
||||
/// Provider circuit 的可重建协调策略;由 Core 拥有并交给 Store adapter 执行。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderCircuitPolicy {
|
||||
pub failure_threshold: NonZeroU32,
|
||||
pub open_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for ProviderCircuitPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
failure_threshold: NonZeroU32::new(3).unwrap_or(NonZeroU32::MIN),
|
||||
open_duration: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCircuitPort: Send + Sync {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
@@ -254,10 +271,20 @@ impl DefaultExecutionService {
|
||||
let continuation = match request.metadata.previous_response_id.as_ref() {
|
||||
Some(previous) => {
|
||||
let resolve = self.continuation.resolve(previous).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(resolve, timeout);
|
||||
let pin = select_biased! {
|
||||
result = resolve => result.ok().flatten(),
|
||||
result = resolve => match result {
|
||||
Ok(pin) => pin,
|
||||
Err(error) => {
|
||||
tracing::debug!(
|
||||
request_id = request_id.as_str(),
|
||||
%error,
|
||||
"Continuation affinity 查询失败,退化为外部续接"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
_ = timeout => None,
|
||||
};
|
||||
match pin {
|
||||
@@ -393,7 +420,7 @@ impl DefaultExecutionService {
|
||||
provider_kind: &ProviderKind,
|
||||
) -> Result<RoutingContext, GatewayError> {
|
||||
let decision = self.circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
@@ -933,17 +960,28 @@ async fn record_native_continuation(
|
||||
continuation: &dyn NativeContinuationPort,
|
||||
pin: NativeContinuationPin,
|
||||
) {
|
||||
let provider = pin.provider().as_str().to_owned();
|
||||
let account = pin.account().as_str().to_owned();
|
||||
let record = continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
result = record => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "Continuation affinity 写入失败,后续请求将退化为外部续接");
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
%error,
|
||||
"Continuation affinity 写入失败,后续请求将退化为外部续接"
|
||||
);
|
||||
}
|
||||
},
|
||||
_ = timeout => {
|
||||
tracing::warn!("Continuation affinity 后台写入超时,已丢弃本次亲和记录");
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
"Continuation affinity 后台写入超时,已丢弃本次亲和记录"
|
||||
);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,13 +244,8 @@ async fn compile_runtime_snapshot(
|
||||
}
|
||||
|
||||
let model_mappings = facts.settings.model_mappings;
|
||||
let rotation_strategy = match facts.settings.rotation_strategy.as_str() {
|
||||
"smart" => RotationStrategy::Smart,
|
||||
"quota_reset_priority" => RotationStrategy::QuotaResetPriority,
|
||||
"round_robin" => RotationStrategy::RoundRobin,
|
||||
"sticky" => RotationStrategy::Sticky,
|
||||
_ => return Err(RuntimeSnapshotCompileError::InvalidData),
|
||||
};
|
||||
let rotation_strategy = RotationStrategy::parse(facts.settings.rotation_strategy.as_str())
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
let selection_policy = AccountSelectionPolicy::new(
|
||||
rotation_strategy,
|
||||
NonZeroU32::new(facts.settings.max_concurrent_per_account)
|
||||
|
||||
@@ -56,6 +56,19 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotation_strategy_parse_should_round_trip_stable_wire_values() {
|
||||
for strategy in [
|
||||
RotationStrategy::Smart,
|
||||
RotationStrategy::QuotaResetPriority,
|
||||
RotationStrategy::RoundRobin,
|
||||
RotationStrategy::Sticky,
|
||||
] {
|
||||
assert_eq!(RotationStrategy::parse(strategy.as_str()), Some(strategy));
|
||||
}
|
||||
assert_eq!(RotationStrategy::parse("unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plaintext_credential_debug_should_redact_values() {
|
||||
let mut object = Map::new();
|
||||
|
||||
@@ -330,7 +330,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
Arc::new(redis::RedisProviderCircuitRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
redis::ProviderCircuitPolicy::default(),
|
||||
gateway_core::engine::execution::ProviderCircuitPolicy::default(),
|
||||
)?);
|
||||
// Continuation affinity 是下一轮请求的路由事实,Core 必须直接等待 Redis 确认。
|
||||
let continuation: Arc<dyn gateway_core::engine::continuation::NativeContinuationPort> =
|
||||
@@ -613,7 +613,7 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
refresh_concurrency: command.refresh_concurrency,
|
||||
max_concurrent_per_account: command.max_concurrent_per_account,
|
||||
request_interval_ms: command.request_interval_ms,
|
||||
rotation_strategy: admin_rotation_name(command.rotation_strategy).to_owned(),
|
||||
rotation_strategy: command.rotation_strategy.as_str().to_owned(),
|
||||
model_mappings: store_model_mappings(command.model_mappings),
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
@@ -712,19 +712,14 @@ impl AdminSettingsStoreAdapter {
|
||||
fn admin_runtime_settings(
|
||||
settings: postgres::RuntimeSettings,
|
||||
) -> AdminStoreResult<AdminRuntimeSettings> {
|
||||
let rotation_strategy = match settings.rotation_strategy.as_str() {
|
||||
"smart" => AdminRotationStrategy::Smart,
|
||||
"quota_reset_priority" => AdminRotationStrategy::QuotaResetPriority,
|
||||
"round_robin" => AdminRotationStrategy::RoundRobin,
|
||||
"sticky" => AdminRotationStrategy::Sticky,
|
||||
_ => {
|
||||
return Err(AdminStoreError::new(
|
||||
let rotation_strategy = AdminRotationStrategy::parse(settings.rotation_strategy.as_str())
|
||||
.ok_or_else(|| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
"runtime settings",
|
||||
"rotation strategy is invalid",
|
||||
));
|
||||
}
|
||||
};
|
||||
)
|
||||
})?;
|
||||
let model_mappings = settings
|
||||
.model_mappings
|
||||
.into_iter()
|
||||
@@ -768,15 +763,6 @@ fn store_model_mappings(mappings: ModelMappings) -> std::collections::BTreeMap<S
|
||||
.collect()
|
||||
}
|
||||
|
||||
const fn admin_rotation_name(strategy: AdminRotationStrategy) -> &'static str {
|
||||
match strategy {
|
||||
AdminRotationStrategy::Smart => "smart",
|
||||
AdminRotationStrategy::QuotaResetPriority => "quota_reset_priority",
|
||||
AdminRotationStrategy::RoundRobin => "round_robin",
|
||||
AdminRotationStrategy::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn store_revision(revision: AdminRevision) -> AdminStoreResult<Revision> {
|
||||
Revision::new(revision.get()).map_err(|error| admin_store_error("config revision", error))
|
||||
}
|
||||
|
||||
@@ -414,12 +414,18 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
reasoning_preset, request_kind, subagent_kind, compact,
|
||||
image_generation_requested, started_at, deadline_at,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
upstream_model_id, upstream_transport, http_version,
|
||||
attempt_count, upstream_send_state
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10::inet, $11, $12,
|
||||
$13, $14, $15, $16, $17, $18, $19,
|
||||
$20, $21, $22, $23, $24, $25, 1, 'not_sent'
|
||||
$20, $21, $22,
|
||||
(select name from provider_accounts where id = $21),
|
||||
(select email from provider_accounts where id = $21),
|
||||
(select authentication_kind from provider_accounts where id = $21),
|
||||
$23, $24, $25, 1, 'not_sent'
|
||||
)",
|
||||
)
|
||||
.bind(request.id)
|
||||
@@ -466,6 +472,15 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
set provider_kind = $2,
|
||||
provider_account_id = $3,
|
||||
provider_account_ref = $4,
|
||||
provider_account_name_snapshot = (
|
||||
select name from provider_accounts where id = $3
|
||||
),
|
||||
provider_account_email_snapshot = (
|
||||
select email from provider_accounts where id = $3
|
||||
),
|
||||
provider_account_authentication_kind_snapshot = (
|
||||
select authentication_kind from provider_accounts where id = $3
|
||||
),
|
||||
upstream_model_id = $5,
|
||||
upstream_transport = $6,
|
||||
http_version = $7,
|
||||
|
||||
@@ -587,6 +587,9 @@ pub struct UsageAttemptObservation {
|
||||
pub operation: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub upstream_send_state: Option<String>,
|
||||
@@ -636,6 +639,7 @@ pub struct UsageOverview {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DiagnosticObservation {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -643,6 +647,7 @@ pub struct DiagnosticObservation {
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCostTotal>,
|
||||
}
|
||||
@@ -656,8 +661,12 @@ pub struct OpsErrorRecord {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
@@ -1540,6 +1549,9 @@ fn admin_usage_attempt(
|
||||
operation: attempt.operation,
|
||||
provider_kind: attempt.provider_kind,
|
||||
provider_account_ref: attempt.provider_account_ref,
|
||||
provider_account_name: attempt.provider_account_name,
|
||||
provider_account_email: attempt.provider_account_email,
|
||||
provider_account_authentication_kind: attempt.provider_account_authentication_kind,
|
||||
upstream_model_id: attempt.upstream_model_id,
|
||||
upstream_transport: attempt.upstream_transport,
|
||||
upstream_send_state: attempt.upstream_send_state,
|
||||
@@ -1596,6 +1608,7 @@ fn admin_diagnostic_observation(
|
||||
observation: DiagnosticObservation,
|
||||
) -> AdminStoreResult<admin_observability::DiagnosticObservation> {
|
||||
Ok(admin_observability::DiagnosticObservation {
|
||||
key: observation.key,
|
||||
name: observation.name,
|
||||
request_count: observation.request_count,
|
||||
success_count: observation.success_count,
|
||||
@@ -1603,6 +1616,7 @@ fn admin_diagnostic_observation(
|
||||
attempt_count: observation.attempt_count,
|
||||
total_tokens: observation.total_tokens,
|
||||
average_latency_ms: observation.average_latency_ms,
|
||||
latency_p95_ms: observation.latency_p95_ms,
|
||||
cost_coverage: admin_cost_coverage(observation.cost_coverage),
|
||||
costs: admin_currency_costs(observation.costs)?,
|
||||
})
|
||||
@@ -1625,8 +1639,12 @@ fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::OpsError {
|
||||
client_api_key_ref: error.client_api_key_ref,
|
||||
component: error.component,
|
||||
operation: error.operation,
|
||||
endpoint: error.endpoint,
|
||||
provider_kind: error.provider_kind,
|
||||
provider_account_ref: error.provider_account_ref,
|
||||
provider_account_name: error.provider_account_name,
|
||||
provider_account_email: error.provider_account_email,
|
||||
provider_account_authentication_kind: error.provider_account_authentication_kind,
|
||||
upstream_model_id: error.upstream_model_id,
|
||||
upstream_transport: error.upstream_transport,
|
||||
failure_kind: error.failure_kind,
|
||||
@@ -2635,8 +2653,10 @@ const USAGE_RECORD_SELECT: &str =
|
||||
"select mr.id, mr.client_api_key_ref, mr.config_revision, mr.protocol, mr.operation,
|
||||
mr.endpoint, mr.client_transport, mr.requested_model_id,
|
||||
mr.provider_kind, mr.provider_account_ref,
|
||||
pa.name as provider_account_name, pa.email as provider_account_email,
|
||||
pa.authentication_kind as provider_account_authentication_kind,
|
||||
mr.provider_account_name_snapshot as provider_account_name,
|
||||
mr.provider_account_email_snapshot as provider_account_email,
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool,
|
||||
mr.service_tier, mr.provider_observation_json,
|
||||
mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at,
|
||||
@@ -2652,8 +2672,7 @@ const USAGE_RECORD_SELECT: &str =
|
||||
mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact, mr.image_generation_requested,
|
||||
mr.image_generation_succeeded, mr.started_at, mr.deadline_at, mr.completed_at
|
||||
from model_requests mr
|
||||
left join provider_accounts pa on pa.id = mr.provider_account_id";
|
||||
from model_requests mr";
|
||||
|
||||
async fn list_usage_records(
|
||||
pool: &PgPool,
|
||||
@@ -2753,7 +2772,12 @@ async fn usage_record_detail(pool: &PgPool, request_id: &str) -> StoreResult<Usa
|
||||
let request = usage_record_from_row(&row)?;
|
||||
let rows = sqlx::query(
|
||||
"select id, attempt_index, component, operation,
|
||||
provider_kind, provider_account_ref, upstream_model_id,
|
||||
provider_kind, provider_account_ref,
|
||||
provider_account_name_snapshot as provider_account_name,
|
||||
provider_account_email_snapshot as provider_account_email,
|
||||
provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
upstream_model_id,
|
||||
failure_kind, status_code, provider_error_code, retry_after_ms,
|
||||
upstream_request_id, latency_ms, message, created_at
|
||||
from ops_events where model_request_id = $1
|
||||
@@ -2784,6 +2808,9 @@ fn intermediate_attempt_from_row(
|
||||
operation: get(row, "operation")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
provider_account_ref: get(row, "provider_account_ref")?,
|
||||
provider_account_name: get(row, "provider_account_name")?,
|
||||
provider_account_email: get(row, "provider_account_email")?,
|
||||
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
upstream_transport: None,
|
||||
upstream_send_state: None,
|
||||
@@ -2818,6 +2845,9 @@ fn final_attempt_from_request(request: &UsageRecord) -> UsageAttemptObservation
|
||||
operation: request.operation.clone(),
|
||||
provider_kind: request.provider_kind.clone(),
|
||||
provider_account_ref: request.provider_account_ref.clone(),
|
||||
provider_account_name: request.provider_account_name.clone(),
|
||||
provider_account_email: request.provider_account_email.clone(),
|
||||
provider_account_authentication_kind: request.provider_account_authentication_kind.clone(),
|
||||
upstream_model_id: request.upstream_model_id.clone(),
|
||||
upstream_transport: request.upstream_transport.clone(),
|
||||
upstream_send_state: Some(request.upstream_send_state.clone()),
|
||||
@@ -2861,6 +2891,8 @@ async fn usage_diagnostics(
|
||||
coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count,
|
||||
coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens,
|
||||
round(avg(mr.latency_ms))::bigint as average_latency_ms,
|
||||
round(percentile_cont(0.95) within group (order by mr.latency_ms))::bigint
|
||||
as latency_p95_ms,
|
||||
count(*) filter (where mr.cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(*) filter (where mr.cost_source = 'calculated')::bigint
|
||||
@@ -2872,7 +2904,7 @@ async fn usage_diagnostics(
|
||||
statement.push_bind(range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
push_diagnostic_fact_filter(&mut statement, dimension);
|
||||
push_usage_filter(&mut statement, filter, "mr");
|
||||
statement.push(" group by dimension_name order by request_count desc, dimension_name limit ");
|
||||
statement.push_bind(DIAGNOSTIC_LIMIT);
|
||||
@@ -2885,6 +2917,7 @@ async fn usage_diagnostics(
|
||||
.iter()
|
||||
.map(|row| {
|
||||
Ok(DiagnosticObservation {
|
||||
key: get(row, "dimension_name")?,
|
||||
name: get(row, "dimension_name")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
@@ -2892,30 +2925,113 @@ async fn usage_diagnostics(
|
||||
attempt_count: unsigned(row, "attempt_count")?,
|
||||
total_tokens: unsigned(row, "total_tokens")?,
|
||||
average_latency_ms: optional_unsigned(row, "average_latency_ms")?,
|
||||
latency_p95_ms: optional_unsigned(row, "latency_p95_ms")?,
|
||||
cost_coverage: coverage_from_row(row)?,
|
||||
costs: Vec::new(),
|
||||
})
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
let mut display_names = match dimension {
|
||||
DiagnosticDimension::Account => {
|
||||
diagnostic_account_display_names(
|
||||
pool,
|
||||
&observations
|
||||
.iter()
|
||||
.map(|item| item.key.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
DiagnosticDimension::ApiKey => {
|
||||
diagnostic_api_key_display_names(
|
||||
pool,
|
||||
&observations
|
||||
.iter()
|
||||
.map(|item| item.key.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
_ => HashMap::new(),
|
||||
};
|
||||
for observation in &mut observations {
|
||||
if let Some(name) = display_names.remove(&observation.key) {
|
||||
observation.name = name;
|
||||
}
|
||||
}
|
||||
let positions = observations
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, item)| (item.name.clone(), index))
|
||||
.map(|(index, item)| (item.key.clone(), index))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let costs = diagnostic_costs(pool, range, filter, dimension_sql).await?;
|
||||
for (name, values) in costs {
|
||||
if let Some(position) = positions.get(&name).copied() {
|
||||
let costs = diagnostic_costs(pool, range, filter, dimension_sql, dimension).await?;
|
||||
for (key, values) in costs {
|
||||
if let Some(position) = positions.get(&key).copied() {
|
||||
observations[position].costs = values;
|
||||
}
|
||||
}
|
||||
Ok(observations)
|
||||
}
|
||||
|
||||
async fn diagnostic_account_display_names(
|
||||
pool: &PgPool,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, String>> {
|
||||
if account_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
let rows = sqlx::query(
|
||||
"select provider_account_ref,
|
||||
coalesce(
|
||||
(array_agg(provider_account_email_snapshot order by started_at desc)
|
||||
filter (where provider_account_email_snapshot is not null))[1],
|
||||
(array_agg(provider_account_name_snapshot order by started_at desc)
|
||||
filter (where provider_account_name_snapshot is not null))[1],
|
||||
provider_account_ref
|
||||
) as display_name
|
||||
from model_requests
|
||||
where provider_account_ref = any($1::text[])
|
||||
group by provider_account_ref",
|
||||
)
|
||||
.bind(account_ids)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load diagnostic account display names"))?;
|
||||
let mut display_names = HashMap::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
display_names.insert(
|
||||
get(&row, "provider_account_ref")?,
|
||||
get(&row, "display_name")?,
|
||||
);
|
||||
}
|
||||
Ok(display_names)
|
||||
}
|
||||
|
||||
async fn diagnostic_api_key_display_names(
|
||||
pool: &PgPool,
|
||||
key_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, String>> {
|
||||
if key_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
let rows = sqlx::query("select id, name from client_api_keys where id = any($1::text[])")
|
||||
.bind(key_ids)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load diagnostic api key display names"))?;
|
||||
let mut display_names = HashMap::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
display_names.insert(get(&row, "id")?, get(&row, "name")?);
|
||||
}
|
||||
Ok(display_names)
|
||||
}
|
||||
|
||||
async fn diagnostic_costs(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
dimension_sql: &'static str,
|
||||
dimension: DiagnosticDimension,
|
||||
) -> StoreResult<HashMap<String, Vec<CurrencyCostTotal>>> {
|
||||
let mut statement = QueryBuilder::<Postgres>::new("select ");
|
||||
statement.push(dimension_sql);
|
||||
@@ -2927,7 +3043,7 @@ async fn diagnostic_costs(
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(range.end);
|
||||
statement.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
push_diagnostic_fact_filter(&mut statement, dimension);
|
||||
push_usage_filter(&mut statement, filter, "mr");
|
||||
statement.push(
|
||||
" group by dimension_name, mr.cost_currency order by dimension_name, mr.cost_currency",
|
||||
@@ -2965,13 +3081,30 @@ fn diagnostic_dimension_sql(dimension: DiagnosticDimension) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn push_diagnostic_fact_filter(
|
||||
statement: &mut QueryBuilder<Postgres>,
|
||||
dimension: DiagnosticDimension,
|
||||
) {
|
||||
if dimension == DiagnosticDimension::Failure {
|
||||
statement.push(" and mr.error_kind is not null");
|
||||
} else {
|
||||
push_completed_usage_fact_filter(statement, "mr");
|
||||
}
|
||||
}
|
||||
|
||||
const OPS_ERRORS_CTE: &str = "with errors as (
|
||||
select 'model_request'::text as source,
|
||||
mr.id as event_id, mr.id as request_id,
|
||||
nullif(mr.attempt_count, 0) as attempt_index,
|
||||
mr.client_api_key_ref, 'model_request'::text as component, mr.operation,
|
||||
mr.endpoint,
|
||||
mr.provider_kind,
|
||||
mr.provider_account_ref, mr.upstream_model_id, mr.upstream_transport,
|
||||
mr.provider_account_ref,
|
||||
mr.provider_account_name_snapshot as provider_account_name,
|
||||
mr.provider_account_email_snapshot as provider_account_email,
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport,
|
||||
coalesce(mr.error_kind, 'failed') as failure_kind,
|
||||
mr.client_status_code, mr.upstream_status_code,
|
||||
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
|
||||
@@ -2983,7 +3116,12 @@ const OPS_ERRORS_CTE: &str = "with errors as (
|
||||
union all
|
||||
select 'ops_event'::text, oe.id, oe.model_request_id, oe.attempt_index,
|
||||
mr.client_api_key_ref, oe.component, oe.operation,
|
||||
null::text as endpoint,
|
||||
oe.provider_kind, oe.provider_account_ref,
|
||||
oe.provider_account_name_snapshot as provider_account_name,
|
||||
oe.provider_account_email_snapshot as provider_account_email,
|
||||
oe.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
oe.upstream_model_id, null::text, oe.failure_kind,
|
||||
null::integer as client_status_code, oe.status_code as upstream_status_code,
|
||||
oe.provider_error_code, mr.client_response_id, oe.upstream_request_id,
|
||||
@@ -2998,7 +3136,9 @@ async fn list_ops_errors(pool: &PgPool, query: OpsErrorQuery) -> StoreResult<Ops
|
||||
let mut statement = QueryBuilder::<Postgres>::new(OPS_ERRORS_CTE);
|
||||
statement.push(
|
||||
" select source, event_id, request_id, attempt_index, client_api_key_ref,
|
||||
component, operation, provider_kind, provider_account_ref,
|
||||
component, operation, endpoint, provider_kind, provider_account_ref,
|
||||
provider_account_name, provider_account_email,
|
||||
provider_account_authentication_kind,
|
||||
upstream_model_id, upstream_transport,
|
||||
failure_kind, client_status_code, upstream_status_code,
|
||||
provider_error_code, client_response_id,
|
||||
@@ -3200,8 +3340,12 @@ fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<OpsErrorRecord
|
||||
client_api_key_ref: get(row, "client_api_key_ref")?,
|
||||
component: get(row, "component")?,
|
||||
operation: get(row, "operation")?,
|
||||
endpoint: get(row, "endpoint")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
provider_account_ref: get(row, "provider_account_ref")?,
|
||||
provider_account_name: get(row, "provider_account_name")?,
|
||||
provider_account_email: get(row, "provider_account_email")?,
|
||||
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
upstream_transport: get(row, "upstream_transport")?,
|
||||
failure_kind: get(row, "failure_kind")?,
|
||||
|
||||
@@ -110,11 +110,16 @@ impl OpsEventRepository for PgOpsEventRepository {
|
||||
"insert into ops_events (
|
||||
id, model_request_id, attempt_index, level, component, operation,
|
||||
provider_kind, provider_account_id, provider_account_ref, upstream_model_id,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
failure_kind, status_code, provider_error_code, retry_after_ms,
|
||||
upstream_request_id, latency_ms, message, occurrence_count, created_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
|
||||
$14, $15, $16, $17, $18, $19
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||
(select name from provider_accounts where id = $8),
|
||||
(select email from provider_accounts where id = $8),
|
||||
(select authentication_kind from provider_accounts where id = $8),
|
||||
$11, $12, $13, $14, $15, $16, $17, $18, $19
|
||||
)",
|
||||
)
|
||||
.bind(event.id)
|
||||
|
||||
@@ -8,6 +8,7 @@ use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
|
||||
use gateway_core::engine::credential::RotationStrategy;
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
};
|
||||
@@ -93,10 +94,7 @@ impl RuntimeSettingsUpdate {
|
||||
|| self.ops_event_retention_days == 0
|
||||
|| self.audit_retention_days == 0
|
||||
|| !valid_model_mappings(&self.model_mappings)
|
||||
|| !matches!(
|
||||
self.rotation_strategy.as_str(),
|
||||
"smart" | "quota_reset_priority" | "round_robin" | "sticky"
|
||||
)
|
||||
|| RotationStrategy::parse(&self.rotation_strategy).is_none()
|
||||
{
|
||||
return Err(StoreError::InvalidData {
|
||||
entity: "runtime settings",
|
||||
|
||||
@@ -92,7 +92,7 @@ impl RedisNativeContinuationRepository {
|
||||
|
||||
fn entry_fingerprint(&self, response_id: &str) -> Result<String, NativeContinuationStoreError> {
|
||||
resource_fingerprint("native continuation", response_id)
|
||||
.map_err(|_| NativeContinuationStoreError)
|
||||
.map_err(|error| NativeContinuationStoreError::new(error.to_string()))
|
||||
}
|
||||
|
||||
fn entry_key(&self, fingerprint: &str) -> String {
|
||||
@@ -123,7 +123,9 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
.arg(self.entry_key(&fingerprint))
|
||||
.query_async::<Option<String>>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| NativeContinuationStoreError)?;
|
||||
.map_err(|error| {
|
||||
NativeContinuationStoreError::new(format!("redis GET failed: {error}"))
|
||||
})?;
|
||||
payload
|
||||
.map(|payload| decode_pin(previous_response_id, &payload))
|
||||
.transpose()
|
||||
@@ -141,15 +143,21 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
let fingerprint = self.entry_fingerprint(pin.previous_response_id().as_str())?;
|
||||
let payload = encode_pin(&pin)?;
|
||||
if payload.len() > MAX_RECORD_BYTES {
|
||||
return Err(NativeContinuationStoreError);
|
||||
return Err(NativeContinuationStoreError::new(format!(
|
||||
"pin payload is too large ({} bytes, limit {MAX_RECORD_BYTES})",
|
||||
payload.len()
|
||||
)));
|
||||
}
|
||||
let now_millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|_| NativeContinuationStoreError)?
|
||||
.map_err(|_| NativeContinuationStoreError::new("system clock predates Unix epoch"))?
|
||||
.as_millis();
|
||||
let now_millis = u64::try_from(now_millis).map_err(|_| NativeContinuationStoreError)?;
|
||||
let ttl_millis = u64::try_from(CONTINUATION_TTL.as_millis())
|
||||
.map_err(|_| NativeContinuationStoreError)?;
|
||||
let now_millis = u64::try_from(now_millis).map_err(|_| {
|
||||
NativeContinuationStoreError::new("system time is outside supported range")
|
||||
})?;
|
||||
let ttl_millis = u64::try_from(CONTINUATION_TTL.as_millis()).map_err(|_| {
|
||||
NativeContinuationStoreError::new("continuation TTL is outside supported range")
|
||||
})?;
|
||||
let cutoff = now_millis.saturating_sub(ttl_millis);
|
||||
let ttl_seconds = CONTINUATION_TTL.as_secs();
|
||||
let mut connection = self.connection.clone();
|
||||
@@ -166,7 +174,11 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
.arg(MAX_INDEX_CLEANUP_PER_RECORD)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| NativeContinuationStoreError)?;
|
||||
.map_err(|error| {
|
||||
NativeContinuationStoreError::new(format!(
|
||||
"redis affinity script failed: {error}"
|
||||
))
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
@@ -177,7 +189,9 @@ fn encode_pin(pin: &NativeContinuationPin) -> Result<String, NativeContinuationS
|
||||
.session_state()
|
||||
.is_some_and(|state| state.provider() != pin.provider().as_str())
|
||||
{
|
||||
return Err(NativeContinuationStoreError);
|
||||
return Err(NativeContinuationStoreError::new(
|
||||
"session state provider does not match pin provider",
|
||||
));
|
||||
}
|
||||
serde_json::to_string(&ContinuationWire {
|
||||
upstream_response_id: pin.upstream_response_id().as_str().to_owned(),
|
||||
@@ -186,17 +200,23 @@ fn encode_pin(pin: &NativeContinuationPin) -> Result<String, NativeContinuationS
|
||||
scope: scope_name(pin.scope()).to_owned(),
|
||||
session_state: pin.session_state().cloned(),
|
||||
})
|
||||
.map_err(|_| NativeContinuationStoreError)
|
||||
.map_err(|error| {
|
||||
NativeContinuationStoreError::new(format!("pin serialization failed: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_pin(
|
||||
previous_response_id: &PreviousResponseId,
|
||||
payload: &str,
|
||||
) -> Result<NativeContinuationPin, NativeContinuationStoreError> {
|
||||
let wire: ContinuationWire =
|
||||
serde_json::from_str(payload).map_err(|_| NativeContinuationStoreError)?;
|
||||
let provider = ProviderKind::new(wire.provider).map_err(|_| NativeContinuationStoreError)?;
|
||||
let account = ProviderAccountId::new(wire.account).map_err(|_| NativeContinuationStoreError)?;
|
||||
let wire: ContinuationWire = serde_json::from_str(payload)
|
||||
.map_err(|error| NativeContinuationStoreError::new(format!("{error}")))?;
|
||||
let provider = ProviderKind::new(wire.provider).map_err(|_| {
|
||||
NativeContinuationStoreError::new("pin payload contains an invalid provider")
|
||||
})?;
|
||||
let account = ProviderAccountId::new(wire.account).map_err(|_| {
|
||||
NativeContinuationStoreError::new("pin payload contains an invalid account id")
|
||||
})?;
|
||||
let upstream_response_id = PreviousResponseId::new(wire.upstream_response_id);
|
||||
let scope = parse_scope(&wire.scope)?;
|
||||
let mut pin = NativeContinuationPin::new(
|
||||
@@ -208,7 +228,9 @@ fn decode_pin(
|
||||
.with_scope(scope);
|
||||
if let Some(state) = wire.session_state {
|
||||
if state.provider() != provider.as_str() {
|
||||
return Err(NativeContinuationStoreError);
|
||||
return Err(NativeContinuationStoreError::new(
|
||||
"pin payload session state provider does not match pin provider",
|
||||
));
|
||||
}
|
||||
pin = pin.with_session_state(state);
|
||||
}
|
||||
@@ -226,6 +248,8 @@ fn parse_scope(scope: &str) -> Result<NativeContinuationScope, NativeContinuatio
|
||||
match scope {
|
||||
"persisted" => Ok(NativeContinuationScope::Persisted),
|
||||
"connection_local" => Ok(NativeContinuationScope::ConnectionLocal),
|
||||
_ => Err(NativeContinuationStoreError),
|
||||
_ => Err(NativeContinuationStoreError::new(
|
||||
"pin payload contains an unsupported continuation scope",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
//! Provider 的可重建 Redis circuit。
|
||||
|
||||
use std::num::NonZeroU32;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::execution::{
|
||||
ProviderCircuitDecision as CoreCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
ProviderCircuitDecision as CoreCircuitDecision, ProviderCircuitError, ProviderCircuitPolicy,
|
||||
ProviderCircuitPort,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
@@ -37,21 +35,6 @@ if open_until > now_ms then return {0, tostring(open_until)} end
|
||||
return {1, '0'}
|
||||
"#;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderCircuitPolicy {
|
||||
pub failure_threshold: NonZeroU32,
|
||||
pub open_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for ProviderCircuitPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
failure_threshold: NonZeroU32::new(3).unwrap_or(NonZeroU32::MIN),
|
||||
open_duration: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProviderCircuitDecision {
|
||||
Allow,
|
||||
|
||||
@@ -17,6 +17,7 @@ mod provider_accounts;
|
||||
mod retention;
|
||||
mod runtime_settings;
|
||||
mod snapshot;
|
||||
mod snapshots;
|
||||
|
||||
pub(super) struct TestDatabase {
|
||||
admin: PgPool,
|
||||
@@ -58,6 +59,12 @@ impl TestDatabase {
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply terminal migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0002_snapshot_provider_account_identity.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply snapshot migration");
|
||||
migration.commit().await.expect("commit terminal migration");
|
||||
Some(Self {
|
||||
admin,
|
||||
@@ -153,19 +160,22 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
.expect("drop migration test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!((first_table_count, migration_count), (8, 1));
|
||||
assert_eq!((first_table_count, migration_count), (8, 2));
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrations_should_leave_transaction_ownership_to_sqlx() {
|
||||
let transaction_statements = [include_str!("../../../../migrations/0001_initial.sql")]
|
||||
.into_iter()
|
||||
.flat_map(str::lines)
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
let transaction_statements = [
|
||||
include_str!("../../../../migrations/0001_initial.sql"),
|
||||
include_str!("../../../../migrations/0002_snapshot_provider_account_identity.sql"),
|
||||
]
|
||||
.into_iter()
|
||||
.flat_map(str::lines)
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
|
||||
assert_eq!(transaction_statements, 0);
|
||||
}
|
||||
|
||||
@@ -441,7 +441,8 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
)
|
||||
.await
|
||||
.expect("admin diagnostics");
|
||||
assert_eq!(diagnostics[0].name, "acct_observe");
|
||||
assert_eq!(diagnostics[0].key, "acct_observe");
|
||||
assert_eq!(diagnostics[0].name, "account@example.invalid");
|
||||
assert_eq!(diagnostics[0].cost_coverage.provider_reported_count, 1);
|
||||
assert_eq!(diagnostics[0].costs[0].amount.as_str(), "1.25");
|
||||
|
||||
@@ -731,7 +732,8 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
)
|
||||
.await
|
||||
.expect("usage diagnostics");
|
||||
assert_eq!(diagnostics[0].name, "acct_observe");
|
||||
assert_eq!(diagnostics[0].key, "acct_observe");
|
||||
assert_eq!(diagnostics[0].name, "account@example.invalid");
|
||||
assert_eq!(diagnostics[0].request_count, 1);
|
||||
assert_eq!(diagnostics[0].costs[0].amount.as_str(), "1.25");
|
||||
|
||||
@@ -790,6 +792,8 @@ async fn seed_observability_facts(
|
||||
client_transport, requested_model_id,
|
||||
provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
attempt_count, upstream_send_state, outcome, client_status_code,
|
||||
input_tokens, output_tokens, total_tokens, cost_source, latency_ms,
|
||||
started_at, deadline_at, completed_at
|
||||
@@ -797,6 +801,7 @@ async fn seed_observability_facts(
|
||||
'req_observe_uncommitted', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe',
|
||||
'acct_observe', 'upstream-model', 'http_sse',
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
1, 'sent', 'succeeded', 200,
|
||||
900, 900, 1800, 'unavailable', 650,
|
||||
$1 - interval '15 minutes', $1 + interval '10 minutes', $1 - interval '14 minutes'
|
||||
@@ -812,6 +817,8 @@ async fn seed_observability_facts(
|
||||
provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, websocket_pool,
|
||||
service_tier,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
attempt_count,
|
||||
upstream_send_state, downstream_committed_at, outcome, client_status_code,
|
||||
upstream_status_code, client_response_id, upstream_request_id, upstream_response_id,
|
||||
@@ -824,7 +831,9 @@ async fn seed_observability_facts(
|
||||
'req_observe_success', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe',
|
||||
'acct_observe', 'upstream-model',
|
||||
'http_sse', 'reuse', 'priority', 1, 'sent', $1 - interval '19 minutes', 'succeeded', 200, 200,
|
||||
'http_sse', 'reuse', 'priority',
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
1, 'sent', $1 - interval '19 minutes', 'succeeded', 200, 200,
|
||||
'resp_observe_success', 'upstream_req_success', 'upstream_resp_success',
|
||||
100, 20, 40, 3, 5, 31, 9, 120, true, true,
|
||||
'provider_reported', 1.25, 'USD', 120, 900,
|
||||
@@ -840,6 +849,8 @@ async fn seed_observability_facts(
|
||||
client_transport, requested_model_id,
|
||||
provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, attempt_count,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
upstream_send_state, outcome, client_status_code, upstream_status_code,
|
||||
error_kind, provider_error_code, error_message, retry_after_ms,
|
||||
input_tokens, cached_tokens, image_generation_requested,
|
||||
@@ -849,7 +860,9 @@ async fn seed_observability_facts(
|
||||
'req_observe_failed', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe',
|
||||
'acct_observe', 'upstream-model',
|
||||
'http_sse', 2, 'sent', 'failed', 502, 429, 'rate_limited', 'rate_limit',
|
||||
'http_sse', 2,
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
'sent', 'failed', 502, 429, 'rate_limited', 'rate_limit',
|
||||
'upstream limited', 1000, 0, 0, true, false, 'unavailable', 700,
|
||||
$1 - interval '10 minutes', $1 + interval '20 minutes', $1 - interval '9 minutes'
|
||||
)",
|
||||
@@ -862,11 +875,15 @@ async fn seed_observability_facts(
|
||||
id, model_request_id, attempt_index, level, component, operation,
|
||||
provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, failure_kind, status_code,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
provider_error_code, retry_after_ms, latency_ms, message, occurrence_count, created_at
|
||||
) values (
|
||||
'ops_observe_retry', 'req_observe_failed', 1, 'warning', 'routing', 'responses',
|
||||
'openai', 'acct_observe', 'acct_observe',
|
||||
'upstream-model', 'rate_limited', 429, 'rate_limit', 1000, 300,
|
||||
'upstream-model', 'rate_limited', 429,
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
'rate_limit', 1000, 300,
|
||||
'first account was limited', 1, $1 - interval '9 minutes 30 seconds'
|
||||
)",
|
||||
)
|
||||
@@ -885,13 +902,17 @@ async fn seed_calculated_billing_facts(
|
||||
id, client_api_key_ref, config_revision, protocol, operation, endpoint,
|
||||
client_transport, requested_model_id, provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, attempt_count,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
upstream_send_state, downstream_committed_at, outcome, client_status_code,
|
||||
input_tokens, output_tokens, cached_tokens, cache_write_tokens, total_tokens, service_tier,
|
||||
cost_source, cost_amount, cost_currency, started_at, deadline_at, completed_at
|
||||
) values (
|
||||
'req_observe_calculated', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe', 'acct_observe', 'gpt-5.5',
|
||||
'http_sse', 1, 'sent', $1 - interval '29 minutes', 'succeeded', 200,
|
||||
'http_sse', 1,
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
'sent', $1 - interval '29 minutes', 'succeeded', 200,
|
||||
800, 200, 0, 0, 1000, 'priority', 'calculated', 1.25, 'USD',
|
||||
$1 - interval '30 minutes', $1 + interval '10 minutes', $1 - interval '29 minutes'
|
||||
)",
|
||||
@@ -904,13 +925,17 @@ async fn seed_calculated_billing_facts(
|
||||
id, client_api_key_ref, config_revision, protocol, operation, endpoint,
|
||||
client_transport, requested_model_id, provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, attempt_count,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
upstream_send_state, outcome, client_status_code, input_tokens, output_tokens,
|
||||
total_tokens, cost_source, cost_amount, cost_currency, started_at, deadline_at,
|
||||
completed_at
|
||||
) values (
|
||||
'req_observe_calculated_uncommitted', 'key_observe', 1, 'openai', 'responses',
|
||||
'/v1/responses', 'http_sse', 'public-model', 'openai', 'acct_observe',
|
||||
'acct_observe', 'gpt-5.5', 'http_sse', 1, 'sent', 'succeeded', 200, 800, 200,
|
||||
'acct_observe', 'gpt-5.5', 'http_sse', 1,
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
'sent', 'succeeded', 200, 800, 200,
|
||||
1000, 'calculated', 1.25, 'USD',
|
||||
$1 - interval '40 minutes', $1 + interval '10 minutes', $1 - interval '39 minutes'
|
||||
)",
|
||||
|
||||
@@ -0,0 +1,948 @@
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{
|
||||
DiagnosticDimension, ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest,
|
||||
ObservabilityPageNumber, ObservabilityPageSize, ObservabilityRange, ObservabilityRepository,
|
||||
OpsErrorFilter, OpsErrorQuery, OpsEvent, OpsEventLevel, OpsEventRepository, PgExecutionStore,
|
||||
PgObservabilityRepository, PgOpsEventRepository, UsageRecordFilter, UsageRecordQuery,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::TestDatabase;
|
||||
|
||||
type ProviderAccountSnapshotRow = (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
);
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_snapshots_should_survive_account_deletion() {
|
||||
let Some(database) = TestDatabase::create("snapshot_request_delete").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
Some("alpha@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_a", started_at),
|
||||
attempt("req_snap_a", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert request with first attempt");
|
||||
finalize_request(&database.pool, "req_snap_a", started_at).await;
|
||||
|
||||
sqlx::query("delete from provider_accounts where id = 'acct_snap_a'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("delete provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
.expect("list usage records after deletion");
|
||||
let record = page
|
||||
.items
|
||||
.iter()
|
||||
.find(|record| record.id == "req_snap_a")
|
||||
.expect("request snapshot record");
|
||||
assert_eq!(
|
||||
(
|
||||
record.provider_account_ref.as_deref(),
|
||||
record.provider_account_name.as_deref(),
|
||||
record.provider_account_email.as_deref(),
|
||||
record.provider_account_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("acct_snap_a"),
|
||||
Some("Snapshot Alpha"),
|
||||
Some("alpha@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
|
||||
let detail = repository
|
||||
.usage_record_detail("req_snap_a")
|
||||
.await
|
||||
.expect("usage record detail after deletion");
|
||||
assert_eq!(
|
||||
(
|
||||
detail.request.provider_account_name.as_deref(),
|
||||
detail.request.provider_account_email.as_deref(),
|
||||
detail
|
||||
.request
|
||||
.provider_account_authentication_kind
|
||||
.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Snapshot Alpha"),
|
||||
Some("alpha@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attempts_should_keep_their_own_account_snapshots() {
|
||||
let Some(database) = TestDatabase::create("snapshot_attempt").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
Some("alpha@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_b",
|
||||
"Snapshot Beta",
|
||||
Some("beta@example.invalid"),
|
||||
"api_key",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_ab", started_at),
|
||||
attempt("req_snap_ab", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert first attempt");
|
||||
PgOpsEventRepository::new(database.pool.clone())
|
||||
.append_ops_event(OpsEvent {
|
||||
id: "ops_snap_a".to_owned(),
|
||||
model_request_id: Some("req_snap_ab".to_owned()),
|
||||
attempt_index: Some(1),
|
||||
level: OpsEventLevel::Warning,
|
||||
component: "routing".to_owned(),
|
||||
operation: "fallback".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_id: Some("acct_snap_a".to_owned()),
|
||||
provider_account_ref: Some("acct_snap_a".to_owned()),
|
||||
upstream_model_id: Some("upstream-a".to_owned()),
|
||||
failure_kind: "rate_limited".to_owned(),
|
||||
status_code: Some(429),
|
||||
provider_error_code: Some("rate_limit".to_owned()),
|
||||
retry_after_ms: Some(1_000),
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(120),
|
||||
message: "first account was limited".to_owned(),
|
||||
occurrence_count: 1,
|
||||
created_at: started_at + chrono::Duration::seconds(1),
|
||||
})
|
||||
.await
|
||||
.expect("append intermediate failure");
|
||||
store
|
||||
.begin_model_request_attempt(attempt("req_snap_ab", 2, "acct_snap_b"))
|
||||
.await
|
||||
.expect("begin second attempt");
|
||||
finalize_request(&database.pool, "req_snap_ab", started_at).await;
|
||||
|
||||
sqlx::query("delete from provider_accounts where id in ('acct_snap_a', 'acct_snap_b')")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let detail = repository
|
||||
.usage_record_detail("req_snap_ab")
|
||||
.await
|
||||
.expect("usage detail with mixed attempts");
|
||||
let intermediate = detail
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|attempt| attempt.source == "ops_event")
|
||||
.expect("intermediate attempt");
|
||||
assert_eq!(
|
||||
(
|
||||
intermediate.provider_account_name.as_deref(),
|
||||
intermediate.provider_account_email.as_deref(),
|
||||
intermediate.provider_account_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Snapshot Alpha"),
|
||||
Some("alpha@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
let final_attempt = detail
|
||||
.attempts
|
||||
.iter()
|
||||
.find(|attempt| attempt.source == "model_request")
|
||||
.expect("final attempt");
|
||||
assert_eq!(
|
||||
(
|
||||
final_attempt.provider_account_name.as_deref(),
|
||||
final_attempt.provider_account_email.as_deref(),
|
||||
final_attempt
|
||||
.provider_account_authentication_kind
|
||||
.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Snapshot Beta"),
|
||||
Some("beta@example.invalid"),
|
||||
Some("api_key"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
detail.request.provider_account_email.as_deref(),
|
||||
Some("beta@example.invalid")
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deletion() {
|
||||
let Some(database) = TestDatabase::create("snapshot_ops_errors").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
Some("alpha@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_c",
|
||||
"Snapshot Charlie",
|
||||
Some("charlie@example.invalid"),
|
||||
"api_key",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_error", started_at),
|
||||
attempt("req_snap_error", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert failed request");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set outcome = 'failed', error_kind = 'upstream_error',
|
||||
error_message = 'snapshot failure', client_status_code = 502,
|
||||
upstream_status_code = 502, completed_at = $1
|
||||
where id = 'req_snap_error'",
|
||||
)
|
||||
.bind(started_at + chrono::Duration::seconds(2))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("mark request failed");
|
||||
PgOpsEventRepository::new(database.pool.clone())
|
||||
.append_ops_event(OpsEvent {
|
||||
id: "ops_snap_probe".to_owned(),
|
||||
model_request_id: None,
|
||||
attempt_index: None,
|
||||
level: OpsEventLevel::Warning,
|
||||
component: "account_probe".to_owned(),
|
||||
operation: "connection_test".to_owned(),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_id: Some("acct_snap_c".to_owned()),
|
||||
provider_account_ref: Some("acct_snap_c".to_owned()),
|
||||
upstream_model_id: Some("grok-test".to_owned()),
|
||||
failure_kind: "auth_failed".to_owned(),
|
||||
status_code: Some(401),
|
||||
provider_error_code: Some("invalid_api_key".to_owned()),
|
||||
retry_after_ms: None,
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(80),
|
||||
message: "probe failed".to_owned(),
|
||||
occurrence_count: 1,
|
||||
created_at: started_at + chrono::Duration::seconds(3),
|
||||
})
|
||||
.await
|
||||
.expect("append probe failure");
|
||||
|
||||
sqlx::query("delete from provider_accounts where id in ('acct_snap_a', 'acct_snap_c')")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let errors = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range: range_around(started_at),
|
||||
filter: OpsErrorFilter::default(),
|
||||
cursor: None,
|
||||
page: ObservabilityPageNumber::new(1).expect("page"),
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("list ops errors after deletion");
|
||||
let request_error = errors
|
||||
.items
|
||||
.iter()
|
||||
.find(|error| error.source == "model_request")
|
||||
.expect("request error");
|
||||
assert_eq!(
|
||||
(
|
||||
request_error.provider_account_name.as_deref(),
|
||||
request_error.provider_account_email.as_deref(),
|
||||
request_error
|
||||
.provider_account_authentication_kind
|
||||
.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Snapshot Alpha"),
|
||||
Some("alpha@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
assert_eq!(request_error.endpoint.as_deref(), Some("/v1/responses"));
|
||||
let probe_error = errors
|
||||
.items
|
||||
.iter()
|
||||
.find(|error| error.source == "ops_event")
|
||||
.expect("probe error");
|
||||
assert_eq!(
|
||||
(
|
||||
probe_error.provider_account_name.as_deref(),
|
||||
probe_error.provider_account_email.as_deref(),
|
||||
probe_error.provider_account_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Snapshot Charlie"),
|
||||
Some("charlie@example.invalid"),
|
||||
Some("api_key"),
|
||||
)
|
||||
);
|
||||
assert_eq!(probe_error.endpoint, None);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
|
||||
let Some(database) = TestDatabase::create("snapshot_diagnostics_same_email").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_a",
|
||||
"Alpha",
|
||||
Some("same@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_b",
|
||||
"Beta",
|
||||
Some("same@example.invalid"),
|
||||
"api_key",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_a", started_at),
|
||||
attempt("req_snap_a", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert account a request");
|
||||
finalize_request(&database.pool, "req_snap_a", started_at).await;
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_b", started_at + chrono::Duration::seconds(1)),
|
||||
attempt("req_snap_b", 1, "acct_snap_b"),
|
||||
)
|
||||
.await
|
||||
.expect("insert account b request");
|
||||
finalize_request(&database.pool, "req_snap_b", started_at).await;
|
||||
|
||||
sqlx::query("delete from provider_accounts where id in ('acct_snap_a', 'acct_snap_b')")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::Account,
|
||||
)
|
||||
.await
|
||||
.expect("account diagnostics");
|
||||
assert_eq!(diagnostics.len(), 2);
|
||||
assert_eq!(
|
||||
diagnostics
|
||||
.iter()
|
||||
.map(|item| item.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["acct_snap_a", "acct_snap_b"]
|
||||
);
|
||||
assert!(
|
||||
diagnostics
|
||||
.iter()
|
||||
.all(|item| item.name == "same@example.invalid")
|
||||
);
|
||||
assert!(diagnostics.iter().all(|item| item.request_count == 1));
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
|
||||
let Some(database) = TestDatabase::create("snapshot_diagnostics_fallback").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_c",
|
||||
"Snapshot Charlie",
|
||||
None,
|
||||
"api_key",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_c", started_at),
|
||||
attempt("req_snap_c", 1, "acct_snap_c"),
|
||||
)
|
||||
.await
|
||||
.expect("insert account c request");
|
||||
finalize_request(&database.pool, "req_snap_c", started_at).await;
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_legacy", started_at + chrono::Duration::seconds(1)),
|
||||
ModelRequestAttemptStart {
|
||||
model_request_id: "req_snap_legacy".to_owned(),
|
||||
attempt_count: 1,
|
||||
provider_kind: "openai".to_owned(),
|
||||
provider_account_id: None,
|
||||
provider_account_ref: Some("acct_legacy_deleted".to_owned()),
|
||||
upstream_model_id: "legacy-model".to_owned(),
|
||||
upstream_transport: "http_sse".to_owned(),
|
||||
http_version: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("insert legacy request without live account");
|
||||
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::Account,
|
||||
)
|
||||
.await
|
||||
.expect("account diagnostics");
|
||||
let by_key = diagnostics
|
||||
.iter()
|
||||
.map(|item| (item.key.as_str(), item.name.as_str()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(by_key.get("acct_snap_c"), Some(&"Snapshot Charlie"));
|
||||
assert_eq!(
|
||||
by_key.get("acct_legacy_deleted"),
|
||||
Some(&"acct_legacy_deleted")
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failure_diagnostics_should_only_include_errored_requests() {
|
||||
let Some(database) = TestDatabase::create("snapshot_failure_dimension").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
Some("alpha@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_ok", started_at),
|
||||
attempt("req_snap_ok", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert healthy request");
|
||||
finalize_request(&database.pool, "req_snap_ok", started_at).await;
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_err", started_at + chrono::Duration::seconds(1)),
|
||||
attempt("req_snap_err", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert errored request");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set outcome = 'failed', error_kind = 'rate_limited',
|
||||
error_message = 'upstream limited', client_status_code = 429,
|
||||
upstream_status_code = 429, completed_at = $2
|
||||
where id = $1",
|
||||
)
|
||||
.bind("req_snap_err")
|
||||
.bind(started_at + chrono::Duration::seconds(6))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("mark request failed");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::Failure,
|
||||
)
|
||||
.await
|
||||
.expect("failure diagnostics");
|
||||
assert_eq!(diagnostics.len(), 1);
|
||||
assert_eq!(diagnostics[0].key, "rate_limited");
|
||||
assert_eq!(diagnostics[0].request_count, 1);
|
||||
assert_eq!(diagnostics[0].failure_count, 1);
|
||||
assert_eq!(diagnostics[0].success_count, 0);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
|
||||
let Some(database) = TestDatabase::create("snapshot_api_key_dimension").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_api_key(&database.pool, "key_diag", "My Key", started_at).await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
for id in ["req_key_a", "req_key_b"] {
|
||||
let mut request = new_request(id, started_at);
|
||||
request.client_api_key_id = Some("key_diag".to_owned());
|
||||
request.client_api_key_ref = "key_diag".to_owned();
|
||||
let attempt = ModelRequestAttemptStart {
|
||||
model_request_id: id.to_owned(),
|
||||
attempt_count: 1,
|
||||
provider_kind: "openai".to_owned(),
|
||||
provider_account_id: None,
|
||||
provider_account_ref: Some("acct_key".to_owned()),
|
||||
upstream_model_id: "upstream-model".to_owned(),
|
||||
upstream_transport: "http_sse".to_owned(),
|
||||
http_version: None,
|
||||
};
|
||||
store
|
||||
.insert_model_request_with_first_attempt(request, attempt)
|
||||
.await
|
||||
.expect("insert api key request");
|
||||
finalize_request(&database.pool, id, started_at).await;
|
||||
}
|
||||
sqlx::query("update model_requests set latency_ms = 1000 where id = 'req_key_a'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("set request a latency");
|
||||
sqlx::query("update model_requests set latency_ms = 2000 where id = 'req_key_b'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("set request b latency");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::ApiKey,
|
||||
)
|
||||
.await
|
||||
.expect("api key diagnostics");
|
||||
assert_eq!(diagnostics.len(), 1);
|
||||
assert_eq!(diagnostics[0].key, "key_diag");
|
||||
assert_eq!(diagnostics[0].name, "My Key");
|
||||
assert_eq!(diagnostics[0].request_count, 2);
|
||||
assert_eq!(diagnostics[0].latency_p95_ms, Some(1950));
|
||||
|
||||
sqlx::query("delete from client_api_keys where id = 'key_diag'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("delete api key");
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::ApiKey,
|
||||
)
|
||||
.await
|
||||
.expect("api key diagnostics after deletion");
|
||||
assert_eq!(diagnostics.len(), 1);
|
||||
assert_eq!(diagnostics[0].key, "key_diag");
|
||||
assert_eq!(diagnostics[0].name, "key_diag");
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
||||
let Some(database) = TestDatabase::create("snapshot_rename").await else {
|
||||
return;
|
||||
};
|
||||
let started_at = Utc::now();
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_snap_a",
|
||||
"Original Name",
|
||||
Some("original@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
store
|
||||
.insert_model_request_with_first_attempt(
|
||||
new_request("req_snap_a", started_at),
|
||||
attempt("req_snap_a", 1, "acct_snap_a"),
|
||||
)
|
||||
.await
|
||||
.expect("insert request with original snapshot");
|
||||
finalize_request(&database.pool, "req_snap_a", started_at).await;
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set name = 'Renamed', email = 'renamed@example.invalid'
|
||||
where id = 'acct_snap_a'",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("rename provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
.expect("list usage records after rename");
|
||||
let record = page
|
||||
.items
|
||||
.iter()
|
||||
.find(|record| record.id == "req_snap_a")
|
||||
.expect("historical request");
|
||||
assert_eq!(
|
||||
(
|
||||
record.provider_account_name.as_deref(),
|
||||
record.provider_account_email.as_deref(),
|
||||
),
|
||||
(Some("Original Name"), Some("original@example.invalid"))
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_should_backfill_snapshots_for_existing_accounts_and_survive_deletion() {
|
||||
let Some(database_url) = crate::support::test_env("CPR_TEST_DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let schema = format!("cpr_store_snapshot_backfill_{}", Uuid::new_v4().simple());
|
||||
let admin = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect test PostgreSQL");
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!("create schema \"{schema}\"")))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("create backfill schema");
|
||||
let search_path = schema.clone();
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.after_connect(move |connection, _metadata| {
|
||||
let search_path = search_path.clone();
|
||||
Box::pin(async move {
|
||||
sqlx::query("select set_config('search_path', $1, false)")
|
||||
.bind(search_path)
|
||||
.execute(connection)
|
||||
.await?;
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect backfill pool");
|
||||
let started_at = Utc::now();
|
||||
{
|
||||
let mut migration = pool.begin().await.expect("begin terminal migration");
|
||||
sqlx::raw_sql(include_str!("../../../../migrations/0001_initial.sql"))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply initial migration");
|
||||
migration.commit().await.expect("commit initial migration");
|
||||
}
|
||||
seed_account(
|
||||
&pool,
|
||||
"acct_backfill",
|
||||
"Backfill",
|
||||
Some("backfill@example.invalid"),
|
||||
"oauth",
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_ref, config_revision, protocol, operation, endpoint,
|
||||
client_transport, requested_model_id,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
upstream_model_id, upstream_transport, attempt_count, upstream_send_state,
|
||||
started_at, deadline_at
|
||||
) values (
|
||||
'req_backfill', 'key_backfill', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'coding', 'openai', 'acct_backfill', 'acct_backfill',
|
||||
'upstream-backfill', 'http_sse', 1, 'sent', $1, $1 + interval '30 seconds'
|
||||
)",
|
||||
)
|
||||
.bind(started_at)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("insert pre-migration request");
|
||||
sqlx::query(
|
||||
"insert into ops_events (
|
||||
id, level, component, operation,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
failure_kind, message, occurrence_count, created_at
|
||||
) values (
|
||||
'ops_backfill', 'warning', 'account_probe', 'connection_test',
|
||||
'openai', 'acct_backfill', 'acct_backfill',
|
||||
'auth_failed', 'backfill probe', 1, $1
|
||||
)",
|
||||
)
|
||||
.bind(started_at)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("insert pre-migration ops event");
|
||||
{
|
||||
let mut migration = pool.begin().await.expect("begin snapshot migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0002_snapshot_provider_account_identity.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply snapshot migration");
|
||||
migration.commit().await.expect("commit snapshot migration");
|
||||
}
|
||||
let (
|
||||
request_name,
|
||||
request_email,
|
||||
request_authentication_kind,
|
||||
event_name,
|
||||
event_email,
|
||||
event_authentication_kind,
|
||||
): ProviderAccountSnapshotRow = sqlx::query_as(
|
||||
"select
|
||||
(select provider_account_name_snapshot from model_requests where id = 'req_backfill'),
|
||||
(select provider_account_email_snapshot from model_requests where id = 'req_backfill'),
|
||||
(select provider_account_authentication_kind_snapshot from model_requests where id = 'req_backfill'),
|
||||
(select provider_account_name_snapshot from ops_events where id = 'ops_backfill'),
|
||||
(select provider_account_email_snapshot from ops_events where id = 'ops_backfill'),
|
||||
(select provider_account_authentication_kind_snapshot from ops_events where id = 'ops_backfill')",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("load backfilled snapshots");
|
||||
assert_eq!(
|
||||
(
|
||||
request_name.as_deref(),
|
||||
request_email.as_deref(),
|
||||
request_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Backfill"),
|
||||
Some("backfill@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
(
|
||||
event_name.as_deref(),
|
||||
event_email.as_deref(),
|
||||
event_authentication_kind.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Backfill"),
|
||||
Some("backfill@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
sqlx::query("delete from provider_accounts where id = 'acct_backfill'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("delete backfilled account");
|
||||
let request_snapshot: (Option<String>, Option<String>, Option<String>) = sqlx::query_as(
|
||||
"select provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot
|
||||
from model_requests where id = 'req_backfill'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("load request snapshot after deletion");
|
||||
assert_eq!(
|
||||
(
|
||||
request_snapshot.0.as_deref(),
|
||||
request_snapshot.1.as_deref(),
|
||||
request_snapshot.2.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("Backfill"),
|
||||
Some("backfill@example.invalid"),
|
||||
Some("oauth"),
|
||||
)
|
||||
);
|
||||
|
||||
drop(pool);
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"drop schema \"{schema}\" cascade"
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("drop backfill schema");
|
||||
admin.close().await;
|
||||
}
|
||||
|
||||
fn new_request(id: &str, started_at: DateTime<Utc>) -> NewModelRequest {
|
||||
NewModelRequest {
|
||||
id: id.to_owned(),
|
||||
client_api_key_id: None,
|
||||
client_api_key_ref: "key_snapshot".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "coding".to_owned(),
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
image_generation_requested: false,
|
||||
started_at,
|
||||
deadline_at: started_at + chrono::Duration::seconds(30),
|
||||
}
|
||||
}
|
||||
|
||||
fn attempt(id: &str, count: u32, account_id: &str) -> ModelRequestAttemptStart {
|
||||
ModelRequestAttemptStart {
|
||||
model_request_id: id.to_owned(),
|
||||
attempt_count: count,
|
||||
provider_kind: "openai".to_owned(),
|
||||
provider_account_id: Some(account_id.to_owned()),
|
||||
provider_account_ref: Some(account_id.to_owned()),
|
||||
upstream_model_id: "upstream-model".to_owned(),
|
||||
upstream_transport: "http_sse".to_owned(),
|
||||
http_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_query(started_at: DateTime<Utc>, filter: UsageRecordFilter) -> UsageRecordQuery {
|
||||
UsageRecordQuery {
|
||||
range: range_around(started_at),
|
||||
filter,
|
||||
cursor: None,
|
||||
page: ObservabilityPageNumber::new(1).expect("page"),
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
}
|
||||
}
|
||||
|
||||
fn range_around(started_at: DateTime<Utc>) -> ObservabilityRange {
|
||||
ObservabilityRange::new(
|
||||
started_at - TimeDelta::hours(1),
|
||||
started_at + TimeDelta::hours(1),
|
||||
)
|
||||
.expect("observability range")
|
||||
}
|
||||
|
||||
async fn seed_account(
|
||||
pool: &PgPool,
|
||||
id: &str,
|
||||
name: &str,
|
||||
email: Option<&str>,
|
||||
authentication_kind: &str,
|
||||
now: DateTime<Utc>,
|
||||
) {
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind,
|
||||
provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, enabled, availability,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, 'openai', $2, $3, 'user-' || $1, null, null, $4,
|
||||
'{}'::jsonb, 1, false, $5 + interval '1 day', true, 'ready',
|
||||
$5, $5, $5
|
||||
)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(name)
|
||||
.bind(email)
|
||||
.bind(authentication_kind)
|
||||
.bind(now)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed provider account");
|
||||
}
|
||||
|
||||
async fn finalize_request(pool: &PgPool, id: &str, started_at: DateTime<Utc>) {
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set outcome = 'succeeded', client_status_code = 200,
|
||||
upstream_status_code = 200, downstream_committed_at = $2, completed_at = $2
|
||||
where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(started_at + chrono::Duration::seconds(5))
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("finalize request");
|
||||
}
|
||||
|
||||
async fn seed_api_key(pool: &PgPool, id: &str, name: &str, now: DateTime<Utc>) {
|
||||
sqlx::query(
|
||||
"insert into client_api_keys (id, name, provider_kind, key, enabled, created_at, updated_at)
|
||||
values ($1, $2, 'openai', 'sk_' || $3, true, $4, $4)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(name)
|
||||
.bind("A".repeat(43))
|
||||
.bind(now)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed api key");
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
use std::{num::NonZeroU32, time::Duration};
|
||||
|
||||
use gateway_core::{engine::execution::provider_failure_affects_circuit, error::ProviderErrorKind};
|
||||
use gateway_core::{
|
||||
engine::execution::{ProviderCircuitPolicy, provider_failure_affects_circuit},
|
||||
error::ProviderErrorKind,
|
||||
};
|
||||
use gateway_store::redis::{
|
||||
ProviderCircuitDecision, ProviderCircuitPolicy, ProviderCircuitRepository,
|
||||
RedisProviderCircuitRepository,
|
||||
ProviderCircuitDecision, ProviderCircuitRepository, RedisProviderCircuitRepository,
|
||||
};
|
||||
use redis::aio::ConnectionManager;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1439,8 +1439,8 @@ fn map_quota_error(error: CodexCredentialQuotaError) -> ProviderAdminError {
|
||||
Error::RevisionConflict => ProviderAdminErrorKind::Conflict,
|
||||
Error::CredentialRefreshRequired
|
||||
| Error::Repository(_)
|
||||
| Error::Store
|
||||
| Error::Upstream => ProviderAdminErrorKind::Unavailable,
|
||||
| Error::Store { .. }
|
||||
| Error::Upstream { .. } => ProviderAdminErrorKind::Unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1452,6 +1452,6 @@ fn map_catalog_error(error: CodexCredentialCatalogError) -> ProviderAdminError {
|
||||
}
|
||||
Error::NoEligibleCredential => ProviderAdminErrorKind::NotFound,
|
||||
Error::ConcurrentUpdate => ProviderAdminErrorKind::Conflict,
|
||||
Error::Upstream | Error::Cache => ProviderAdminErrorKind::Unavailable,
|
||||
Error::Upstream { .. } | Error::Cache => ProviderAdminErrorKind::Unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -136,8 +136,8 @@ pub enum CodexCredentialCatalogError {
|
||||
NoEligibleCredential,
|
||||
#[error("Codex model catalog account data is invalid")]
|
||||
InvalidCredentialData,
|
||||
#[error("Codex model catalog upstream query failed")]
|
||||
Upstream,
|
||||
#[error("Codex model catalog upstream query failed: {detail}")]
|
||||
Upstream { detail: String },
|
||||
#[error("Codex model catalog contains conflicting account facts")]
|
||||
ConflictingModelFacts,
|
||||
#[error("Codex model catalog cache is unavailable")]
|
||||
@@ -482,7 +482,9 @@ impl CodexCredentialCatalogService {
|
||||
))
|
||||
.await;
|
||||
}
|
||||
let snapshot = result.map_err(|_| CodexCredentialCatalogError::Upstream)?;
|
||||
let snapshot = result.map_err(|error| CodexCredentialCatalogError::Upstream {
|
||||
detail: error.to_string(),
|
||||
})?;
|
||||
Ok(FetchedAccountModels {
|
||||
models: snapshot.models().to_vec(),
|
||||
etag: snapshot.etag().map(str::to_owned),
|
||||
|
||||
@@ -215,19 +215,21 @@ pub enum CodexCredentialQuotaError {
|
||||
CredentialRefreshRequired,
|
||||
#[error(transparent)]
|
||||
Repository(#[from] CredentialRepositoryError),
|
||||
#[error("provider account store is unavailable")]
|
||||
Store,
|
||||
#[error("provider account store is unavailable: {detail}")]
|
||||
Store { detail: String },
|
||||
#[error("Codex quota account was not found")]
|
||||
NotFound,
|
||||
#[error("Codex quota credential revision is stale")]
|
||||
RevisionConflict,
|
||||
#[error("Codex quota upstream query failed")]
|
||||
Upstream,
|
||||
#[error("Codex quota upstream query failed: {detail}")]
|
||||
Upstream { detail: String },
|
||||
}
|
||||
|
||||
impl From<gateway_core::error::StoreError> for CodexCredentialQuotaError {
|
||||
fn from(_: gateway_core::error::StoreError) -> Self {
|
||||
Self::Store
|
||||
fn from(error: gateway_core::error::StoreError) -> Self {
|
||||
Self::Store {
|
||||
detail: error.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -660,10 +662,19 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
Err(CodexQuotaFetchError::Recovery) => {
|
||||
summary.transient += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
"OpenAI quota credential recovery failed; refresh cycle will retry later"
|
||||
);
|
||||
}
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
let Some(transition) = quota_state_transition(&error, observed_at) else {
|
||||
summary.transient += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI quota upstream rejection was not classified"
|
||||
);
|
||||
continue;
|
||||
};
|
||||
match transition {
|
||||
@@ -986,7 +997,9 @@ impl CodexCredentialQuotaService {
|
||||
return Err(CodexCredentialQuotaError::InvalidCredentialData);
|
||||
}
|
||||
Err(CodexQuotaFetchError::Recovery) => {
|
||||
return Err(CodexCredentialQuotaError::Upstream);
|
||||
return Err(CodexCredentialQuotaError::Upstream {
|
||||
detail: "credential recovery failed".to_owned(),
|
||||
});
|
||||
}
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
if let Some(transition) = quota_state_transition(&error, observed_at) {
|
||||
@@ -1001,7 +1014,9 @@ impl CodexCredentialQuotaService {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
return Err(CodexCredentialQuotaError::Upstream);
|
||||
return Err(CodexCredentialQuotaError::Upstream {
|
||||
detail: error.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
|
||||
@@ -2416,19 +2416,45 @@ impl ScheduledTask for OpenAiOAuthRefreshTask {
|
||||
.refresh_due()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("OpenAI OAuth refresh failed"))?;
|
||||
let operational_failures = outcomes
|
||||
.iter()
|
||||
.filter(|outcome| {
|
||||
matches!(
|
||||
outcome,
|
||||
CodexCredentialRefreshOutcome::Transient { .. }
|
||||
| CodexCredentialRefreshOutcome::Failed { .. }
|
||||
)
|
||||
})
|
||||
.count();
|
||||
if operational_failures > 0 {
|
||||
let mut refreshed = 0_u64;
|
||||
let mut invalidated = 0_u64;
|
||||
let mut banned = 0_u64;
|
||||
let mut transient = 0_u64;
|
||||
let mut lease_unavailable = 0_u64;
|
||||
let mut stale = 0_u64;
|
||||
let mut failed = 0_u64;
|
||||
let mut transient_accounts = Vec::new();
|
||||
let mut failed_accounts = Vec::new();
|
||||
for outcome in &outcomes {
|
||||
match outcome {
|
||||
CodexCredentialRefreshOutcome::Refreshed { .. } => refreshed += 1,
|
||||
CodexCredentialRefreshOutcome::Invalidated { .. } => invalidated += 1,
|
||||
CodexCredentialRefreshOutcome::Banned { .. } => banned += 1,
|
||||
CodexCredentialRefreshOutcome::Transient { account_id } => {
|
||||
transient += 1;
|
||||
transient_accounts.push(account_id);
|
||||
}
|
||||
CodexCredentialRefreshOutcome::LeaseUnavailable { .. } => {
|
||||
lease_unavailable += 1;
|
||||
}
|
||||
CodexCredentialRefreshOutcome::Stale { .. } => stale += 1,
|
||||
CodexCredentialRefreshOutcome::Failed { account_id } => {
|
||||
failed += 1;
|
||||
failed_accounts.push(account_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
if transient > 0 || failed > 0 {
|
||||
tracing::warn!(
|
||||
operational_failures,
|
||||
refreshed,
|
||||
invalidated,
|
||||
banned,
|
||||
transient,
|
||||
lease_unavailable,
|
||||
stale,
|
||||
failed,
|
||||
transient_accounts = ?transient_accounts,
|
||||
failed_accounts = ?failed_accounts,
|
||||
"OpenAI OAuth refresh cycle contained operational failures"
|
||||
);
|
||||
}
|
||||
@@ -2499,12 +2525,23 @@ impl ScheduledTask for OpenAiQuotaTask {
|
||||
let mut failures = false;
|
||||
match self.quota.synchronize().await {
|
||||
Ok(summary) if summary.has_operational_failures() => {
|
||||
tracing::warn!("OpenAI quota cycle contained operational failures");
|
||||
tracing::warn!(
|
||||
updated = summary.updated,
|
||||
exhausted = summary.exhausted,
|
||||
banned = summary.banned,
|
||||
cooldown = summary.cooldown,
|
||||
transient = summary.transient,
|
||||
stale = summary.stale,
|
||||
"OpenAI quota cycle contained operational failures"
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(_) => {
|
||||
Err(error) => {
|
||||
failures = true;
|
||||
tracing::warn!("OpenAI quota synchronization failed");
|
||||
tracing::warn!(
|
||||
error = %error,
|
||||
"OpenAI quota synchronization failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
if context.cancellation().is_cancelled() {
|
||||
|
||||
@@ -96,25 +96,29 @@ const PRICING_RULES: &[PricingRule] = &[
|
||||
model: "gpt-5.6-sol",
|
||||
pricing: ModelPricing::new(50_000, 300_000, 5_000)
|
||||
.with_cache_write(125)
|
||||
.with_priority(100_000, 600_000, 10_000),
|
||||
.with_priority(100_000, 600_000, 10_000)
|
||||
.with_long_context(100_000, 450_000, 10_000, 0, 0, 0),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.6-terra",
|
||||
pricing: ModelPricing::new(25_000, 150_000, 2_500)
|
||||
pricing: ModelPricing::new(20_000, 120_000, 2_000)
|
||||
.with_cache_write(125)
|
||||
.with_priority(50_000, 300_000, 5_000),
|
||||
.with_priority(40_000, 240_000, 4_000)
|
||||
.with_long_context(40_000, 180_000, 4_000, 0, 0, 0),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.6-luna",
|
||||
pricing: ModelPricing::new(10_000, 60_000, 1_000)
|
||||
pricing: ModelPricing::new(2_000, 12_000, 200)
|
||||
.with_cache_write(125)
|
||||
.with_priority(20_000, 120_000, 2_000),
|
||||
.with_priority(4_000, 24_000, 400)
|
||||
.with_long_context(4_000, 18_000, 400, 0, 0, 0),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.6",
|
||||
pricing: ModelPricing::new(50_000, 300_000, 5_000)
|
||||
.with_cache_write(125)
|
||||
.with_priority(100_000, 600_000, 10_000),
|
||||
.with_priority(100_000, 600_000, 10_000)
|
||||
.with_long_context(100_000, 450_000, 10_000, 0, 0, 0),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.5-pro",
|
||||
@@ -162,11 +166,11 @@ const PRICING_RULES: &[PricingRule] = &[
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-4o-mini",
|
||||
pricing: ModelPricing::new(1_500, 6_000, 0),
|
||||
pricing: ModelPricing::new(1_500, 6_000, 750),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-4o",
|
||||
pricing: ModelPricing::new(25_000, 100_000, 0),
|
||||
pricing: ModelPricing::new(25_000, 100_000, 12_500),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-4-turbo",
|
||||
|
||||
@@ -215,7 +215,10 @@ async fn plan_catalog_refresh_stops_after_three_failed_accounts() {
|
||||
.await
|
||||
.expect_err("three failed accounts stop the refresh");
|
||||
|
||||
assert!(matches!(error, CodexCredentialCatalogError::Upstream));
|
||||
assert!(matches!(
|
||||
error,
|
||||
CodexCredentialCatalogError::Upstream { .. }
|
||||
));
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
|
||||
@@ -652,7 +652,7 @@ async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential(
|
||||
|
||||
assert!(matches!(
|
||||
service.refresh_account(account.id()).await,
|
||||
Err(CodexCredentialQuotaError::Upstream)
|
||||
Err(CodexCredentialQuotaError::Upstream { .. })
|
||||
));
|
||||
assert_eq!(
|
||||
store
|
||||
|
||||
@@ -30,6 +30,33 @@ fn billing_breakdown_should_preserve_input_output_and_cache_components() {
|
||||
assert_eq!(breakdown.multiplier_percent(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_breakdown_should_use_latest_gpt_5_6_and_cached_input_prices() {
|
||||
let terra = openai_billing_breakdown("gpt-5.6-terra", 1, 1, 0, 0, None)
|
||||
.expect("gpt-5.6-terra standard pricing");
|
||||
assert_eq!(terra.total_amount().amount().scaled(), 140_000);
|
||||
|
||||
let terra_fast = openai_billing_breakdown("gpt-5.6-terra", 1, 1, 0, 0, Some("fast"))
|
||||
.expect("gpt-5.6-terra fast pricing");
|
||||
assert_eq!(terra_fast.total_amount().amount().scaled(), 280_000);
|
||||
|
||||
let terra_long = openai_billing_breakdown("gpt-5.6-terra", 272_001, 0, 0, 0, None)
|
||||
.expect("gpt-5.6-terra long-context pricing");
|
||||
assert_eq!(terra_long.total_amount().amount().scaled(), 10_880_040_000);
|
||||
|
||||
let luna = openai_billing_breakdown("gpt-5.6-luna", 1, 1, 0, 0, None)
|
||||
.expect("gpt-5.6-luna standard pricing");
|
||||
assert_eq!(luna.total_amount().amount().scaled(), 14_000);
|
||||
|
||||
let gpt_4o =
|
||||
openai_billing_breakdown("gpt-4o", 1, 1, 1, 0, None).expect("gpt-4o cached-input pricing");
|
||||
assert_eq!(gpt_4o.total_amount().amount().scaled(), 112_500);
|
||||
|
||||
let gpt_4o_mini = openai_billing_breakdown("gpt-4o-mini", 1, 1, 1, 0, None)
|
||||
.expect("gpt-4o-mini cached-input pricing");
|
||||
assert_eq!(gpt_4o_mini.total_amount().amount().scaled(), 6_750);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_breakdown_should_apply_fast_and_flex_tiers_without_guessing_unknown_models() {
|
||||
let fast = openai_billing_breakdown("gpt-5.4", 1, 1, 0, 0, Some("fast")).expect("fast pricing");
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
a4681a5c040e80c5234e879293fc34972080e5251dfbc7c40e1374f8e4170a4c 0001_initial.sql
|
||||
7dc3b626e7a958281f82c0e646b8284f8b373e0c7d3dbdaf750811ce0c4a7f83 0002_snapshot_provider_account_identity.sql
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
-- Provider 账号观测快照:请求/事件发生时账号的名称、邮箱与认证类型。
|
||||
-- 账号删除后 provider_account_id 外键置空,历史显示仍依赖这些快照列;
|
||||
-- provider_account_ref 继续承担稳定筛选与关联事实。
|
||||
|
||||
alter table model_requests
|
||||
add column provider_account_name_snapshot text,
|
||||
add column provider_account_email_snapshot text,
|
||||
add column provider_account_authentication_kind_snapshot text;
|
||||
|
||||
alter table ops_events
|
||||
add column provider_account_name_snapshot text,
|
||||
add column provider_account_email_snapshot text,
|
||||
add column provider_account_authentication_kind_snapshot text;
|
||||
|
||||
-- 迁移时仍存在账号的历史行回填快照;已删除账号的历史行保持为空,
|
||||
-- 读取侧回退 provider_account_ref。迁移前已删除账号的资料无法恢复。
|
||||
update model_requests mr
|
||||
set provider_account_name_snapshot = pa.name,
|
||||
provider_account_email_snapshot = pa.email,
|
||||
provider_account_authentication_kind_snapshot = pa.authentication_kind
|
||||
from provider_accounts pa
|
||||
where pa.id = mr.provider_account_ref;
|
||||
|
||||
update ops_events oe
|
||||
set provider_account_name_snapshot = pa.name,
|
||||
provider_account_email_snapshot = pa.email,
|
||||
provider_account_authentication_kind_snapshot = pa.authentication_kind
|
||||
from provider_accounts pa
|
||||
where pa.id = oe.provider_account_ref;
|
||||
@@ -0,0 +1,202 @@
|
||||
# Provider 账号观测快照方案
|
||||
|
||||
- 状态:已实现(迁移、Store/API/前端与集成测试完成)
|
||||
- 日期:2026-07-31
|
||||
|
||||
## 1. 背景
|
||||
|
||||
请求明细当前只在 `model_requests` 中保存稳定账号引用 `provider_account_ref`,账号名称、
|
||||
邮箱和认证类型则在查询时实时关联 `provider_accounts`。账号删除后,外键列
|
||||
`provider_account_id` 被置空,实时关联失效,页面只能显示保留下来的 `acct_...` 引用。
|
||||
|
||||
另外两条观测链路也存在同类问题:
|
||||
|
||||
- 热点诊断的账号维度直接使用 `provider_account_ref` 作为分组名称,因此始终显示账号 ID;
|
||||
- 错误明细虽然已经预留 `metadata.accountLabel`,但后端当前固定返回空值。
|
||||
|
||||
账号 ID 是稳定的排查、筛选和关联事实,不能用邮箱替换。名称、邮箱以及平台/类型属于请求
|
||||
发生时的显示事实,应作为历史快照独立保存,不能依赖账号当前是否仍然存在。
|
||||
|
||||
## 2. 目标
|
||||
|
||||
1. 请求、attempt 和错误事件在账号删除后仍能显示请求发生时的账号信息。
|
||||
2. 请求明细、热点诊断和错误明细统一采用“邮箱、名称、账号 ID”的显示优先级。
|
||||
3. 平台/类型在账号删除后保持可解释:平台为 Provider,类型为认证方式。
|
||||
4. 聚合、筛选和审计继续使用稳定账号 ID,不因同邮箱账号或账号改名而改变语义。
|
||||
5. 不增加上游请求,不增加前端补查账号接口,不让显示需求进入调度与 Provider 逻辑。
|
||||
|
||||
## 3. 非目标
|
||||
|
||||
- 不把 `provider_account_ref` 改成邮箱或名称。
|
||||
- 不根据历史请求反向恢复已经删除且没有任何可用资料的旧账号。
|
||||
- 不在账号改名或换邮箱后批量改写既有请求快照。
|
||||
- 不改变账号选择、亲和、额度、冷却或凭据刷新逻辑。
|
||||
|
||||
## 4. 快照契约
|
||||
|
||||
### 4.1 稳定标识与显示快照分离
|
||||
|
||||
| 事实 | 持久化字段 | 语义 |
|
||||
| --- | --- | --- |
|
||||
| 账号稳定引用 | `provider_account_ref` | 永久保存请求发生时使用的本地账号 ID,用于筛选、关联和审计 |
|
||||
| 账号实时外键 | `provider_account_id` | 账号存在时指向 `provider_accounts.id`,账号删除时由外键置空 |
|
||||
| 账号名称快照 | `provider_account_name_snapshot` | 请求或事件写入时的账号名称 |
|
||||
| 账号邮箱快照 | `provider_account_email_snapshot` | 请求或事件写入时的账号邮箱 |
|
||||
| 平台快照 | `provider_kind` | 请求或事件已经持久化的 Provider kind,例如 `openai`、`xai` |
|
||||
| 类型快照 | `provider_account_authentication_kind_snapshot` | 请求或事件写入时的认证类型,例如 OAuth 或 API Key |
|
||||
|
||||
`provider_kind` 本身已经是请求和事件的历史事实,因此不再增加语义重复的
|
||||
`provider_kind_snapshot` 列。实现和测试必须把它明确视为平台快照,读取时不得通过当前账号
|
||||
重新推导。
|
||||
|
||||
### 4.2 表结构
|
||||
|
||||
`model_requests` 新增:
|
||||
|
||||
```sql
|
||||
provider_account_name_snapshot text,
|
||||
provider_account_email_snapshot text,
|
||||
provider_account_authentication_kind_snapshot text
|
||||
```
|
||||
|
||||
`ops_events` 新增同样三个字段。`ops_events.provider_kind` 继续承担平台快照;每条 attempt
|
||||
错误必须保存该 attempt 实际使用账号的快照,不能复用请求最终账号的资料。
|
||||
|
||||
这些列保持可空,原因包括:请求可能尚未路由、历史账号可能已经删除、探测事件可能缺少完整
|
||||
账号资料。新写入路径在存在 `provider_account_ref` 时应尽力写全快照,但数据库不能用
|
||||
`NOT NULL` 阻止历史数据迁移。
|
||||
|
||||
## 5. 写入方案
|
||||
|
||||
快照由 PostgreSQL Store 在现有写入语句内按 `provider_account_id` 主键读取
|
||||
`provider_accounts`,与请求或事件原子写入。Core 继续只传递稳定账号 ID,不把名称、邮箱等
|
||||
展示资料扩散到调度领域。
|
||||
|
||||
需要覆盖的写入边界:
|
||||
|
||||
1. `insert_model_request_with_first_attempt`:首次 attempt 与请求行合并写入时保存快照。
|
||||
2. `begin_model_request_attempt`:内部换号时更新请求行,使请求终态对应最终 attempt 的账号快照。
|
||||
3. `append_ops_event`:保存事件所属 attempt 或账号探测使用的独立快照。
|
||||
|
||||
普通的 `insert_model_request` 尚未选出账号,不写快照;后续首次
|
||||
`begin_model_request_attempt` 负责补齐。账号资料后续发生修改时,不回写历史快照。
|
||||
|
||||
主键读取合并在现有 PostgreSQL 语句中,不产生上游请求或前端额外请求。该读取只访问
|
||||
`provider_accounts.id` 主键,性能成本相对现有观测写入可忽略,并继续服从现有异步、有界
|
||||
观测队列语义。
|
||||
|
||||
## 6. 读取与展示方案
|
||||
|
||||
### 6.1 统一显示优先级
|
||||
|
||||
所有页面统一使用:
|
||||
|
||||
```text
|
||||
非空邮箱 > 非空账号名称 > provider_account_ref > “—”
|
||||
```
|
||||
|
||||
平台/类型统一使用:
|
||||
|
||||
```text
|
||||
provider_kind + provider_account_authentication_kind_snapshot
|
||||
```
|
||||
|
||||
原始 `accountId` 继续保留在 API 和详情中,不能因展示邮箱而丢失。
|
||||
|
||||
### 6.2 请求明细
|
||||
|
||||
- `UsageRecord` 直接读取名称、邮箱和认证类型快照,不再依赖实时账号关联;
|
||||
- API 在现有 `accountId`、`accountEmail` 基础上增加 `accountName`;
|
||||
- `authenticationKind` 改为读取类型快照,保证账号删除后平台/类型图标仍然正确;
|
||||
- attempt API 使用对应 `ops_events` 快照填充 `credentialName`、`accountEmail` 和
|
||||
`authenticationKind`;
|
||||
- 请求详情主要位置显示统一账号名称,深层字段仍保留“账号 ID”。
|
||||
|
||||
### 6.3 热点诊断
|
||||
|
||||
账号热点必须按 `provider_account_ref` 聚合,显示名称单独计算,不能直接按邮箱分组。否则两个
|
||||
使用相同邮箱的本地账号会被错误合并。
|
||||
|
||||
诊断响应增加稳定 `key`:账号维度使用账号 ID,其他维度使用原始维度值。`name` 作为显示值,
|
||||
账号维度从该账号最新的非空历史快照中选择邮箱或名称,最终回退账号 ID。费用聚合也按同一个
|
||||
`key` 合并,避免名称变化导致请求数与费用错位。前端表格以 `key` 作为行键,显示 `name`。
|
||||
|
||||
### 6.4 错误明细
|
||||
|
||||
- request 级错误读取 `model_requests` 快照;
|
||||
- attempt 和探测错误读取 `ops_events` 自身快照;
|
||||
- API 保留 `accountId`,并填充现有 `metadata.accountLabel`;
|
||||
- API 同时提供平台和认证类型快照,供详情或后续表格展示;
|
||||
- 表格列由“账号 ID”改为“账号”,显示统一账号名称;详情同时显示“账号”和“账号 ID”。
|
||||
|
||||
## 7. 数据库迁移
|
||||
|
||||
`0001_initial.sql` 已在 3.0.30 基线中重新冻结。本功能属于之后的新 schema 变更,应新增:
|
||||
|
||||
```text
|
||||
0002_snapshot_provider_account_identity.sql
|
||||
```
|
||||
|
||||
迁移步骤:
|
||||
|
||||
1. 为 `model_requests` 和 `ops_events` 增加三个可空快照字段;
|
||||
2. 对仍能通过 `provider_account_ref = provider_accounts.id` 找到账号的历史行回填快照;
|
||||
3. 已删除账号的历史行保持空快照,读取时回退 `provider_account_ref`;
|
||||
4. 将新迁移哈希追加到 `.frozen-sha256`,迁移文件合入后按字节冻结;
|
||||
5. 更新迁移集成测试的迁移数量和最终 schema 断言。
|
||||
|
||||
迁移无法恢复在执行迁移前已经删除的账号名称、邮箱或认证类型,这是现有数据缺失造成的明确
|
||||
边界。部署后的新请求和迁移时仍存在账号的历史请求不受此限制。
|
||||
|
||||
## 8. 删除与隐私语义
|
||||
|
||||
账号删除只删除可调度账号和凭据,不再删除请求日志中的历史名称、邮箱、平台和类型快照。
|
||||
这些字段属于管理员可见的历史审计信息,生命周期跟随 `model_requests`、`ops_events` 的保留与
|
||||
清理策略。
|
||||
|
||||
这也意味着“删除账号”不是历史邮箱的数据擦除操作。若未来需要满足单独的隐私擦除要求,应
|
||||
增加显式的历史快照脱敏操作,不能复用普通账号删除接口隐式完成。
|
||||
|
||||
## 9. 测试方案
|
||||
|
||||
### 9.1 PostgreSQL 集成测试
|
||||
|
||||
1. 创建账号并写入请求,删除账号后查询请求明细,名称、邮箱、平台和类型仍保持原值。
|
||||
2. 同一请求依次使用账号 A、账号 B,两个 attempt 分别保留自己的账号快照。
|
||||
3. 错误事件和账号探测事件在账号删除后仍返回对应快照。
|
||||
4. 无邮箱时回退账号名称;名称和邮箱都缺失时回退账号 ID。
|
||||
5. 两个相同邮箱的账号在热点诊断中仍是两个稳定分组。
|
||||
6. 账号改名或更换邮箱不改写已经完成请求的历史快照。
|
||||
7. 迁移可以为仍存在账号的旧请求和事件完成回填。
|
||||
8. 全新数据库执行 `0001`、`0002` 后二次启动,迁移记录和最终 schema 均正确。
|
||||
|
||||
### 9.2 API 契约测试
|
||||
|
||||
1. 请求列表和详情稳定输出 `accountId`、`accountName`、`accountEmail`、`provider`、
|
||||
`authenticationKind`。
|
||||
2. attempt 输出自身的账号名称、邮箱和认证类型,不继承最终 attempt。
|
||||
3. 错误明细同时输出原始 `accountId` 与显示用 `accountLabel`。
|
||||
4. 诊断响应同时输出稳定 `key` 与显示 `name`。
|
||||
|
||||
### 9.3 前端验收
|
||||
|
||||
1. 请求明细列表和弹窗优先显示邮箱,账号删除后刷新页面仍不退化为 ID。
|
||||
2. 热点诊断的账号页签显示邮箱或名称,不显示 `acct_...`,同邮箱账号不被合并。
|
||||
3. 错误明细表格显示邮箱或名称,详情中仍可查看原始账号 ID。
|
||||
4. 账号删除后,请求明细的平台/类型图标与删除前一致。
|
||||
5. 长邮箱继续使用截断与完整 title,不改变表格布局。
|
||||
|
||||
## 10. 实施顺序与验收标准
|
||||
|
||||
建议按以下顺序实施:
|
||||
|
||||
1. 新增迁移和 Store 写入快照;
|
||||
2. 完成 PostgreSQL 删除后查询回归测试;
|
||||
3. 扩展 Store、Admin domain 与 API wire;
|
||||
4. 调整三个前端展示入口;
|
||||
5. 运行带真实 PostgreSQL 的 gateway-store 测试、gateway-admin/gateway-api 测试、workspace
|
||||
Clippy 与构建、前端 lint 与 build;
|
||||
6. 使用全新数据库启动项目,创建请求与错误事件,删除账号后通过三个页面实测。
|
||||
|
||||
验收完成必须同时满足:历史显示不依赖实时账号表、稳定账号 ID 未被显示字段替代、多 attempt
|
||||
账号不串号、平台/类型删除后不丢失、迁移前已删除账号的不可恢复边界被明确保留。
|
||||
@@ -24,8 +24,6 @@ export interface DashboardTrendPoint {
|
||||
errorsValue: number
|
||||
latency: string
|
||||
latencyValue: number | null
|
||||
firstTokenLatency: string
|
||||
firstTokenLatencyValue: number | null
|
||||
maxLatency: string
|
||||
maxLatencyValue: number | null
|
||||
minLatency: string
|
||||
@@ -121,12 +119,6 @@ export interface DashboardWireProfile {
|
||||
checkedAt?: string
|
||||
latestVersion?: string
|
||||
latestBuild?: string
|
||||
publishedAt?: string
|
||||
minimumSystemVersion?: string
|
||||
hardwareRequirements?: string
|
||||
downloadUrl?: string
|
||||
downloadSize?: number
|
||||
signaturePresent?: boolean
|
||||
error?: string
|
||||
}
|
||||
}
|
||||
@@ -149,18 +141,6 @@ export interface DashboardAccountUsage {
|
||||
lastUsed: string
|
||||
}
|
||||
|
||||
export interface DashboardCredentialUsage {
|
||||
id: string
|
||||
displayName: string
|
||||
planType: string | null
|
||||
tokens: string
|
||||
tokensValue: number | null
|
||||
lastUsed: string
|
||||
provider: string
|
||||
availability: string
|
||||
requestCount: number
|
||||
}
|
||||
|
||||
export interface DashboardPoolSummary {
|
||||
total: number
|
||||
active: number
|
||||
@@ -178,33 +158,16 @@ export interface DashboardCapacityInfo {
|
||||
availableSlots: number | null
|
||||
}
|
||||
|
||||
export interface DashboardRequestMetrics {
|
||||
requestCount: number
|
||||
successCount: number
|
||||
failureCount: number
|
||||
cancelledCount: number
|
||||
incompleteCount: number
|
||||
callerErrorCount: number
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cachedTokens: number
|
||||
cacheWriteTokens: number
|
||||
reasoningTokens: number
|
||||
totalTokens: number
|
||||
}
|
||||
|
||||
export interface DashboardSummaryResponse {
|
||||
cards: DashboardCards
|
||||
trend: DashboardTrendResponse
|
||||
healthTimeline: DashboardHealthTimeline
|
||||
wireProfiles: DashboardWireProfile[]
|
||||
accountUsage: DashboardAccountUsage[]
|
||||
credentialUsage: DashboardCredentialUsage[]
|
||||
usageRecords: UsageRecord[]
|
||||
poolSummary: DashboardPoolSummary
|
||||
capacityInfo: DashboardCapacityInfo
|
||||
rotationStrategy: string
|
||||
logicalRequests: DashboardRequestMetrics
|
||||
}
|
||||
|
||||
export function getDashboardSummary(data: object) {
|
||||
|
||||
@@ -100,6 +100,7 @@ export interface UsageRecord {
|
||||
authenticationKind: string | null
|
||||
accountId: string | null
|
||||
accountEmail: string | null
|
||||
accountName: string | null
|
||||
route: string
|
||||
model: string
|
||||
requestedModel: string | null
|
||||
@@ -164,7 +165,10 @@ export interface UsageAttempt {
|
||||
firstTokenMs: number | null
|
||||
latencyMs: number | null
|
||||
credentialName: string | null
|
||||
accountId: string | null
|
||||
accountName: string | null
|
||||
accountEmail: string | null
|
||||
authenticationKind: string | null
|
||||
startedAt: string
|
||||
completedAt: string | null
|
||||
}
|
||||
@@ -192,6 +196,7 @@ export interface OpsError {
|
||||
clientApiKeyId: string | null
|
||||
kind: string
|
||||
provider: string | null
|
||||
authenticationKind: string | null
|
||||
accountId: string | null
|
||||
route: string
|
||||
model: string | null
|
||||
@@ -307,6 +312,7 @@ export interface UsageInsightsOverviewResponse {
|
||||
}
|
||||
|
||||
export interface UsageDiagnosticItem {
|
||||
key: string
|
||||
name: string
|
||||
requestCount: number
|
||||
successCount: number
|
||||
@@ -314,6 +320,7 @@ export interface UsageDiagnosticItem {
|
||||
errorRate: number
|
||||
requestShare: number
|
||||
averageLatencyMs: number | null
|
||||
latencyP95Ms: number | null
|
||||
estimatedCost: string | null
|
||||
attemptCount: number
|
||||
totalTokens: number
|
||||
|
||||
@@ -13,10 +13,11 @@ const props = defineProps<{
|
||||
const open = defineModel<boolean>({ default: false })
|
||||
|
||||
const fields = computed(() => [
|
||||
{ label: '账号', value: props.record?.metadata.accountLabel },
|
||||
{ label: '时间', value: props.record?.createdAtDisplay },
|
||||
{ label: '事件', value: props.record?.kind },
|
||||
{ label: '失败分类', value: props.record?.failureClass },
|
||||
{ label: 'Provider', value: props.record?.provider },
|
||||
{ label: '平台/类型', value: providerKindLabel(props.record) },
|
||||
{ label: '端点', value: props.record?.route },
|
||||
{ label: '模型', value: props.record?.model },
|
||||
{ label: '账号 ID', value: props.record?.accountId },
|
||||
@@ -43,6 +44,14 @@ function display(value: unknown) {
|
||||
function latencyDisplay(value: unknown) {
|
||||
return typeof value === 'number' ? `${value} ms` : '—'
|
||||
}
|
||||
|
||||
function providerKindLabel(record: Awaited<ReturnType<typeof getOpsErrors>>['items'][number] | null) {
|
||||
if (!record?.provider)
|
||||
return '—'
|
||||
return record.authenticationKind
|
||||
? `${record.provider} · ${record.authenticationKind}`
|
||||
: record.provider
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -57,7 +66,7 @@ function latencyDisplay(value: unknown) {
|
||||
<template v-if="record">
|
||||
<section class="rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3.5">
|
||||
<div class="flex flex-wrap items-center gap-3">
|
||||
<UsageStatusCodeBadge :status-code="record.upstreamStatusCode" />
|
||||
<UsageStatusCodeBadge :status-code="record.clientStatusCode" />
|
||||
<span class="text-[12px] font-bold text-(--cp-text-secondary)">
|
||||
客户端 {{ display(record.clientStatusCode) }}
|
||||
</span>
|
||||
|
||||
@@ -93,11 +93,24 @@ function showDetail(record: Awaited<ReturnType<typeof getOpsErrors>>['items'][nu
|
||||
:status-code="typeof row.upstreamStatusCode === 'number' ? row.upstreamStatusCode : null"
|
||||
/>
|
||||
</template>
|
||||
<template #clientStatusCode="{ row }">
|
||||
<UsageStatusCodeBadge
|
||||
:status-code="typeof row.clientStatusCode === 'number' ? row.clientStatusCode : null"
|
||||
/>
|
||||
</template>
|
||||
<template #failureClass="{ row }">
|
||||
<span class="font-mono text-[12px] font-bold text-(--cp-danger-text)">
|
||||
{{ row.failureClass || '—' }}
|
||||
</span>
|
||||
</template>
|
||||
<template #accountId="{ row }">
|
||||
<span
|
||||
class="block max-w-full truncate font-mono text-[12px] font-bold text-(--cp-text-primary)"
|
||||
:title="row.accountId || ''"
|
||||
>
|
||||
{{ row.metadata.accountLabel || '—' }}
|
||||
</span>
|
||||
</template>
|
||||
<template #actions="{ row }">
|
||||
<BaseButton
|
||||
icon-only
|
||||
|
||||
@@ -3,8 +3,18 @@ import { displayValue, fieldLabelClass, fieldValueClass } from '../utils/detail'
|
||||
|
||||
// 详情弹窗的两列字段网格:label + 值(可选等宽字体),空值以“—”占位。
|
||||
defineProps<{
|
||||
items: Array<{ label: string, value: unknown, mono?: boolean }>
|
||||
items: Array<{ label: string, value: unknown, mono?: boolean, wrap?: boolean }>
|
||||
}>()
|
||||
|
||||
function valueClass(item: { mono?: boolean, wrap?: boolean }) {
|
||||
if (item.wrap) {
|
||||
return [
|
||||
'mt-1.5 mb-0 min-w-0 text-[12px] leading-snug font-bold text-(--cp-text-primary)',
|
||||
item.mono ? 'font-mono tabular-nums break-words' : undefined,
|
||||
]
|
||||
}
|
||||
return fieldValueClass(item.mono)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -13,7 +23,7 @@ defineProps<{
|
||||
<dt :class="fieldLabelClass">
|
||||
{{ item.label }}
|
||||
</dt>
|
||||
<dd :class="fieldValueClass(item.mono)" :title="displayValue(item.value)">
|
||||
<dd :class="valueClass(item)" :title="displayValue(item.value)">
|
||||
{{ displayValue(item.value) }}
|
||||
</dd>
|
||||
</div>
|
||||
|
||||
@@ -82,10 +82,8 @@ const sortedItems = computed(() =>
|
||||
),
|
||||
)
|
||||
|
||||
// P95 列消费的 latencyP95Ms 目前不在诊断聚合响应中,缺失时渲染占位符。
|
||||
type DiagnosticDisplayItem = Diagnostics['items'][number] & {
|
||||
nameDisplay: ReturnType<typeof diagnosticNameDisplay>
|
||||
latencyP95Ms?: number
|
||||
}
|
||||
|
||||
const displayItems = computed<DiagnosticDisplayItem[]>(() =>
|
||||
@@ -141,7 +139,7 @@ function diagnosticNameDisplay(name: string) {
|
||||
:columns="diagnosticColumns"
|
||||
:rows="displayItems"
|
||||
compact
|
||||
row-key="name"
|
||||
row-key="key"
|
||||
empty-text="暂无诊断数据"
|
||||
max-height="230px"
|
||||
min-width="640px"
|
||||
|
||||
@@ -5,9 +5,11 @@ import type { getUsageRecordDetail } from '@/api'
|
||||
import { computed } from 'vue'
|
||||
import BaseButton from '@/components/base/BaseButton.vue'
|
||||
import BaseModal from '@/components/base/BaseModal.vue'
|
||||
import BaseTable from '@/components/base/BaseTable/index.vue'
|
||||
import BaseChart from '@/components/charts/BaseChart.vue'
|
||||
import { useThemeColor } from '@/composables/useThemeColor'
|
||||
import { displayValue, fieldLabelClass, fieldValueBaseClass, fieldValueClass } from '../utils/detail'
|
||||
import { formatDuration } from '../utils/format'
|
||||
import {
|
||||
usageAccountText,
|
||||
usageBilling,
|
||||
@@ -47,6 +49,11 @@ const panelClass = 'rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3.5'
|
||||
const panelTitleClass = 'm-0 text-[12px] leading-none font-heavy text-(--cp-text-secondary)'
|
||||
|
||||
const accountDisplay = computed(() => props.record ? usageAccountText(props.record) : '—')
|
||||
const finalAttemptIndex = computed(() => {
|
||||
const attempts = props.record?.attempts ?? []
|
||||
const last = attempts[attempts.length - 1]
|
||||
return last?.attemptIndex ?? props.record?.attemptCount
|
||||
})
|
||||
const overviewItems = computed(() => [
|
||||
{ label: '时间', value: props.record?.createdAtDisplay, mono: true },
|
||||
{ label: '类型', value: props.record ? usageRecordType(props.record) : '—' },
|
||||
@@ -84,10 +91,15 @@ const detailGroups = computed(() => [
|
||||
title: '客户端与上游',
|
||||
items: [
|
||||
{ label: '客户端 IP', value: props.record ? usageClientIp(props.record) : '—', mono: true },
|
||||
{ label: 'User-Agent', value: props.record ? usageUserAgent(props.record) : '', mono: true },
|
||||
{
|
||||
label: 'User-Agent',
|
||||
value: props.record ? usageUserAgent(props.record) : '',
|
||||
mono: true,
|
||||
wrap: true,
|
||||
},
|
||||
{ label: '传输方式', value: props.record?.transport, mono: true },
|
||||
{ label: '事件类型', value: props.record?.kind, mono: true },
|
||||
{ label: '尝试序号', value: props.record?.attemptIndex },
|
||||
{ label: '尝试序号', value: finalAttemptIndex.value },
|
||||
{ label: '服务层级', value: props.record?.serviceTier, mono: true },
|
||||
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId, mono: true },
|
||||
],
|
||||
@@ -96,6 +108,61 @@ const detailGroups = computed(() => [
|
||||
const modelRouteGroup = computed(() => detailGroups.value[0])
|
||||
const clientUpstreamGroup = computed(() => detailGroups.value[1])
|
||||
|
||||
const attemptColumns = [
|
||||
{
|
||||
key: 'attemptIndex',
|
||||
label: '序号',
|
||||
width: '56px',
|
||||
align: 'center' as const,
|
||||
},
|
||||
{ key: 'outcome', label: '结果', width: '76px' },
|
||||
{ key: 'provider', label: '平台', width: '72px' },
|
||||
{ key: 'model', label: '模型', width: '140px' },
|
||||
{ key: 'transport', label: '传输', width: '84px' },
|
||||
{ key: 'statusCode', label: '状态', width: '84px' },
|
||||
{ key: 'accountLabel', label: '账号', width: '200px' },
|
||||
{
|
||||
key: 'latencyMs',
|
||||
label: '耗时',
|
||||
width: '84px',
|
||||
align: 'right' as const,
|
||||
},
|
||||
]
|
||||
|
||||
const attemptRows = computed(() =>
|
||||
(props.record?.attempts ?? []).map(attempt => ({
|
||||
id: attempt.id,
|
||||
attemptIndex: attempt.attemptIndex,
|
||||
outcome: attempt.outcome,
|
||||
provider: attempt.provider,
|
||||
model: attempt.model,
|
||||
transport: attempt.transport,
|
||||
statusCode: attempt.statusCode,
|
||||
accountLabel: attempt.accountEmail || attempt.accountName || attempt.accountId || '—',
|
||||
accountId: attempt.accountId,
|
||||
latencyMs: attempt.latencyMs,
|
||||
})),
|
||||
)
|
||||
|
||||
function attemptOutcomeText(outcome: string) {
|
||||
const labels: Record<string, string> = {
|
||||
succeeded: '成功',
|
||||
failed: '失败',
|
||||
cancelled: '取消',
|
||||
incomplete: '未完成',
|
||||
running: '进行中',
|
||||
}
|
||||
return labels[outcome] ?? outcome
|
||||
}
|
||||
|
||||
function attemptOutcomeClass(outcome: string) {
|
||||
if (outcome === 'succeeded')
|
||||
return 'text-(--cp-success-text)'
|
||||
if (outcome === 'failed')
|
||||
return 'text-(--cp-danger-text)'
|
||||
return 'text-(--cp-text-secondary)'
|
||||
}
|
||||
|
||||
const billingItems = computed(() => {
|
||||
const value = billing.value
|
||||
if (!value) {
|
||||
@@ -334,6 +401,44 @@ const tokenDonutOption = computed<EChartsOption>(() => {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="attemptRows.length" :class="panelClass">
|
||||
<h3 :class="panelTitleClass">
|
||||
尝试链路
|
||||
</h3>
|
||||
<BaseTable
|
||||
class="mt-3 min-w-0"
|
||||
:columns="attemptColumns"
|
||||
:rows="attemptRows"
|
||||
compact
|
||||
row-key="id"
|
||||
min-width="640px"
|
||||
>
|
||||
<template #outcome="{ row }">
|
||||
<span :class="attemptOutcomeClass(row.outcome)">
|
||||
{{ attemptOutcomeText(row.outcome) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #statusCode="{ row }">
|
||||
<UsageStatusCodeBadge
|
||||
:status-code="typeof row.statusCode === 'number' ? row.statusCode : null"
|
||||
/>
|
||||
</template>
|
||||
<template #accountLabel="{ row }">
|
||||
<span
|
||||
class="block max-w-full truncate font-mono text-[12px] font-bold text-(--cp-text-primary)"
|
||||
:title="row.accountId || ''"
|
||||
>
|
||||
{{ row.accountLabel }}
|
||||
</span>
|
||||
</template>
|
||||
<template #latencyMs="{ row }">
|
||||
<span class="font-mono font-bold tabular-nums text-(--cp-text-primary)">
|
||||
{{ formatDuration(row.latencyMs) }}
|
||||
</span>
|
||||
</template>
|
||||
</BaseTable>
|
||||
</section>
|
||||
|
||||
<section
|
||||
v-if="requestText || responseText"
|
||||
class="grid min-h-0 grid-cols-1 gap-3 lg:grid-cols-2"
|
||||
|
||||
@@ -96,6 +96,12 @@ export const usageRecordColumns = [
|
||||
]
|
||||
|
||||
export const opsErrorColumns = [
|
||||
{
|
||||
key: 'accountId',
|
||||
label: '账号',
|
||||
width: '230px',
|
||||
cellClass: 'font-mono text-[12px] font-emphasis',
|
||||
},
|
||||
{
|
||||
key: 'createdAtDisplay',
|
||||
label: '时间',
|
||||
@@ -104,6 +110,7 @@ export const opsErrorColumns = [
|
||||
'whitespace-nowrap font-mono text-[12px] font-emphasis tabular-nums text-(--cp-text-secondary)',
|
||||
},
|
||||
{ key: 'upstreamStatusCode', label: '上游状态', width: '96px', align: 'center' as const },
|
||||
{ key: 'clientStatusCode', label: '客户端状态', width: '96px', align: 'center' as const },
|
||||
{
|
||||
key: 'failureClass',
|
||||
label: '失败分类',
|
||||
@@ -128,12 +135,6 @@ export const opsErrorColumns = [
|
||||
width: '180px',
|
||||
cellClass: 'font-mono text-[12px] font-emphasis',
|
||||
},
|
||||
{
|
||||
key: 'accountId',
|
||||
label: '账号 ID',
|
||||
width: '230px',
|
||||
cellClass: 'font-mono text-[12px] font-emphasis',
|
||||
},
|
||||
{
|
||||
key: 'requestId',
|
||||
label: '请求 ID',
|
||||
|
||||
@@ -34,7 +34,7 @@ export function usageRecordTypeClass(record: UsageDisplayRecord) {
|
||||
}
|
||||
|
||||
export function usageAccountText(record: UsageDisplayRecord) {
|
||||
return record.accountEmail || '—'
|
||||
return record.accountEmail || record.accountName || record.accountId || '—'
|
||||
}
|
||||
|
||||
export function usageAuthenticationKind(record: UsageDisplayRecord) {
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.0.32"
|
||||
version: "3.0.33"
|
||||
|
||||
Reference in New Issue
Block a user