mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-11 08:06:14 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
676044c43f | ||
|
|
e7161da277 | ||
|
|
97faca0c45 | ||
|
|
45b473c7e2 | ||
|
|
cc69368f77 | ||
|
|
67691d18d3 | ||
|
|
85c64d35d5 | ||
|
|
9bf86f7a7c | ||
|
|
6c56a23508 | ||
|
|
b5f3cb402c | ||
|
|
57e284d7d1 | ||
|
|
21597f9e98 | ||
|
|
fa0b87dd0a |
@@ -56,6 +56,12 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
plugin_runtime,
|
||||
gateway_host::plugin_distribution::HttpPluginDistribution::new(plugin_http)
|
||||
));
|
||||
let analytics_worker = or_shutdown!(
|
||||
plugin_runtime,
|
||||
store.request_analytics_worker(std::sync::Arc::new(
|
||||
gateway_admin::ProviderUsageBillingProjector::new(providers.admin.clone()),
|
||||
))
|
||||
);
|
||||
let mut admin = or_shutdown!(
|
||||
plugin_runtime,
|
||||
gateway_admin::initialize_with_plugin_accounts(
|
||||
@@ -142,6 +148,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
);
|
||||
|
||||
let mut plan = store.take_worker_contributions();
|
||||
plan.push(analytics_worker);
|
||||
plan.push(or_shutdown!(
|
||||
plugin_runtime,
|
||||
gateway_host::retention::worker(store.retention())
|
||||
|
||||
@@ -35,7 +35,7 @@ pub use use_case::{
|
||||
client_keys::ClientKeyService,
|
||||
credentials::{CredentialsService, ProviderCredentials},
|
||||
import_tasks::ImportTasksService,
|
||||
observability::ObservabilityService,
|
||||
observability::{ObservabilityService, ProviderUsageBillingProjector},
|
||||
proxies::ProxiesService,
|
||||
settings::SettingsService,
|
||||
system::SystemService,
|
||||
@@ -441,11 +441,14 @@ async fn initialize_inner(
|
||||
plugin_inspector.clone(),
|
||||
));
|
||||
let system = Arc::new(DefaultSystemService::new(system, system_preflight));
|
||||
let observations: Arc<dyn ports::store::ObservabilityStore> = Arc::new(
|
||||
use_case::observability::CachedObservabilityStore::new(store.observability()),
|
||||
);
|
||||
let key_usage = Arc::new(use_case::key_usage::DefaultKeyUsageService::new(
|
||||
auth.clone(),
|
||||
client_key_verifier,
|
||||
store.client_keys(),
|
||||
store.observability(),
|
||||
observations.clone(),
|
||||
system.clone(),
|
||||
timezone,
|
||||
));
|
||||
@@ -506,7 +509,7 @@ async fn initialize_inner(
|
||||
)),
|
||||
client_distribution: Arc::new(DefaultClientDistributionService::new(client_distribution)),
|
||||
observability: Arc::new(DefaultObservabilityService::new(
|
||||
store.observability(),
|
||||
observations.clone(),
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
|
||||
@@ -146,9 +146,9 @@ pub struct AccountUsage {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
//! 统计投影的新鲜度、覆盖范围与预计算费用合同
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::observability::{DecimalAmount, RequestMetricPoint, UsageOverview};
|
||||
|
||||
/// 数据可见性状态,不以缺失投影冒充零统计
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum AnalyticsState {
|
||||
Ready,
|
||||
#[default]
|
||||
Building,
|
||||
Lagging,
|
||||
Stale,
|
||||
}
|
||||
|
||||
impl AnalyticsState {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ready => "ready",
|
||||
Self::Building => "building",
|
||||
Self::Lagging => "lagging",
|
||||
Self::Stale => "stale",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 与统计结果在同一读取快照内取得的覆盖和版本事实
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AnalyticsMetadata {
|
||||
pub coverage_start: Option<DateTime<Utc>>,
|
||||
pub coverage_end: Option<DateTime<Utc>>,
|
||||
pub generated_at: DateTime<Utc>,
|
||||
pub lag_seconds: u64,
|
||||
pub state: AnalyticsState,
|
||||
pub percentiles_approximate: bool,
|
||||
pub schema_version: u32,
|
||||
pub histogram_version: u32,
|
||||
/// 派生数据版本供缓存判断失效,不作为源事务提交顺序
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
/// 单次请求经 Provider 费用合同确认后可累加的 USD 情景金额
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct UsageBillingProjection {
|
||||
pub standard_amount: Option<DecimalAmount>,
|
||||
pub no_cache_amount: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 时间桶内的情景费用,未知金额不改写为零
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageProjectedCostPoint {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub standard_amount: Option<DecimalAmount>,
|
||||
pub no_cache_amount: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 同一数据库快照内的完整洞察输入
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageInsightsObservation {
|
||||
pub overview: UsageOverview,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub costs: Vec<UsageProjectedCostPoint>,
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroU64};
|
||||
|
||||
pub mod account_groups;
|
||||
pub mod accounts;
|
||||
pub mod analytics;
|
||||
pub mod audit;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
|
||||
@@ -5,6 +5,10 @@ use std::{num::NonZeroU16, str::FromStr};
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
|
||||
use super::AdminModelError;
|
||||
pub use super::analytics::{
|
||||
AnalyticsMetadata, AnalyticsState, UsageBillingProjection, UsageInsightsObservation,
|
||||
UsageProjectedCostPoint,
|
||||
};
|
||||
|
||||
/// 观测记录查询的页大小,由 HTTP、管理用例和存储端口共同使用
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -48,7 +52,7 @@ impl CalendarPeriod {
|
||||
}
|
||||
|
||||
/// 外部观测查询的 UTC 时间范围
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct TimeRange {
|
||||
pub start: DateTime<Utc>,
|
||||
pub end: DateTime<Utc>,
|
||||
@@ -89,7 +93,7 @@ impl TimeRange {
|
||||
}
|
||||
|
||||
/// 请求结果状态
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum RequestOutcome {
|
||||
Running,
|
||||
Succeeded,
|
||||
@@ -100,7 +104,7 @@ pub enum RequestOutcome {
|
||||
}
|
||||
|
||||
/// 未知但有界的请求结果值;私有字段禁止绕过 [`RequestOutcome::new`] 构造
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct OtherRequestOutcome(String);
|
||||
|
||||
impl RequestOutcome {
|
||||
@@ -151,8 +155,9 @@ impl OtherRequestOutcome {
|
||||
}
|
||||
|
||||
/// 用量记录过滤条件
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
|
||||
pub struct UsageFilter {
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
@@ -178,7 +183,7 @@ pub struct UsageQuery {
|
||||
}
|
||||
|
||||
/// Admin 确定概览的数量、趋势粒度及最近请求筛选,Store 只执行该选择
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct DashboardQuery {
|
||||
pub range: TimeRange,
|
||||
pub granularity: Granularity,
|
||||
@@ -206,6 +211,7 @@ impl DashboardQuery {
|
||||
/// 运维错误过滤条件
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct OpsErrorFilter {
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
@@ -230,7 +236,7 @@ pub struct OpsErrorQuery {
|
||||
}
|
||||
|
||||
/// 用量诊断维度
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum DiagnosticDimension {
|
||||
Provider,
|
||||
Model,
|
||||
@@ -363,8 +369,8 @@ pub struct ProviderBillingInput {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub total: CurrencyCost,
|
||||
}
|
||||
|
||||
@@ -380,8 +386,8 @@ pub struct UsageCalculatedBillingFact {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub total: CurrencyCost,
|
||||
}
|
||||
|
||||
@@ -408,7 +414,7 @@ pub struct CalculatedBillingBreakdown {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ImageBillingBreakdown {
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub input_amount: CurrencyCost,
|
||||
pub cache_read_amount: CurrencyCost,
|
||||
pub input_price_per_million: CurrencyCost,
|
||||
@@ -454,9 +460,9 @@ pub struct RequestMetrics {
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub cache_write_input_tokens: u64,
|
||||
pub reasoning_output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub first_token_latency_sum_ms: u64,
|
||||
pub first_token_latency_count: u64,
|
||||
@@ -496,7 +502,7 @@ pub struct AttemptMetrics {
|
||||
}
|
||||
|
||||
/// 时间序列粒度
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum Granularity {
|
||||
FifteenMinutes,
|
||||
Hour,
|
||||
@@ -556,9 +562,9 @@ pub struct AccountModelUsage {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -591,9 +597,9 @@ pub struct DashboardAccountUsage {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -628,7 +634,7 @@ pub struct DashboardRuntimeSlots {
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub billing_usd: Option<DecimalAmount>,
|
||||
}
|
||||
@@ -636,6 +642,7 @@ pub struct DashboardTotals {
|
||||
/// 仪表盘所需的当前区间事实及全历史累计
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardObservation {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub range: TimeRange,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: AccountPoolMetrics,
|
||||
@@ -657,9 +664,9 @@ pub struct UsageListRecord {
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
/// 账号当前备注,按内部账号 ID 关联,不属于请求历史快照
|
||||
/// 请求选号时冻结的账号备注,未采集时保持未知
|
||||
pub provider_account_notes: Option<String>,
|
||||
/// 账号当前套餐,不属于请求历史快照
|
||||
/// 请求选号时冻结的账号套餐
|
||||
pub provider_account_plan_type: Option<String>,
|
||||
pub provider_account_plan_type_display: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
@@ -669,9 +676,9 @@ pub struct UsageListRecord {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
@@ -750,9 +757,9 @@ pub struct UsageRecord {
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
@@ -830,9 +837,9 @@ pub struct UsageAttempt {
|
||||
pub message: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_source: Option<String>,
|
||||
pub cost_amount: Option<DecimalAmount>,
|
||||
@@ -862,6 +869,7 @@ pub struct ProviderObservation {
|
||||
/// 用量总览
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageOverview {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub range: TimeRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
@@ -878,6 +886,7 @@ pub struct UsageSummary {
|
||||
/// 诊断聚合结果,分母包含截取展示项之前的全部匹配请求
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DiagnosticsObservation {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub total_request_count: u64,
|
||||
pub items: Vec<DiagnosticObservation>,
|
||||
}
|
||||
@@ -1001,7 +1010,7 @@ pub struct TrendSummary {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub service_failure_count: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
@@ -1019,6 +1028,7 @@ pub struct TrendSummary {
|
||||
/// 趋势结果;API 再决定图表标签与数字格式
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Trend {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub kind: TrendKind,
|
||||
pub points: Vec<TrendPoint>,
|
||||
pub summary: TrendSummary,
|
||||
@@ -1134,7 +1144,7 @@ pub struct DashboardPeriodMetrics {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub cached_token_rate: f64,
|
||||
pub observed_cached_token_rate: Option<f64>,
|
||||
@@ -1224,7 +1234,7 @@ pub struct UsageInsightsCostPoint {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub standard_cost: Option<DecimalAmount>,
|
||||
@@ -1249,7 +1259,7 @@ pub struct UsageInsightsCost {
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub points: Vec<UsageInsightsCostPoint>,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
@@ -1259,6 +1269,7 @@ pub struct UsageInsightsCost {
|
||||
/// 用量洞察的完整用例结果;API 只负责标签、时区与字符串格式化
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageInsights {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub granularity: Granularity,
|
||||
pub health: UsageInsightsHealth,
|
||||
pub performance: UsageInsightsPerformance,
|
||||
@@ -1294,6 +1305,7 @@ pub struct DiagnosticsItem {
|
||||
/// 诊断结果
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsResult {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub dimension: DiagnosticDimension,
|
||||
pub items: Vec<DiagnosticsItem>,
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ pub struct QuotaForecastUsage {
|
||||
pub tokens: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub missing_token_count: u64,
|
||||
pub known_cost_count: u64,
|
||||
pub unavailable_cost_count: u64,
|
||||
@@ -35,7 +35,9 @@ impl QuotaForecastUsage {
|
||||
tokens: self.tokens.checked_sub(baseline.tokens)?,
|
||||
input_tokens: self.input_tokens.checked_sub(baseline.input_tokens)?,
|
||||
output_tokens: self.output_tokens.checked_sub(baseline.output_tokens)?,
|
||||
cached_tokens: self.cached_tokens.checked_sub(baseline.cached_tokens)?,
|
||||
cached_input_tokens: self
|
||||
.cached_input_tokens
|
||||
.checked_sub(baseline.cached_input_tokens)?,
|
||||
missing_token_count: self
|
||||
.missing_token_count
|
||||
.checked_sub(baseline.missing_token_count)?,
|
||||
|
||||
@@ -6,7 +6,6 @@ use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::stream::BoxStream;
|
||||
|
||||
use super::backup::BackupStorePorts;
|
||||
use crate::model::{
|
||||
@@ -30,7 +29,8 @@ use crate::model::{
|
||||
observability::{
|
||||
DashboardObservation, DashboardQuery, DiagnosticDimension, DiagnosticsObservation,
|
||||
Granularity, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange,
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageOverview, UsagePage, UsageQuery,
|
||||
UsageBillingProjection, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsightsObservation, UsageOverview, UsagePage, UsageQuery,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, CredentialDetails, CredentialImportCommit, CredentialImportResult,
|
||||
@@ -421,13 +421,20 @@ pub trait AccountGroupStore: Send + Sync {
|
||||
) -> AdminStoreResult<AccountGroupMutation>;
|
||||
}
|
||||
|
||||
/// 逐条读取的已计算费用事实;消费结束或丢弃时释放查询资源
|
||||
pub type UsageCalculatedBillingStream<'a> =
|
||||
BoxStream<'a, AdminStoreResult<UsageCalculatedBillingFact>>;
|
||||
/// 后台投影复用 Admin 与 Provider 的费用规则,Store 不重新解释价格
|
||||
pub trait UsageBillingProjector: Send + Sync {
|
||||
fn project(
|
||||
&self,
|
||||
fact: &UsageCalculatedBillingFact,
|
||||
) -> Result<UsageBillingProjection, crate::model::AdminError>;
|
||||
}
|
||||
|
||||
/// 用量、趋势、诊断与运维错误的只读能力
|
||||
#[async_trait]
|
||||
pub trait ObservabilityStore: Send + Sync {
|
||||
/// 缓存必须与删除和后台投影提交的版本一致
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64>;
|
||||
|
||||
/// 返回历史统计区间和指定观测时刻下的实时账号状态
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
@@ -448,14 +455,13 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>>;
|
||||
|
||||
/// 流式返回可由 Provider 重新校验的已计算费用事实,不保证顺序
|
||||
/// 查询及解码错误由流返回;调用方应逐条聚合,避免收集整个区间
|
||||
fn usage_calculated_billing_facts(
|
||||
/// 同一读取快照中的汇总、趋势和已投影情景费用
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> UsageCalculatedBillingStream<'_>;
|
||||
) -> AdminStoreResult<UsageInsightsObservation>;
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage>;
|
||||
|
||||
|
||||
@@ -192,10 +192,12 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
.ok_or_else(|| AdminError::internal("时间超出支持范围"))?,
|
||||
end: now,
|
||||
};
|
||||
let (overview, trend, health_points) = futures::try_join!(
|
||||
self.observations.usage_summary(query.range, filter.clone()),
|
||||
self.observations
|
||||
.usage_trend(query.range, filter, Granularity::for_range(query.range)),
|
||||
let (observation, health_points) = futures::try_join!(
|
||||
self.observations.usage_insights_observation(
|
||||
query.range,
|
||||
filter,
|
||||
Granularity::for_range(query.range)
|
||||
),
|
||||
self.observations.usage_trend(
|
||||
today,
|
||||
usage_filter(&id, None),
|
||||
@@ -205,8 +207,8 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
.map_err(|error| map_store_error(error, "key usage overview"))?;
|
||||
Ok(Some(KeyUsageOverview {
|
||||
key,
|
||||
overview,
|
||||
trend,
|
||||
overview: observation.overview,
|
||||
trend: observation.trend,
|
||||
health_timeline: health_timeline_at(&health_points, now, self.timezone)?,
|
||||
}))
|
||||
}
|
||||
|
||||
+125
-147
@@ -4,12 +4,10 @@ use std::{
|
||||
collections::{BTreeMap, HashMap, btree_map::Entry},
|
||||
str::FromStr,
|
||||
sync::Arc,
|
||||
time::{Duration as StdDuration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use futures::TryStreamExt;
|
||||
use gateway_core::{account::ProviderAccountId, routing::ProviderKind};
|
||||
|
||||
use crate::{
|
||||
@@ -21,16 +19,16 @@ use crate::{
|
||||
DiagnosticDimension, DiagnosticsItem, DiagnosticsResult, Granularity, HealthStatus,
|
||||
HealthTimeline, HealthTimelinePoint, OpsErrorPage, OpsErrorQuery, ProviderBillingInput,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, Trend, TrendKind, TrendPoint,
|
||||
TrendSummary, UsageBilling, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsights, UsageInsightsCost, UsageInsightsCostPoint, UsageInsightsHealth,
|
||||
UsageInsightsHealthPoint, UsageInsightsPerformance, UsageInsightsPerformancePoint,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageSummary,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCalculatedBillingFact,
|
||||
UsageDetail, UsageFilter, UsageInsights, UsageInsightsCost, UsageInsightsCostPoint,
|
||||
UsageInsightsHealth, UsageInsightsHealthPoint, UsageInsightsPerformance,
|
||||
UsageInsightsPerformancePoint, UsageOverview, UsagePage, UsageQuery, UsageSummary,
|
||||
},
|
||||
provider_credentials::ProviderQuotaRequest,
|
||||
},
|
||||
ports::{
|
||||
provider::{ProviderAdminErrorKind, ProviderAdminRegistry},
|
||||
store::{AccountStore, ObservabilityStore, SettingsStore, UsageCalculatedBillingStream},
|
||||
store::{AccountStore, ObservabilityStore, SettingsStore, UsageBillingProjector},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -40,58 +38,8 @@ const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15;
|
||||
const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
|
||||
const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
|
||||
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0;
|
||||
const DASHBOARD_QUERY_BUCKET_SECONDS: i64 = 2;
|
||||
const DASHBOARD_QUERY_CACHE_RETENTION: StdDuration = StdDuration::from_secs(2);
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
||||
struct DashboardQueryKey {
|
||||
start: DateTime<Utc>,
|
||||
end_bucket: i64,
|
||||
}
|
||||
|
||||
impl DashboardQueryKey {
|
||||
fn new(range: TimeRange) -> Self {
|
||||
Self {
|
||||
start: range.start,
|
||||
end_bucket: range
|
||||
.end
|
||||
.timestamp()
|
||||
.div_euclid(DASHBOARD_QUERY_BUCKET_SECONDS),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DashboardQueryEntry {
|
||||
created_at: Instant,
|
||||
result: Arc<tokio::sync::OnceCell<Result<DashboardResult, AdminError>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DashboardQueryCache {
|
||||
entries: tokio::sync::Mutex<HashMap<DashboardQueryKey, DashboardQueryEntry>>,
|
||||
}
|
||||
|
||||
impl DashboardQueryCache {
|
||||
async fn result_cell(
|
||||
&self,
|
||||
key: DashboardQueryKey,
|
||||
) -> Arc<tokio::sync::OnceCell<Result<DashboardResult, AdminError>>> {
|
||||
let now = Instant::now();
|
||||
let mut entries = self.entries.lock().await;
|
||||
entries.retain(|_, entry| {
|
||||
now.duration_since(entry.created_at) < DASHBOARD_QUERY_CACHE_RETENTION
|
||||
});
|
||||
Arc::clone(
|
||||
&entries
|
||||
.entry(key)
|
||||
.or_insert_with(|| DashboardQueryEntry {
|
||||
created_at: now,
|
||||
result: Arc::new(tokio::sync::OnceCell::new()),
|
||||
})
|
||||
.result,
|
||||
)
|
||||
}
|
||||
}
|
||||
mod store_cache;
|
||||
pub(crate) use store_cache::CachedObservabilityStore;
|
||||
|
||||
/// API 消费的观测控制面服务
|
||||
#[async_trait]
|
||||
@@ -130,7 +78,6 @@ pub(crate) struct DefaultObservabilityService {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
dashboard_queries: DashboardQueryCache,
|
||||
}
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
@@ -148,7 +95,6 @@ impl DefaultObservabilityService {
|
||||
accounts,
|
||||
settings,
|
||||
providers,
|
||||
dashboard_queries: DashboardQueryCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,7 +131,7 @@ impl DefaultObservabilityService {
|
||||
let yesterday = dashboard_period_metrics(&observation.trend, yesterday_start, today_start);
|
||||
let total_billing_usd = observation.totals.billing_usd.clone();
|
||||
let total_cached_token_rate = rate_or_zero(
|
||||
observation.totals.cached_tokens,
|
||||
observation.totals.cached_input_tokens,
|
||||
observation.totals.input_tokens,
|
||||
);
|
||||
let (first_token_latency_sum_ms, first_token_latency_count) = observation
|
||||
@@ -199,7 +145,11 @@ impl DefaultObservabilityService {
|
||||
});
|
||||
let average_first_token_latency_ms =
|
||||
average(first_token_latency_sum_ms, first_token_latency_count);
|
||||
let trend = trend(TrendKind::Usage, observation.trend.clone())?;
|
||||
let trend = trend(
|
||||
TrendKind::Usage,
|
||||
observation.trend.clone(),
|
||||
observation.analytics.clone(),
|
||||
)?;
|
||||
let health_timeline =
|
||||
health_timeline_at(&observation.trend, observation.range.end, self.timezone)?;
|
||||
let wire_profiles = self
|
||||
@@ -247,15 +197,12 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range: TimeRange,
|
||||
kind: TrendKind,
|
||||
) -> Result<DashboardResult, AdminError> {
|
||||
let cell = self
|
||||
.dashboard_queries
|
||||
.result_cell(DashboardQueryKey::new(range))
|
||||
.await;
|
||||
let mut result = cell
|
||||
.get_or_init(|| self.load_dashboard_summary(range))
|
||||
.await
|
||||
.clone()?;
|
||||
result.trend = trend(kind, result.observation.trend.clone())?;
|
||||
let mut result = self.load_dashboard_summary(range).await?;
|
||||
result.trend = trend(
|
||||
kind,
|
||||
result.observation.trend.clone(),
|
||||
result.observation.analytics.clone(),
|
||||
)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -264,12 +211,16 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range: TimeRange,
|
||||
kind: TrendKind,
|
||||
) -> Result<Trend, AdminError> {
|
||||
let points = self
|
||||
let observation = self
|
||||
.store
|
||||
.dashboard_trend(range, Granularity::for_range(range))
|
||||
.usage_insights_observation(
|
||||
range,
|
||||
UsageFilter::default(),
|
||||
Granularity::for_range(range),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "dashboard trend"))?;
|
||||
trend(kind, points)
|
||||
trend(kind, observation.trend, observation.overview.analytics)
|
||||
}
|
||||
|
||||
async fn usage_records(&self, query: UsageQuery) -> Result<UsagePage, AdminError> {
|
||||
@@ -321,21 +272,22 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
filter: UsageFilter,
|
||||
) -> Result<UsageInsights, AdminError> {
|
||||
let granularity = Granularity::for_range(range);
|
||||
let aggregates = async {
|
||||
futures::try_join!(
|
||||
self.store.usage_summary(range, filter.clone()),
|
||||
self.store.usage_trend(range, filter.clone(), granularity),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "usage insights"))
|
||||
};
|
||||
let ((overview, trend), standard_costs) = futures::try_join!(
|
||||
aggregates,
|
||||
recover_standard_costs(
|
||||
&self.providers,
|
||||
self.store
|
||||
.usage_calculated_billing_facts(range, filter.clone(), granularity),
|
||||
),
|
||||
)?;
|
||||
let observation = self
|
||||
.store
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage insights"))?;
|
||||
let mut standard_costs = UsageCostScenarios::default();
|
||||
for point in observation.costs {
|
||||
if let Some(amount) = point.standard_amount {
|
||||
standard_costs.standard.add(point.bucket_start, amount)?;
|
||||
}
|
||||
if let Some(amount) = point.no_cache_amount {
|
||||
standard_costs.no_cache.add(point.bucket_start, amount)?;
|
||||
}
|
||||
}
|
||||
let overview = observation.overview;
|
||||
let trend = observation.trend;
|
||||
build_usage_insights(overview, trend, standard_costs)
|
||||
}
|
||||
|
||||
@@ -389,7 +341,11 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Ok(DiagnosticsResult { dimension, items })
|
||||
Ok(DiagnosticsResult {
|
||||
analytics: observation.analytics,
|
||||
dimension,
|
||||
items,
|
||||
})
|
||||
}
|
||||
|
||||
async fn ops_errors(&self, query: OpsErrorQuery) -> Result<OpsErrorPage, AdminError> {
|
||||
@@ -533,14 +489,14 @@ fn build_usage_insights(
|
||||
tier_premium,
|
||||
tokens_per_request: rate_or_zero(requests.total_tokens, requests.request_count),
|
||||
cost_per_successful_request,
|
||||
cached_token_rate: rate_or_zero(requests.cached_tokens, requests.input_tokens),
|
||||
cached_token_rate: rate_or_zero(requests.cached_input_tokens, requests.input_tokens),
|
||||
cache_hit_request_rate: ratio(
|
||||
requests.cache_hit_request_count,
|
||||
requests.cache_eligible_request_count,
|
||||
),
|
||||
input_tokens: requests.input_tokens,
|
||||
output_tokens: requests.output_tokens,
|
||||
cached_tokens: requests.cached_tokens,
|
||||
cached_input_tokens: requests.cached_input_tokens,
|
||||
total_tokens: requests.total_tokens,
|
||||
points: trend
|
||||
.iter()
|
||||
@@ -548,7 +504,7 @@ fn build_usage_insights(
|
||||
bucket_start: point.bucket_start,
|
||||
input_tokens: point.metrics.input_tokens,
|
||||
output_tokens: point.metrics.output_tokens,
|
||||
cached_tokens: point.metrics.cached_tokens,
|
||||
cached_input_tokens: point.metrics.cached_input_tokens,
|
||||
total_tokens: point.metrics.total_tokens,
|
||||
estimated_cost: usd_cost(&point.costs),
|
||||
standard_cost: standard_costs_by_bucket.get(&point.bucket_start).cloned(),
|
||||
@@ -558,7 +514,7 @@ fn build_usage_insights(
|
||||
usd_cost(&point.costs).as_ref(),
|
||||
),
|
||||
cached_token_rate: rate_or_zero(
|
||||
point.metrics.cached_tokens,
|
||||
point.metrics.cached_input_tokens,
|
||||
point.metrics.input_tokens,
|
||||
),
|
||||
cache_hit_request_rate: ratio(
|
||||
@@ -571,6 +527,7 @@ fn build_usage_insights(
|
||||
coverage: overview.attempts.cost_coverage.clone(),
|
||||
};
|
||||
Ok(UsageInsights {
|
||||
analytics: overview.analytics,
|
||||
granularity,
|
||||
health,
|
||||
performance,
|
||||
@@ -620,58 +577,65 @@ struct UsageCostScenarios {
|
||||
no_cache: ScenarioCosts,
|
||||
}
|
||||
|
||||
async fn recover_standard_costs(
|
||||
providers: &ProviderAdminRegistry,
|
||||
mut facts: UsageCalculatedBillingStream<'_>,
|
||||
) -> Result<UsageCostScenarios, AdminError> {
|
||||
let mut scenarios = UsageCostScenarios::default();
|
||||
while let Some(mut fact) = facts
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage billing facts"))?
|
||||
{
|
||||
/// 在后台投影时恢复费用场景,查询不再逐请求读取计费快照
|
||||
pub struct ProviderUsageBillingProjector {
|
||||
providers: ProviderAdminRegistry,
|
||||
}
|
||||
|
||||
impl ProviderUsageBillingProjector {
|
||||
#[must_use]
|
||||
pub fn new(providers: ProviderAdminRegistry) -> Self {
|
||||
Self { providers }
|
||||
}
|
||||
}
|
||||
|
||||
impl UsageBillingProjector for ProviderUsageBillingProjector {
|
||||
fn project(
|
||||
&self,
|
||||
fact: &UsageCalculatedBillingFact,
|
||||
) -> Result<UsageBillingProjection, AdminError> {
|
||||
let Ok(provider_kind) = ProviderKind::new(fact.provider_kind.clone()) else {
|
||||
continue;
|
||||
return Ok(UsageBillingProjection::default());
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: fact.upstream_model_id.clone(),
|
||||
service_tier: fact.service_tier.clone(),
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
cache_write_tokens: fact.cache_write_tokens,
|
||||
cached_input_tokens: fact.cached_input_tokens,
|
||||
cache_write_input_tokens: fact.cache_write_input_tokens,
|
||||
total: fact.total.clone(),
|
||||
};
|
||||
let breakdown = match fact
|
||||
.breakdown
|
||||
.take()
|
||||
.filter(|b| b.total_amount == input.total)
|
||||
.as_ref()
|
||||
.filter(|breakdown| breakdown.total_amount == input.total)
|
||||
.cloned()
|
||||
.map_or_else(
|
||||
|| providers.calculated_billing(&provider_kind, &input),
|
||||
|b| Ok(Some(b)),
|
||||
|| self.providers.calculated_billing(&provider_kind, &input),
|
||||
|breakdown| Ok(Some(breakdown)),
|
||||
) {
|
||||
Ok(breakdown) => breakdown,
|
||||
Err(error) if error.kind() == ProviderAdminErrorKind::Unsupported => continue,
|
||||
Err(error) if error.kind() == ProviderAdminErrorKind::Unsupported => {
|
||||
return Ok(UsageBillingProjection::default());
|
||||
}
|
||||
Err(error) => return Err(map_provider_error(error, "usage billing")),
|
||||
};
|
||||
let Some(breakdown) = breakdown else {
|
||||
continue;
|
||||
return Ok(UsageBillingProjection::default());
|
||||
};
|
||||
if !breakdown
|
||||
.standard_amount
|
||||
.currency
|
||||
.eq_ignore_ascii_case("USD")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
scenarios
|
||||
.standard
|
||||
.add(fact.bucket_start, breakdown.standard_amount.amount.clone())?;
|
||||
if let Some(no_cache_cost) = no_cache_cost(&fact, &breakdown) {
|
||||
scenarios.no_cache.add(fact.bucket_start, no_cache_cost)?;
|
||||
return Ok(UsageBillingProjection::default());
|
||||
}
|
||||
Ok(UsageBillingProjection {
|
||||
standard_amount: Some(breakdown.standard_amount.amount.clone()),
|
||||
no_cache_amount: no_cache_cost(fact, &breakdown),
|
||||
})
|
||||
}
|
||||
Ok(scenarios)
|
||||
}
|
||||
|
||||
fn no_cache_cost(
|
||||
@@ -682,19 +646,21 @@ fn no_cache_cost(
|
||||
let input_tokens = fact
|
||||
.input_tokens?
|
||||
.checked_sub(image.map_or(0, |i| i.input_tokens))?;
|
||||
let cached_tokens = fact
|
||||
.cached_tokens
|
||||
let cached_input_tokens = fact
|
||||
.cached_input_tokens
|
||||
.unwrap_or_default()
|
||||
.checked_sub(image.map_or(0, |i| i.cached_tokens))?
|
||||
.checked_sub(image.map_or(0, |i| i.cached_input_tokens))?
|
||||
.min(input_tokens);
|
||||
let cache_write_tokens = fact
|
||||
.cache_write_tokens
|
||||
let cache_write_input_tokens = fact
|
||||
.cache_write_input_tokens
|
||||
.unwrap_or_default()
|
||||
.min(input_tokens.saturating_sub(cached_tokens));
|
||||
.min(input_tokens.saturating_sub(cached_input_tokens));
|
||||
let input_rate = decimal(&breakdown.input_price_per_million.amount)?;
|
||||
let replaced_input_amount = input_rate
|
||||
.scaled()
|
||||
.checked_mul(u128::from(cached_tokens.saturating_add(cache_write_tokens)))?
|
||||
.checked_mul(u128::from(
|
||||
cached_input_tokens.saturating_add(cache_write_input_tokens),
|
||||
))?
|
||||
.checked_div(1_000_000)?;
|
||||
let mut total = decimal(&breakdown.total_amount.amount)?
|
||||
.scaled()
|
||||
@@ -704,7 +670,7 @@ fn no_cache_cost(
|
||||
if let Some(image) = image {
|
||||
let image_no_cache = decimal(&image.input_price_per_million.amount)?
|
||||
.scaled()
|
||||
.checked_mul(u128::from(image.cached_tokens))?
|
||||
.checked_mul(u128::from(image.cached_input_tokens))?
|
||||
.checked_div(1_000_000)?;
|
||||
total = total
|
||||
.checked_sub(decimal(&image.cache_read_amount.amount)?.scaled())?
|
||||
@@ -829,8 +795,8 @@ impl DefaultObservabilityService {
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
total: total.clone(),
|
||||
};
|
||||
if let Ok(Some(breakdown)) = self.providers.calculated_billing(&provider_kind, &input) {
|
||||
@@ -861,8 +827,8 @@ impl DefaultObservabilityService {
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
total: total.clone(),
|
||||
};
|
||||
if let Ok(Some(breakdown)) = self.providers.calculated_billing(&provider_kind, &input) {
|
||||
@@ -951,7 +917,11 @@ pub(super) fn health_timeline_at(
|
||||
})
|
||||
}
|
||||
|
||||
fn trend(kind: TrendKind, points: Vec<RequestMetricPoint>) -> Result<Trend, AdminError> {
|
||||
fn trend(
|
||||
kind: TrendKind,
|
||||
points: Vec<RequestMetricPoint>,
|
||||
analytics: crate::model::analytics::AnalyticsMetadata,
|
||||
) -> Result<Trend, AdminError> {
|
||||
let mut metrics = RequestMetrics::default();
|
||||
let mut coverage = CostCoverage::default();
|
||||
let mut costs = BTreeMap::<String, DecimalAmount>::new();
|
||||
@@ -1007,13 +977,14 @@ fn trend(kind: TrendKind, points: Vec<RequestMetricPoint>) -> Result<Trend, Admi
|
||||
.filter_map(|point| point.metrics.output_throughput_p50)
|
||||
.min();
|
||||
Ok(Trend {
|
||||
analytics,
|
||||
kind,
|
||||
points: points.into_iter().map(trend_point).collect(),
|
||||
summary: TrendSummary {
|
||||
request_count: metrics.request_count,
|
||||
input_tokens: metrics.input_tokens,
|
||||
output_tokens: metrics.output_tokens,
|
||||
cached_tokens: metrics.cached_tokens,
|
||||
cached_input_tokens: metrics.cached_input_tokens,
|
||||
total_tokens: metrics.total_tokens,
|
||||
service_failure_count,
|
||||
average_latency_ms,
|
||||
@@ -1044,7 +1015,10 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint {
|
||||
point.metrics.first_token_latency_sum_ms,
|
||||
point.metrics.first_token_latency_count,
|
||||
),
|
||||
cached_token_rate: rate_or_zero(point.metrics.cached_tokens, point.metrics.input_tokens),
|
||||
cached_token_rate: rate_or_zero(
|
||||
point.metrics.cached_input_tokens,
|
||||
point.metrics.input_tokens,
|
||||
),
|
||||
cache_hit_request_rate: ratio(
|
||||
point.metrics.cache_hit_request_count,
|
||||
point.metrics.cache_eligible_request_count,
|
||||
@@ -1070,11 +1044,13 @@ fn dashboard_period_metrics(
|
||||
result.request_count = result.request_count.saturating_add(metrics.request_count);
|
||||
result.input_tokens = result.input_tokens.saturating_add(metrics.input_tokens);
|
||||
result.output_tokens = result.output_tokens.saturating_add(metrics.output_tokens);
|
||||
result.cached_tokens = result.cached_tokens.saturating_add(metrics.cached_tokens);
|
||||
result.cached_input_tokens = result
|
||||
.cached_input_tokens
|
||||
.saturating_add(metrics.cached_input_tokens);
|
||||
result.total_tokens = result.total_tokens.saturating_add(metrics.total_tokens);
|
||||
}
|
||||
result.cached_token_rate = rate_or_zero(result.cached_tokens, result.input_tokens);
|
||||
result.observed_cached_token_rate = ratio(result.cached_tokens, result.input_tokens);
|
||||
result.cached_token_rate = rate_or_zero(result.cached_input_tokens, result.input_tokens);
|
||||
result.observed_cached_token_rate = ratio(result.cached_input_tokens, result.input_tokens);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -1091,13 +1067,15 @@ fn add_request_metrics(total: &mut RequestMetrics, value: &RequestMetrics) {
|
||||
.saturating_add(value.caller_error_count);
|
||||
total.input_tokens = total.input_tokens.saturating_add(value.input_tokens);
|
||||
total.output_tokens = total.output_tokens.saturating_add(value.output_tokens);
|
||||
total.cached_tokens = total.cached_tokens.saturating_add(value.cached_tokens);
|
||||
total.cache_write_tokens = total
|
||||
.cache_write_tokens
|
||||
.saturating_add(value.cache_write_tokens);
|
||||
total.reasoning_tokens = total
|
||||
.reasoning_tokens
|
||||
.saturating_add(value.reasoning_tokens);
|
||||
total.cached_input_tokens = total
|
||||
.cached_input_tokens
|
||||
.saturating_add(value.cached_input_tokens);
|
||||
total.cache_write_input_tokens = total
|
||||
.cache_write_input_tokens
|
||||
.saturating_add(value.cache_write_input_tokens);
|
||||
total.reasoning_output_tokens = total
|
||||
.reasoning_output_tokens
|
||||
.saturating_add(value.reasoning_output_tokens);
|
||||
total.total_tokens = total.total_tokens.saturating_add(value.total_tokens);
|
||||
total.first_token_latency_sum_ms = total
|
||||
.first_token_latency_sum_ms
|
||||
@@ -0,0 +1,298 @@
|
||||
//! 有界统计缓存:身份和完整筛选参与键,删除版本阻止旧数据复活
|
||||
|
||||
use crate::model::observability::*;
|
||||
use crate::ports::store::{
|
||||
AdminStoreError, AdminStoreErrorKind, AdminStoreResult, ObservabilityStore,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use std::{collections::HashMap, future::Future, hash::Hash, sync::Arc, time::Duration};
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
|
||||
const CAPACITY: usize = 64;
|
||||
const TTL: Duration = Duration::from_secs(15);
|
||||
const MAX_STALE: Duration = Duration::from_secs(60);
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
type Slot<T> = Arc<Mutex<Option<Cached<T>>>>;
|
||||
struct Cached<T> {
|
||||
value: T,
|
||||
created: Instant,
|
||||
}
|
||||
struct Entry<T> {
|
||||
slot: Slot<T>,
|
||||
touched: Instant,
|
||||
}
|
||||
struct Cache<K, T> {
|
||||
entries: Mutex<HashMap<K, Entry<T>>>,
|
||||
}
|
||||
impl<K, T> Default for Cache<K, T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait AnalyticsValue: Clone {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata;
|
||||
}
|
||||
impl AnalyticsValue for DashboardObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for UsageOverview {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for UsageInsightsObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.overview.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for DiagnosticsObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
|
||||
impl<K: Eq + Hash, T: AnalyticsValue> Cache<K, T> {
|
||||
async fn get<F, Fut>(
|
||||
&self,
|
||||
key: K,
|
||||
store: &dyn ObservabilityStore,
|
||||
ttl: Duration,
|
||||
load: F,
|
||||
) -> AdminStoreResult<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = AdminStoreResult<T>>,
|
||||
{
|
||||
// 刷新失败后的同键等待也计入预算,避免串行重试把延迟放大为队列长度倍数
|
||||
tokio::time::timeout(
|
||||
REQUEST_TIMEOUT,
|
||||
self.get_within_budget(key, store, ttl, load),
|
||||
)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
"observability cache",
|
||||
"observability cache execution budget is exhausted",
|
||||
)
|
||||
.with_source(source)
|
||||
})?
|
||||
}
|
||||
|
||||
async fn get_within_budget<F, Fut>(
|
||||
&self,
|
||||
key: K,
|
||||
store: &dyn ObservabilityStore,
|
||||
ttl: Duration,
|
||||
load: F,
|
||||
) -> AdminStoreResult<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = AdminStoreResult<T>>,
|
||||
{
|
||||
let slot = {
|
||||
let mut entries = self.entries.lock().await;
|
||||
if !entries.contains_key(&key) && entries.len() >= CAPACITY {
|
||||
let oldest = entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| Arc::strong_count(&entry.slot) == 1)
|
||||
.min_by_key(|(_, entry)| entry.touched)
|
||||
.map(|(_, entry)| entry.touched);
|
||||
if let Some(oldest) = oldest {
|
||||
entries.retain(|_, entry| entry.touched != oldest);
|
||||
}
|
||||
}
|
||||
if !entries.contains_key(&key) && entries.len() >= CAPACITY {
|
||||
drop(entries);
|
||||
return load().await;
|
||||
}
|
||||
let entry = entries.entry(key).or_insert_with(|| Entry {
|
||||
slot: Arc::new(Mutex::new(None)),
|
||||
touched: Instant::now(),
|
||||
});
|
||||
entry.touched = Instant::now();
|
||||
entry.slot.clone()
|
||||
};
|
||||
// 同键刷新只执行一次;失败不会缓存成成功,也不延长旧结果寿命
|
||||
let mut cached = slot.lock().await;
|
||||
// 等锁期间代次可能推进;取得同键锁后再核对,失败时也不能回用已删除事实
|
||||
let generation = store.analytics_cache_generation().await?;
|
||||
if let Some(entry) = cached.as_mut()
|
||||
&& entry.value.metadata().generation == generation
|
||||
&& entry.created.elapsed() < ttl
|
||||
{
|
||||
return Ok(with_age(entry.value.clone(), false));
|
||||
}
|
||||
match load().await {
|
||||
Ok(value) => {
|
||||
*cached = Some(Cached {
|
||||
value: value.clone(),
|
||||
created: Instant::now(),
|
||||
});
|
||||
Ok(value)
|
||||
}
|
||||
Err(error) => {
|
||||
if ttl == TTL
|
||||
&& error.kind() == AdminStoreErrorKind::Unavailable
|
||||
&& let Some(entry) = cached.as_mut()
|
||||
&& entry.created.elapsed() <= MAX_STALE
|
||||
&& Utc::now()
|
||||
.signed_duration_since(entry.value.metadata().generated_at)
|
||||
.to_std()
|
||||
.unwrap_or_default()
|
||||
<= MAX_STALE
|
||||
&& entry.value.metadata().state != AnalyticsState::Building
|
||||
&& entry.value.metadata().generation
|
||||
== store.analytics_cache_generation().await?
|
||||
{
|
||||
return Ok(with_age(entry.value.clone(), true));
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn with_age<T: AnalyticsValue>(mut value: T, stale: bool) -> T {
|
||||
let metadata = value.metadata();
|
||||
metadata.lag_seconds = metadata.lag_seconds.saturating_add(
|
||||
Utc::now()
|
||||
.signed_duration_since(metadata.generated_at)
|
||||
.num_seconds()
|
||||
.max(0) as u64,
|
||||
);
|
||||
if stale {
|
||||
metadata.state = AnalyticsState::Stale;
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
fn normalized(mut filter: UsageFilter) -> UsageFilter {
|
||||
filter.routing_group_refs.sort_unstable();
|
||||
filter.routing_group_refs.dedup();
|
||||
filter
|
||||
}
|
||||
|
||||
type UsageKey = (TimeRange, UsageFilter);
|
||||
pub(crate) struct CachedObservabilityStore {
|
||||
inner: Arc<dyn ObservabilityStore>,
|
||||
dashboard: Cache<DashboardQuery, DashboardObservation>,
|
||||
summaries: Cache<UsageKey, UsageOverview>,
|
||||
insights: Cache<(UsageKey, Granularity), UsageInsightsObservation>,
|
||||
diagnostics: Cache<(UsageKey, DiagnosticDimension, u16), DiagnosticsObservation>,
|
||||
}
|
||||
impl CachedObservabilityStore {
|
||||
pub(crate) fn new(inner: Arc<dyn ObservabilityStore>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
dashboard: Cache::default(),
|
||||
summaries: Cache::default(),
|
||||
insights: Cache::default(),
|
||||
diagnostics: Cache::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for CachedObservabilityStore {
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
self.inner.analytics_cache_generation().await
|
||||
}
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
query: DashboardQuery,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<DashboardObservation> {
|
||||
// 复合概览还包含账号容量和最近请求,保持原有两秒实时刷新上限
|
||||
self.dashboard
|
||||
.get(
|
||||
query.clone(),
|
||||
self.inner.as_ref(),
|
||||
Duration::from_secs(2),
|
||||
|| self.inner.dashboard_summary(query, observed_at),
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn dashboard_trend(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
self.inner.dashboard_trend(range, granularity).await
|
||||
}
|
||||
async fn usage_trend(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
self.inner.usage_trend(range, filter, granularity).await
|
||||
}
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
let filter = normalized(filter);
|
||||
self.insights
|
||||
.get(
|
||||
((range, filter.clone()), granularity),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
let filter = normalized(filter);
|
||||
self.summaries
|
||||
.get((range, filter.clone()), self.inner.as_ref(), TTL, || {
|
||||
self.inner.usage_summary(range, filter)
|
||||
})
|
||||
.await
|
||||
}
|
||||
async fn usage_diagnostics(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<DiagnosticsObservation> {
|
||||
let filter = normalized(filter);
|
||||
self.diagnostics
|
||||
.get(
|
||||
((range, filter.clone()), dimension, limit),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_diagnostics(range, filter, dimension, limit)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
self.inner.list_usage_records(query).await
|
||||
}
|
||||
async fn usage_record_detail(&self, request_id: &str) -> AdminStoreResult<UsageDetail> {
|
||||
self.inner.usage_record_detail(request_id).await
|
||||
}
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
self.inner.list_ops_errors(query).await
|
||||
}
|
||||
}
|
||||
@@ -170,9 +170,9 @@ fn usage_window_should_not_use_daily_rolling_usage_for_weekly_statistics() {
|
||||
success_count: 5,
|
||||
input_tokens: Some(100),
|
||||
output_tokens: Some(20),
|
||||
cached_tokens: Some(80),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(80),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
image_request_count: 0,
|
||||
|
||||
@@ -115,9 +115,9 @@ fn usage_window(key: &str, group: &str, seconds: u64) -> ProviderQuotaWindow {
|
||||
success_count: 0,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
|
||||
@@ -32,9 +32,9 @@ fn window(key: &str, days: u64) -> ProviderQuotaWindow {
|
||||
success_count: 10,
|
||||
input_tokens: Some(900),
|
||||
output_tokens: Some(100),
|
||||
cached_tokens: Some(800),
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: Some(800),
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
@@ -84,7 +84,7 @@ fn samples(quota: &ProviderQuota) -> Vec<QuotaForecastSample> {
|
||||
tokens: usage.total_tokens.unwrap_or(0),
|
||||
input_tokens: usage.input_tokens.unwrap_or(0),
|
||||
output_tokens: usage.output_tokens.unwrap_or(0),
|
||||
cached_tokens: usage.cached_tokens.unwrap_or(0),
|
||||
cached_input_tokens: usage.cached_input_tokens.unwrap_or(0),
|
||||
missing_token_count: u64::from(usage.total_tokens.is_none()),
|
||||
known_cost_count: if usd.is_some() {
|
||||
usage.cost_coverage.known_count()
|
||||
|
||||
@@ -3032,9 +3032,9 @@ fn quota_local_usage(account_id: &str, total_tokens: u64) -> AccountUsage {
|
||||
success_count: 1,
|
||||
input_tokens: Some(total_tokens),
|
||||
output_tokens: Some(0),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(0),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
image_request_count: 0,
|
||||
|
||||
@@ -772,15 +772,17 @@ impl ObservabilityStore for UnavailableStore {
|
||||
Err(unavailable("usage trend"))
|
||||
}
|
||||
|
||||
fn usage_calculated_billing_facts(
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
Err(unavailable("analytics generation"))
|
||||
}
|
||||
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
_: Granularity,
|
||||
) -> gateway_admin::ports::store::UsageCalculatedBillingStream<'_> {
|
||||
Box::pin(futures::stream::once(async {
|
||||
Err(unavailable("usage billing facts"))
|
||||
}))
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
Err(unavailable("usage insights"))
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
|
||||
+275
-71
@@ -17,13 +17,13 @@ use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, CostCoverage, CurrencyCost, DashboardAccountUsage,
|
||||
DashboardObservation, DashboardQuery, DashboardRuntimeSlots, DiagnosticDimension,
|
||||
DiagnosticObservation, DiagnosticsObservation, Granularity, HealthStatus,
|
||||
LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
||||
AccountPoolMetrics, AnalyticsState, AttemptMetrics, CostCoverage, CurrencyCost,
|
||||
DashboardAccountUsage, DashboardObservation, DashboardQuery, DashboardRuntimeSlots,
|
||||
DiagnosticDimension, DiagnosticObservation, DiagnosticsObservation, Granularity,
|
||||
HealthStatus, LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
||||
PercentileMilliseconds, RequestMetricPoint, RequestMetrics, TimeRange, TrendKind,
|
||||
UsageBilling, UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageQuery,
|
||||
UsageBilling, UsageDetail, UsageFilter, UsageInsightsObservation, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageProjectedCostPoint, UsageQuery,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -300,7 +300,7 @@ async fn dashboard_summary_should_observe_current_accounts_for_a_historical_rang
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_coalesce_concurrent_requests_in_one_short_window() {
|
||||
async fn dashboard_summary_should_keep_distinct_exact_ranges_in_one_short_window() {
|
||||
let end_bucket = Utc::now().timestamp().div_euclid(2) * 2;
|
||||
let first_end = DateTime::from_timestamp(end_bucket, 100_000_000).expect("first end");
|
||||
let second_end = DateTime::from_timestamp(end_bucket, 900_000_000).expect("second end");
|
||||
@@ -330,7 +330,7 @@ async fn dashboard_summary_should_coalesce_concurrent_requests_in_one_short_wind
|
||||
|
||||
first.expect("first dashboard summary");
|
||||
second.expect("second dashboard summary");
|
||||
assert_eq!(store.dashboard_summary_calls(), 1);
|
||||
assert_eq!(store.dashboard_summary_calls(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -403,6 +403,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
..RequestMetrics::default()
|
||||
};
|
||||
store.replace_overview(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: metrics.clone(),
|
||||
attempts: AttemptMetrics {
|
||||
@@ -431,23 +432,13 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
.expect("bucket USD cost"),
|
||||
}],
|
||||
}]);
|
||||
store.replace_calculated_billing_facts(vec![UsageCalculatedBillingFact {
|
||||
breakdown: None,
|
||||
store.replace_projected_costs(vec![UsageProjectedCostPoint {
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
.expect("calculated total"),
|
||||
},
|
||||
standard_amount: Some("1".parse().expect("standard cost")),
|
||||
no_cache_amount: Some("1.25".parse().expect("no-cache cost")),
|
||||
}]);
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 4,
|
||||
items: vec![diagnostic("openai", 3), diagnostic("xai", 1)],
|
||||
});
|
||||
@@ -543,6 +534,7 @@ async fn diagnostics_should_preserve_request_order_when_a_small_sample_fails() {
|
||||
failure.success_count = 0;
|
||||
failure.failure_count = 1;
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 101,
|
||||
items: vec![success, failure],
|
||||
});
|
||||
@@ -566,6 +558,7 @@ async fn diagnostics_should_use_the_total_before_truncating_groups() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 200,
|
||||
items: (0..100)
|
||||
.map(|index| diagnostic(&format!("model-{index}"), 1))
|
||||
@@ -591,6 +584,7 @@ async fn diagnostics_should_count_each_retried_request_once() {
|
||||
item.retry_count = 6;
|
||||
item.retried_request_count = 2;
|
||||
store.replace_diagnostics(DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 8,
|
||||
items: vec![item],
|
||||
});
|
||||
@@ -691,26 +685,16 @@ async fn usage_records_should_enrich_provider_reported_totals_when_pricing_match
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_insights_should_reject_partial_costs_when_billing_stream_fails() {
|
||||
async fn usage_insights_should_reject_partial_costs_when_insights_fail() {
|
||||
let now = Utc::now();
|
||||
let range = observation_range(now);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.replace_calculated_billing_facts(vec![UsageCalculatedBillingFact {
|
||||
breakdown: None,
|
||||
store.replace_projected_costs(vec![UsageProjectedCostPoint {
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: "1.25".parse().expect("cost"),
|
||||
},
|
||||
standard_amount: Some("1".parse().expect("standard cost")),
|
||||
no_cache_amount: Some("1.25".parse().expect("no-cache cost")),
|
||||
}]);
|
||||
store.billing_stream_fails.store(true, Ordering::SeqCst);
|
||||
store.insights_fail.store(true, Ordering::SeqCst);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
assert!(
|
||||
services
|
||||
@@ -741,9 +725,9 @@ async fn dashboard_should_preserve_provider_plan_names() {
|
||||
success_count: 0,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
@@ -783,8 +767,9 @@ async fn dashboard_should_preserve_provider_plan_names() {
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
calculated_billing_facts: Mutex<Vec<UsageCalculatedBillingFact>>,
|
||||
billing_stream_fails: AtomicBool,
|
||||
projected_costs: Mutex<Vec<UsageProjectedCostPoint>>,
|
||||
insights_fail: AtomicBool,
|
||||
summary_fail: AtomicBool,
|
||||
diagnostics: Mutex<DiagnosticsObservation>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
summary_observed_at: Mutex<Option<DateTime<Utc>>>,
|
||||
@@ -794,6 +779,7 @@ struct FixtureObservabilityStore {
|
||||
account_usage: Mutex<Vec<DashboardAccountUsage>>,
|
||||
dashboard_delay: Mutex<StdDuration>,
|
||||
dashboard_summary_calls: AtomicUsize,
|
||||
usage_summary_filters: Mutex<Vec<UsageFilter>>,
|
||||
}
|
||||
|
||||
impl FixtureObservabilityStore {
|
||||
@@ -801,13 +787,15 @@ impl FixtureObservabilityStore {
|
||||
Self {
|
||||
trend: Mutex::new(Vec::new()),
|
||||
overview: Mutex::new(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
providers: Vec::new(),
|
||||
}),
|
||||
calculated_billing_facts: Mutex::new(Vec::new()),
|
||||
billing_stream_fails: AtomicBool::new(false),
|
||||
projected_costs: Mutex::new(Vec::new()),
|
||||
insights_fail: AtomicBool::new(false),
|
||||
summary_fail: AtomicBool::new(false),
|
||||
diagnostics: Mutex::new(DiagnosticsObservation::default()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
summary_observed_at: Mutex::new(None),
|
||||
@@ -817,6 +805,7 @@ impl FixtureObservabilityStore {
|
||||
account_usage: Mutex::new(Vec::new()),
|
||||
dashboard_delay: Mutex::new(StdDuration::ZERO),
|
||||
dashboard_summary_calls: AtomicUsize::new(0),
|
||||
usage_summary_filters: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -836,11 +825,8 @@ impl FixtureObservabilityStore {
|
||||
*self.overview.lock().expect("overview") = overview;
|
||||
}
|
||||
|
||||
fn replace_calculated_billing_facts(&self, facts: Vec<UsageCalculatedBillingFact>) {
|
||||
*self
|
||||
.calculated_billing_facts
|
||||
.lock()
|
||||
.expect("calculated billing facts") = facts;
|
||||
fn replace_projected_costs(&self, facts: Vec<UsageProjectedCostPoint>) {
|
||||
*self.projected_costs.lock().expect("projected costs") = facts;
|
||||
}
|
||||
|
||||
fn replace_diagnostics(&self, diagnostics: DiagnosticsObservation) {
|
||||
@@ -874,6 +860,7 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
tokio::time::sleep(dashboard_delay).await;
|
||||
}
|
||||
Ok(DashboardObservation {
|
||||
analytics: self.overview.lock().expect("overview").analytics.clone(),
|
||||
range: query.range,
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
@@ -902,25 +889,29 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
Ok(self.trend.lock().expect("trend").clone())
|
||||
}
|
||||
|
||||
fn usage_calculated_billing_facts(
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
Ok(self.overview.lock().expect("overview").analytics.generation)
|
||||
}
|
||||
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> gateway_admin::ports::store::UsageCalculatedBillingStream<'_> {
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.usage_granularities.lock().unwrap().push(granularity);
|
||||
let facts = self
|
||||
.calculated_billing_facts
|
||||
.lock()
|
||||
.expect("calculated billing facts")
|
||||
.clone();
|
||||
let error = self
|
||||
.billing_stream_fails
|
||||
.load(Ordering::SeqCst)
|
||||
.then(|| Err(super::unavailable("billing stream failed")));
|
||||
Box::pin(futures::stream::iter(
|
||||
facts.into_iter().map(Ok).chain(error),
|
||||
))
|
||||
if self.insights_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection read failed"));
|
||||
}
|
||||
Ok(UsageInsightsObservation {
|
||||
overview: self.overview.lock().expect("overview").clone(),
|
||||
trend: self.trend.lock().expect("trend").clone(),
|
||||
costs: self
|
||||
.projected_costs
|
||||
.lock()
|
||||
.expect("projected costs")
|
||||
.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
@@ -938,8 +929,26 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
Err(super::unavailable("usage detail"))
|
||||
}
|
||||
|
||||
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
|
||||
Ok(self.overview.lock().expect("overview").clone())
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
self.usage_summary_filters
|
||||
.lock()
|
||||
.expect("summary filters")
|
||||
.push(filter);
|
||||
let delay = *self.dashboard_delay.lock().expect("query delay");
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
if self.summary_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection summary failed"));
|
||||
}
|
||||
Ok(UsageOverview {
|
||||
range,
|
||||
..self.overview.lock().expect("overview").clone()
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_diagnostics(
|
||||
@@ -1114,9 +1123,9 @@ fn total_record(
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(0),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
@@ -1229,9 +1238,204 @@ async fn admin_selects_dashboard_defaults_and_matching_usage_buckets_at_range_bo
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
*store.usage_granularities.lock().unwrap(),
|
||||
vec![expected, expected]
|
||||
assert_eq!(*store.usage_granularities.lock().unwrap(), vec![expected]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_summary_cache_should_coalesce_identical_queries_and_invalidate_deleted_generation() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(StdDuration::from_millis(20));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
let filter = UsageFilter {
|
||||
client_api_key_ref: Some("deleted-key".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let (first, second) = tokio::join!(
|
||||
service.usage_summary(range, filter.clone()),
|
||||
service.usage_summary(range, filter.clone()),
|
||||
);
|
||||
first.expect("first summary");
|
||||
second.expect("coalesced summary");
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 1);
|
||||
{
|
||||
let mut overview = store.overview.lock().unwrap();
|
||||
overview.analytics.generation = 1;
|
||||
overview.requests.request_count = 7;
|
||||
}
|
||||
let refreshed = service
|
||||
.usage_summary(range, filter)
|
||||
.await
|
||||
.expect("generation refresh");
|
||||
assert_eq!(refreshed.overview.analytics.generation, 1);
|
||||
assert_eq!(refreshed.overview.requests.request_count, 7);
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the_first_load() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(StdDuration::from_millis(20));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
let (first, second, ()) = tokio::join!(
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
async {
|
||||
tokio::time::sleep(StdDuration::from_millis(5)).await;
|
||||
store.overview.lock().unwrap().analytics.generation = 1;
|
||||
},
|
||||
);
|
||||
assert_eq!(
|
||||
first.expect("first summary").overview.analytics.generation,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
second
|
||||
.expect("coalesced summary")
|
||||
.overview
|
||||
.analytics
|
||||
.generation,
|
||||
1
|
||||
);
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_sets() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
for (key, groups) in [
|
||||
("key-a", vec!["deleted-b", "deleted-a", "deleted-b"]),
|
||||
("key-a", vec!["deleted-a", "deleted-b"]),
|
||||
("key-b", vec!["deleted-a", "deleted-b"]),
|
||||
("key-a", vec!["deleted-a"]),
|
||||
] {
|
||||
service
|
||||
.usage_summary(
|
||||
range,
|
||||
UsageFilter {
|
||||
client_api_key_ref: Some(key.to_owned()),
|
||||
routing_group_refs: groups.into_iter().map(str::to_owned).collect(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("isolated summary");
|
||||
}
|
||||
let calls = store.usage_summary_filters.lock().unwrap();
|
||||
assert_eq!(calls.len(), 3);
|
||||
assert_eq!(calls[0].routing_group_refs, ["deleted-a", "deleted-b"]);
|
||||
assert_eq!(calls[1].client_api_key_ref.as_deref(), Some("key-b"));
|
||||
assert_eq!(calls[2].routing_group_refs, ["deleted-a"]);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow_recovery() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(StdDuration::from_secs(7));
|
||||
store.summary_fail.store(true, Ordering::SeqCst);
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
let started = tokio::time::Instant::now();
|
||||
let (first, second, third, fourth) = tokio::join!(
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
service.usage_summary(range, UsageFilter::default()),
|
||||
);
|
||||
assert!(first.is_err() && second.is_err() && third.is_err() && fourth.is_err());
|
||||
assert!(
|
||||
started.elapsed() <= StdDuration::from_secs(10),
|
||||
"同键失败不能使后续请求逐个等待完整数据库超时: {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
|
||||
store.summary_fail.store(false, Ordering::SeqCst);
|
||||
store.set_dashboard_delay(StdDuration::ZERO);
|
||||
service
|
||||
.usage_summary(range, UsageFilter::default())
|
||||
.await
|
||||
.expect("cancelled cache refresh releases its slot for recovery");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn insights_cache_should_bound_stale_lifetime_without_renewing_failed_refreshes() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
{
|
||||
let mut overview = store.overview.lock().unwrap();
|
||||
overview.analytics.state = AnalyticsState::Ready;
|
||||
overview.analytics.generated_at = Utc::now();
|
||||
overview.analytics.lag_seconds = 7;
|
||||
overview.requests.request_count = 19;
|
||||
}
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
store.insights_fail.store(true, Ordering::SeqCst);
|
||||
tokio::time::advance(StdDuration::from_secs(16)).await;
|
||||
let stale = service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stale.analytics.state, AnalyticsState::Stale);
|
||||
assert!(stale.analytics.lag_seconds >= 7);
|
||||
assert_eq!(stale.health.total_requests, 19);
|
||||
tokio::time::advance(StdDuration::from_secs(45)).await;
|
||||
assert!(
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn insights_cache_should_reject_deleted_building_and_old_generation_times() {
|
||||
for reason in ["deleted", "building", "generated-too-long-ago"] {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
{
|
||||
let mut overview = store.overview.lock().unwrap();
|
||||
overview.analytics.state = if reason == "building" {
|
||||
AnalyticsState::Building
|
||||
} else {
|
||||
AnalyticsState::Ready
|
||||
};
|
||||
overview.analytics.generated_at = Utc::now()
|
||||
- Duration::seconds(if reason == "generated-too-long-ago" {
|
||||
61
|
||||
} else {
|
||||
0
|
||||
});
|
||||
}
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
store.insights_fail.store(true, Ordering::SeqCst);
|
||||
if reason == "deleted" {
|
||||
store.overview.lock().unwrap().analytics.generation += 1;
|
||||
}
|
||||
tokio::time::advance(StdDuration::from_secs(16)).await;
|
||||
assert!(
|
||||
service
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"{reason}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -308,8 +308,8 @@ pub(super) fn quota_local_usage(usage: &AccountUsage, time: crate::time::TimePre
|
||||
"inputTokensDisplay": display_optional_tokens(usage.input_tokens),
|
||||
"outputTokens": usage.output_tokens.unwrap_or_default(),
|
||||
"outputTokensDisplay": display_optional_tokens(usage.output_tokens),
|
||||
"cachedTokens": usage.cached_tokens.unwrap_or_default(),
|
||||
"cachedTokensDisplay": display_optional_tokens(usage.cached_tokens),
|
||||
"cachedInputTokens": usage.cached_input_tokens.unwrap_or_default(),
|
||||
"cachedInputTokensDisplay": display_optional_tokens(usage.cached_input_tokens),
|
||||
"imageInputTokens": usage.image_input_tokens.unwrap_or_default(),
|
||||
"imageInputTokensDisplay": display_optional_tokens(usage.image_input_tokens),
|
||||
"imageOutputTokens": usage.image_output_tokens.unwrap_or_default(),
|
||||
@@ -355,10 +355,10 @@ pub(super) fn account_usage_view(
|
||||
input_tokens_display: display_optional_tokens(usage.input_tokens),
|
||||
output_tokens: usage.output_tokens,
|
||||
output_tokens_display: display_optional_tokens(usage.output_tokens),
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cached_tokens_display: display_optional_tokens(usage.cached_tokens),
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
reasoning_tokens_display: display_optional_tokens(usage.reasoning_tokens),
|
||||
cached_input_tokens: usage.cached_input_tokens,
|
||||
cached_input_tokens_display: display_optional_tokens(usage.cached_input_tokens),
|
||||
reasoning_output_tokens: usage.reasoning_output_tokens,
|
||||
reasoning_output_tokens_display: display_optional_tokens(usage.reasoning_output_tokens),
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_input_tokens_display: display_optional_tokens(usage.image_input_tokens),
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
@@ -369,10 +369,8 @@ pub(super) fn account_usage_view(
|
||||
image_request_failed_count_display: format_number(usage.image_request_failed_count),
|
||||
total_tokens: usage.total_tokens,
|
||||
total_tokens_display: display_optional_tokens(usage.total_tokens),
|
||||
created_tokens: usage.cache_write_tokens,
|
||||
created_tokens_display: display_optional_tokens(usage.cache_write_tokens),
|
||||
read_tokens: usage.cached_tokens,
|
||||
read_tokens_display: display_optional_tokens(usage.cached_tokens),
|
||||
cache_write_input_tokens: usage.cache_write_input_tokens,
|
||||
cache_write_input_tokens_display: display_optional_tokens(usage.cache_write_input_tokens),
|
||||
last_used_at: usage.last_used_at.map(|value| time.rfc3339(&value)),
|
||||
last_used_at_full_display: usage.last_used_at.map(|value| time.datetime(&value)),
|
||||
last_used_at_display: usage
|
||||
@@ -429,8 +427,8 @@ pub(super) fn account_model_usage_view(
|
||||
input_tokens_display: display_optional_tokens(usage.input_tokens),
|
||||
output_tokens: usage.output_tokens,
|
||||
output_tokens_display: display_optional_tokens(usage.output_tokens),
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cached_tokens_display: display_optional_tokens(usage.cached_tokens),
|
||||
cached_input_tokens: usage.cached_input_tokens,
|
||||
cached_input_tokens_display: display_optional_tokens(usage.cached_input_tokens),
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_input_tokens_display: display_optional_tokens(usage.image_input_tokens),
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
@@ -478,10 +476,10 @@ pub(super) fn empty_account_usage() -> AccountUsageView {
|
||||
input_tokens_display: "—".to_owned(),
|
||||
output_tokens: None,
|
||||
output_tokens_display: "—".to_owned(),
|
||||
cached_tokens: None,
|
||||
cached_tokens_display: "—".to_owned(),
|
||||
reasoning_tokens: None,
|
||||
reasoning_tokens_display: "—".to_owned(),
|
||||
cached_input_tokens: None,
|
||||
cached_input_tokens_display: "—".to_owned(),
|
||||
reasoning_output_tokens: None,
|
||||
reasoning_output_tokens_display: "—".to_owned(),
|
||||
image_input_tokens: None,
|
||||
image_input_tokens_display: "—".to_owned(),
|
||||
image_output_tokens: None,
|
||||
@@ -492,10 +490,8 @@ pub(super) fn empty_account_usage() -> AccountUsageView {
|
||||
image_request_failed_count_display: "—".to_owned(),
|
||||
total_tokens: None,
|
||||
total_tokens_display: "—".to_owned(),
|
||||
created_tokens: None,
|
||||
created_tokens_display: "—".to_owned(),
|
||||
read_tokens: None,
|
||||
read_tokens_display: "—".to_owned(),
|
||||
cache_write_input_tokens: None,
|
||||
cache_write_input_tokens_display: "—".to_owned(),
|
||||
last_used_at: None,
|
||||
last_used_at_full_display: None,
|
||||
last_used_at_display: "—".to_owned(),
|
||||
|
||||
@@ -435,10 +435,10 @@ pub struct AccountUsageView {
|
||||
pub input_tokens_display: String,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub output_tokens_display: String,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cached_tokens_display: String,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub reasoning_tokens_display: String,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cached_input_tokens_display: String,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens_display: String,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_input_tokens_display: String,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
@@ -449,10 +449,8 @@ pub struct AccountUsageView {
|
||||
pub image_request_failed_count_display: String,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub total_tokens_display: String,
|
||||
pub created_tokens: Option<u64>,
|
||||
pub created_tokens_display: String,
|
||||
pub read_tokens: Option<u64>,
|
||||
pub read_tokens_display: String,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens_display: String,
|
||||
pub last_used_at: Option<String>,
|
||||
pub last_used_at_display: String,
|
||||
pub last_used_at_full_display: Option<String>,
|
||||
@@ -477,8 +475,8 @@ pub struct ModelUsageView {
|
||||
pub input_tokens_display: String,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub output_tokens_display: String,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cached_tokens_display: String,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cached_input_tokens_display: String,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_input_tokens_display: String,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
|
||||
@@ -58,9 +58,9 @@ pub(crate) fn request_metrics_view(metrics: &domain::RequestMetrics) -> RequestM
|
||||
caller_error_count: metrics.caller_error_count,
|
||||
input_tokens: metrics.input_tokens,
|
||||
output_tokens: metrics.output_tokens,
|
||||
cached_tokens: metrics.cached_tokens,
|
||||
cache_write_tokens: metrics.cache_write_tokens,
|
||||
reasoning_tokens: metrics.reasoning_tokens,
|
||||
cached_input_tokens: metrics.cached_input_tokens,
|
||||
cache_write_input_tokens: metrics.cache_write_input_tokens,
|
||||
reasoning_output_tokens: metrics.reasoning_output_tokens,
|
||||
total_tokens: metrics.total_tokens,
|
||||
}
|
||||
}
|
||||
@@ -103,9 +103,9 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
|
||||
TokenDetailsView {
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
reasoning_tokens: record.reasoning_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
reasoning_output_tokens: record.reasoning_output_tokens,
|
||||
image_input_tokens: record.image_input_tokens,
|
||||
image_output_tokens: record.image_output_tokens,
|
||||
total_tokens: record.total_tokens,
|
||||
@@ -115,14 +115,14 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
|
||||
output_tokens_display: record
|
||||
.output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
cached_tokens_display: record
|
||||
.cached_tokens
|
||||
cached_input_tokens_display: record
|
||||
.cached_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
cache_write_tokens_display: record
|
||||
.cache_write_tokens
|
||||
cache_write_input_tokens_display: record
|
||||
.cache_write_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
reasoning_tokens_display: record
|
||||
.reasoning_tokens
|
||||
reasoning_output_tokens_display: record
|
||||
.reasoning_output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
image_input_tokens_display: record
|
||||
.image_input_tokens
|
||||
@@ -140,9 +140,9 @@ pub(crate) fn usage_list_token_details(record: &domain::UsageListRecord) -> Toke
|
||||
TokenDetailsView {
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
reasoning_tokens: record.reasoning_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
reasoning_output_tokens: record.reasoning_output_tokens,
|
||||
image_input_tokens: record.image_input_tokens,
|
||||
image_output_tokens: record.image_output_tokens,
|
||||
total_tokens: record.total_tokens,
|
||||
@@ -152,14 +152,14 @@ pub(crate) fn usage_list_token_details(record: &domain::UsageListRecord) -> Toke
|
||||
output_tokens_display: record
|
||||
.output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
cached_tokens_display: record
|
||||
.cached_tokens
|
||||
cached_input_tokens_display: record
|
||||
.cached_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
cache_write_tokens_display: record
|
||||
.cache_write_tokens
|
||||
cache_write_input_tokens_display: record
|
||||
.cache_write_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
reasoning_tokens_display: record
|
||||
.reasoning_tokens
|
||||
reasoning_output_tokens_display: record
|
||||
.reasoning_output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
image_input_tokens_display: record
|
||||
.image_input_tokens
|
||||
@@ -430,9 +430,9 @@ pub(crate) fn usage_record_view(
|
||||
latency_ms: record.latency_ms,
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
reasoning_tokens: record.reasoning_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
cache_write_input_tokens: record.cache_write_input_tokens,
|
||||
reasoning_output_tokens: record.reasoning_output_tokens,
|
||||
image_input_tokens: record.image_input_tokens,
|
||||
image_output_tokens: record.image_output_tokens,
|
||||
message,
|
||||
@@ -526,7 +526,7 @@ pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptV
|
||||
estimated_cost_currency: attempt.cost_currency,
|
||||
input_tokens: attempt.input_tokens,
|
||||
output_tokens: attempt.output_tokens,
|
||||
cached_tokens: attempt.cached_tokens,
|
||||
cached_input_tokens: attempt.cached_input_tokens,
|
||||
total_tokens: attempt.total_tokens,
|
||||
first_token_ms: None,
|
||||
latency_ms: attempt.latency_ms,
|
||||
@@ -683,8 +683,8 @@ pub(crate) fn trend_point_view(
|
||||
input_tokens_value: point.metrics.input_tokens,
|
||||
output_tokens: format_compact_number(point.metrics.output_tokens),
|
||||
output_tokens_value: point.metrics.output_tokens,
|
||||
cached_tokens: format_compact_number(point.metrics.cached_tokens),
|
||||
cached_tokens_value: point.metrics.cached_tokens,
|
||||
cached_input_tokens: format_compact_number(point.metrics.cached_input_tokens),
|
||||
cached_input_tokens_value: point.metrics.cached_input_tokens,
|
||||
cache_hit_rate_value: point.cached_token_rate,
|
||||
tokens_value: point.metrics.total_tokens,
|
||||
errors: format_compact_number(point.service_failure_count),
|
||||
@@ -749,7 +749,7 @@ pub(crate) fn trend_summary_view(
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "缓存".to_owned(),
|
||||
value: format_compact_number(summary.cached_tokens),
|
||||
value: format_compact_number(summary.cached_input_tokens),
|
||||
ratio: None,
|
||||
},
|
||||
],
|
||||
@@ -800,6 +800,7 @@ pub(crate) fn trend_view(
|
||||
time: crate::time::TimePresenter,
|
||||
) -> TrendData {
|
||||
TrendData {
|
||||
analytics: analytics_view(trend.analytics, time),
|
||||
kind,
|
||||
summary: trend_summary_view(kind, &trend.summary),
|
||||
points: trend
|
||||
@@ -913,6 +914,7 @@ pub(crate) fn dashboard_view(
|
||||
rotation_strategy,
|
||||
} = result;
|
||||
let domain::DashboardObservation {
|
||||
analytics,
|
||||
runtime_slots: _,
|
||||
range,
|
||||
totals,
|
||||
@@ -982,6 +984,7 @@ pub(crate) fn dashboard_view(
|
||||
.saturating_sub(provider_accounts.normal);
|
||||
|
||||
DashboardDataView {
|
||||
analytics: analytics_view(analytics, time),
|
||||
as_of: range.end,
|
||||
as_of_display: time.datetime(&range.end),
|
||||
cards: DashboardCardsView {
|
||||
@@ -1013,7 +1016,7 @@ pub(crate) fn dashboard_view(
|
||||
today_hit_rate_value: today.observed_cached_token_rate,
|
||||
yesterday_hit_rate_value: yesterday.observed_cached_token_rate,
|
||||
total_hit_rate: display_rate(total_cached_token_rate),
|
||||
total_cached_tokens: format_compact_number(totals.cached_tokens),
|
||||
total_cached_input_tokens: format_compact_number(totals.cached_input_tokens),
|
||||
average_first_token_latency_ms: display_duration(average_first_token_latency_ms),
|
||||
},
|
||||
},
|
||||
@@ -1046,14 +1049,18 @@ pub(crate) fn dashboard_view(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn usage_summary_view(summary: domain::UsageSummary) -> UsageSummaryView {
|
||||
pub(crate) fn usage_summary_view(
|
||||
summary: domain::UsageSummary,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageSummaryView {
|
||||
let overview = summary.overview;
|
||||
UsageSummaryView {
|
||||
analytics: analytics_view(overview.analytics, time),
|
||||
total_requests: format_compact_number(overview.requests.request_count),
|
||||
input_tokens: format_compact_number(overview.requests.input_tokens),
|
||||
output_tokens: format_compact_number(overview.requests.output_tokens),
|
||||
cached_tokens: format_compact_number(overview.requests.cached_tokens),
|
||||
cache_write_tokens: format_compact_number(overview.requests.cache_write_tokens),
|
||||
cached_input_tokens: format_compact_number(overview.requests.cached_input_tokens),
|
||||
cache_write_input_tokens: format_compact_number(overview.requests.cache_write_input_tokens),
|
||||
total_tokens: format_compact_number(overview.requests.total_tokens),
|
||||
average_latency_ms: display_duration(summary.average_latency_ms),
|
||||
logical_requests: request_metrics_view(&overview.requests),
|
||||
@@ -1139,7 +1146,7 @@ pub(crate) fn usage_insights_view(
|
||||
label: time.label(point.bucket_start, "%m-%d %H:%M"),
|
||||
input_tokens: point.input_tokens,
|
||||
output_tokens: point.output_tokens,
|
||||
cached_tokens: point.cached_tokens,
|
||||
cached_input_tokens: point.cached_input_tokens,
|
||||
total_tokens: point.total_tokens,
|
||||
estimated_cost: point.estimated_cost.as_ref().map(ToString::to_string),
|
||||
standard_cost: point.standard_cost.as_ref().map(ToString::to_string),
|
||||
@@ -1150,6 +1157,7 @@ pub(crate) fn usage_insights_view(
|
||||
})
|
||||
.collect();
|
||||
UsageInsightsOverviewView {
|
||||
analytics: analytics_view(insights.analytics, time),
|
||||
granularity: granularity_name(insights.granularity).to_owned(),
|
||||
health: OverviewHealthView {
|
||||
total_requests: insights.health.total_requests,
|
||||
@@ -1270,7 +1278,7 @@ pub(crate) fn usage_insights_view(
|
||||
cache_hit_request_rate: insights.cost.cache_hit_request_rate,
|
||||
input_tokens: insights.cost.input_tokens,
|
||||
output_tokens: insights.cost.output_tokens,
|
||||
cached_tokens: insights.cost.cached_tokens,
|
||||
cached_input_tokens: insights.cost.cached_input_tokens,
|
||||
total_tokens: insights.cost.total_tokens,
|
||||
points: cost_points,
|
||||
costs: cost_views(&insights.cost.costs),
|
||||
@@ -1302,8 +1310,10 @@ pub(crate) fn granularity_name(granularity: domain::Granularity) -> &'static str
|
||||
pub(crate) fn diagnostics_view(
|
||||
result: domain::DiagnosticsResult,
|
||||
dimension: DiagnosticDimension,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> DiagnosticsView {
|
||||
DiagnosticsView {
|
||||
analytics: analytics_view(result.analytics, time),
|
||||
dimension: dimension.display_name().to_owned(),
|
||||
items: result
|
||||
.items
|
||||
@@ -1356,3 +1366,22 @@ pub(crate) fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
pub(crate) fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
|
||||
map_admin_service_error(error)
|
||||
}
|
||||
|
||||
pub(crate) fn analytics_view(
|
||||
metadata: domain::AnalyticsMetadata,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> AnalyticsView {
|
||||
AnalyticsView {
|
||||
coverage_start: metadata.coverage_start,
|
||||
coverage_start_display: metadata.coverage_start.map(|value| time.datetime(&value)),
|
||||
coverage_end: metadata.coverage_end,
|
||||
coverage_end_display: metadata.coverage_end.map(|value| time.datetime(&value)),
|
||||
generated_at: metadata.generated_at,
|
||||
generated_at_display: time.datetime(&metadata.generated_at),
|
||||
lag_seconds: metadata.lag_seconds,
|
||||
state: metadata.state.as_str().to_owned(),
|
||||
percentiles_approximate: metadata.percentiles_approximate,
|
||||
schema_version: metadata.schema_version,
|
||||
histogram_version: metadata.histogram_version,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ impl DashboardQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub routing_group_refs: Option<String>,
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub kind: Option<String>,
|
||||
@@ -76,6 +77,7 @@ impl DetailQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DiagnosticsQuery {
|
||||
pub routing_group_refs: Option<String>,
|
||||
pub dimension: Option<String>,
|
||||
pub period: Option<String>,
|
||||
pub as_of: Option<i64>,
|
||||
@@ -98,6 +100,7 @@ impl DiagnosticsQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct OpsQuery {
|
||||
pub routing_group_refs: Option<String>,
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub kind: Option<String>,
|
||||
@@ -298,6 +301,7 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, Wi
|
||||
})
|
||||
});
|
||||
Ok(domain::UsageFilter {
|
||||
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?,
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
@@ -357,6 +361,7 @@ pub(crate) fn ops_command(
|
||||
timezone,
|
||||
)?,
|
||||
filter: domain::OpsErrorFilter {
|
||||
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?,
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
@@ -395,3 +400,25 @@ fn validate_pagination(
|
||||
}
|
||||
Ok((current_page, page_size))
|
||||
}
|
||||
|
||||
/// 逗号分隔的稳定分组身份按集合归一化,不要求分组当前仍存在
|
||||
pub(crate) fn routing_group_refs(value: Option<&str>) -> Result<Vec<String>, WireValidationError> {
|
||||
let Some(value) = value else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let mut refs = Vec::new();
|
||||
for item in value.split(',') {
|
||||
let item = item.trim();
|
||||
if item.is_empty()
|
||||
|| item.len() > 128
|
||||
|| item.chars().any(char::is_control)
|
||||
|| refs.len() >= 32
|
||||
{
|
||||
return Err(WireValidationError::new("routingGroupRefs"));
|
||||
}
|
||||
refs.push(item.to_owned());
|
||||
}
|
||||
refs.sort();
|
||||
refs.dedup();
|
||||
Ok(refs)
|
||||
}
|
||||
|
||||
@@ -160,7 +160,10 @@ where
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_summary_view(result)),
|
||||
AdminEnvelope::ok(usage_summary_view(
|
||||
result,
|
||||
crate::time::TimePresenter::new(state.admin_services().timezone()),
|
||||
)),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -212,6 +215,8 @@ where
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())
|
||||
.map_err(map_wire_error)?,
|
||||
provider_kind: non_empty(query.provider),
|
||||
model: non_empty(query.model),
|
||||
status_code: parse_status(query.status_code).map_err(map_wire_error)?,
|
||||
@@ -226,7 +231,11 @@ where
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(diagnostics_view(result, dimension)),
|
||||
AdminEnvelope::ok(diagnostics_view(
|
||||
result,
|
||||
dimension,
|
||||
crate::time::TimePresenter::new(state.admin_services().timezone()),
|
||||
)),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -18,17 +18,17 @@ pub struct PageData<T> {
|
||||
pub struct TokenDetailsView {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub input_tokens_display: String,
|
||||
pub output_tokens_display: String,
|
||||
pub cached_tokens_display: String,
|
||||
pub cache_write_tokens_display: String,
|
||||
pub reasoning_tokens_display: String,
|
||||
pub cached_input_tokens_display: String,
|
||||
pub cache_write_input_tokens_display: String,
|
||||
pub reasoning_output_tokens_display: String,
|
||||
pub image_input_tokens_display: String,
|
||||
pub image_output_tokens_display: String,
|
||||
pub total_tokens_display: String,
|
||||
@@ -162,9 +162,9 @@ pub struct UsageRecordView {
|
||||
pub latency_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub message: String,
|
||||
@@ -258,7 +258,7 @@ pub struct UsageAttemptView {
|
||||
pub estimated_cost_currency: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
@@ -288,6 +288,7 @@ pub struct UsageRecordDetailView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrendData {
|
||||
pub analytics: AnalyticsView,
|
||||
pub kind: TrendKind,
|
||||
pub points: Vec<TrendPointView>,
|
||||
pub summary: Vec<TrendSummaryView>,
|
||||
@@ -306,8 +307,8 @@ pub struct TrendPointView {
|
||||
pub input_tokens_value: u64,
|
||||
pub output_tokens: String,
|
||||
pub output_tokens_value: u64,
|
||||
pub cached_tokens: String,
|
||||
pub cached_tokens_value: u64,
|
||||
pub cached_input_tokens: String,
|
||||
pub cached_input_tokens_value: u64,
|
||||
pub cache_hit_rate_value: f64,
|
||||
pub tokens_value: u64,
|
||||
pub errors: String,
|
||||
@@ -388,7 +389,7 @@ pub struct DashboardCacheCardView {
|
||||
pub today_hit_rate_value: Option<f64>,
|
||||
pub yesterday_hit_rate_value: Option<f64>,
|
||||
pub total_hit_rate: String,
|
||||
pub total_cached_tokens: String,
|
||||
pub total_cached_input_tokens: String,
|
||||
pub average_first_token_latency_ms: String,
|
||||
}
|
||||
|
||||
@@ -446,9 +447,9 @@ pub struct RequestMetricsView {
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub cache_write_input_tokens: u64,
|
||||
pub reasoning_output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
@@ -575,6 +576,7 @@ impl From<domain::DesktopReleaseStatus> for DashboardDesktopReleaseStatusView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardDataView {
|
||||
pub analytics: AnalyticsView,
|
||||
pub as_of: DateTime<Utc>,
|
||||
pub as_of_display: String,
|
||||
pub cards: DashboardCardsView,
|
||||
@@ -592,11 +594,12 @@ pub struct DashboardDataView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageSummaryView {
|
||||
pub analytics: AnalyticsView,
|
||||
pub total_requests: String,
|
||||
pub input_tokens: String,
|
||||
pub output_tokens: String,
|
||||
pub cached_tokens: String,
|
||||
pub cache_write_tokens: String,
|
||||
pub cached_input_tokens: String,
|
||||
pub cache_write_input_tokens: String,
|
||||
pub total_tokens: String,
|
||||
pub average_latency_ms: String,
|
||||
pub logical_requests: RequestMetricsView,
|
||||
@@ -693,7 +696,7 @@ pub struct OverviewCostPointView {
|
||||
pub label: String,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub estimated_cost: Option<String>,
|
||||
pub standard_cost: Option<String>,
|
||||
@@ -719,7 +722,7 @@ pub struct OverviewCostView {
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub points: Vec<OverviewCostPointView>,
|
||||
pub costs: Vec<CostView>,
|
||||
@@ -741,6 +744,7 @@ pub struct ProviderOverviewView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageInsightsOverviewView {
|
||||
pub analytics: AnalyticsView,
|
||||
pub granularity: String,
|
||||
pub health: OverviewHealthView,
|
||||
pub performance: OverviewPerformanceView,
|
||||
@@ -778,6 +782,7 @@ pub struct DiagnosticItemView {
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DiagnosticsView {
|
||||
pub analytics: AnalyticsView,
|
||||
pub dimension: String,
|
||||
pub items: Vec<DiagnosticItemView>,
|
||||
}
|
||||
@@ -852,3 +857,20 @@ pub struct OpsErrorMetadataView {
|
||||
pub recovery_retry_delay_ms: Option<u64>,
|
||||
pub recovery_total_latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// 统计覆盖、新鲜度与近似口径,时间文案使用部署时区
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AnalyticsView {
|
||||
pub coverage_start: Option<DateTime<Utc>>,
|
||||
pub coverage_start_display: Option<String>,
|
||||
pub coverage_end: Option<DateTime<Utc>>,
|
||||
pub coverage_end_display: Option<String>,
|
||||
pub generated_at: DateTime<Utc>,
|
||||
pub generated_at_display: String,
|
||||
pub lag_seconds: u64,
|
||||
pub state: String,
|
||||
pub percentiles_approximate: bool,
|
||||
pub schema_version: u32,
|
||||
pub histogram_version: u32,
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ pub(super) fn config(secret: ClientKeySecret) -> ConfigView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct OverviewView {
|
||||
analytics: crate::admin::observability::AnalyticsView,
|
||||
as_of: DateTime<Utc>,
|
||||
as_of_display: String,
|
||||
start_time: DateTime<Utc>,
|
||||
@@ -82,9 +83,9 @@ struct MetricsView {
|
||||
requests: u64,
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
cached_tokens: u64,
|
||||
cache_write_tokens: u64,
|
||||
reasoning_tokens: u64,
|
||||
cached_input_tokens: u64,
|
||||
cache_write_input_tokens: u64,
|
||||
reasoning_output_tokens: u64,
|
||||
total_tokens: u64,
|
||||
cost_usd: Option<String>,
|
||||
cost_incomplete: bool,
|
||||
@@ -95,9 +96,9 @@ fn metrics(value: &RequestMetrics, costs: &[CurrencyCost], coverage: &CostCovera
|
||||
requests: value.request_count,
|
||||
input_tokens: value.input_tokens,
|
||||
output_tokens: value.output_tokens,
|
||||
cached_tokens: value.cached_tokens,
|
||||
cache_write_tokens: value.cache_write_tokens,
|
||||
reasoning_tokens: value.reasoning_tokens,
|
||||
cached_input_tokens: value.cached_input_tokens,
|
||||
cache_write_input_tokens: value.cache_write_input_tokens,
|
||||
reasoning_output_tokens: value.reasoning_output_tokens,
|
||||
total_tokens: value.total_tokens,
|
||||
cost_usd: costs
|
||||
.iter()
|
||||
@@ -121,6 +122,10 @@ struct TrendPointView {
|
||||
pub(super) fn overview(value: KeyUsageOverview, time: crate::time::TimePresenter) -> OverviewView {
|
||||
let key = value.key;
|
||||
OverviewView {
|
||||
analytics: crate::admin::observability::analytics_view(
|
||||
value.overview.analytics.clone(),
|
||||
time,
|
||||
),
|
||||
as_of: value.overview.range.end,
|
||||
as_of_display: time.time(&value.overview.range.end),
|
||||
start_time: value.overview.range.start,
|
||||
|
||||
@@ -86,7 +86,9 @@ async fn models_response(
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, entry)| match entry {
|
||||
PublicModelDescriptor::Adapted(profile) => Ok(codex_model_json(profile, index)),
|
||||
PublicModelDescriptor::Adapted(profile) => {
|
||||
Ok(adapted_codex_model_json(profile, index))
|
||||
}
|
||||
PublicModelDescriptor::Native { model, payload } => {
|
||||
if payload.protocol() != "codex" {
|
||||
return Err(());
|
||||
@@ -192,7 +194,7 @@ fn openai_model_json(id: &str) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
fn adapted_codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
let id = profile.model().as_str();
|
||||
let presentation = profile.presentation();
|
||||
let reasoning_levels = presentation
|
||||
@@ -265,6 +267,8 @@ fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
"experimental_supported_tools": [],
|
||||
"input_modalities": input_modalities,
|
||||
"supports_search_tool": presentation.search_tool(),
|
||||
// 通用画像没有 Responses Lite 支持依据,避免客户端启用未经声明的协议行为
|
||||
// 原生目录由 Native 分支保留上游字段,不经过此处
|
||||
"use_responses_lite": false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -150,10 +150,12 @@ pub(super) async fn forward_response(
|
||||
ProviderResponseHeader::new(name, value)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut interrupt = super::interrupt::ResponseInterrupt::default();
|
||||
let first = match next_body_input(
|
||||
connection,
|
||||
body.as_mut(),
|
||||
response_control,
|
||||
&mut interrupt,
|
||||
&request_id,
|
||||
false,
|
||||
)
|
||||
@@ -207,6 +209,7 @@ pub(super) async fn forward_response(
|
||||
connection,
|
||||
body.as_mut(),
|
||||
response_control,
|
||||
&mut interrupt,
|
||||
&request_id,
|
||||
terminal_seen,
|
||||
)
|
||||
@@ -274,11 +277,24 @@ pub(super) async fn forward_response(
|
||||
(true, true) => FramePhase::Terminal,
|
||||
(true, false) => FramePhase::Data,
|
||||
};
|
||||
let created_id = super::interrupt::ResponseInterrupt::created_id(&message);
|
||||
match connection
|
||||
.send_text(message, WriteContext::request(&request_id, phase))
|
||||
.await
|
||||
{
|
||||
Ok(super::connection::WriteOutcome::Written) => first_frame_written = true,
|
||||
Ok(super::connection::WriteOutcome::Written) => {
|
||||
first_frame_written = true;
|
||||
if let Some(id) = created_id {
|
||||
interrupt.observe_created(id, response_control);
|
||||
tokio::select! {
|
||||
_ = connection.wait_for_exit() => {
|
||||
detach_body(body);
|
||||
return ForwardOutcome::Disconnect;
|
||||
}
|
||||
() = interrupt.forward_pending(response_control) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(super::connection::WriteOutcome::Suppressed) => {}
|
||||
Err(_) => {
|
||||
detach_body(body);
|
||||
@@ -315,6 +331,7 @@ async fn next_body_input(
|
||||
connection: &mut ResponsesWebSocketConnection,
|
||||
body: &mut dyn MiddlewareBody,
|
||||
response_control: &ResponseControl,
|
||||
interrupt: &mut super::interrupt::ResponseInterrupt,
|
||||
request_id: &Arc<str>,
|
||||
terminal_seen: bool,
|
||||
) -> BodyInput {
|
||||
@@ -331,6 +348,13 @@ async fn next_body_input(
|
||||
connection.defer(event);
|
||||
continue;
|
||||
}
|
||||
if !terminal_seen && interrupt.retain(payload) {
|
||||
tokio::select! {
|
||||
_ = connection.wait_for_exit() => return BodyInput::Disconnect,
|
||||
() = interrupt.forward_pending(response_control) => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if send_control(connection, response_control, payload, request_id).await == ForwardOutcome::Disconnect {
|
||||
return BodyInput::Disconnect;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
//! 当前逻辑响应的中断意图,跨重试连接保留并随新响应 ID 转发
|
||||
|
||||
use std::{borrow::Cow, collections::BTreeSet};
|
||||
|
||||
use gateway_core::engine::response_control::{ResponseControl, ResponseControlBinding};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct EventIdentity<'a> {
|
||||
#[serde(rename = "type", borrow)]
|
||||
event_type: Cow<'a, str>,
|
||||
#[serde(borrow)]
|
||||
response: Option<ResponseIdentity<'a>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ResponseIdentity<'a> {
|
||||
#[serde(borrow)]
|
||||
id: Cow<'a, str>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ResponseInterrupt {
|
||||
delivered_ids: BTreeSet<String>,
|
||||
current: Option<(String, ResponseControlBinding)>,
|
||||
pending: Option<Value>,
|
||||
sent: bool,
|
||||
}
|
||||
|
||||
impl ResponseInterrupt {
|
||||
pub(super) fn created_id(payload: &str) -> Option<String> {
|
||||
let event: EventIdentity<'_> = serde_json::from_str(payload).ok()?;
|
||||
(event.event_type == "response.created").then_some(())?;
|
||||
Some(event.response?.id.into_owned())
|
||||
}
|
||||
|
||||
/// 只在 created 实际写入下游后更新,不能把未交付的 ID 当成客户端目标
|
||||
pub(super) fn observe_created(&mut self, id: String, control: &ResponseControl) {
|
||||
let Some(binding) = control.binding() else {
|
||||
return;
|
||||
};
|
||||
if self
|
||||
.current
|
||||
.as_ref()
|
||||
.is_none_or(|(previous, old)| previous != &id || !old.same_binding(&binding))
|
||||
{
|
||||
self.sent = false;
|
||||
}
|
||||
self.delivered_ids.insert(id.clone());
|
||||
self.current = Some((id, binding));
|
||||
}
|
||||
|
||||
/// 未知控制消息、模式和其他逻辑响应的 ID 仍走原始透明通道
|
||||
pub(super) fn retain(&mut self, payload: &str) -> bool {
|
||||
let Ok(event) = serde_json::from_str::<Value>(payload) else {
|
||||
return false;
|
||||
};
|
||||
if event["type"] != "response.interrupt"
|
||||
|| event["mode"] != "discard_partial_items"
|
||||
|| !event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| self.delivered_ids.contains(id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.pending = Some(event);
|
||||
self.sent = false;
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) async fn forward_pending(&mut self, control: &ResponseControl) {
|
||||
if self.sent {
|
||||
return;
|
||||
}
|
||||
let (Some(payload), Some((id, binding))) = (&mut self.pending, &self.current) else {
|
||||
return;
|
||||
};
|
||||
// 换号已绑定、created 尚未交付时等待新 ID;连接快照保证 await 期间也不会串号
|
||||
if !control
|
||||
.binding()
|
||||
.is_some_and(|current| binding.same_binding(¤t))
|
||||
{
|
||||
return;
|
||||
}
|
||||
payload["response_id"] = Value::String(id.clone());
|
||||
if binding.send(&payload.to_string()).await.is_ok() {
|
||||
self.sent = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
pub mod connection;
|
||||
mod forward;
|
||||
mod interrupt;
|
||||
mod protocol;
|
||||
|
||||
use std::{
|
||||
|
||||
@@ -12,13 +12,8 @@ use super::super::{AdminTestFixture, AdminTestState};
|
||||
#[tokio::test]
|
||||
async fn account_cooldown_returns_an_instant_and_server_formatted_recovery() {
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountCapacity, AccountPageItem, AccountRecord},
|
||||
};
|
||||
use gateway_core::account::{
|
||||
AccountCooldown, AccountCooldownKind, AccountStatusFacts, CredentialState, QuotaState,
|
||||
resolve_account_status,
|
||||
};
|
||||
|
||||
for zone in ["Asia/Shanghai", "UTC", "Asia/Kathmandu", "America/New_York"] {
|
||||
@@ -65,41 +60,7 @@ async fn account_cooldown_returns_an_instant_and_server_formatted_recovery() {
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
};
|
||||
*fixture.account.lock().unwrap() = Some(AccountPageItem {
|
||||
account: AccountRecord {
|
||||
id: "acct_cooldown".to_owned(),
|
||||
provider_kind: gateway_core::routing::ProviderKind::new("openai").unwrap(),
|
||||
groups: Vec::new(),
|
||||
name: "synthetic cooldown account".to_owned(),
|
||||
notes: None,
|
||||
email: None,
|
||||
upstream_user_id: None,
|
||||
upstream_account_id: None,
|
||||
plan_type: None,
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
credential_revision: Revision::new(1).unwrap(),
|
||||
has_refresh_token: true,
|
||||
access_token_expires_at: None,
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
concurrency_limit: None,
|
||||
weight: Default::default(),
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
credential_state: facts.credential_state,
|
||||
credential_observed_at: now,
|
||||
quota: facts.quota,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
capacity: AccountCapacity {
|
||||
used_slots: None,
|
||||
total_slots: None,
|
||||
},
|
||||
projection: resolve_account_status(&facts, now.into()),
|
||||
});
|
||||
*fixture.account.lock().unwrap() = Some(account_page_item(facts, now, "oauth"));
|
||||
let response = admin::accounts::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
@@ -138,6 +99,132 @@ async fn account_cooldown_returns_an_instant_and_server_formatted_recovery() {
|
||||
}
|
||||
}
|
||||
|
||||
fn account_page_item(
|
||||
facts: gateway_core::account::AccountStatusFacts,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
authentication_kind: &str,
|
||||
) -> gateway_admin::model::accounts::AccountPageItem {
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountCapacity, AccountPageItem, AccountRecord},
|
||||
};
|
||||
use gateway_core::account::resolve_account_status;
|
||||
|
||||
AccountPageItem {
|
||||
account: AccountRecord {
|
||||
id: "acct_cooldown".to_owned(),
|
||||
provider_kind: gateway_core::routing::ProviderKind::new("openai").unwrap(),
|
||||
groups: Vec::new(),
|
||||
name: "synthetic cooldown account".to_owned(),
|
||||
notes: None,
|
||||
email: None,
|
||||
upstream_user_id: None,
|
||||
upstream_account_id: None,
|
||||
plan_type: None,
|
||||
authentication_kind: authentication_kind.to_owned(),
|
||||
credential_revision: Revision::new(1).unwrap(),
|
||||
has_refresh_token: authentication_kind == "oauth",
|
||||
access_token_expires_at: None,
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
concurrency_limit: None,
|
||||
weight: Default::default(),
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
credential_state: facts.credential_state,
|
||||
credential_observed_at: now,
|
||||
quota: facts.quota,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
capacity: AccountCapacity {
|
||||
used_slots: None,
|
||||
total_slots: None,
|
||||
},
|
||||
projection: resolve_account_status(&facts, now.into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_usage_returns_distinct_cache_read_and_write_tokens_without_aliases() {
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::accounts::AccountUsage;
|
||||
use gateway_core::account::{AccountStatusFacts, CredentialState, QuotaState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
let facts = AccountStatusFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
};
|
||||
*fixture.account.lock().unwrap() = Some(account_page_item(facts, now, "api_key"));
|
||||
for (read, write) in [(Some(13), Some(7)), (None, None)] {
|
||||
*fixture.account_usage.lock().unwrap() = vec![AccountUsage {
|
||||
account_id: "acct_cooldown".to_owned(),
|
||||
request_count: 1,
|
||||
success_count: 1,
|
||||
input_tokens: Some(100),
|
||||
output_tokens: Some(20),
|
||||
cached_input_tokens: read,
|
||||
cache_write_input_tokens: write,
|
||||
reasoning_output_tokens: Some(3),
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
total_tokens: Some(120),
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
last_used_at: Some(now),
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
}];
|
||||
let response = admin::accounts::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/accounts")
|
||||
.header("x-request-id", "req_account_usage_contract")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 32768).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
let usage = &value["data"]["items"][0]["usage"];
|
||||
assert_eq!(usage["cachedInputTokens"], serde_json::json!(read));
|
||||
assert_eq!(usage["cacheWriteInputTokens"], serde_json::json!(write));
|
||||
assert_eq!(
|
||||
usage["cachedInputTokensDisplay"],
|
||||
if read.is_some() { "13" } else { "—" }
|
||||
);
|
||||
assert_eq!(
|
||||
usage["cacheWriteInputTokensDisplay"],
|
||||
if write.is_some() { "7" } else { "—" }
|
||||
);
|
||||
for alias in [
|
||||
"createdTokens",
|
||||
"createdTokensDisplay",
|
||||
"readTokens",
|
||||
"readTokensDisplay",
|
||||
] {
|
||||
assert!(usage.get(alias).is_none(), "obsolete token alias: {alias}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_credential_rotation_route_is_not_exposed() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
|
||||
@@ -484,10 +484,10 @@ mod response {
|
||||
input_tokens_display: "-".to_owned(),
|
||||
output_tokens: None,
|
||||
output_tokens_display: "-".to_owned(),
|
||||
cached_tokens: None,
|
||||
cached_tokens_display: "-".to_owned(),
|
||||
reasoning_tokens: None,
|
||||
reasoning_tokens_display: "-".to_owned(),
|
||||
cached_input_tokens: None,
|
||||
cached_input_tokens_display: "-".to_owned(),
|
||||
reasoning_output_tokens: None,
|
||||
reasoning_output_tokens_display: "-".to_owned(),
|
||||
image_input_tokens: None,
|
||||
image_input_tokens_display: "-".to_owned(),
|
||||
image_output_tokens: None,
|
||||
@@ -498,10 +498,8 @@ mod response {
|
||||
image_request_failed_count_display: "-".to_owned(),
|
||||
total_tokens: None,
|
||||
total_tokens_display: "-".to_owned(),
|
||||
created_tokens: None,
|
||||
created_tokens_display: "-".to_owned(),
|
||||
read_tokens: None,
|
||||
read_tokens_display: "-".to_owned(),
|
||||
cache_write_input_tokens: None,
|
||||
cache_write_input_tokens_display: "-".to_owned(),
|
||||
last_used_at: None,
|
||||
last_used_at_full_display: None,
|
||||
last_used_at_display: "-".to_owned(),
|
||||
@@ -516,9 +514,9 @@ mod response {
|
||||
assert_eq!(value["windowLabelDisplay"], "周/月额度窗口");
|
||||
assert!(value["inputTokens"].is_null());
|
||||
assert!(value["totalTokens"].is_null());
|
||||
assert!(value["reasoningTokens"].is_null());
|
||||
assert_eq!(value["reasoningTokensDisplay"], "-");
|
||||
assert_eq!(value["createdTokensDisplay"], "-");
|
||||
assert!(value["reasoningOutputTokens"].is_null());
|
||||
assert_eq!(value["reasoningOutputTokensDisplay"], "-");
|
||||
assert_eq!(value["cacheWriteInputTokensDisplay"], "-");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -112,6 +112,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
pub provider_error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
pub account: Arc<Mutex<Option<AccountPageItem>>>,
|
||||
pub account_usage: Arc<Mutex<Vec<AccountUsage>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
@@ -154,6 +155,7 @@ impl AdminTestFixture {
|
||||
let dashboard_summary_range = Arc::new(Mutex::new(None));
|
||||
let provider_error = Arc::new(Mutex::new(None));
|
||||
let account = Arc::new(Mutex::new(None));
|
||||
let account_usage = Arc::new(Mutex::new(Vec::new()));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
observations: observations.clone(),
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
@@ -163,6 +165,7 @@ impl AdminTestFixture {
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
dashboard_summary_range: Arc::clone(&dashboard_summary_range),
|
||||
account: Arc::clone(&account),
|
||||
account_usage: Arc::clone(&account_usage),
|
||||
});
|
||||
let plugin_ports = Arc::new(plugins::TestPluginPorts::default());
|
||||
let published_snapshot = gateway_core::runtime::RuntimeSnapshotHandle::default();
|
||||
@@ -232,6 +235,7 @@ impl AdminTestFixture {
|
||||
dashboard_summary_range,
|
||||
provider_error,
|
||||
account,
|
||||
account_usage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1100,6 +1104,7 @@ struct UnusedStore {
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
account: Arc<Mutex<Option<AccountPageItem>>>,
|
||||
account_usage: Arc<Mutex<Vec<AccountUsage>>>,
|
||||
}
|
||||
|
||||
struct UnusedClientKeyVerifier;
|
||||
@@ -1161,18 +1166,41 @@ impl AccountStore for UnusedStore {
|
||||
async fn load_account_usage(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: &[String],
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<Vec<AccountUsage>> {
|
||||
if self.account.lock().expect("account").is_some() {
|
||||
return Ok(Vec::new());
|
||||
return Ok(self
|
||||
.account_usage
|
||||
.lock()
|
||||
.expect("account usage")
|
||||
.iter()
|
||||
.filter(|usage| account_ids.contains(&usage.account_id))
|
||||
.cloned()
|
||||
.collect());
|
||||
}
|
||||
Err(unavailable("account usage"))
|
||||
}
|
||||
|
||||
async fn load_account_usage_by_windows(
|
||||
&self,
|
||||
_: &[AccountUsageWindowQuery],
|
||||
windows: &[AccountUsageWindowQuery],
|
||||
) -> AdminStoreResult<Vec<AccountUsageWindowResult>> {
|
||||
if self.account.lock().expect("account").is_some() {
|
||||
let usage = self.account_usage.lock().expect("account usage");
|
||||
return Ok(windows
|
||||
.iter()
|
||||
.filter_map(|window| {
|
||||
usage
|
||||
.iter()
|
||||
.find(|usage| usage.account_id == window.account_id)
|
||||
.map(|usage| AccountUsageWindowResult {
|
||||
account_id: window.account_id.clone(),
|
||||
key: window.key.clone(),
|
||||
usage: usage.clone(),
|
||||
})
|
||||
})
|
||||
.collect());
|
||||
}
|
||||
Err(unavailable("account quota window usage"))
|
||||
}
|
||||
|
||||
@@ -1381,15 +1409,39 @@ impl ObservabilityStore for UnusedStore {
|
||||
data.trend.clone().ok_or_else(|| unavailable("usage trend"))
|
||||
}
|
||||
|
||||
fn usage_calculated_billing_facts(
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
Ok(self
|
||||
.observations
|
||||
.lock()
|
||||
.expect("observations")
|
||||
.summary
|
||||
.as_ref()
|
||||
.map_or(0, |summary| summary.analytics.generation))
|
||||
}
|
||||
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
_: Granularity,
|
||||
) -> gateway_admin::ports::store::UsageCalculatedBillingStream<'_> {
|
||||
Box::pin(futures::stream::once(async {
|
||||
Err(unavailable("usage billing facts"))
|
||||
}))
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
let mut data = self.observations.lock().expect("observations");
|
||||
data.summaries.push((range, filter.clone()));
|
||||
data.trends.push((range, filter));
|
||||
Ok(
|
||||
gateway_admin::model::observability::UsageInsightsObservation {
|
||||
overview: data
|
||||
.summary
|
||||
.clone()
|
||||
.map(|summary| UsageOverview { range, ..summary })
|
||||
.ok_or_else(|| unavailable("usage summary"))?,
|
||||
trend: data
|
||||
.trend
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("usage trend"))?,
|
||||
costs: Vec::new(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
|
||||
@@ -25,6 +25,17 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
analytics: gateway_admin::model::observability::AnalyticsMetadata {
|
||||
coverage_start: Some(now - Duration::hours(1)),
|
||||
coverage_end: Some(now),
|
||||
generated_at: now,
|
||||
lag_seconds: 12,
|
||||
state: gateway_admin::model::observability::AnalyticsState::Lagging,
|
||||
percentiles_approximate: true,
|
||||
schema_version: 1,
|
||||
histogram_version: 1,
|
||||
generation: 42,
|
||||
},
|
||||
runtime_slots: None,
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
totals: Default::default(),
|
||||
@@ -59,6 +70,14 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
assert_eq!(value["data"]["analytics"]["state"], "lagging");
|
||||
assert_eq!(value["data"]["analytics"]["lagSeconds"], 12);
|
||||
assert_eq!(value["data"]["analytics"]["percentilesApproximate"], true);
|
||||
assert_eq!(value["data"]["analytics"]["schemaVersion"], 1);
|
||||
assert_eq!(value["data"]["analytics"]["histogramVersion"], 1);
|
||||
assert!(value["data"]["analytics"]["coverageStart"].is_string());
|
||||
assert!(value["data"]["analytics"]["generatedAtDisplay"].is_string());
|
||||
assert!(value["data"]["analytics"].get("generation").is_none());
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -108,12 +127,13 @@ async fn dashboard_summary_should_use_lifetime_totals_for_card_footers() {
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
analytics: Default::default(),
|
||||
runtime_slots: None,
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
totals: DashboardTotals {
|
||||
request_count: 42,
|
||||
input_tokens: 800,
|
||||
cached_tokens: 200,
|
||||
cached_input_tokens: 200,
|
||||
total_tokens: 840,
|
||||
billing_usd: None,
|
||||
},
|
||||
@@ -141,13 +161,21 @@ async fn dashboard_summary_should_use_lifetime_totals_for_card_footers() {
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
assert_eq!(
|
||||
value["data"]["analytics"]["coverageStartDisplay"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["analytics"]["coverageEndDisplay"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["traffic"]["totalRequests"],
|
||||
&value["data"]["cards"]["tokens"]["totalTokens"],
|
||||
&value["data"]["cards"]["cache"]["totalHitRate"],
|
||||
&value["data"]["cards"]["cache"]["totalCachedTokens"],
|
||||
&value["data"]["cards"]["cache"]["totalCachedInputTokens"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("42"),
|
||||
@@ -180,6 +208,7 @@ async fn dashboard_summary_should_default_to_current_deployment_day() {
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
analytics: Default::default(),
|
||||
runtime_slots: None,
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
totals: Default::default(),
|
||||
@@ -282,9 +311,9 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
message: Some("limited".to_owned()),
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
total_tokens: None,
|
||||
cost_source: Some("unavailable".to_owned()),
|
||||
cost_amount: None,
|
||||
@@ -401,9 +430,9 @@ async fn zero_attempt_failure_detail_keeps_missing_upstream_facts_and_preparatio
|
||||
record.error_message = Some("no upstream account is available".to_owned());
|
||||
record.input_tokens = None;
|
||||
record.output_tokens = None;
|
||||
record.cached_tokens = None;
|
||||
record.cache_write_tokens = None;
|
||||
record.reasoning_tokens = None;
|
||||
record.cached_input_tokens = None;
|
||||
record.cache_write_input_tokens = None;
|
||||
record.reasoning_output_tokens = None;
|
||||
record.image_input_tokens = None;
|
||||
record.image_output_tokens = None;
|
||||
record.total_tokens = None;
|
||||
@@ -617,6 +646,7 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
*fixture.diagnostics.lock().expect("diagnostics") = DiagnosticsObservation {
|
||||
analytics: Default::default(),
|
||||
total_request_count: 4,
|
||||
items: vec![DiagnosticObservation {
|
||||
key: "acct_diag".to_owned(),
|
||||
@@ -726,9 +756,9 @@ fn usage_record_with_account(
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(0),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
total_tokens: Some(2),
|
||||
@@ -847,9 +877,9 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
service_tier: Some("default".to_owned()),
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
cached_input_tokens: Some(0),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(0),
|
||||
image_input_tokens: Some(31),
|
||||
image_output_tokens: Some(9),
|
||||
total_tokens: Some(2),
|
||||
@@ -1101,6 +1131,12 @@ async fn calendar_queries_and_display_follow_deployment_timezone_with_one_anchor
|
||||
let fixture = AdminTestFixture::with_timezone(timezone).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
*fixture.dashboard_observation.lock().unwrap() = Some(DashboardObservation {
|
||||
analytics: gateway_admin::model::observability::AnalyticsMetadata {
|
||||
coverage_start: Some(range.start),
|
||||
coverage_end: Some(end),
|
||||
generated_at: end,
|
||||
..Default::default()
|
||||
},
|
||||
runtime_slots: None,
|
||||
range,
|
||||
totals: Default::default(),
|
||||
@@ -1142,6 +1178,15 @@ async fn calendar_queries_and_display_follow_deployment_timezone_with_one_anchor
|
||||
assert_eq!(queried.end, end);
|
||||
assert_eq!(value["data"]["asOf"], anchor);
|
||||
assert_eq!(value["data"]["asOfDisplay"], display);
|
||||
let analytics = &value["data"]["analytics"];
|
||||
assert_eq!(analytics["coverageStart"], start);
|
||||
assert_eq!(analytics["coverageEnd"], anchor);
|
||||
assert_eq!(
|
||||
analytics["coverageStartDisplay"],
|
||||
format!("{} 00:00:00", &display[..10])
|
||||
);
|
||||
assert_eq!(analytics["coverageEndDisplay"], display);
|
||||
assert_eq!(analytics["generatedAtDisplay"], display);
|
||||
let points = value["data"]["healthTimeline"]["points"]
|
||||
.as_array()
|
||||
.unwrap();
|
||||
@@ -1198,3 +1243,52 @@ async fn calendar_query_rejects_an_anchor_outside_the_local_calendar() {
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
assert!(fixture.dashboard_summary_range.lock().unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_and_ops_queries_should_forward_deleted_group_id_sets() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let router = observability::router::<AdminTestState>().with_state(fixture.state());
|
||||
for path in ["/api/admin/usage/records", "/api/admin/operations/errors"] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!(
|
||||
"{path}?routingGroupRefs=deleted-b,deleted-a,deleted-b"
|
||||
))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_group_filter")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"{path}: {}",
|
||||
String::from_utf8_lossy(&body)
|
||||
);
|
||||
}
|
||||
let observations = fixture.observations.lock().unwrap();
|
||||
assert_eq!(
|
||||
observations.records[0].filter.routing_group_refs,
|
||||
["deleted-a", "deleted-b"]
|
||||
);
|
||||
assert_eq!(
|
||||
observations.errors[0].filter.routing_group_refs,
|
||||
["deleted-a", "deleted-b"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -48,8 +48,8 @@ pub(super) async fn fixture() -> AdminTestFixture {
|
||||
failure_count: 1,
|
||||
input_tokens: 1000,
|
||||
output_tokens: 100,
|
||||
cached_tokens: 900,
|
||||
reasoning_tokens: 40,
|
||||
cached_input_tokens: 900,
|
||||
reasoning_output_tokens: 40,
|
||||
total_tokens: 1100,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -65,6 +65,7 @@ pub(super) async fn fixture() -> AdminTestFixture {
|
||||
{
|
||||
let mut observations = fixture.observations.lock().unwrap();
|
||||
observations.summary = Some(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range: TimeRange {
|
||||
start: now - Duration::hours(1),
|
||||
end: now,
|
||||
@@ -111,9 +112,9 @@ fn usage_record() -> UsageListRecord {
|
||||
service_tier: Some("private-sentinel".to_owned()),
|
||||
input_tokens: Some(1000),
|
||||
output_tokens: Some(100),
|
||||
cached_tokens: Some(900),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(40),
|
||||
cached_input_tokens: Some(900),
|
||||
cache_write_input_tokens: Some(0),
|
||||
reasoning_output_tokens: Some(40),
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1100),
|
||||
|
||||
@@ -181,6 +181,7 @@ async fn overview_scopes_every_query_and_projects_only_key_visible_fields() {
|
||||
"summary",
|
||||
"trend",
|
||||
"healthTimeline",
|
||||
"analytics",
|
||||
],
|
||||
);
|
||||
assert_fields(
|
||||
@@ -203,8 +204,8 @@ async fn overview_scopes_every_query_and_projects_only_key_visible_fields() {
|
||||
assert_eq!(data["key"]["dailyUsedUsd"], "0.640001");
|
||||
assert_eq!(data["key"]["maxConcurrency"], 0);
|
||||
assert_eq!(data["summary"]["totalTokens"], 1100);
|
||||
assert_eq!(data["summary"]["reasoningTokens"], 40);
|
||||
assert_eq!(data["trend"][0]["reasoningTokens"], 40);
|
||||
assert_eq!(data["summary"]["reasoningOutputTokens"], 40);
|
||||
assert_eq!(data["trend"][0]["reasoningOutputTokens"], 40);
|
||||
assert_eq!(data["summary"]["costUsd"], "0.123456");
|
||||
assert_eq!(data["summary"]["costIncomplete"], true);
|
||||
assert_eq!(data["trend"][0]["bucketSeconds"], 900);
|
||||
@@ -300,7 +301,7 @@ async fn records_keep_pagination_and_hide_admin_and_upstream_data() {
|
||||
assert!(!data.to_string().contains("private-sentinel"));
|
||||
if kind == "success" {
|
||||
assert_eq!(record["billing"]["totalAmountDisplay"], "$0.1235");
|
||||
assert_eq!(record["tokenDetails"]["reasoningTokens"], 40);
|
||||
assert_eq!(record["tokenDetails"]["reasoningOutputTokens"], 40);
|
||||
assert_eq!(record["tokenDetails"]["totalTokens"], 1100);
|
||||
assert_eq!(record["firstTokenLatencyMs"], 210);
|
||||
assert_fields(
|
||||
@@ -430,6 +431,7 @@ async fn empty_usage_has_zero_cost_but_unknown_pricing_stays_unknown() {
|
||||
{
|
||||
let mut data = fixture.observations.lock().unwrap();
|
||||
let summary = data.summary.as_mut().unwrap();
|
||||
summary.analytics.generation += 1;
|
||||
summary.requests = RequestMetrics {
|
||||
request_count: requests,
|
||||
..Default::default()
|
||||
|
||||
@@ -283,12 +283,25 @@ fn snapshot_with_client_key(
|
||||
plaintext: &str,
|
||||
provider_name: &str,
|
||||
client_key_id: &str,
|
||||
) -> RuntimeSnapshot {
|
||||
snapshot_with_accounts(plaintext, provider_name, client_key_id, &["acct_api_test"])
|
||||
}
|
||||
|
||||
fn snapshot_with_accounts(
|
||||
plaintext: &str,
|
||||
provider_name: &str,
|
||||
client_key_id: &str,
|
||||
accounts: &[&str],
|
||||
) -> RuntimeSnapshot {
|
||||
let provider = ProviderKind::new(provider_name).expect("provider");
|
||||
let account_directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
|
||||
ProviderAccountId::new("acct_api_test").expect("account ID"),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
|
||||
)])));
|
||||
let account_directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from_iter(
|
||||
accounts.iter().map(|id| {
|
||||
(
|
||||
ProviderAccountId::new(*id).expect("account ID"),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
|
||||
)
|
||||
}),
|
||||
)));
|
||||
let capabilities = ModelCapabilities::new(
|
||||
BTreeSet::from([gateway_core::operation::OperationKind::Generate]),
|
||||
Some(16_000),
|
||||
|
||||
@@ -62,6 +62,9 @@ impl ResponseControlTransport for ControlTransport {
|
||||
#[derive(Default)]
|
||||
struct InterruptProvider {
|
||||
calls: AtomicUsize,
|
||||
fail_first: bool,
|
||||
retry_started: tokio::sync::Notify,
|
||||
retry_release: tokio::sync::Notify,
|
||||
transport: Mutex<Option<Arc<ControlTransport>>>,
|
||||
complete: tokio::sync::Notify,
|
||||
initial_messages: Mutex<std::collections::VecDeque<ProtocolWireEvent>>,
|
||||
@@ -87,6 +90,15 @@ impl Provider for InterruptProvider {
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
let number = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if number == 1 && self.fail_first {
|
||||
self.retry_started.notify_one();
|
||||
self.retry_release.notified().await;
|
||||
assert!(
|
||||
context
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_api_test").unwrap())
|
||||
);
|
||||
}
|
||||
let id = format!("resp_interrupt_{number}");
|
||||
let transport = Arc::new(ControlTransport::new(id));
|
||||
let owner: Arc<dyn ResponseControlTransport> = transport.clone();
|
||||
@@ -96,7 +108,12 @@ impl Provider for InterruptProvider {
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
candidate.provider().clone(),
|
||||
candidate.upstream_model().unwrap().clone(),
|
||||
ProviderAccountId::new("acct_api_test").unwrap(),
|
||||
ProviderAccountId::new(if self.fail_first && number > 0 {
|
||||
"acct_api_retry"
|
||||
} else {
|
||||
"acct_api_test"
|
||||
})
|
||||
.unwrap(),
|
||||
UpstreamTransport::new("websocket").unwrap(),
|
||||
);
|
||||
let body = futures::stream::unfold(
|
||||
@@ -110,6 +127,18 @@ impl Provider for InterruptProvider {
|
||||
if let Some(wire) = pending {
|
||||
return Some((Ok(ProviderEvent::wire(wire)), (phase, transport, provider)));
|
||||
}
|
||||
if phase == 1 && provider.fail_first && transport.id == "resp_interrupt_0" {
|
||||
provider.complete.notified().await;
|
||||
transport.active.store(false, Ordering::SeqCst);
|
||||
return Some((
|
||||
Err(ProviderError::new(
|
||||
gateway_core::error::ProviderErrorKind::QuotaExhausted,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_replay_safe()),
|
||||
(2, transport, provider),
|
||||
));
|
||||
}
|
||||
let (wire, events, next_phase) = match phase {
|
||||
0 => (
|
||||
json!({"type":"response.created","response":{"id":transport.id,"model":"model-a","status":"in_progress","output":[]}}),
|
||||
@@ -144,10 +173,13 @@ impl Provider for InterruptProvider {
|
||||
_ => return None,
|
||||
};
|
||||
let event_type = wire["type"].as_str().unwrap().to_owned();
|
||||
let event = ProviderEvent::canonical_with_wire(
|
||||
let mut event = ProviderEvent::canonical_with_wire(
|
||||
events,
|
||||
ProtocolWireEvent::json("openai", Some(event_type), wire).unwrap(),
|
||||
);
|
||||
if phase == 0 && provider.fail_first {
|
||||
event = event.with_replayable_prelude();
|
||||
}
|
||||
Some((Ok(event), (next_phase, transport, provider)))
|
||||
},
|
||||
);
|
||||
@@ -233,7 +265,16 @@ async fn connect(
|
||||
admissions: Arc<InterruptAdmissions>,
|
||||
) -> (TestSocket, Server) {
|
||||
let execution = Arc::new(DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot("sk_ws_interrupt", "openai")),
|
||||
RuntimeSnapshotHandle::new(if provider.fail_first {
|
||||
crate::openai::snapshot_with_accounts(
|
||||
"sk_ws_interrupt",
|
||||
"openai",
|
||||
"key_api_test",
|
||||
&["acct_api_test", "acct_api_retry"],
|
||||
)
|
||||
} else {
|
||||
snapshot("sk_ws_interrupt", "openai")
|
||||
}),
|
||||
Arc::new(SettlementPorts::default()),
|
||||
ProviderRegistry::new([provider as Arc<dyn Provider>]).unwrap(),
|
||||
admissions,
|
||||
@@ -448,3 +489,102 @@ async fn active_websocket_delivery_preserves_unparsed_messages_and_business_meta
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prelude_retry_retains_interrupt_during_account_switch() {
|
||||
for interrupt_before_new_created in [true, false] {
|
||||
let provider = Arc::new(InterruptProvider {
|
||||
fail_first: true,
|
||||
..Default::default()
|
||||
});
|
||||
let admissions = Arc::new(InterruptAdmissions::default());
|
||||
let (mut socket, _server) = connect(provider.clone(), admissions.clone()).await;
|
||||
send(
|
||||
&mut socket,
|
||||
json!({"type":"response.create","model":"model-a","input":"hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await["response"]["id"],
|
||||
"resp_interrupt_0"
|
||||
);
|
||||
let old = provider.transport.lock().unwrap().clone().unwrap();
|
||||
provider.complete.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), provider.retry_started.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
if interrupt_before_new_created {
|
||||
send(
|
||||
&mut socket,
|
||||
interrupt("resp_interrupt_0", "discard_partial_items"),
|
||||
)
|
||||
.await;
|
||||
// 等待服务端处理控制帧,随后才允许新 attempt 建立连接
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while old.requests.lock().unwrap().is_empty() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
provider.retry_release.notify_one();
|
||||
assert_eq!(
|
||||
next(&mut socket).await["response"]["id"],
|
||||
"resp_interrupt_1"
|
||||
);
|
||||
if !interrupt_before_new_created {
|
||||
// 迟到的旧 ID 也应作用于当前逻辑响应
|
||||
send(
|
||||
&mut socket,
|
||||
interrupt("resp_interrupt_0", "discard_partial_items"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let terminal = next(&mut socket).await;
|
||||
assert_eq!(terminal["type"], "response.incomplete");
|
||||
assert_eq!(terminal["response"]["id"], "resp_interrupt_1");
|
||||
let current = provider.transport.lock().unwrap().clone().unwrap();
|
||||
let requests = current.requests.lock().unwrap().clone();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&requests[0]).unwrap(),
|
||||
interrupt("resp_interrupt_1", "discard_partial_items")
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 2);
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_replayable_delivery_blocks_retry_even_after_a_replayable_prelude() {
|
||||
for event in [
|
||||
json!({"type":"response.output_text.delta","delta":"visible","output_index":0,"content_index":0}),
|
||||
json!({"type":"response.future_progress","marker":"preserved"}),
|
||||
] {
|
||||
let provider = Arc::new(InterruptProvider {
|
||||
fail_first: true,
|
||||
..Default::default()
|
||||
});
|
||||
provider.initial_messages.lock().unwrap().push_back(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some(event["type"].as_str().unwrap().to_owned()),
|
||||
event.clone(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let (mut socket, _server) = connect(provider.clone(), Arc::default()).await;
|
||||
send(
|
||||
&mut socket,
|
||||
json!({"type":"response.create","model":"model-a","input":"hello"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(next(&mut socket).await["type"], "response.created");
|
||||
assert_eq!(next(&mut socket).await, event);
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "error");
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 1);
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -750,12 +750,40 @@ const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
|
||||
}
|
||||
}
|
||||
|
||||
/// 选中账号时冻结的历史展示事实,不包含凭据与运行状态
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountHistorySnapshot {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub email: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub plan_type_snapshot: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub notes_snapshot: Option<String>,
|
||||
}
|
||||
|
||||
impl AccountHistorySnapshot {
|
||||
#[must_use]
|
||||
pub fn from_account(account: &crate::account::ProviderAccount) -> Self {
|
||||
Self {
|
||||
name: account.name().to_owned(),
|
||||
email: account.email().map(str::to_owned),
|
||||
authentication_kind: account.authentication_kind().to_owned(),
|
||||
plan_type_snapshot: account.plan_type().map(str::to_owned),
|
||||
notes_snapshot: account.notes().map(str::to_owned),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号持久事实;代理认证信息只通过显式 secret accessor 读取
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccount {
|
||||
id: ProviderAccountId,
|
||||
provider: ProviderKind,
|
||||
name: String,
|
||||
notes: Option<String>,
|
||||
email: Option<String>,
|
||||
upstream_user_id: Option<String>,
|
||||
upstream_account_id: Option<String>,
|
||||
@@ -793,6 +821,7 @@ impl ProviderAccount {
|
||||
id,
|
||||
provider,
|
||||
name,
|
||||
notes: None,
|
||||
email: None,
|
||||
upstream_user_id,
|
||||
upstream_account_id: None,
|
||||
@@ -815,6 +844,17 @@ impl ProviderAccount {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_notes(mut self, notes: Option<String>) -> Self {
|
||||
self.notes = notes;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn notes(&self) -> Option<&str> {
|
||||
self.notes.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_profile(
|
||||
mut self,
|
||||
|
||||
@@ -18,6 +18,7 @@ pub struct OperationalFailure {
|
||||
pub correlation_id: Option<String>,
|
||||
pub provider_kind: Option<ProviderKind>,
|
||||
pub account_id: Option<ProviderAccountId>,
|
||||
pub account_snapshot: Option<crate::account::AccountHistorySnapshot>,
|
||||
pub upstream_status: Option<u16>,
|
||||
pub upstream_code: Option<OpaqueUpstreamValue>,
|
||||
pub details: Option<ErrorDetails>,
|
||||
@@ -40,6 +41,7 @@ impl OperationalFailure {
|
||||
correlation_id: None,
|
||||
provider_kind: None,
|
||||
account_id: None,
|
||||
account_snapshot: None,
|
||||
upstream_status: None,
|
||||
upstream_code: None,
|
||||
details: None,
|
||||
|
||||
@@ -346,6 +346,7 @@ where
|
||||
current: None,
|
||||
send_state_watermark: UpstreamSendState::NotSent,
|
||||
downstream_committed_at: None,
|
||||
replay_barrier_reached: false,
|
||||
client_status_code: None,
|
||||
delivery_pending: false,
|
||||
upstream_complete: false,
|
||||
@@ -483,6 +484,8 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
/// 请求级发送状态水位;跨 attempt 单调不降,终态写回不得低于此档
|
||||
send_state_watermark: UpstreamSendState,
|
||||
downstream_committed_at: Option<SystemTime>,
|
||||
/// 已交付的实际输出禁止重放,可替换前导事件不跨越此屏障
|
||||
replay_barrier_reached: bool,
|
||||
client_status_code: Option<u16>,
|
||||
delivery_pending: bool,
|
||||
upstream_complete: bool,
|
||||
@@ -548,6 +551,10 @@ where
|
||||
loop {
|
||||
match self.pull().await? {
|
||||
PullOutcome::Events(events) => {
|
||||
self.replay_barrier_reached |= self.middleware.is_some()
|
||||
|| self.client_transport != super::execution::ClientTransport::WebSocket
|
||||
|| self.response_control.is_none()
|
||||
|| events.iter().any(|event| !event.is_replayable_prelude());
|
||||
let requirement = if self.downstream_committed_at.is_some() {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
} else {
|
||||
@@ -568,6 +575,7 @@ where
|
||||
self.delivery_pending = true;
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
};
|
||||
self.replay_barrier_reached = true;
|
||||
self.pending_terminal_failure =
|
||||
Some(PendingTerminalFailure { error, send_state });
|
||||
return CoordinatedEvent::try_batch(events, requirement).map(Some);
|
||||
@@ -668,6 +676,7 @@ where
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
}
|
||||
self.delivery_pending = false;
|
||||
self.replay_barrier_reached = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1316,11 +1325,11 @@ where
|
||||
));
|
||||
}
|
||||
let attempt_record = AttemptRecord {
|
||||
account_snapshot: metadata.account_snapshot().cloned(),
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_count: next_attempt,
|
||||
trigger,
|
||||
provider_kind: metadata.provider().clone(),
|
||||
provider_account_id: Some(metadata.provider_account_id().clone()),
|
||||
provider_account_ref: Some(metadata.provider_account_id().clone()),
|
||||
upstream_model_id: metadata.upstream_model().cloned(),
|
||||
upstream_transport: metadata.transport().as_str().to_owned(),
|
||||
@@ -1497,7 +1506,7 @@ where
|
||||
let account_rotation_retry = !execution_effect_observed
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& matches!(
|
||||
@@ -1511,11 +1520,13 @@ where
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
|
||||
);
|
||||
// Provider 的有界同账号传输恢复是尽力重试策略,不要求把 Sent/Ambiguous
|
||||
// 改写成安全证明;外部副作用、取消、预算与下游提交仍由 Core 拦截
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { transport })
|
||||
if !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state == UpstreamSendState::NotSent
|
||||
&& self.current_send_state() == UpstreamSendState::NotSent
|
||||
@@ -1533,18 +1544,16 @@ where
|
||||
delay,
|
||||
}) if !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous =>
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
{
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if !execution_effect_observed
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous =>
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending =>
|
||||
{
|
||||
Some((
|
||||
AttemptTransport::Fallback,
|
||||
@@ -1558,7 +1567,7 @@ where
|
||||
&& !execution_effect_observed
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& provider_proved_replay_safe
|
||||
@@ -1590,7 +1599,7 @@ where
|
||||
let same_account_retry = !execution_effect_observed
|
||||
&& error.retries_same_account()
|
||||
&& provider_proved_replay_safe
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.replay_barrier_reached
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
@@ -1656,6 +1665,7 @@ where
|
||||
"executionEffectObserved": execution_effect_observed,
|
||||
"delayMs": transient_retry.or(transport_recovery.map(|(_, delay)| delay)).map(duration_ms),
|
||||
"downstreamCommitted": self.downstream_committed_at.is_some(),
|
||||
"replayBarrierReached": self.replay_barrier_reached,
|
||||
"sendState": format!("{attempt_send_state:?}"),
|
||||
}));
|
||||
if retryable {
|
||||
@@ -1701,6 +1711,7 @@ where
|
||||
self.engine
|
||||
.store()
|
||||
.record_intermediate_failure(IntermediateFailure {
|
||||
account_snapshot: current.metadata.account_snapshot().cloned(),
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
@@ -1813,7 +1824,7 @@ where
|
||||
) -> bool {
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt == ContinuationAttempt::None
|
||||
|| self.downstream_committed_at.is_some()
|
||||
|| self.replay_barrier_reached
|
||||
|| self.delivery_pending
|
||||
|| send_state == UpstreamSendState::Ambiguous
|
||||
|| !provider_proved_replay_safe
|
||||
|
||||
@@ -807,7 +807,7 @@ impl DefaultExecutionService {
|
||||
}
|
||||
let new_request = NewModelRequest {
|
||||
id: request_id.clone(),
|
||||
client_api_key_id: Some(client.policy.key_id().clone()),
|
||||
client_api_key_name_snapshot: client.policy.name_snapshot().map(str::to_owned),
|
||||
client_api_key_ref: client.policy.key_id().clone(),
|
||||
config_revision: plan.config_revision(),
|
||||
routing: authorization.account_scope.routing_snapshot(),
|
||||
@@ -1347,7 +1347,7 @@ impl DefaultExecutionService {
|
||||
.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "invalid admin actor"))?;
|
||||
let new_request = NewModelRequest {
|
||||
id: request_id,
|
||||
client_api_key_id: None,
|
||||
client_api_key_name_snapshot: None,
|
||||
client_api_key_ref: actor,
|
||||
config_revision: plan.config_revision(),
|
||||
routing: crate::routing::AccountRoutingSnapshot::all(),
|
||||
|
||||
@@ -559,6 +559,14 @@ impl AttemptContext {
|
||||
self.request.client_transport
|
||||
}
|
||||
|
||||
/// 原生双向通道可提前交付可替换前导事件;中间件可能改变其输出语义
|
||||
#[must_use]
|
||||
pub fn allows_replayable_prelude(&self) -> bool {
|
||||
self.client_transport() == execution::ClientTransport::WebSocket
|
||||
&& self.request.middleware.is_none()
|
||||
&& self.response_control().is_some()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn response_control(&self) -> Option<&response_control::ResponseControl> {
|
||||
self.request.response_control.as_ref()
|
||||
@@ -815,8 +823,8 @@ impl AttemptContext {
|
||||
/// 创建唯一 `model_requests` 行所需的入口事实
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NewModelRequest {
|
||||
pub client_api_key_name_snapshot: Option<String>,
|
||||
pub id: ModelRequestId,
|
||||
pub client_api_key_id: Option<ClientApiKeyId>,
|
||||
pub client_api_key_ref: ClientApiKeyId,
|
||||
pub config_revision: ConfigRevision,
|
||||
pub routing: crate::routing::AccountRoutingSnapshot,
|
||||
@@ -844,11 +852,11 @@ pub struct NewModelRequest {
|
||||
/// 每次真实上游发送前对同一 `model_requests` 行的更新
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AttemptRecord {
|
||||
pub account_snapshot: Option<crate::account::AccountHistorySnapshot>,
|
||||
pub request_id: ModelRequestId,
|
||||
pub attempt_count: NonZeroU32,
|
||||
pub trigger: AttemptTrigger,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub provider_account_id: Option<ProviderAccountId>,
|
||||
pub provider_account_ref: Option<ProviderAccountId>,
|
||||
pub upstream_model_id: Option<UpstreamModelId>,
|
||||
pub upstream_transport: String,
|
||||
@@ -863,6 +871,7 @@ pub struct AttemptRecord {
|
||||
/// 需要解释换号的中间失败
|
||||
#[derive(Debug)]
|
||||
pub struct IntermediateFailure {
|
||||
pub account_snapshot: Option<crate::account::AccountHistorySnapshot>,
|
||||
pub request_id: ModelRequestId,
|
||||
pub attempt_index: NonZeroU32,
|
||||
pub trigger: AttemptTrigger,
|
||||
|
||||
@@ -13,7 +13,8 @@ use futures::{Stream, StreamExt, future::BoxFuture};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::account::{
|
||||
AccountAttemptFeedback, AccountCapacitySnapshot, AccountFeedbackStats, ProviderAccountId,
|
||||
AccountAttemptFeedback, AccountCapacitySnapshot, AccountFeedbackStats, AccountHistorySnapshot,
|
||||
ProviderAccountId,
|
||||
};
|
||||
use crate::engine::AttemptContext;
|
||||
use crate::engine::execution::ClientTransport;
|
||||
@@ -39,7 +40,7 @@ use crate::upstream::{UpstreamSendState, UpstreamTransport};
|
||||
pub type EventStream =
|
||||
Pin<Box<dyn Stream<Item = Result<ProviderEvent, ProviderError>> + Send + 'static>>;
|
||||
|
||||
/// Provider 选定单个 credential 后返回的事实
|
||||
/// Provider 选定单个 credential 后返回的调用事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCallMetadata {
|
||||
provider: ProviderKind,
|
||||
@@ -48,6 +49,7 @@ pub struct ProviderCallMetadata {
|
||||
upstream_request_id: Option<OpaqueUpstreamValue>,
|
||||
transport: UpstreamTransport,
|
||||
selection_observation: Option<ProviderSelectionObservation>,
|
||||
account_snapshot: Option<AccountHistorySnapshot>,
|
||||
}
|
||||
|
||||
/// Provider 账号选择阶段输出的中立运行压力事实
|
||||
@@ -96,6 +98,7 @@ impl ProviderCallMetadata {
|
||||
upstream_request_id: None,
|
||||
transport,
|
||||
selection_observation: None,
|
||||
account_snapshot: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,9 +116,21 @@ impl ProviderCallMetadata {
|
||||
upstream_request_id: None,
|
||||
transport,
|
||||
selection_observation: None,
|
||||
account_snapshot: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_account_snapshot(mut self, snapshot: Option<AccountHistorySnapshot>) -> Self {
|
||||
self.account_snapshot = snapshot;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_snapshot(&self) -> Option<&AccountHistorySnapshot> {
|
||||
self.account_snapshot.as_ref()
|
||||
}
|
||||
|
||||
/// 设置 adapter 已分类为非 bearer 的 request ID
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
|
||||
@@ -24,6 +24,25 @@ pub struct ResponseControl {
|
||||
target: Arc<Mutex<Option<Weak<dyn ResponseControlTransport>>>>,
|
||||
}
|
||||
|
||||
/// 某次已交付响应所属的弱连接快照,重绑后不能把旧消息发送到新连接
|
||||
#[derive(Clone)]
|
||||
pub struct ResponseControlBinding(Weak<dyn ResponseControlTransport>);
|
||||
|
||||
impl ResponseControlBinding {
|
||||
#[must_use]
|
||||
pub fn same_binding(&self, other: &Self) -> bool {
|
||||
self.0.ptr_eq(&other.0)
|
||||
}
|
||||
|
||||
pub async fn send(&self, payload: &str) -> Result<(), ResponseControlUnavailable> {
|
||||
self.0
|
||||
.upgrade()
|
||||
.ok_or(ResponseControlUnavailable)?
|
||||
.send(payload)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ResponseControl {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ResponseControl([REDACTED])")
|
||||
@@ -31,6 +50,17 @@ impl fmt::Debug for ResponseControl {
|
||||
}
|
||||
|
||||
impl ResponseControl {
|
||||
/// 仅保存弱引用,不延长旧 attempt 的连接生命周期
|
||||
#[must_use]
|
||||
pub fn binding(&self) -> Option<ResponseControlBinding> {
|
||||
self.target
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.map(ResponseControlBinding)
|
||||
}
|
||||
|
||||
/// Provider 在请求成功写入上游后绑定;连接 owner 持有强引用并负责撤销
|
||||
pub fn bind(&self, transport: &Arc<dyn ResponseControlTransport>) {
|
||||
*self
|
||||
|
||||
@@ -135,6 +135,7 @@ pub enum PreDeliveryRetry {
|
||||
max_delay: Duration,
|
||||
},
|
||||
/// 固定本次账号,并按 Provider 给出的序号重试当前传输
|
||||
/// 可用于发送结果不明确时的有界尽力恢复,不等于上游未执行的证明
|
||||
SameAccountTransportRetry {
|
||||
/// Provider-owned 传输重试序号,从 1 开始
|
||||
retry_index: NonZeroU32,
|
||||
@@ -630,8 +631,8 @@ impl ProviderError {
|
||||
|
||||
/// 允许 Core 在首个客户端事件前固定原账号并改用 Provider 备用传输
|
||||
///
|
||||
/// 与普通换号恢复相同,Core 仍会拒绝 continuation、指定账号、预算耗尽或
|
||||
/// 下游已提交后的隐藏重放
|
||||
/// Provider 必须确认备用传输满足当前 continuation scope 并限制恢复次数
|
||||
/// Core 保持原账号,拒绝预算耗尽、外部副作用或下游已提交后的隐藏重放
|
||||
#[must_use]
|
||||
pub fn with_pre_delivery_transport_fallback(mut self) -> Self {
|
||||
self.set_pre_delivery_transport_fallback();
|
||||
@@ -647,6 +648,7 @@ impl ProviderError {
|
||||
///
|
||||
/// 重试预算和退避策略由 Provider 所属协议定义;Core 只负责同账号钉选、
|
||||
/// deadline/取消边界以及下游 commit barrier
|
||||
/// 该意图允许发送结果不明确时尽力恢复,但不修改发送事实或安全重放证明
|
||||
#[must_use]
|
||||
pub fn with_pre_delivery_transport_retry(
|
||||
mut self,
|
||||
|
||||
@@ -834,6 +834,7 @@ pub struct ProviderEvent {
|
||||
observation: Option<Box<ProviderResponseObservation>>,
|
||||
session_update: Option<Box<ProviderSessionState>>,
|
||||
middleware_transformed: bool,
|
||||
replayable_prelude: bool,
|
||||
middleware_origin_wire: Option<Arc<ProtocolWireEvent>>,
|
||||
}
|
||||
|
||||
@@ -847,6 +848,7 @@ impl ProviderEvent {
|
||||
observation: None,
|
||||
session_update: None,
|
||||
middleware_transformed: false,
|
||||
replayable_prelude: false,
|
||||
middleware_origin_wire: None,
|
||||
}
|
||||
}
|
||||
@@ -867,6 +869,7 @@ impl ProviderEvent {
|
||||
observation: None,
|
||||
session_update: None,
|
||||
middleware_transformed: false,
|
||||
replayable_prelude: false,
|
||||
middleware_origin_wire: None,
|
||||
}
|
||||
}
|
||||
@@ -880,6 +883,7 @@ impl ProviderEvent {
|
||||
observation: None,
|
||||
session_update: None,
|
||||
middleware_transformed: false,
|
||||
replayable_prelude: false,
|
||||
middleware_origin_wire: None,
|
||||
}
|
||||
}
|
||||
@@ -893,10 +897,24 @@ impl ProviderEvent {
|
||||
observation: Some(Box::new(observation)),
|
||||
session_update: None,
|
||||
middleware_transformed: false,
|
||||
replayable_prelude: false,
|
||||
middleware_origin_wire: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 确认该事件仅为可替换的前导信息,客户端支持后续 attempt 的新前导事件
|
||||
/// 此证明不代替错误分类、发送状态、预算或续接恢复检查
|
||||
#[must_use]
|
||||
pub fn with_replayable_prelude(mut self) -> Self {
|
||||
self.replayable_prelude = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_replayable_prelude(&self) -> bool {
|
||||
self.replayable_prelude && !self.middleware_transformed
|
||||
}
|
||||
|
||||
/// 返回协议表达是否由中间件替换、展开或丢弃映射产生
|
||||
///
|
||||
/// 宿主最终协议边界据此启用严格复核,插件可读取其快照
|
||||
@@ -970,6 +988,7 @@ impl ProviderEvent {
|
||||
/// 仅替换客户端协议表达;canonical facts、响应观察和会话状态保持原始事实
|
||||
pub fn replace_wire(&mut self, wire: Option<ProtocolWireEvent>) {
|
||||
self.wire = wire.map(Box::new);
|
||||
self.replayable_prelude = false;
|
||||
}
|
||||
|
||||
/// 取出仅供 Core 持久化的响应观察
|
||||
|
||||
@@ -194,9 +194,9 @@ impl Money {
|
||||
pub struct Usage {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
/// Provider/协议报告的独立事实,不从其他列相加推导
|
||||
@@ -209,9 +209,9 @@ impl Usage {
|
||||
Self {
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: None,
|
||||
@@ -226,14 +226,14 @@ impl Usage {
|
||||
if newer.output_tokens.is_some() {
|
||||
self.output_tokens = newer.output_tokens;
|
||||
}
|
||||
if newer.cached_tokens.is_some() {
|
||||
self.cached_tokens = newer.cached_tokens;
|
||||
if newer.cached_input_tokens.is_some() {
|
||||
self.cached_input_tokens = newer.cached_input_tokens;
|
||||
}
|
||||
if newer.cache_write_tokens.is_some() {
|
||||
self.cache_write_tokens = newer.cache_write_tokens;
|
||||
if newer.cache_write_input_tokens.is_some() {
|
||||
self.cache_write_input_tokens = newer.cache_write_input_tokens;
|
||||
}
|
||||
if newer.reasoning_tokens.is_some() {
|
||||
self.reasoning_tokens = newer.reasoning_tokens;
|
||||
if newer.reasoning_output_tokens.is_some() {
|
||||
self.reasoning_output_tokens = newer.reasoning_output_tokens;
|
||||
}
|
||||
if newer.image_input_tokens.is_some() {
|
||||
self.image_input_tokens = newer.image_input_tokens;
|
||||
@@ -332,7 +332,7 @@ pub struct CalculatedCostBreakdown {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ImageCostBreakdown {
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub input_amount: Money,
|
||||
pub cache_read_amount: Money,
|
||||
pub input_price_per_million: Money,
|
||||
|
||||
@@ -119,6 +119,7 @@ pub struct ClientSettings {
|
||||
/// 从 `client_api_keys` 冻结的公开准入事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ClientPolicy {
|
||||
name_snapshot: Option<String>,
|
||||
defaults: Arc<ClientSettings>,
|
||||
key_id: ClientApiKeyId,
|
||||
plaintext_key: PlaintextClientApiKey,
|
||||
@@ -163,6 +164,7 @@ impl ClientPolicy {
|
||||
limits: RateLimits,
|
||||
) -> Self {
|
||||
Self {
|
||||
name_snapshot: None,
|
||||
defaults: Arc::new(ClientSettings {
|
||||
request_profiles: account_scope.request_profiles().clone(),
|
||||
fast_mode: account_scope.fast_mode(),
|
||||
@@ -176,6 +178,17 @@ impl ClientPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_name_snapshot(mut self, name: Option<String>) -> Self {
|
||||
self.name_snapshot = name;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn name_snapshot(&self) -> Option<&str> {
|
||||
self.name_snapshot.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn key_id(&self) -> &ClientApiKeyId {
|
||||
&self.key_id
|
||||
|
||||
@@ -29,6 +29,7 @@ type ModelCatalogAccounts = BTreeMap<ProviderKind, BTreeMap<String, BTreeSet<Pro
|
||||
/// Store 读取到的一个启用 Client API Key 策略事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotClientPolicyFacts {
|
||||
name_snapshot: Option<String>,
|
||||
request_profiles: BTreeMap<ProviderKind, crate::account::OpaqueProviderData>,
|
||||
key_id: ClientApiKeyId,
|
||||
plaintext_key: PlaintextClientApiKey,
|
||||
@@ -37,6 +38,12 @@ pub struct SnapshotClientPolicyFacts {
|
||||
}
|
||||
|
||||
impl SnapshotClientPolicyFacts {
|
||||
#[must_use]
|
||||
pub fn with_name_snapshot(mut self, name: Option<String>) -> Self {
|
||||
self.name_snapshot = name;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_profiles(
|
||||
mut self,
|
||||
@@ -55,6 +62,7 @@ impl SnapshotClientPolicyFacts {
|
||||
) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
name_snapshot: None,
|
||||
request_profiles: BTreeMap::new(),
|
||||
plaintext_key,
|
||||
group_ids,
|
||||
@@ -509,17 +517,20 @@ async fn compile_runtime_snapshot(
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?,
|
||||
)
|
||||
};
|
||||
client_policies.push(ClientPolicy::new(
|
||||
policy.key_id,
|
||||
policy.plaintext_key,
|
||||
Arc::new(
|
||||
account_scope
|
||||
.with_fast_mode(fast_mode)
|
||||
.with_request_profiles(policy.request_profiles),
|
||||
),
|
||||
true,
|
||||
policy.limits,
|
||||
));
|
||||
client_policies.push(
|
||||
ClientPolicy::new(
|
||||
policy.key_id,
|
||||
policy.plaintext_key,
|
||||
Arc::new(
|
||||
account_scope
|
||||
.with_fast_mode(fast_mode)
|
||||
.with_request_profiles(policy.request_profiles),
|
||||
),
|
||||
true,
|
||||
policy.limits,
|
||||
)
|
||||
.with_name_snapshot(policy.name_snapshot),
|
||||
);
|
||||
}
|
||||
|
||||
RuntimeSnapshot::new_with_privacy_compiler(
|
||||
@@ -1090,6 +1101,7 @@ impl RuntimeSnapshot {
|
||||
) -> Result<RoutingPlan, RoutingError> {
|
||||
let requirements = operation.capability_requirements();
|
||||
let mut candidates = Vec::new();
|
||||
let mut unlisted_candidates = Vec::new();
|
||||
|
||||
if context.required_provider.is_none() && account_scope.provider_kinds().is_empty() {
|
||||
return Err(RoutingError::EmptyAccountScope);
|
||||
@@ -1137,11 +1149,11 @@ impl RuntimeSnapshot {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let emulated_features = match self
|
||||
let capabilities = self
|
||||
.provider_models
|
||||
.get(provider)
|
||||
.and_then(|models| models.get(&upstream_model))
|
||||
{
|
||||
.and_then(|models| models.get(&upstream_model));
|
||||
let emulated_features = match capabilities {
|
||||
Some(capabilities) => {
|
||||
let Some(emulated) = capabilities.match_requirements(&requirements) else {
|
||||
continue;
|
||||
@@ -1151,7 +1163,7 @@ impl RuntimeSnapshot {
|
||||
None if self.exhaustive_provider_catalogs.contains(provider) => continue,
|
||||
None => BTreeSet::new(),
|
||||
};
|
||||
candidates.push(ProviderCandidate {
|
||||
let candidate = ProviderCandidate {
|
||||
provider: provider.clone(),
|
||||
model_presentation: self
|
||||
.provider_model_presentations
|
||||
@@ -1161,8 +1173,15 @@ impl RuntimeSnapshot {
|
||||
upstream_model: Some(upstream_model),
|
||||
emulated_features,
|
||||
account_scope: Arc::clone(&account_scope),
|
||||
});
|
||||
};
|
||||
if capabilities.is_some() {
|
||||
candidates.push(candidate);
|
||||
} else {
|
||||
unlisted_candidates.push(candidate);
|
||||
}
|
||||
}
|
||||
// 已知模型优先;发现型目录的未知模型仍可透传,但不能抢在明确匹配的 Provider 前面
|
||||
candidates.extend(unlisted_candidates);
|
||||
|
||||
if candidates.is_empty() {
|
||||
// 模型存在性必须包含被请求路由限制排除的 Provider;目录未知时不能断言模型不存在
|
||||
|
||||
@@ -20,6 +20,7 @@ pub enum WorkerKind {
|
||||
RuntimeSnapshotReconciliation,
|
||||
RuntimeChangeSubscription,
|
||||
StaleModelRequestRecovery,
|
||||
RequestAnalytics,
|
||||
Retention,
|
||||
OpsFlush,
|
||||
Backup,
|
||||
@@ -28,12 +29,13 @@ pub enum WorkerKind {
|
||||
}
|
||||
|
||||
impl WorkerKind {
|
||||
pub const ALL: [Self; 10] = [
|
||||
pub const ALL: [Self; 11] = [
|
||||
Self::OAuthRefresh,
|
||||
Self::QuotaCatalogHealth,
|
||||
Self::RuntimeSnapshotReconciliation,
|
||||
Self::RuntimeChangeSubscription,
|
||||
Self::StaleModelRequestRecovery,
|
||||
Self::RequestAnalytics,
|
||||
Self::Retention,
|
||||
Self::OpsFlush,
|
||||
Self::Backup,
|
||||
@@ -48,6 +50,7 @@ impl WorkerKind {
|
||||
Self::QuotaCatalogHealth => "quota_catalog_health",
|
||||
Self::RuntimeSnapshotReconciliation => "runtime_snapshot_reconciliation",
|
||||
Self::RuntimeChangeSubscription => "runtime_change_subscription",
|
||||
Self::RequestAnalytics => "request_analytics",
|
||||
Self::StaleModelRequestRecovery => "stale_model_request_recovery",
|
||||
Self::Retention => "retention",
|
||||
Self::OpsFlush => "ops_flush",
|
||||
|
||||
@@ -860,7 +860,7 @@ fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest
|
||||
NewModelRequest {
|
||||
admission_decision_ms: None,
|
||||
id: ModelRequestId::new("req_core_1").expect("request id"),
|
||||
client_api_key_id: Some(client_key.clone()),
|
||||
client_api_key_name_snapshot: None,
|
||||
client_api_key_ref: client_key,
|
||||
config_revision: ConfigRevision::new(1).expect("config revision"),
|
||||
routing: AccountRoutingSnapshot::all(),
|
||||
@@ -3493,7 +3493,7 @@ fn ambiguous_pre_delivery_retry_marker_does_not_rotate_account() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_transport_retry_is_rejected_even_with_provider_session_state() {
|
||||
fn ambiguous_transport_retry_keeps_the_account_without_claiming_replay_safety() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let checkpoint = ProviderSessionState::new(
|
||||
@@ -3513,7 +3513,6 @@ fn ambiguous_transport_retry_is_rejected_even_with_provider_session_state() {
|
||||
Ok(checkpoint_event),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_replay_safe()
|
||||
.with_pre_delivery_transport_retry(retry_index, Duration::ZERO),
|
||||
),
|
||||
],
|
||||
@@ -3533,14 +3532,25 @@ fn ambiguous_transport_retry_is_rejected_even_with_provider_session_state() {
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error =
|
||||
block_on(session.collect_uncommitted()).expect_err("ambiguous send cannot be replayed");
|
||||
assert!(matches!(error, EngineError::Provider(_)));
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 1);
|
||||
block_on(session.collect_uncommitted()).expect("bounded transport recovery succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[1].transport(),
|
||||
AttemptTransport::Retry(retry_index)
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].required_account(),
|
||||
Some(&ProviderAccountId::new("acct_first").unwrap())
|
||||
);
|
||||
assert!(contexts[1].excluded_accounts().is_empty());
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 0);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 2);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert!(state.send_states.contains(&UpstreamSendState::Ambiguous));
|
||||
assert_eq!(state.finalizations[0].send_state, UpstreamSendState::Sent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -579,10 +579,7 @@ fn assert_early_failure_recorded(
|
||||
let request = &requests[0];
|
||||
assert_eq!(request.id, *request_id);
|
||||
assert_eq!(request.client_api_key_ref.as_str(), "key_start_test");
|
||||
assert_eq!(
|
||||
request.client_api_key_id.as_ref(),
|
||||
Some(&request.client_api_key_ref)
|
||||
);
|
||||
assert_eq!(request.client_api_key_name_snapshot, None);
|
||||
assert_eq!(request.protocol, "openai");
|
||||
assert_eq!(request.endpoint, "/v1/responses");
|
||||
assert_eq!(request.client_transport, transport.as_str());
|
||||
|
||||
@@ -66,3 +66,28 @@ fn replacing_or_clearing_a_binding_does_not_share_control_between_executions() {
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn captured_binding_never_redirects_old_control_to_a_replacement() {
|
||||
block_on(async {
|
||||
let control = ResponseControl::default();
|
||||
let old = Arc::new(Transport::default());
|
||||
let old_owner: Arc<dyn ResponseControlTransport> = old.clone();
|
||||
control.bind(&old_owner);
|
||||
let binding = control.binding().unwrap();
|
||||
let new = Arc::new(Transport::default());
|
||||
let new_owner: Arc<dyn ResponseControlTransport> = new.clone();
|
||||
control.bind(&new_owner);
|
||||
assert!(!binding.same_binding(&control.binding().unwrap()));
|
||||
binding.send("old ID").await.unwrap();
|
||||
assert_eq!(*old.0.lock().unwrap(), ["old ID"]);
|
||||
assert!(new.0.lock().unwrap().is_empty());
|
||||
drop(old_owner);
|
||||
drop(old);
|
||||
assert_eq!(
|
||||
binding.send("expired ID").await,
|
||||
Err(ResponseControlUnavailable)
|
||||
);
|
||||
assert!(new.0.lock().unwrap().is_empty());
|
||||
});
|
||||
}
|
||||
|
||||
@@ -282,6 +282,77 @@ impl ProviderCatalogPort for TestCatalog {
|
||||
}
|
||||
}
|
||||
|
||||
struct MixedCatalog;
|
||||
|
||||
impl ProviderCatalogPort for MixedCatalog {
|
||||
fn catalog_generations(&self) -> BTreeMap<ProviderKind, ProviderCatalogGeneration> {
|
||||
["openai", "xai"]
|
||||
.into_iter()
|
||||
.map(|provider| {
|
||||
(
|
||||
ProviderKind::new(provider).unwrap(),
|
||||
ProviderCatalogGeneration::new(1),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn model_catalog_is_exhaustive(&self, provider: &ProviderKind) -> bool {
|
||||
provider.as_str() != "openai"
|
||||
}
|
||||
|
||||
fn query_model_capabilities(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
) -> BoxFuture<'_, Result<Vec<ProviderModelCapabilities>, ProviderCatalogUnavailable>> {
|
||||
let model = match provider.as_str() {
|
||||
"openai" => "gpt-5.5",
|
||||
"xai" => "grok-4.7",
|
||||
_ => return Box::pin(async { Err(ProviderCatalogUnavailable) }),
|
||||
};
|
||||
Box::pin(async move {
|
||||
Ok([model, "shared-model"]
|
||||
.into_iter()
|
||||
.map(|model| {
|
||||
ProviderModelCapabilities::new(
|
||||
UpstreamModelId::new(model).unwrap(),
|
||||
super::capabilities(),
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn mixed_catalog_snapshot() -> gateway_core::routing::RuntimeSnapshot {
|
||||
let facts = SnapshotFacts::new(
|
||||
revision(1),
|
||||
revision(1),
|
||||
super::settings(),
|
||||
vec![],
|
||||
vec![],
|
||||
["openai", "xai"]
|
||||
.into_iter()
|
||||
.map(|provider| {
|
||||
SnapshotProviderAccountFacts::new(
|
||||
gateway_core::account::ProviderAccountId::new(format!("acct_{provider}"))
|
||||
.unwrap(),
|
||||
provider,
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
vec![],
|
||||
);
|
||||
block_on(
|
||||
RuntimeSnapshotCompiler::new(
|
||||
Arc::new(TestSnapshotStore::new(Ok(facts))),
|
||||
Arc::new(MixedCatalog),
|
||||
)
|
||||
.compile(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn catalog_generations(generation: u64) -> BTreeMap<ProviderKind, ProviderCatalogGeneration> {
|
||||
BTreeMap::from([(
|
||||
ProviderKind::new("alpha").expect("provider"),
|
||||
@@ -606,6 +677,139 @@ fn discovery_catalog_should_not_reject_an_unlisted_model() {
|
||||
.expect("upstream decides model availability");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_catalog_should_prefer_known_model_over_discovery_fallback() {
|
||||
let snapshot = mixed_catalog_snapshot();
|
||||
let model = PublicModelId::new("grok-4.7").unwrap();
|
||||
assert!(
|
||||
snapshot
|
||||
.public_models_for_scope(&snapshot.all_account_scope())
|
||||
.contains(&model)
|
||||
);
|
||||
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&model,
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
plan.candidates()
|
||||
.iter()
|
||||
.map(|candidate| candidate.provider().as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["xai", "openai"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_catalog_should_prioritize_the_mapped_upstream_model() {
|
||||
let snapshot = mixed_catalog_snapshot();
|
||||
let settings = snapshot
|
||||
.settings()
|
||||
.clone()
|
||||
.with_model_mappings(BTreeMap::from([(
|
||||
"public-model".to_owned(),
|
||||
"grok-4.7".to_owned(),
|
||||
)]));
|
||||
let snapshot = snapshot.with_settings(&settings).unwrap();
|
||||
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("public-model").unwrap(),
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(plan.candidates()[0].provider().as_str(), "xai");
|
||||
assert_eq!(
|
||||
plan.candidates()[0].upstream_model().unwrap().as_str(),
|
||||
"grok-4.7"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_catalog_should_keep_discovery_routes_for_unlisted_models() {
|
||||
let snapshot = mixed_catalog_snapshot();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("unlisted-model").unwrap(),
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(plan.candidates().len(), 1);
|
||||
assert_eq!(plan.candidates()[0].provider().as_str(), "openai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_catalog_should_preserve_provider_order_for_shared_models() {
|
||||
let snapshot = mixed_catalog_snapshot();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("shared-model").unwrap(),
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
plan.candidates()
|
||||
.iter()
|
||||
.map(|candidate| candidate.provider().as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["openai", "xai"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_catalog_should_respect_required_provider_before_model_priority() {
|
||||
let snapshot = mixed_catalog_snapshot();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("grok-4.7").unwrap(),
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&gateway_core::routing::RoutingContext {
|
||||
required_provider: Some(ProviderKind::new("openai").unwrap()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(plan.candidates().len(), 1);
|
||||
assert_eq!(plan.candidates()[0].provider().as_str(), "openai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_catalog_should_respect_blocked_provider_before_model_priority() {
|
||||
let snapshot = mixed_catalog_snapshot();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("grok-4.7").unwrap(),
|
||||
&super::operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&gateway_core::routing::RoutingContext {
|
||||
blocked_providers: std::collections::BTreeSet::from([
|
||||
ProviderKind::new("xai").unwrap()
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(plan.candidates().len(), 1);
|
||||
assert_eq!(plan.candidates()[0].provider().as_str(), "openai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_empty_catalog_should_report_model_not_found() {
|
||||
let compiler = RuntimeSnapshotCompiler::new(
|
||||
|
||||
@@ -19,12 +19,13 @@ use gateway_core::task::{
|
||||
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
const ACTIVE_KINDS: [WorkerKind; 9] = [
|
||||
const ACTIVE_KINDS: [WorkerKind; 10] = [
|
||||
WorkerKind::OAuthRefresh,
|
||||
WorkerKind::QuotaCatalogHealth,
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
WorkerKind::RuntimeChangeSubscription,
|
||||
WorkerKind::StaleModelRequestRecovery,
|
||||
WorkerKind::RequestAnalytics,
|
||||
WorkerKind::Retention,
|
||||
WorkerKind::Backup,
|
||||
WorkerKind::AccountImport,
|
||||
|
||||
@@ -495,9 +495,9 @@ fn wire_observation(observation: &RequestObservation) -> RequestCompleted {
|
||||
RequestUsage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
cached_tokens: usage.cached_input_tokens,
|
||||
cache_write_tokens: usage.cache_write_input_tokens,
|
||||
reasoning_tokens: usage.reasoning_output_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
total_tokens: usage.total_tokens,
|
||||
|
||||
@@ -165,9 +165,9 @@ impl EventDecoder {
|
||||
let usage = Usage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
cached_input_tokens: usage.cached_tokens,
|
||||
cache_write_input_tokens: usage.cache_write_tokens,
|
||||
reasoning_output_tokens: usage.reasoning_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
total_tokens: usage.total_tokens,
|
||||
|
||||
@@ -86,7 +86,7 @@ fn breakdown(value: &CalculatedCostBreakdown) -> facts::CostBreakdown {
|
||||
long_context_billing_applied: value.long_context_billing_applied(),
|
||||
image: value.image().map(|value| facts::ImageCostBreakdown {
|
||||
input_tokens: value.input_tokens,
|
||||
cached_tokens: value.cached_tokens,
|
||||
cached_tokens: value.cached_input_tokens,
|
||||
input_amount: money(value.input_amount),
|
||||
cache_read_amount: money(value.cache_read_amount),
|
||||
input_price_per_million: money(value.input_price_per_million),
|
||||
@@ -234,9 +234,9 @@ fn project_fact(event: GatewayEvent) -> Option<Result<CanonicalEvent, PluginFaul
|
||||
usage: WireUsage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
cached_tokens: usage.cached_input_tokens,
|
||||
cache_write_tokens: usage.cache_write_input_tokens,
|
||||
reasoning_tokens: usage.reasoning_output_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
total_tokens: usage.total_tokens,
|
||||
|
||||
@@ -66,11 +66,11 @@ pub struct TokenUsage {
|
||||
/// 输出 token 数
|
||||
pub output_tokens: u64,
|
||||
/// 命中缓存的输入 token 数
|
||||
pub cached_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
/// 写入 prompt cache 的输入 token 数
|
||||
pub cache_write_tokens: u64,
|
||||
pub cache_write_input_tokens: u64,
|
||||
/// 输出 token 中的 reasoning token 数
|
||||
pub reasoning_tokens: u64,
|
||||
pub reasoning_output_tokens: u64,
|
||||
/// 图片工具输入 token 数
|
||||
pub image_input_tokens: u64,
|
||||
/// 图片工具输出 token 数
|
||||
@@ -94,18 +94,19 @@ pub fn extract_usage(body: &Value) -> Option<TokenUsage> {
|
||||
let output_tokens = number_field(usage, "output_tokens")
|
||||
.or_else(|| number_field(usage, "completion_tokens"))
|
||||
.unwrap_or_default();
|
||||
let cached_tokens = nested_number_field(usage, &["input_tokens_details", "cached_tokens"])
|
||||
.or_else(|| nested_number_field(usage, &["prompt_tokens_details", "cached_tokens"]))
|
||||
.or_else(|| number_field(usage, "cached_tokens"))
|
||||
.unwrap_or_default();
|
||||
let cache_write_tokens =
|
||||
let cached_input_tokens =
|
||||
nested_number_field(usage, &["input_tokens_details", "cached_tokens"])
|
||||
.or_else(|| nested_number_field(usage, &["prompt_tokens_details", "cached_tokens"]))
|
||||
.or_else(|| number_field(usage, "cached_tokens"))
|
||||
.unwrap_or_default();
|
||||
let cache_write_input_tokens =
|
||||
nested_number_field(usage, &["input_tokens_details", "cache_write_tokens"])
|
||||
.or_else(|| {
|
||||
nested_number_field(usage, &["prompt_tokens_details", "cache_write_tokens"])
|
||||
})
|
||||
.or_else(|| number_field(usage, "cache_write_tokens"))
|
||||
.unwrap_or_default();
|
||||
let reasoning_tokens =
|
||||
let reasoning_output_tokens =
|
||||
nested_number_field(usage, &["output_tokens_details", "reasoning_tokens"])
|
||||
.or_else(|| {
|
||||
nested_number_field(usage, &["completion_tokens_details", "reasoning_tokens"])
|
||||
@@ -142,9 +143,9 @@ pub fn extract_usage(body: &Value) -> Option<TokenUsage> {
|
||||
has_usage.then_some(TokenUsage {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cached_tokens,
|
||||
cache_write_tokens,
|
||||
reasoning_tokens,
|
||||
cached_input_tokens,
|
||||
cache_write_input_tokens,
|
||||
reasoning_output_tokens,
|
||||
image_input_tokens,
|
||||
image_output_tokens,
|
||||
total_tokens,
|
||||
@@ -187,11 +188,11 @@ pub fn billable_usage_is_complete(response: &Value, usage: TokenUsage) -> bool {
|
||||
|
||||
input == Some(usage.input_tokens)
|
||||
&& output == Some(usage.output_tokens)
|
||||
&& cached == Some(usage.cached_tokens)
|
||||
&& written == Some(usage.cache_write_tokens)
|
||||
&& cached == Some(usage.cached_input_tokens)
|
||||
&& written == Some(usage.cache_write_input_tokens)
|
||||
&& usage
|
||||
.cached_tokens
|
||||
.checked_add(usage.cache_write_tokens)
|
||||
.cached_input_tokens
|
||||
.checked_add(usage.cache_write_input_tokens)
|
||||
.is_some_and(|cached| cached <= usage.input_tokens)
|
||||
&& total == Some(usage.total_tokens)
|
||||
&& raw
|
||||
|
||||
@@ -165,9 +165,9 @@ fn extract_usage_should_read_codex_usage_shape() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 12,
|
||||
output_tokens: 5,
|
||||
cached_tokens: 3,
|
||||
cache_write_tokens: 4,
|
||||
reasoning_tokens: 0,
|
||||
cached_input_tokens: 3,
|
||||
cache_write_input_tokens: 4,
|
||||
reasoning_output_tokens: 0,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
total_tokens: 17,
|
||||
@@ -191,9 +191,9 @@ fn extract_usage_should_read_openai_usage_shape() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 8,
|
||||
output_tokens: 4,
|
||||
cached_tokens: 2,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 3,
|
||||
cached_input_tokens: 2,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 3,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
total_tokens: 12,
|
||||
@@ -217,9 +217,9 @@ fn extract_usage_should_preserve_explicit_total_and_reasoning_tokens() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 12,
|
||||
output_tokens: 8,
|
||||
cached_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 6,
|
||||
cached_input_tokens: 0,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 6,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
total_tokens: 20,
|
||||
@@ -248,9 +248,9 @@ fn extract_usage_should_keep_image_tool_usage_separate_from_model_total() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 12,
|
||||
output_tokens: 5,
|
||||
cached_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_input_tokens: 0,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 0,
|
||||
image_input_tokens: 31,
|
||||
image_output_tokens: 9,
|
||||
total_tokens: 17,
|
||||
@@ -278,9 +278,9 @@ fn extract_sse_usage_should_prefer_completed_response_usage() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 3,
|
||||
output_tokens: 5,
|
||||
cached_tokens: 1,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_input_tokens: 1,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 0,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
total_tokens: 8,
|
||||
@@ -302,9 +302,9 @@ fn extract_sse_usage_should_fall_back_to_last_visible_usage() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 4,
|
||||
output_tokens: 6,
|
||||
cached_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_input_tokens: 0,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 0,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
total_tokens: 10,
|
||||
@@ -324,9 +324,9 @@ fn extract_sse_usage_should_preserve_completed_image_tool_usage() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 12,
|
||||
output_tokens: 5,
|
||||
cached_tokens: 3,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_input_tokens: 3,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 0,
|
||||
image_input_tokens: 31,
|
||||
image_output_tokens: 9,
|
||||
total_tokens: 17,
|
||||
@@ -348,9 +348,9 @@ fn extract_sse_usage_should_ignore_non_json_control_events() {
|
||||
Some(TokenUsage {
|
||||
input_tokens: 2,
|
||||
output_tokens: 3,
|
||||
cached_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
reasoning_tokens: 0,
|
||||
cached_input_tokens: 0,
|
||||
cache_write_input_tokens: 0,
|
||||
reasoning_output_tokens: 0,
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
total_tokens: 5,
|
||||
|
||||
@@ -27,7 +27,7 @@ hex = "0.4.3"
|
||||
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order", "raw_value"] }
|
||||
sha2 = "0.11.0"
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json", "macros", "migrate", "uuid"] }
|
||||
thiserror = "2.0.18"
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
//! 跟踪迁移目录,确保新增 SQL 文件会重新生成内嵌迁移
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=../../migrations");
|
||||
}
|
||||
@@ -17,6 +17,7 @@ pub struct StoreBundle {
|
||||
worker_contributions: Vec<WorkerContribution>,
|
||||
command_writers: Option<CommandStoreWriters>,
|
||||
command_drain: Option<CommandStoreDrain>,
|
||||
analytics: postgres::PgRequestAnalyticsRepository,
|
||||
}
|
||||
|
||||
impl StoreBundle {
|
||||
@@ -55,6 +56,14 @@ impl StoreBundle {
|
||||
self.health_probes.clone()
|
||||
}
|
||||
|
||||
/// Provider 计价规则在应用组装阶段注入,Store 只负责持久化投影
|
||||
pub fn request_analytics_worker(
|
||||
&self,
|
||||
projector: Arc<dyn gateway_admin::ports::store::UsageBillingProjector>,
|
||||
) -> StoreResult<WorkerContribution> {
|
||||
request_analytics_worker(self.analytics.clone().with_billing_projector(projector))
|
||||
}
|
||||
|
||||
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
|
||||
std::mem::take(&mut self.worker_contributions)
|
||||
}
|
||||
@@ -188,7 +197,7 @@ async fn connect(
|
||||
pool.clone(),
|
||||
Some(credential_leases.clone()),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
observability_query_budget,
|
||||
observability_query_budget.clone(),
|
||||
)
|
||||
.with_timezone(config.timezone),
|
||||
),
|
||||
@@ -291,7 +300,9 @@ async fn connect(
|
||||
}),
|
||||
),
|
||||
};
|
||||
let analytics = postgres::PgRequestAnalyticsRepository::new(pool, observability_query_budget);
|
||||
Ok(StoreBundle {
|
||||
analytics,
|
||||
diagnostics,
|
||||
admin_ports,
|
||||
core_ports,
|
||||
|
||||
@@ -46,4 +46,4 @@ pub use bundle::*;
|
||||
pub use config::*;
|
||||
pub use value::*;
|
||||
pub(crate) use workers::*;
|
||||
pub use workers::{CommandStoreDrainError, PostgresHealthProbe};
|
||||
pub use workers::{CommandStoreDrainError, PostgresHealthProbe, run_request_analytics_cycle};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Redis 丢失后从 `model_requests` 恢复客户端准入热状态
|
||||
//! Redis 丢失后从历史请求与有效运行租约恢复客户端准入热状态
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -27,7 +27,7 @@ pub struct ClientAdmissionRecentRequest {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ClientAdmissionRunningRequest {
|
||||
pub model_request_id: String,
|
||||
pub deadline_at: DateTime<Utc>,
|
||||
pub lease_expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -63,18 +63,21 @@ impl ClientAdmissionRecoveryRepository for PgClientAdmissionRecoveryRepository {
|
||||
&self,
|
||||
window_started_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<ClientAdmissionRecovery>> {
|
||||
let rows = sqlx::query_as::<_, (String, String, DateTime<Utc>, DateTime<Utc>, String)>(
|
||||
"select client_api_key_ref, id, started_at, deadline_at, outcome
|
||||
from model_requests
|
||||
where started_at >= $1 or outcome = 'running'
|
||||
order by client_api_key_ref, started_at, id",
|
||||
)
|
||||
.bind(window_started_at)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load client admission recovery", source))?;
|
||||
let rows =
|
||||
sqlx::query_as::<_, (String, String, DateTime<Utc>, Option<DateTime<Utc>>, String)>(
|
||||
"select request.client_api_key_ref, request.id, request.started_at,
|
||||
lease.lease_expires_at, request.outcome
|
||||
from model_requests request
|
||||
left join model_request_leases lease on lease.request_id = request.id
|
||||
where request.started_at >= $1 or request.outcome = 'running'
|
||||
order by request.client_api_key_ref, request.started_at, request.id",
|
||||
)
|
||||
.bind(window_started_at)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load client admission recovery", source))?;
|
||||
let mut recoveries = BTreeMap::<String, ClientAdmissionRecovery>::new();
|
||||
for (client_api_key_ref, model_request_id, started_at, deadline_at, outcome) in rows {
|
||||
for (client_api_key_ref, model_request_id, started_at, lease_expires_at, outcome) in rows {
|
||||
let recovery = recoveries
|
||||
.entry(client_api_key_ref.clone())
|
||||
.or_insert_with(|| ClientAdmissionRecovery {
|
||||
@@ -88,12 +91,15 @@ impl ClientAdmissionRecoveryRepository for PgClientAdmissionRecoveryRepository {
|
||||
started_at,
|
||||
});
|
||||
}
|
||||
if outcome == "running" {
|
||||
// 缺失租约的遗留行由请求恢复 Worker 收敛,不伪造仍占用准入容量的期限
|
||||
if outcome == "running"
|
||||
&& let Some(lease_expires_at) = lease_expires_at
|
||||
{
|
||||
recovery
|
||||
.running_requests
|
||||
.push(ClientAdmissionRunningRequest {
|
||||
model_request_id,
|
||||
deadline_at,
|
||||
lease_expires_at,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -133,7 +139,7 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository {
|
||||
Ok(RunningAdmissionFact {
|
||||
model_request_id: ModelRequestId::new(request.model_request_id)
|
||||
.map_err(|source| ClientAdmissionError(Some(source.into())))?,
|
||||
expires_at: request.deadline_at.into(),
|
||||
expires_at: request.lease_expires_at.into(),
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, ClientAdmissionError>>()?;
|
||||
|
||||
@@ -346,6 +346,7 @@ async fn advance_windows(
|
||||
let weekly_end = timezone
|
||||
.days_after(now, 7)
|
||||
.ok_or_else(|| sqlx::Error::Protocol("invalid budget week end".to_owned()))?;
|
||||
// Key 行锁仍串行化准入和结算;未滚动的窗口无需产生新元组与 WAL
|
||||
sqlx::query("insert into client_key_budget_windows
|
||||
(client_api_key_id, daily_start, daily_end, weekly_start, weekly_end)
|
||||
select $1, $4, $5, $4, $6
|
||||
@@ -355,7 +356,9 @@ async fn advance_windows(
|
||||
daily_used_usd = case when client_key_budget_windows.daily_end <= $2 and client_key_budget_windows.daily_start <= $3 then 0 else client_key_budget_windows.daily_used_usd end,
|
||||
weekly_start = case when client_key_budget_windows.weekly_end <= $2 and client_key_budget_windows.weekly_start <= $3 then greatest(excluded.weekly_start, client_key_budget_windows.weekly_end, client_key_budget_windows.weekly_start) else client_key_budget_windows.weekly_start end,
|
||||
weekly_end = case when client_key_budget_windows.weekly_end <= $2 and client_key_budget_windows.weekly_start <= $3 then excluded.weekly_end else client_key_budget_windows.weekly_end end,
|
||||
weekly_used_usd = case when client_key_budget_windows.weekly_end <= $2 and client_key_budget_windows.weekly_start <= $3 then 0 else client_key_budget_windows.weekly_used_usd end")
|
||||
weekly_used_usd = case when client_key_budget_windows.weekly_end <= $2 and client_key_budget_windows.weekly_start <= $3 then 0 else client_key_budget_windows.weekly_used_usd end
|
||||
where (client_key_budget_windows.daily_end <= $2 and client_key_budget_windows.daily_start <= $3)
|
||||
or (client_key_budget_windows.weekly_end <= $2 and client_key_budget_windows.weekly_start <= $3)")
|
||||
.bind(key).bind(now).bind(used_at).bind(day).bind(daily_end).bind(weekly_end).execute(&mut **tx).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ const CLIENT_API_KEY_LAST_USED_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ClientApiKeySnapshot {
|
||||
pub name_snapshot: Option<String>,
|
||||
pub request_profiles: std::collections::BTreeMap<
|
||||
gateway_core::routing::ProviderKind,
|
||||
gateway_core::account::OpaqueProviderData,
|
||||
@@ -73,6 +74,7 @@ impl ClientApiKeySnapshot {
|
||||
requests_per_minute: i64,
|
||||
) -> StoreResult<Self> {
|
||||
Ok(Self {
|
||||
name_snapshot: None,
|
||||
request_profiles: std::collections::BTreeMap::new(),
|
||||
id: ClientApiKeyId::new(id)
|
||||
.map_err(|source| invalid("persisted key ID is invalid").with_source(source))?,
|
||||
|
||||
@@ -89,7 +89,7 @@ impl CostSource {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct NewModelRequest {
|
||||
pub id: String,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub client_api_key_name_snapshot: Option<String>,
|
||||
pub client_api_key_ref: String,
|
||||
pub config_revision: u64,
|
||||
pub routing_scope: String,
|
||||
@@ -111,7 +111,7 @@ pub struct NewModelRequest {
|
||||
pub image_generation_requested: bool,
|
||||
pub admission_decision_ms: Option<u64>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub deadline_at: DateTime<Utc>,
|
||||
pub lease_expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
@@ -137,18 +137,11 @@ impl NewModelRequest {
|
||||
&self.routing_group_refs,
|
||||
&self.routing_group_names_snapshot,
|
||||
)?;
|
||||
if self.config_revision == 0 || self.started_at > self.deadline_at {
|
||||
if self.config_revision == 0 || self.started_at > self.lease_expires_at {
|
||||
return Err(invalid(
|
||||
"revision and deadline violate the frozen constraints",
|
||||
));
|
||||
}
|
||||
if self
|
||||
.client_api_key_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.client_api_key_ref)
|
||||
{
|
||||
return Err(invalid("live client key ID must equal its historical ref"));
|
||||
}
|
||||
self.continuation.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -183,7 +176,7 @@ pub struct ModelRequestAttemptStart {
|
||||
pub model_request_id: String,
|
||||
pub attempt_count: u32,
|
||||
pub provider_kind: String,
|
||||
pub provider_account_id: Option<String>,
|
||||
pub account_snapshot: Option<gateway_core::account::AccountHistorySnapshot>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: String,
|
||||
@@ -213,15 +206,6 @@ impl ModelRequestAttemptStart {
|
||||
(Some(used), Some(total)) if total > 0 && used <= total => {}
|
||||
_ => return Err(invalid("capacity snapshot is invalid")),
|
||||
}
|
||||
if self
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| Some(id) != self.provider_account_ref.as_ref())
|
||||
{
|
||||
return Err(invalid(
|
||||
"live provider account ID must equal its historical ref",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -427,21 +411,24 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
request.validate()?;
|
||||
let observation = RequestObservation::initial(&request)?;
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_id, client_api_key_ref, operation, client_transport,
|
||||
requested_model_id, image_generation_requested, started_at, deadline_at,
|
||||
continuation_affinity_hash, continuation_requested, request_kind, request_observation_json
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
|
||||
"with inserted as (
|
||||
insert into model_requests (
|
||||
id, client_api_key_ref, operation, client_transport,
|
||||
requested_model_id, image_generation_requested, started_at,
|
||||
continuation_affinity_hash, continuation_requested, request_kind, request_observation_json
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $9, $10, $11, $12)
|
||||
returning id
|
||||
) insert into model_request_leases (request_id, lease_expires_at)
|
||||
select id, $8 from inserted",
|
||||
)
|
||||
.bind(request.id)
|
||||
.bind(request.client_api_key_id)
|
||||
.bind(request.client_api_key_ref)
|
||||
.bind(request.operation)
|
||||
.bind(request.client_transport)
|
||||
.bind(request.requested_model_id)
|
||||
.bind(request.image_generation_requested)
|
||||
.bind(request.started_at)
|
||||
.bind(request.deadline_at)
|
||||
.bind(request.lease_expires_at)
|
||||
.bind(request.continuation.affinity_hash)
|
||||
.bind(request.continuation.requested)
|
||||
.bind(request.request_kind)
|
||||
@@ -464,40 +451,39 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
}
|
||||
let observation = RequestObservation::initial(&request)?;
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_id, client_api_key_ref, operation, client_transport,
|
||||
requested_model_id, image_generation_requested, started_at, deadline_at,
|
||||
continuation_affinity_hash, continuation_requested, request_kind, request_observation_json,
|
||||
provider_kind, provider_account_id, provider_account_ref, upstream_model_id,
|
||||
upstream_transport, attempt_count, upstream_send_state
|
||||
) select $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
|
||||
$13::jsonb || jsonb_build_object(
|
||||
'account', jsonb_strip_nulls(jsonb_build_object(
|
||||
'name', account.name, 'email', account.email,
|
||||
'authenticationKind', account.authentication_kind)),
|
||||
'transport', jsonb_strip_nulls(jsonb_build_object('httpVersion', $19::text)),
|
||||
'scheduling', ($13::jsonb -> 'scheduling') || jsonb_strip_nulls(jsonb_build_object(
|
||||
'accountSelectionWaitMs', $20::bigint,
|
||||
'capacityUsedSlots', $21::bigint, 'capacityTotalSlots', $22::bigint))),
|
||||
$14, $15, $16, $17, $18, 1, 'not_sent'
|
||||
from (values (true)) as seed(present)
|
||||
left join provider_accounts account on account.id = $15",
|
||||
"with inserted as (
|
||||
insert into model_requests (
|
||||
id, client_api_key_ref, operation, client_transport,
|
||||
requested_model_id, image_generation_requested, started_at,
|
||||
continuation_affinity_hash, continuation_requested, request_kind, request_observation_json,
|
||||
provider_kind, provider_account_ref, upstream_model_id,
|
||||
upstream_transport, attempt_count, upstream_send_state
|
||||
) select $1, $2, $3, $4, $5, $6, $7, $9, $10, $11,
|
||||
$12::jsonb || jsonb_build_object(
|
||||
'account', coalesce($14::jsonb, '{}'::jsonb),
|
||||
'transport', jsonb_strip_nulls(jsonb_build_object('httpVersion', $18::text)),
|
||||
'scheduling', ($12::jsonb -> 'scheduling') || jsonb_strip_nulls(jsonb_build_object(
|
||||
'accountSelectionWaitMs', $19::bigint,
|
||||
'capacityUsedSlots', $20::bigint, 'capacityTotalSlots', $21::bigint))),
|
||||
$13, $15, $16, $17, 1, 'not_sent'
|
||||
returning id
|
||||
) insert into model_request_leases (request_id, lease_expires_at)
|
||||
select id, $8 from inserted",
|
||||
)
|
||||
.bind(request.id)
|
||||
.bind(request.client_api_key_id)
|
||||
.bind(request.client_api_key_ref)
|
||||
.bind(request.operation)
|
||||
.bind(request.client_transport)
|
||||
.bind(request.requested_model_id)
|
||||
.bind(request.image_generation_requested)
|
||||
.bind(request.started_at)
|
||||
.bind(request.deadline_at)
|
||||
.bind(request.lease_expires_at)
|
||||
.bind(request.continuation.affinity_hash)
|
||||
.bind(request.continuation.requested)
|
||||
.bind(request.request_kind)
|
||||
.bind(observation)
|
||||
.bind(attempt.provider_kind)
|
||||
.bind(attempt.provider_account_id)
|
||||
.bind(attempt.account_snapshot.map(sqlx::types::Json))
|
||||
.bind(attempt.provider_account_ref)
|
||||
.bind(attempt.upstream_model_id)
|
||||
.bind(attempt.upstream_transport)
|
||||
@@ -531,31 +517,35 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
// 由 mark/finalize 抬升;开启新 attempt 不得把已持久化的 sent 重置回
|
||||
// not_sent——崩溃恢复的终态写回会原样继承本列
|
||||
let count = sqlx::query_scalar::<_, i32>(
|
||||
"update model_requests
|
||||
"with observed as materialized (
|
||||
select id,
|
||||
(request_observation_json #>> '{scheduling,accountSelectionWaitMs}')::bigint as selection_wait_ms,
|
||||
(request_observation_json #>> '{scheduling,capacityUsedSlots}')::bigint as used_slots,
|
||||
(request_observation_json #>> '{scheduling,capacityTotalSlots}')::bigint as total_slots
|
||||
from model_requests where id = $1 for update
|
||||
) update model_requests
|
||||
set provider_kind = $2,
|
||||
provider_account_id = $3,
|
||||
provider_account_ref = $4,
|
||||
upstream_model_id = $5,
|
||||
upstream_transport = $6,
|
||||
attempt_count = $8,
|
||||
request_observation_json = request_observation_json || jsonb_build_object(
|
||||
'account', coalesce((select jsonb_strip_nulls(jsonb_build_object(
|
||||
'name', name, 'email', email, 'authenticationKind', authentication_kind))
|
||||
from provider_accounts where id = $3), '{}'::jsonb),
|
||||
'account', coalesce($3::jsonb, '{}'::jsonb),
|
||||
'transport', jsonb_strip_nulls(jsonb_build_object('httpVersion', $7::text)),
|
||||
'scheduling', coalesce(request_observation_json -> 'scheduling', '{}'::jsonb) || jsonb_strip_nulls(jsonb_build_object(
|
||||
'accountSelectionWaitMs', case when $9::bigint is null
|
||||
then (request_observation_json #>> '{scheduling,accountSelectionWaitMs}')::bigint
|
||||
else coalesce((request_observation_json #>> '{scheduling,accountSelectionWaitMs}')::bigint, 0) + $9 end,
|
||||
'capacityUsedSlots', coalesce($10::bigint, (request_observation_json #>> '{scheduling,capacityUsedSlots}')::bigint),
|
||||
'capacityTotalSlots', coalesce($11::bigint, (request_observation_json #>> '{scheduling,capacityTotalSlots}')::bigint))))
|
||||
where id = $1 and outcome = 'running' and downstream_committed_at is null
|
||||
then observed.selection_wait_ms
|
||||
else coalesce(observed.selection_wait_ms, 0) + $9 end,
|
||||
'capacityUsedSlots', coalesce($10::bigint, observed.used_slots),
|
||||
'capacityTotalSlots', coalesce($11::bigint, observed.total_slots))))
|
||||
from observed
|
||||
where model_requests.id = observed.id and outcome = 'running' and downstream_committed_at is null
|
||||
and $8 = attempt_count + 1
|
||||
returning attempt_count",
|
||||
)
|
||||
.bind(&attempt.model_request_id)
|
||||
.bind(attempt.provider_kind)
|
||||
.bind(attempt.provider_account_id)
|
||||
.bind(attempt.account_snapshot.map(sqlx::types::Json))
|
||||
.bind(attempt.provider_account_ref)
|
||||
.bind(attempt.upstream_model_id)
|
||||
.bind(attempt.upstream_transport)
|
||||
@@ -664,7 +654,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
client_status_code = coalesce(client_status_code, $6), upstream_status_code = $7,
|
||||
client_response_id = $8, upstream_request_id = $9, upstream_response_id = $10,
|
||||
error_kind = $11, input_tokens = $12, output_tokens = $13,
|
||||
cached_tokens = $14, cache_write_tokens = $15, reasoning_tokens = $16,
|
||||
cached_input_tokens = $14, cache_write_input_tokens = $15, reasoning_output_tokens = $16,
|
||||
image_input_tokens = $17, image_output_tokens = $18, total_tokens = $19,
|
||||
image_generation_succeeded = $20, cost_source = $21,
|
||||
cost_amount = $22::numeric, cost_currency = $23,
|
||||
@@ -771,9 +761,9 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
.bind(finalization.error_kind)
|
||||
.bind(optional_i64(finalization.usage.input_tokens, "input_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.output_tokens, "output_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.cached_tokens, "cached_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.cache_write_tokens, "cache_write_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.reasoning_tokens, "reasoning_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.cached_input_tokens, "cached_input_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.cache_write_input_tokens, "cache_write_input_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.reasoning_output_tokens, "reasoning_output_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.image_input_tokens, "image_input_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.image_output_tokens, "image_output_tokens")?)
|
||||
.bind(optional_i64(finalization.usage.total_tokens, "total_tokens")?)
|
||||
@@ -800,15 +790,29 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
now: DateTime<Utc>,
|
||||
) -> StoreResult<ModelRequestRecoveryReport> {
|
||||
let result = sqlx::query(
|
||||
"update model_requests
|
||||
"with candidates as materialized (
|
||||
select request.id from model_requests request
|
||||
left join model_request_leases lease on lease.request_id = request.id
|
||||
where request.outcome = 'running'
|
||||
and coalesce(lease.lease_expires_at, request.recovery_lease_expires_at, request.started_at) <= $1
|
||||
order by request.id for update of request skip locked
|
||||
), expired as materialized (
|
||||
select candidate.id from candidates candidate
|
||||
left join lateral (
|
||||
select lease_expires_at from model_request_leases
|
||||
where request_id = candidate.id for update
|
||||
) lease on true
|
||||
where lease.lease_expires_at is null or lease.lease_expires_at <= $1
|
||||
) update model_requests
|
||||
set outcome = 'incomplete', error_kind = 'process_interrupted',
|
||||
request_observation_json = request_observation_json || jsonb_build_object('error',
|
||||
jsonb_build_object('message', 'request did not reach a terminal state')),
|
||||
jsonb_build_object('message', case when exists (select 1 from model_request_leases where request_id = model_requests.id)
|
||||
then 'request did not reach a terminal state' else 'request recovery lease is missing' end)),
|
||||
image_generation_succeeded = case
|
||||
when image_generation_requested then false else null
|
||||
end,
|
||||
completed_at = $1
|
||||
where outcome = 'running' and deadline_at <= $1",
|
||||
completed_at = greatest($1, started_at)
|
||||
where outcome = 'running' and id in (select id from expired)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&self.pool)
|
||||
@@ -847,9 +851,9 @@ impl ExecutionStore for PgExecutionStore {
|
||||
// 首次观测可能尚在队列中;缺行不创建记录,已终结行不改写
|
||||
sqlx::query_scalar::<_, bool>(
|
||||
"with renewed as (
|
||||
update model_requests set deadline_at = now() + $2 * interval '1 millisecond'
|
||||
where id = $1 and outcome = 'running' and deadline_at > now()
|
||||
returning id
|
||||
update model_request_leases set lease_expires_at = now() + $2 * interval '1 millisecond'
|
||||
where request_id = $1 and lease_expires_at > now()
|
||||
returning request_id
|
||||
)
|
||||
select exists(select 1 from renewed)
|
||||
or not exists(select 1 from model_requests where id = $1 and outcome = 'running')",
|
||||
@@ -968,7 +972,7 @@ impl ExecutionStore for PgExecutionStore {
|
||||
component: "routing".to_owned(),
|
||||
operation: failure.trigger.as_str().to_owned(),
|
||||
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
|
||||
provider_account_id: failure.account_id.as_ref().map(|id| id.as_str().to_owned()),
|
||||
account_snapshot: failure.account_snapshot,
|
||||
provider_account_ref: failure.account_id.as_ref().map(|id| id.as_str().to_owned()),
|
||||
upstream_model_id: failure
|
||||
.upstream_model_id
|
||||
@@ -1015,7 +1019,7 @@ impl ExecutionStore for PgExecutionStore {
|
||||
component: "request_entry".to_owned(),
|
||||
operation: "reject".to_owned(),
|
||||
provider_kind: None,
|
||||
provider_account_id: None,
|
||||
account_snapshot: None,
|
||||
provider_account_ref: None,
|
||||
upstream_model_id: None,
|
||||
failure_kind: error.kind().as_str().to_owned(),
|
||||
@@ -1064,7 +1068,7 @@ impl ExecutionStore for PgExecutionStore {
|
||||
component: "account_probe".to_owned(),
|
||||
operation: "connection_test".to_owned(),
|
||||
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
|
||||
provider_account_id: Some(failure.account_id.as_str().to_owned()),
|
||||
account_snapshot: None,
|
||||
provider_account_ref: Some(failure.account_id.as_str().to_owned()),
|
||||
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
|
||||
failure_kind: error.kind().as_str().to_owned(),
|
||||
@@ -1266,10 +1270,7 @@ fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
|
||||
routing_snapshot_row(&request.routing);
|
||||
NewModelRequest {
|
||||
id: request.id.as_str().to_owned(),
|
||||
client_api_key_id: request
|
||||
.client_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
client_api_key_name_snapshot: request.client_api_key_name_snapshot,
|
||||
client_api_key_ref: request.client_api_key_ref.as_str().to_owned(),
|
||||
config_revision: request.config_revision.get(),
|
||||
routing_scope,
|
||||
@@ -1297,7 +1298,7 @@ fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
|
||||
image_generation_requested: request.image_generation_requested,
|
||||
admission_decision_ms: request.admission_decision_ms,
|
||||
started_at: DateTime::<Utc>::from(request.started_at),
|
||||
deadline_at: DateTime::<Utc>::from(request.deadline_at.lease_deadline()),
|
||||
lease_expires_at: DateTime::<Utc>::from(request.deadline_at.lease_deadline()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1348,10 +1349,7 @@ fn attempt_start_row(attempt: CoreAttemptRecord) -> ModelRequestAttemptStart {
|
||||
model_request_id: attempt.request_id.as_str().to_owned(),
|
||||
attempt_count: attempt.attempt_count.get(),
|
||||
provider_kind: attempt.provider_kind.as_str().to_owned(),
|
||||
provider_account_id: attempt
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned()),
|
||||
account_snapshot: attempt.account_snapshot,
|
||||
provider_account_ref: attempt
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
@@ -1451,7 +1449,7 @@ impl gateway_core::diagnostics::OperationalDiagnostics for PgExecutionStore {
|
||||
component: failure.component.to_owned(),
|
||||
operation: failure.operation.to_owned(),
|
||||
provider_kind: failure.provider_kind.map(|kind| kind.as_str().to_owned()),
|
||||
provider_account_id: failure.account_id.as_ref().map(|id| id.as_str().to_owned()),
|
||||
account_snapshot: failure.account_snapshot,
|
||||
provider_account_ref: failure.account_id.map(|id| id.as_str().to_owned()),
|
||||
upstream_model_id: None,
|
||||
failure_kind: failure.kind.to_owned(),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 请求观测 JSON 的类型化编码,只拥有持久化分组,不决定业务状态
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use serde_json::value::RawValue;
|
||||
|
||||
use super::{ModelRequestFinalization, NewModelRequest, StoreResult, invalid};
|
||||
|
||||
@@ -30,6 +30,8 @@ pub(super) struct RequestObservation {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct RequestContext {
|
||||
config_revision: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
client_api_key_name_snapshot: Option<String>,
|
||||
protocol: String,
|
||||
endpoint: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -169,10 +171,14 @@ struct ContinuationObservation {
|
||||
}
|
||||
|
||||
impl RequestObservation {
|
||||
pub(super) fn initial(request: &NewModelRequest) -> StoreResult<sqlx::types::Json<Value>> {
|
||||
pub(super) fn initial(
|
||||
request: &NewModelRequest,
|
||||
) -> StoreResult<sqlx::types::Json<Box<RawValue>>> {
|
||||
validate_integers([Some(request.config_revision), request.admission_decision_ms])?;
|
||||
let observation = Self {
|
||||
request: Some(RequestContext {
|
||||
config_revision: request.config_revision,
|
||||
client_api_key_name_snapshot: request.client_api_key_name_snapshot.clone(),
|
||||
protocol: request.protocol.clone(),
|
||||
endpoint: request.endpoint.clone(),
|
||||
client_ip: request.client_ip.clone(),
|
||||
@@ -205,8 +211,23 @@ impl RequestObservation {
|
||||
|
||||
pub(super) fn finalized(
|
||||
value: &ModelRequestFinalization,
|
||||
) -> StoreResult<sqlx::types::Json<Value>> {
|
||||
) -> StoreResult<sqlx::types::Json<Box<RawValue>>> {
|
||||
let timings = &value.timings;
|
||||
validate_integers([
|
||||
timings.transport_decision_wait_ms,
|
||||
timings.connect_ms,
|
||||
timings.headers_ms,
|
||||
timings.first_event_ms,
|
||||
timings.first_reasoning_ms,
|
||||
timings.first_text_ms,
|
||||
timings.first_token_ms,
|
||||
timings.latency_ms,
|
||||
timings.provider_processing_ms,
|
||||
timings.upstream_response_ms,
|
||||
value.upstream_connection_age_ms,
|
||||
value.upstream_connection_idle_ms,
|
||||
value.retry_after_ms,
|
||||
])?;
|
||||
// serde_json 会把非有限浮点数写成 null,先拒绝以免把非法指标伪装成缺失
|
||||
for value in [
|
||||
timings.upstream_api_overhead_ms,
|
||||
@@ -278,44 +299,29 @@ impl RequestObservation {
|
||||
observation.encode()
|
||||
}
|
||||
|
||||
fn encode(self) -> StoreResult<sqlx::types::Json<Value>> {
|
||||
let value = serde_json::to_value(self)
|
||||
fn encode(self) -> StoreResult<sqlx::types::Json<Box<RawValue>>> {
|
||||
// 类型化事实只编码一次,SQLx 复用原始 JSON 文本,避免 Value 树与第二次长度序列化
|
||||
let text = serde_json::to_string(&self)
|
||||
.map_err(|source| invalid("encode request observation").with_source(source))?;
|
||||
validate_numbers(&value)?;
|
||||
if value.to_string().len() > 1024 * 1024 {
|
||||
if text.len() > 1024 * 1024 {
|
||||
return Err(invalid("request observation exceed 1 MiB"));
|
||||
}
|
||||
Ok(sqlx::types::Json(value))
|
||||
RawValue::from_string(text)
|
||||
.map(sqlx::types::Json)
|
||||
.map_err(|source| invalid("encode request observation").with_source(source))
|
||||
}
|
||||
}
|
||||
|
||||
// 整数指标保持 bigint 边界,官方专项计时允许毫秒小数
|
||||
fn validate_numbers(value: &Value) -> StoreResult<()> {
|
||||
match value {
|
||||
Value::Number(value) => {
|
||||
let valid = match value.as_u64() {
|
||||
Some(integer) => integer <= i64::MAX as u64,
|
||||
None => value
|
||||
.as_f64()
|
||||
.is_some_and(|value| value >= 0.0 && value < i64::MAX as f64),
|
||||
};
|
||||
if !valid {
|
||||
return Err(invalid(
|
||||
"request observation number exceeds supported range",
|
||||
));
|
||||
}
|
||||
}
|
||||
Value::Object(values) => {
|
||||
for value in values.values() {
|
||||
validate_numbers(value)?;
|
||||
}
|
||||
}
|
||||
Value::Array(values) => {
|
||||
for value in values {
|
||||
validate_numbers(value)?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
fn validate_integers(values: impl IntoIterator<Item = Option<u64>>) -> StoreResult<()> {
|
||||
if values
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|value| value > i64::MAX as u64)
|
||||
{
|
||||
return Err(invalid(
|
||||
"request observation number exceeds supported range",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -790,6 +790,7 @@ impl ExecutionObservationWrite {
|
||||
.map(|model| model.as_str()),
|
||||
failure.upstream_request_id.as_deref(),
|
||||
])
|
||||
.saturating_add(account_snapshot_bytes(failure.account_snapshot.as_ref()))
|
||||
.saturating_add(provider_error_bytes(&failure.error)),
|
||||
Self::EntryRejection(rejection) => text_bytes([
|
||||
Some(rejection.request_id.as_str()),
|
||||
@@ -808,16 +809,16 @@ impl ExecutionObservationWrite {
|
||||
Some(failure.upstream_model_id.as_str()),
|
||||
])
|
||||
.saturating_add(provider_error_bytes(&failure.error)),
|
||||
Self::OperationalFailure(failure) => {
|
||||
size_of::<OperationalFailure>().saturating_add(text_bytes([
|
||||
Self::OperationalFailure(failure) => size_of::<OperationalFailure>()
|
||||
.saturating_add(account_snapshot_bytes(failure.account_snapshot.as_ref()))
|
||||
.saturating_add(text_bytes([
|
||||
Some(failure.message.as_str()),
|
||||
failure.correlation_id.as_deref(),
|
||||
failure.provider_kind.as_ref().map(|kind| kind.as_str()),
|
||||
failure.account_id.as_ref().map(|id| id.as_str()),
|
||||
failure.upstream_code.as_ref().map(|code| code.as_str()),
|
||||
failure.details.as_ref().map(|details| details.as_str()),
|
||||
]))
|
||||
}
|
||||
])),
|
||||
Self::Finalize(finalization) => {
|
||||
let error_bytes = finalization.error.as_ref().map_or(0, |error| {
|
||||
text_bytes([
|
||||
@@ -897,10 +898,7 @@ impl ExecutionObservationWrite {
|
||||
fn new_request_bytes(request: &NewModelRequest) -> usize {
|
||||
size_of::<NewModelRequest>().saturating_add(text_bytes([
|
||||
Some(request.id.as_str()),
|
||||
request
|
||||
.client_api_key_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
request.client_api_key_name_snapshot.as_deref(),
|
||||
Some(request.client_api_key_ref.as_str()),
|
||||
Some(request.protocol.as_str()),
|
||||
Some(request.endpoint.as_str()),
|
||||
@@ -915,24 +913,36 @@ fn new_request_bytes(request: &NewModelRequest) -> usize {
|
||||
}
|
||||
|
||||
fn attempt_bytes(attempt: &AttemptRecord) -> usize {
|
||||
size_of::<AttemptRecord>().saturating_add(text_bytes([
|
||||
Some(attempt.request_id.as_str()),
|
||||
Some(attempt.provider_kind.as_str()),
|
||||
attempt
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
attempt
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
attempt
|
||||
.upstream_model_id
|
||||
.as_ref()
|
||||
.map(|model| model.as_str()),
|
||||
Some(attempt.upstream_transport.as_str()),
|
||||
attempt.http_version.as_deref(),
|
||||
]))
|
||||
size_of::<AttemptRecord>()
|
||||
.saturating_add(account_snapshot_bytes(attempt.account_snapshot.as_ref()))
|
||||
.saturating_add(text_bytes([
|
||||
Some(attempt.request_id.as_str()),
|
||||
Some(attempt.provider_kind.as_str()),
|
||||
attempt
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
attempt
|
||||
.upstream_model_id
|
||||
.as_ref()
|
||||
.map(|model| model.as_str()),
|
||||
Some(attempt.upstream_transport.as_str()),
|
||||
attempt.http_version.as_deref(),
|
||||
]))
|
||||
}
|
||||
|
||||
fn account_snapshot_bytes(
|
||||
snapshot: Option<&gateway_core::account::AccountHistorySnapshot>,
|
||||
) -> usize {
|
||||
snapshot.map_or(0, |snapshot| {
|
||||
text_bytes([
|
||||
Some(snapshot.name.as_str()),
|
||||
snapshot.email.as_deref(),
|
||||
Some(snapshot.authentication_kind.as_str()),
|
||||
snapshot.plan_type_snapshot.as_deref(),
|
||||
snapshot.notes_snapshot.as_deref(),
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
fn provider_error_bytes(error: &ProviderError) -> usize {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
//! Pg 观测查询与 Admin 观测端口适配
|
||||
|
||||
use futures::{StreamExt, TryStreamExt, stream::BoxStream};
|
||||
use gateway_admin::ports::store::UsageCalculatedBillingStream;
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -171,7 +169,6 @@ impl PgObservabilityRepository {
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<DashboardObservation> {
|
||||
let range = store_range(query.range)?;
|
||||
let filter = UsageRecordFilter::default();
|
||||
let account_usage_query = ProviderAccountUsageQuery::recent(range, query.account_limit)?
|
||||
.with_hourly_request_buckets()?;
|
||||
let recent_query = UsageRecordQuery {
|
||||
@@ -181,36 +178,39 @@ impl PgObservabilityRepository {
|
||||
page_size: ObservabilityPageSize::new(query.recent_request_limit)
|
||||
.map_err(|_| invalid("invalid dashboard page size"))?,
|
||||
};
|
||||
// 每条 SQL 独立取一个全局观测槽位,避免整包预留造成队头阻塞
|
||||
let (totals, (provider_accounts, normal_accounts)) = futures::try_join!(
|
||||
self.query_budget.run(
|
||||
"load dashboard lifetime totals",
|
||||
dashboard_totals(&self.pool)
|
||||
),
|
||||
self.account_status_snapshot(observed_at),
|
||||
)?;
|
||||
let (trend, account_usage, recent_requests, runtime_slots) = futures::try_join!(
|
||||
self.query_budget.run(
|
||||
"load dashboard request trend",
|
||||
dashboard_request_metric_series(
|
||||
&self.pool,
|
||||
range,
|
||||
&filter,
|
||||
query.granularity,
|
||||
self.timezone
|
||||
),
|
||||
),
|
||||
self.query_budget.run(
|
||||
"load dashboard account usage",
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
),
|
||||
self.query_budget
|
||||
.run("load dashboard recent requests", async {
|
||||
list_usage_record_items(&self.pool, &recent_query).await
|
||||
}),
|
||||
self.dashboard_runtime_slots(&normal_accounts),
|
||||
)?;
|
||||
let (totals, trend, analytics, account_usage) = self
|
||||
.query_budget
|
||||
.run("load dashboard analytics", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
let totals = snapshot.totals().await?;
|
||||
let trend = snapshot
|
||||
.trend(
|
||||
range,
|
||||
&UsageRecordFilter::default(),
|
||||
query.granularity,
|
||||
self.timezone,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
let account_usage =
|
||||
provider_account_usage(&mut snapshot.tx, account_usage_query).await?;
|
||||
let metadata = snapshot.metadata();
|
||||
snapshot.finish().await?;
|
||||
Ok((totals, trend, metadata, account_usage))
|
||||
})
|
||||
.await?;
|
||||
let (provider_accounts, normal_accounts) =
|
||||
self.account_status_snapshot(observed_at).await?;
|
||||
let recent_requests = self
|
||||
.query_budget
|
||||
.run(
|
||||
"load dashboard recent requests",
|
||||
list_usage_record_items(&self.pool, &recent_query),
|
||||
)
|
||||
.await?;
|
||||
let runtime_slots = self.dashboard_runtime_slots(&normal_accounts).await?;
|
||||
Ok(DashboardObservation {
|
||||
analytics,
|
||||
range,
|
||||
totals,
|
||||
provider_accounts,
|
||||
@@ -227,16 +227,20 @@ impl PgObservabilityRepository {
|
||||
granularity: admin_observability::Granularity,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
self.query_budget
|
||||
.run(
|
||||
"load dashboard request trend",
|
||||
dashboard_request_metric_series(
|
||||
&self.pool,
|
||||
range,
|
||||
&UsageRecordFilter::default(),
|
||||
granularity,
|
||||
self.timezone,
|
||||
),
|
||||
)
|
||||
.run("load dashboard analytics trend", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
let points = snapshot
|
||||
.trend(
|
||||
range,
|
||||
&UsageRecordFilter::default(),
|
||||
granularity,
|
||||
self.timezone,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(points)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -247,23 +251,42 @@ impl PgObservabilityRepository {
|
||||
granularity: admin_observability::Granularity,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
self.query_budget
|
||||
.run(
|
||||
"load usage request trend",
|
||||
request_metric_series(&self.pool, range, &filter, granularity, self.timezone),
|
||||
)
|
||||
.run("load analytics trend", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
let points = snapshot
|
||||
.trend(range, &filter, granularity, self.timezone, true)
|
||||
.await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(points)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn usage_calculated_billing_facts(
|
||||
pub async fn usage_insights_observation(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
granularity: admin_observability::Granularity,
|
||||
) -> BoxStream<'_, StoreResult<CalculatedUsageBillingFact>> {
|
||||
self.query_budget.run_stream(
|
||||
"load calculated usage billing facts",
|
||||
calculated_usage_billing_facts(&self.pool, range, filter, granularity, self.timezone),
|
||||
)
|
||||
) -> StoreResult<admin_observability::UsageInsightsObservation> {
|
||||
self.query_budget
|
||||
.run("load usage insights snapshot", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
let mut overview = snapshot.overview(range, &filter).await?;
|
||||
let trend = snapshot
|
||||
.trend(range, &filter, granularity, self.timezone, true)
|
||||
.await?;
|
||||
let costs = snapshot
|
||||
.projected_costs(range, &filter, granularity, self.timezone)
|
||||
.await?;
|
||||
overview.analytics = snapshot.metadata();
|
||||
snapshot.finish().await?;
|
||||
Ok(admin_observability::UsageInsightsObservation {
|
||||
overview: admin_usage_overview(overview),
|
||||
trend: trend.into_iter().map(admin_request_metric_point).collect(),
|
||||
costs,
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn provider_account_usage(
|
||||
@@ -271,10 +294,16 @@ impl PgObservabilityRepository {
|
||||
query: ProviderAccountUsageQuery,
|
||||
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
|
||||
self.query_budget
|
||||
.run(
|
||||
"load provider account usage",
|
||||
provider_account_usage(&self.pool, query),
|
||||
)
|
||||
.run("load provider account usage", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
snapshot.require_visible(Some(query.range)).await?;
|
||||
if let Some(range) = query.request_bucket_range {
|
||||
snapshot.require_visible(Some(range)).await?;
|
||||
}
|
||||
let usage = provider_account_usage(&mut snapshot.tx, query).await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(usage)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -301,34 +330,14 @@ impl PgObservabilityRepository {
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<UsageOverview> {
|
||||
filter.validate()?;
|
||||
let requests = self
|
||||
.query_budget
|
||||
.run(
|
||||
"load usage request metrics",
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
)
|
||||
.await?;
|
||||
let attempts = self
|
||||
.query_budget
|
||||
.run(
|
||||
"load usage attempt metrics",
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
)
|
||||
.await?;
|
||||
let providers = self
|
||||
.query_budget
|
||||
.run(
|
||||
"load usage provider observations",
|
||||
provider_observations(&self.pool, range, &filter),
|
||||
)
|
||||
.await?;
|
||||
Ok(UsageOverview {
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
providers,
|
||||
})
|
||||
self.query_budget
|
||||
.run("load usage analytics", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
let overview = snapshot.overview(range, &filter).await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(overview)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn usage_diagnostics(
|
||||
@@ -339,10 +348,14 @@ impl PgObservabilityRepository {
|
||||
limit: u16,
|
||||
) -> StoreResult<DiagnosticsObservation> {
|
||||
self.query_budget
|
||||
.run(
|
||||
"load usage diagnostics",
|
||||
usage_diagnostics(&self.pool, range, &filter, dimension, limit),
|
||||
)
|
||||
.run("load analytics diagnostics", async {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&self.pool).await?;
|
||||
let observation = snapshot
|
||||
.diagnostics(range, &filter, dimension, limit)
|
||||
.await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(observation)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -355,6 +368,22 @@ impl PgObservabilityRepository {
|
||||
|
||||
#[async_trait]
|
||||
impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
self.repository
|
||||
.query_budget
|
||||
.run("read analytics cache generation", async {
|
||||
let generation: i64 = sqlx::query_scalar(
|
||||
"select generation from request_analytics_build_state where singleton",
|
||||
)
|
||||
.fetch_one(&self.repository.pool)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read analytics cache generation", error))?;
|
||||
u64::try_from(generation).map_err(|_| invalid("invalid analytics generation"))
|
||||
})
|
||||
.await
|
||||
.map_err(observability_error)
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
query: admin_observability::DashboardQuery,
|
||||
@@ -406,25 +435,20 @@ impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn usage_calculated_billing_facts(
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
granularity: admin_observability::Granularity,
|
||||
) -> UsageCalculatedBillingStream<'_> {
|
||||
let range = match store_range(range) {
|
||||
Ok(range) => range,
|
||||
Err(error) => {
|
||||
return Box::pin(futures::stream::once(async {
|
||||
Err(observability_error(error))
|
||||
}));
|
||||
}
|
||||
};
|
||||
) -> AdminStoreResult<admin_observability::UsageInsightsObservation> {
|
||||
self.repository
|
||||
.usage_calculated_billing_facts(range, store_usage_filter(filter), granularity)
|
||||
.usage_insights_observation(
|
||||
store_range(range).map_err(observability_error)?,
|
||||
store_usage_filter(filter),
|
||||
granularity,
|
||||
)
|
||||
.await
|
||||
.map_err(observability_error)
|
||||
.map_ok(admin_calculated_usage_billing_fact)
|
||||
.boxed()
|
||||
}
|
||||
|
||||
async fn list_usage_records(
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
//! 同小时的十五分钟桶就绪后共享一次可恢复的小时构建
|
||||
|
||||
use super::worker::collections;
|
||||
use super::*;
|
||||
|
||||
type GroupStates = BTreeMap<(String, Vec<String>, String), MetricState>;
|
||||
|
||||
pub(super) enum BucketProgress {
|
||||
Waiting,
|
||||
Advanced,
|
||||
Completed(u64),
|
||||
}
|
||||
|
||||
const PEER_NOT_READY: &str =
|
||||
"(peer.build_phase <> 2 or peer.build_version is distinct from peer.change_version)";
|
||||
|
||||
pub(super) async fn pending_batch(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
) -> StoreResult<Vec<DateTime<Utc>>> {
|
||||
// 等待其他 quarter 的条目不占用领取槽位,避免八个旧等待项挡住其依赖
|
||||
let statement = format!(
|
||||
"select pending.bucket_start from request_analytics_pending_buckets pending where pending.build_phase <> 2 or pending.build_version is distinct from pending.change_version or not exists(select 1 from request_analytics_pending_buckets peer where peer.bucket_start >= date_bin(interval '1 hour',pending.bucket_start,timestamptz '2000-01-01 UTC') and peer.bucket_start < date_bin(interval '1 hour',pending.bucket_start,timestamptz '2000-01-01 UTC')+interval '1 hour' and peer.bucket_start <> pending.bucket_start and {PEER_NOT_READY}) order by (pending.build_phase >= 3) desc,pending.oldest_pending_at,pending.bucket_start limit 8"
|
||||
);
|
||||
sqlx::query_scalar(sqlx::AssertSqlSafe(statement))
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("load analytics bucket queue", error))
|
||||
}
|
||||
|
||||
pub(super) async fn rebuild_bucket_batch(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
bucket: DateTime<Utc>,
|
||||
batch_size: u32,
|
||||
) -> StoreResult<BucketProgress> {
|
||||
let row = sqlx::query("select * from request_analytics_pending_buckets where bucket_start=$1")
|
||||
.bind(bucket)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read analytics bucket progress", error))?;
|
||||
// 同一领取批次中更早的小时发布可能已经确认了这个 quarter
|
||||
let Some(row) = row else {
|
||||
return Ok(BucketProgress::Waiting);
|
||||
};
|
||||
let version: i64 = get(&row, "change_version")?;
|
||||
let same_version = get::<Option<i64>>(&row, "build_version")? == Some(version);
|
||||
let mut phase = if same_version {
|
||||
get::<i16>(&row, "build_phase")?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if !same_version {
|
||||
// 处理中事实发生变化时清掉未发布的部分状态,不能把新旧快照增量相加
|
||||
sqlx::query("update request_analytics_pending_buckets set build_version=change_version,build_phase=0,build_started_at=null,build_id=null,build_collection=null,build_dimensions=null,build_outcome=null where bucket_start=$1")
|
||||
.bind(bucket).execute(&mut **tx).await.map_err(|error|postgres_unavailable("restart changed analytics bucket",error))?;
|
||||
}
|
||||
let hour = bucket_for(bucket, 3600)?;
|
||||
if phase < 2 {
|
||||
// 小时分段期间不改写其输入;源事实仍可投影,版本变化使小时任务先退回等待
|
||||
let owned:bool=sqlx::query_scalar("select exists(select 1 from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and bucket_start <> $2 and build_phase >= 3)")
|
||||
.bind(hour).bind(bucket).fetch_one(&mut **tx).await.map_err(|error|postgres_unavailable("claim analytics hour stage",error))?;
|
||||
if owned {
|
||||
return Ok(BucketProgress::Waiting);
|
||||
}
|
||||
} else {
|
||||
let statement = format!(
|
||||
"select not exists(select 1 from request_analytics_pending_buckets peer where peer.bucket_start >= $1 and peer.bucket_start < $1+interval '1 hour' and peer.bucket_start <> $2 and {PEER_NOT_READY})"
|
||||
);
|
||||
let ready: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(statement))
|
||||
.bind(hour)
|
||||
.bind(bucket)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("check analytics hour inputs", error))?;
|
||||
if !ready {
|
||||
if phase >= 3 {
|
||||
sqlx::query("update request_analytics_pending_buckets set build_phase=2,build_started_at=null,build_id=null,build_collection=null,build_dimensions=null,build_outcome=null where bucket_start=$1")
|
||||
.bind(bucket).execute(&mut **tx).await.map_err(|error|postgres_unavailable("invalidate analytics hour inputs",error))?;
|
||||
return Ok(BucketProgress::Advanced);
|
||||
}
|
||||
return Ok(BucketProgress::Waiting);
|
||||
}
|
||||
}
|
||||
if phase == 2 {
|
||||
phase = 3;
|
||||
sqlx::query(
|
||||
"update request_analytics_pending_buckets set build_phase=3 where bucket_start=$1",
|
||||
)
|
||||
.bind(bucket)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("start analytics hour stage", error))?;
|
||||
}
|
||||
if phase == 0 || phase == 3 {
|
||||
let table = if phase == 0 {
|
||||
"request_analytics_15m"
|
||||
} else {
|
||||
"request_analytics_hourly"
|
||||
};
|
||||
let start = if phase == 0 { bucket } else { hour };
|
||||
let statement = format!(
|
||||
"delete from {table} where ctid in (select ctid from {table} where bucket_start=$1 limit $2)"
|
||||
);
|
||||
let removed = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(start)
|
||||
.bind(i64::from(batch_size))
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("clear staged analytics rows", error))?
|
||||
.rows_affected();
|
||||
if removed == u64::from(batch_size) {
|
||||
return Ok(BucketProgress::Advanced);
|
||||
}
|
||||
phase += 1;
|
||||
sqlx::query("update request_analytics_pending_buckets set build_phase=$2,build_started_at=null,build_id=null,build_collection=null,build_dimensions=null,build_outcome=null where bucket_start=$1")
|
||||
.bind(bucket).bind(phase).execute(&mut **tx).await.map_err(|error|postgres_unavailable("advance analytics bucket stage",error))?;
|
||||
// 清理阶段只负责清理,下一事务从持久游标开始读取,避免复用上一阶段游标
|
||||
return Ok(BucketProgress::Advanced);
|
||||
}
|
||||
let limit = i64::from(batch_size) + 1;
|
||||
let mut groups = GroupStates::new();
|
||||
if phase == 1 {
|
||||
let statement = format!(
|
||||
"{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and mr.recovered_at is null and (mr.started_at,mr.id)>($2,$3) order by mr.started_at,mr.id limit $4",
|
||||
fact_select()
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(bucket)
|
||||
.bind(
|
||||
get::<Option<DateTime<Utc>>>(&row, "build_started_at")?
|
||||
.unwrap_or(bucket - TimeDelta::microseconds(1)),
|
||||
)
|
||||
.bind(get::<Option<String>>(&row, "build_id")?.unwrap_or_default())
|
||||
.bind(limit)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read analytics fact segment", error))?;
|
||||
let selected = &rows[..rows.len().min(batch_size as usize)];
|
||||
for fact in selected {
|
||||
let outcome: String = get(fact, "outcome")?;
|
||||
let mut state = MetricState::default();
|
||||
state.add_row(fact)?;
|
||||
for (collection, dimensions) in collections(fact)? {
|
||||
groups
|
||||
.entry((collection.to_owned(), dimensions, outcome.clone()))
|
||||
.or_default()
|
||||
.merge(&state)?;
|
||||
}
|
||||
}
|
||||
merge_staged_groups(tx, "request_analytics_15m", bucket, groups).await?;
|
||||
if rows.len() <= batch_size as usize {
|
||||
sqlx::query("update request_analytics_pending_buckets set build_phase=2,build_started_at=null,build_id=null where bucket_start=$1")
|
||||
.bind(bucket).execute(&mut **tx).await.map_err(|error|postgres_unavailable("complete analytics quarter stage",error))?;
|
||||
} else if let Some(last) = selected.last() {
|
||||
sqlx::query("update request_analytics_pending_buckets set build_started_at=$2,build_id=$3 where bucket_start=$1")
|
||||
.bind(bucket).bind(get::<DateTime<Utc>>(last,"started_at")?).bind(get::<String>(last,"id")?).execute(&mut **tx).await.map_err(|error|postgres_unavailable("save analytics fact cursor",error))?;
|
||||
}
|
||||
} else if phase == 4 {
|
||||
let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection,dimensions,outcome)>($4,$5,$6,$7) order by bucket_start,collection,dimensions,outcome limit $8")
|
||||
.bind(hour).bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).bind(get::<Option<DateTime<Utc>>>(&row,"build_started_at")?.unwrap_or(hour-TimeDelta::microseconds(1)))
|
||||
.bind(get::<Option<String>>(&row,"build_collection")?.unwrap_or_default()).bind(get::<Option<Vec<String>>>(&row,"build_dimensions")?.unwrap_or_default()).bind(get::<Option<String>>(&row,"build_outcome")?.unwrap_or_default()).bind(limit)
|
||||
.fetch_all(&mut **tx).await.map_err(|error|postgres_unavailable("read analytics hour segment",error))?;
|
||||
let selected = &rows[..rows.len().min(batch_size as usize)];
|
||||
for quarter in selected {
|
||||
groups
|
||||
.entry((
|
||||
get(quarter, "collection")?,
|
||||
get(quarter, "dimensions")?,
|
||||
get(quarter, "outcome")?,
|
||||
))
|
||||
.or_default()
|
||||
.merge(&MetricState::decode(quarter)?)?;
|
||||
}
|
||||
merge_staged_groups(tx, "request_analytics_hourly", hour, groups).await?;
|
||||
if rows.len() <= batch_size as usize {
|
||||
// 一次小时发布确认全部就绪 quarter,避免每个条目再次清空并重写同一小时
|
||||
let acknowledged = sqlx::query("delete from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and build_version=change_version and (build_phase=2 or (bucket_start=$2 and build_phase=4 and change_version=$3))")
|
||||
.bind(hour).bind(bucket).bind(version).execute(&mut **tx).await.map_err(|error|postgres_unavailable("publish complete analytics hour",error))?.rows_affected();
|
||||
return Ok(BucketProgress::Completed(acknowledged));
|
||||
} else if let Some(last) = selected.last() {
|
||||
sqlx::query("update request_analytics_pending_buckets set build_started_at=$2,build_collection=$3,build_dimensions=$4,build_outcome=$5 where bucket_start=$1")
|
||||
.bind(bucket).bind(get::<DateTime<Utc>>(last,"bucket_start")?).bind(get::<String>(last,"collection")?).bind(get::<Vec<String>>(last,"dimensions")?).bind(get::<String>(last,"outcome")?)
|
||||
.execute(&mut **tx).await.map_err(|error|postgres_unavailable("save analytics hour cursor",error))?;
|
||||
}
|
||||
} else {
|
||||
return Err(invalid("invalid analytics bucket phase"));
|
||||
}
|
||||
Ok(BucketProgress::Advanced)
|
||||
}
|
||||
|
||||
async fn merge_staged_groups(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
table: &str,
|
||||
bucket: DateTime<Utc>,
|
||||
mut groups: GroupStates,
|
||||
) -> StoreResult<()> {
|
||||
if groups.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let keys=groups.keys().map(|(collection,dimensions,outcome)|serde_json::json!({"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::<Vec<_>>();
|
||||
let statement = format!(
|
||||
"select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1"
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(bucket)
|
||||
.bind(serde_json::Value::Array(keys))
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read staged analytics groups", error))?;
|
||||
for row in rows {
|
||||
let key = (
|
||||
get(&row, "collection")?,
|
||||
get(&row, "dimensions")?,
|
||||
get(&row, "outcome")?,
|
||||
);
|
||||
groups
|
||||
.get_mut(&key)
|
||||
.ok_or_else(|| invalid("missing staged analytics group"))?
|
||||
.merge(&MetricState::decode(&row)?)?;
|
||||
}
|
||||
write_groups(tx, table, bucket, groups).await
|
||||
}
|
||||
|
||||
async fn write_groups(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
table: &str,
|
||||
bucket: DateTime<Utc>,
|
||||
groups: GroupStates,
|
||||
) -> StoreResult<()> {
|
||||
if groups.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut statement = QueryBuilder::<Postgres>::new(format!(
|
||||
"insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) "
|
||||
));
|
||||
let rows = groups
|
||||
.into_iter()
|
||||
.map(|((collection, dimensions, outcome), state)| {
|
||||
let (counts, costs, distributions) = state.encode()?;
|
||||
Ok((
|
||||
collection,
|
||||
dimensions,
|
||||
outcome,
|
||||
counts,
|
||||
costs,
|
||||
distributions,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
// 九个绑定参数限制单语句批量,避免高基数时间桶越过 PostgreSQL 参数上限
|
||||
for chunk in rows.chunks(1000) {
|
||||
statement.push_values(
|
||||
chunk,
|
||||
|mut row, (collection, dimensions, outcome, counts, costs, distributions)| {
|
||||
row.push_bind(bucket)
|
||||
.push_bind(collection)
|
||||
.push_bind(dimensions)
|
||||
.push_bind(outcome)
|
||||
.push_bind(SCHEMA_VERSION)
|
||||
.push_bind(HISTOGRAM_VERSION)
|
||||
.push_bind(counts)
|
||||
.push_bind(costs)
|
||||
.push_bind(distributions);
|
||||
},
|
||||
);
|
||||
statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version");
|
||||
statement
|
||||
.build()
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("write analytics bucket", error))?;
|
||||
statement.reset();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//! 可合并的固定对数直方图与连续样本秩分位数
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::super::{StoreResult, invalid};
|
||||
|
||||
pub(super) const HISTOGRAM_VERSION: i32 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub(super) struct Histogram {
|
||||
pub count: u64,
|
||||
pub sum: f64,
|
||||
pub minimum: Option<f64>,
|
||||
pub maximum: Option<f64>,
|
||||
zero: u64,
|
||||
bins: BTreeMap<i32, u64>,
|
||||
}
|
||||
|
||||
impl Histogram {
|
||||
pub fn add(&mut self, value: f64) -> StoreResult<()> {
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return Err(invalid("analytics sample must be finite and nonnegative"));
|
||||
}
|
||||
self.count = self
|
||||
.count
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| invalid("histogram count overflow"))?;
|
||||
self.sum += value;
|
||||
self.minimum = Some(self.minimum.map_or(value, |old| old.min(value)));
|
||||
self.maximum = Some(self.maximum.map_or(value, |old| old.max(value)));
|
||||
if value == 0.0 {
|
||||
self.zero += 1;
|
||||
} else {
|
||||
let index = (value.log2() * 16.0).floor() as i32;
|
||||
*self.bins.entry(index).or_default() += 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &Self) -> StoreResult<()> {
|
||||
self.count = self
|
||||
.count
|
||||
.checked_add(other.count)
|
||||
.ok_or_else(|| invalid("histogram count overflow"))?;
|
||||
self.zero = self
|
||||
.zero
|
||||
.checked_add(other.zero)
|
||||
.ok_or_else(|| invalid("histogram count overflow"))?;
|
||||
self.sum += other.sum;
|
||||
if !self.sum.is_finite() {
|
||||
return Err(invalid("histogram sum overflow"));
|
||||
}
|
||||
if let Some(value) = other.minimum {
|
||||
self.minimum = Some(self.minimum.map_or(value, |old| old.min(value)));
|
||||
}
|
||||
if let Some(value) = other.maximum {
|
||||
self.maximum = Some(self.maximum.map_or(value, |old| old.max(value)));
|
||||
}
|
||||
for (index, count) in &other.bins {
|
||||
let current = self.bins.entry(*index).or_default();
|
||||
*current = current
|
||||
.checked_add(*count)
|
||||
.ok_or_else(|| invalid("histogram bin overflow"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn percentile(&self, fraction: f64) -> Option<f64> {
|
||||
if self.count == 0 {
|
||||
return None;
|
||||
}
|
||||
let rank = self.count.saturating_sub(1) as f64 * fraction;
|
||||
let lower = rank.floor() as u64;
|
||||
let upper = rank.ceil() as u64;
|
||||
let a = self.at_rank(lower)?;
|
||||
let b = self.at_rank(upper)?;
|
||||
Some(a + (b - a) * rank.fract())
|
||||
}
|
||||
|
||||
fn at_rank(&self, rank: u64) -> Option<f64> {
|
||||
if rank < self.zero {
|
||||
return Some(0.0);
|
||||
}
|
||||
let mut cumulative = self.zero;
|
||||
for (index, count) in &self.bins {
|
||||
cumulative += count;
|
||||
if rank < cumulative {
|
||||
let low = 2_f64.powf(f64::from(*index) / 16.0);
|
||||
let high = 2_f64.powf(f64::from(*index + 1) / 16.0);
|
||||
return Some((low + high) / 2.0);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
//! 窄事实的可合并指标状态,不持久化已经计算的分位数
|
||||
|
||||
use super::super::*;
|
||||
use super::histogram::Histogram;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::postgres::PgRow;
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub(super) struct MetricState {
|
||||
counts: BTreeMap<String, u64>,
|
||||
distributions: BTreeMap<String, Histogram>,
|
||||
costs: BTreeMap<String, String>,
|
||||
#[serde(skip)]
|
||||
exact_samples: Option<BTreeMap<String, Vec<f64>>>,
|
||||
}
|
||||
|
||||
impl MetricState {
|
||||
pub fn decode(row: &PgRow) -> StoreResult<Self> {
|
||||
Ok(Self {
|
||||
// 直接解码为指标类型,避免每个汇总行先构造通用 JSON 树再遍历转换
|
||||
counts: get::<sqlx::types::Json<BTreeMap<String, u64>>>(row, "counts")?.0,
|
||||
costs: get::<sqlx::types::Json<BTreeMap<String, String>>>(row, "costs")?.0,
|
||||
distributions: get::<sqlx::types::Json<BTreeMap<String, Histogram>>>(
|
||||
row,
|
||||
"distributions",
|
||||
)?
|
||||
.0,
|
||||
exact_samples: None,
|
||||
})
|
||||
}
|
||||
pub fn encode(&self) -> StoreResult<(serde_json::Value, serde_json::Value, serde_json::Value)> {
|
||||
Ok((
|
||||
serde_json::to_value(&self.counts)
|
||||
.map_err(|error| postgres_unavailable("encode analytics counters", error))?,
|
||||
serde_json::to_value(&self.costs)
|
||||
.map_err(|error| postgres_unavailable("encode analytics costs", error))?,
|
||||
serde_json::to_value(&self.distributions)
|
||||
.map_err(|error| postgres_unavailable("encode analytics histograms", error))?,
|
||||
))
|
||||
}
|
||||
pub fn exact() -> Self {
|
||||
Self {
|
||||
exact_samples: Some(BTreeMap::new()),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
pub fn count(&self, name: &str) -> u64 {
|
||||
self.counts.get(name).copied().unwrap_or_default()
|
||||
}
|
||||
fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> {
|
||||
let count = self.counts.entry(name.to_owned()).or_default();
|
||||
*count = count
|
||||
.checked_add(amount)
|
||||
.ok_or_else(|| invalid("analytics counter overflow"))?;
|
||||
Ok(())
|
||||
}
|
||||
fn sample(&mut self, name: &str, sample: Option<f64>) -> StoreResult<()> {
|
||||
if let Some(sample) = sample {
|
||||
self.distributions
|
||||
.entry(name.to_owned())
|
||||
.or_default()
|
||||
.add(sample)?;
|
||||
if let Some(samples) = &mut self.exact_samples {
|
||||
samples.entry(name.to_owned()).or_default().push(sample);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> {
|
||||
let amount: DecimalAmount = amount
|
||||
.parse()
|
||||
.map_err(|_| invalid("analytics invalid cost"))?;
|
||||
let total = if let Some(previous) = self.costs.get(currency) {
|
||||
previous
|
||||
.parse::<DecimalAmount>()
|
||||
.map_err(|_| invalid("analytics invalid stored cost"))?
|
||||
.checked_add(&amount)
|
||||
.ok_or_else(|| invalid("analytics cost overflow"))?
|
||||
} else {
|
||||
amount
|
||||
};
|
||||
self.costs.insert(currency.to_owned(), total.to_string());
|
||||
Ok(())
|
||||
}
|
||||
pub fn merge(&mut self, other: &Self) -> StoreResult<()> {
|
||||
for (name, count) in &other.counts {
|
||||
self.increment(name, *count)?;
|
||||
}
|
||||
for (name, distribution) in &other.distributions {
|
||||
self.distributions
|
||||
.entry(name.clone())
|
||||
.or_default()
|
||||
.merge(distribution)?;
|
||||
}
|
||||
if let (Some(ours), Some(theirs)) = (&mut self.exact_samples, &other.exact_samples) {
|
||||
for (name, samples) in theirs {
|
||||
ours.entry(name.clone()).or_default().extend(samples);
|
||||
}
|
||||
}
|
||||
for (currency, amount) in &other.costs {
|
||||
self.add_cost(currency, amount)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn add_row(&mut self, row: &PgRow) -> StoreResult<()> {
|
||||
let outcome: String = get(row, "outcome")?;
|
||||
let completed: bool = get(row, "is_completed_usage")?;
|
||||
self.increment("request_count", 1)?;
|
||||
for (value, name) in [
|
||||
("succeeded", "success_count"),
|
||||
("failed", "failure_count"),
|
||||
("cancelled", "cancelled_count"),
|
||||
("incomplete", "incomplete_count"),
|
||||
] {
|
||||
if outcome == value {
|
||||
self.increment(name, 1)?;
|
||||
}
|
||||
}
|
||||
let client_status: Option<i32> = get(row, "client_status_code")?;
|
||||
if client_status.is_some_and(|value| (400..=499).contains(&value)) {
|
||||
self.increment("caller_error_count", 1)?;
|
||||
}
|
||||
let attempts = u64::try_from(get::<i32>(row, "attempt_count")?)
|
||||
.map_err(|_| invalid("negative analytics attempt count"))?;
|
||||
self.increment("attempt_count", attempts)?;
|
||||
self.increment("retry_count", attempts.saturating_sub(1))?;
|
||||
if attempts > 1 {
|
||||
self.increment("retried_request_count", 1)?;
|
||||
}
|
||||
if attempts > 0 {
|
||||
for (value, name) in [
|
||||
("succeeded", "attempt_success_count"),
|
||||
("failed", "attempt_failure_count"),
|
||||
("cancelled", "attempt_cancelled_count"),
|
||||
("incomplete", "attempt_incomplete_count"),
|
||||
] {
|
||||
if outcome == value {
|
||||
self.increment(name, 1)?;
|
||||
}
|
||||
}
|
||||
if outcome == "failed" {
|
||||
let status: Option<i32> =
|
||||
get::<Option<i32>>(row, "upstream_status_code")?.or(client_status);
|
||||
let kind: Option<String> = get(row, "error_kind")?;
|
||||
if matches!(kind.as_deref(), Some("rate_limited" | "quota_exhausted"))
|
||||
|| status == Some(429)
|
||||
{
|
||||
self.increment("rate_limited_count", 1)?;
|
||||
}
|
||||
if matches!(
|
||||
kind.as_deref(),
|
||||
Some("authentication" | "authorization" | "invalid_credential")
|
||||
) || matches!(status, Some(401 | 403))
|
||||
{
|
||||
self.increment("auth_failure_count", 1)?;
|
||||
}
|
||||
if status.is_some_and(|status| (500..=599).contains(&status)) {
|
||||
self.increment("provider_5xx_count", 1)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (source, target) in [
|
||||
("ops_failure_count", "attempt_failure_count"),
|
||||
("ops_rate_limited_count", "rate_limited_count"),
|
||||
("ops_auth_failure_count", "auth_failure_count"),
|
||||
("ops_provider_5xx_count", "provider_5xx_count"),
|
||||
] {
|
||||
self.increment(target, unsigned(row, source)?)?;
|
||||
}
|
||||
for name in ["admission_decision_ms", "account_selection_wait_ms"] {
|
||||
self.sample(
|
||||
name,
|
||||
get::<Option<i64>>(row, name)?.map(|value| value as f64),
|
||||
)?;
|
||||
}
|
||||
let capacity_total: Option<i64> = get(row, "capacity_total_slots")?;
|
||||
if capacity_total.is_some() {
|
||||
self.increment("capacity_sample_count", 1)?;
|
||||
}
|
||||
if let (Some(used), Some(total)) = (
|
||||
get::<Option<i64>>(row, "capacity_used_slots")?,
|
||||
capacity_total,
|
||||
) && total > 0
|
||||
{
|
||||
self.sample("capacity", Some(used as f64 / total as f64 * 10_000.0))?;
|
||||
}
|
||||
if completed {
|
||||
for name in [
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cached_input_tokens",
|
||||
"cache_write_input_tokens",
|
||||
"reasoning_output_tokens",
|
||||
"total_tokens",
|
||||
"image_input_tokens",
|
||||
"image_output_tokens",
|
||||
] {
|
||||
self.increment(name, optional_unsigned(row, name)?.unwrap_or_default())?;
|
||||
}
|
||||
if get::<Option<i64>>(row, "input_tokens")?.is_some() {
|
||||
self.increment("cache_eligible_request_count", 1)?;
|
||||
if optional_unsigned(row, "cached_input_tokens")?.is_some_and(|value| value > 0) {
|
||||
self.increment("cache_hit_request_count", 1)?;
|
||||
}
|
||||
}
|
||||
for name in ["latency_ms", "first_token_ms"] {
|
||||
let value = optional_unsigned(row, name)?;
|
||||
if let Some(value) = value {
|
||||
self.increment(&format!("{name}_sum"), value)?;
|
||||
}
|
||||
self.sample(name, value.map(|value| value as f64))?;
|
||||
}
|
||||
if let (Some(tokens), Some(milliseconds)) = (
|
||||
optional_unsigned(row, "output_tokens")?,
|
||||
optional_unsigned(row, "upstream_response_ms")?,
|
||||
) && tokens > 0
|
||||
&& milliseconds > 0
|
||||
{
|
||||
self.sample(
|
||||
"throughput",
|
||||
Some(tokens as f64 * 1000.0 / milliseconds as f64),
|
||||
)?;
|
||||
}
|
||||
for (name, source) in [
|
||||
("standard_usd", "standard_cost_amount_text"),
|
||||
("no_cache_usd", "no_cache_cost_amount_text"),
|
||||
] {
|
||||
if let Some(amount) = get::<Option<String>>(row, source)? {
|
||||
self.increment(&format!("{name}_known_count"), 1)?;
|
||||
self.add_cost(name, &amount)?;
|
||||
}
|
||||
}
|
||||
let source: String = get(row, "cost_source")?;
|
||||
self.increment(&format!("{source}_count"), 1)?;
|
||||
if let (Some(currency), Some(amount)) = (
|
||||
get::<Option<String>>(row, "cost_currency")?,
|
||||
get::<Option<String>>(row, "cost_amount_text")?,
|
||||
) {
|
||||
self.add_cost(¤cy, &amount)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn histogram(&self, name: &str) -> Histogram {
|
||||
self.distributions.get(name).cloned().unwrap_or_default()
|
||||
}
|
||||
pub fn percentile(&self, name: &str, fraction: f64) -> Option<f64> {
|
||||
if let Some(samples) = &self.exact_samples {
|
||||
let mut samples = samples.get(name)?.clone();
|
||||
samples.sort_by(f64::total_cmp);
|
||||
let rank = samples.len().saturating_sub(1) as f64 * fraction;
|
||||
let a = *samples.get(rank.floor() as usize)?;
|
||||
let b = *samples.get(rank.ceil() as usize)?;
|
||||
return Some(a + (b - a) * rank.fract());
|
||||
}
|
||||
self.histogram(name).percentile(fraction)
|
||||
}
|
||||
fn percentiles(&self, name: &str) -> StoreResult<LatencyPercentiles> {
|
||||
let percentile = |fraction| {
|
||||
self.percentile(name, fraction)
|
||||
.map(|value| {
|
||||
PercentileMilliseconds::new(value)
|
||||
.map_err(|_| invalid("invalid analytics percentile"))
|
||||
})
|
||||
.transpose()
|
||||
};
|
||||
Ok(LatencyPercentiles {
|
||||
p50_ms: percentile(0.5)?,
|
||||
p95_ms: percentile(0.95)?,
|
||||
p99_ms: percentile(0.99)?,
|
||||
})
|
||||
}
|
||||
pub fn request_metrics(&self) -> StoreResult<RequestMetrics> {
|
||||
let latency = self.histogram("latency_ms");
|
||||
let first = self.histogram("first_token_ms");
|
||||
let capacity = self.histogram("capacity");
|
||||
Ok(RequestMetrics {
|
||||
request_count: self.count("request_count"),
|
||||
success_count: self.count("success_count"),
|
||||
failure_count: self.count("failure_count"),
|
||||
cancelled_count: self.count("cancelled_count"),
|
||||
incomplete_count: self.count("incomplete_count"),
|
||||
caller_error_count: self.count("caller_error_count"),
|
||||
input_tokens: self.count("input_tokens"),
|
||||
output_tokens: self.count("output_tokens"),
|
||||
cached_input_tokens: self.count("cached_input_tokens"),
|
||||
cache_write_input_tokens: self.count("cache_write_input_tokens"),
|
||||
reasoning_output_tokens: self.count("reasoning_output_tokens"),
|
||||
total_tokens: self.count("total_tokens"),
|
||||
cache_eligible_request_count: self.count("cache_eligible_request_count"),
|
||||
cache_hit_request_count: self.count("cache_hit_request_count"),
|
||||
capacity_sample_count: self.count("capacity_sample_count"),
|
||||
first_token_latency_sum_ms: self.count("first_token_ms_sum"),
|
||||
first_token_latency_count: first.count,
|
||||
latency_sum_ms: self.count("latency_ms_sum"),
|
||||
latency_count: latency.count,
|
||||
max_latency_ms: latency.maximum.map(|value| value as u64),
|
||||
min_latency_ms: latency.minimum.map(|value| value as u64),
|
||||
latency_percentiles: self.percentiles("latency_ms")?,
|
||||
first_token_latency_percentiles: self.percentiles("first_token_ms")?,
|
||||
admission_decision_count: self.histogram("admission_decision_ms").count,
|
||||
admission_decision_percentiles: self.percentiles("admission_decision_ms")?,
|
||||
account_selection_wait_count: self.histogram("account_selection_wait_ms").count,
|
||||
account_selection_wait_percentiles: self.percentiles("account_selection_wait_ms")?,
|
||||
output_throughput_p10: self
|
||||
.percentile("throughput", 0.1)
|
||||
.map(|value| value.round() as u64),
|
||||
output_throughput_p50: self
|
||||
.percentile("throughput", 0.5)
|
||||
.map(|value| value.round() as u64),
|
||||
output_throughput_p90: self
|
||||
.percentile("throughput", 0.9)
|
||||
.map(|value| value.round() as u64),
|
||||
capacity_utilization_avg_basis_points: (capacity.count > 0)
|
||||
.then(|| (capacity.sum / capacity.count as f64).round() as u64),
|
||||
capacity_utilization_p95_basis_points: self
|
||||
.percentile("capacity", 0.95)
|
||||
.map(|value| value.round() as u64),
|
||||
})
|
||||
}
|
||||
pub fn coverage(&self) -> CostCoverage {
|
||||
CostCoverage {
|
||||
provider_reported_count: self.count("provider_reported_count"),
|
||||
calculated_count: self.count("calculated_count"),
|
||||
unavailable_count: self.count("unavailable_count"),
|
||||
}
|
||||
}
|
||||
pub fn projected_cost(&self, name: &str) -> StoreResult<Option<DecimalAmount>> {
|
||||
self.costs
|
||||
.get(name)
|
||||
.map(|value| {
|
||||
value
|
||||
.parse()
|
||||
.map_err(|_| invalid("analytics invalid projected cost"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
pub fn costs(&self) -> StoreResult<Vec<CurrencyCostTotal>> {
|
||||
self.costs
|
||||
.iter()
|
||||
.filter(|(currency, _)| !matches!(currency.as_str(), "standard_usd" | "no_cache_usd"))
|
||||
.map(|(currency, amount)| {
|
||||
Ok(CurrencyCostTotal {
|
||||
currency: currency.clone(),
|
||||
amount: amount
|
||||
.parse()
|
||||
.map_err(|_| invalid("analytics invalid cost"))?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
pub fn attempts(&self) -> StoreResult<AttemptMetrics> {
|
||||
Ok(AttemptMetrics {
|
||||
attempt_count: self.count("attempt_count"),
|
||||
success_count: self.count("attempt_success_count"),
|
||||
failure_count: self.count("attempt_failure_count"),
|
||||
cancelled_count: self.count("attempt_cancelled_count"),
|
||||
incomplete_count: self.count("attempt_incomplete_count"),
|
||||
rate_limited_count: self.count("rate_limited_count"),
|
||||
auth_failure_count: self.count("auth_failure_count"),
|
||||
provider_5xx_count: self.count("provider_5xx_count"),
|
||||
cost_coverage: self.coverage(),
|
||||
costs: self.costs()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//! 请求观测的持久化投影、可重建汇总与受预算约束的读取
|
||||
|
||||
mod buckets;
|
||||
mod histogram;
|
||||
mod metrics;
|
||||
mod query;
|
||||
mod reset;
|
||||
mod worker;
|
||||
|
||||
pub(crate) use query::*;
|
||||
pub use worker::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
|
||||
|
||||
use super::*;
|
||||
use metrics::MetricState;
|
||||
|
||||
const SCHEMA_VERSION: i32 = 1;
|
||||
const HISTOGRAM_VERSION: i32 = histogram::HISTOGRAM_VERSION;
|
||||
const MAX_FACT_ROWS: usize = 100_000;
|
||||
|
||||
pub(super) async fn begin_snapshot(pool: &PgPool) -> StoreResult<sqlx::Transaction<'_, Postgres>> {
|
||||
let mut transaction = pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("begin analytics snapshot", error))?;
|
||||
sqlx::query("set transaction isolation level repeatable read")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("isolate analytics snapshot", error))?;
|
||||
sqlx::query("set local statement_timeout = '10s'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("set analytics query deadline", error))?;
|
||||
Ok(transaction)
|
||||
}
|
||||
|
||||
fn fact_select() -> String {
|
||||
format!(
|
||||
"select mr.*, mr.cost_amount::text as cost_amount_text, mr.standard_cost_amount::text as standard_cost_amount_text, mr.no_cache_cost_amount::text as no_cache_cost_amount_text, coalesce(({}),false) as is_completed_usage from request_analytics_facts mr",
|
||||
completed_usage_fact_predicate("mr")
|
||||
)
|
||||
}
|
||||
|
||||
fn bucket_for(time: DateTime<Utc>, seconds: i64) -> StoreResult<DateTime<Utc>> {
|
||||
DateTime::from_timestamp(time.timestamp().div_euclid(seconds) * seconds, 0)
|
||||
.ok_or_else(|| invalid("invalid analytics bucket"))
|
||||
}
|
||||
|
||||
fn unavailable(message: &str) -> StoreError {
|
||||
StoreError::Unavailable {
|
||||
backend: crate::StoreBackend::PostgreSql,
|
||||
message: message.to_owned(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,984 @@
|
||||
//! 在一致性快照中选择互斥的小时、十五分钟与窄事实区间
|
||||
|
||||
use super::*;
|
||||
use futures::TryStreamExt;
|
||||
|
||||
pub(crate) struct AnalyticsSnapshot<'a> {
|
||||
pub tx: sqlx::Transaction<'a, Postgres>,
|
||||
status: AnalyticsStatus,
|
||||
approximate: bool,
|
||||
}
|
||||
|
||||
struct ReadSelection<'a> {
|
||||
distributions: bool,
|
||||
costs: bool,
|
||||
keys: Option<&'a [String]>,
|
||||
}
|
||||
|
||||
impl<'a> AnalyticsSnapshot<'a> {
|
||||
pub async fn new(pool: &'a PgPool) -> StoreResult<Self> {
|
||||
let mut tx = begin_snapshot(pool).await?;
|
||||
// 时间跨度和分组选择性差异很大,通用计划会把长区间误估成短区间
|
||||
// 仅当前只读事务使用实际参数规划,不改变连接池后续事务的计划策略
|
||||
sqlx::query("set local plan_cache_mode = force_custom_plan")
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("plan analytics with actual bounds", error))?;
|
||||
let status = worker::query_status(&mut tx).await?;
|
||||
if status.schema_version != SCHEMA_VERSION as u32
|
||||
|| status.histogram_version != HISTOGRAM_VERSION as u32
|
||||
{
|
||||
return Err(unavailable(
|
||||
"request analytics is building; retry after the background projection completes",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
tx,
|
||||
status,
|
||||
approximate: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn require_visible(&mut self, range: Option<ObservabilityRange>) -> StoreResult<()> {
|
||||
if self.status.building
|
||||
&& !range.is_some_and(|range| {
|
||||
self.status
|
||||
.coverage_start
|
||||
.is_some_and(|start| range.start >= start)
|
||||
&& self.status.coverage_end.is_some()
|
||||
})
|
||||
{
|
||||
return Err(unavailable(
|
||||
"request analytics has not backfilled the complete requested range",
|
||||
));
|
||||
}
|
||||
// 未知旧桶与已知范围分开查,避免通用计划为 OR 选择扫描全部删除积压
|
||||
// 时间移动和未 ACK 的投影会使队列旧桶偏离当前事实,删除可见性必须同时检查事实所在范围
|
||||
let deletion_pending: bool = if let Some(range) = range {
|
||||
let lower = range
|
||||
.start
|
||||
.checked_sub_signed(TimeDelta::minutes(15))
|
||||
.ok_or_else(|| invalid("analytics deletion boundary overflow"))?;
|
||||
let (intersects, any): (bool, bool) = sqlx::query_as("select exists(select 1 from request_analytics_pending_requests where deletion_pending and previous_bucket is null) or exists(select 1 from request_analytics_pending_requests where deletion_pending and previous_bucket > $1 and previous_bucket < $2), exists(select 1 from request_analytics_pending_requests where deletion_pending)")
|
||||
.bind(lower).bind(range.end).fetch_one(&mut *self.tx).await
|
||||
.map_err(|error|postgres_unavailable("check analytics deletion visibility",error))?;
|
||||
if intersects || !any {
|
||||
intersects
|
||||
} else {
|
||||
// 没有待删版本时不引用 facts,使完整干净桶保持只读汇总的路径
|
||||
sqlx::query_scalar("select exists(select 1 from request_analytics_pending_requests p join request_analytics_facts f on f.id=p.request_id where p.deletion_pending and f.started_at >= $1 and f.started_at < $2)")
|
||||
.bind(range.start).bind(range.end).fetch_one(&mut *self.tx).await
|
||||
.map_err(|error|postgres_unavailable("check projected analytics deletion visibility",error))?
|
||||
}
|
||||
} else {
|
||||
sqlx::query_scalar("select exists(select 1 from request_analytics_pending_requests where deletion_pending)")
|
||||
.fetch_one(&mut *self.tx).await
|
||||
.map_err(|error|postgres_unavailable("check analytics deletion visibility",error))?
|
||||
};
|
||||
if deletion_pending {
|
||||
return Err(unavailable(
|
||||
"request analytics is removing deleted source contributions in this range; retry after projection",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> admin_observability::AnalyticsMetadata {
|
||||
let generated_at = Utc::now();
|
||||
admin_observability::AnalyticsMetadata {
|
||||
coverage_start: self.status.coverage_start,
|
||||
coverage_end: self.status.coverage_end,
|
||||
generated_at,
|
||||
lag_seconds: self.status.oldest_pending_at.map_or(0, |time| {
|
||||
generated_at
|
||||
.signed_duration_since(time)
|
||||
.num_seconds()
|
||||
.max(0) as u64
|
||||
}),
|
||||
state: if self.status.building {
|
||||
admin_observability::AnalyticsState::Building
|
||||
} else if self.status.oldest_pending_at.is_some() {
|
||||
admin_observability::AnalyticsState::Lagging
|
||||
} else {
|
||||
admin_observability::AnalyticsState::Ready
|
||||
},
|
||||
percentiles_approximate: self.approximate,
|
||||
generation: self.status.generation.max(0) as u64,
|
||||
schema_version: self.status.schema_version,
|
||||
histogram_version: self.status.histogram_version,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn finish(self) -> StoreResult<()> {
|
||||
self.tx
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("complete analytics query snapshot", error))
|
||||
}
|
||||
|
||||
pub async fn totals(&mut self) -> StoreResult<DashboardTotals> {
|
||||
self.require_visible(None).await?;
|
||||
let bounds=sqlx::query("select (select started_at from request_analytics_facts order by started_at,id limit 1) as first_started_at,(select started_at from request_analytics_facts order by started_at desc,id desc limit 1) as last_started_at")
|
||||
.fetch_one(&mut *self.tx).await.map_err(|error|postgres_unavailable("read retained analytics bounds",error))?;
|
||||
let start =
|
||||
get::<Option<DateTime<Utc>>>(&bounds, "first_started_at")?.unwrap_or_else(Utc::now);
|
||||
let last = get::<Option<DateTime<Utc>>>(&bounds, "last_started_at")?.unwrap_or(start);
|
||||
// 保留期总量没有用户边界,向外对齐完整桶避免反复扫描首尾热桶
|
||||
let range = ObservabilityRange {
|
||||
start: bucket_for(start, 900)?,
|
||||
end: bucket_for(last, 900)?
|
||||
.checked_add_signed(TimeDelta::minutes(15))
|
||||
.ok_or_else(|| invalid("analytics time boundary overflow"))?,
|
||||
};
|
||||
let states = self
|
||||
.load(
|
||||
range,
|
||||
&UsageRecordFilter::default(),
|
||||
None,
|
||||
None,
|
||||
Default::default(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
let state = merge_states(states.into_values())?;
|
||||
Ok(DashboardTotals {
|
||||
request_count: state.count("request_count"),
|
||||
input_tokens: state.count("input_tokens"),
|
||||
cached_input_tokens: state.count("cached_input_tokens"),
|
||||
total_tokens: state.count("total_tokens"),
|
||||
billing_usd: state
|
||||
.costs()?
|
||||
.into_iter()
|
||||
.find(|value| value.currency == "USD")
|
||||
.map(|value| value.amount),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn overview(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<UsageOverview> {
|
||||
let states = self
|
||||
.load(range, filter, None, None, Default::default(), true)
|
||||
.await?;
|
||||
let state = merge_states(states.into_values())?;
|
||||
let provider_states = self
|
||||
.load(
|
||||
range,
|
||||
filter,
|
||||
Some(DiagnosticDimension::Provider),
|
||||
None,
|
||||
Default::default(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
let mut providers = provider_states
|
||||
.into_iter()
|
||||
.map(|((_, name), state)| ProviderObservation {
|
||||
provider_kind: name,
|
||||
request_count: state.count("request_count"),
|
||||
attempt_count: state.count("attempt_count"),
|
||||
failure_count: state.count("failure_count"),
|
||||
total_tokens: state.count("total_tokens"),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
providers.sort_by(|a, b| {
|
||||
b.request_count
|
||||
.cmp(&a.request_count)
|
||||
.then(a.provider_kind.cmp(&b.provider_kind))
|
||||
});
|
||||
Ok(UsageOverview {
|
||||
range,
|
||||
requests: state.request_metrics()?,
|
||||
attempts: state.attempts()?,
|
||||
providers,
|
||||
analytics: self.metadata(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn trend(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
include_costs: bool,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
let states = self
|
||||
.load_selected(
|
||||
range,
|
||||
filter,
|
||||
None,
|
||||
Some(granularity),
|
||||
timezone,
|
||||
ReadSelection {
|
||||
distributions: true,
|
||||
costs: include_costs,
|
||||
keys: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let mut points = BTreeMap::new();
|
||||
for ((bucket, _), state) in states {
|
||||
points.insert(
|
||||
bucket,
|
||||
RequestMetricPoint {
|
||||
bucket_start: bucket,
|
||||
granularity,
|
||||
metrics: state.request_metrics()?,
|
||||
cost_coverage: state.coverage(),
|
||||
costs: if include_costs {
|
||||
state.costs()?
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
fill_metric_gaps(range, granularity, points, timezone)
|
||||
}
|
||||
|
||||
pub async fn projected_costs(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<Vec<admin_observability::UsageProjectedCostPoint>> {
|
||||
self.load(range, filter, None, Some(granularity), timezone, false)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|((bucket_start, _), state)| {
|
||||
Ok(admin_observability::UsageProjectedCostPoint {
|
||||
bucket_start,
|
||||
standard_amount: state.projected_cost("standard_usd")?,
|
||||
no_cache_amount: state.projected_cost("no_cache_usd")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn diagnostics(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
limit: u16,
|
||||
) -> StoreResult<DiagnosticsObservation> {
|
||||
let limit = ObservabilityPageSize::new(limit)
|
||||
.map_err(|_| invalid("invalid diagnostic limit"))?
|
||||
.get();
|
||||
// 高基数账号和 Key 先合并整个区间的精确计数选 TopN,再读取少量选中组的分布
|
||||
let ranked = if matches!(
|
||||
dimension,
|
||||
DiagnosticDimension::Account | DiagnosticDimension::ApiKey
|
||||
) && collection_for(filter, Some(dimension)).is_some()
|
||||
{
|
||||
Some(
|
||||
self.diagnostic_top_keys(range, filter, dimension, limit)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let selected_keys = ranked.as_ref().map(|(_, keys)| keys.as_slice());
|
||||
let states = self
|
||||
.load_selected(
|
||||
range,
|
||||
filter,
|
||||
Some(dimension),
|
||||
None,
|
||||
Default::default(),
|
||||
ReadSelection {
|
||||
distributions: true,
|
||||
costs: true,
|
||||
keys: selected_keys,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let total_request_count = if let Some((total, _)) = &ranked {
|
||||
*total
|
||||
} else {
|
||||
states.values().try_fold(0_u64, |sum, state| {
|
||||
sum.checked_add(state.count("request_count"))
|
||||
.ok_or_else(|| invalid("analytics counter overflow"))
|
||||
})?
|
||||
};
|
||||
let mut states = states.into_iter().collect::<Vec<_>>();
|
||||
states.sort_by(|a, b| {
|
||||
b.1.count("request_count")
|
||||
.cmp(&a.1.count("request_count"))
|
||||
.then(a.0.1.cmp(&b.0.1))
|
||||
});
|
||||
states.truncate(usize::from(limit));
|
||||
let mut observations = Vec::with_capacity(states.len());
|
||||
for ((_, key), state) in states {
|
||||
let metrics = state.request_metrics()?;
|
||||
observations.push(DiagnosticObservation {
|
||||
key: key.clone(),
|
||||
name: key,
|
||||
account_provider_kind: None,
|
||||
account_plan_type: None,
|
||||
request_count: metrics.request_count,
|
||||
success_count: metrics.success_count,
|
||||
failure_count: metrics.failure_count,
|
||||
attempt_count: state.count("attempt_count"),
|
||||
total_tokens: metrics.total_tokens,
|
||||
average_latency_ms: (metrics.latency_count > 0).then(|| {
|
||||
(metrics.latency_sum_ms as f64 / metrics.latency_count as f64).round() as u64
|
||||
}),
|
||||
latency_p95_ms: state
|
||||
.percentile("latency_ms", 0.95)
|
||||
.map(|value| value.round() as u64),
|
||||
first_token_p95_ms: state
|
||||
.percentile("first_token_ms", 0.95)
|
||||
.map(|value| value.round() as u64),
|
||||
non_completion_count: metrics.cancelled_count + metrics.incomplete_count,
|
||||
retry_count: state.count("retry_count"),
|
||||
retried_request_count: state.count("retried_request_count"),
|
||||
cost_coverage: state.coverage(),
|
||||
costs: state.costs()?,
|
||||
});
|
||||
}
|
||||
if matches!(
|
||||
dimension,
|
||||
DiagnosticDimension::Account | DiagnosticDimension::ApiKey
|
||||
) && !observations.is_empty()
|
||||
{
|
||||
let ids = observations
|
||||
.iter()
|
||||
.map(|item| item.key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let statement = if dimension == DiagnosticDimension::Account {
|
||||
"select requested.id, historical.provider_kind, historical.provider_account_plan_type_snapshot as plan_type, coalesce(historical.provider_account_email_snapshot,historical.provider_account_name_snapshot,requested.id) as name from unnest($1::text[]) requested(id) left join lateral (select provider_kind,provider_account_plan_type_snapshot,provider_account_email_snapshot,provider_account_name_snapshot from model_request_observations mr where mr.provider_account_ref=requested.id order by mr.started_at desc,mr.id desc limit 1) historical on true"
|
||||
} else {
|
||||
"select requested.id, null::text as provider_kind, null::text as plan_type,coalesce(historical.client_api_key_name_snapshot,requested.id) as name from unnest($1::text[]) requested(id) left join lateral (select client_api_key_name_snapshot from model_request_observations mr where mr.client_api_key_ref=requested.id order by mr.started_at desc,mr.id desc limit 1) historical on true"
|
||||
};
|
||||
let labels = sqlx::query(statement)
|
||||
.bind(ids)
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read historical analytics labels", error))?;
|
||||
for row in labels {
|
||||
let id: String = get(&row, "id")?;
|
||||
if let Some(item) = observations.iter_mut().find(|item| item.key == id) {
|
||||
item.name = get(&row, "name")?;
|
||||
item.account_provider_kind = get(&row, "provider_kind")?;
|
||||
item.account_plan_type = get(&row, "plan_type")?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(DiagnosticsObservation {
|
||||
total_request_count,
|
||||
items: observations,
|
||||
analytics: self.metadata(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn diagnostic_top_keys(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
limit: u16,
|
||||
) -> StoreResult<(u64, Vec<String>)> {
|
||||
filter.validate()?;
|
||||
self.require_visible(Some(range)).await?;
|
||||
let (collection, column, reference) = if dimension == DiagnosticDimension::Account {
|
||||
(
|
||||
"account",
|
||||
"provider_account_ref",
|
||||
&filter.provider_account_ref,
|
||||
)
|
||||
} else {
|
||||
("key", "client_api_key_ref", &filter.client_api_key_ref)
|
||||
};
|
||||
let spans = self.uncovered_spans(range).await?;
|
||||
let (starts, ends): (Vec<_>, Vec<_>) = spans.into_iter().unzip();
|
||||
let (hours, quarters) = self.summary_buckets(range, true).await?;
|
||||
let mut day_hours = BTreeMap::<DateTime<Utc>, usize>::new();
|
||||
for hour in &hours {
|
||||
*day_hours.entry(bucket_for(*hour, 86400)?).or_default() += 1;
|
||||
}
|
||||
let days = day_hours
|
||||
.into_iter()
|
||||
.filter_map(|(day, count)| (count == 24).then_some(day))
|
||||
.collect::<Vec<_>>();
|
||||
let hours = hours
|
||||
.into_iter()
|
||||
.filter(|hour| {
|
||||
!days
|
||||
.iter()
|
||||
.any(|day| *hour >= *day && *hour < *day + TimeDelta::days(1))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut query = QueryBuilder::<Postgres>::new("with edge_facts as materialized (");
|
||||
if starts.is_empty() {
|
||||
query.push("select null::text as key where false");
|
||||
} else {
|
||||
query.push("select edge.key from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ")
|
||||
.push(format!("coalesce(mr.{column},'') as key from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at"));
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_analytics_filter(&mut query, filter, "mr");
|
||||
push_diagnostic_dimension_filter(&mut query, dimension);
|
||||
query.push(" limit 100001) edge limit 100001");
|
||||
}
|
||||
query.push("), contributions as (select key,count(*)::bigint as request_count from edge_facts group by key");
|
||||
for (table, column, seconds, buckets) in [
|
||||
("request_analytics_daily_counts", "dimension", 86400, days),
|
||||
("request_analytics_hourly", "dimensions[1]", 3600, hours),
|
||||
("request_analytics_15m", "dimensions[1]", 900, quarters),
|
||||
] {
|
||||
if buckets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// 范围扫描可并行合并计数,额外桶集合排除 dirty 与首尾不完整桶
|
||||
query.push(format!(" union all select {column} as key,sum(request_count)::bigint as request_count from {table} a where collection in ('account','key') and collection="))
|
||||
.push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION)
|
||||
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION);
|
||||
push_summary_buckets(&mut query, &buckets, seconds);
|
||||
if let Some(outcome) = &filter.outcome {
|
||||
query.push(" and outcome=").push_bind(outcome);
|
||||
}
|
||||
if let Some(reference) = reference {
|
||||
query.push(format!(" and {column}=")).push_bind(reference);
|
||||
}
|
||||
query.push(format!(" group by {column}"));
|
||||
}
|
||||
query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count,(select count(*) from edge_facts)::bigint as edge_count from grouped order by request_count desc,");
|
||||
if dimension == DiagnosticDimension::Account {
|
||||
query.push("coalesce(nullif(key,''),'unrouted')");
|
||||
} else {
|
||||
query.push("key");
|
||||
}
|
||||
query.push(" limit ").push_bind(i64::from(limit));
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("rank full-range analytics groups", error))?;
|
||||
if rows.first().is_some_and(|row| {
|
||||
row.try_get::<i64, _>("edge_count")
|
||||
.is_ok_and(|count| count > MAX_FACT_ROWS as i64)
|
||||
}) {
|
||||
return Err(unavailable("analytics fact row budget exceeded"));
|
||||
}
|
||||
let total = rows
|
||||
.first()
|
||||
.map(|row| unsigned(row, "total_request_count"))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let keys = rows
|
||||
.iter()
|
||||
.map(|row| get(row, "key"))
|
||||
.collect::<StoreResult<Vec<String>>>()?;
|
||||
Ok((total, keys))
|
||||
}
|
||||
|
||||
async fn summary_buckets(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
use_hourly: bool,
|
||||
) -> StoreResult<(Vec<DateTime<Utc>>, Vec<DateTime<Utc>>)> {
|
||||
// 先按时间确定互斥的完整桶,空的十五分钟集合不读取汇总表
|
||||
let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start limit 100001")
|
||||
.bind(range.start).bind(range.end).bind(use_hourly)
|
||||
.fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("select disjoint analytics buckets",error))?;
|
||||
if rows.len() > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics time bucket budget exceeded"));
|
||||
}
|
||||
let mut hours = Vec::new();
|
||||
let mut quarters = Vec::new();
|
||||
for row in rows {
|
||||
if let Some(hour) = get::<Option<DateTime<Utc>>>(&row, "hour_start")? {
|
||||
if hours.last() != Some(&hour) {
|
||||
hours.push(hour);
|
||||
}
|
||||
} else {
|
||||
quarters.push(get(&row, "bucket_start")?);
|
||||
}
|
||||
}
|
||||
Ok((hours, quarters))
|
||||
}
|
||||
|
||||
async fn uncovered_spans(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<Vec<(DateTime<Utc>, DateTime<Utc>)>> {
|
||||
let first = bucket_for(range.start, 900)?;
|
||||
let complete_start = if first == range.start {
|
||||
first
|
||||
} else {
|
||||
first + TimeDelta::minutes(15)
|
||||
};
|
||||
let complete_end = bucket_for(range.end, 900)?;
|
||||
let mut spans = Vec::new();
|
||||
if range.start < complete_start {
|
||||
spans.push((range.start, complete_start.min(range.end)));
|
||||
}
|
||||
let right_start = complete_end.max(complete_start).max(range.start);
|
||||
if right_start < range.end {
|
||||
spans.push((right_start, range.end));
|
||||
}
|
||||
if complete_start < complete_end {
|
||||
let dirty:Vec<DateTime<Utc>>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start limit 100001")
|
||||
.bind(complete_start).bind(complete_end).fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("plan bounded analytics fact spans",error))?;
|
||||
if dirty.len() > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics dirty interval budget exceeded"));
|
||||
}
|
||||
spans.extend(
|
||||
dirty
|
||||
.into_iter()
|
||||
.map(|start| (start, start + TimeDelta::minutes(15))),
|
||||
);
|
||||
}
|
||||
spans.sort_unstable();
|
||||
Ok(spans)
|
||||
}
|
||||
|
||||
async fn load(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
granularity: Option<ObservationGranularity>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
read_distributions: bool,
|
||||
) -> StoreResult<BTreeMap<(DateTime<Utc>, String), MetricState>> {
|
||||
self.load_selected(
|
||||
range,
|
||||
filter,
|
||||
dimension,
|
||||
granularity,
|
||||
timezone,
|
||||
ReadSelection {
|
||||
distributions: read_distributions,
|
||||
costs: true,
|
||||
keys: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_selected(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
granularity: Option<ObservationGranularity>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
selection: ReadSelection<'_>,
|
||||
) -> StoreResult<BTreeMap<(DateTime<Utc>, String), MetricState>> {
|
||||
let ReadSelection {
|
||||
distributions: read_distributions,
|
||||
costs: read_costs,
|
||||
keys: selected_keys,
|
||||
} = selection;
|
||||
filter.validate()?;
|
||||
self.require_visible(Some(range)).await?;
|
||||
let selected = collection_for(filter, dimension);
|
||||
let boundaries = if granularity == Some(ObservationGranularity::Day) {
|
||||
Some(calendar_buckets(range, timezone)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// 非整桶日界和短区间不能用完整桶替代,统一在窄事实上执行精确口径
|
||||
let selected = if boundaries.as_ref().is_some_and(|values| {
|
||||
values
|
||||
.iter()
|
||||
.any(|value| value.timestamp().rem_euclid(900) != 0)
|
||||
}) {
|
||||
None
|
||||
} else {
|
||||
selected
|
||||
};
|
||||
let mut result = BTreeMap::<(DateTime<Utc>, String), MetricState>::new();
|
||||
if let Some(collection) = selected {
|
||||
self.approximate |= read_distributions;
|
||||
let use_hourly = granularity != Some(ObservationGranularity::FifteenMinutes)
|
||||
&& boundaries.as_ref().is_none_or(|values| {
|
||||
values
|
||||
.iter()
|
||||
.all(|value| value.timestamp().rem_euclid(3600) == 0)
|
||||
});
|
||||
let (hours, quarters) = self.summary_buckets(range, use_hourly).await?;
|
||||
for (table, seconds, buckets) in [
|
||||
("request_analytics_hourly", 3600, hours),
|
||||
("request_analytics_15m", 900, quarters),
|
||||
] {
|
||||
if buckets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (dimension_filter, either_filter, either_positions) = match collection {
|
||||
"provider_model" => {
|
||||
(filter.provider_kind.clone(), filter.model.clone(), (2, 3))
|
||||
}
|
||||
"account" => (filter.provider_account_ref.clone(), None, (1, 2)),
|
||||
"key" => (filter.client_api_key_ref.clone(), None, (1, 2)),
|
||||
"operation" => (filter.operation.clone(), None, (1, 2)),
|
||||
"transport" => (None, filter.transport.clone(), (1, 2)),
|
||||
"status" => (
|
||||
None,
|
||||
filter.status_code.map(|value| value.to_string()),
|
||||
(1, 2),
|
||||
),
|
||||
_ => (None, None, (1, 2)),
|
||||
};
|
||||
let distributions = if read_distributions {
|
||||
"distributions"
|
||||
} else {
|
||||
"'{}'::jsonb as distributions"
|
||||
};
|
||||
let costs = if read_costs {
|
||||
"costs"
|
||||
} else {
|
||||
"'{}'::jsonb as costs"
|
||||
};
|
||||
let provider_counts = dimension == Some(DiagnosticDimension::Provider)
|
||||
&& granularity.is_none()
|
||||
&& !read_distributions;
|
||||
let mut query = QueryBuilder::<Postgres>::new(if provider_counts {
|
||||
"select min(bucket_start) as bucket_start, array[dimensions[1]] as dimensions, ''::text as outcome, jsonb_build_object('request_count',sum((counts->>'request_count')::bigint),'attempt_count',sum((counts->>'attempt_count')::bigint),'failure_count',sum(coalesce((counts->>'failure_count')::bigint,0)),'total_tokens',sum(coalesce((counts->>'total_tokens')::bigint,0))) as counts, '{}'::jsonb as costs, '{}'::jsonb as distributions from ("
|
||||
} else {
|
||||
""
|
||||
});
|
||||
query.push(format!("select bucket_start, dimensions, outcome, counts, {costs}, {distributions} from {table} a where collection="))
|
||||
.push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION)
|
||||
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION);
|
||||
push_summary_buckets(&mut query, &buckets, seconds);
|
||||
// 可选筛选仅在有值时生成谓词,保留维度与时间索引的选择空间
|
||||
if let Some(outcome) = &filter.outcome {
|
||||
query.push(" and outcome=").push_bind(outcome);
|
||||
}
|
||||
if let Some(value) = dimension_filter {
|
||||
query.push(" and dimensions[1]=").push_bind(value);
|
||||
}
|
||||
if let Some(value) = either_filter {
|
||||
query
|
||||
.push(format!(" and (dimensions[{}]=", either_positions.0))
|
||||
.push_bind(value.clone())
|
||||
.push(format!(" or dimensions[{}]=", either_positions.1))
|
||||
.push_bind(value)
|
||||
.push(")");
|
||||
}
|
||||
if let Some(keys) = selected_keys {
|
||||
query
|
||||
.push(" and dimensions[1]=any(")
|
||||
.push_bind(keys)
|
||||
.push(")");
|
||||
}
|
||||
if provider_counts {
|
||||
query.push(") matching group by dimensions[1]");
|
||||
}
|
||||
query.push(" limit 100001");
|
||||
let mut rows = query.build().fetch(&mut *self.tx);
|
||||
let mut count = 0;
|
||||
while let Some(row) = rows
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read analytics summaries", error))?
|
||||
{
|
||||
count += 1;
|
||||
if count > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics aggregate row budget exceeded"));
|
||||
}
|
||||
let dimensions: Vec<String> = get(&row, "dimensions")?;
|
||||
let name = dimension_name(collection, &dimensions, dimension);
|
||||
let Some(name) = name else {
|
||||
continue;
|
||||
};
|
||||
let bucket = output_bucket(
|
||||
get(&row, "bucket_start")?,
|
||||
range,
|
||||
granularity,
|
||||
boundaries.as_deref(),
|
||||
)?;
|
||||
let state = MetricState::decode(&row)?;
|
||||
result.entry((bucket, name)).or_default().merge(&state)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
let spans = if selected.is_some() {
|
||||
self.uncovered_spans(range).await?
|
||||
} else {
|
||||
vec![(range.start, range.end)]
|
||||
};
|
||||
if spans.is_empty() {
|
||||
return Ok(result);
|
||||
}
|
||||
// 单段直接绑定边界,让优化器看见真实跨度,避免把三十天误估成小范围 BitmapAnd
|
||||
let single_span = spans.len() == 1;
|
||||
let mut query = if single_span {
|
||||
let mut query = QueryBuilder::<Postgres>::new(fact_select());
|
||||
query
|
||||
.push(" where mr.started_at >= ")
|
||||
.push_bind(spans[0].0)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(spans[0].1);
|
||||
query
|
||||
} else {
|
||||
let (starts, ends): (Vec<_>, Vec<_>) = spans.into_iter().unzip();
|
||||
let mut query = QueryBuilder::<Postgres>::new("select fact.* from unnest(");
|
||||
query
|
||||
.push_bind(starts)
|
||||
.push("::timestamptz[],")
|
||||
.push_bind(ends)
|
||||
.push("::timestamptz[]) span(start_at,end_at) cross join lateral (")
|
||||
.push(fact_select())
|
||||
.push(" where mr.started_at >= span.start_at and mr.started_at < span.end_at");
|
||||
query
|
||||
};
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_analytics_filter(&mut query, filter, "mr");
|
||||
if let Some(dimension) = dimension {
|
||||
push_diagnostic_dimension_filter(&mut query, dimension);
|
||||
}
|
||||
if let Some(keys) = selected_keys {
|
||||
let column = if dimension == Some(DiagnosticDimension::Account) {
|
||||
"provider_account_ref"
|
||||
} else {
|
||||
"client_api_key_ref"
|
||||
};
|
||||
query
|
||||
.push(format!(" and (mr.{column}=any("))
|
||||
.push_bind(keys)
|
||||
.push(")");
|
||||
if keys.iter().any(String::is_empty) {
|
||||
query.push(format!(" or mr.{column} is null"));
|
||||
}
|
||||
query.push(")");
|
||||
}
|
||||
query.push(" limit 100001");
|
||||
if !single_span {
|
||||
query.push(") fact limit 100001");
|
||||
}
|
||||
let mut rows = query.build().fetch(&mut *self.tx);
|
||||
let mut count = 0;
|
||||
while let Some(row) = rows
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read bounded analytics facts", error))?
|
||||
{
|
||||
count += 1;
|
||||
if count > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics fact row budget exceeded"));
|
||||
}
|
||||
let bucket = output_bucket(
|
||||
get(&row, "started_at")?,
|
||||
range,
|
||||
granularity,
|
||||
boundaries.as_deref(),
|
||||
)?;
|
||||
let name = fact_dimension(&row, dimension)?;
|
||||
let state = result.entry((bucket, name)).or_insert_with(|| {
|
||||
if selected.is_some() {
|
||||
MetricState::default()
|
||||
} else {
|
||||
MetricState::exact()
|
||||
}
|
||||
});
|
||||
state.add_row(&row)?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
fn push_summary_buckets(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
buckets: &[DateTime<Utc>],
|
||||
seconds: i64,
|
||||
) {
|
||||
let (Some(first), Some(last)) = (buckets.first(), buckets.last()) else {
|
||||
query.push(" and false");
|
||||
return;
|
||||
};
|
||||
query
|
||||
.push(" and bucket_start >= ")
|
||||
.push_bind(*first)
|
||||
.push(" and bucket_start <= ")
|
||||
.push_bind(*last);
|
||||
// 连续干净区间直接范围扫描;只有被 dirty 桶分割时才追加集合过滤
|
||||
if buckets
|
||||
.windows(2)
|
||||
.any(|pair| pair[1] - pair[0] != TimeDelta::seconds(seconds))
|
||||
{
|
||||
query
|
||||
.push(" and bucket_start=any(")
|
||||
.push_bind(buckets.to_vec())
|
||||
.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_states(states: impl Iterator<Item = MetricState>) -> StoreResult<MetricState> {
|
||||
let mut result: Option<MetricState> = None;
|
||||
for state in states {
|
||||
if let Some(total) = &mut result {
|
||||
total.merge(&state)?;
|
||||
} else {
|
||||
result = Some(state);
|
||||
}
|
||||
}
|
||||
Ok(result.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn output_bucket(
|
||||
time: DateTime<Utc>,
|
||||
range: ObservabilityRange,
|
||||
granularity: Option<ObservationGranularity>,
|
||||
boundaries: Option<&[DateTime<Utc>]>,
|
||||
) -> StoreResult<DateTime<Utc>> {
|
||||
match granularity {
|
||||
None => Ok(range.start),
|
||||
Some(ObservationGranularity::Day) => boundaries
|
||||
.and_then(|values| {
|
||||
values
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|boundary| **boundary <= time)
|
||||
.copied()
|
||||
})
|
||||
.ok_or_else(|| invalid("missing analytics calendar boundary")),
|
||||
Some(granularity) => bucket_for(time, granularity.seconds()),
|
||||
}
|
||||
}
|
||||
|
||||
fn collection_for(
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> Option<&'static str> {
|
||||
if filter.request_id.is_some()
|
||||
|| filter.attempt_index.is_some()
|
||||
|| filter.response_id.is_some()
|
||||
|| filter.upstream_request_id.is_some()
|
||||
|| filter.search.is_some()
|
||||
|| !filter.routing_group_refs.is_empty()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut required = BTreeSet::new();
|
||||
if filter.provider_kind.is_some() || filter.model.is_some() {
|
||||
required.insert("provider_model");
|
||||
}
|
||||
if filter.provider_account_ref.is_some() {
|
||||
required.insert("account");
|
||||
}
|
||||
if filter.client_api_key_ref.is_some() {
|
||||
required.insert("key");
|
||||
}
|
||||
if filter.operation.is_some() {
|
||||
required.insert("operation");
|
||||
}
|
||||
if filter.transport.is_some() {
|
||||
required.insert("transport");
|
||||
}
|
||||
if filter.status_code.is_some() {
|
||||
required.insert("status");
|
||||
}
|
||||
if let Some(dimension) = dimension {
|
||||
required.insert(match dimension {
|
||||
DiagnosticDimension::Provider | DiagnosticDimension::Model => "provider_model",
|
||||
DiagnosticDimension::Account => "account",
|
||||
DiagnosticDimension::ApiKey => "key",
|
||||
DiagnosticDimension::Transport => "transport",
|
||||
DiagnosticDimension::Failure => "failure",
|
||||
DiagnosticDimension::Status => "status",
|
||||
});
|
||||
}
|
||||
if required.len() > 1 {
|
||||
None
|
||||
} else {
|
||||
Some(required.first().copied().unwrap_or("global"))
|
||||
}
|
||||
}
|
||||
|
||||
fn dimension_name(
|
||||
collection: &str,
|
||||
values: &[String],
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> Option<String> {
|
||||
let Some(dimension) = dimension else {
|
||||
return Some(String::new());
|
||||
};
|
||||
let value = |index| values.get(index).cloned().unwrap_or_default();
|
||||
let coalesce = |preferred: usize, fallback: usize| {
|
||||
let v = value(preferred);
|
||||
if v.is_empty() { value(fallback) } else { v }
|
||||
};
|
||||
let name = match dimension {
|
||||
DiagnosticDimension::Provider if collection == "provider_model" => {
|
||||
let v = value(0);
|
||||
if v.is_empty() {
|
||||
"unrouted".to_owned()
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
DiagnosticDimension::Model => coalesce(2, 1),
|
||||
DiagnosticDimension::Account => {
|
||||
let v = value(0);
|
||||
if v.is_empty() {
|
||||
"unrouted".to_owned()
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
DiagnosticDimension::Transport => {
|
||||
let v = coalesce(1, 0);
|
||||
if v.is_empty() {
|
||||
"unknown".to_owned()
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
DiagnosticDimension::Status => coalesce(1, 0),
|
||||
_ => value(0),
|
||||
};
|
||||
if name.is_empty()
|
||||
&& matches!(
|
||||
dimension,
|
||||
DiagnosticDimension::Model | DiagnosticDimension::Status | DiagnosticDimension::Failure
|
||||
)
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(name)
|
||||
}
|
||||
}
|
||||
|
||||
fn fact_dimension(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> StoreResult<String> {
|
||||
let Some(dimension) = dimension else {
|
||||
return Ok(String::new());
|
||||
};
|
||||
let collection = collection_for(&UsageRecordFilter::default(), Some(dimension))
|
||||
.ok_or_else(|| invalid("missing analytics dimension"))?;
|
||||
let dimensions = worker::collections(row)?
|
||||
.into_iter()
|
||||
.find(|(name, _)| *name == collection)
|
||||
.map(|(_, values)| values)
|
||||
.ok_or_else(|| invalid("missing analytics dimension"))?;
|
||||
dimension_name(collection, &dimensions, Some(dimension))
|
||||
.ok_or_else(|| invalid("unexpected empty analytics dimension"))
|
||||
}
|
||||
|
||||
pub(crate) fn push_analytics_filter(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
filter: &UsageRecordFilter,
|
||||
alias: &str,
|
||||
) {
|
||||
let mut typed = filter.clone();
|
||||
typed.search = None;
|
||||
push_usage_filter(query, &typed, alias);
|
||||
if filter.search.is_some() {
|
||||
query.push(format!(" and exists(select 1 from model_request_observations searched where searched.id={alias}.id"));
|
||||
let text = UsageRecordFilter {
|
||||
search: filter.search.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
push_usage_filter(query, &text, "searched");
|
||||
query.push(")");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//! 按持久主键游标清理过期统计版本,保留重建期间的源请求队列
|
||||
|
||||
use super::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(tag = "kind", content = "key", rename_all = "snake_case")]
|
||||
enum ResetCursor {
|
||||
Fact {
|
||||
id: String,
|
||||
},
|
||||
Rollup {
|
||||
bucket_start: DateTime<Utc>,
|
||||
collection: String,
|
||||
dimensions: Vec<String>,
|
||||
outcome: String,
|
||||
},
|
||||
Bucket {
|
||||
bucket_start: DateTime<Utc>,
|
||||
},
|
||||
}
|
||||
|
||||
pub(super) async fn reset_obsolete_batch(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
batch_size: u32,
|
||||
) -> StoreResult<bool> {
|
||||
let state = sqlx::query("select schema_version,histogram_version,reset_phase,reset_cursor from request_analytics_build_state where singleton")
|
||||
.fetch_one(&mut **tx).await.map_err(|error|postgres_unavailable("read analytics build version",error))?;
|
||||
let mut phase: i16 = get(&state, "reset_phase")?;
|
||||
if phase == 0 {
|
||||
if get::<i32>(&state, "schema_version")? == SCHEMA_VERSION
|
||||
&& get::<i32>(&state, "histogram_version")? == HISTOGRAM_VERSION
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
phase = 1;
|
||||
sqlx::query("update request_analytics_build_state set reset_phase=1,reset_cursor=null,backfill_complete=false,coverage_start=null,coverage_end=null,generation=generation+1,updated_at=clock_timestamp() where singleton")
|
||||
.execute(&mut **tx).await.map_err(|error|postgres_unavailable("begin obsolete analytics reset",error))?;
|
||||
}
|
||||
let (table, keys, kind) = match phase {
|
||||
1 => ("request_analytics_facts", "id", "fact"),
|
||||
2 => (
|
||||
"request_analytics_15m",
|
||||
"bucket_start,collection,dimensions,outcome",
|
||||
"rollup",
|
||||
),
|
||||
3 => (
|
||||
"request_analytics_hourly",
|
||||
"bucket_start,collection,dimensions,outcome",
|
||||
"rollup",
|
||||
),
|
||||
4 => (
|
||||
"request_analytics_pending_buckets",
|
||||
"bucket_start",
|
||||
"bucket",
|
||||
),
|
||||
_ => return Err(invalid("invalid analytics reset phase")),
|
||||
};
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"with batch as materialized (select ctid,{keys} from {table}"
|
||||
));
|
||||
let cursor = get::<Option<sqlx::types::Json<ResetCursor>>>(&state, "reset_cursor")?;
|
||||
// LIMIT 只限制删除数量,主键下界才避免后续批次反复扫描已经清空的堆页
|
||||
match cursor.map(|cursor| cursor.0) {
|
||||
None => {}
|
||||
Some(ResetCursor::Fact { id }) if phase == 1 => {
|
||||
query.push(" where id > ").push_bind(id);
|
||||
}
|
||||
Some(ResetCursor::Rollup {
|
||||
bucket_start,
|
||||
collection,
|
||||
dimensions,
|
||||
outcome,
|
||||
}) if phase == 2 || phase == 3 => {
|
||||
query
|
||||
.push(" where (bucket_start,collection,dimensions,outcome) > (")
|
||||
.push_bind(bucket_start)
|
||||
.push(",")
|
||||
.push_bind(collection)
|
||||
.push(",")
|
||||
.push_bind(dimensions)
|
||||
.push(",")
|
||||
.push_bind(outcome)
|
||||
.push(")");
|
||||
}
|
||||
Some(ResetCursor::Bucket { bucket_start }) if phase == 4 => {
|
||||
query.push(" where bucket_start > ").push_bind(bucket_start);
|
||||
}
|
||||
Some(_) => return Err(invalid("analytics reset cursor does not match its phase")),
|
||||
}
|
||||
query
|
||||
.push(format!(" order by {keys} limit "))
|
||||
.push_bind(i64::from(batch_size));
|
||||
let descending = keys
|
||||
.split(',')
|
||||
.map(|key| format!("{key} desc"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
// CTID 只在本次事务内定位已选行,跨事务进度始终使用不会被表重写改变的主键
|
||||
query.push(format!(") , removed as (delete from {table} where ctid=any(array(select ctid from batch)) returning 1) select (select count(*) from removed) as removed,(select jsonb_build_object('kind','{kind}','key',to_jsonb(batch)-'ctid') from batch order by {descending} limit 1) as cursor"));
|
||||
let result = query
|
||||
.build()
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("reset obsolete analytics batch", error))?;
|
||||
let removed: i64 = get(&result, "removed")?;
|
||||
if removed < i64::from(batch_size) {
|
||||
if phase == 4 {
|
||||
// 所有旧派生表清空后才发布新版本,源请求队列始终保留
|
||||
sqlx::query("update request_analytics_build_state set schema_version=$1,histogram_version=$2,reset_phase=0,reset_cursor=null,backfill_cursor_started_at=null,backfill_cursor_id=null,backfill_complete=false,generation=generation+1,updated_at=clock_timestamp() where singleton")
|
||||
.bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).execute(&mut **tx).await.map_err(|error|postgres_unavailable("restart analytics backfill",error))?;
|
||||
} else {
|
||||
sqlx::query("update request_analytics_build_state set reset_phase=$1,reset_cursor=null,updated_at=clock_timestamp() where singleton")
|
||||
.bind(phase+1).execute(&mut **tx).await.map_err(|error|postgres_unavailable("advance analytics reset phase",error))?;
|
||||
}
|
||||
} else {
|
||||
let cursor = get::<Option<sqlx::types::Json<ResetCursor>>>(&result, "cursor")?
|
||||
.ok_or_else(|| invalid("missing analytics reset cursor"))?;
|
||||
sqlx::query("update request_analytics_build_state set reset_cursor=$1,updated_at=clock_timestamp() where singleton")
|
||||
.bind(cursor).execute(&mut **tx).await.map_err(|error|postgres_unavailable("save analytics reset cursor",error))?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
//! 有界增量投影、历史回填与时间桶覆盖重建
|
||||
|
||||
use super::buckets::{BucketProgress, pending_batch, rebuild_bucket_batch};
|
||||
use super::reset::reset_obsolete_batch;
|
||||
use super::*;
|
||||
use gateway_admin::ports::store::UsageBillingProjector;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AnalyticsMaintenance {
|
||||
pub projected_requests: u64,
|
||||
pub rebuilt_buckets: u64,
|
||||
pub advanced_buckets: u64,
|
||||
pub backfilled_requests: u64,
|
||||
pub reset_progress: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AnalyticsStatus {
|
||||
pub coverage_start: Option<DateTime<Utc>>,
|
||||
pub coverage_end: Option<DateTime<Utc>>,
|
||||
pub oldest_pending_at: Option<DateTime<Utc>>,
|
||||
pub building: bool,
|
||||
pub generation: i64,
|
||||
pub pending_requests: i64,
|
||||
pub pending_buckets: i64,
|
||||
pub schema_version: u32,
|
||||
pub histogram_version: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PgRequestAnalyticsRepository {
|
||||
pool: PgPool,
|
||||
query_budget: ObservabilityQueryBudget,
|
||||
billing_projector: Option<Arc<dyn UsageBillingProjector>>,
|
||||
}
|
||||
|
||||
impl PgRequestAnalyticsRepository {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool, query_budget: ObservabilityQueryBudget) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
query_budget,
|
||||
billing_projector: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_billing_projector(mut self, projector: Arc<dyn UsageBillingProjector>) -> Self {
|
||||
self.billing_projector = Some(projector);
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn status(&self) -> StoreResult<AnalyticsStatus> {
|
||||
self.query_budget
|
||||
.run("analytics status", async {
|
||||
let mut transaction = begin_snapshot(&self.pool).await?;
|
||||
let status = status(&mut transaction).await?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("complete analytics status", error))?;
|
||||
Ok(status)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn maintain(&self, batch_size: u32) -> StoreResult<AnalyticsMaintenance> {
|
||||
if batch_size == 0 || batch_size > 10_000 {
|
||||
return Err(invalid("analytics batch size must be between 1 and 10000"));
|
||||
}
|
||||
self.query_budget.run("maintain request analytics", async {
|
||||
// 数据库事务锁同时约束单副本内重复任务与过期 Worker,源写入不获取此锁
|
||||
let mut transaction = begin_snapshot(&self.pool).await?;
|
||||
let acquired: bool = sqlx::query_scalar("select pg_try_advisory_xact_lock(hashtextextended(current_schema()::text,382032))")
|
||||
.fetch_one(&mut *transaction).await.map_err(|error| postgres_unavailable("claim analytics maintenance", error))?;
|
||||
if !acquired { return Ok(AnalyticsMaintenance::default()); }
|
||||
if reset_obsolete_batch(&mut transaction, batch_size).await? {
|
||||
transaction.commit().await.map_err(|error| postgres_unavailable("save analytics reset progress", error))?;
|
||||
return Ok(AnalyticsMaintenance { reset_progress: true, ..Default::default() });
|
||||
}
|
||||
let mut result = AnalyticsMaintenance::default();
|
||||
let rows = sqlx::query("select request_id, change_version, oldest_pending_at, previous_bucket from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
|
||||
.bind(i64::from(batch_size)).fetch_all(&mut *transaction).await.map_err(|error| postgres_unavailable("load analytics request queue", error))?;
|
||||
let mut jobs = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")?, pending_at: get(&row,"oldest_pending_at")?, previous_bucket: get(&row,"previous_bucket")? });
|
||||
}
|
||||
self.project_batch(&mut transaction, &jobs).await?;
|
||||
result.projected_requests = jobs.len() as u64;
|
||||
result.backfilled_requests = self.backfill(&mut transaction, batch_size).await?;
|
||||
let ids = jobs.iter().map(|job|job.id.clone()).collect::<Vec<_>>();
|
||||
let versions = jobs.iter().map(|job|job.version).collect::<Vec<_>>();
|
||||
if result.projected_requests + result.backfilled_requests > 0 {
|
||||
publish_generation(&mut transaction, true).await?;
|
||||
}
|
||||
// 源删除会持有队列行再修改 generation,ACK 不能等待这些源事务形成反向锁依赖
|
||||
// 被跳过的版本保留在持久队列中重放,只有精确匹配且已取得行锁的版本才确认
|
||||
sqlx::query("with acknowledged as materialized (select pending.request_id,pending.change_version from request_analytics_pending_requests pending join unnest($1::text[],$2::bigint[]) job(id,version) on pending.request_id=job.id and pending.change_version=job.version for update of pending skip locked) delete from request_analytics_pending_requests pending using acknowledged where pending.request_id=acknowledged.request_id and pending.change_version=acknowledged.change_version")
|
||||
.bind(ids).bind(versions).execute(&mut *transaction).await.map_err(|error| postgres_unavailable("confirm analytics request versions", error))?;
|
||||
transaction.commit().await.map_err(|error| postgres_unavailable("publish analytics maintenance", error))?;
|
||||
// 每个阶段使用独立短事务,聚合只锁自己的队列,不把读放大传递到请求终结
|
||||
let mut transaction = begin_snapshot(&self.pool).await?;
|
||||
let acquired: bool = sqlx::query_scalar("select pg_try_advisory_xact_lock(hashtextextended(current_schema()::text,382032))")
|
||||
.fetch_one(&mut *transaction).await.map_err(|error| postgres_unavailable("claim analytics bucket maintenance", error))?;
|
||||
if !acquired { return Ok(result); }
|
||||
let reset_in_progress: bool = sqlx::query_scalar("select reset_phase <> 0 or schema_version <> $1 or histogram_version <> $2 from request_analytics_build_state where singleton")
|
||||
.bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).fetch_one(&mut *transaction).await.map_err(|error|postgres_unavailable("check analytics rebuild epoch",error))?;
|
||||
if reset_in_progress { return Ok(result); }
|
||||
for bucket in pending_batch(&mut transaction).await? {
|
||||
match rebuild_bucket_batch(&mut transaction,bucket,batch_size).await? {
|
||||
BucketProgress::Waiting => {},
|
||||
BucketProgress::Advanced => result.advanced_buckets += 1,
|
||||
BucketProgress::Completed(count) => { result.advanced_buckets += 1; result.rebuilt_buckets += count; },
|
||||
}
|
||||
}
|
||||
if result.advanced_buckets > 0 { publish_generation(&mut transaction,false).await?; }
|
||||
transaction.commit().await.map_err(|error| postgres_unavailable("publish analytics buckets",error))?;
|
||||
Ok(result)
|
||||
}).await
|
||||
}
|
||||
|
||||
async fn project_batch(
|
||||
&self,
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
jobs: &[ProjectionJob],
|
||||
) -> StoreResult<()> {
|
||||
if jobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let ids = jobs.iter().map(|job| job.id.clone()).collect::<Vec<_>>();
|
||||
let versions = jobs.iter().map(|job| job.version).collect::<Vec<_>>();
|
||||
let pending = jobs
|
||||
.iter()
|
||||
.map(|job| (job.id.as_str(), job.pending_at))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut buckets = BTreeMap::<DateTime<Utc>, DateTime<Utc>>::new();
|
||||
let mut affect = |id: &str, start: DateTime<Utc>| -> StoreResult<()> {
|
||||
let pending_at = pending
|
||||
.get(id)
|
||||
.copied()
|
||||
.ok_or_else(|| invalid("analytics job disappeared"))?;
|
||||
let bucket = bucket_for(start, 900)?;
|
||||
buckets
|
||||
.entry(bucket)
|
||||
.and_modify(|old| *old = (*old).min(pending_at))
|
||||
.or_insert(pending_at);
|
||||
Ok(())
|
||||
};
|
||||
let rows = sqlx::query(
|
||||
"delete from request_analytics_facts where id=any($1::text[]) returning id,started_at",
|
||||
)
|
||||
.bind(&ids)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("replace analytics facts", error))?;
|
||||
for row in rows {
|
||||
affect(&get::<String>(&row, "id")?, get(&row, "started_at")?)?;
|
||||
}
|
||||
let rows = sqlx::query(PROJECT_REQUEST)
|
||||
.bind(&ids)
|
||||
.bind(versions)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("project analytics facts", error))?;
|
||||
for row in rows {
|
||||
affect(&get::<String>(&row, "id")?, get(&row, "started_at")?)?;
|
||||
}
|
||||
for job in jobs {
|
||||
if let Some(bucket) = job.previous_bucket {
|
||||
affect(&job.id, bucket)?;
|
||||
}
|
||||
}
|
||||
self.project_billing_batch(tx, &ids).await?;
|
||||
if !buckets.is_empty() {
|
||||
let (starts, times): (Vec<_>, Vec<_>) = buckets.into_iter().unzip();
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select * from unnest($1::timestamptz[],$2::timestamptz[]) on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version'),oldest_pending_at=least(request_analytics_pending_buckets.oldest_pending_at,excluded.oldest_pending_at)")
|
||||
.bind(starts).bind(times).execute(&mut **tx).await.map_err(|error| postgres_unavailable("enqueue analytics buckets",error))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn backfill(
|
||||
&self,
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
batch_size: u32,
|
||||
) -> StoreResult<u64> {
|
||||
let state = sqlx::query("select backfill_complete, backfill_cursor_started_at, backfill_cursor_id from request_analytics_build_state where singleton")
|
||||
.fetch_one(&mut **tx).await.map_err(|error| postgres_unavailable("load analytics backfill cursor", error))?;
|
||||
if get::<bool>(&state, "backfill_complete")? {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select id,started_at,nextval('request_analytics_change_version') as version from model_requests",
|
||||
);
|
||||
// 首批与后续批次使用两种明确的计划,避免可空游标的 OR 使通用计划退化为全索引过滤
|
||||
if let Some(start) = get::<Option<DateTime<Utc>>>(&state, "backfill_cursor_started_at")? {
|
||||
let id = get::<Option<String>>(&state, "backfill_cursor_id")?
|
||||
.ok_or_else(|| invalid("incomplete analytics backfill cursor"))?;
|
||||
query
|
||||
.push(" where (started_at,id)<(")
|
||||
.push_bind(start)
|
||||
.push(",")
|
||||
.push_bind(id)
|
||||
.push(")");
|
||||
}
|
||||
query
|
||||
.push(" order by started_at desc,id desc limit ")
|
||||
.push_bind(i64::from(batch_size));
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read analytics backfill batch", error))?;
|
||||
let mut jobs = Vec::with_capacity(rows.len());
|
||||
for row in &rows {
|
||||
jobs.push(ProjectionJob {
|
||||
id: get(row, "id")?,
|
||||
version: get(row, "version")?,
|
||||
pending_at: Utc::now(),
|
||||
previous_bucket: None,
|
||||
});
|
||||
}
|
||||
self.project_batch(tx, &jobs).await?;
|
||||
if let Some(last) = rows.last() {
|
||||
sqlx::query("update request_analytics_build_state set backfill_cursor_started_at = $1, backfill_cursor_id = $2, coverage_start = $1 + interval '1 microsecond' where singleton")
|
||||
.bind(get::<DateTime<Utc>>(last,"started_at")?).bind(get::<String>(last,"id")?).execute(&mut **tx).await.map_err(|error| postgres_unavailable("save analytics backfill cursor", error))?;
|
||||
}
|
||||
if rows.len() < usize::try_from(batch_size).unwrap_or(usize::MAX) {
|
||||
sqlx::query("update request_analytics_build_state set backfill_complete = true, coverage_end = clock_timestamp(), coverage_start = (select started_at from request_analytics_facts order by started_at,id limit 1) where singleton")
|
||||
.execute(&mut **tx).await.map_err(|error| postgres_unavailable("complete analytics backfill", error))?;
|
||||
}
|
||||
Ok(rows.len() as u64)
|
||||
}
|
||||
|
||||
async fn project_billing_batch(
|
||||
&self,
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
ids: &[String],
|
||||
) -> StoreResult<()> {
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let rows=sqlx::query(sqlx::AssertSqlSafe(format!("select mr.id, mr.billing_snapshot_json, mr.started_at as bucket_start, mr.provider_kind, mr.upstream_model_id, mr.service_tier, mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.cache_write_input_tokens, mr.cost_amount::text as amount, mr.cost_currency from model_request_observations mr where mr.id=any($1::text[]) and {fact} and mr.cost_source='calculated' and mr.cost_amount is not null and mr.cost_currency is not null and nullif(mr.provider_kind,'') is not null and nullif(mr.upstream_model_id,'') is not null")))
|
||||
.bind(ids).fetch_all(&mut **tx).await.map_err(|error| postgres_unavailable("read analytics billing source",error))?;
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let projector = self
|
||||
.billing_projector
|
||||
.as_ref()
|
||||
.ok_or_else(|| invalid("analytics billing projector is not configured"))?;
|
||||
let mut request_ids = Vec::with_capacity(rows.len());
|
||||
let mut standard = Vec::with_capacity(rows.len());
|
||||
let mut no_cache = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let fact =
|
||||
admin_calculated_usage_billing_fact(calculated_usage_billing_fact_from_row(&row)?);
|
||||
let projection = projector
|
||||
.project(&fact)
|
||||
.map_err(|error| postgres_unavailable("project analytics billing", error))?;
|
||||
request_ids.push(get::<String>(&row, "id")?);
|
||||
standard.push(projection.standard_amount.map(|value| value.to_string()));
|
||||
no_cache.push(projection.no_cache_amount.map(|value| value.to_string()));
|
||||
}
|
||||
sqlx::query("update request_analytics_facts facts set standard_cost_amount=projection.standard::numeric,no_cache_cost_amount=projection.no_cache::numeric from unnest($1::text[],$2::text[],$3::text[]) projection(id,standard,no_cache) where facts.id=projection.id")
|
||||
.bind(request_ids).bind(standard).bind(no_cache).execute(&mut **tx).await.map_err(|error| postgres_unavailable("persist analytics billing projections",error))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn status(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
) -> StoreResult<AnalyticsStatus> {
|
||||
read_status(tx, true).await
|
||||
}
|
||||
|
||||
pub(super) async fn query_status(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
) -> StoreResult<AnalyticsStatus> {
|
||||
read_status(tx, false).await
|
||||
}
|
||||
|
||||
async fn read_status(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
count_pending: bool,
|
||||
) -> StoreResult<AnalyticsStatus> {
|
||||
// 在线元信息仅使用最旧待处理时间;精确队列长度留给后台运维状态查询
|
||||
let row = sqlx::query("select *, case when $1 then (select count(*) from request_analytics_pending_requests) else 0 end as pending_requests, case when $1 then (select count(*) from request_analytics_pending_buckets) else 0 end as pending_buckets, (select min(oldest_pending_at) from (select min(oldest_pending_at) as oldest_pending_at from request_analytics_pending_requests union all select min(oldest_pending_at) from request_analytics_pending_buckets) pending) as oldest_pending_at from request_analytics_build_state where singleton")
|
||||
.bind(count_pending).fetch_one(&mut **tx).await.map_err(|error| postgres_unavailable("read analytics status", error))?;
|
||||
Ok(AnalyticsStatus {
|
||||
coverage_start: get(&row, "coverage_start")?,
|
||||
coverage_end: get(&row, "coverage_end")?,
|
||||
oldest_pending_at: get(&row, "oldest_pending_at")?,
|
||||
building: !get::<bool>(&row, "backfill_complete")?
|
||||
|| get::<i16>(&row, "reset_phase")? != 0
|
||||
|| get::<i32>(&row, "schema_version")? != SCHEMA_VERSION
|
||||
|| get::<i32>(&row, "histogram_version")? != HISTOGRAM_VERSION,
|
||||
generation: get(&row, "generation")?,
|
||||
pending_requests: get(&row, "pending_requests")?,
|
||||
pending_buckets: get(&row, "pending_buckets")?,
|
||||
schema_version: u32::try_from(get::<i32>(&row, "schema_version")?)
|
||||
.map_err(|_| invalid("invalid analytics version"))?,
|
||||
histogram_version: u32::try_from(get::<i32>(&row, "histogram_version")?)
|
||||
.map_err(|_| invalid("invalid histogram version"))?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn collections(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> StoreResult<Vec<(&'static str, Vec<String>)>> {
|
||||
let text =
|
||||
|name| -> StoreResult<String> { Ok(get::<Option<String>>(row, name)?.unwrap_or_default()) };
|
||||
let status = |name| -> StoreResult<String> {
|
||||
Ok(get::<Option<i32>>(row, name)?
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_default())
|
||||
};
|
||||
Ok(vec![
|
||||
("global", vec![]),
|
||||
(
|
||||
"provider_model",
|
||||
vec![
|
||||
text("provider_kind")?,
|
||||
text("requested_model_id")?,
|
||||
text("upstream_model_id")?,
|
||||
],
|
||||
),
|
||||
("account", vec![text("provider_account_ref")?]),
|
||||
("key", vec![text("client_api_key_ref")?]),
|
||||
("operation", vec![text("operation")?]),
|
||||
(
|
||||
"transport",
|
||||
vec![text("client_transport")?, text("upstream_transport")?],
|
||||
),
|
||||
(
|
||||
"status",
|
||||
vec![
|
||||
status("client_status_code")?,
|
||||
status("upstream_status_code")?,
|
||||
],
|
||||
),
|
||||
("failure", vec![text("error_kind")?]),
|
||||
])
|
||||
}
|
||||
|
||||
struct ProjectionJob {
|
||||
id: String,
|
||||
version: i64,
|
||||
pending_at: DateTime<Utc>,
|
||||
previous_bucket: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
async fn publish_generation(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
facts_changed: bool,
|
||||
) -> StoreResult<()> {
|
||||
sqlx::query("update request_analytics_build_state set generation=generation+1,updated_at=clock_timestamp(),coverage_end=clock_timestamp(),coverage_start=case when not backfill_complete then backfill_cursor_started_at + interval '1 microsecond' when $1 then (select started_at from request_analytics_facts order by started_at,id limit 1) else coverage_start end where singleton")
|
||||
.bind(facts_changed).execute(&mut **tx).await.map_err(|error|postgres_unavailable("advance analytics generation",error))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const PROJECT_REQUEST: &str = "insert into request_analytics_facts (id, started_at, completed_at, client_api_key_ref, provider_account_ref, routing_group_refs, operation, provider_kind, requested_model_id, upstream_model_id, outcome, client_status_code, upstream_status_code, client_transport, upstream_transport, attempt_count, client_response_id, upstream_response_id, upstream_request_id, recovered_at, downstream_committed_at, request_kind, image_generation_requested, image_generation_succeeded, error_kind, input_tokens, output_tokens, cached_input_tokens, cache_write_input_tokens, reasoning_output_tokens, image_input_tokens, image_output_tokens, total_tokens, cost_source, cost_currency, cost_amount, first_token_ms, latency_ms, admission_decision_ms, account_selection_wait_ms, upstream_response_ms, capacity_used_slots, capacity_total_slots, source_version, ops_failure_count, ops_rate_limited_count, ops_auth_failure_count, ops_provider_5xx_count) select mr.id, mr.started_at, mr.completed_at, mr.client_api_key_ref, mr.provider_account_ref, array(select distinct group_ref from unnest(mr.routing_group_refs) group_ref order by group_ref), mr.operation, mr.provider_kind, mr.requested_model_id, mr.upstream_model_id, mr.outcome, mr.client_status_code, mr.upstream_status_code, mr.client_transport, mr.upstream_transport, mr.attempt_count, mr.client_response_id, mr.upstream_response_id, mr.upstream_request_id, mr.recovered_at, mr.downstream_committed_at, mr.request_kind, mr.image_generation_requested, mr.image_generation_succeeded, mr.error_kind, mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.cache_write_input_tokens, mr.reasoning_output_tokens, mr.image_input_tokens, mr.image_output_tokens, mr.total_tokens, mr.cost_source, mr.cost_currency, mr.cost_amount, mr.first_token_ms, mr.latency_ms, mr.admission_decision_ms, mr.account_selection_wait_ms, mr.upstream_response_ms, mr.capacity_used_slots, mr.capacity_total_slots, job.version, ops.failure_count, ops.rate_limited_count, ops.auth_failure_count, ops.provider_5xx_count from unnest($1::text[],$2::bigint[]) job(id,version) join model_request_observations mr on mr.id=job.id cross join lateral (select count(*)::bigint as failure_count, count(*) filter(where failure_kind in ('rate_limited','quota_exhausted') or status_code=429)::bigint as rate_limited_count, count(*) filter(where failure_kind in ('authentication','authorization','invalid_credential') or status_code in (401,403))::bigint as auth_failure_count, count(*) filter(where status_code between 500 and 599)::bigint as provider_5xx_count from ops_events where model_request_id=mr.id) ops returning id, started_at";
|
||||
@@ -17,6 +17,7 @@ pub(crate) fn store_range(
|
||||
|
||||
pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter {
|
||||
UsageRecordFilter {
|
||||
routing_group_refs: filter.routing_group_refs,
|
||||
client_api_key_ref: filter.client_api_key_ref,
|
||||
request_id: filter.request_id,
|
||||
provider_account_ref: filter.provider_account_ref,
|
||||
@@ -63,6 +64,7 @@ pub(crate) fn admin_dashboard_observation(
|
||||
observation: DashboardObservation,
|
||||
) -> admin_observability::DashboardObservation {
|
||||
let DashboardObservation {
|
||||
analytics,
|
||||
range,
|
||||
totals,
|
||||
provider_accounts,
|
||||
@@ -72,6 +74,7 @@ pub(crate) fn admin_dashboard_observation(
|
||||
recent_requests,
|
||||
} = observation;
|
||||
admin_observability::DashboardObservation {
|
||||
analytics,
|
||||
range: admin_range(range),
|
||||
totals,
|
||||
provider_accounts,
|
||||
@@ -136,9 +139,9 @@ pub(crate) fn admin_dashboard_account_usage(
|
||||
success_count: usage.success_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
cached_input_tokens: usage.cached_input_tokens,
|
||||
cache_write_input_tokens: usage.cache_write_input_tokens,
|
||||
reasoning_output_tokens: usage.reasoning_output_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
image_request_count: usage.image_request_count,
|
||||
@@ -167,9 +170,9 @@ pub(crate) fn admin_account_model_usage(
|
||||
success_count: usage.success_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
cached_input_tokens: usage.cached_input_tokens,
|
||||
cache_write_input_tokens: usage.cache_write_input_tokens,
|
||||
reasoning_output_tokens: usage.reasoning_output_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
image_request_count: usage.image_request_count,
|
||||
@@ -207,8 +210,8 @@ pub(crate) fn admin_calculated_usage_billing_fact(
|
||||
service_tier: fact.service_tier,
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
cache_write_tokens: fact.cache_write_tokens,
|
||||
cached_input_tokens: fact.cached_input_tokens,
|
||||
cache_write_input_tokens: fact.cache_write_input_tokens,
|
||||
total: fact.total,
|
||||
}
|
||||
}
|
||||
@@ -241,6 +244,7 @@ pub(crate) fn request_outcome(outcome: &str) -> StoreResult<admin_observability:
|
||||
|
||||
pub(crate) fn admin_usage_overview(overview: UsageOverview) -> admin_observability::UsageOverview {
|
||||
admin_observability::UsageOverview {
|
||||
analytics: overview.analytics,
|
||||
range: admin_range(overview.range),
|
||||
requests: overview.requests,
|
||||
attempts: admin_attempt_metrics(overview.attempts),
|
||||
@@ -252,6 +256,7 @@ pub(crate) fn admin_diagnostics_observation(
|
||||
observation: DiagnosticsObservation,
|
||||
) -> admin_observability::DiagnosticsObservation {
|
||||
admin_observability::DiagnosticsObservation {
|
||||
analytics: observation.analytics,
|
||||
total_request_count: observation.total_request_count,
|
||||
items: observation
|
||||
.items
|
||||
@@ -318,9 +323,9 @@ pub(crate) fn usage_list_record_from_row(
|
||||
service_tier: get(row, "service_tier")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
total_tokens: optional_unsigned(row, "total_tokens")?,
|
||||
@@ -414,9 +419,9 @@ pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<
|
||||
retry_after_ms: optional_unsigned(row, "retry_after_ms")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
total_tokens: optional_unsigned(row, "total_tokens")?,
|
||||
@@ -536,8 +541,8 @@ pub(crate) fn calculated_usage_billing_fact_from_row(
|
||||
service_tier: get(row, "service_tier")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
total: cost_from_row(row)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -24,12 +24,14 @@ use super::{
|
||||
};
|
||||
|
||||
mod admin_adapter;
|
||||
mod analytics;
|
||||
mod mapping;
|
||||
mod model;
|
||||
mod queries;
|
||||
mod query_budget;
|
||||
|
||||
pub use admin_adapter::*;
|
||||
pub use analytics::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
|
||||
pub(crate) use mapping::*;
|
||||
pub use model::*;
|
||||
pub(crate) use queries::*;
|
||||
|
||||
@@ -39,6 +39,7 @@ pub(crate) fn observability_page_offset(
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct UsageRecordFilter {
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
@@ -75,6 +76,7 @@ impl UsageRecordFilter {
|
||||
] {
|
||||
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
|
||||
}
|
||||
validate_account_ids_if_present(&self.routing_group_refs)?;
|
||||
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
|
||||
if self
|
||||
.status_code
|
||||
@@ -103,6 +105,14 @@ pub struct UsageRecordQuery {
|
||||
pub use gateway_admin::model::observability::OpsErrorFilter;
|
||||
|
||||
pub(crate) fn validate_ops_error_filter(filter: &OpsErrorFilter) -> StoreResult<()> {
|
||||
if filter.routing_group_refs.len() > 32
|
||||
|| filter.routing_group_refs.iter().any(|value| {
|
||||
value.is_empty() || value.len() > 128 || value.chars().any(char::is_control)
|
||||
})
|
||||
{
|
||||
return Err(invalid("invalid routing group filter"));
|
||||
}
|
||||
|
||||
for (value, field) in [
|
||||
(
|
||||
filter.client_api_key_ref.as_deref(),
|
||||
@@ -198,8 +208,8 @@ pub struct CalculatedUsageBillingFact {
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub total: CurrencyCostTotal,
|
||||
}
|
||||
|
||||
@@ -219,9 +229,9 @@ pub struct ProviderAccountUsageObservation {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -241,9 +251,9 @@ pub struct ProviderAccountModelUsageObservation {
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub cached_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
pub reasoning_output_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
@@ -311,6 +321,7 @@ pub use gateway_admin::model::observability::DashboardTotals;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardObservation {
|
||||
pub analytics: admin_observability::AnalyticsMetadata,
|
||||
pub range: ObservabilityRange,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: ProviderAccountMetrics,
|
||||
@@ -335,6 +346,7 @@ pub use gateway_admin::model::observability::ProviderObservation;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageOverview {
|
||||
pub analytics: admin_observability::AnalyticsMetadata,
|
||||
pub range: ObservabilityRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
@@ -343,6 +355,7 @@ pub struct UsageOverview {
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DiagnosticsObservation {
|
||||
pub analytics: admin_observability::AnalyticsMetadata,
|
||||
pub total_request_count: u64,
|
||||
pub items: Vec<DiagnosticObservation>,
|
||||
}
|
||||
@@ -371,3 +384,10 @@ pub struct DiagnosticObservation {
|
||||
pub use gateway_admin::model::observability::OpsError as OpsErrorRecord;
|
||||
|
||||
pub use gateway_admin::model::observability::OpsErrorPage;
|
||||
|
||||
fn validate_account_ids_if_present(ids: &[String]) -> StoreResult<()> {
|
||||
if ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
validate_account_ids(ids)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) async fn provider_account_usage(
|
||||
pool: &PgPool,
|
||||
connection: &mut sqlx::PgConnection,
|
||||
query: ProviderAccountUsageQuery,
|
||||
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
@@ -13,18 +13,33 @@ pub(crate) async fn provider_account_usage(
|
||||
return Err(invalid("account usage limit must be between 1 and 200"));
|
||||
}
|
||||
|
||||
// 账号卡片先确定有限候选,避免为只展示二十个账号聚合整个历史范围
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"with matched as (
|
||||
"with candidate_accounts as materialized (select pa.* from provider_accounts pa",
|
||||
);
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
statement
|
||||
.push(" where pa.id=any(")
|
||||
.push_bind(account_ids.clone())
|
||||
.push("::text[])");
|
||||
} else {
|
||||
statement.push(" join lateral (select mr.started_at from request_analytics_facts mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start)
|
||||
.push(" and mr.started_at < ").push_bind(query.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
statement.push(" order by mr.started_at desc,mr.id desc limit 1) latest on true order by latest.started_at desc,pa.name,pa.id limit ").push_bind(i64::from(query.limit));
|
||||
}
|
||||
statement.push(
|
||||
"), matched as materialized (
|
||||
select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
|
||||
pa.plan_type, mr.id as request_id,
|
||||
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
|
||||
mr.cost_currency, mr.outcome, mr.input_tokens, mr.output_tokens,
|
||||
mr.cached_tokens, mr.cache_write_tokens, mr.reasoning_tokens,
|
||||
mr.cached_input_tokens, mr.cache_write_input_tokens, mr.reasoning_output_tokens,
|
||||
mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.image_generation_succeeded, mr.total_tokens, mr.cost_source,
|
||||
mr.cost_amount, mr.started_at
|
||||
from provider_accounts pa
|
||||
left join model_requests mr
|
||||
from candidate_accounts pa
|
||||
left join request_analytics_facts mr
|
||||
on mr.provider_account_ref = pa.id and mr.started_at >= ",
|
||||
);
|
||||
statement.push_bind(query.range.start);
|
||||
@@ -37,7 +52,7 @@ pub(crate) async fn provider_account_usage(
|
||||
statement.push("::text[])");
|
||||
}
|
||||
statement.push(
|
||||
"), aggregated as (
|
||||
" limit 100001), aggregated as (
|
||||
select id, provider_kind, authentication_kind, name, email, plan_type,
|
||||
model, cost_currency,
|
||||
grouping(model)::integer as model_grouping,
|
||||
@@ -47,9 +62,9 @@ pub(crate) async fn provider_account_usage(
|
||||
as success_count,
|
||||
sum(input_tokens)::bigint as input_tokens,
|
||||
sum(output_tokens)::bigint as output_tokens,
|
||||
sum(cached_tokens)::bigint as cached_tokens,
|
||||
sum(cache_write_tokens)::bigint as cache_write_tokens,
|
||||
sum(reasoning_tokens)::bigint as reasoning_tokens,
|
||||
sum(cached_input_tokens)::bigint as cached_input_tokens,
|
||||
sum(cache_write_input_tokens)::bigint as cache_write_input_tokens,
|
||||
sum(reasoning_output_tokens)::bigint as reasoning_output_tokens,
|
||||
sum(image_input_tokens)::bigint as image_input_tokens,
|
||||
sum(image_output_tokens)::bigint as image_output_tokens,
|
||||
count(request_id) filter (where image_generation_succeeded is true)::bigint
|
||||
@@ -82,7 +97,7 @@ pub(crate) async fn provider_account_usage(
|
||||
statement.push_bind(i64::from(query.limit));
|
||||
statement.push(
|
||||
")
|
||||
select aggregated.*
|
||||
select aggregated.*, (select count(*) from matched)::bigint as matched_count
|
||||
from aggregated
|
||||
join selected_accounts selected using (id)
|
||||
order by selected.sort_position, model_grouping desc, currency_grouping desc,
|
||||
@@ -90,9 +105,18 @@ pub(crate) async fn provider_account_usage(
|
||||
);
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.fetch_all(&mut *connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load provider account usage", source))?;
|
||||
if let Some(row) = rows.first()
|
||||
&& unsigned(row, "matched_count")? > 100_000
|
||||
{
|
||||
return Err(crate::StoreError::Unavailable {
|
||||
backend: crate::StoreBackend::PostgreSql,
|
||||
message: "account analytics fact row budget exceeded".to_owned(),
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
|
||||
let mut observations = Vec::with_capacity(usize::from(query.limit));
|
||||
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
|
||||
@@ -142,7 +166,7 @@ pub(crate) async fn provider_account_usage(
|
||||
.map(|item| item.account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let mut request_buckets = if let Some(range) = query.request_bucket_range {
|
||||
provider_account_request_buckets(pool, range, &account_ids).await?
|
||||
provider_account_request_buckets(connection, range, &account_ids).await?
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
@@ -168,7 +192,7 @@ pub(crate) async fn provider_account_usage(
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_account_request_buckets(
|
||||
pool: &PgPool,
|
||||
connection: &mut sqlx::PgConnection,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, Vec<ProviderAccountRequestBucket>>> {
|
||||
@@ -177,7 +201,7 @@ pub(crate) async fn provider_account_request_buckets(
|
||||
"select provider_account_ref,
|
||||
floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index,
|
||||
count(*)::bigint as request_count
|
||||
from model_requests mr
|
||||
from request_analytics_facts mr
|
||||
where mr.provider_account_ref = any($2::text[])
|
||||
and mr.started_at >= $1 and mr.started_at < $3
|
||||
and {completed_usage}
|
||||
@@ -188,7 +212,7 @@ pub(crate) async fn provider_account_request_buckets(
|
||||
.bind(range.start)
|
||||
.bind(account_ids)
|
||||
.bind(range.end)
|
||||
.fetch_all(pool)
|
||||
.fetch_all(&mut *connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load provider account request timeline", source))?;
|
||||
|
||||
@@ -233,9 +257,9 @@ fn provider_account_model_from_row(
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
image_request_count: unsigned(row, "image_request_count")?,
|
||||
@@ -261,9 +285,9 @@ pub(crate) fn provider_account_from_row(
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
image_request_count: unsigned(row, "image_request_count")?,
|
||||
|
||||
@@ -1,630 +1,7 @@
|
||||
//! Dashboard 与趋势查询族
|
||||
//! 时间桶补齐、日历边界与共享费用覆盖映射
|
||||
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) async fn request_metrics(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<RequestMetrics> {
|
||||
filter.validate()?;
|
||||
// 结果计数覆盖范围内全部请求(成功率/失败率的分母分子);用量、缓存、
|
||||
// 延迟与成本聚合仅统计用量事实(已完整交付客户端的成功响应)
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
count(*) filter (where outcome = 'failed')::bigint as failure_count,
|
||||
count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count,
|
||||
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
|
||||
count(*) filter (where client_status_code between 400 and 499)::bigint
|
||||
as caller_error_count,
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
|
||||
as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
|
||||
as reasoning_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
|
||||
as first_token_latency_sum,
|
||||
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
|
||||
count(latency_ms) filter (where {fact})::bigint as latency_count,
|
||||
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
|
||||
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
|
||||
count(*) filter (where {fact} and input_tokens is not null)::bigint
|
||||
as cache_eligible_request_count,
|
||||
count(*) filter (where {fact} and input_tokens is not null
|
||||
and cached_tokens > 0)::bigint
|
||||
as cache_hit_request_count,
|
||||
percentile_cont(0.50) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p50_ms,
|
||||
percentile_cont(0.95) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p95_ms,
|
||||
percentile_cont(0.99) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p99_ms,
|
||||
percentile_cont(0.50) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p50_ms,
|
||||
percentile_cont(0.95) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p95_ms,
|
||||
percentile_cont(0.99) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p99_ms,
|
||||
count(admission_decision_ms)::bigint as admission_decision_count,
|
||||
percentile_cont(0.50) within group (order by admission_decision_ms)
|
||||
as admission_decision_p50_ms,
|
||||
percentile_cont(0.95) within group (order by admission_decision_ms)
|
||||
as admission_decision_p95_ms,
|
||||
percentile_cont(0.99) within group (order by admission_decision_ms)
|
||||
as admission_decision_p99_ms,
|
||||
count(account_selection_wait_ms)::bigint as account_selection_wait_count,
|
||||
percentile_cont(0.50) within group (order by account_selection_wait_ms)
|
||||
as account_selection_wait_p50_ms,
|
||||
percentile_cont(0.95) within group (order by account_selection_wait_ms)
|
||||
as account_selection_wait_p95_ms,
|
||||
percentile_cont(0.99) within group (order by account_selection_wait_ms)
|
||||
as account_selection_wait_p99_ms,
|
||||
round((percentile_cont(0.10) within group (
|
||||
order by output_tokens::double precision * 1000.0
|
||||
/ nullif(upstream_response_ms, 0)
|
||||
) filter (where {fact} and output_tokens > 0
|
||||
and upstream_response_ms > 0)))::bigint as output_throughput_p10,
|
||||
round((percentile_cont(0.50) within group (
|
||||
order by output_tokens::double precision * 1000.0
|
||||
/ nullif(upstream_response_ms, 0)
|
||||
) filter (where {fact} and output_tokens > 0
|
||||
and upstream_response_ms > 0)))::bigint as output_throughput_p50,
|
||||
round((percentile_cont(0.90) within group (
|
||||
order by output_tokens::double precision * 1000.0
|
||||
/ nullif(upstream_response_ms, 0)
|
||||
) filter (where {fact} and output_tokens > 0
|
||||
and upstream_response_ms > 0)))::bigint as output_throughput_p90,
|
||||
count(capacity_total_slots)::bigint as capacity_sample_count,
|
||||
round(avg(capacity_used_slots::double precision
|
||||
/ nullif(capacity_total_slots, 0)) * 10000)::bigint
|
||||
as capacity_utilization_avg_basis_points,
|
||||
round(percentile_cont(0.95) within group (
|
||||
order by capacity_used_slots::double precision
|
||||
/ nullif(capacity_total_slots, 0)
|
||||
) * 10000)::bigint as capacity_utilization_p95_basis_points
|
||||
from model_request_observations mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load request metrics", source))?;
|
||||
request_metrics_from_row(&row)
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult<DashboardTotals> {
|
||||
// 卡片脚注的“总计”覆盖全部历史;只聚合页面实际展示的字段,避免重复计算
|
||||
// 当前区间指标所需的延迟分位数
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select count(*)::bigint as request_count,
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
sum(cost_amount) filter (where {fact} and cost_currency = 'USD')::text
|
||||
as billing_usd
|
||||
from model_request_observations mr
|
||||
where mr.recovered_at is null"
|
||||
));
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load dashboard totals", source))?;
|
||||
Ok(DashboardTotals {
|
||||
request_count: unsigned(&row, "request_count")?,
|
||||
input_tokens: unsigned(&row, "input_tokens")?,
|
||||
cached_tokens: unsigned(&row, "cached_tokens")?,
|
||||
total_tokens: unsigned(&row, "total_tokens")?,
|
||||
billing_usd: optional_decimal(&row, "billing_usd")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn request_metric_series(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
request_metric_series_inner(pool, range, filter, true, granularity, timezone).await
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_request_metric_series(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
request_metric_series_inner(pool, range, filter, false, granularity, timezone).await
|
||||
}
|
||||
|
||||
async fn request_metric_series_inner(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
load_costs: bool,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
filter.validate()?;
|
||||
// 与 request_metrics 同一契约:结果计数覆盖全部请求,用量/延迟/成本
|
||||
// 聚合仅统计用量事实
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new("select ");
|
||||
push_metric_bucket(&mut query, range, granularity, timezone)?;
|
||||
query.push(format!(
|
||||
" as bucket_start,
|
||||
count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
count(*) filter (where outcome = 'failed')::bigint as failure_count,
|
||||
count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count,
|
||||
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
|
||||
count(*) filter (where client_status_code between 400 and 499)::bigint
|
||||
as caller_error_count,
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
|
||||
as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
|
||||
as reasoning_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
|
||||
as first_token_latency_sum,
|
||||
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
|
||||
count(latency_ms) filter (where {fact})::bigint as latency_count,
|
||||
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
|
||||
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
|
||||
count(*) filter (where {fact} and input_tokens is not null)::bigint
|
||||
as cache_eligible_request_count,
|
||||
count(*) filter (where {fact} and input_tokens is not null
|
||||
and cached_tokens > 0)::bigint
|
||||
as cache_hit_request_count,
|
||||
percentile_cont(0.50) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p50_ms,
|
||||
percentile_cont(0.95) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p95_ms,
|
||||
percentile_cont(0.99) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p99_ms,
|
||||
percentile_cont(0.50) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p50_ms,
|
||||
percentile_cont(0.95) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p95_ms,
|
||||
percentile_cont(0.99) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p99_ms,
|
||||
count(admission_decision_ms)::bigint as admission_decision_count,
|
||||
percentile_cont(0.50) within group (order by admission_decision_ms)
|
||||
as admission_decision_p50_ms,
|
||||
percentile_cont(0.95) within group (order by admission_decision_ms)
|
||||
as admission_decision_p95_ms,
|
||||
percentile_cont(0.99) within group (order by admission_decision_ms)
|
||||
as admission_decision_p99_ms,
|
||||
count(account_selection_wait_ms)::bigint as account_selection_wait_count,
|
||||
percentile_cont(0.50) within group (order by account_selection_wait_ms)
|
||||
as account_selection_wait_p50_ms,
|
||||
percentile_cont(0.95) within group (order by account_selection_wait_ms)
|
||||
as account_selection_wait_p95_ms,
|
||||
percentile_cont(0.99) within group (order by account_selection_wait_ms)
|
||||
as account_selection_wait_p99_ms,
|
||||
round((percentile_cont(0.10) within group (
|
||||
order by output_tokens::double precision * 1000.0
|
||||
/ nullif(upstream_response_ms, 0)
|
||||
) filter (where {fact} and output_tokens > 0
|
||||
and upstream_response_ms > 0)))::bigint as output_throughput_p10,
|
||||
round((percentile_cont(0.50) within group (
|
||||
order by output_tokens::double precision * 1000.0
|
||||
/ nullif(upstream_response_ms, 0)
|
||||
) filter (where {fact} and output_tokens > 0
|
||||
and upstream_response_ms > 0)))::bigint as output_throughput_p50,
|
||||
round((percentile_cont(0.90) within group (
|
||||
order by output_tokens::double precision * 1000.0
|
||||
/ nullif(upstream_response_ms, 0)
|
||||
) filter (where {fact} and output_tokens > 0
|
||||
and upstream_response_ms > 0)))::bigint as output_throughput_p90,
|
||||
count(capacity_total_slots)::bigint as capacity_sample_count,
|
||||
round(avg(capacity_used_slots::double precision
|
||||
/ nullif(capacity_total_slots, 0)) * 10000)::bigint
|
||||
as capacity_utilization_avg_basis_points,
|
||||
round(percentile_cont(0.95) within group (
|
||||
order by capacity_used_slots::double precision
|
||||
/ nullif(capacity_total_slots, 0)
|
||||
) * 10000)::bigint as capacity_utilization_p95_basis_points,
|
||||
count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(*) filter (where {fact} and cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(*) filter (where {fact} and cost_source = 'unavailable')::bigint
|
||||
as unavailable_count
|
||||
from model_request_observations mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start order by bucket_start");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load request metric series", source))?;
|
||||
|
||||
let mut points = BTreeMap::new();
|
||||
for row in rows {
|
||||
let bucket_start = get(&row, "bucket_start")?;
|
||||
points.insert(
|
||||
bucket_start,
|
||||
RequestMetricPoint {
|
||||
bucket_start,
|
||||
granularity,
|
||||
metrics: request_metrics_from_row(&row)?,
|
||||
cost_coverage: coverage_from_row(&row)?,
|
||||
costs: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
if load_costs {
|
||||
let bucket_costs =
|
||||
request_costs_by_bucket(pool, range, filter, granularity, timezone).await?;
|
||||
for (bucket, costs) in bucket_costs {
|
||||
if let Some(point) = points.get_mut(&bucket) {
|
||||
point.costs = costs;
|
||||
}
|
||||
}
|
||||
}
|
||||
fill_metric_gaps(range, granularity, points, timezone)
|
||||
}
|
||||
|
||||
pub(crate) fn calculated_usage_billing_facts(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> futures::stream::BoxStream<'_, StoreResult<CalculatedUsageBillingFact>> {
|
||||
use futures::TryStreamExt;
|
||||
|
||||
Box::pin(async_stream::try_stream! {
|
||||
filter.validate()?;
|
||||
let mut query = QueryBuilder::<Postgres>::new("select ");
|
||||
push_metric_bucket(&mut query, range, granularity, timezone)?;
|
||||
query.push(
|
||||
" as bucket_start,
|
||||
mr.provider_kind, mr.upstream_model_id, mr.service_tier,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens,
|
||||
mr.billing_snapshot_json, mr.cost_currency, mr.cost_amount::text as amount
|
||||
from model_request_observations mr where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
query.push(
|
||||
" and mr.cost_source = 'calculated'
|
||||
and mr.cost_amount is not null and mr.cost_currency is not null
|
||||
and nullif(mr.provider_kind, '') is not null
|
||||
and nullif(mr.upstream_model_id, '') is not null",
|
||||
);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, &filter, "mr");
|
||||
// 费用逐条累加与行顺序无关,避免全量物化和不必要的数据库排序
|
||||
let mut rows = query.build().fetch(pool);
|
||||
while let Some(row) = rows.try_next().await
|
||||
.map_err(|source| postgres_unavailable("load calculated usage billing facts", source))?
|
||||
{
|
||||
yield calculated_usage_billing_fact_from_row(&row)?;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn request_costs_by_bucket(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<BTreeMap<DateTime<Utc>, Vec<CurrencyCostTotal>>> {
|
||||
let mut query = QueryBuilder::<Postgres>::new("select ");
|
||||
push_metric_bucket(&mut query, range, granularity, timezone)?;
|
||||
query.push(
|
||||
" as bucket_start,
|
||||
mr.cost_currency, sum(mr.cost_amount)::text as amount
|
||||
from model_request_observations mr
|
||||
where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
query.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start, mr.cost_currency order by bucket_start, mr.cost_currency");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load request costs by bucket", source))?;
|
||||
let mut result = BTreeMap::<DateTime<Utc>, Vec<CurrencyCostTotal>>::new();
|
||||
for row in rows {
|
||||
result
|
||||
.entry(get(&row, "bucket_start")?)
|
||||
.or_default()
|
||||
.push(cost_from_row(&row)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub(crate) async fn attempt_metrics(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<AttemptMetrics> {
|
||||
filter.validate()?;
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
// PERF: `model_requests` 包含宽请求元数据
|
||||
// 这里只物化两个聚合族会复用的列,
|
||||
// 避免生产基数下 `mr.*` 与多次标量子查询把 CTE 反复写入临时文件
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"with selected_requests as materialized (
|
||||
select mr.id, mr.attempt_count, mr.outcome, mr.error_kind,
|
||||
mr.upstream_status_code, mr.client_status_code,
|
||||
mr.cost_source, ({completed_usage}) as is_completed_usage
|
||||
from model_request_observations mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(
|
||||
"), request_aggregate as (
|
||||
select coalesce(sum(sr.attempt_count), 0)::bigint as attempt_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'succeeded' and sr.attempt_count > 0
|
||||
)::bigint as success_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'cancelled' and sr.attempt_count > 0
|
||||
)::bigint as cancelled_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'incomplete' and sr.attempt_count > 0
|
||||
)::bigint as incomplete_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'failed' and sr.attempt_count > 0
|
||||
)::bigint as request_failure_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'failed' and sr.attempt_count > 0
|
||||
and (coalesce(sr.error_kind, 'failed') in
|
||||
('rate_limited', 'quota_exhausted')
|
||||
or coalesce(sr.upstream_status_code,
|
||||
sr.client_status_code) = 429)
|
||||
)::bigint as request_rate_limited_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'failed' and sr.attempt_count > 0
|
||||
and (coalesce(sr.error_kind, 'failed') in
|
||||
('authentication', 'authorization', 'invalid_credential')
|
||||
or coalesce(sr.upstream_status_code,
|
||||
sr.client_status_code) in (401, 403))
|
||||
)::bigint as request_auth_failure_count,
|
||||
count(*) filter (
|
||||
where sr.outcome = 'failed' and sr.attempt_count > 0
|
||||
and coalesce(sr.upstream_status_code,
|
||||
sr.client_status_code) between 500 and 599
|
||||
)::bigint as request_provider_5xx_count,
|
||||
count(*) filter (
|
||||
where sr.is_completed_usage and sr.cost_source = 'provider_reported'
|
||||
)::bigint as provider_reported_count,
|
||||
count(*) filter (
|
||||
where sr.is_completed_usage and sr.cost_source = 'calculated'
|
||||
)::bigint as calculated_count,
|
||||
count(*) filter (
|
||||
where sr.is_completed_usage and sr.cost_source = 'unavailable'
|
||||
)::bigint as unavailable_count
|
||||
from selected_requests sr
|
||||
), ops_aggregate as (
|
||||
select count(*)::bigint
|
||||
as ops_failure_count,
|
||||
count(*) filter (
|
||||
where oe.failure_kind in ('rate_limited', 'quota_exhausted')
|
||||
or oe.status_code = 429
|
||||
)::bigint as ops_rate_limited_count,
|
||||
count(*) filter (
|
||||
where oe.failure_kind in
|
||||
('authentication', 'authorization', 'invalid_credential')
|
||||
or oe.status_code in (401, 403)
|
||||
)::bigint as ops_auth_failure_count,
|
||||
count(*) filter (
|
||||
where oe.status_code between 500 and 599
|
||||
)::bigint as ops_provider_5xx_count
|
||||
from ops_events oe
|
||||
join selected_requests sr on sr.id = oe.model_request_id
|
||||
)
|
||||
select requests.attempt_count,
|
||||
requests.success_count,
|
||||
requests.request_failure_count + ops.ops_failure_count as failure_count,
|
||||
requests.cancelled_count,
|
||||
requests.incomplete_count,
|
||||
requests.request_rate_limited_count + ops.ops_rate_limited_count
|
||||
as rate_limited_count,
|
||||
requests.request_auth_failure_count + ops.ops_auth_failure_count
|
||||
as auth_failure_count,
|
||||
requests.request_provider_5xx_count + ops.ops_provider_5xx_count
|
||||
as provider_5xx_count,
|
||||
requests.provider_reported_count,
|
||||
requests.calculated_count,
|
||||
requests.unavailable_count
|
||||
from request_aggregate requests
|
||||
cross join ops_aggregate ops",
|
||||
);
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load attempt metrics", source))?;
|
||||
Ok(AttemptMetrics {
|
||||
attempt_count: unsigned(&row, "attempt_count")?,
|
||||
success_count: unsigned(&row, "success_count")?,
|
||||
failure_count: unsigned(&row, "failure_count")?,
|
||||
cancelled_count: unsigned(&row, "cancelled_count")?,
|
||||
incomplete_count: unsigned(&row, "incomplete_count")?,
|
||||
rate_limited_count: unsigned(&row, "rate_limited_count")?,
|
||||
auth_failure_count: unsigned(&row, "auth_failure_count")?,
|
||||
provider_5xx_count: unsigned(&row, "provider_5xx_count")?,
|
||||
cost_coverage: coverage_from_row(&row)?,
|
||||
costs: request_costs(pool, range, filter).await?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn request_costs(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Vec<CurrencyCostTotal>> {
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select mr.cost_currency, sum(mr.cost_amount)::text as amount
|
||||
from model_request_observations mr where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
query.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by mr.cost_currency order by mr.cost_currency");
|
||||
query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load request costs", source))?
|
||||
.iter()
|
||||
.map(cost_from_row)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Vec<ProviderObservation>> {
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
// 请求、attempt 与失败覆盖执行审计;token 只累计完整交付的用量事实
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select coalesce(mr.provider_kind, 'unrouted') as provider_kind,
|
||||
count(*)::bigint as request_count,
|
||||
coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count,
|
||||
count(*) filter (where mr.outcome = 'failed')::bigint as failure_count,
|
||||
coalesce(sum(mr.total_tokens) filter (where {fact}), 0)::bigint
|
||||
as total_tokens
|
||||
from model_request_observations mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load provider observations", source))?;
|
||||
rows.iter()
|
||||
.map(|row| {
|
||||
Ok(ProviderObservation {
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
attempt_count: unsigned(row, "attempt_count")?,
|
||||
failure_count: unsigned(row, "failure_count")?,
|
||||
total_tokens: unsigned(row, "total_tokens")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn request_metrics_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<RequestMetrics> {
|
||||
Ok(RequestMetrics {
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
failure_count: unsigned(row, "failure_count")?,
|
||||
cancelled_count: unsigned(row, "cancelled_count")?,
|
||||
incomplete_count: unsigned(row, "incomplete_count")?,
|
||||
caller_error_count: unsigned(row, "caller_error_count")?,
|
||||
input_tokens: unsigned(row, "input_tokens")?,
|
||||
output_tokens: unsigned(row, "output_tokens")?,
|
||||
cached_tokens: unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: unsigned(row, "reasoning_tokens")?,
|
||||
total_tokens: unsigned(row, "total_tokens")?,
|
||||
first_token_latency_sum_ms: unsigned(row, "first_token_latency_sum")?,
|
||||
first_token_latency_count: unsigned(row, "first_token_latency_count")?,
|
||||
latency_sum_ms: unsigned(row, "latency_sum")?,
|
||||
latency_count: unsigned(row, "latency_count")?,
|
||||
max_latency_ms: optional_unsigned(row, "max_latency_ms")?,
|
||||
min_latency_ms: optional_unsigned(row, "min_latency_ms")?,
|
||||
cache_eligible_request_count: unsigned(row, "cache_eligible_request_count")?,
|
||||
cache_hit_request_count: unsigned(row, "cache_hit_request_count")?,
|
||||
latency_percentiles: LatencyPercentiles {
|
||||
p50_ms: optional_percentile(row, "latency_p50_ms")?,
|
||||
p95_ms: optional_percentile(row, "latency_p95_ms")?,
|
||||
p99_ms: optional_percentile(row, "latency_p99_ms")?,
|
||||
},
|
||||
first_token_latency_percentiles: LatencyPercentiles {
|
||||
p50_ms: optional_percentile(row, "first_token_p50_ms")?,
|
||||
p95_ms: optional_percentile(row, "first_token_p95_ms")?,
|
||||
p99_ms: optional_percentile(row, "first_token_p99_ms")?,
|
||||
},
|
||||
admission_decision_count: unsigned(row, "admission_decision_count")?,
|
||||
admission_decision_percentiles: LatencyPercentiles {
|
||||
p50_ms: optional_percentile(row, "admission_decision_p50_ms")?,
|
||||
p95_ms: optional_percentile(row, "admission_decision_p95_ms")?,
|
||||
p99_ms: optional_percentile(row, "admission_decision_p99_ms")?,
|
||||
},
|
||||
account_selection_wait_count: unsigned(row, "account_selection_wait_count")?,
|
||||
account_selection_wait_percentiles: LatencyPercentiles {
|
||||
p50_ms: optional_percentile(row, "account_selection_wait_p50_ms")?,
|
||||
p95_ms: optional_percentile(row, "account_selection_wait_p95_ms")?,
|
||||
p99_ms: optional_percentile(row, "account_selection_wait_p99_ms")?,
|
||||
},
|
||||
output_throughput_p10: optional_unsigned(row, "output_throughput_p10")?,
|
||||
output_throughput_p50: optional_unsigned(row, "output_throughput_p50")?,
|
||||
output_throughput_p90: optional_unsigned(row, "output_throughput_p90")?,
|
||||
capacity_sample_count: unsigned(row, "capacity_sample_count")?,
|
||||
capacity_utilization_avg_basis_points: optional_unsigned(
|
||||
row,
|
||||
"capacity_utilization_avg_basis_points",
|
||||
)?,
|
||||
capacity_utilization_p95_basis_points: optional_unsigned(
|
||||
row,
|
||||
"capacity_utilization_p95_basis_points",
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn optional_percentile(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &str,
|
||||
) -> StoreResult<Option<PercentileMilliseconds>> {
|
||||
row.try_get::<Option<f64>, _>(column)
|
||||
.map_err(|source| postgres_unavailable("decode latency percentile", source))?
|
||||
.map(|value| {
|
||||
PercentileMilliseconds::new(value)
|
||||
.map_err(|source| postgres_unavailable("decode latency percentile", source))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CostCoverage> {
|
||||
Ok(CostCoverage {
|
||||
provider_reported_count: unsigned(row, "provider_reported_count")?,
|
||||
@@ -674,31 +51,7 @@ pub(crate) fn fill_metric_gaps(
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
// 将同一组 UTC 日界交给请求、费用和空桶补齐,避免数据库另用一套时区规则
|
||||
fn push_metric_bucket(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
range: ObservabilityRange,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<()> {
|
||||
if granularity == ObservationGranularity::Day {
|
||||
let boundaries = calendar_buckets(range, timezone)?;
|
||||
query
|
||||
.push("(")
|
||||
.push_bind(boundaries.clone())
|
||||
.push("::timestamptz[])[width_bucket(mr.started_at, ")
|
||||
.push_bind(boundaries)
|
||||
.push("::timestamptz[])]");
|
||||
} else {
|
||||
query
|
||||
.push("date_bin(")
|
||||
.push_bind(sql_interval(granularity))
|
||||
.push("::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00')");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn calendar_buckets(
|
||||
pub(crate) fn calendar_buckets(
|
||||
range: ObservabilityRange,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<Vec<DateTime<Utc>>> {
|
||||
@@ -714,11 +67,3 @@ fn calendar_buckets(
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn sql_interval(granularity: ObservationGranularity) -> &'static str {
|
||||
match granularity {
|
||||
ObservationGranularity::FifteenMinutes => "15 minutes",
|
||||
ObservationGranularity::Hour => "1 hour",
|
||||
ObservationGranularity::Day => "1 day",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,12 +5,12 @@ use super::super::*;
|
||||
const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
mr.id as event_id, mr.id as request_id,
|
||||
nullif(mr.attempt_count, 0) as attempt_index,
|
||||
mr.client_api_key_ref, client_key.name as client_api_key_name, 'model_request'::text as component, mr.operation,
|
||||
mr.client_api_key_ref, mr.client_api_key_name_snapshot as client_api_key_name, 'model_request'::text as component, mr.operation,
|
||||
mr.protocol, mr.client_transport, mr.requested_model_id, mr.service_tier,
|
||||
mr.endpoint, mr.provider_kind, mr.provider_account_ref,
|
||||
mr.provider_account_name_snapshot as provider_account_name,
|
||||
mr.provider_account_email_snapshot as provider_account_email,
|
||||
account.plan_type as provider_account_plan_type,
|
||||
mr.provider_account_plan_type_snapshot as provider_account_plan_type,
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport,
|
||||
@@ -33,18 +33,16 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
mr.completed_at as occurred_at,
|
||||
'model_request:' || mr.id as stable_sort_id
|
||||
from model_request_observations mr
|
||||
left join client_api_keys client_key on client_key.id = mr.client_api_key_ref
|
||||
left join provider_accounts account on account.id = mr.provider_account_ref
|
||||
where true";
|
||||
|
||||
const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
|
||||
oe.id as event_id, oe.model_request_id as request_id, oe.attempt_index,
|
||||
mr.client_api_key_ref, client_key.name as client_api_key_name, oe.component, oe.operation,
|
||||
mr.client_api_key_ref, mr.client_api_key_name_snapshot as client_api_key_name, oe.component, oe.operation,
|
||||
mr.protocol, mr.client_transport, mr.requested_model_id, mr.service_tier,
|
||||
mr.endpoint, oe.provider_kind, oe.provider_account_ref,
|
||||
oe.provider_account_name_snapshot as provider_account_name,
|
||||
oe.provider_account_email_snapshot as provider_account_email,
|
||||
account.plan_type as provider_account_plan_type,
|
||||
oe.provider_account_plan_type_snapshot as provider_account_plan_type,
|
||||
oe.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
oe.upstream_model_id, null::text as upstream_transport, oe.failure_kind,
|
||||
@@ -68,8 +66,6 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
|
||||
'ops_event:' || oe.id as stable_sort_id
|
||||
from ops_events oe
|
||||
left join model_request_observations mr on mr.id = oe.model_request_id
|
||||
left join client_api_keys client_key on client_key.id = mr.client_api_key_ref
|
||||
left join provider_accounts account on account.id = oe.provider_account_ref
|
||||
where true";
|
||||
|
||||
pub(crate) async fn list_ops_errors(
|
||||
@@ -77,27 +73,38 @@ pub(crate) async fn list_ops_errors(
|
||||
query: OpsErrorQuery,
|
||||
) -> StoreResult<OpsErrorPage> {
|
||||
validate_ops_error_filter(&query.filter)?;
|
||||
let total = count_ops_errors(pool, query.range, &query.filter).await?;
|
||||
let mut transaction = analytics::begin_snapshot(pool).await?;
|
||||
let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?;
|
||||
let offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut statement = QueryBuilder::<Postgres>::new("select * from (");
|
||||
statement.push(REQUEST_ERROR_SELECT);
|
||||
let mut statement =
|
||||
QueryBuilder::<Postgres>::new("with candidate_page as materialized (select * from (");
|
||||
statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true");
|
||||
push_request_error_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(" union all ");
|
||||
statement.push(OPS_EVENT_SELECT);
|
||||
statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true");
|
||||
push_ops_event_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(") e order by occurred_at desc, stable_sort_id desc limit ");
|
||||
statement.push_bind(i64::from(query.page_size.get()));
|
||||
statement.push(" offset ");
|
||||
statement.push_bind(offset);
|
||||
statement.push(") select details.* from candidate_page page cross join lateral (");
|
||||
statement.push(REQUEST_ERROR_SELECT);
|
||||
statement.push(" and page.source='model_request' and mr.id=page.event_id union all ");
|
||||
statement.push(OPS_EVENT_SELECT);
|
||||
statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc");
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("list ops errors", source))?;
|
||||
let items = rows
|
||||
.iter()
|
||||
.map(ops_error_from_row)
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("complete Ops page snapshot", error))?;
|
||||
Ok(OpsErrorPage {
|
||||
items,
|
||||
current_page: query.current_page,
|
||||
@@ -107,7 +114,7 @@ pub(crate) async fn list_ops_errors(
|
||||
}
|
||||
|
||||
pub(crate) async fn count_ops_errors(
|
||||
pool: &PgPool,
|
||||
connection: &mut sqlx::PgConnection,
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) -> StoreResult<u64> {
|
||||
@@ -122,7 +129,7 @@ pub(crate) async fn count_ops_errors(
|
||||
statement.push(") counts");
|
||||
let total = statement
|
||||
.build_query_scalar::<i64>()
|
||||
.fetch_one(pool)
|
||||
.fetch_one(connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("count ops errors", source))?;
|
||||
to_u64(total)
|
||||
@@ -137,6 +144,7 @@ fn push_request_error_predicates(
|
||||
// 列表和总数共用此条件,避免流式响应中的错误因 outcome 被漏掉
|
||||
statement.push(" and mr.error_kind is not null and mr.error_kind <> 'cancelled'");
|
||||
push_range(statement, "mr.completed_at", range);
|
||||
push_routing_groups(statement, &filter.routing_group_refs);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
("mr.id", &filter.request_id),
|
||||
@@ -179,6 +187,7 @@ fn push_ops_event_predicates(
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
push_range(statement, "oe.created_at", range);
|
||||
push_routing_groups(statement, &filter.routing_group_refs);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
("oe.model_request_id", &filter.request_id),
|
||||
@@ -260,6 +269,17 @@ fn push_prefix_search(statement: &mut QueryBuilder<Postgres>, columns: &[&str],
|
||||
statement.push_bind(pattern.clone());
|
||||
statement.push(" escape '\\'");
|
||||
}
|
||||
push_client_key_name_search(statement, "mr.client_api_key_ref", pattern);
|
||||
statement
|
||||
.push(" or mr.client_api_key_name_snapshot ilike ")
|
||||
.push_bind(pattern)
|
||||
.push(" escape '\\'");
|
||||
statement.push(")");
|
||||
}
|
||||
|
||||
fn push_routing_groups(statement: &mut QueryBuilder<Postgres>, groups: &[String]) {
|
||||
if !groups.is_empty() {
|
||||
statement.push(" and mr.routing_group_refs && ");
|
||||
statement.push_bind(groups.to_vec());
|
||||
statement.push("::text[]");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,12 @@ pub(crate) fn push_usage_filter(
|
||||
filter: &UsageRecordFilter,
|
||||
alias: &str,
|
||||
) {
|
||||
if !filter.routing_group_refs.is_empty() {
|
||||
query
|
||||
.push(format!(" and {alias}.routing_group_refs && "))
|
||||
.push_bind(filter.routing_group_refs.clone())
|
||||
.push("::text[]");
|
||||
}
|
||||
if let Some(value) = &filter.client_api_key_ref {
|
||||
query.push(format!(" and {alias}.client_api_key_ref = "));
|
||||
query.push_bind(value.clone());
|
||||
@@ -95,24 +101,15 @@ pub(crate) fn push_usage_filter(
|
||||
query.push_bind(pattern.clone());
|
||||
}
|
||||
query.push(" escape '\\'");
|
||||
push_client_key_name_search(query, &format!("{alias}.client_api_key_ref"), pattern);
|
||||
|
||||
query
|
||||
.push(format!(" or {alias}.client_api_key_name_snapshot ilike "))
|
||||
.push_bind(pattern)
|
||||
.push(" escape '\\'");
|
||||
query.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_client_key_name_search(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
key_ref: &str,
|
||||
pattern: String,
|
||||
) {
|
||||
query.push(format!(
|
||||
" or exists (select 1 from client_api_keys searched_client_key
|
||||
where searched_client_key.id = {key_ref} and searched_client_key.name ilike "
|
||||
));
|
||||
query.push_bind(pattern);
|
||||
query.push(" escape '\\')");
|
||||
}
|
||||
|
||||
pub(crate) fn literal_prefix_pattern(value: &str) -> String {
|
||||
format!(
|
||||
"{}%",
|
||||
@@ -124,17 +121,17 @@ pub(crate) fn literal_prefix_pattern(value: &str) -> String {
|
||||
}
|
||||
|
||||
pub(crate) const USAGE_LIST_RECORD_SELECT: &str =
|
||||
"select mr.id, client_key.name as client_api_key_name, mr.endpoint, mr.client_transport, mr.requested_model_id,
|
||||
"select mr.id, mr.client_api_key_name_snapshot as client_api_key_name, mr.endpoint, mr.client_transport, mr.requested_model_id,
|
||||
mr.provider_kind, mr.provider_account_ref,
|
||||
mr.provider_account_name_snapshot as provider_account_name,
|
||||
mr.provider_account_email_snapshot as provider_account_email,
|
||||
account.notes as provider_account_notes,
|
||||
account.plan_type as provider_account_plan_type,
|
||||
mr.provider_account_notes_snapshot as provider_account_notes,
|
||||
mr.provider_account_plan_type_snapshot as provider_account_plan_type,
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport, mr.upstream_response_model, mr.service_tier,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens,
|
||||
mr.reasoning_tokens, mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.cache_write_input_tokens,
|
||||
mr.reasoning_output_tokens, mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.total_tokens, mr.billing_snapshot_json, mr.cost_source, mr.cost_amount::text, mr.cost_currency,
|
||||
mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms,
|
||||
mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms,
|
||||
@@ -144,9 +141,7 @@ pub(crate) const USAGE_LIST_RECORD_SELECT: &str =
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.subagent_kind, mr.compact,
|
||||
mr.started_at
|
||||
from model_request_observations mr
|
||||
left join client_api_keys client_key on client_key.id = mr.client_api_key_ref
|
||||
left join provider_accounts account on account.id = mr.provider_account_ref";
|
||||
from model_request_observations mr";
|
||||
|
||||
pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
"select mr.id, mr.client_api_key_ref, mr.config_revision,
|
||||
@@ -164,8 +159,8 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
mr.outcome, mr.client_status_code, mr.upstream_status_code,
|
||||
mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id,
|
||||
mr.error_kind, mr.provider_error_code, mr.error_message, mr.retry_after_ms,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens,
|
||||
mr.reasoning_tokens, mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.cache_write_input_tokens,
|
||||
mr.reasoning_output_tokens, mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.total_tokens, mr.billing_snapshot_json, mr.cost_source, mr.cost_amount::text,
|
||||
mr.cost_currency, mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms,
|
||||
mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms,
|
||||
@@ -175,7 +170,7 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
host(mr.client_ip) as client_ip,
|
||||
mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact, mr.image_generation_requested,
|
||||
mr.image_generation_succeeded, mr.started_at, mr.deadline_at, mr.completed_at
|
||||
mr.image_generation_succeeded, mr.started_at, mr.effective_recovery_lease_expires_at as deadline_at, mr.completed_at
|
||||
from model_request_observations mr";
|
||||
|
||||
pub(crate) async fn list_usage_records(
|
||||
@@ -183,8 +178,13 @@ pub(crate) async fn list_usage_records(
|
||||
query: UsageRecordQuery,
|
||||
) -> StoreResult<UsageRecordPage> {
|
||||
query.filter.validate()?;
|
||||
let total = count_usage_records(pool, query.range, &query.filter).await?;
|
||||
let items = list_usage_record_items(pool, &query).await?;
|
||||
let mut transaction = analytics::begin_snapshot(pool).await?;
|
||||
let total = count_usage_records(&mut *transaction, query.range, &query.filter).await?;
|
||||
let items = list_usage_record_items(&mut *transaction, &query).await?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("complete usage page snapshot", error))?;
|
||||
Ok(UsageRecordPage {
|
||||
items,
|
||||
current_page: query.current_page,
|
||||
@@ -193,13 +193,15 @@ pub(crate) async fn list_usage_records(
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn list_usage_record_items(
|
||||
pool: &PgPool,
|
||||
pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database = Postgres>>(
|
||||
executor: E,
|
||||
query: &UsageRecordQuery,
|
||||
) -> StoreResult<Vec<UsageListRecord>> {
|
||||
query.filter.validate()?;
|
||||
let offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut statement = QueryBuilder::<Postgres>::new(USAGE_LIST_RECORD_SELECT);
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"with candidate_page as materialized (select mr.id, mr.started_at from model_request_observations mr",
|
||||
);
|
||||
statement.push(" where mr.started_at >= ");
|
||||
statement.push_bind(query.range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
@@ -210,9 +212,14 @@ pub(crate) async fn list_usage_record_items(
|
||||
statement.push_bind(i64::from(query.page_size.get()));
|
||||
statement.push(" offset ");
|
||||
statement.push_bind(offset);
|
||||
statement.push(") ");
|
||||
statement.push(USAGE_LIST_RECORD_SELECT);
|
||||
statement.push(
|
||||
" join candidate_page page on page.id = mr.id order by page.started_at desc, page.id desc",
|
||||
);
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.fetch_all(executor)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("list usage records", source))?;
|
||||
let items = rows
|
||||
@@ -222,8 +229,8 @@ pub(crate) async fn list_usage_record_items(
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
pub(crate) async fn count_usage_records(
|
||||
pool: &PgPool,
|
||||
pub(crate) async fn count_usage_records<'e, E: sqlx::Executor<'e, Database = Postgres>>(
|
||||
executor: E,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<u64> {
|
||||
@@ -237,7 +244,7 @@ pub(crate) async fn count_usage_records(
|
||||
push_usage_filter(&mut statement, filter, "mr");
|
||||
let total = statement
|
||||
.build_query_scalar::<i64>()
|
||||
.fetch_one(pool)
|
||||
.fetch_one(executor)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("count usage records", source))?;
|
||||
to_u64(total)
|
||||
@@ -335,9 +342,9 @@ pub(crate) fn intermediate_attempt_from_row(
|
||||
message: get(row, "message")?,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
total_tokens: None,
|
||||
cost_source: None,
|
||||
cost_amount: None,
|
||||
@@ -372,9 +379,9 @@ pub(crate) fn final_attempt_from_request(request: &UsageRecord) -> UsageAttemptO
|
||||
message: request.error_message.clone(),
|
||||
input_tokens: request.input_tokens,
|
||||
output_tokens: request.output_tokens,
|
||||
cached_tokens: request.cached_tokens,
|
||||
cache_write_tokens: request.cache_write_tokens,
|
||||
reasoning_tokens: request.reasoning_tokens,
|
||||
cached_input_tokens: request.cached_input_tokens,
|
||||
cache_write_input_tokens: request.cache_write_input_tokens,
|
||||
reasoning_output_tokens: request.reasoning_output_tokens,
|
||||
total_tokens: request.total_tokens,
|
||||
cost_source: Some(request.cost_source.clone()),
|
||||
cost_amount: request.cost_amount.clone(),
|
||||
@@ -383,259 +390,6 @@ pub(crate) fn final_attempt_from_request(request: &UsageRecord) -> UsageAttemptO
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_diagnostics(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
limit: u16,
|
||||
) -> StoreResult<DiagnosticsObservation> {
|
||||
filter.validate()?;
|
||||
let limit = ObservabilityPageSize::new(limit)
|
||||
.map_err(|_| invalid("invalid diagnostic limit"))?
|
||||
.get();
|
||||
let dimension_sql = diagnostic_dimension_sql(dimension);
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
let mut statement = QueryBuilder::<Postgres>::new("with matched as (select ");
|
||||
statement.push(dimension_sql);
|
||||
statement.push(format!(
|
||||
" as dimension_name, mr.outcome, mr.attempt_count, mr.total_tokens,
|
||||
mr.latency_ms, mr.first_token_ms, mr.cost_source, mr.cost_amount,
|
||||
mr.cost_currency, mr.downstream_committed_at, mr.client_transport,
|
||||
mr.client_status_code, ({completed_usage}) as is_completed_usage
|
||||
from model_request_observations mr where mr.started_at >= ",
|
||||
));
|
||||
statement.push_bind(range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut statement, "mr");
|
||||
push_diagnostic_dimension_filter(&mut statement, dimension);
|
||||
push_usage_filter(&mut statement, filter, "mr");
|
||||
statement.push(
|
||||
"), aggregated as (
|
||||
select dimension_name, cost_currency,
|
||||
grouping(cost_currency)::integer as currency_grouping,
|
||||
count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
count(*) filter (where outcome = 'failed')::bigint as failure_count,
|
||||
coalesce(sum(attempt_count), 0)::bigint as attempt_count,
|
||||
coalesce(sum(total_tokens) filter (where is_completed_usage), 0)::bigint
|
||||
as total_tokens,
|
||||
round(avg(latency_ms) filter (where is_completed_usage))::bigint
|
||||
as average_latency_ms,
|
||||
round(percentile_cont(0.95) within group (order by latency_ms)
|
||||
filter (where is_completed_usage))::bigint
|
||||
as latency_p95_ms,
|
||||
round(percentile_cont(0.95) within group (order by first_token_ms)
|
||||
filter (where is_completed_usage))::bigint
|
||||
as first_token_p95_ms,
|
||||
count(*) filter (where outcome in ('cancelled', 'incomplete'))::bigint
|
||||
as non_completion_count,
|
||||
coalesce(sum(greatest(attempt_count - 1, 0)), 0)::bigint as retry_count,
|
||||
count(*) filter (where attempt_count > 1)::bigint as retried_request_count,
|
||||
count(*) filter (
|
||||
where is_completed_usage and cost_source = 'provider_reported'
|
||||
)::bigint
|
||||
as provider_reported_count,
|
||||
count(*) filter (
|
||||
where is_completed_usage and cost_source = 'calculated'
|
||||
)::bigint
|
||||
as calculated_count,
|
||||
count(*) filter (
|
||||
where is_completed_usage and cost_source = 'unavailable'
|
||||
)::bigint
|
||||
as unavailable_count,
|
||||
sum(cost_amount) filter (
|
||||
where is_completed_usage
|
||||
)::text as amount
|
||||
from matched
|
||||
group by dimension_name, grouping sets ((), (cost_currency))
|
||||
), selected_dimensions as (
|
||||
select dimension_name,
|
||||
sum(request_count) over ()::bigint as total_request_count,
|
||||
row_number() over (order by request_count desc, dimension_name)
|
||||
as sort_position
|
||||
from aggregated
|
||||
where currency_grouping = 1
|
||||
order by request_count desc, dimension_name limit ",
|
||||
);
|
||||
statement.push_bind(i64::from(limit));
|
||||
statement.push(
|
||||
")
|
||||
select aggregated.*, selected.total_request_count,
|
||||
account.provider_kind as account_provider_kind, account.plan_type as account_plan_type
|
||||
from aggregated
|
||||
join selected_dimensions selected using (dimension_name)
|
||||
left join provider_accounts account on account.id = aggregated.dimension_name and ",
|
||||
);
|
||||
statement.push_bind(dimension == DiagnosticDimension::Account);
|
||||
statement
|
||||
.push(" order by selected.sort_position, currency_grouping desc, cost_currency nulls last");
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load usage diagnostics", source))?;
|
||||
let total_request_count = rows
|
||||
.first()
|
||||
.map(|row| unsigned(row, "total_request_count"))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let mut observations = Vec::with_capacity(usize::from(limit));
|
||||
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
|
||||
for row in &rows {
|
||||
match get::<i32>(row, "currency_grouping")? {
|
||||
1 => observations.push(DiagnosticObservation {
|
||||
key: get(row, "dimension_name")?,
|
||||
name: get(row, "dimension_name")?,
|
||||
account_provider_kind: get(row, "account_provider_kind")?,
|
||||
account_plan_type: get(row, "account_plan_type")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
failure_count: unsigned(row, "failure_count")?,
|
||||
attempt_count: unsigned(row, "attempt_count")?,
|
||||
total_tokens: unsigned(row, "total_tokens")?,
|
||||
average_latency_ms: optional_unsigned(row, "average_latency_ms")?,
|
||||
latency_p95_ms: optional_unsigned(row, "latency_p95_ms")?,
|
||||
first_token_p95_ms: optional_unsigned(row, "first_token_p95_ms")?,
|
||||
non_completion_count: unsigned(row, "non_completion_count")?,
|
||||
retry_count: unsigned(row, "retry_count")?,
|
||||
retried_request_count: unsigned(row, "retried_request_count")?,
|
||||
cost_coverage: coverage_from_row(row)?,
|
||||
costs: Vec::new(),
|
||||
}),
|
||||
0 if get::<Option<String>>(row, "cost_currency")?.is_some()
|
||||
&& get::<Option<String>>(row, "amount")?.is_some() =>
|
||||
{
|
||||
costs
|
||||
.entry(get(row, "dimension_name")?)
|
||||
.or_default()
|
||||
.push(cost_from_row(row)?);
|
||||
}
|
||||
0 => {}
|
||||
_ => {
|
||||
return Err(crate::StoreError::Unavailable {
|
||||
backend: crate::StoreBackend::PostgreSql,
|
||||
message: "decode usage diagnostic grouping".to_owned(),
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut display_names = match dimension {
|
||||
DiagnosticDimension::Account => {
|
||||
diagnostic_account_display_names(
|
||||
pool,
|
||||
&observations
|
||||
.iter()
|
||||
.map(|item| item.key.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
DiagnosticDimension::ApiKey => {
|
||||
diagnostic_api_key_display_names(
|
||||
pool,
|
||||
&observations
|
||||
.iter()
|
||||
.map(|item| item.key.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
_ => HashMap::new(),
|
||||
};
|
||||
for observation in &mut observations {
|
||||
if let Some(name) = display_names.remove(&observation.key) {
|
||||
observation.name = name;
|
||||
}
|
||||
observation.costs = costs.remove(&observation.key).unwrap_or_default();
|
||||
}
|
||||
Ok(DiagnosticsObservation {
|
||||
total_request_count,
|
||||
items: observations,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn diagnostic_account_display_names(
|
||||
pool: &PgPool,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, String>> {
|
||||
if account_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
let rows = sqlx::query(
|
||||
"select requested.account_id as provider_account_ref,
|
||||
coalesce(
|
||||
(
|
||||
select request.provider_account_email_snapshot
|
||||
from model_request_observations request
|
||||
where request.provider_account_ref = requested.account_id
|
||||
and request.provider_account_email_snapshot is not null
|
||||
order by request.started_at desc, request.id desc
|
||||
limit 1
|
||||
),
|
||||
(
|
||||
select request.provider_account_name_snapshot
|
||||
from model_request_observations request
|
||||
where request.provider_account_ref = requested.account_id
|
||||
and request.provider_account_name_snapshot is not null
|
||||
order by request.started_at desc, request.id desc
|
||||
limit 1
|
||||
),
|
||||
requested.account_id
|
||||
) as display_name
|
||||
from unnest($1::text[]) as requested(account_id)",
|
||||
)
|
||||
.bind(account_ids)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load diagnostic account display names", source))?;
|
||||
let mut display_names = HashMap::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
display_names.insert(
|
||||
get(&row, "provider_account_ref")?,
|
||||
get(&row, "display_name")?,
|
||||
);
|
||||
}
|
||||
Ok(display_names)
|
||||
}
|
||||
|
||||
pub(crate) async fn diagnostic_api_key_display_names(
|
||||
pool: &PgPool,
|
||||
key_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, String>> {
|
||||
if key_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
let rows = sqlx::query("select id, name from client_api_keys where id = any($1::text[])")
|
||||
.bind(key_ids)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load diagnostic api key display names", source))?;
|
||||
let mut display_names = HashMap::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
display_names.insert(get(&row, "id")?, get(&row, "name")?);
|
||||
}
|
||||
Ok(display_names)
|
||||
}
|
||||
|
||||
pub(crate) fn diagnostic_dimension_sql(dimension: DiagnosticDimension) -> &'static str {
|
||||
match dimension {
|
||||
DiagnosticDimension::Provider => "coalesce(mr.provider_kind, 'unrouted')",
|
||||
DiagnosticDimension::Model => MODEL_DIAGNOSTIC_DIMENSION_SQL,
|
||||
DiagnosticDimension::Account => "coalesce(mr.provider_account_ref, 'unrouted')",
|
||||
DiagnosticDimension::ApiKey => "mr.client_api_key_ref",
|
||||
DiagnosticDimension::Transport => {
|
||||
"coalesce(mr.upstream_transport, mr.client_transport, 'unknown')"
|
||||
}
|
||||
DiagnosticDimension::Failure => "coalesce(mr.error_kind, 'none')",
|
||||
DiagnosticDimension::Status => {
|
||||
"coalesce(mr.upstream_status_code, mr.client_status_code)::text"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_diagnostic_dimension_filter(
|
||||
statement: &mut QueryBuilder<Postgres>,
|
||||
dimension: DiagnosticDimension,
|
||||
|
||||
@@ -6,7 +6,6 @@ use std::{
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use futures::{Stream, TryStreamExt, stream::BoxStream};
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
use crate::{StoreBackend, StoreError, StoreResult};
|
||||
@@ -39,7 +38,7 @@ impl ObservabilityQueryBudget {
|
||||
})
|
||||
}
|
||||
|
||||
/// 在共享管理观测预算内执行一条 PostgreSQL 查询
|
||||
/// 在共享槽位与十秒执行预算内完成整个观测操作
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -49,44 +48,13 @@ impl ObservabilityQueryBudget {
|
||||
F: Future<Output = StoreResult<T>>,
|
||||
{
|
||||
let _slot = self.acquire(operation).await?;
|
||||
query.await
|
||||
}
|
||||
|
||||
/// 将槽位持有到查询流消费结束、报错或被丢弃,而不是仅持有到流创建完成
|
||||
pub fn run_stream<'a, T, S>(
|
||||
&'a self,
|
||||
operation: &'static str,
|
||||
query: S,
|
||||
) -> BoxStream<'a, StoreResult<T>>
|
||||
where
|
||||
T: Send + 'a,
|
||||
S: Stream<Item = StoreResult<T>> + Send + 'a,
|
||||
{
|
||||
Box::pin(async_stream::stream! {
|
||||
let mut query = Box::pin(query);
|
||||
let slot = match self.acquire(operation).await {
|
||||
Ok(slot) => slot,
|
||||
Err(error) => {
|
||||
drop(query);
|
||||
yield Err(error);
|
||||
return;
|
||||
}
|
||||
};
|
||||
loop {
|
||||
match query.try_next().await {
|
||||
Ok(Some(item)) => yield Ok(item),
|
||||
Ok(None) => break,
|
||||
Err(error) => {
|
||||
// 返回错误后调用方可能不再 poll,但仍保留流对象
|
||||
// 必须在交付错误之前释放数据库流和预算
|
||||
drop(query);
|
||||
drop(slot);
|
||||
yield Err(error);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
tokio::time::timeout(Duration::from_secs(10), query)
|
||||
.await
|
||||
.map_err(|source| StoreError::Unavailable {
|
||||
backend: StoreBackend::PostgreSql,
|
||||
message: "observability PostgreSQL execution budget is exhausted".to_owned(),
|
||||
source: Some(gateway_core::error::ErrorSource::new(source)),
|
||||
})?
|
||||
}
|
||||
|
||||
async fn acquire(&self, operation: &'static str) -> StoreResult<OwnedSemaphorePermit> {
|
||||
|
||||
@@ -33,7 +33,7 @@ pub struct OpsEvent {
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_id: Option<String>,
|
||||
pub account_snapshot: Option<gateway_core::account::AccountHistorySnapshot>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub failure_kind: String,
|
||||
@@ -78,15 +78,6 @@ impl OpsEvent {
|
||||
{
|
||||
return Err(invalid("upstream_send_state is invalid"));
|
||||
}
|
||||
if self
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| Some(id) != self.provider_account_ref.as_ref())
|
||||
{
|
||||
return Err(invalid(
|
||||
"live provider account ID must equal its historical ref",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -115,17 +106,17 @@ impl OpsEventRepository for PgOpsEventRepository {
|
||||
sqlx::query(
|
||||
"insert into ops_events (
|
||||
id, model_request_id, attempt_index, level, component, operation,
|
||||
provider_kind, provider_account_id, provider_account_ref, upstream_model_id,
|
||||
provider_kind, provider_account_ref, upstream_model_id,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
provider_account_authentication_kind_snapshot, provider_account_plan_type_snapshot,
|
||||
failure_kind, upstream_send_state, status_code, provider_error_code, retry_after_ms,
|
||||
upstream_request_id, latency_ms, message, error_details,
|
||||
created_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||
(select name from provider_accounts where id = $8),
|
||||
(select email from provider_accounts where id = $8),
|
||||
(select authentication_kind from provider_accounts where id = $8),
|
||||
$1, $2, $3, $4, $5, $6, $7, $9, $10,
|
||||
$8::jsonb ->> 'name',
|
||||
$8::jsonb ->> 'email',
|
||||
$8::jsonb ->> 'authenticationKind', $8::jsonb ->> 'planTypeSnapshot',
|
||||
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20
|
||||
)",
|
||||
)
|
||||
@@ -142,7 +133,7 @@ impl OpsEventRepository for PgOpsEventRepository {
|
||||
.bind(event.component)
|
||||
.bind(event.operation)
|
||||
.bind(event.provider_kind)
|
||||
.bind(event.provider_account_id)
|
||||
.bind(event.account_snapshot.map(sqlx::types::Json))
|
||||
.bind(event.provider_account_ref)
|
||||
.bind(event.upstream_model_id)
|
||||
.bind(event.failure_kind)
|
||||
|
||||
@@ -20,7 +20,7 @@ pub(crate) fn encode_billing_snapshot(
|
||||
"version": 1,
|
||||
"longContextBillingApplied": b.long_context_billing_applied(),
|
||||
"image": b.image().map(|image| serde_json::json!({
|
||||
"inputTokens": image.input_tokens, "cachedTokens": image.cached_tokens,
|
||||
"inputTokens": image.input_tokens, "cachedTokens": image.cached_input_tokens,
|
||||
"input": image.input_amount.amount().canonical(),
|
||||
"cacheRead": image.cache_read_amount.amount().canonical(),
|
||||
"inputPrice": image.input_price_per_million.amount().canonical(),
|
||||
@@ -74,7 +74,7 @@ pub(crate) fn decode_billing_snapshot(
|
||||
};
|
||||
Some(gateway_admin::model::observability::ImageBillingBreakdown {
|
||||
input_tokens: image.get("inputTokens")?.as_u64()?,
|
||||
cached_tokens: image.get("cachedTokens")?.as_u64()?,
|
||||
cached_input_tokens: image.get("cachedTokens")?.as_u64()?,
|
||||
input_amount: amount("input")?,
|
||||
cache_read_amount: amount("cacheRead")?,
|
||||
input_price_per_million: amount("inputPrice")?,
|
||||
|
||||
@@ -86,9 +86,9 @@ pub(crate) fn admin_account_usage(usage: ProviderAccountUsageObservation) -> Acc
|
||||
success_count: usage.success_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
cached_input_tokens: usage.cached_input_tokens,
|
||||
cache_write_input_tokens: usage.cache_write_input_tokens,
|
||||
reasoning_output_tokens: usage.reasoning_output_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
image_request_count: usage.image_request_count,
|
||||
@@ -121,9 +121,9 @@ pub(crate) fn admin_account_usage_window(
|
||||
success_count: window_usage_count(row, "success_count")?,
|
||||
input_tokens: optional_window_usage_count(row, "input_tokens")?,
|
||||
output_tokens: optional_window_usage_count(row, "output_tokens")?,
|
||||
cached_tokens: optional_window_usage_count(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_window_usage_count(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_window_usage_count(row, "reasoning_tokens")?,
|
||||
cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?,
|
||||
image_request_count: window_usage_count(row, "image_request_count")?,
|
||||
@@ -161,9 +161,9 @@ pub(crate) fn admin_account_usage_window_model(
|
||||
success_count: window_usage_count(row, "success_count")?,
|
||||
input_tokens: optional_window_usage_count(row, "input_tokens")?,
|
||||
output_tokens: optional_window_usage_count(row, "output_tokens")?,
|
||||
cached_tokens: optional_window_usage_count(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_window_usage_count(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_window_usage_count(row, "reasoning_tokens")?,
|
||||
cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?,
|
||||
image_request_count: window_usage_count(row, "image_request_count")?,
|
||||
|
||||
@@ -43,7 +43,7 @@ pub(super) async fn load_history(
|
||||
tokens: window_usage_count(row, "tokens")?,
|
||||
input_tokens: window_usage_count(row, "input_tokens")?,
|
||||
output_tokens: window_usage_count(row, "output_tokens")?,
|
||||
cached_tokens: window_usage_count(row, "cached_tokens")?,
|
||||
cached_input_tokens: window_usage_count(row, "cached_input_tokens")?,
|
||||
missing_token_count: window_usage_count(row, "missing_token_count")?,
|
||||
known_cost_count: window_usage_count(row, "known_cost_count")?,
|
||||
unavailable_cost_count: window_usage_count(row, "unavailable_cost_count")?,
|
||||
@@ -79,7 +79,7 @@ fn history_sql() -> String {
|
||||
mr.completed_at <= $3 and mr.outcome <> 'running' as settled,
|
||||
coalesce(({completed_usage}), false) as included,
|
||||
mr.provider_observation_json is not null as has_document,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.total_tokens,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.total_tokens,
|
||||
mr.cost_amount, mr.cost_currency,
|
||||
coalesce(mr.cost_source in ('calculated', 'provider_reported')
|
||||
and mr.cost_amount is not null and mr.cost_currency = 'USD', false) as known_cost
|
||||
@@ -93,7 +93,7 @@ fn history_sql() -> String {
|
||||
coalesce(input_tokens, 0) + coalesce(output_tokens, 0)) else 0 end as tokens,
|
||||
case when included and settled then coalesce(input_tokens, 0) else 0 end as inputs,
|
||||
case when included and settled then coalesce(output_tokens, 0) else 0 end as outputs,
|
||||
case when included and settled then coalesce(cached_tokens, 0) else 0 end as cached,
|
||||
case when included and settled then coalesce(cached_input_tokens, 0) else 0 end as cached,
|
||||
(included and settled and total_tokens is null
|
||||
and (input_tokens is null or output_tokens is null))::integer as missing_tokens,
|
||||
(included and settled and known_cost)::integer as known_costs,
|
||||
@@ -109,7 +109,7 @@ fn history_sql() -> String {
|
||||
sum(tokens) over w as tokens,
|
||||
sum(inputs) over w as input_tokens,
|
||||
sum(outputs) over w as output_tokens,
|
||||
sum(cached) over w as cached_tokens,
|
||||
sum(cached) over w as cached_input_tokens,
|
||||
sum(missing_tokens) over w as missing_token_count,
|
||||
sum(known_costs) over w as known_cost_count,
|
||||
sum(missing_costs) over w as unavailable_cost_count,
|
||||
@@ -123,7 +123,7 @@ fn history_sql() -> String {
|
||||
)
|
||||
select false as is_total, s.started_at, s.completed_at,
|
||||
s.request_count::bigint, s.tokens::bigint, s.input_tokens::bigint,
|
||||
s.output_tokens::bigint, s.cached_tokens::bigint, s.missing_token_count::bigint,
|
||||
s.output_tokens::bigint, s.cached_input_tokens::bigint, s.missing_token_count::bigint,
|
||||
s.known_cost_count::bigint, s.unavailable_cost_count::bigint,
|
||||
s.usd::double precision, s.excluded_request_count::bigint,
|
||||
0::bigint as pending_count, mr.provider_observation_json as document
|
||||
|
||||
@@ -24,8 +24,8 @@ pub(crate) fn account_usage_by_windows_sql() -> String {
|
||||
requested.window_key,
|
||||
mr.id as request_id,
|
||||
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
|
||||
mr.outcome, mr.input_tokens, mr.output_tokens, mr.cached_tokens,
|
||||
mr.cache_write_tokens, mr.reasoning_tokens,
|
||||
mr.outcome, mr.input_tokens, mr.output_tokens, mr.cached_input_tokens,
|
||||
mr.cache_write_input_tokens, mr.reasoning_output_tokens,
|
||||
mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.image_generation_succeeded, mr.total_tokens,
|
||||
mr.cost_source, mr.cost_amount, mr.cost_currency, mr.started_at
|
||||
@@ -46,9 +46,9 @@ pub(crate) fn account_usage_by_windows_sql() -> String {
|
||||
count(request_id) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
sum(input_tokens)::bigint as input_tokens,
|
||||
sum(output_tokens)::bigint as output_tokens,
|
||||
sum(cached_tokens)::bigint as cached_tokens,
|
||||
sum(cache_write_tokens)::bigint as cache_write_tokens,
|
||||
sum(reasoning_tokens)::bigint as reasoning_tokens,
|
||||
sum(cached_input_tokens)::bigint as cached_input_tokens,
|
||||
sum(cache_write_input_tokens)::bigint as cache_write_input_tokens,
|
||||
sum(reasoning_output_tokens)::bigint as reasoning_output_tokens,
|
||||
sum(image_input_tokens)::bigint as image_input_tokens,
|
||||
sum(image_output_tokens)::bigint as image_output_tokens,
|
||||
count(request_id) filter (where image_generation_succeeded is true)::bigint
|
||||
@@ -479,6 +479,7 @@ pub(crate) fn core_account_from_summary(
|
||||
revision,
|
||||
summary.access_token_expires_at.map(Into::into),
|
||||
)
|
||||
.with_notes(summary.notes)
|
||||
.with_profile(
|
||||
summary.email,
|
||||
summary.upstream_account_id,
|
||||
|
||||
@@ -144,6 +144,7 @@ impl SnapshotStorePort for PgRuntimeSnapshotRepository {
|
||||
key.group_ids,
|
||||
key.limits,
|
||||
)
|
||||
.with_name_snapshot(key.name_snapshot)
|
||||
.with_request_profiles(key.request_profiles)
|
||||
})
|
||||
.collect();
|
||||
@@ -304,12 +305,13 @@ async fn load_client_keys(
|
||||
i64,
|
||||
i64,
|
||||
sqlx::types::Json<BTreeMap<String, serde_json::Map<String, serde_json::Value>>>,
|
||||
String,
|
||||
),
|
||||
>(
|
||||
"select k.id, k.key,
|
||||
coalesce(array_agg(kg.account_group_id order by kg.account_group_id)
|
||||
filter (where kg.account_group_id is not null), '{}') as group_ids,
|
||||
k.max_concurrency, k.requests_per_minute, k.provider_request_profiles_json
|
||||
k.max_concurrency, k.requests_per_minute, k.provider_request_profiles_json, k.name
|
||||
from client_api_keys k
|
||||
left join client_api_key_groups kg on kg.client_api_key_id = k.id
|
||||
where k.enabled
|
||||
@@ -323,6 +325,7 @@ async fn load_client_keys(
|
||||
.map(|row| {
|
||||
let mut key = ClientApiKeySnapshot::from_persisted(row.0, row.1, row.2, row.3, row.4)?;
|
||||
key.request_profiles = decode_request_profiles(row.5.0)?;
|
||||
key.name_snapshot = Some(row.6);
|
||||
Ok(key)
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -35,9 +35,9 @@ fn usage_evidence_predicate(alias: &str) -> String {
|
||||
or {alias}.image_generation_requested
|
||||
or {alias}.input_tokens is not null
|
||||
or {alias}.output_tokens is not null
|
||||
or {alias}.cached_tokens is not null
|
||||
or {alias}.cache_write_tokens is not null
|
||||
or {alias}.reasoning_tokens is not null
|
||||
or {alias}.cached_input_tokens is not null
|
||||
or {alias}.cache_write_input_tokens is not null
|
||||
or {alias}.reasoning_output_tokens is not null
|
||||
or {alias}.image_input_tokens is not null
|
||||
or {alias}.image_output_tokens is not null
|
||||
or {alias}.total_tokens is not null
|
||||
|
||||
@@ -315,3 +315,77 @@ impl HealthProbe for RedisHealthProbe {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 每轮有界推进队列和历史回填;取消时丢弃未提交事务,待下轮重新领取
|
||||
pub(crate) fn request_analytics_worker(
|
||||
repository: postgres::PgRequestAnalyticsRepository,
|
||||
) -> StoreResult<WorkerContribution> {
|
||||
let id = WorkerId::try_new(WorkerKind::RequestAnalytics, "postgres")
|
||||
.map_err(worker_definition_error)?;
|
||||
Ok(WorkerContribution::Registration(scheduled_worker(
|
||||
id,
|
||||
Duration::from_secs(5),
|
||||
Box::new(RequestAnalyticsTask { repository }),
|
||||
)?))
|
||||
}
|
||||
|
||||
struct RequestAnalyticsTask {
|
||||
repository: postgres::PgRequestAnalyticsRepository,
|
||||
}
|
||||
|
||||
impl ScheduledTask for RequestAnalyticsTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => Ok(()),
|
||||
result = run_request_analytics_cycle(&self.repository) => result.map(|_| ()).map_err(|source| {
|
||||
WorkerTaskError::safe("request analytics maintenance failed").with_source(source)
|
||||
}),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 生产与隔离压测共用的单周期推进预算;各批独立提交并归还观测槽位
|
||||
pub async fn run_request_analytics_cycle(
|
||||
repository: &postgres::PgRequestAnalyticsRepository,
|
||||
) -> StoreResult<postgres::AnalyticsMaintenance> {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut total = postgres::AnalyticsMaintenance::default();
|
||||
loop {
|
||||
let result = match tokio::time::timeout_at(deadline, repository.maintain(1_000)).await {
|
||||
Ok(result) => result?,
|
||||
Err(source) => {
|
||||
if total.projected_requests == 0
|
||||
&& total.backfilled_requests == 0
|
||||
&& total.rebuilt_buckets == 0
|
||||
&& total.advanced_buckets == 0
|
||||
&& !total.reset_progress
|
||||
{
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::PostgreSql,
|
||||
message: "request analytics cycle timed out without progress".to_owned(),
|
||||
source: Some(gateway_core::error::ErrorSource::new(source)),
|
||||
});
|
||||
}
|
||||
return Ok(total);
|
||||
}
|
||||
};
|
||||
let idle = result.projected_requests == 0
|
||||
&& result.backfilled_requests == 0
|
||||
&& result.rebuilt_buckets == 0
|
||||
&& result.advanced_buckets == 0
|
||||
&& !result.reset_progress;
|
||||
total.projected_requests += result.projected_requests;
|
||||
total.backfilled_requests += result.backfilled_requests;
|
||||
total.rebuilt_buckets += result.rebuilt_buckets;
|
||||
total.advanced_buckets += result.advanced_buckets;
|
||||
total.reset_progress |= result.reset_progress;
|
||||
if idle || tokio::time::Instant::now() >= deadline {
|
||||
return Ok(total);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -528,9 +528,9 @@ async fn seed_group_cost_snapshot(
|
||||
.collect();
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_ref, operation, client_transport, requested_model_id, provider_kind, provider_account_id, provider_account_ref, upstream_model_id, upstream_transport, attempt_count, upstream_send_state, downstream_committed_at, outcome, client_status_code, upstream_status_code, total_tokens, cost_source, cost_amount, cost_currency, started_at, deadline_at, completed_at, request_observation_json
|
||||
id, client_api_key_ref, operation, client_transport, requested_model_id, provider_kind, provider_account_ref, upstream_model_id, upstream_transport, attempt_count, upstream_send_state, downstream_committed_at, outcome, client_status_code, upstream_status_code, total_tokens, cost_source, cost_amount, cost_currency, started_at, recovery_lease_expires_at, completed_at, request_observation_json
|
||||
) values (
|
||||
$1, 'key-group-history', 'responses', 'http_sse', 'gpt-group', 'openai', $2, $2, 'gpt-group', 'http_sse', 1, 'sent', now(), 'succeeded', 200, 200, 10, 'provider_reported', $4::numeric, 'USD', now() - interval '1 minute', now() + interval '5 minutes', now(),
|
||||
$1, 'key-group-history', 'responses', 'http_sse', 'gpt-group', 'openai', $2, 'gpt-group', 'http_sse', 1, 'sent', now(), 'succeeded', 200, 200, 10, 'provider_reported', $4::numeric, 'USD', now() - interval '1 minute', now() + interval '5 minutes', now(),
|
||||
jsonb_strip_nulls(jsonb_build_object(
|
||||
'request', jsonb_build_object(
|
||||
'configRevision', 1,
|
||||
|
||||
@@ -49,6 +49,18 @@ async fn recovery_loads_precise_window_and_running_request_facts() {
|
||||
"running",
|
||||
)
|
||||
.await;
|
||||
seed_request(
|
||||
&database.pool,
|
||||
"old-missing-lease",
|
||||
now - Duration::seconds(121),
|
||||
now + Duration::seconds(50),
|
||||
"running",
|
||||
)
|
||||
.await;
|
||||
sqlx::query("delete from model_request_leases where request_id = 'old-missing-lease'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("remove missing lease fixture");
|
||||
|
||||
let repository = PgClientAdmissionRecoveryRepository::new(database.pool.clone());
|
||||
let actual = repository
|
||||
@@ -70,11 +82,11 @@ async fn recovery_loads_precise_window_and_running_request_facts() {
|
||||
running_requests: vec![
|
||||
ClientAdmissionRunningRequest {
|
||||
model_request_id: "old-running".to_owned(),
|
||||
deadline_at: now + Duration::seconds(30),
|
||||
lease_expires_at: now + Duration::seconds(30),
|
||||
},
|
||||
ClientAdmissionRunningRequest {
|
||||
model_request_id: "recent-running".to_owned(),
|
||||
deadline_at: now + Duration::seconds(40),
|
||||
lease_expires_at: now + Duration::seconds(40),
|
||||
},
|
||||
],
|
||||
}];
|
||||
@@ -87,13 +99,13 @@ async fn seed_request(
|
||||
pool: &PgPool,
|
||||
id: &str,
|
||||
started_at: DateTime<Utc>,
|
||||
deadline_at: DateTime<Utc>,
|
||||
lease_expires_at: DateTime<Utc>,
|
||||
outcome: &str,
|
||||
) {
|
||||
let completed_at = (outcome != "running").then_some(started_at + Duration::seconds(1));
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_ref, operation, client_transport, requested_model_id, outcome, started_at, deadline_at, completed_at, request_observation_json
|
||||
id, client_api_key_ref, operation, client_transport, requested_model_id, outcome, started_at, recovery_lease_expires_at, completed_at, request_observation_json
|
||||
) values (
|
||||
$1, 'key-recovery', 'responses', 'http_sse', 'coding', $2, $3, $4, $5,
|
||||
jsonb_strip_nulls(jsonb_build_object(
|
||||
@@ -111,9 +123,19 @@ async fn seed_request(
|
||||
.bind(id)
|
||||
.bind(outcome)
|
||||
.bind(started_at)
|
||||
.bind(deadline_at)
|
||||
.bind((outcome != "running").then_some(lease_expires_at))
|
||||
.bind(completed_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed model request recovery fact");
|
||||
if outcome == "running" {
|
||||
sqlx::query(
|
||||
"insert into model_request_leases (request_id, lease_expires_at) values ($1, $2)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(lease_expires_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed running lease");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,176 @@
|
||||
//! 小时重建合并、分段快照失效与阻塞桶调度的真实数据库验收
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{ObservabilityRange, PgRequestAnalyticsRepository};
|
||||
|
||||
use super::{
|
||||
TestDatabase, analytics::seed, drain_analytics, observability_query_budget,
|
||||
observability_repository,
|
||||
};
|
||||
|
||||
fn hour() -> DateTime<Utc> {
|
||||
DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
async fn seed_hour(pool: &sqlx::PgPool, start: DateTime<Utc>, prefix: &str) {
|
||||
for quarter in 0..4 {
|
||||
for request in 0..2 {
|
||||
let id = format!("{prefix}_{quarter}_{request}");
|
||||
seed(
|
||||
pool,
|
||||
&id,
|
||||
start + TimeDelta::minutes(quarter * 15 + request + 1),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("update model_requests set provider_account_ref=$1 where id=$1")
|
||||
.bind(id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dirty_quarters_share_one_hour_rebuild_and_acknowledge_each_bucket() {
|
||||
let Some(database) = TestDatabase::create("analytics_coalesced_hour").await else {
|
||||
return;
|
||||
};
|
||||
sqlx::raw_sql("create table hourly_build_audit (bucket_start timestamptz); create function audit_hourly_build() returns trigger language plpgsql as $$begin if new.collection='global' then insert into hourly_build_audit values(new.bucket_start); end if; return new; end$$; create trigger audit_hourly_build after insert on request_analytics_hourly for each row execute function audit_hourly_build()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
seed_hour(&database.pool, hour(), "coalesced").await;
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
let mut acknowledged = 0;
|
||||
for _ in 0..30 {
|
||||
acknowledged += worker.maintain(1000).await.unwrap().rebuilt_buckets;
|
||||
let status = worker.status().await.unwrap();
|
||||
if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(worker.status().await.unwrap().pending_buckets, 0);
|
||||
assert_eq!(acknowledged, 4);
|
||||
let builds: i64 = sqlx::query_scalar("select count(*) from hourly_build_audit")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
builds, 1,
|
||||
"one shared hour must not be rebuilt once per dirty quarter"
|
||||
);
|
||||
let result = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.requests.request_count, 8);
|
||||
assert_eq!(result.requests.total_tokens, 960);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuilt() {
|
||||
let Some(database) = TestDatabase::create("analytics_hour_changed_input").await else {
|
||||
return;
|
||||
};
|
||||
seed_hour(&database.pool, hour(), "changed").await;
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::query("update model_requests set total_tokens=130")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
let mut owner = None;
|
||||
for _ in 0..100 {
|
||||
worker.maintain(2).await.unwrap();
|
||||
owner = sqlx::query_scalar::<_, DateTime<Utc>>("select bucket_start from request_analytics_pending_buckets where build_phase=4 and build_started_at is not null")
|
||||
.fetch_optional(&database.pool).await.unwrap();
|
||||
if owner.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let owner = owner.expect("hour must persist a partial merge");
|
||||
let changed: String = sqlx::query_scalar("select id from model_requests where date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC') <> $1 order by id limit 1")
|
||||
.bind(owner).fetch_one(&database.pool).await.unwrap();
|
||||
sqlx::query("update model_requests set total_tokens=9999 where id=$1")
|
||||
.bind(changed)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
worker.maintain(2).await.unwrap();
|
||||
let phase: i16 = sqlx::query_scalar(
|
||||
"select build_phase from request_analytics_pending_buckets where bucket_start=$1",
|
||||
)
|
||||
.bind(owner)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
phase, 2,
|
||||
"a changed input invalidates the staged hour before quarter rebuilding"
|
||||
);
|
||||
let range = ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap();
|
||||
let result = observability_repository(&database.pool)
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.requests.request_count, 8);
|
||||
assert_eq!(result.requests.total_tokens, 7 * 130 + 9999);
|
||||
// 每轮换实例验证所有进度均保存在数据库,小时片段不能依赖进程内快照
|
||||
for _ in 0..150 {
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget())
|
||||
.maintain(2)
|
||||
.await
|
||||
.unwrap();
|
||||
if worker.status().await.unwrap().pending_buckets == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(worker.status().await.unwrap().pending_buckets, 0);
|
||||
let hour_total: i64 = sqlx::query_scalar("select (counts->>'total_tokens')::bigint from request_analytics_hourly where collection='global'")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(hour_total, 7 * 130 + 9999);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn waiting_hours_cannot_fill_all_slots_and_starve_their_unbuilt_quarters() {
|
||||
let Some(database) = TestDatabase::create("analytics_hour_waiting_slots").await else {
|
||||
return;
|
||||
};
|
||||
for index in 0..9 {
|
||||
for quarter in 0..2 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("waiting_{index}_{quarter}"),
|
||||
hour() + TimeDelta::hours(index) + TimeDelta::minutes(quarter * 15 + 1),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
// 九个更早的等待项超过单批八个槽位,各自依赖排在后面的 quarter
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),case when extract(minute from started_at)<15 then now()-interval '1 hour' else now() end from request_analytics_facts")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
sqlx::query("update request_analytics_pending_buckets set build_phase=2,build_version=change_version where extract(minute from bucket_start)=0")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let result = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(9)).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.requests.request_count, 18);
|
||||
assert_eq!(result.requests.total_tokens, 18 * 120);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! 统计版本清理的主键进度、事务回滚、表重写与进程重启验收
|
||||
|
||||
use super::{
|
||||
TestDatabase, analytics::seed, drain_analytics, observability_query_budget,
|
||||
observability_repository,
|
||||
};
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_store::postgres::{ObservabilityRange, PgRequestAnalyticsRepository};
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_reset_persists_primary_keys_atomically_and_resumes_after_table_rewrite() {
|
||||
let Some(database) = TestDatabase::create("analytics_reset_cursor").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
let start = now - TimeDelta::hours(1);
|
||||
for index in 0..5 {
|
||||
seed(&database.pool, &format!("reset_{index:02}"), start, &[]).await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
// 旧格式内容只能被清理,复合主键在同一时间桶内也需要持续前进
|
||||
for table in ["request_analytics_15m", "request_analytics_hourly"] {
|
||||
sqlx::query(sqlx::AssertSqlSafe(format!("insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) select $1,'obsolete',array['key_'||i],outcome,0,0,'{{}}','{{}}','{{}}' from generate_series(1,3)i cross join unnest(array['failed','succeeded'])outcome")))
|
||||
.bind(start).execute(&database.pool).await.unwrap();
|
||||
}
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select $1 + i * interval '1 day',$1 from generate_series(1,5)i")
|
||||
.bind(start).execute(&database.pool).await.unwrap();
|
||||
sqlx::query("update request_analytics_build_state set schema_version=0,histogram_version=0")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 游标保存失败必须回滚同批删除,不能跳过尚未成功清理的行
|
||||
sqlx::raw_sql("create function reject_reset_cursor() returns trigger language plpgsql as $$ begin if new.reset_cursor is not null then raise exception 'test reset progress failure'; end if; return new; end $$; create trigger reject_reset_cursor before update on request_analytics_build_state for each row execute function reject_reset_cursor();")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
assert!(worker.maintain(2).await.is_err());
|
||||
let unchanged: (i64, i16, bool) = sqlx::query_as("select (select count(*) from request_analytics_facts),reset_phase,reset_cursor is null from request_analytics_build_state")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(unchanged, (5, 0, true));
|
||||
sqlx::raw_sql("drop trigger reject_reset_cursor on request_analytics_build_state; drop function reject_reset_cursor();")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
assert!(worker.maintain(2).await.unwrap().reset_progress);
|
||||
let cursor: serde_json::Value =
|
||||
sqlx::query_scalar("select reset_cursor from request_analytics_build_state")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
cursor,
|
||||
serde_json::json!({"kind":"fact","key":{"id":"reset_01"}})
|
||||
);
|
||||
drop(worker);
|
||||
// 物理位置变化不改变跨事务主键进度,新增源记录即使 ID 落在游标之前也会重新回填
|
||||
sqlx::query("vacuum full request_analytics_facts")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
seed(&database.pool, "reset_00_new", start, &[]).await;
|
||||
sqlx::query("update model_requests set total_tokens=240 where id='reset_04'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut seen_phases = std::collections::BTreeSet::from([1_i16]);
|
||||
let mut completed = false;
|
||||
for _ in 0..100 {
|
||||
let before: (i16, Option<serde_json::Value>) =
|
||||
sqlx::query_as("select reset_phase,reset_cursor from request_analytics_build_state")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let table = match before.0 {
|
||||
1 => "request_analytics_facts",
|
||||
2 => "request_analytics_15m",
|
||||
3 => "request_analytics_hourly",
|
||||
4 => "request_analytics_pending_buckets",
|
||||
_ => panic!("reset ended before final verification"),
|
||||
};
|
||||
let remaining: i64 =
|
||||
sqlx::query_scalar(sqlx::AssertSqlSafe(format!("select count(*) from {table}")))
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 每轮创建新实例,所有推进事实必须来自数据库而非 Worker 内存
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
assert!(worker.maintain(2).await.unwrap().reset_progress);
|
||||
let after: (i16, Option<serde_json::Value>) =
|
||||
sqlx::query_as("select reset_phase,reset_cursor from request_analytics_build_state")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let left: i64 =
|
||||
sqlx::query_scalar(sqlx::AssertSqlSafe(format!("select count(*) from {table}")))
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining - left, remaining.min(2));
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from request_analytics_pending_requests")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
2
|
||||
);
|
||||
if before.0 == after.0 {
|
||||
assert!(after.1.is_some());
|
||||
assert_ne!(before.1, after.1);
|
||||
seen_phases.insert(before.0);
|
||||
} else {
|
||||
assert!(after.1.is_none());
|
||||
}
|
||||
if after.0 == 0 {
|
||||
completed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(completed);
|
||||
assert_eq!(seen_phases, std::collections::BTreeSet::from([1, 2, 3, 4]));
|
||||
drain_analytics(&database.pool).await;
|
||||
let repo = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(start - TimeDelta::minutes(1), now).unwrap();
|
||||
let result = repo.usage_summary(range, Default::default()).await.unwrap();
|
||||
assert_eq!(result.requests.request_count, 6);
|
||||
assert_eq!(result.requests.total_tokens, 840);
|
||||
assert_eq!(result.analytics.schema_version, 1);
|
||||
assert_eq!(result.analytics.histogram_version, 1);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -76,6 +76,107 @@ fn context() -> MutationContext {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_budget_windows_only_write_for_new_charges() {
|
||||
let Some(database) = TestDatabase::create("budget_window_writes").await else {
|
||||
return;
|
||||
};
|
||||
sqlx::raw_sql(
|
||||
"create table budget_window_writes (operation text not null);
|
||||
create function record_budget_window_write() returns trigger language plpgsql as $$
|
||||
begin
|
||||
insert into budget_window_writes (operation) values (tg_op);
|
||||
return new;
|
||||
end $$;
|
||||
create trigger record_budget_window_write after insert or update
|
||||
on client_key_budget_windows for each row
|
||||
execute function record_budget_window_write();",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let first = PgClientBudgetStore::new(database.pool.clone());
|
||||
let second = PgClientBudgetStore::new(database.pool.clone());
|
||||
for (key, limit) in [("unlimited", "0"), ("limited", "10")] {
|
||||
seed(&database, key, limit, limit).await;
|
||||
first.admit(key_id(key)).await.unwrap();
|
||||
for _ in 0..4 {
|
||||
let (a, b) = tokio::join!(first.admit(key_id(key)), second.admit(key_id(key)));
|
||||
a.unwrap();
|
||||
b.unwrap();
|
||||
}
|
||||
let billed = charge(key, key, "0.25");
|
||||
let (a, b) = tokio::join!(first.settle(billed.clone()), second.settle(billed));
|
||||
a.unwrap();
|
||||
b.unwrap();
|
||||
let budget = status(&database, key).await;
|
||||
assert_eq!(budget.daily_used_usd.canonical(), "0.25");
|
||||
assert_eq!(budget.weekly_used_usd.canonical(), "0.25");
|
||||
}
|
||||
let writes: (i64, i64) = sqlx::query_as(
|
||||
"select count(*) filter (where operation = 'INSERT'),
|
||||
count(*) filter (where operation = 'UPDATE') from budget_window_writes",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
writes,
|
||||
(2, 2),
|
||||
"only creation and amount accumulation write rows"
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn indexed_charge_cleanup_keeps_other_keys_and_rolls_back_with_key_deletion() {
|
||||
let Some(database) = TestDatabase::create("budget_key_cleanup").await else {
|
||||
return;
|
||||
};
|
||||
let store = PgClientBudgetStore::new(database.pool.clone());
|
||||
for key in ["deleted", "retained"] {
|
||||
seed(&database, key, "0", "0").await;
|
||||
store.settle(charge(key, key, "1.25")).await.unwrap();
|
||||
}
|
||||
let mut tx = database.pool.begin().await.unwrap();
|
||||
sqlx::query("delete from client_api_keys where id = 'deleted'")
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.unwrap();
|
||||
let remaining: Vec<String> = sqlx::query_scalar(
|
||||
"select client_api_key_id from client_key_charge_events order by client_api_key_id",
|
||||
)
|
||||
.fetch_all(&mut *tx)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, ["retained"]);
|
||||
tx.rollback().await.unwrap();
|
||||
let restored: i64 = sqlx::query_scalar("select count(*) from client_key_charge_events")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored, 2);
|
||||
sqlx::query("delete from client_api_keys where id = 'deleted'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
status(&database, "retained")
|
||||
.await
|
||||
.daily_used_usd
|
||||
.canonical(),
|
||||
"1.25"
|
||||
);
|
||||
let remaining: Vec<String> = sqlx::query_scalar(
|
||||
"select client_api_key_id from client_key_charge_events order by client_api_key_id",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, ["retained"]);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unopened_future_budget_windows_do_not_report_usage_or_reset_times() {
|
||||
let Some(database) = TestDatabase::create("budget_unopened_future").await else {
|
||||
|
||||
@@ -243,6 +243,7 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
"client_api_keys",
|
||||
"client_key_budget_windows",
|
||||
"client_key_charge_events",
|
||||
"model_request_leases",
|
||||
"model_request_observations",
|
||||
"model_requests",
|
||||
"ops_events",
|
||||
@@ -260,6 +261,13 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
"plugin_update_sources",
|
||||
"plugin_version_configurations",
|
||||
"provider_accounts",
|
||||
"request_analytics_15m",
|
||||
"request_analytics_build_state",
|
||||
"request_analytics_daily_counts",
|
||||
"request_analytics_facts",
|
||||
"request_analytics_hourly",
|
||||
"request_analytics_pending_buckets",
|
||||
"request_analytics_pending_requests",
|
||||
"runtime_settings",
|
||||
]
|
||||
);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user