diff --git a/backend/crates/gateway-admin/src/model/account_groups.rs b/backend/crates/gateway-admin/src/model/account_groups.rs index 085df9fc1..3743d7b31 100644 --- a/backend/crates/gateway-admin/src/model/account_groups.rs +++ b/backend/crates/gateway-admin/src/model/account_groups.rs @@ -66,7 +66,7 @@ pub struct AccountGroupCapacity { #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountGroupUsage { pub today_usd: DecimalAmount, - /// 当前 usage retention 窗口内、按请求发生时 routing group 快照归属的累计成本 + /// 当前 usage retention 窗口内、当前组成员实际完成请求的累计成本 pub retained_total_usd: DecimalAmount, } diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index 556d549f4..3d6ee4e3b 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -157,7 +157,6 @@ impl OtherRequestOutcome { /// 用量记录过滤条件 #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct UsageFilter { - pub routing_group_refs: Vec, pub client_api_key_ref: Option, pub request_id: Option, pub provider_account_ref: Option, @@ -211,7 +210,6 @@ impl DashboardQuery { /// 运维错误过滤条件 #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OpsErrorFilter { - pub routing_group_refs: Vec, pub client_api_key_ref: Option, pub request_id: Option, pub provider_kind: Option, diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index eee3f7f47..4476a1ef0 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -432,6 +432,9 @@ pub trait UsageBillingProjector: Send + Sync { /// 用量、趋势、诊断与运维错误的只读能力 #[async_trait] pub trait ObservabilityStore: Send + Sync { + /// 读取实现声明的总预算，缓存等待沿用同一配置，避免外层提前截断 + fn query_timeout(&self) -> std::time::Duration; + /// 缓存必须与删除和后台投影提交的版本一致 async fn analytics_cache_generation(&self) -> AdminStoreResult; diff --git a/backend/crates/gateway-admin/src/use_case/observability/store_cache.rs b/backend/crates/gateway-admin/src/use_case/observability/store_cache.rs index eb55ba302..cebe2f15a 100644 --- a/backend/crates/gateway-admin/src/use_case/observability/store_cache.rs +++ b/backend/crates/gateway-admin/src/use_case/observability/store_cache.rs @@ -12,7 +12,6 @@ use tokio::{sync::Mutex, time::Instant}; const CAPACITY: usize = 64; const TTL: Duration = Duration::from_secs(15); const MAX_STALE: Duration = Duration::from_secs(60); -const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); type Slot = Arc>>>; struct Cached { @@ -72,7 +71,7 @@ impl Cache { { // 刷新失败后的同键等待也计入预算，避免串行重试把延迟放大为队列长度倍数 tokio::time::timeout( - REQUEST_TIMEOUT, + store.query_timeout(), self.get_within_budget(key, store, ttl, load), ) .await @@ -174,12 +173,6 @@ fn with_age(mut value: T, stale: bool) -> T { value } -fn normalized(mut filter: UsageFilter) -> UsageFilter { - filter.routing_group_refs.sort_unstable(); - filter.routing_group_refs.dedup(); - filter -} - type UsageKey = (TimeRange, UsageFilter); pub(crate) struct CachedObservabilityStore { inner: Arc, @@ -202,6 +195,10 @@ impl CachedObservabilityStore { #[async_trait] impl ObservabilityStore for CachedObservabilityStore { + fn query_timeout(&self) -> Duration { + self.inner.query_timeout() + } + async fn analytics_cache_generation(&self) -> AdminStoreResult { self.inner.analytics_cache_generation().await } @@ -241,7 +238,6 @@ impl ObservabilityStore for CachedObservabilityStore { filter: UsageFilter, granularity: Granularity, ) -> AdminStoreResult { - let filter = normalized(filter); self.insights .get( ((range, filter.clone()), granularity), @@ -259,7 +255,6 @@ impl ObservabilityStore for CachedObservabilityStore { range: TimeRange, filter: UsageFilter, ) -> AdminStoreResult { - let filter = normalized(filter); self.summaries .get((range, filter.clone()), self.inner.as_ref(), TTL, || { self.inner.usage_summary(range, filter) @@ -273,7 +268,6 @@ impl ObservabilityStore for CachedObservabilityStore { dimension: DiagnosticDimension, limit: u16, ) -> AdminStoreResult { - let filter = normalized(filter); self.diagnostics .get( ((range, filter.clone()), dimension, limit), diff --git a/backend/crates/gateway-admin/tests/use_case/mod.rs b/backend/crates/gateway-admin/tests/use_case/mod.rs index 17be7fe75..3a6801ec6 100644 --- a/backend/crates/gateway-admin/tests/use_case/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/mod.rs @@ -747,6 +747,10 @@ impl ClientKeyStore for UnavailableStore { #[async_trait] impl ObservabilityStore for UnavailableStore { + fn query_timeout(&self) -> std::time::Duration { + std::time::Duration::from_secs(10) + } + async fn dashboard_summary( &self, _: DashboardQuery, diff --git a/backend/crates/gateway-admin/tests/use_case/observability/mod.rs b/backend/crates/gateway-admin/tests/use_case/observability/mod.rs index 59fb44959..e66b14ff4 100644 --- a/backend/crates/gateway-admin/tests/use_case/observability/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/observability/mod.rs @@ -945,6 +945,10 @@ impl FixtureObservabilityStore { #[async_trait] impl ObservabilityStore for FixtureObservabilityStore { + fn query_timeout(&self) -> std::time::Duration { + std::time::Duration::from_secs(10) + } + async fn dashboard_summary( &self, query: DashboardQuery, @@ -1406,23 +1410,23 @@ async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the } #[tokio::test] -async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_sets() { +async fn usage_summary_cache_should_isolate_key_identity_and_filters() { let range = observation_range(Utc::now()); let store = Arc::new(FixtureObservabilityStore::new(range)); let services = observability_services(store.clone()).await; let service = services.observability(); - for (key, groups) in [ - ("key-a", vec!["deleted-b", "deleted-a", "deleted-b"]), - ("key-a", vec!["deleted-a", "deleted-b"]), - ("key-b", vec!["deleted-a", "deleted-b"]), - ("key-a", vec!["deleted-a"]), + for (key, model) in [ + ("key-a", None), + ("key-a", None), + ("key-b", None), + ("key-a", Some("model-a")), ] { service .usage_summary( range, UsageFilter { client_api_key_ref: Some(key.to_owned()), - routing_group_refs: groups.into_iter().map(str::to_owned).collect(), + model: model.map(str::to_owned), ..Default::default() }, ) @@ -1431,16 +1435,16 @@ async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_set } let calls = store.usage_summary_filters.lock().unwrap(); assert_eq!(calls.len(), 3); - assert_eq!(calls[0].routing_group_refs, ["deleted-a", "deleted-b"]); + assert_eq!(calls[0].client_api_key_ref.as_deref(), Some("key-a")); assert_eq!(calls[1].client_api_key_ref.as_deref(), Some("key-b")); - assert_eq!(calls[2].routing_group_refs, ["deleted-a"]); + assert_eq!(calls[2].model.as_deref(), Some("model-a")); } #[tokio::test(start_paused = true)] async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow_recovery() { let range = observation_range(Utc::now()); let store = Arc::new(FixtureObservabilityStore::new(range)); - store.set_dashboard_delay(StdDuration::from_secs(7)); + store.set_dashboard_delay(store.query_timeout().mul_f64(0.7)); store.summary_fail.store(true, Ordering::SeqCst); let services = observability_services(store.clone()).await; let service = services.observability(); @@ -1453,7 +1457,7 @@ async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow ); assert!(first.is_err() && second.is_err() && third.is_err() && fourth.is_err()); assert!( - started.elapsed() <= StdDuration::from_secs(10), + started.elapsed() <= store.query_timeout(), "同键失败不能使后续请求逐个等待完整数据库超时: {:?}", started.elapsed() ); diff --git a/backend/crates/gateway-api/src/admin/observability/query.rs b/backend/crates/gateway-api/src/admin/observability/query.rs index ea2e965b3..ed51154bd 100644 --- a/backend/crates/gateway-api/src/admin/observability/query.rs +++ b/backend/crates/gateway-api/src/admin/observability/query.rs @@ -29,7 +29,6 @@ impl DashboardQuery { #[derive(Clone, Default, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct UsageQuery { - pub routing_group_refs: Option, pub current_page: Option, pub page_size: Option, pub kind: Option, @@ -77,7 +76,6 @@ impl DetailQuery { #[derive(Clone, Default, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct DiagnosticsQuery { - pub routing_group_refs: Option, pub dimension: Option, pub period: Option, pub as_of: Option, @@ -86,7 +84,6 @@ pub struct DiagnosticsQuery { pub provider: Option, pub model: Option, pub status_code: Option, - pub search: Option, } impl DiagnosticsQuery { @@ -100,7 +97,6 @@ impl DiagnosticsQuery { #[derive(Clone, Default, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct OpsQuery { - pub routing_group_refs: Option, pub current_page: Option, pub page_size: Option, pub kind: Option, @@ -301,7 +297,6 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result Result Result { + if query.search.is_some() { + return Err(WireValidationError::new("search")); + } + usage_filter(query) +} + pub(crate) fn usage_command( query: &UsageQuery, timezone: gateway_core::time::DeploymentTimeZone, @@ -361,7 +365,6 @@ pub(crate) fn ops_command( timezone, )?, filter: domain::OpsErrorFilter { - routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?, client_api_key_ref: non_empty(query.client_api_key_id.clone()), request_id: non_empty(query.request_id.clone()), provider_kind: non_empty(query.provider.clone()), @@ -400,25 +403,3 @@ fn validate_pagination( } Ok((current_page, page_size)) } - -/// 逗号分隔的稳定分组身份按集合归一化，不要求分组当前仍存在 -pub(crate) fn routing_group_refs(value: Option<&str>) -> Result, WireValidationError> { - let Some(value) = value else { - return Ok(Vec::new()); - }; - let mut refs = Vec::new(); - for item in value.split(',') { - let item = item.trim(); - if item.is_empty() - || item.len() > 128 - || item.chars().any(char::is_control) - || refs.len() >= 32 - { - return Err(WireValidationError::new("routingGroupRefs")); - } - refs.push(item.to_owned()); - } - refs.sort(); - refs.dedup(); - Ok(refs) -} diff --git a/backend/crates/gateway-api/src/admin/observability/routes.rs b/backend/crates/gateway-api/src/admin/observability/routes.rs index 05f46e586..1a88f5076 100644 --- a/backend/crates/gateway-api/src/admin/observability/routes.rs +++ b/backend/crates/gateway-api/src/admin/observability/routes.rs @@ -151,7 +151,7 @@ where state.admin_services().timezone(), ) .map_err(map_wire_error)?; - let filter = usage_filter(&query).map_err(map_wire_error)?; + let filter = analytics_filter(&query).map_err(map_wire_error)?; let result = state .admin_services() .observability() @@ -184,7 +184,7 @@ where state.admin_services().timezone(), ) .map_err(map_wire_error)?; - let filter = usage_filter(&query).map_err(map_wire_error)?; + let filter = analytics_filter(&query).map_err(map_wire_error)?; let result = state .admin_services() .observability() @@ -215,12 +215,9 @@ where ) .map_err(map_wire_error)?; let filter = domain::UsageFilter { - routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref()) - .map_err(map_wire_error)?, provider_kind: non_empty(query.provider), model: non_empty(query.model), status_code: parse_status(query.status_code).map_err(map_wire_error)?, - search: non_empty(query.search), ..domain::UsageFilter::default() }; let result = state diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index ddc6c6964..10357768f 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -1374,6 +1374,10 @@ impl AccountRuntimeStore for UnusedStore { #[async_trait] impl ObservabilityStore for UnusedStore { + fn query_timeout(&self) -> std::time::Duration { + std::time::Duration::from_secs(10) + } + async fn dashboard_summary( &self, query: DashboardQuery, diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index 9f4f30b39..2eb1eb001 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -1245,7 +1245,7 @@ async fn calendar_query_rejects_an_anchor_outside_the_local_calendar() { } #[tokio::test] -async fn request_and_ops_queries_should_forward_deleted_group_id_sets() { +async fn observability_queries_reject_group_filters_and_analytics_search() { use crate::admin::{AdminTestFixture, AdminTestState}; use axum::{ body::Body, @@ -1256,39 +1256,50 @@ async fn request_and_ops_queries_should_forward_deleted_group_id_sets() { let fixture = AdminTestFixture::new().await; fixture.auth.insert_session("valid-session"); let router = observability::router::().with_state(fixture.state()); - for path in ["/api/admin/usage/records", "/api/admin/operations/errors"] { + for (path, parameter) in [ + ("/api/admin/usage/records", "routingGroupRefs=group-a"), + ("/api/admin/operations/errors", "routingGroupRefs=group-a"), + ( + "/api/admin/usage/records/summary", + "routingGroupRefs=group-a", + ), + ( + "/api/admin/usage/insights/overview", + "routingGroupRefs=group-a", + ), + ( + "/api/admin/usage/insights/diagnostics", + "routingGroupRefs=group-a", + ), + ("/api/admin/usage/records/summary", "search=request-prefix"), + ( + "/api/admin/usage/insights/overview", + "search=request-prefix", + ), + ( + "/api/admin/usage/insights/diagnostics", + "search=request-prefix", + ), + ] { let response = router .clone() .oneshot( Request::builder() - .uri(format!( - "{path}?routingGroupRefs=deleted-b,deleted-a,deleted-b" - )) + .uri(format!("{path}?{parameter}")) .header(header::COOKIE, "cpr_session=valid-session") - .header("x-request-id", "req_group_filter") + .header("x-request-id", "req_rejected_observability_filter") .body(Body::empty()) .unwrap(), ) .await .unwrap(); - let status = response.status(); - let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) - .await - .unwrap(); assert_eq!( - status, - StatusCode::OK, - "{path}: {}", - String::from_utf8_lossy(&body) + response.status(), + StatusCode::BAD_REQUEST, + "{path}?{parameter}" ); } let observations = fixture.observations.lock().unwrap(); - assert_eq!( - observations.records[0].filter.routing_group_refs, - ["deleted-a", "deleted-b"] - ); - assert_eq!( - observations.errors[0].filter.routing_group_refs, - ["deleted-a", "deleted-b"] - ); + assert!(observations.records.is_empty()); + assert!(observations.errors.is_empty()); } diff --git a/backend/crates/gateway-store/src/bundle.rs b/backend/crates/gateway-store/src/bundle.rs index bb0f04a87..301ccf75a 100644 --- a/backend/crates/gateway-store/src/bundle.rs +++ b/backend/crates/gateway-store/src/bundle.rs @@ -126,6 +126,7 @@ async fn connect( let observability_query_budget = postgres::ObservabilityQueryBudget::try_new( config.pool.observability_max_connections(), config.pool.acquire_timeout(), + config.pool.observability_timeout(), )?; let redis_client = ::redis::Client::open(config.redis_url()?) .map_err(|source| redis_unavailable("create Redis client", source))?; @@ -180,8 +181,11 @@ async fn connect( )), admin_account_runtime, Arc::new( - postgres::PgAccountGroupRepository::new(pool.clone()) - .with_timezone(config.timezone), + postgres::PgAccountGroupRepository::new( + pool.clone(), + observability_query_budget.clone(), + ) + .with_timezone(config.timezone), ), Arc::new(postgres::PgProxyRepository::new(pool.clone())), ), diff --git a/backend/crates/gateway-store/src/config.rs b/backend/crates/gateway-store/src/config.rs index fffb15284..1dfd46326 100644 --- a/backend/crates/gateway-store/src/config.rs +++ b/backend/crates/gateway-store/src/config.rs @@ -36,6 +36,7 @@ pub struct StoreConfig { pub struct StorePoolConfig { pub max_connections: u32, pub acquire_timeout_seconds: u64, + pub observability_timeout_seconds: u64, } impl Default for StorePoolConfig { @@ -43,6 +44,7 @@ impl Default for StorePoolConfig { Self { max_connections: 20, acquire_timeout_seconds: 5, + observability_timeout_seconds: 120, } } } @@ -58,9 +60,22 @@ impl StorePoolConfig { .to_owned(), }); } + if self.observability_timeout_seconds == 0 + || self.observability_timeout().as_millis() > i32::MAX as u128 + { + return Err(StoreError::InvalidData { + source: None, + entity: "store config", + message: "pool.observability_timeout_seconds must be positive and fit PostgreSQL statement_timeout".to_owned(), + }); + } Ok(()) } + pub(crate) const fn observability_timeout(self) -> Duration { + Duration::from_secs(self.observability_timeout_seconds) + } + /// 管理观测查询可并发占用的连接数；始终为数据面保留约 20% 的池容量 #[must_use] pub const fn observability_max_connections(self) -> u32 { diff --git a/backend/crates/gateway-store/src/postgres/account_groups.rs b/backend/crates/gateway-store/src/postgres/account_groups.rs index c48cab3ab..bc4d88aa9 100644 --- a/backend/crates/gateway-store/src/postgres/account_groups.rs +++ b/backend/crates/gateway-store/src/postgres/account_groups.rs @@ -31,9 +31,9 @@ use crate::{ }; use super::{ - PgRuntimeSettingsRepository, RuntimeSettingsRepository, account_summary_from_row, - append_admin_audit_event_in_transaction, bump_config_revision_in_transaction, - completed_usage_fact_predicate, + ObservabilityQueryBudget, PgRuntimeSettingsRepository, RuntimeSettingsRepository, + account_summary_from_row, append_admin_audit_event_in_transaction, + bump_config_revision_in_transaction, }; const ENTITY: &str = "account group"; @@ -43,13 +43,15 @@ const ENTITY: &str = "account group"; pub struct PgAccountGroupRepository { timezone: gateway_core::time::DeploymentTimeZone, pool: PgPool, + query_budget: ObservabilityQueryBudget, } impl PgAccountGroupRepository { #[must_use] - pub fn new(pool: PgPool) -> Self { + pub fn new(pool: PgPool, query_budget: ObservabilityQueryBudget) -> Self { Self { pool, + query_budget, timezone: Default::default(), } } @@ -74,9 +76,14 @@ impl PgAccountGroupRepository { .await .map_err(|error| admin_store_error(ENTITY, error))? .ok_or_else(|| not_found(id.as_str()))?; - let costs = group_costs(&self.pool, &[id.as_str().to_owned()], self.timezone) - .await - .map_err(|error| admin_store_error(ENTITY, error))?; + let costs = group_costs( + &self.pool, + &self.query_budget, + &[id.as_str().to_owned()], + self.timezone, + ) + .await + .map_err(|error| admin_store_error(ENTITY, error))?; if let Some(usage) = costs.get(id.as_str()) { record.usage = usage.clone(); } @@ -152,7 +159,7 @@ impl AccountGroupStore for PgAccountGroupRepository { .iter() .map(|record| record.id.as_str().to_owned()) .collect::>(); - let costs = group_costs(&self.pool, &group_ids, self.timezone) + let costs = group_costs(&self.pool, &self.query_budget, &group_ids, self.timezone) .await .map_err(|error| admin_store_error(ENTITY, error))?; for record in &mut items { @@ -601,71 +608,23 @@ fn group_ref(row: &sqlx::postgres::PgRow) -> StoreResult { async fn group_costs( pool: &PgPool, + budget: &ObservabilityQueryBudget, group_ids: &[String], timezone: gateway_core::time::DeploymentTimeZone, ) -> StoreResult> { if group_ids.is_empty() { return Ok(BTreeMap::new()); } - let completed_usage = completed_usage_fact_predicate("mr"); - // 用量按实际完成请求的账号统计：账号属于多个分组时计入每个所属分组， - // 不再按 Client Key 绑定分组快照归属，避免多分组 Key 的费用重复出现在未承接请求的分组上 - let statement = format!( - "with requested_groups(group_id) as ( - select unnest($1::text[]) - ) - select requested_groups.group_id, - coalesce(sum(mr.cost_amount) filter ( - where mr.started_at >= $2 - ), 0)::text as today_usd, - coalesce(sum(mr.cost_amount), 0)::text as retained_total_usd - from requested_groups - cross join runtime_settings settings - left join account_group_accounts membership - on membership.account_group_id = requested_groups.group_id - left join model_requests mr - on mr.provider_account_ref = membership.provider_account_id - and mr.started_at >= now() - make_interval(days => settings.usage_retention_days::int) - and {completed_usage} - and mr.cost_currency = 'USD' - and mr.cost_amount is not null - where settings.id = 1 - group by requested_groups.group_id" - ); - // 动态片段仅为共享的固定 usage-fact predicate；group IDs 仍使用 bind - let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) - .bind(group_ids) - .bind( - timezone - .day_start(chrono::Utc::now()) - .ok_or_else(|| invalid("invalid business date"))?, + budget + .run( + pool, + "load account group costs", + |mut connection| async move { + super::observability::account_group_usage(&mut connection, group_ids, timezone) + .await + }, ) - .fetch_all(pool) .await - .map_err(|source| unavailable("load account group costs", source))?; - rows.into_iter() - .map(|row| { - let group_id: String = row - .try_get("group_id") - .map_err(|source| invalid("invalid group ID").with_source(source))?; - let today = row - .try_get::("today_usd") - .map_err(|source| invalid("invalid today cost").with_source(source))?; - let retained_total = row - .try_get::("retained_total_usd") - .map_err(|source| invalid("invalid retained total cost").with_source(source))?; - Ok(( - group_id, - AccountGroupUsage { - today_usd: DecimalAmount::from_str(&today) - .map_err(|source| invalid("invalid today cost").with_source(source))?, - retained_total_usd: DecimalAmount::from_str(&retained_total).map_err( - |source| invalid("invalid retained total cost").with_source(source), - )?, - }, - )) - }) - .collect() } fn count_value(row: &sqlx::postgres::PgRow, field: &str) -> StoreResult { diff --git a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs index ca6f46c97..baa0e77bb 100644 --- a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs @@ -197,7 +197,7 @@ impl PgObservabilityRepository { ) .await?; let account_usage = - provider_account_usage(&mut snapshot.tx, account_usage_query).await?; + provider_account_usage(&mut snapshot, account_usage_query).await?; let metadata = snapshot.metadata(); snapshot.finish().await?; Ok((totals, trend, metadata, account_usage)) @@ -290,12 +290,8 @@ impl PgObservabilityRepository { "load usage insights snapshot", |mut connection| async move { let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?; - let mut overview = snapshot.overview(range, &filter).await?; - let trend = snapshot - .trend(range, &filter, granularity, self.timezone, true) - .await?; - let costs = snapshot - .projected_costs(range, &filter, granularity, self.timezone) + let (mut overview, trend, costs) = snapshot + .insights(range, &filter, granularity, self.timezone) .await?; overview.analytics = snapshot.metadata(); snapshot.finish().await?; @@ -318,12 +314,9 @@ impl PgObservabilityRepository { &self.pool, "load provider account usage", |mut connection| async move { - let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?; - snapshot.require_visible(Some(query.range)).await?; - if let Some(range) = query.request_bucket_range { - snapshot.require_visible(Some(range)).await?; - } - let usage = provider_account_usage(&mut snapshot.tx, query).await?; + let mut snapshot = + analytics::AnalyticsSnapshot::new_for_accounts(&mut connection).await?; + let usage = provider_account_usage(&mut snapshot, query).await?; snapshot.finish().await?; Ok(usage) }, @@ -407,6 +400,10 @@ impl PgObservabilityRepository { #[async_trait] impl AdminObservabilityStore for PgAdminObservabilityStore { + fn query_timeout(&self) -> std::time::Duration { + self.repository.query_budget.query_timeout() + } + async fn analytics_cache_generation(&self) -> AdminStoreResult { self.repository .query_budget diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs new file mode 100644 index 000000000..10d375ff5 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/accounts.rs @@ -0,0 +1,655 @@ +//! 账号用量复用汇总并补齐未投影源版本，保持同步查询的精确窗口口径 + +use super::query::push_summary_buckets; +use super::*; +use gateway_admin::model::{ + account_groups::AccountGroupUsage, + accounts::{ + AccountCost, AccountModelUsage, AccountUsage, AccountUsageWindowQuery, + AccountUsageWindowResult, + }, +}; + +type AccountStates = BTreeMap<(String, String), MetricState>; + +impl AnalyticsSnapshot<'_> { + pub(crate) async fn account_usage_states( + &mut self, + range: ObservabilityRange, + account_ids: &[String], + ) -> StoreResult { + self.account_states(range, account_ids, true).await + } + + async fn account_states( + &mut self, + range: ObservabilityRange, + account_ids: &[String], + by_model: bool, + ) -> StoreResult { + if account_ids.is_empty() { + return Ok(BTreeMap::new()); + } + // 原同步接口在首次回填期间仍可查询；只有完整覆盖的区间才允许用汇总替代源事实 + if !self.projection_covers(range) { + return self + .account_fact_states("model_requests", range, account_ids, false, by_model) + .await; + } + let mut states = BTreeMap::new(); + // 排序与分组费用只需要账号总量，不额外展开模型维度 + let (collection, model) = if by_model { + ( + "account_model", + "coalesce(nullif(dimensions[4],''),dimensions[3])", + ) + } else { + ("account", "''::text") + }; + let (hours, quarters) = self.summary_buckets(range, true).await?; + for (table, seconds, buckets) in [ + ("request_analytics_hourly", 3600, hours), + ("request_analytics_15m", 900, quarters), + ] { + if buckets.is_empty() { + continue; + } + let mut query = QueryBuilder::::new(format!( + "with selected as materialized (select dimensions[1] as account_id,{model} as model,counts,costs,last_used_at from {table} a where collection='{collection}' and schema_version=" + )); + query + .push_bind(SCHEMA_VERSION) + .push(" and histogram_version=") + .push_bind(HISTOGRAM_VERSION) + .push(" and dimensions[1]=any(") + .push_bind(account_ids) + .push("::text[])"); + push_summary_buckets(&mut query, &buckets, seconds); + let counters = usage_counter_names(); + // 每行一次展开指标，避免对可能位于 TOAST 的同一 JSON 反复解码 + query.push(format!( + "), totals as (select account_id,model,jsonb_build_object({}) as counts,max(last_used_at) as last_used_at from selected cross join lateral jsonb_to_record(counts) as metric({}) group by account_id,model), + currencies as (select account_id,model,c.key,sum(c.value::numeric)::text as amount from selected cross join lateral jsonb_each_text(costs) c where c.key not in ('standard_usd','no_cache_usd') group by account_id,model,c.key), + costs as (select account_id,model,jsonb_object_agg(key,amount) as costs from currencies group by account_id,model) + select totals.*,coalesce(costs.costs,'{{}}'::jsonb) as costs,'{{}}'::jsonb as distributions from totals left join costs using(account_id,model)", + counters.iter().map(|name| format!("'{name}',coalesce(sum(metric.{name}),0)")).collect::>().join(","), + counters.iter().map(|name| format!("{name} numeric")).collect::>().join(",") + )); + let rows = query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| { + postgres_unavailable("aggregate account usage summaries", error) + })?; + merge_account_rows(&mut states, rows)?; + } + for (start, end) in self.uncovered_spans(range).await? { + let boundary = self + .account_fact_states( + "request_analytics_facts", + ObservabilityRange { start, end }, + account_ids, + false, + by_model, + ) + .await?; + merge_account_states(&mut states, boundary)?; + } + if !self.has_pending_requests().await? { + states.retain(|_, state| state.count("completed_request_count") > 0); + return Ok(states); + } + let old = self + .account_fact_states( + "request_analytics_facts", + range, + account_ids, + true, + by_model, + ) + .await?; + let current = self + .account_fact_states("model_requests", range, account_ids, true, by_model) + .await?; + let mut repair = BTreeSet::new(); + for (key, previous) in old { + let state = states.entry(key.clone()).or_default(); + for (name, count) in &previous.counts { + let value = state.counts.entry(name.clone()).or_default(); + *value = value + .checked_sub(*count) + .ok_or_else(|| invalid("account usage pending counter underflow"))?; + } + for (currency, amount) in &previous.costs { + let before = state + .costs + .get(currency) + .ok_or_else(|| invalid("account usage pending cost is missing"))?; + let before = before + .parse::() + .map_err(|_| invalid("invalid account cost"))?; + let removed = amount + .parse::() + .map_err(|_| invalid("invalid pending account cost"))?; + let difference = before + .scaled() + .checked_sub(removed.scaled()) + .ok_or_else(|| invalid("account usage pending cost underflow"))?; + let value = gateway_core::metering::Decimal::from_scaled(difference) + .map_err(|_| invalid("invalid account cost difference"))?; + state.costs.insert(currency.clone(), value.canonical()); + } + repair.insert(key); + } + merge_account_states(&mut states, current)?; + // 撤销最大值不能用减法；只对发生源版本替换的账号模型做索引定位 + let completed = completed_usage_fact_predicate("mr"); + for (account, model) in repair { + let state = states + .get_mut(&(account.clone(), model.clone())) + .ok_or_else(|| invalid("pending account state disappeared"))?; + if state.count("completed_request_count") == 0 { + *state = MetricState::default(); + continue; + } + let statement = format!( + "select started_at from model_requests mr where provider_account_ref=$1 and ($2::text is null or coalesce(upstream_model_id,requested_model_id,'')=$2) and started_at >= $3 and started_at < $4 and {completed} order by started_at desc,id desc limit 1" + ); + state.last_used_at = sqlx::query_scalar(sqlx::AssertSqlSafe(statement)) + .bind(&account) + .bind(by_model.then_some(&model)) + .bind(range.start) + .bind(range.end) + .fetch_optional(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("repair account latest usage", error))?; + let zero_currencies = state + .costs + .iter() + .filter(|(_, amount)| amount.as_str() == "0") + .map(|(currency, _)| currency.clone()) + .collect::>(); + for currency in zero_currencies { + let statement = format!( + "select exists(select 1 from model_requests mr where provider_account_ref=$1 and ($2::text is null or coalesce(upstream_model_id,requested_model_id,'')=$2) and started_at >= $3 and started_at < $4 and {completed} and cost_currency=$5 and cost_amount is not null)" + ); + let exists: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(statement)) + .bind(&account) + .bind(by_model.then_some(&model)) + .bind(range.start) + .bind(range.end) + .bind(¤cy) + .fetch_one(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("repair account cost coverage", error))?; + if !exists { + state.costs.remove(¤cy); + } + } + } + states.retain(|_, state| state.count("completed_request_count") > 0); + Ok(states) + } + + pub(super) async fn has_pending_requests(&mut self) -> StoreResult { + if self.status.oldest_pending_at.is_none() { + return Ok(false); + } + sqlx::query_scalar("select exists(select 1 from request_analytics_pending_requests)") + .fetch_one(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("check pending account usage", error)) + } + + pub(super) fn projection_covers(&self, range: ObservabilityRange) -> bool { + self.status.schema_version == SCHEMA_VERSION as u32 + && self.status.histogram_version == HISTOGRAM_VERSION as u32 + && (!self.status.building + || (self + .status + .coverage_start + .is_some_and(|start| range.start >= start) + && self.status.coverage_end.is_some())) + } + + pub(crate) async fn account_request_counts( + &mut self, + range: ObservabilityRange, + account_ids: &[String], + ) -> StoreResult> { + if account_ids.is_empty() { + return Ok(BTreeMap::new()); + } + let mut counts = BTreeMap::new(); + // 公开小时图采用 UTC 整点；非整刻内部窗口直接在源表按相对小时分组 + if !self.projection_covers(range) + || range.start.timestamp().rem_euclid(900) != 0 + || range.start.timestamp_subsec_nanos() != 0 + { + let rows = self + .account_request_fact_counts( + "model_requests", + range, + range.start, + account_ids, + false, + ) + .await?; + merge_request_counts(&mut counts, rows, false)?; + return Ok(counts); + } + let (hours, quarters) = self + .summary_buckets(range, range.start.timestamp().rem_euclid(3600) == 0) + .await?; + for (table, seconds, buckets) in [ + ("request_analytics_hourly", 3600, hours), + ("request_analytics_15m", 900, quarters), + ] { + if buckets.is_empty() { + continue; + } + let mut query = QueryBuilder::::new( + "select dimensions[1] as account_id,floor(extract(epoch from (bucket_start-", + ); + query.push_bind(range.start).push("::timestamptz))/3600)::bigint as bucket_index,coalesce(sum((counts->>'completed_request_count')::bigint),0)::bigint as request_count from ") + .push(table).push(" a where collection='account' and schema_version=").push_bind(SCHEMA_VERSION) + .push(" and histogram_version=").push_bind(HISTOGRAM_VERSION).push(" and dimensions[1]=any(").push_bind(account_ids).push("::text[])"); + push_summary_buckets(&mut query, &buckets, seconds); + query.push(" group by dimensions[1],bucket_index"); + let rows = query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| { + postgres_unavailable("read account hourly request summaries", error) + })?; + merge_request_counts(&mut counts, rows, false)?; + } + for (start, end) in self.uncovered_spans(range).await? { + let rows = self + .account_request_fact_counts( + "request_analytics_facts", + ObservabilityRange { start, end }, + range.start, + account_ids, + false, + ) + .await?; + merge_request_counts(&mut counts, rows, false)?; + } + if self.has_pending_requests().await? { + let old = self + .account_request_fact_counts( + "request_analytics_facts", + range, + range.start, + account_ids, + true, + ) + .await?; + merge_request_counts(&mut counts, old, true)?; + let current = self + .account_request_fact_counts( + "model_requests", + range, + range.start, + account_ids, + true, + ) + .await?; + merge_request_counts(&mut counts, current, false)?; + } + Ok(counts) + } + + async fn account_request_fact_counts( + &mut self, + table: &'static str, + range: ObservabilityRange, + origin: DateTime, + account_ids: &[String], + pending_only: bool, + ) -> StoreResult> { + let mut query = QueryBuilder::::new( + "select mr.provider_account_ref as account_id,floor(extract(epoch from (mr.started_at-", + ); + query.push_bind(origin).push("::timestamptz))/3600)::bigint as bucket_index,count(*)::bigint as request_count from ").push(table).push(" mr"); + if pending_only { + query.push(" join request_analytics_pending_requests p on p.request_id=mr.id"); + } + query + .push(" where mr.provider_account_ref=any(") + .push_bind(account_ids) + .push("::text[]) and mr.started_at >= ") + .push_bind(range.start) + .push(" and mr.started_at < ") + .push_bind(range.end); + push_completed_usage_fact_filter(&mut query, "mr"); + query.push(" group by mr.provider_account_ref,bucket_index"); + query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| { + postgres_unavailable("aggregate account hourly request boundaries", error) + }) + } + + async fn account_fact_states( + &mut self, + table: &'static str, + range: ObservabilityRange, + account_ids: &[String], + pending_only: bool, + by_model: bool, + ) -> StoreResult { + let mut query = QueryBuilder::::new(usage_fact_aggregate_sql(table, by_model)); + if pending_only { + query.push( + " join request_analytics_pending_requests pending on pending.request_id=mr.id", + ); + } + query + .push(" where mr.provider_account_ref=any(") + .push_bind(account_ids) + .push("::text[]) and mr.started_at >= ") + .push_bind(range.start) + .push(" and mr.started_at < ") + .push_bind(range.end); + push_completed_usage_fact_filter(&mut query, "mr"); + query.push(" group by mr.provider_account_ref,mr.cost_currency"); + if by_model { + query.push(",coalesce(mr.upstream_model_id,mr.requested_model_id,'')"); + } + let rows = query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("aggregate account usage facts", error))?; + let mut states = BTreeMap::new(); + merge_account_rows(&mut states, rows)?; + Ok(states) + } +} + +fn merge_request_counts( + counts: &mut BTreeMap<(String, i64), u64>, + rows: Vec, + subtract: bool, +) -> StoreResult<()> { + for row in rows { + let value = unsigned(&row, "request_count")?; + let total = counts + .entry((get(&row, "account_id")?, get(&row, "bucket_index")?)) + .or_default(); + *total = if subtract { + total.checked_sub(value) + } else { + total.checked_add(value) + } + .ok_or_else(|| invalid("invalid account request count delta"))?; + } + Ok(()) +} + +fn merge_account_rows( + states: &mut AccountStates, + rows: Vec, +) -> StoreResult<()> { + for row in rows { + states + .entry((get(&row, "account_id")?, get(&row, "model")?)) + .or_default() + .merge(&MetricState::decode(&row)?)?; + } + Ok(()) +} + +fn merge_account_states(states: &mut AccountStates, incoming: AccountStates) -> StoreResult<()> { + for (key, state) in incoming { + states.entry(key).or_default().merge(&state)?; + } + Ok(()) +} + +fn usage_counter_names() -> Vec { + let mut names = [ + "completed_request_count", + "image_request_count", + "image_request_failed_count", + "window_total_tokens", + "provider_reported_count", + "calculated_count", + "unavailable_count", + ] + .map(str::to_owned) + .to_vec(); + for name in super::metrics::TOKEN_METRICS { + names.push(name.to_owned()); + names.push(format!("{name}_known_count")); + } + names +} + +fn usage_fact_aggregate_sql(table: &str, by_model: bool) -> String { + let model = if by_model { + "coalesce(mr.upstream_model_id,mr.requested_model_id,'')" + } else { + "''::text" + }; + let mut counters = vec![ + "'completed_request_count',count(*)".to_owned(), + "'image_request_count',count(*) filter(where image_generation_succeeded is true)".to_owned(), + "'image_request_failed_count',count(*) filter(where image_generation_succeeded is false)".to_owned(), + "'window_total_tokens',coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)+coalesce(output_tokens,0))),0)".to_owned(), + ]; + for name in super::metrics::TOKEN_METRICS { + counters.push(format!( + "'{name}',coalesce(sum({name}),0),'{name}_known_count',count({name})" + )); + } + for source in ["provider_reported", "calculated", "unavailable"] { + counters.push(format!( + "'{source}_count',count(*) filter(where cost_source='{source}')" + )); + } + format!("select mr.provider_account_ref as account_id,{model} as model, + jsonb_build_object({}) as counts,case when mr.cost_currency is not null and sum(mr.cost_amount) is not null + then jsonb_build_object(mr.cost_currency,sum(mr.cost_amount)::text) else '{{}}'::jsonb end as costs, + '{{}}'::jsonb as distributions,max(mr.started_at) as last_used_at from {table} mr",counters.join(",")) +} + +pub(crate) async fn account_usage_windows( + connection: &mut ObservabilityConnection, + windows: &[AccountUsageWindowQuery], +) -> StoreResult> { + let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?; + // 相同窗口一次查询所有账号，避免列表逐账号重复读取同一套时间桶 + let mut grouped = BTreeMap::<(DateTime, DateTime), BTreeSet>::new(); + for window in windows { + grouped + .entry((window.range.start, window.range.end)) + .or_default() + .insert(window.account_id.clone()); + } + let mut results = Vec::with_capacity(windows.len()); + for ((start, end), accounts) in grouped { + let ids = accounts.into_iter().collect::>(); + let states = snapshot + .account_usage_states(ObservabilityRange { start, end }, &ids) + .await?; + for window in windows + .iter() + .filter(|window| window.range.start == start && window.range.end == end) + { + let mut total = MetricState::default(); + let mut models = Vec::new(); + for ((account, model), state) in &states { + if account != &window.account_id { + continue; + } + total.merge(state)?; + if !model.is_empty() + && let Some(last_used_at) = state.last_used_at() + { + models.push(AccountModelUsage { + model: model.clone(), + request_count: state.count("completed_request_count"), + success_count: state.count("completed_request_count"), + input_tokens: state.optional_count("input_tokens"), + output_tokens: state.optional_count("output_tokens"), + cached_input_tokens: state.optional_count("cached_input_tokens"), + cache_write_input_tokens: state.optional_count("cache_write_input_tokens"), + reasoning_output_tokens: state.optional_count("reasoning_output_tokens"), + image_input_tokens: state.optional_count("image_input_tokens"), + image_output_tokens: state.optional_count("image_output_tokens"), + total_tokens: Some(state.count("window_total_tokens")), + image_request_count: state.count("image_request_count"), + image_request_failed_count: state.count("image_request_failed_count"), + cost_coverage: admin_cost_coverage(state.coverage()), + costs: account_costs(state)?, + last_used_at, + }); + } + } + models.sort_by(|a, b| { + b.request_count + .cmp(&a.request_count) + .then(a.model.cmp(&b.model)) + }); + results.push(AccountUsageWindowResult { + account_id: window.account_id.clone(), + key: window.key.clone(), + usage: AccountUsage { + account_id: window.account_id.clone(), + request_count: total.count("completed_request_count"), + success_count: total.count("completed_request_count"), + input_tokens: total.optional_count("input_tokens"), + output_tokens: total.optional_count("output_tokens"), + cached_input_tokens: total.optional_count("cached_input_tokens"), + cache_write_input_tokens: total.optional_count("cache_write_input_tokens"), + reasoning_output_tokens: total.optional_count("reasoning_output_tokens"), + image_input_tokens: total.optional_count("image_input_tokens"), + image_output_tokens: total.optional_count("image_output_tokens"), + total_tokens: Some(total.count("window_total_tokens")), + image_request_count: total.count("image_request_count"), + image_request_failed_count: total.count("image_request_failed_count"), + cost_coverage: admin_cost_coverage(total.coverage()), + costs: account_costs(&total)?, + last_used_at: total.last_used_at(), + request_buckets: Vec::new(), + models, + }, + }); + } + } + snapshot.finish().await?; + results.sort_by(|a, b| a.account_id.cmp(&b.account_id).then(a.key.cmp(&b.key))); + Ok(results) +} + +fn account_costs(state: &MetricState) -> StoreResult> { + Ok(state + .costs()? + .into_iter() + .map(|cost| AccountCost { + currency: cost.currency, + amount: cost.amount, + }) + .collect()) +} + +pub(crate) async fn account_usage_sort_values( + snapshot: &mut AnalyticsSnapshot<'_>, + now: DateTime, +) -> StoreResult<(Vec, Vec, Vec>>)> { + let days: i32 = + sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1") + .fetch_one(&mut *snapshot.tx) + .await + .map_err(|error| postgres_unavailable("read account usage retention", error))?; + let ids: Vec = sqlx::query_scalar("select id from provider_accounts") + .fetch_all(&mut *snapshot.tx) + .await + .map_err(|error| postgres_unavailable("read account usage sort identities", error))?; + let range = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?; + let mut totals = BTreeMap::::new(); + for ((account, _), state) in snapshot.account_states(range, &ids, false).await? { + totals.entry(account).or_default().merge(&state)?; + } + let mut tokens = Vec::with_capacity(ids.len()); + let mut latest = Vec::with_capacity(ids.len()); + for id in &ids { + let state = totals.remove(id).unwrap_or_default(); + tokens.push( + i64::try_from(state.count("window_total_tokens")) + .map_err(|_| invalid("account usage sort token overflow"))?, + ); + latest.push(state.last_used_at()); + } + Ok((ids, tokens, latest)) +} + +pub(crate) async fn account_group_usage( + connection: &mut ObservabilityConnection, + group_ids: &[String], + timezone: gateway_core::time::DeploymentTimeZone, +) -> StoreResult> { + if group_ids.is_empty() { + return Ok(BTreeMap::new()); + } + let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?; + let now = Utc::now(); + let days: i32 = + sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1") + .fetch_one(&mut *snapshot.tx) + .await + .map_err(|error| postgres_unavailable("read group usage retention", error))?; + let memberships:Vec<(String,String)>=sqlx::query_as("select account_group_id,provider_account_id from account_group_accounts where account_group_id=any($1::text[])").bind(group_ids).fetch_all(&mut *snapshot.tx).await.map_err(|error|postgres_unavailable("read group usage membership",error))?; + let ids = memberships + .iter() + .map(|(_, account)| account.clone()) + .collect::>() + .into_iter() + .collect::>(); + let retained = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?; + let today = ObservabilityRange::new( + timezone + .day_start(now) + .ok_or_else(|| invalid("invalid business date"))? + .max(retained.start), + now, + )?; + let retained_states = snapshot.account_states(retained, &ids, false).await?; + let today_states = snapshot.account_states(today, &ids, false).await?; + let mut results = BTreeMap::new(); + for group in group_ids { + let mut usage = AccountGroupUsage { + today_usd: "0".parse().map_err(|_| invalid("invalid zero cost"))?, + retained_total_usd: "0".parse().map_err(|_| invalid("invalid zero cost"))?, + }; + for (_, account) in memberships.iter().filter(|(id, _)| id == group) { + for (states, total) in [ + (&retained_states, &mut usage.retained_total_usd), + (&today_states, &mut usage.today_usd), + ] { + for ((id, _), state) in states { + if id == account + && let Some(amount) = state.costs.get("USD") + { + let amount = amount + .parse() + .map_err(|_| invalid("invalid group usage cost"))?; + *total = total + .checked_add(&amount) + .ok_or_else(|| invalid("group usage cost overflow"))?; + } + } + } + } + results.insert(group.clone(), usage); + } + snapshot.finish().await?; + Ok(results) +} diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs new file mode 100644 index 000000000..4a26fbf6c --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs @@ -0,0 +1,410 @@ +//! 在 PostgreSQL 内完成事实统计与汇总合并，结果规模只随输出分组增长 + +use super::*; + +/// 输入 matching 提供 output_bucket、name 和完整事实列及 is_completed_usage +/// 精确分位数在整个结果组内计算，不把局部分位数合并成新的分位数 +pub(super) fn push_fact_aggregation( + query: &mut QueryBuilder, + exact_percentiles: bool, + read_costs: bool, +) { + push_fact_aggregation_selected(query, exact_percentiles, read_costs, true, None, None); +} + +pub(super) fn push_fact_aggregation_selected( + query: &mut QueryBuilder, + exact_percentiles: bool, + read_costs: bool, + read_distributions: bool, + counter_names: Option<&[&str]>, + distribution_names: Option<&[&str]>, +) { + let counters = selected_counter_expressions(counter_names); + query.push(", counted as (select output_bucket,name,max(started_at) filter(where is_completed_usage) as last_used_at,"); + push_counter_object(query, &counters); + query.push(" as counts"); + if exact_percentiles && read_distributions { + query.push(","); + push_exact_distributions(query, distribution_names); + } + query.push(" from matching group by output_bucket,name)"); + if read_costs { + query.push(", cost_values as (select output_bucket,name,currency,sum(amount)::text as amount from matching cross join lateral (values (cost_currency,cost_amount),('standard_usd',standard_cost_amount),('no_cache_usd',no_cache_cost_amount)) cost(currency,amount) where is_completed_usage and currency is not null and amount is not null group by output_bucket,name,currency), merged_costs as (select output_bucket,name,jsonb_object_agg(currency,amount) as costs from cost_values group by output_bucket,name)"); + } + if !read_distributions { + query.push(" select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,'{}'::jsonb as distributions,'{}'::jsonb as exact_percentiles,"); + query.push(if read_costs { + "coalesce(k.costs,'{}'::jsonb)" + } else { + "'{}'::jsonb" + }); + query.push(" as costs from counted c"); + if read_costs { + query.push(" left join merged_costs k using(output_bucket,name)"); + } + return; + } + if exact_percentiles { + query.push(" select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,c.distributions,c.exact_percentiles,"); + query.push(if read_costs { + "coalesce(k.costs,'{}'::jsonb)" + } else { + "'{}'::jsonb" + }); + query.push(" as costs from counted c"); + if read_costs { + query.push(" left join merged_costs k using(output_bucket,name)"); + } + return; + } + query.push(", samples as (select output_bucket,name,metric,value from matching cross join lateral (values "); + for (index, (name, expression)) in selected_sample_expressions(distribution_names) + .iter() + .enumerate() + { + if index > 0 { + query.push(","); + } + query.push(format!("('{name}',{expression})")); + } + query.push(") sample(metric,value) where value is not null), sample_stats as (select output_bucket,name,metric,count(*) as count,sum(value) as sum,min(value) as minimum,max(value) as maximum,count(*) filter(where value=0) as zero"); + query.push(" from samples group by output_bucket,name,metric)"); + // 对数桶定义与 Histogram::add 相同，零值单独保留，不对零取对数 + query.push(", bin_counts as (select output_bucket,name,metric,floor(log(2::numeric,value::numeric)*16)::integer as bin,count(*) as count from samples where value>0 group by output_bucket,name,metric,bin), merged_bins as (select output_bucket,name,metric,jsonb_object_agg(bin,count) as bins from bin_counts group by output_bucket,name,metric)"); + query.push(", merged_distributions as (select s.output_bucket,s.name,jsonb_object_agg(s.metric,jsonb_build_object('count',s.count,'sum',s.sum,'minimum',s.minimum,'maximum',s.maximum,'zero',s.zero,'bins',"); + query.push("coalesce(b.bins,'{}'::jsonb)"); + query.push(")) as distributions"); + query.push(" from sample_stats s"); + query.push(" left join merged_bins b using(output_bucket,name,metric)"); + query.push(" group by s.output_bucket,s.name) select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,"); + query.push(if read_costs { + "coalesce(k.costs,'{}'::jsonb)" + } else { + "'{}'::jsonb" + }); + query.push(" as costs,coalesce(d.distributions,'{}'::jsonb) as distributions,"); + query.push("'{}'::jsonb"); + query.push(" as exact_percentiles from counted c left join merged_distributions d using(output_bucket,name)"); + if read_costs { + query.push(" left join merged_costs k using(output_bucket,name)"); + } +} + +// 事实聚合与汇总读取共用指标清单，防止新增计数只在其中一条路径生效 +fn counter_expressions() -> Vec<(String, String)> { + let mut counters = vec![ + ("request_count".to_owned(), "count(*)".to_owned()), + ("completed_request_count".to_owned(), "count(*) filter(where is_completed_usage)".to_owned()), + ("caller_error_count".to_owned(), "count(*) filter(where client_status_code between 400 and 499)".to_owned()), + ("attempt_count".to_owned(), "coalesce(sum(attempt_count),0)".to_owned()), + ("retry_count".to_owned(), "coalesce(sum(greatest(attempt_count-1,0)),0)".to_owned()), + ("retried_request_count".to_owned(), "count(*) filter(where attempt_count>1)".to_owned()), + ("capacity_sample_count".to_owned(), "count(capacity_total_slots)".to_owned()), + ("window_total_tokens".to_owned(), "coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)::numeric+coalesce(output_tokens,0))) filter(where is_completed_usage),0)".to_owned()), + ("cache_eligible_request_count".to_owned(), "count(*) filter(where is_completed_usage and input_tokens is not null)".to_owned()), + ("cache_hit_request_count".to_owned(), "count(*) filter(where is_completed_usage and input_tokens is not null and cached_input_tokens>0)".to_owned()), + ("image_request_count".to_owned(), "count(*) filter(where is_completed_usage and image_generation_succeeded)".to_owned()), + ("image_request_failed_count".to_owned(), "count(*) filter(where is_completed_usage and not image_generation_succeeded)".to_owned()), + ]; + for (outcome, name) in [ + ("succeeded", "success"), + ("failed", "failure"), + ("cancelled", "cancelled"), + ("incomplete", "incomplete"), + ] { + counters.push(( + format!("{name}_count"), + format!("count(*) filter(where outcome='{outcome}')"), + )); + let extra = if outcome == "failed" { + "+coalesce(sum(ops_failure_count),0)" + } else { + "" + }; + counters.push(( + format!("attempt_{name}_count"), + format!("count(*) filter(where attempt_count>0 and outcome='{outcome}'){extra}"), + )); + } + for (name, predicate, source) in [ + ( + "rate_limited_count", + "(error_kind in ('rate_limited','quota_exhausted') or coalesce(upstream_status_code,client_status_code)=429)", + "ops_rate_limited_count", + ), + ( + "auth_failure_count", + "(error_kind in ('authentication','authorization','invalid_credential') or coalesce(upstream_status_code,client_status_code) in (401,403))", + "ops_auth_failure_count", + ), + ( + "provider_5xx_count", + "coalesce(upstream_status_code,client_status_code) between 500 and 599", + "ops_provider_5xx_count", + ), + ] { + counters.push((name.to_owned(), format!("count(*) filter(where attempt_count>0 and outcome='failed' and {predicate})+coalesce(sum({source}),0)"))); + } + for name in metrics::TOKEN_METRICS { + counters.push(( + name.to_owned(), + format!("coalesce(sum({name}) filter(where is_completed_usage),0)"), + )); + counters.push(( + format!("{name}_known_count"), + format!("count({name}) filter(where is_completed_usage)"), + )); + } + for name in ["latency_ms", "first_token_ms"] { + counters.push(( + format!("{name}_sum"), + format!("coalesce(sum({name}) filter(where is_completed_usage),0)"), + )); + } + for (name, column) in [ + ("standard_usd", "standard_cost_amount"), + ("no_cache_usd", "no_cache_cost_amount"), + ] { + counters.push(( + format!("{name}_known_count"), + format!("count({column}) filter(where is_completed_usage)"), + )); + } + for source in ["provider_reported", "calculated", "unavailable"] { + counters.push(( + format!("{source}_count"), + format!("count(*) filter(where is_completed_usage and cost_source='{source}')"), + )); + } + counters +} + +fn selected_counter_expressions(names: Option<&[&str]>) -> Vec<(String, String)> { + let mut counters = counter_expressions(); + if let Some(names) = names { + counters.retain(|(name, _)| names.contains(&name.as_str())); + } + counters +} + +fn push_exact_distributions(query: &mut QueryBuilder, names: Option<&[&str]>) { + let samples = selected_sample_expressions(names); + // 独立 ordered-set 聚合只排序实际样本，避免把每条事实展开为六条记录再排序 + query.push("jsonb_build_object("); + for (index, (name, expression)) in samples.iter().enumerate() { + if index > 0 { + query.push(","); + } + query.push(format!("'{name}',jsonb_build_object('count',count({expression}),'sum',coalesce(sum({expression}),0),'minimum',min({expression}),'maximum',max({expression}),'zero',count(*) filter(where ({expression})=0),'bins','{{}}'::jsonb)")); + } + query.push(") as distributions,jsonb_build_object("); + for (index, (name, expression)) in samples.iter().enumerate() { + if index > 0 { + query.push(","); + } + query.push(format!("'{name}',coalesce(to_jsonb(percentile_cont(array[0.1,0.5,0.9,0.95,0.99]::double precision[]) within group(order by ({expression}))),'[]'::jsonb)")); + } + query.push(") as exact_percentiles"); +} + +fn sample_expressions() -> [(&'static str, &'static str); 6] { + [ + ( + "admission_decision_ms", + "admission_decision_ms::double precision", + ), + ( + "account_selection_wait_ms", + "account_selection_wait_ms::double precision", + ), + ( + "capacity", + "case when capacity_total_slots>0 then capacity_used_slots::double precision/capacity_total_slots::double precision*10000.0 end", + ), + ( + "latency_ms", + "case when is_completed_usage then latency_ms::double precision end", + ), + ( + "first_token_ms", + "case when is_completed_usage then first_token_ms::double precision end", + ), + ( + "throughput", + "case when is_completed_usage and output_tokens>0 and upstream_response_ms>0 then output_tokens::double precision*1000.0/upstream_response_ms::double precision end", + ), + ] +} + +fn selected_sample_expressions(names: Option<&[&str]>) -> Vec<(&'static str, &'static str)> { + sample_expressions() + .into_iter() + .filter(|(name, _)| names.is_none_or(|names| names.contains(name))) + .collect() +} + +fn push_counter_object(query: &mut QueryBuilder, counters: &[(String, String)]) { + // PostgreSQL 单个函数调用最多接收一百个参数，分块构造不限制指标数量 + for (index, chunk) in counters.chunks(40).enumerate() { + if index > 0 { + query.push(" || "); + } + query.push("jsonb_build_object("); + for (position, (name, expression)) in chunk.iter().enumerate() { + if position > 0 { + query.push(","); + } + query.push(format!("'{name}',{expression}")); + } + query.push(")"); + } +} + +/// 输入 matching 的行来自互斥时间桶，合并后每个最终输出组只返回一行 +pub(super) fn push_summary_aggregation( + query: &mut QueryBuilder, + distributions: bool, + costs: bool, + counter_names: Option<&[&str]>, + distribution_names: Option<&[&str]>, +) { + // 每个源汇总只展开一次固定指标，避免按指标键放大行数再排序 + let counters = selected_counter_expressions(counter_names) + .into_iter() + .map(|(name, _)| { + let expression = format!("coalesce(sum(metric.{name}),0)"); + (name, expression) + }) + .collect::>(); + query.push(", grouped as (select output_bucket,name,max(last_used_at) as last_used_at,"); + push_counter_object(query, &counters); + query.push(" as counts from matching cross join lateral jsonb_to_record(counts) as metric("); + query.push( + counters + .iter() + .map(|(name, _)| format!("{name} numeric")) + .collect::>() + .join(","), + ); + query.push(") group by output_bucket,name)"); + if costs { + query.push(", cost_values as (select output_bucket,name,c.key,sum(c.value::numeric)::text as value from matching cross join lateral jsonb_each_text(costs) c group by output_bucket,name,c.key), merged_costs as (select output_bucket,name,jsonb_object_agg(key,value) as costs from cost_values group by output_bucket,name)"); + } + if distributions { + // 先按最终输出组归并分布，再在每组内合并直方图键，避免全局展开后大排序 + query.push(", samples as (select output_bucket,name,d.key as metric,d.value from matching cross join lateral jsonb_each(distributions) d"); + if let Some(names) = distribution_names { + query + .push(" where d.key=any(") + .push_bind( + names + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), + ) + .push("::text[])"); + } + query.push("), sample_stats as (select output_bucket,name,metric,sum((value->>'count')::numeric) as count,sum((value->>'sum')::double precision) as sum,min((value->>'minimum')::double precision) as minimum,max((value->>'maximum')::double precision) as maximum,sum((value->>'zero')::numeric) as zero,array_agg(value->'bins') as bins from samples group by output_bucket,name,metric), merged_distributions as (select s.output_bucket,s.name,jsonb_object_agg(s.metric,jsonb_build_object('count',s.count,'sum',s.sum,'minimum',s.minimum,'maximum',s.maximum,'zero',s.zero,'bins',coalesce(b.bins,'{}'::jsonb))) as distributions from sample_stats s cross join lateral(select jsonb_object_agg(key,count) as bins from(select b.key,sum(b.value::numeric) as count from unnest(s.bins) input(value) cross join lateral jsonb_each_text(input.value) b group by b.key) merged)b group by s.output_bucket,s.name)"); + } + query.push(" select g.output_bucket as bucket_start,g.name,g.last_used_at,g.counts,"); + query.push(if costs { + "coalesce(k.costs,'{}'::jsonb)" + } else { + "'{}'::jsonb" + }); + query.push(" as costs,"); + query.push(if distributions { + "coalesce(d.distributions,'{}'::jsonb)" + } else { + "'{}'::jsonb" + }); + query.push(" as distributions from grouped g"); + if costs { + query.push(" left join merged_costs k using(output_bucket,name)"); + } + if distributions { + query.push(" left join merged_distributions d using(output_bucket,name)"); + } +} + +pub(super) fn push_output_bucket( + query: &mut QueryBuilder, + column: &str, + range: ObservabilityRange, + granularity: Option, + boundaries: Option<&[DateTime]>, +) { + match granularity { + None => { + query.push_bind(range.start).push("::timestamptz"); + } + Some(ObservationGranularity::Day) => { + // 使用有序阈值二分定位，保留部署时区及夏令时日界，不逐行扫描所有日界 + let boundaries = boundaries.unwrap_or_default(); + query + .push("(") + .push_bind(boundaries.to_vec()) + .push(format!("::timestamptz[])[width_bucket({column},")) + .push_bind(boundaries.to_vec()) + .push("::timestamptz[])]"); + } + Some(granularity) => { + query + .push("date_bin(make_interval(secs => ") + .push_bind(granularity.seconds() as f64) + .push(format!("),{column},timestamptz '2000-01-01 UTC')")); + } + } +} + +pub(super) fn fact_dimension_sql(dimension: Option) -> &'static str { + match dimension { + None => "''::text", + Some(DiagnosticDimension::Provider) => "coalesce(nullif(mr.provider_kind,''),'unrouted')", + Some(DiagnosticDimension::Model) => { + "coalesce(nullif(mr.upstream_model_id,''),mr.requested_model_id,'')" + } + Some(DiagnosticDimension::Account) => { + "coalesce(nullif(mr.provider_account_ref,''),'unrouted')" + } + Some(DiagnosticDimension::ApiKey) => "coalesce(mr.client_api_key_ref,'')", + Some(DiagnosticDimension::Transport) => { + "coalesce(nullif(mr.upstream_transport,''),nullif(mr.client_transport,''),'unknown')" + } + Some(DiagnosticDimension::Failure) => "coalesce(mr.error_kind,'')", + Some(DiagnosticDimension::Status) => { + "coalesce(mr.upstream_status_code::text,mr.client_status_code::text,'')" + } + } +} + +pub(super) fn summary_dimension_sql( + collection: &str, + dimension: Option, +) -> String { + let offset = usize::from(matches!(collection, "account_model" | "key_model")); + match dimension { + None => "''::text".to_owned(), + Some(DiagnosticDimension::Provider) => { + format!("coalesce(nullif(dimensions[{}],''),'unrouted')", 1 + offset) + } + Some(DiagnosticDimension::Model) => format!( + "coalesce(nullif(dimensions[{}],''),dimensions[{}],'')", + 3 + offset, + 2 + offset + ), + Some(DiagnosticDimension::Account) => { + "coalesce(nullif(dimensions[1],''),'unrouted')".to_owned() + } + Some(DiagnosticDimension::Transport) => { + "coalesce(nullif(dimensions[2],''),nullif(dimensions[1],''),'unknown')".to_owned() + } + Some(DiagnosticDimension::Status) => { + "coalesce(nullif(dimensions[2],''),dimensions[1],'')".to_owned() + } + Some(DiagnosticDimension::ApiKey | DiagnosticDimension::Failure) => { + "coalesce(dimensions[1],'')".to_owned() + } + } +} diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/buckets.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/buckets.rs index ed8e0efd0..6081a2d8f 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/buckets.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/buckets.rs @@ -49,13 +49,13 @@ pub(super) async fn rebuild_bucket_batch( 0 }; if !same_version { - // 处理中事实发生变化时清掉未发布的部分状态，不能把新旧快照增量相加 + // 显式重建版本变化时清掉旧前缀，普通事实变化由增量维护更新前缀 sqlx::query("update request_analytics_pending_buckets set build_version=change_version,build_phase=0,build_started_at=null,build_id=null,build_collection=null,build_dimensions=null,build_outcome=null where bucket_start=$1") .bind(bucket).execute(&mut **tx).await.map_err(|error|postgres_unavailable("restart changed analytics bucket",error))?; } let hour = bucket_for(bucket, 3600)?; if phase < 2 { - // 小时分段期间不改写其输入；源事实仍可投影，版本变化使小时任务先退回等待 + // 小时分段期间暂停其他 quarter 的完整重建，普通增量同步修正已扫描前缀 let owned:bool=sqlx::query_scalar("select exists(select 1 from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and bucket_start <> $2 and build_phase >= 3)") .bind(hour).bind(bucket).fetch_one(&mut **tx).await.map_err(|error|postgres_unavailable("claim analytics hour stage",error))?; if owned { @@ -119,8 +119,9 @@ pub(super) async fn rebuild_bucket_batch( let limit = i64::from(batch_size) + 1; let mut groups = GroupStates::new(); if phase == 1 { + // 增量前缀判断使用 Rust 字节序，SQL 游标必须使用同一排序规则 let statement = format!( - "{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and mr.recovered_at is null and (mr.started_at,mr.id)>($2,$3) order by mr.started_at,mr.id limit $4", + "{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and (mr.started_at,mr.id collate \"C\")>($2,$3) order by mr.started_at,mr.id collate \"C\" limit $4", fact_select() ); let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) @@ -139,11 +140,16 @@ pub(super) async fn rebuild_bucket_batch( let outcome: String = get(fact, "outcome")?; let mut state = MetricState::default(); state.add_row(fact)?; + let forecast = forecast::state(fact)?; for (collection, dimensions) in collections(fact)? { groups .entry((collection.to_owned(), dimensions, outcome.clone())) .or_default() - .merge(&state)?; + .merge(if collection == forecast::COLLECTION { + &forecast + } else { + &state + })?; } } merge_staged_groups(tx, "request_analytics_15m", bucket, groups).await?; @@ -155,7 +161,7 @@ pub(super) async fn rebuild_bucket_batch( .bind(bucket).bind(get::>(last,"started_at")?).bind(get::(last,"id")?).execute(&mut **tx).await.map_err(|error|postgres_unavailable("save analytics fact cursor",error))?; } } else if phase == 4 { - let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection,dimensions,outcome)>($4,$5,$6,$7) order by bucket_start,collection,dimensions,outcome limit $8") + let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions,last_used_at from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection collate \"C\",dimensions collate \"C\",outcome collate \"C\")>($4,$5,$6,$7) order by bucket_start,collection collate \"C\",dimensions collate \"C\",outcome collate \"C\" limit $8") .bind(hour).bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).bind(get::>>(&row,"build_started_at")?.unwrap_or(hour-TimeDelta::microseconds(1))) .bind(get::>(&row,"build_collection")?.unwrap_or_default()).bind(get::>>(&row,"build_dimensions")?.unwrap_or_default()).bind(get::>(&row,"build_outcome")?.unwrap_or_default()).bind(limit) .fetch_all(&mut **tx).await.map_err(|error|postgres_unavailable("read analytics hour segment",error))?; @@ -198,7 +204,7 @@ async fn merge_staged_groups( } let keys=groups.keys().map(|(collection,dimensions,outcome)|serde_json::json!({"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::>(); let statement = format!( - "select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1" + "select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions,stored.last_used_at from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1" ); let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) .bind(bucket) @@ -220,37 +226,64 @@ async fn merge_staged_groups( write_groups(tx, table, bucket, groups).await } -async fn write_groups( +pub(super) async fn write_groups( tx: &mut sqlx::Transaction<'_, Postgres>, table: &str, bucket: DateTime, groups: GroupStates, ) -> StoreResult<()> { - if groups.is_empty() { - return Ok(()); - } - let mut statement = QueryBuilder::::new(format!( - "insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) " - )); - let rows = groups - .into_iter() - .map(|((collection, dimensions, outcome), state)| { - let (counts, costs, distributions) = state.encode()?; - Ok(( + write_bucket_groups(tx, table, [(bucket, groups)]).await +} + +pub(super) async fn write_bucket_groups( + tx: &mut sqlx::Transaction<'_, Postgres>, + table: &str, + buckets: impl IntoIterator, GroupStates)>, +) -> StoreResult<()> { + let mut groups = buckets.into_iter().flat_map(|(bucket, groups)| { + groups + .into_iter() + .map(move |(key, state)| (bucket, key, state)) + }); + loop { + // 跨桶合并 SQL 往返，但编码缓冲只保留当前批次，避免回填将所有状态展开为 JSON + let rows = groups + .by_ref() + .take(1000) + .map(|(bucket, (collection, dimensions, outcome), state)| { + let (counts, costs, distributions) = state.encode()?; + Ok(( + bucket, + collection, + dimensions, + outcome, + counts, + costs, + distributions, + state.last_used_at(), + )) + }) + .collect::>>()?; + if rows.is_empty() { + return Ok(()); + } + let mut statement = QueryBuilder::::new(format!( + "insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions,last_used_at) " + )); + // 十个绑定参数限制单语句批量，避免高基数时间桶越过 PostgreSQL 参数上限 + statement.push_values( + &rows, + |mut row, + ( + bucket, collection, dimensions, outcome, counts, costs, distributions, - )) - }) - .collect::>>()?; - // 九个绑定参数限制单语句批量，避免高基数时间桶越过 PostgreSQL 参数上限 - for chunk in rows.chunks(1000) { - statement.push_values( - chunk, - |mut row, (collection, dimensions, outcome, counts, costs, distributions)| { + last_used_at, + )| { row.push_bind(bucket) .push_bind(collection) .push_bind(dimensions) @@ -259,16 +292,15 @@ async fn write_groups( .push_bind(HISTOGRAM_VERSION) .push_bind(counts) .push_bind(costs) - .push_bind(distributions); + .push_bind(distributions) + .push_bind(last_used_at); }, ); - statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version"); + statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,last_used_at=excluded.last_used_at,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version"); statement .build() .execute(&mut **tx) .await .map_err(|error| postgres_unavailable("write analytics bucket", error))?; - statement.reset(); } - Ok(()) } diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/counts.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/counts.rs new file mode 100644 index 000000000..589a906eb --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/counts.rs @@ -0,0 +1,113 @@ +//! 用完整刻钟计数和同快照源差量定位精确页码，避免每页扫描整个保留期 + +use super::query::{collection_for, push_collection_filter, push_summary_buckets}; +use super::*; + +enum ReadVersion { + Published, + Before, + Current, +} + +impl AnalyticsSnapshot<'_> { + pub(crate) async fn completed_request_counts( + &mut self, + range: ObservabilityRange, + filter: &UsageRecordFilter, + ) -> StoreResult, u64>>> { + let Some(collection) = collection_for(filter, None) else { + return Ok(None); + }; + if !self.projection_covers(range) { + return Ok(None); + } + let mut counts = BTreeMap::new(); + let (_, buckets) = self.summary_buckets(range, false).await?; + if !buckets.is_empty() { + let mut query = QueryBuilder::::new( + "select bucket_start,coalesce(sum((counts->>'completed_request_count')::bigint),0)::bigint as request_count from request_analytics_15m a where collection=", + ); + query + .push_bind(collection) + .push(" and schema_version=") + .push_bind(SCHEMA_VERSION) + .push(" and histogram_version=") + .push_bind(HISTOGRAM_VERSION); + push_summary_buckets(&mut query, &buckets, 900); + push_collection_filter(&mut query, collection, filter); + query.push(" group by bucket_start"); + for row in query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("read usage page counts", error))? + { + counts.insert( + get(&row, "bucket_start")?, + to_u64(get(&row, "request_count")?)?, + ); + } + } + for (start, end) in self.uncovered_spans(range).await? { + self.merge_completed_counts( + &mut counts, + ObservabilityRange { start, end }, + filter, + ReadVersion::Published, + ) + .await?; + } + if self.has_pending_requests().await? { + // 先撤销已投影版本，再加入源表当前版本；移动、删除和未确认的重放均只计算一次 + self.merge_completed_counts(&mut counts, range, filter, ReadVersion::Before) + .await?; + self.merge_completed_counts(&mut counts, range, filter, ReadVersion::Current) + .await?; + } + Ok(Some(counts)) + } + + async fn merge_completed_counts( + &mut self, + counts: &mut BTreeMap, u64>, + range: ObservabilityRange, + filter: &UsageRecordFilter, + version: ReadVersion, + ) -> StoreResult<()> { + let table = if matches!(version, ReadVersion::Current) { + "model_request_observations" + } else { + "request_analytics_facts" + }; + let mut query = QueryBuilder::::new(format!( + "select date_bin(interval '15 minutes',mr.started_at,timestamptz '2000-01-01 UTC') as bucket_start,count(*)::bigint as request_count from {table} mr" + )); + if !matches!(version, ReadVersion::Published) { + query.push(" join request_analytics_pending_requests p on p.request_id=mr.id"); + } + query + .push(" where mr.started_at >= ") + .push_bind(range.start) + .push(" and mr.started_at < ") + .push_bind(range.end); + push_completed_usage_fact_filter(&mut query, "mr"); + push_usage_filter(&mut query, filter, "mr"); + query.push(" group by bucket_start"); + let rows = query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("correct usage page counts", error))?; + for row in rows { + let value = to_u64(get(&row, "request_count")?)?; + let count = counts.entry(get(&row, "bucket_start")?).or_default(); + *count = if matches!(version, ReadVersion::Before) { + count.checked_sub(value) + } else { + count.checked_add(value) + } + .ok_or_else(|| invalid("usage page count is inconsistent"))?; + } + Ok(()) + } +} diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/forecast.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/forecast.rs new file mode 100644 index 000000000..eb5bb3f74 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/forecast.rs @@ -0,0 +1,343 @@ +//! 预测累计使用开始时间与完成时间的二维汇总，采样边界及待投影版本读取精确事实 + +use super::*; +use gateway_admin::model::provider_credentials::ProviderDocument; +use gateway_admin::model::quota_forecast_sampling::{ + MAX_FORECAST_HISTORY_POINTS, QuotaForecastHistory, QuotaForecastHistoryPoint, + QuotaForecastUsage, +}; +use gateway_core::account::OpaqueProviderData; +use sqlx::postgres::PgRow; + +pub(super) const COLLECTION: &str = "account_completion"; +const COMPLETION_SECONDS: i64 = 300; +const COUNTERS: [&str; 10] = [ + "request_count", + "completed_request_count", + "window_total_tokens", + "input_tokens", + "output_tokens", + "cached_input_tokens", + "missing_token_count", + "known_cost_count", + "unavailable_cost_count", + "excluded_request_count", +]; + +pub(super) fn dimensions(row: &PgRow) -> StoreResult> { + let completed: Option> = get(row, "completed_at")?; + let terminal = get::(row, "outcome")? != "running"; + Ok(vec![ + get::>(row, "provider_account_ref")?.unwrap_or_default(), + completed + .filter(|_| terminal) + .map(|time| time.timestamp().div_euclid(COMPLETION_SECONDS).to_string()) + .unwrap_or_default(), + ]) +} + +pub(super) fn state(row: &PgRow) -> StoreResult { + let mut state = MetricState::default(); + state.increment("request_count", 1)?; + state.add_cost("USD", "0")?; + if get::>>(row, "completed_at")?.is_none() + || get::(row, "outcome")? == "running" + { + return Ok(state); + } + if !get::(row, "is_completed_usage")? { + state.increment("excluded_request_count", 1)?; + return Ok(state); + } + state.increment("completed_request_count", 1)?; + let input = optional_unsigned(row, "input_tokens")?; + let output = optional_unsigned(row, "output_tokens")?; + let total = optional_unsigned(row, "total_tokens")?; + state.increment( + "window_total_tokens", + total.unwrap_or( + input + .unwrap_or_default() + .checked_add(output.unwrap_or_default()) + .ok_or_else(|| invalid("forecast token overflow"))?, + ), + )?; + state.increment("input_tokens", input.unwrap_or_default())?; + state.increment("output_tokens", output.unwrap_or_default())?; + state.increment( + "cached_input_tokens", + optional_unsigned(row, "cached_input_tokens")?.unwrap_or_default(), + )?; + state.increment( + "missing_token_count", + u64::from(total.is_none() && (input.is_none() || output.is_none())), + )?; + let source: String = get(row, "cost_source")?; + let amount: Option = get(row, "cost_amount_text")?; + let known = matches!(source.as_str(), "calculated" | "provider_reported") + && get::>(row, "cost_currency")?.as_deref() == Some("USD") + && amount.is_some(); + state.increment("known_cost_count", u64::from(known))?; + state.increment("unavailable_cost_count", u64::from(!known))?; + if known { + state.add_cost( + "USD", + amount + .as_deref() + .ok_or_else(|| invalid("missing forecast amount"))?, + )?; + } + Ok(state) +} + +impl AnalyticsSnapshot<'_> { + pub(crate) async fn forecast_history( + &mut self, + account: &str, + range: ObservabilityRange, + ) -> StoreResult { + let documents = sqlx::query(DOCUMENTS) + .bind(account) + .bind(range.start) + .bind(range.end) + .bind(MAX_FORECAST_HISTORY_POINTS as i32) + .fetch_all(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("sample forecast documents", error))?; + let mut cutoffs = documents + .iter() + .map(|row| get(row, "completed_at")) + .collect::>>>()?; + // 最后一个状态专门代表整个窗口，文档恰好完成于窗口末端时复用同一截止值 + if cutoffs.last() != Some(&range.end) { + cutoffs.push(range.end); + } + let boundaries = cutoffs + .iter() + .map(|time| time.timestamp().div_euclid(COMPLETION_SECONDS)) + .collect::>() + .into_iter() + .collect::>(); + let mut states = vec![MetricState::default(); cutoffs.len() + 1]; + if self.projection_covers(range) { + let (_, quarters) = self.summary_buckets(range, false).await?; + if !quarters.is_empty() { + let sums = COUNTERS + .iter() + .map(|name| format!("'{name}',coalesce(sum((counts->>'{name}')::numeric),0)")) + .collect::>() + .join(","); + let statement = format!( + "with selected as (select counts,costs,case when dimensions[2]='' then cardinality($3::timestamptz[]) else width_bucket(to_timestamp((dimensions[2]::bigint+1)*{COMPLETION_SECONDS})-interval '1 microsecond',$3::timestamptz[]) end as slot + from request_analytics_15m where collection='{COLLECTION}' and dimensions[1]=$1 and bucket_start=any($2::timestamptz[]) and schema_version=$5 and histogram_version=$6 + and (dimensions[2]='' or not(nullif(dimensions[2],'')::bigint=any($4::bigint[])))) + select slot,jsonb_build_object({sums}) as counts,jsonb_build_object('USD',coalesce(sum((costs->>'USD')::numeric),0)::text) as costs,'{{}}'::jsonb as distributions,null::timestamptz as last_used_at from selected group by slot" + ); + let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) + .bind(account) + .bind(&quarters) + .bind(&cutoffs) + .bind(&boundaries) + .bind(SCHEMA_VERSION) + .bind(HISTOGRAM_VERSION) + .fetch_all(&mut *self.tx) + .await + .map_err(|error| { + postgres_unavailable("read forecast completion summaries", error) + })?; + merge_rows(&mut states, rows, false)?; + // 只有跨过采样截止值的完成时间小桶需要回读，按完成时间索引逐段定位 + let rows = self + .forecast_facts( + account, + range, + &cutoffs, + FactSelection::CompletionEdges { + quarters: &quarters, + boundaries: &boundaries, + }, + ) + .await?; + merge_rows(&mut states, rows, false)?; + } + for (start, end) in self.uncovered_spans(range).await? { + let rows = self + .forecast_facts( + account, + ObservabilityRange { start, end }, + &cutoffs, + FactSelection::Projected, + ) + .await?; + merge_rows(&mut states, rows, false)?; + } + if self.has_pending_requests().await? { + let old = self + .forecast_facts(account, range, &cutoffs, FactSelection::Previous) + .await?; + merge_rows(&mut states, old, true)?; + let current = self + .forecast_facts(account, range, &cutoffs, FactSelection::Current) + .await?; + merge_rows(&mut states, current, false)?; + } + } else { + let rows = self + .forecast_facts(account, range, &cutoffs, FactSelection::Source) + .await?; + merge_rows(&mut states, rows, false)?; + } + let pending_request_count = states + .last() + .map_or(0, |state| state.count("request_count")); + let mut cumulative = MetricState::default(); + let mut usages = Vec::with_capacity(cutoffs.len()); + for state in states.iter().take(cutoffs.len()) { + cumulative.merge(state)?; + usages.push(usage(&cumulative)?); + } + let mut points = Vec::with_capacity(documents.len()); + for (row, usage) in documents.into_iter().zip(&usages) { + let document: sqlx::types::Json> = + get(&row, "document")?; + points.push(QuotaForecastHistoryPoint { + started_at: get(&row, "started_at")?, + completed_at: get(&row, "completed_at")?, + usage: usage.clone(), + provider_observation: ProviderDocument::new(OpaqueProviderData::new(document.0)), + }); + } + Ok(QuotaForecastHistory { + points, + usage: usage(&cumulative)?, + pending_request_count, + }) + } + + async fn forecast_facts( + &mut self, + account: &str, + range: ObservabilityRange, + cutoffs: &[DateTime], + selection: FactSelection<'_>, + ) -> StoreResult> { + let source = matches!(selection, FactSelection::Source | FactSelection::Current); + let table = if source { + "model_requests" + } else { + "request_analytics_facts" + }; + let mut query = QueryBuilder::::new( + "with matching as (select mr.*,case when mr.completed_at is not null and mr.outcome <> 'running' then width_bucket(mr.completed_at-interval '1 microsecond',", + ); + query.push_bind(cutoffs).push("::timestamptz[]) else ").push_bind(cutoffs.len() as i32) + .push(" end as slot,coalesce((").push(completed_usage_fact_predicate("mr")) + .push("),false) and mr.completed_at is not null and mr.outcome <> 'running' as included from "); + if let FactSelection::CompletionEdges { boundaries, .. } = selection { + // 保留按完成时间小桶的参数化索引扫描，避免展开后扫描整段账号事实再逐桶匹配 + // OFFSET 0 不裁剪数据，只阻止此相关子查询被合并到外层连接 + query.push("unnest(").push_bind(boundaries).push(format!("::bigint[]) edge(bucket) cross join lateral (select * from {table} f where f.provider_account_ref=")) + .push_bind(account).push(format!(" and f.completed_at >= to_timestamp(edge.bucket*{COMPLETION_SECONDS}) and f.completed_at < to_timestamp((edge.bucket+1)*{COMPLETION_SECONDS}) and f.outcome <> 'running' offset 0) mr")); + } else { + query.push(format!("{table} mr")); + } + if matches!(selection, FactSelection::Previous | FactSelection::Current) { + query.push(" join request_analytics_pending_requests p on p.request_id=mr.id"); + } + query + .push(" where mr.provider_account_ref=") + .push_bind(account) + .push(" and mr.started_at >= ") + .push_bind(range.start) + .push(" and mr.started_at < ") + .push_bind(range.end); + if let FactSelection::CompletionEdges { quarters, .. } = selection { + query.push(" and date_bin(interval '15 minutes',mr.started_at,timestamptz '2000-01-01 UTC')=any(").push_bind(quarters).push("::timestamptz[])"); + } + // 多个累计指标复用同一分类结果，避免展开后重复判断完成用量并携带宽事实排序 + // OFFSET 0 保留全部行，只让分类在聚合前计算一次 + query.push("), classified as (select *,coalesce(cost_source in ('calculated','provider_reported') and cost_amount is not null and cost_currency='USD',false) as known_cost from matching offset 0) + select slot,jsonb_build_object( + 'request_count',count(*), + 'completed_request_count',count(*) filter(where included), + 'window_total_tokens',coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)+coalesce(output_tokens,0))) filter(where included),0), + 'input_tokens',coalesce(sum(input_tokens) filter(where included),0), + 'output_tokens',coalesce(sum(output_tokens) filter(where included),0), + 'cached_input_tokens',coalesce(sum(cached_input_tokens) filter(where included),0), + 'missing_token_count',count(*) filter(where included and total_tokens is null and (input_tokens is null or output_tokens is null)), + 'known_cost_count',count(*) filter(where included and known_cost), + 'unavailable_cost_count',count(*) filter(where included and not known_cost), + 'excluded_request_count',count(*) filter(where not included and completed_at is not null and outcome <> 'running')) as counts, + jsonb_build_object('USD',coalesce(sum(cost_amount) filter(where included and known_cost),0)::text) as costs, + '{}'::jsonb as distributions,null::timestamptz as last_used_at from classified group by slot"); + query + .build() + .fetch_all(&mut *self.tx) + .await + .map_err(|error| postgres_unavailable("read exact forecast boundaries", error)) + } +} + +#[derive(Clone, Copy)] +enum FactSelection<'a> { + Source, + Projected, + Previous, + Current, + CompletionEdges { + quarters: &'a [DateTime], + boundaries: &'a [i64], + }, +} + +fn merge_rows(states: &mut [MetricState], rows: Vec, subtract: bool) -> StoreResult<()> { + for row in rows { + let index = usize::try_from(get::(&row, "slot")?) + .map_err(|_| invalid("negative forecast slot"))?; + let target = states + .get_mut(index) + .ok_or_else(|| invalid("forecast slot out of range"))?; + let state = MetricState::decode(&row)?; + if subtract { + incremental::subtract(target, &state, &MetricState::default())?; + } else { + target.merge(&state)?; + } + } + Ok(()) +} + +fn usage(state: &MetricState) -> StoreResult { + Ok(QuotaForecastUsage { + request_count: state.count("completed_request_count"), + tokens: state.count("window_total_tokens"), + input_tokens: state.count("input_tokens"), + output_tokens: state.count("output_tokens"), + cached_input_tokens: state.count("cached_input_tokens"), + missing_token_count: state.count("missing_token_count"), + known_cost_count: state.count("known_cost_count"), + unavailable_cost_count: state.count("unavailable_cost_count"), + excluded_request_count: state.count("excluded_request_count"), + usd: state + .costs + .get("USD") + .map(|value| { + value + .parse::() + .map_err(|_| invalid("invalid forecast dollars")) + }) + .transpose()? + .unwrap_or_default(), + }) +} + +// 微秒边界向上取整，最后一槽包含观测时刻；文档采样与累计量补差共用读事务 +const DOCUMENTS: &str = "select mr.started_at,mr.completed_at,mr.provider_observation_json as document + from generate_series(0,$4-1) b(bucket) cross join lateral ( + select mr.started_at,mr.completed_at,mr.provider_observation_json + from model_requests mr where mr.provider_account_ref=$1 and mr.started_at >= $2 and mr.started_at < $3 + and mr.completed_at >= $2 + ceil(b.bucket*greatest(1,extract(epoch from ($3::timestamptz-$2::timestamptz))/$4)*1000000)::bigint*interval '1 microsecond' + and mr.completed_at < case when b.bucket=$4-1 then $3+interval '1 microsecond' else least($3+interval '1 microsecond',$2+ceil((b.bucket+1)*greatest(1,extract(epoch from ($3::timestamptz-$2::timestamptz))/$4)*1000000)::bigint*interval '1 microsecond') end + and mr.outcome <> 'running' and mr.provider_observation_json is not null + order by mr.completed_at desc,mr.id desc limit 1 + ) mr order by mr.completed_at"; diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/histogram.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/histogram.rs index 737d959d9..eecb8d0df 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/histogram.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/histogram.rs @@ -14,8 +14,8 @@ pub(super) struct Histogram { pub sum: f64, pub minimum: Option, pub maximum: Option, - zero: u64, - bins: BTreeMap, + pub(super) zero: u64, + pub(super) bins: BTreeMap, } impl Histogram { diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/incremental.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/incremental.rs new file mode 100644 index 000000000..ba50b08f6 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/incremental.rs @@ -0,0 +1,371 @@ +//! 同事务撤销旧事实并合并新事实，构建中的统计只更新已经扫描的前缀 + +use super::buckets::write_bucket_groups; +use super::worker::collections; +use super::*; +use sqlx::postgres::PgRow; + +type GroupKey = (DateTime, String, Vec, String); + +#[derive(Default)] +struct Change { + previous: MetricState, + current: MetricState, +} + +struct Progress { + phase: i16, + started_at: Option>, + id: Option, + collection: Option, + dimensions: Option>, + outcome: Option, +} + +impl Progress { + fn contains_fact(&self, time: DateTime, id: &str) -> bool { + match self.phase { + 0 => false, + 1 => self + .started_at + .zip(self.id.as_deref()) + .is_some_and(|cursor| (time, id) <= cursor), + _ => true, + } + } + + fn contains_quarter(&self, key: &GroupKey) -> bool { + if self.phase != 4 { + return false; + } + match ( + &self.started_at, + &self.collection, + &self.dimensions, + &self.outcome, + ) { + (Some(time), Some(collection), Some(dimensions), Some(outcome)) => { + (&key.0, &key.1, &key.2, &key.3) <= (time, collection, dimensions, outcome) + } + _ => false, + } + } +} + +pub(super) async fn update_summaries( + tx: &mut sqlx::Transaction<'_, Postgres>, + previous: &[PgRow], + current: &[PgRow], +) -> StoreResult<()> { + let mut hours = BTreeSet::new(); + for row in previous.iter().chain(current) { + hours.insert(bucket_for(get(row, "started_at")?, 3600)?); + } + if hours.is_empty() { + return Ok(()); + } + let progress_rows = sqlx::query("select p.* from unnest($1::timestamptz[]) h(start_at) join request_analytics_pending_buckets p on p.bucket_start >= h.start_at and p.bucket_start < h.start_at+interval '1 hour'") + .bind(hours.into_iter().collect::>()).fetch_all(&mut **tx).await + .map_err(|error|postgres_unavailable("read incremental analytics stages",error))?; + let mut progress = BTreeMap::new(); + for row in progress_rows { + let phase = + if get::>(&row, "build_version")? == Some(get(&row, "change_version")?) { + get(&row, "build_phase")? + } else { + 0 + }; + progress.insert( + get::>(&row, "bucket_start")?, + Progress { + phase, + started_at: get(&row, "build_started_at")?, + id: get(&row, "build_id")?, + collection: get(&row, "build_collection")?, + dimensions: get(&row, "build_dimensions")?, + outcome: get(&row, "build_outcome")?, + }, + ); + } + // 两种粒度共用一次事实解码，仍分别检查构建前缀，不能把未发布季度并入小时 + let mut changes = [BTreeMap::::new(), BTreeMap::new()]; + let clean_hours = progress.is_empty(); + for (rows, adding) in [(previous, false), (current, true)] { + for row in rows { + let time = get(row, "started_at")?; + let quarter = bucket_for(time, 900)?; + let hour = bucket_for(time, 3600)?; + let id: String = get(row, "id")?; + let outcome: String = get(row, "outcome")?; + let mut state = MetricState::default(); + state.add_row(row)?; + let forecast = forecast::state(row)?; + let mut hour_pending = progress.range(hour..hour + TimeDelta::hours(1)).peekable(); + let hour_ready = hour_pending.peek().is_none(); + let hour_owner = + hour_pending.find_map(|(_, stage)| (stage.phase >= 3).then_some(stage)); + for (collection, dimensions) in collections(row)? { + let key = (quarter, collection.to_owned(), dimensions, outcome.clone()); + let included = [ + progress + .get(&quarter) + .is_none_or(|p| p.contains_fact(time, &id)), + !clean_hours + && (hour_ready + || hour_owner.is_some_and(|stage| stage.contains_quarter(&key))), + ]; + let hourly_key = (hour, key.1.clone(), key.2.clone(), key.3.clone()); + for (index, key) in [key, hourly_key].into_iter().enumerate() { + if !included[index] { + continue; + } + let change = changes[index].entry(key).or_default(); + let state = if collection == forecast::COLLECTION { + &forecast + } else { + &state + }; + if adding { + change.current.merge(state)?; + } else { + change.previous.merge(state)?; + } + } + } + } + } + if clean_hours { + // 无重建中的时间桶时，小时贡献可直接合并季度贡献，避免逐事实再合并一遍全部指标 + let [quarters, hours] = &mut changes; + for ((quarter, collection, dimensions, outcome), contribution) in quarters.iter() { + let key = ( + bucket_for(*quarter, 3600)?, + collection.clone(), + dimensions.clone(), + outcome.clone(), + ); + let hour = hours.entry(key).or_default(); + hour.previous.merge(&contribution.previous)?; + hour.current.merge(&contribution.current)?; + } + } + for (index, changes) in changes.into_iter().enumerate() { + apply_changes(tx, index == 1, &progress, changes).await?; + } + Ok(()) +} + +async fn apply_changes( + tx: &mut sqlx::Transaction<'_, Postgres>, + hourly: bool, + progress: &BTreeMap, Progress>, + changes: BTreeMap, +) -> StoreResult<()> { + if changes.is_empty() { + return Ok(()); + } + let table = if hourly { + "request_analytics_hourly" + } else { + "request_analytics_15m" + }; + let keys = changes.keys().map(|(bucket,collection,dimensions,outcome)|serde_json::json!({"bucket_start":bucket,"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::>(); + let statement = format!( + "select a.* from {table} a join jsonb_to_recordset($1::jsonb) k(bucket_start timestamptz,collection text,dimensions text[],outcome text) using(bucket_start,collection,dimensions,outcome)" + ); + let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) + .bind(serde_json::Value::Array(keys)) + .fetch_all(&mut **tx) + .await + .map_err(|error| postgres_unavailable("read incremental analytics groups", error))?; + let mut stored = BTreeMap::new(); + for row in rows { + stored.insert( + ( + get(&row, "bucket_start")?, + get(&row, "collection")?, + get(&row, "dimensions")?, + get(&row, "outcome")?, + ), + MetricState::decode(&row)?, + ); + } + let mut buckets = + BTreeMap::, BTreeMap<(String, Vec, String), MetricState>>::new(); + for (key, change) in changes { + if change.previous.encode()? == change.current.encode()? + && change.previous.last_used_at() == change.current.last_used_at() + { + continue; + } + let mut state = stored.remove(&key).unwrap_or_default(); + let needs_rebuild = subtract(&mut state, &change.previous, &change.current)?; + state.merge(&change.current)?; + if state.count("request_count") == 0 { + let statement = format!( + "delete from {table} where bucket_start=$1 and collection=$2 and dimensions=$3 and outcome=$4" + ); + sqlx::query(sqlx::AssertSqlSafe(statement)) + .bind(key.0) + .bind(&key.1) + .bind(&key.2) + .bind(&key.3) + .execute(&mut **tx) + .await + .map_err(|error| postgres_unavailable("remove empty analytics group", error))?; + } else { + // 预测状态只有可加计数和美元总额，零金额无需修复币种存在性 + if needs_rebuild && key.1 != forecast::COLLECTION { + state = rebuild_group(tx, hourly, progress, &key).await?; + } + buckets + .entry(key.0) + .or_default() + .insert((key.1, key.2, key.3), state); + } + } + write_bucket_groups(tx, table, buckets).await +} + +/// 返回是否撤销了不能从可加状态推导的极值或币种存在性 +pub(super) fn subtract( + state: &mut MetricState, + previous: &MetricState, + current: &MetricState, +) -> StoreResult { + if previous.count("request_count") == 0 { + return Ok(false); + } + // 新贡献仍持有旧极值时无需重读事实，例如只补齐 Token、费用或重放同一请求 + let mut rebuild = previous.last_used_at.is_some() + && previous.last_used_at == state.last_used_at + && current.last_used_at < state.last_used_at; + for (name, value) in &previous.counts { + let count = state.counts.entry(name.clone()).or_default(); + *count = count + .checked_sub(*value) + .ok_or_else(|| invalid("analytics counter underflow"))?; + } + for (name, removed) in &previous.distributions { + let distribution = state + .distributions + .get_mut(name) + .ok_or_else(|| invalid("missing analytics distribution"))?; + distribution.count = distribution + .count + .checked_sub(removed.count) + .ok_or_else(|| invalid("analytics distribution underflow"))?; + distribution.zero = distribution + .zero + .checked_sub(removed.zero) + .ok_or_else(|| invalid("analytics zero count underflow"))?; + distribution.sum -= removed.sum; + for (bin, value) in &removed.bins { + let count = distribution + .bins + .get_mut(bin) + .ok_or_else(|| invalid("missing analytics bin"))?; + *count = count + .checked_sub(*value) + .ok_or_else(|| invalid("analytics bin underflow"))?; + } + distribution.bins.retain(|_, count| *count > 0); + if distribution.count == 0 { + *distribution = Default::default(); + } else { + let replacement = current.distributions.get(name); + rebuild |= distribution.minimum == removed.minimum + && replacement + .and_then(|value| value.minimum) + .is_none_or(|value| Some(value) > distribution.minimum); + rebuild |= distribution.maximum == removed.maximum + && replacement + .and_then(|value| value.maximum) + .is_none_or(|value| Some(value) < distribution.maximum); + rebuild |= !distribution.sum.is_finite() || distribution.sum < 0.0; + } + } + for (currency, amount) in &previous.costs { + let stored_amount = state + .costs + .get(currency) + .ok_or_else(|| invalid("missing analytics currency"))?; + let stored_amount: gateway_core::metering::Decimal = stored_amount + .parse() + .map_err(|_| invalid("invalid analytics amount"))?; + let removed: gateway_core::metering::Decimal = amount + .parse() + .map_err(|_| invalid("invalid analytics amount"))?; + let value = stored_amount + .scaled() + .checked_sub(removed.scaled()) + .ok_or_else(|| invalid("analytics amount underflow"))?; + rebuild |= value == 0 && !current.costs.contains_key(currency); + state.costs.insert( + currency.clone(), + gateway_core::metering::Decimal::from_scaled(value) + .map_err(|_| invalid("analytics amount overflow"))? + .canonical(), + ); + } + Ok(rebuild) +} + +async fn rebuild_group( + tx: &mut sqlx::Transaction<'_, Postgres>, + hourly: bool, + progress: &BTreeMap, Progress>, + key: &GroupKey, +) -> StoreResult { + let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 }); + let mut cursor = None; + if hourly { + if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4) + && let Some(time) = owner.started_at + { + let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone()); + end = if owner.contains_quarter(&candidate) { + time + TimeDelta::minutes(15) + } else { + time + }; + } + } else if let Some(stage) = progress.get(&key.0) + && stage.phase == 1 + { + cursor = stage.started_at.zip(stage.id.as_deref()); + } + let mut query = QueryBuilder::::new("with matching as materialized (select mr.*,"); + query.push_bind(key.0).push("::timestamptz as output_bucket,''::text as name,coalesce((") + .push(completed_usage_fact_predicate("mr")).push("),false) as is_completed_usage from request_analytics_facts mr where mr.started_at >= ") + .push_bind(key.0).push(" and mr.started_at < ").push_bind(end) + .push(" and mr.recovered_at is null and mr.outcome=").push_bind(&key.3); + if let Some((time, id)) = cursor { + query + .push(" and (mr.started_at,mr.id collate \"C\") <= (") + .push_bind(time) + .push(",") + .push_bind(id) + .push(")"); + } + let columns = worker::COLLECTION_COLUMNS + .iter() + .find_map(|(collection, columns)| (*collection == key.1).then_some(*columns)) + .ok_or_else(|| invalid("unknown incremental analytics collection"))?; + for (column, value) in columns.iter().zip(&key.2) { + query + .push(format!(" and coalesce(mr.{column}::text,'')=")) + .push_bind(value); + } + query.push(")"); + aggregate::push_fact_aggregation(&mut query, false, true); + let row = query + .build() + .fetch_optional(&mut **tx) + .await + .map_err(|error| postgres_unavailable("rebuild changed analytics extrema", error))?; + row.as_ref() + .map(MetricState::decode) + .transpose() + .map(Option::unwrap_or_default) +} diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs index 27eda0a98..c5e40dfa8 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs @@ -5,13 +5,25 @@ use super::histogram::Histogram; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgRow; +pub(super) const TOKEN_METRICS: [&str; 8] = [ + "input_tokens", + "output_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "reasoning_output_tokens", + "total_tokens", + "image_input_tokens", + "image_output_tokens", +]; + #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub(super) struct MetricState { - counts: BTreeMap, - distributions: BTreeMap, - costs: BTreeMap, +pub(crate) struct MetricState { + pub(super) counts: BTreeMap, + pub(super) distributions: BTreeMap, + pub(super) costs: BTreeMap, + pub(super) last_used_at: Option>, #[serde(skip)] - exact_samples: Option>>, + exact_percentiles: Option>>>, } impl MetricState { @@ -25,9 +37,23 @@ impl MetricState { "distributions", )? .0, - exact_samples: None, + last_used_at: get(row, "last_used_at")?, + exact_percentiles: None, }) } + pub(super) fn decode_aggregate(row: &PgRow, exact: bool) -> StoreResult { + let mut state = Self::decode(row)?; + if exact { + state.exact_percentiles = Some( + get::>>>>( + row, + "exact_percentiles", + )? + .0, + ); + } + Ok(state) + } pub fn encode(&self) -> StoreResult<(serde_json::Value, serde_json::Value, serde_json::Value)> { Ok(( serde_json::to_value(&self.counts) @@ -38,17 +64,24 @@ impl MetricState { .map_err(|error| postgres_unavailable("encode analytics histograms", error))?, )) } - pub fn exact() -> Self { - Self { - exact_samples: Some(BTreeMap::new()), - ..Self::default() - } - } pub fn count(&self, name: &str) -> u64 { self.counts.get(name).copied().unwrap_or_default() } - fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> { - let count = self.counts.entry(name.to_owned()).or_default(); + pub fn optional_count(&self, name: &str) -> Option { + (self.count(&format!("{name}_known_count")) > 0).then(|| self.count(name)) + } + pub fn last_used_at(&self) -> Option> { + self.last_used_at + } + pub(super) fn has_exact_percentiles(&self) -> bool { + self.exact_percentiles.is_some() + } + pub(super) fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> { + // 合并同一分组时键通常已经存在，避免每条事实都为已有指标分配字符串 + let count = match self.counts.get_mut(name) { + Some(count) => count, + None => self.counts.entry(name.to_owned()).or_default(), + }; *count = count .checked_add(amount) .ok_or_else(|| invalid("analytics counter overflow"))?; @@ -60,13 +93,10 @@ impl MetricState { .entry(name.to_owned()) .or_default() .add(sample)?; - if let Some(samples) = &mut self.exact_samples { - samples.entry(name.to_owned()).or_default().push(sample); - } } Ok(()) } - fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> { + pub(super) fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> { let amount: DecimalAmount = amount .parse() .map_err(|_| invalid("analytics invalid cost"))?; @@ -83,19 +113,28 @@ impl MetricState { Ok(()) } pub fn merge(&mut self, other: &Self) -> StoreResult<()> { + // 精确分位数只属于数据库已经完成分组的结果，不能把分位数再平均或合并 + if self.count("request_count") > 0 + && other.count("request_count") > 0 + && (self.exact_percentiles.is_some() || other.exact_percentiles.is_some()) + { + return Err(invalid( + "cannot merge finalized exact analytics percentiles", + )); + } + if self.count("request_count") == 0 { + self.exact_percentiles.clone_from(&other.exact_percentiles); + } + self.last_used_at = self.last_used_at.max(other.last_used_at); for (name, count) in &other.counts { self.increment(name, *count)?; } for (name, distribution) in &other.distributions { - self.distributions - .entry(name.clone()) - .or_default() - .merge(distribution)?; - } - if let (Some(ours), Some(theirs)) = (&mut self.exact_samples, &other.exact_samples) { - for (name, samples) in theirs { - ours.entry(name.clone()).or_default().extend(samples); - } + let stored = match self.distributions.get_mut(name) { + Some(stored) => stored, + None => self.distributions.entry(name.clone()).or_default(), + }; + stored.merge(distribution)?; } for (currency, amount) in &other.costs { self.add_cost(currency, amount)?; @@ -185,18 +224,31 @@ impl MetricState { self.sample("capacity", Some(used as f64 / total as f64 * 10_000.0))?; } if completed { - for name in [ - "input_tokens", - "output_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "reasoning_output_tokens", - "total_tokens", - "image_input_tokens", - "image_output_tokens", + self.increment("completed_request_count", 1)?; + self.last_used_at = self.last_used_at.max(Some(get(row, "started_at")?)); + for (value, name) in [ + (Some(true), "image_request_count"), + (Some(false), "image_request_failed_count"), ] { - self.increment(name, optional_unsigned(row, name)?.unwrap_or_default())?; + if get::>(row, "image_generation_succeeded")? == value { + self.increment(name, 1)?; + } } + for name in TOKEN_METRICS { + if let Some(value) = optional_unsigned(row, name)? { + self.increment(name, value)?; + self.increment(&format!("{name}_known_count"), 1)?; + } + } + self.increment( + "window_total_tokens", + optional_unsigned(row, "total_tokens")?.unwrap_or( + optional_unsigned(row, "input_tokens")? + .unwrap_or_default() + .checked_add(optional_unsigned(row, "output_tokens")?.unwrap_or_default()) + .ok_or_else(|| invalid("analytics token overflow"))?, + ), + )?; if get::>(row, "input_tokens")?.is_some() { self.increment("cache_eligible_request_count", 1)?; if optional_unsigned(row, "cached_input_tokens")?.is_some_and(|value| value > 0) { @@ -245,13 +297,11 @@ impl MetricState { self.distributions.get(name).cloned().unwrap_or_default() } pub fn percentile(&self, name: &str, fraction: f64) -> Option { - if let Some(samples) = &self.exact_samples { - let mut samples = samples.get(name)?.clone(); - samples.sort_by(f64::total_cmp); - let rank = samples.len().saturating_sub(1) as f64 * fraction; - let a = *samples.get(rank.floor() as usize)?; - let b = *samples.get(rank.ceil() as usize)?; - return Some(a + (b - a) * rank.fract()); + if let Some(percentiles) = &self.exact_percentiles { + let index = [0.1, 0.5, 0.9, 0.95, 0.99] + .iter() + .position(|value| (*value - fraction).abs() < f64::EPSILON)?; + return percentiles.get(name)?.get(index).copied().flatten(); } self.histogram(name).percentile(fraction) } diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/mod.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/mod.rs index 44616c02e..32d0111ff 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/mod.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/mod.rs @@ -1,7 +1,12 @@ //! 请求观测的持久化投影、可重建汇总与受预算约束的读取 +pub(crate) mod accounts; +mod aggregate; mod buckets; +mod counts; +mod forecast; mod histogram; +mod incremental; mod metrics; mod query; mod reset; @@ -11,15 +16,15 @@ pub(crate) use query::*; pub use worker::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository}; use super::*; -use metrics::MetricState; +pub(crate) use metrics::MetricState; const SCHEMA_VERSION: i32 = 1; const HISTOGRAM_VERSION: i32 = histogram::HISTOGRAM_VERSION; -const MAX_FACT_ROWS: usize = 100_000; pub(super) async fn begin_snapshot( - connection: &mut PgConnection, + connection: &mut ObservabilityConnection, ) -> StoreResult> { + let query_timeout = connection.query_timeout(); let mut transaction = connection .begin() .await @@ -28,7 +33,8 @@ pub(super) async fn begin_snapshot( .execute(&mut *transaction) .await .map_err(|error| postgres_unavailable("isolate analytics snapshot", error))?; - sqlx::query("set local statement_timeout = '10s'") + sqlx::query("select set_config('statement_timeout',$1,true)") + .bind(format!("{}ms", query_timeout.as_millis())) .execute(&mut *transaction) .await .map_err(|error| postgres_unavailable("set analytics query deadline", error))?; diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs index 4d54f421f..d41443659 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs @@ -5,7 +5,7 @@ use futures::TryStreamExt; pub(crate) struct AnalyticsSnapshot<'a> { pub tx: sqlx::Transaction<'a, Postgres>, - status: AnalyticsStatus, + pub(super) status: AnalyticsStatus, approximate: bool, } @@ -13,10 +13,25 @@ struct ReadSelection<'a> { distributions: bool, costs: bool, keys: Option<&'a [String]>, + counters: Option<&'a [&'a str]>, + distribution_names: Option<&'a [&'a str]>, } impl<'a> AnalyticsSnapshot<'a> { - pub async fn new(connection: &'a mut PgConnection) -> StoreResult { + pub async fn new(connection: &'a mut ObservabilityConnection) -> StoreResult { + Self::begin(connection, true).await + } + + pub(crate) async fn new_for_accounts( + connection: &'a mut ObservabilityConnection, + ) -> StoreResult { + Self::begin(connection, false).await + } + + async fn begin( + connection: &'a mut ObservabilityConnection, + require_current_schema: bool, + ) -> StoreResult { let mut tx = begin_snapshot(connection).await?; // 时间跨度和分组选择性差异很大，通用计划会把长区间误估成短区间 // 仅当前只读事务使用实际参数规划，不改变连接池后续事务的计划策略 @@ -25,8 +40,9 @@ impl<'a> AnalyticsSnapshot<'a> { .await .map_err(|error| postgres_unavailable("plan analytics with actual bounds", error))?; let status = worker::query_status(&mut tx).await?; - if status.schema_version != SCHEMA_VERSION as u32 - || status.histogram_version != HISTOGRAM_VERSION as u32 + if require_current_schema + && (status.schema_version != SCHEMA_VERSION as u32 + || status.histogram_version != HISTOGRAM_VERSION as u32) { return Err(unavailable( "request analytics is building; retry after the background projection completes", @@ -163,14 +179,35 @@ impl<'a> AnalyticsSnapshot<'a> { .load(range, filter, None, None, Default::default(), true) .await?; let state = merge_states(states.into_values())?; + self.overview_from_state(range, filter, state).await + } + + async fn overview_from_state( + &mut self, + range: ObservabilityRange, + filter: &UsageRecordFilter, + state: MetricState, + ) -> StoreResult { + // Provider 列表只消费计数，不重复合并费用与分布 let provider_states = self - .load( + .load_selected( range, filter, Some(DiagnosticDimension::Provider), None, Default::default(), - false, + ReadSelection { + distributions: false, + costs: false, + keys: None, + counters: Some(&[ + "request_count", + "attempt_count", + "failure_count", + "total_tokens", + ]), + distribution_names: None, + }, ) .await?; let mut providers = provider_states @@ -216,47 +253,48 @@ impl<'a> AnalyticsSnapshot<'a> { distributions: true, costs: include_costs, keys: None, + counters: None, + distribution_names: None, }, ) .await?; - let mut points = BTreeMap::new(); - for ((bucket, _), state) in states { - points.insert( - bucket, - RequestMetricPoint { - bucket_start: bucket, - granularity, - metrics: state.request_metrics()?, - cost_coverage: state.coverage(), - costs: if include_costs { - state.costs()? - } else { - Vec::new() - }, - }, - ); - } - fill_metric_gaps(range, granularity, points, timezone) + metric_points(range, granularity, timezone, &states, include_costs) } - pub async fn projected_costs( + pub async fn insights( &mut self, range: ObservabilityRange, filter: &UsageRecordFilter, granularity: ObservationGranularity, timezone: gateway_core::time::DeploymentTimeZone, - ) -> StoreResult> { - self.load(range, filter, None, Some(granularity), timezone, false) - .await? - .into_iter() + ) -> StoreResult<( + UsageOverview, + Vec, + Vec, + )> { + let states = self + .load(range, filter, None, Some(granularity), timezone, true) + .await?; + let trend = metric_points(range, granularity, timezone, &states, true)?; + let costs = states + .iter() .map(|((bucket_start, _), state)| { Ok(admin_observability::UsageProjectedCostPoint { - bucket_start, + bucket_start: *bucket_start, standard_amount: state.projected_cost("standard_usd")?, no_cache_amount: state.projected_cost("no_cache_usd")?, }) }) - .collect() + .collect::>>()?; + // 趋势已包含可合并的完整指标和情景费用，概览直接复用同一份状态 + // 精确分位数不可跨桶合并，明细回退时仍按完整区间单独计算概览 + let overview = if states.values().any(MetricState::has_exact_percentiles) { + self.overview(range, filter).await? + } else { + self.overview_from_state(range, filter, merge_states(states.into_values())?) + .await? + }; + Ok((overview, trend, costs)) } pub async fn diagnostics( @@ -273,8 +311,10 @@ impl<'a> AnalyticsSnapshot<'a> { let ranked = if matches!( dimension, DiagnosticDimension::Account | DiagnosticDimension::ApiKey - ) && collection_for(filter, Some(dimension)).is_some() - { + ) && matches!( + collection_for(filter, Some(dimension)), + Some("account" | "key") + ) { Some( self.diagnostic_top_keys(range, filter, dimension, limit) .await?, @@ -294,6 +334,22 @@ impl<'a> AnalyticsSnapshot<'a> { distributions: true, costs: true, keys: selected_keys, + counters: Some(&[ + "request_count", + "success_count", + "failure_count", + "attempt_count", + "total_tokens", + "latency_ms_sum", + "cancelled_count", + "incomplete_count", + "retry_count", + "retried_request_count", + "provider_reported_count", + "calculated_count", + "unavailable_count", + ]), + distribution_names: Some(&["latency_ms", "first_token_ms"]), }, ) .await?; @@ -415,16 +471,18 @@ impl<'a> AnalyticsSnapshot<'a> { .collect::>(); let mut query = QueryBuilder::::new("with edge_facts as materialized ("); if starts.is_empty() { - query.push("select null::text as key where false"); + query.push("select null::text as key,0::bigint as request_count where false"); } else { - query.push("select edge.key from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ") - .push(format!("coalesce(mr.{column},'') as key from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at")); + query.push("select edge.key,sum(edge.request_count)::bigint as request_count from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ") + .push(format!("coalesce(mr.{column},'') as key,count(*)::bigint as request_count from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at")); push_unrecovered_request_filter(&mut query, "mr"); push_analytics_filter(&mut query, filter, "mr"); push_diagnostic_dimension_filter(&mut query, dimension); - query.push(" limit 100001) edge limit 100001"); + query.push(format!( + " group by coalesce(mr.{column},'')) edge group by edge.key" + )); } - query.push("), contributions as (select key,count(*)::bigint as request_count from edge_facts group by key"); + query.push("), contributions as (select key,request_count from edge_facts"); for (table, column, seconds, buckets) in [ ("request_analytics_daily_counts", "dimension", 86400, days), ("request_analytics_hourly", "dimensions[1]", 3600, hours), @@ -446,7 +504,7 @@ impl<'a> AnalyticsSnapshot<'a> { } query.push(format!(" group by {column}")); } - query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count,(select count(*) from edge_facts)::bigint as edge_count from grouped order by request_count desc,"); + query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count from grouped order by request_count desc,"); if dimension == DiagnosticDimension::Account { query.push("coalesce(nullif(key,''),'unrouted')"); } else { @@ -458,12 +516,6 @@ impl<'a> AnalyticsSnapshot<'a> { .fetch_all(&mut *self.tx) .await .map_err(|error| postgres_unavailable("rank full-range analytics groups", error))?; - if rows.first().is_some_and(|row| { - row.try_get::("edge_count") - .is_ok_and(|count| count > MAX_FACT_ROWS as i64) - }) { - return Err(unavailable("analytics fact row budget exceeded")); - } let total = rows .first() .map(|row| unsigned(row, "total_request_count")) @@ -476,18 +528,15 @@ impl<'a> AnalyticsSnapshot<'a> { Ok((total, keys)) } - async fn summary_buckets( + pub(super) async fn summary_buckets( &mut self, range: ObservabilityRange, use_hourly: bool, ) -> StoreResult<(Vec>, Vec>)> { // 先按时间确定互斥的完整桶，空的十五分钟集合不读取汇总表 - let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start limit 100001") + let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start") .bind(range.start).bind(range.end).bind(use_hourly) .fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("select disjoint analytics buckets",error))?; - if rows.len() > MAX_FACT_ROWS { - return Err(unavailable("analytics time bucket budget exceeded")); - } let mut hours = Vec::new(); let mut quarters = Vec::new(); for row in rows { @@ -502,7 +551,7 @@ impl<'a> AnalyticsSnapshot<'a> { Ok((hours, quarters)) } - async fn uncovered_spans( + pub(super) async fn uncovered_spans( &mut self, range: ObservabilityRange, ) -> StoreResult, DateTime)>> { @@ -522,11 +571,8 @@ impl<'a> AnalyticsSnapshot<'a> { spans.push((right_start, range.end)); } if complete_start < complete_end { - let dirty:Vec>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start limit 100001") + let dirty:Vec>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start") .bind(complete_start).bind(complete_end).fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("plan bounded analytics fact spans",error))?; - if dirty.len() > MAX_FACT_ROWS { - return Err(unavailable("analytics dirty interval budget exceeded")); - } spans.extend( dirty .into_iter() @@ -534,7 +580,17 @@ impl<'a> AnalyticsSnapshot<'a> { ); } spans.sort_unstable(); - Ok(spans) + let mut merged: Vec<(DateTime, DateTime)> = Vec::new(); + for (start, end) in spans { + if let Some(previous) = merged.last_mut() + && previous.1 == start + { + previous.1 = end; + } else { + merged.push((start, end)); + } + } + Ok(merged) } async fn load( @@ -556,6 +612,8 @@ impl<'a> AnalyticsSnapshot<'a> { distributions: read_distributions, costs: true, keys: None, + counters: None, + distribution_names: None, }, ) .await @@ -574,6 +632,8 @@ impl<'a> AnalyticsSnapshot<'a> { distributions: read_distributions, costs: read_costs, keys: selected_keys, + counters, + distribution_names, } = selection; filter.validate()?; self.require_visible(Some(range)).await?; @@ -610,90 +670,63 @@ impl<'a> AnalyticsSnapshot<'a> { if buckets.is_empty() { continue; } - let (dimension_filter, either_filter, either_positions) = match collection { - "provider_model" => { - (filter.provider_kind.clone(), filter.model.clone(), (2, 3)) - } - "account" => (filter.provider_account_ref.clone(), None, (1, 2)), - "key" => (filter.client_api_key_ref.clone(), None, (1, 2)), - "operation" => (filter.operation.clone(), None, (1, 2)), - "transport" => (None, filter.transport.clone(), (1, 2)), - "status" => ( - None, - filter.status_code.map(|value| value.to_string()), - (1, 2), - ), - _ => (None, None, (1, 2)), + let mut query = + QueryBuilder::::new("with matching as not materialized (select "); + aggregate::push_output_bucket( + &mut query, + "bucket_start", + range, + granularity, + boundaries.as_deref(), + ); + let name = aggregate::summary_dimension_sql(collection, dimension); + let costs = if read_costs { + "costs" + } else { + "'{}'::jsonb as costs" }; let distributions = if read_distributions { "distributions" } else { "'{}'::jsonb as distributions" }; - let costs = if read_costs { - "costs" - } else { - "'{}'::jsonb as costs" - }; - let provider_counts = dimension == Some(DiagnosticDimension::Provider) - && granularity.is_none() - && !read_distributions; - let mut query = QueryBuilder::::new(if provider_counts { - "select min(bucket_start) as bucket_start, array[dimensions[1]] as dimensions, ''::text as outcome, jsonb_build_object('request_count',sum((counts->>'request_count')::bigint),'attempt_count',sum((counts->>'attempt_count')::bigint),'failure_count',sum(coalesce((counts->>'failure_count')::bigint,0)),'total_tokens',sum(coalesce((counts->>'total_tokens')::bigint,0))) as counts, '{}'::jsonb as costs, '{}'::jsonb as distributions from (" - } else { - "" - }); - query.push(format!("select bucket_start, dimensions, outcome, counts, {costs}, {distributions} from {table} a where collection=")) + query.push(format!(" as output_bucket,{name} as name,counts,{costs},{distributions},last_used_at from {table} a where collection=")) .push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION) .push(" and histogram_version=").push_bind(HISTOGRAM_VERSION); push_summary_buckets(&mut query, &buckets, seconds); - // 可选筛选仅在有值时生成谓词，保留维度与时间索引的选择空间 - if let Some(outcome) = &filter.outcome { - query.push(" and outcome=").push_bind(outcome); - } - if let Some(value) = dimension_filter { - query.push(" and dimensions[1]=").push_bind(value); - } - if let Some(value) = either_filter { - query - .push(format!(" and (dimensions[{}]=", either_positions.0)) - .push_bind(value.clone()) - .push(format!(" or dimensions[{}]=", either_positions.1)) - .push_bind(value) - .push(")"); - } + push_collection_filter(&mut query, collection, filter); if let Some(keys) = selected_keys { query .push(" and dimensions[1]=any(") .push_bind(keys) .push(")"); } - if provider_counts { - query.push(") matching group by dimensions[1]"); + if matches!( + dimension, + Some( + DiagnosticDimension::Model + | DiagnosticDimension::Status + | DiagnosticDimension::Failure + ) + ) { + query.push(format!(" and ({name})<>''")); } - query.push(" limit 100001"); + query.push(")"); + aggregate::push_summary_aggregation( + &mut query, + read_distributions, + read_costs, + counters, + distribution_names, + ); let mut rows = query.build().fetch(&mut *self.tx); - let mut count = 0; while let Some(row) = rows .try_next() .await - .map_err(|error| postgres_unavailable("read analytics summaries", error))? + .map_err(|error| postgres_unavailable("merge analytics summaries", error))? { - count += 1; - if count > MAX_FACT_ROWS { - return Err(unavailable("analytics aggregate row budget exceeded")); - } - let dimensions: Vec = get(&row, "dimensions")?; - let name = dimension_name(collection, &dimensions, dimension); - let Some(name) = name else { - continue; - }; - let bucket = output_bucket( - get(&row, "bucket_start")?, - range, - granularity, - boundaries.as_deref(), - )?; + let bucket = get(&row, "bucket_start")?; + let name = get(&row, "name")?; let state = MetricState::decode(&row)?; result.entry((bucket, name)).or_default().merge(&state)?; } @@ -707,28 +740,42 @@ impl<'a> AnalyticsSnapshot<'a> { if spans.is_empty() { return Ok(result); } - // 单段直接绑定边界，让优化器看见真实跨度，避免把三十天误估成小范围 BitmapAnd + // 单段直接绑定边界，多个互斥区间用索引读取，所有统计都在数据库内完成 let single_span = spans.len() == 1; - let mut query = if single_span { - let mut query = QueryBuilder::::new(fact_select()); - query - .push(" where mr.started_at >= ") - .push_bind(spans[0].0) - .push(" and mr.started_at < ") - .push_bind(spans[0].1); - query - } else { - let (starts, ends): (Vec<_>, Vec<_>) = spans.into_iter().unzip(); - let mut query = QueryBuilder::::new("select fact.* from unnest("); + let mut query = QueryBuilder::::new("with matching as not materialized ("); + if !single_span { + let (starts, ends): (Vec<_>, Vec<_>) = spans.iter().copied().unzip(); query + .push("select fact.* from unnest(") .push_bind(starts) .push("::timestamptz[],") .push_bind(ends) - .push("::timestamptz[]) span(start_at,end_at) cross join lateral (") - .push(fact_select()) - .push(" where mr.started_at >= span.start_at and mr.started_at < span.end_at"); + .push("::timestamptz[]) span(start_at,end_at) cross join lateral ("); + } + query.push(format!( + "select mr.*,coalesce(({}),false) as is_completed_usage,", + completed_usage_fact_predicate("mr") + )); + aggregate::push_output_bucket( + &mut query, + "mr.started_at", + range, + granularity, + boundaries.as_deref(), + ); + query.push(format!( + " as output_bucket,{} as name from request_analytics_facts mr where ", + aggregate::fact_dimension_sql(dimension) + )); + if single_span { query - }; + .push("mr.started_at >= ") + .push_bind(spans[0].0) + .push(" and mr.started_at < ") + .push_bind(spans[0].1); + } else { + query.push("mr.started_at >= span.start_at and mr.started_at < span.end_at"); + } push_unrecovered_request_filter(&mut query, "mr"); push_analytics_filter(&mut query, filter, "mr"); if let Some(dimension) = dimension { @@ -749,42 +796,40 @@ impl<'a> AnalyticsSnapshot<'a> { } query.push(")"); } - query.push(" limit 100001"); if !single_span { - query.push(") fact limit 100001"); + // 保留逐区间的参数化索引扫描，避免展开 LATERAL 后全表读取再过滤 + // OFFSET 0 不限制返回行数 + query.push(" offset 0) fact"); } + query.push(")"); + let exact = selected.is_none(); + aggregate::push_fact_aggregation_selected( + &mut query, + exact, + read_costs, + read_distributions, + counters, + distribution_names, + ); let mut rows = query.build().fetch(&mut *self.tx); - let mut count = 0; while let Some(row) = rows .try_next() .await - .map_err(|error| postgres_unavailable("read bounded analytics facts", error))? + .map_err(|error| postgres_unavailable("aggregate analytics facts", error))? { - count += 1; - if count > MAX_FACT_ROWS { - return Err(unavailable("analytics fact row budget exceeded")); + let key = (get(&row, "bucket_start")?, get(&row, "name")?); + let state = MetricState::decode_aggregate(&row, exact)?; + if exact { + result.insert(key, state); + } else { + result.entry(key).or_default().merge(&state)?; } - let bucket = output_bucket( - get(&row, "started_at")?, - range, - granularity, - boundaries.as_deref(), - )?; - let name = fact_dimension(&row, dimension)?; - let state = result.entry((bucket, name)).or_insert_with(|| { - if selected.is_some() { - MetricState::default() - } else { - MetricState::exact() - } - }); - state.add_row(&row)?; } Ok(result) } } -fn push_summary_buckets( +pub(super) fn push_summary_buckets( query: &mut QueryBuilder, buckets: &[DateTime], seconds: i64, @@ -810,6 +855,33 @@ fn push_summary_buckets( } } +fn metric_points( + range: ObservabilityRange, + granularity: ObservationGranularity, + timezone: gateway_core::time::DeploymentTimeZone, + states: &BTreeMap<(DateTime, String), MetricState>, + include_costs: bool, +) -> StoreResult> { + let mut points = BTreeMap::new(); + for ((bucket, _), state) in states { + points.insert( + *bucket, + RequestMetricPoint { + bucket_start: *bucket, + granularity, + metrics: state.request_metrics()?, + cost_coverage: state.coverage(), + costs: if include_costs { + state.costs()? + } else { + Vec::new() + }, + }, + ); + } + fill_metric_gaps(range, granularity, points, timezone) +} + fn merge_states(states: impl Iterator) -> StoreResult { let mut result: Option = None; for state in states { @@ -822,28 +894,7 @@ fn merge_states(states: impl Iterator) -> StoreResult, - range: ObservabilityRange, - granularity: Option, - boundaries: Option<&[DateTime]>, -) -> StoreResult> { - match granularity { - None => Ok(range.start), - Some(ObservationGranularity::Day) => boundaries - .and_then(|values| { - values - .iter() - .rev() - .find(|boundary| **boundary <= time) - .copied() - }) - .ok_or_else(|| invalid("missing analytics calendar boundary")), - Some(granularity) => bucket_for(time, granularity.seconds()), - } -} - -fn collection_for( +pub(super) fn collection_for( filter: &UsageRecordFilter, dimension: Option, ) -> Option<&'static str> { @@ -852,7 +903,6 @@ fn collection_for( || filter.response_id.is_some() || filter.upstream_request_id.is_some() || filter.search.is_some() - || !filter.routing_group_refs.is_empty() { return None; } @@ -885,85 +935,18 @@ fn collection_for( DiagnosticDimension::Status => "status", }); } - if required.len() > 1 { + if required.len() == 2 && required.contains("provider_model") && required.contains("account") { + Some("account_model") + } else if required.len() == 2 && required.contains("provider_model") && required.contains("key") + { + Some("key_model") + } else if required.len() > 1 { None } else { Some(required.first().copied().unwrap_or("global")) } } -fn dimension_name( - collection: &str, - values: &[String], - dimension: Option, -) -> Option { - let Some(dimension) = dimension else { - return Some(String::new()); - }; - let value = |index| values.get(index).cloned().unwrap_or_default(); - let coalesce = |preferred: usize, fallback: usize| { - let v = value(preferred); - if v.is_empty() { value(fallback) } else { v } - }; - let name = match dimension { - DiagnosticDimension::Provider if collection == "provider_model" => { - let v = value(0); - if v.is_empty() { - "unrouted".to_owned() - } else { - v - } - } - DiagnosticDimension::Model => coalesce(2, 1), - DiagnosticDimension::Account => { - let v = value(0); - if v.is_empty() { - "unrouted".to_owned() - } else { - v - } - } - DiagnosticDimension::Transport => { - let v = coalesce(1, 0); - if v.is_empty() { - "unknown".to_owned() - } else { - v - } - } - DiagnosticDimension::Status => coalesce(1, 0), - _ => value(0), - }; - if name.is_empty() - && matches!( - dimension, - DiagnosticDimension::Model | DiagnosticDimension::Status | DiagnosticDimension::Failure - ) - { - None - } else { - Some(name) - } -} - -fn fact_dimension( - row: &sqlx::postgres::PgRow, - dimension: Option, -) -> StoreResult { - let Some(dimension) = dimension else { - return Ok(String::new()); - }; - let collection = collection_for(&UsageRecordFilter::default(), Some(dimension)) - .ok_or_else(|| invalid("missing analytics dimension"))?; - let dimensions = worker::collections(row)? - .into_iter() - .find(|(name, _)| *name == collection) - .map(|(_, values)| values) - .ok_or_else(|| invalid("missing analytics dimension"))?; - dimension_name(collection, &dimensions, Some(dimension)) - .ok_or_else(|| invalid("unexpected empty analytics dimension")) -} - pub(crate) fn push_analytics_filter( query: &mut QueryBuilder, filter: &UsageRecordFilter, @@ -982,3 +965,53 @@ pub(crate) fn push_analytics_filter( query.push(")"); } } + +pub(super) fn push_collection_filter( + query: &mut QueryBuilder, + collection: &str, + filter: &UsageRecordFilter, +) { + let (dimension_filter, either_filter, either_positions) = match collection { + "provider_model" => (filter.provider_kind.clone(), filter.model.clone(), (2, 3)), + "account_model" => ( + filter.provider_account_ref.clone(), + filter.model.clone(), + (3, 4), + ), + "key_model" => ( + filter.client_api_key_ref.clone(), + filter.model.clone(), + (3, 4), + ), + "account" => (filter.provider_account_ref.clone(), None, (1, 2)), + "key" => (filter.client_api_key_ref.clone(), None, (1, 2)), + "operation" => (filter.operation.clone(), None, (1, 2)), + "transport" => (None, filter.transport.clone(), (1, 2)), + "status" => ( + None, + filter.status_code.map(|value| value.to_string()), + (1, 2), + ), + _ => (None, None, (1, 2)), + }; + // 可选筛选仅在有值时生成谓词，保留维度与时间索引的选择空间 + if let Some(outcome) = &filter.outcome { + query.push(" and outcome=").push_bind(outcome); + } + if let Some(value) = dimension_filter { + query.push(" and dimensions[1]=").push_bind(value); + } + if matches!(collection, "account_model" | "key_model") + && let Some(provider) = &filter.provider_kind + { + query.push(" and dimensions[2]=").push_bind(provider); + } + if let Some(value) = either_filter { + query + .push(format!(" and (dimensions[{}]=", either_positions.0)) + .push_bind(value.clone()) + .push(format!(" or dimensions[{}]=", either_positions.1)) + .push_bind(value) + .push(")"); + } +} diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/worker.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/worker.rs index cd9497980..1a5fd265e 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/worker.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/worker.rs @@ -83,11 +83,11 @@ impl PgRequestAnalyticsRepository { return Ok(AnalyticsMaintenance { reset_progress: true, ..Default::default() }); } let mut result = AnalyticsMaintenance::default(); - let rows = sqlx::query("select request_id, change_version, oldest_pending_at, previous_bucket from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1") + let rows = sqlx::query("select request_id, change_version from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1") .bind(i64::from(batch_size)).fetch_all(&mut *transaction).await.map_err(|error| postgres_unavailable("load analytics request queue", error))?; let mut jobs = Vec::with_capacity(rows.len()); for row in rows { - jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")?, pending_at: get(&row,"oldest_pending_at")?, previous_bucket: get(&row,"previous_bucket")? }); + jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")? }); } self.project_batch(&mut transaction, &jobs).await?; result.projected_requests = jobs.len() as u64; @@ -138,53 +138,36 @@ impl PgRequestAnalyticsRepository { } let ids = jobs.iter().map(|job| job.id.clone()).collect::>(); let versions = jobs.iter().map(|job| job.version).collect::>(); - let pending = jobs - .iter() - .map(|job| (job.id.as_str(), job.pending_at)) - .collect::>(); - let mut buckets = BTreeMap::, DateTime>::new(); - let mut affect = |id: &str, start: DateTime| -> StoreResult<()> { - let pending_at = pending - .get(id) - .copied() - .ok_or_else(|| invalid("analytics job disappeared"))?; - let bucket = bucket_for(start, 900)?; - buckets - .entry(bucket) - .and_modify(|old| *old = (*old).min(pending_at)) - .or_insert(pending_at); - Ok(()) - }; - let rows = sqlx::query( - "delete from request_analytics_facts where id=any($1::text[]) returning id,started_at", - ) + let previous = sqlx::query(sqlx::AssertSqlSafe(format!( + "{} where mr.id=any($1::text[])", + fact_select() + ))) .bind(&ids) .fetch_all(&mut **tx) .await - .map_err(|error| postgres_unavailable("replace analytics facts", error))?; - for row in rows { - affect(&get::(&row, "id")?, get(&row, "started_at")?)?; - } - let rows = sqlx::query(PROJECT_REQUEST) + .map_err(|error| postgres_unavailable("read previous analytics contributions", error))?; + sqlx::query("delete from request_analytics_facts where id=any($1::text[])") + .bind(&ids) + .execute(&mut **tx) + .await + .map_err(|error| postgres_unavailable("replace analytics facts", error))?; + sqlx::query(PROJECT_REQUEST) .bind(&ids) .bind(versions) - .fetch_all(&mut **tx) + .execute(&mut **tx) .await .map_err(|error| postgres_unavailable("project analytics facts", error))?; - for row in rows { - affect(&get::(&row, "id")?, get(&row, "started_at")?)?; - } - for job in jobs { - if let Some(bucket) = job.previous_bucket { - affect(&job.id, bucket)?; - } - } self.project_billing_batch(tx, &ids).await?; - if !buckets.is_empty() { - let (starts, times): (Vec<_>, Vec<_>) = buckets.into_iter().unzip(); - sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select * from unnest($1::timestamptz[],$2::timestamptz[]) on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version'),oldest_pending_at=least(request_analytics_pending_buckets.oldest_pending_at,excluded.oldest_pending_at)") - .bind(starts).bind(times).execute(&mut **tx).await.map_err(|error| postgres_unavailable("enqueue analytics buckets",error))?; - } + let current = sqlx::query(sqlx::AssertSqlSafe(format!( + "{} where mr.id=any($1::text[])", + fact_select() + ))) + .bind(&ids) + .fetch_all(&mut **tx) + .await + .map_err(|error| postgres_unavailable("read current analytics contributions", error))?; + // 事实、已发布汇总与构建前缀在同一事务中撤销和添加，重试不会重复计数 + super::incremental::update_summaries(tx, &previous, ¤t).await?; Ok(()) } @@ -199,7 +182,7 @@ impl PgRequestAnalyticsRepository { return Ok(0); } let mut query = QueryBuilder::::new( - "select id,started_at,nextval('request_analytics_change_version') as version from model_requests", + "select id,started_at,nextval('request_analytics_change_version') as version,exists(select 1 from request_analytics_facts f where f.id=model_requests.id) as projected from model_requests", ); // 首批与后续批次使用两种明确的计划，避免可空游标的 OR 使通用计划退化为全索引过滤 if let Some(start) = get::>>(&state, "backfill_cursor_started_at")? { @@ -222,11 +205,12 @@ impl PgRequestAnalyticsRepository { .map_err(|error| postgres_unavailable("read analytics backfill batch", error))?; let mut jobs = Vec::with_capacity(rows.len()); for row in &rows { + if get::(row, "projected")? { + continue; + } jobs.push(ProjectionJob { id: get(row, "id")?, version: get(row, "version")?, - pending_at: Utc::now(), - previous_bucket: None, }); } self.project_batch(tx, &jobs).await?; @@ -312,49 +296,68 @@ async fn read_status( }) } +// 投影与极值重算共用同一维度顺序，避免两条路径对集合身份作不同解释 +pub(super) const COLLECTION_COLUMNS: &[(&str, &[&str])] = &[ + ("global", &[]), + ( + "provider_model", + &["provider_kind", "requested_model_id", "upstream_model_id"], + ), + ("account", &["provider_account_ref"]), + ("key", &["client_api_key_ref"]), + ( + "account_model", + &[ + "provider_account_ref", + "provider_kind", + "requested_model_id", + "upstream_model_id", + ], + ), + ( + "key_model", + &[ + "client_api_key_ref", + "provider_kind", + "requested_model_id", + "upstream_model_id", + ], + ), + ("operation", &["operation"]), + ("transport", &["client_transport", "upstream_transport"]), + ("status", &["client_status_code", "upstream_status_code"]), + ("failure", &["error_kind"]), +]; + pub(super) fn collections( row: &sqlx::postgres::PgRow, ) -> StoreResult)>> { - let text = - |name| -> StoreResult { Ok(get::>(row, name)?.unwrap_or_default()) }; - let status = |name| -> StoreResult { - Ok(get::>(row, name)? - .map(|value| value.to_string()) - .unwrap_or_default()) - }; - Ok(vec![ - ("global", vec![]), - ( - "provider_model", - vec![ - text("provider_kind")?, - text("requested_model_id")?, - text("upstream_model_id")?, - ], - ), - ("account", vec![text("provider_account_ref")?]), - ("key", vec![text("client_api_key_ref")?]), - ("operation", vec![text("operation")?]), - ( - "transport", - vec![text("client_transport")?, text("upstream_transport")?], - ), - ( - "status", - vec![ - status("client_status_code")?, - status("upstream_status_code")?, - ], - ), - ("failure", vec![text("error_kind")?]), - ]) + let mut result = vec![(forecast::COLLECTION, forecast::dimensions(row)?)]; + // 预测保留原始完成状态，包括被恢复的请求；其他统计继续排除恢复记录 + if get::>>(row, "recovered_at")?.is_some() { + return Ok(result); + } + for &(collection, columns) in COLLECTION_COLUMNS { + let dimensions = columns + .iter() + .map(|column| { + if matches!(*column, "client_status_code" | "upstream_status_code") { + Ok(get::>(row, column)? + .map(|value| value.to_string()) + .unwrap_or_default()) + } else { + Ok(get::>(row, column)?.unwrap_or_default()) + } + }) + .collect::>>()?; + result.push((collection, dimensions)); + } + Ok(result) } struct ProjectionJob { id: String, version: i64, - pending_at: DateTime, - previous_bucket: Option>, } async fn publish_generation( diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index 534f069a7..8d847de88 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -17,7 +17,6 @@ pub(crate) fn store_range( pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter { UsageRecordFilter { - routing_group_refs: filter.routing_group_refs, client_api_key_ref: filter.client_api_key_ref, request_id: filter.request_id, provider_account_ref: filter.provider_account_ref, diff --git a/backend/crates/gateway-store/src/postgres/observability/mod.rs b/backend/crates/gateway-store/src/postgres/observability/mod.rs index d3db2308e..bf29c4234 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mod.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mod.rs @@ -31,6 +31,10 @@ mod queries; mod query_budget; pub use admin_adapter::*; +pub(crate) use analytics::AnalyticsSnapshot; +pub(crate) use analytics::accounts::{ + account_group_usage, account_usage_sort_values, account_usage_windows, +}; pub use analytics::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository}; pub(crate) use mapping::*; pub use model::*; diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs index 4fb9e6c86..4db2dd92a 100644 --- a/backend/crates/gateway-store/src/postgres/observability/model.rs +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -39,7 +39,6 @@ pub(crate) fn observability_page_offset( #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct UsageRecordFilter { - pub routing_group_refs: Vec, pub client_api_key_ref: Option, pub request_id: Option, pub provider_account_ref: Option, @@ -76,7 +75,6 @@ impl UsageRecordFilter { ] { validate_optional_text(value, MAX_FILTER_BYTES, field)?; } - validate_account_ids_if_present(&self.routing_group_refs)?; validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?; if self .status_code @@ -105,14 +103,6 @@ pub struct UsageRecordQuery { pub use gateway_admin::model::observability::OpsErrorFilter; pub(crate) fn validate_ops_error_filter(filter: &OpsErrorFilter) -> StoreResult<()> { - if filter.routing_group_refs.len() > 32 - || filter.routing_group_refs.iter().any(|value| { - value.is_empty() || value.len() > 128 || value.chars().any(char::is_control) - }) - { - return Err(invalid("invalid routing group filter")); - } - for (value, field) in [ ( filter.client_api_key_ref.as_deref(), @@ -384,10 +374,3 @@ pub struct DiagnosticObservation { pub use gateway_admin::model::observability::OpsError as OpsErrorRecord; pub use gateway_admin::model::observability::OpsErrorPage; - -fn validate_account_ids_if_present(ids: &[String]) -> StoreResult<()> { - if ids.is_empty() { - return Ok(()); - } - validate_account_ids(ids) -} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs index f0c15eaf4..eb60c2a6d 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs @@ -3,7 +3,7 @@ use super::super::*; pub(crate) async fn provider_account_usage( - connection: &mut sqlx::PgConnection, + snapshot: &mut analytics::AnalyticsSnapshot<'_>, query: ProviderAccountUsageQuery, ) -> StoreResult> { if let Some(account_ids) = &query.account_ids { @@ -13,290 +13,133 @@ pub(crate) async fn provider_account_usage( return Err(invalid("account usage limit must be between 1 and 200")); } - // 账号卡片先确定有限候选，避免为只展示二十个账号聚合整个历史范围 - let mut statement = QueryBuilder::::new( - "with candidate_accounts as materialized (select pa.* from provider_accounts pa", - ); + // 账号卡片先确定有限候选，避免为少量账号聚合整个历史范围 + let mut statement = QueryBuilder::::new("select pa.* from provider_accounts pa"); if let Some(account_ids) = &query.account_ids { statement .push(" where pa.id=any(") .push_bind(account_ids.clone()) .push("::text[])"); } else { - statement.push(" join lateral (select mr.started_at from request_analytics_facts mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start) + statement.push(" join lateral (select mr.started_at from model_requests mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start) .push(" and mr.started_at < ").push_bind(query.range.end); push_completed_usage_fact_filter(&mut statement, "mr"); statement.push(" order by mr.started_at desc,mr.id desc limit 1) latest on true order by latest.started_at desc,pa.name,pa.id limit ").push_bind(i64::from(query.limit)); } - statement.push( - "), matched as materialized ( - select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, - pa.plan_type, mr.id as request_id, - coalesce(mr.upstream_model_id, mr.requested_model_id) as model, - mr.cost_currency, mr.outcome, mr.input_tokens, mr.output_tokens, - mr.cached_input_tokens, mr.cache_write_input_tokens, mr.reasoning_output_tokens, - mr.image_input_tokens, mr.image_output_tokens, - mr.image_generation_succeeded, mr.total_tokens, mr.cost_source, - mr.cost_amount, mr.started_at - from candidate_accounts pa - left join request_analytics_facts mr - on mr.provider_account_ref = pa.id and mr.started_at >= ", - ); - statement.push_bind(query.range.start); - statement.push(" and mr.started_at < "); - statement.push_bind(query.range.end); - push_completed_usage_fact_filter(&mut statement, "mr"); - if let Some(account_ids) = &query.account_ids { - statement.push(" where pa.id = any("); - statement.push_bind(account_ids.clone()); - statement.push("::text[])"); - } - statement.push( - " limit 100001), aggregated as ( - select id, provider_kind, authentication_kind, name, email, plan_type, - model, cost_currency, - grouping(model)::integer as model_grouping, - grouping(cost_currency)::integer as currency_grouping, - count(request_id)::bigint as request_count, - count(request_id) filter (where outcome = 'succeeded')::bigint - as success_count, - sum(input_tokens)::bigint as input_tokens, - sum(output_tokens)::bigint as output_tokens, - sum(cached_input_tokens)::bigint as cached_input_tokens, - sum(cache_write_input_tokens)::bigint as cache_write_input_tokens, - sum(reasoning_output_tokens)::bigint as reasoning_output_tokens, - sum(image_input_tokens)::bigint as image_input_tokens, - sum(image_output_tokens)::bigint as image_output_tokens, - count(request_id) filter (where image_generation_succeeded is true)::bigint - as image_request_count, - count(request_id) filter (where image_generation_succeeded is false)::bigint - as image_request_failed_count, - sum(total_tokens)::bigint as total_tokens, - count(request_id) filter (where cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(request_id) filter (where cost_source = 'calculated')::bigint - as calculated_count, - count(request_id) filter (where cost_source = 'unavailable')::bigint - as unavailable_count, - max(started_at) as last_used_at, - sum(cost_amount)::text as amount - from matched - group by id, provider_kind, authentication_kind, name, email, plan_type, - grouping sets ((), (cost_currency), (model), (model, cost_currency)) - ), selected_accounts as ( - select id, - row_number() over (order by last_used_at desc nulls last, name, id) - as sort_position - from aggregated - where model_grouping = 1 and currency_grouping = 1", - ); - if query.account_ids.is_none() { - statement.push(" and request_count > 0"); - } - statement.push(" order by last_used_at desc nulls last, name, id limit "); - statement.push_bind(i64::from(query.limit)); - statement.push( - ") - select aggregated.*, (select count(*) from matched)::bigint as matched_count - from aggregated - join selected_accounts selected using (id) - order by selected.sort_position, model_grouping desc, currency_grouping desc, - model nulls last, cost_currency nulls last", - ); let rows = statement .build() - .fetch_all(&mut *connection) + .fetch_all(&mut *snapshot.tx) .await - .map_err(|source| postgres_unavailable("load provider account usage", source))?; - if let Some(row) = rows.first() - && unsigned(row, "matched_count")? > 100_000 + .map_err(|error| postgres_unavailable("select usage accounts", error))?; + let account_ids = rows + .iter() + .map(|row| get(row, "id")) + .collect::>>()?; + let mut models_by_account = HashMap::>::new(); + let mut totals = BTreeMap::::new(); + for ((account, model), state) in snapshot + .account_usage_states(query.range, &account_ids) + .await? { - return Err(crate::StoreError::Unavailable { - backend: crate::StoreBackend::PostgreSql, - message: "account analytics fact row budget exceeded".to_owned(), - source: None, - }); - } - - let mut observations = Vec::with_capacity(usize::from(query.limit)); - let mut costs = HashMap::>::new(); - let mut models = HashMap::>::new(); - let mut model_costs = HashMap::<(String, String), Vec>::new(); - for row in &rows { - let model_grouping: i32 = get(row, "model_grouping")?; - let currency_grouping: i32 = get(row, "currency_grouping")?; - match (model_grouping, currency_grouping) { - (1, 1) => observations.push(provider_account_from_row(row)?), - (1, 0) if get::>(row, "cost_currency")?.is_some() => { - costs - .entry(get(row, "id")?) - .or_default() - .push(cost_from_row(row)?); - } - (0, 1) if get::>(row, "model")?.is_some() => { - models - .entry(get(row, "id")?) - .or_default() - .push(provider_account_model_from_row(row)?); - } - (0, 0) - if get::>(row, "model")?.is_some() - && get::>(row, "cost_currency")?.is_some() => - { - model_costs - .entry((get(row, "id")?, get(row, "model")?)) - .or_default() - .push(cost_from_row(row)?); - } - (0 | 1, 0 | 1) => {} - _ => { - return Err(crate::StoreError::Unavailable { - backend: crate::StoreBackend::PostgreSql, - message: "decode provider account usage grouping".to_owned(), - source: None, - }); - } + totals.entry(account.clone()).or_default().merge(&state)?; + if !model.is_empty() + && let Some(last_used_at) = state.last_used_at() + { + models_by_account.entry(account).or_default().push( + ProviderAccountModelUsageObservation { + model, + request_count: state.count("completed_request_count"), + success_count: state.count("completed_request_count"), + input_tokens: state.optional_count("input_tokens"), + output_tokens: state.optional_count("output_tokens"), + cached_input_tokens: state.optional_count("cached_input_tokens"), + cache_write_input_tokens: state.optional_count("cache_write_input_tokens"), + reasoning_output_tokens: state.optional_count("reasoning_output_tokens"), + image_input_tokens: state.optional_count("image_input_tokens"), + image_output_tokens: state.optional_count("image_output_tokens"), + total_tokens: state.optional_count("total_tokens"), + image_request_count: state.count("image_request_count"), + image_request_failed_count: state.count("image_request_failed_count"), + cost_coverage: state.coverage(), + costs: state.costs()?, + last_used_at, + }, + ); } } - if observations.is_empty() { - return Ok(observations); - } - let account_ids = observations - .iter() - .map(|item| item.account_id.clone()) - .collect::>(); let mut request_buckets = if let Some(range) = query.request_bucket_range { - provider_account_request_buckets(connection, range, &account_ids).await? + provider_account_request_buckets(snapshot, range, &account_ids).await? } else { HashMap::new() }; - for observation in &mut observations { - observation.costs = costs.remove(&observation.account_id).unwrap_or_default(); - observation.request_buckets = request_buckets - .remove(&observation.account_id) - .unwrap_or_default(); - observation.models = models.remove(&observation.account_id).unwrap_or_default(); - for model in &mut observation.models { - model.costs = model_costs - .remove(&(observation.account_id.clone(), model.model.clone())) - .unwrap_or_default(); - } - observation.models.sort_by(|left, right| { - right - .request_count - .cmp(&left.request_count) - .then_with(|| left.model.cmp(&right.model)) + let mut observations = Vec::with_capacity(rows.len()); + for row in rows { + let account_id: String = get(&row, "id")?; + let state = totals.remove(&account_id).unwrap_or_default(); + let mut models = models_by_account.remove(&account_id).unwrap_or_default(); + models.sort_by(|a, b| { + b.request_count + .cmp(&a.request_count) + .then(a.model.cmp(&b.model)) + }); + observations.push(ProviderAccountUsageObservation { + provider_kind: get(&row, "provider_kind")?, + authentication_kind: get(&row, "authentication_kind")?, + name: get(&row, "name")?, + email: get(&row, "email")?, + plan_type: get(&row, "plan_type")?, + request_count: state.count("completed_request_count"), + success_count: state.count("completed_request_count"), + input_tokens: state.optional_count("input_tokens"), + output_tokens: state.optional_count("output_tokens"), + cached_input_tokens: state.optional_count("cached_input_tokens"), + cache_write_input_tokens: state.optional_count("cache_write_input_tokens"), + reasoning_output_tokens: state.optional_count("reasoning_output_tokens"), + image_input_tokens: state.optional_count("image_input_tokens"), + image_output_tokens: state.optional_count("image_output_tokens"), + total_tokens: state.optional_count("total_tokens"), + image_request_count: state.count("image_request_count"), + image_request_failed_count: state.count("image_request_failed_count"), + cost_coverage: state.coverage(), + costs: state.costs()?, + last_used_at: state.last_used_at(), + request_buckets: request_buckets.remove(&account_id).unwrap_or_default(), + models, + account_id, }); } + observations.sort_by(|a, b| { + b.last_used_at + .cmp(&a.last_used_at) + .then(a.name.cmp(&b.name)) + .then(a.account_id.cmp(&b.account_id)) + }); + observations.truncate(usize::from(query.limit)); Ok(observations) } pub(crate) async fn provider_account_request_buckets( - connection: &mut sqlx::PgConnection, + snapshot: &mut analytics::AnalyticsSnapshot<'_>, range: ObservabilityRange, account_ids: &[String], ) -> StoreResult>> { - let completed_usage = completed_usage_fact_predicate("mr"); - let statement = format!( - "select provider_account_ref, - floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index, - count(*)::bigint as request_count - from request_analytics_facts mr - where mr.provider_account_ref = any($2::text[]) - and mr.started_at >= $1 and mr.started_at < $3 - and {completed_usage} - group by mr.provider_account_ref, bucket_index - order by mr.provider_account_ref, bucket_index" - ); - let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) - .bind(range.start) - .bind(account_ids) - .bind(range.end) - .fetch_all(&mut *connection) - .await - .map_err(|source| postgres_unavailable("load provider account request timeline", source))?; - - let mut observed = HashMap::>::new(); - for row in rows { - observed - .entry(get(&row, "provider_account_ref")?) - .or_default() - .insert( - unsigned(&row, "bucket_index")?, - unsigned(&row, "request_count")?, - ); - } - + let mut observed = snapshot.account_request_counts(range, account_ids).await?; let mut timelines = HashMap::with_capacity(account_ids.len()); - for account_id in account_ids { - let mut account_observed = observed.remove(account_id).unwrap_or_default(); - let mut buckets = Vec::new(); - for bucket_index in 0..ACCOUNT_USAGE_TIMELINE_HOURS { - let bucket_start = range + for account in account_ids { + let mut buckets = Vec::with_capacity(ACCOUNT_USAGE_TIMELINE_HOURS as usize); + for index in 0..ACCOUNT_USAGE_TIMELINE_HOURS { + let start = range .start - .checked_add_signed(TimeDelta::hours(bucket_index)) + .checked_add_signed(TimeDelta::hours(index)) .ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?; buckets.push(ProviderAccountRequestBucket { - bucket_start, - request_count: account_observed - .remove(&(bucket_index as u64)) + bucket_start: start, + request_count: observed + .remove(&(account.clone(), index)) .unwrap_or_default(), }); } - timelines.insert(account_id.clone(), buckets); + timelines.insert(account.clone(), buckets); } Ok(timelines) } - -fn provider_account_model_from_row( - row: &sqlx::postgres::PgRow, -) -> StoreResult { - Ok(ProviderAccountModelUsageObservation { - model: get(row, "model")?, - request_count: unsigned(row, "request_count")?, - success_count: unsigned(row, "success_count")?, - input_tokens: optional_unsigned(row, "input_tokens")?, - output_tokens: optional_unsigned(row, "output_tokens")?, - cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?, - cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?, - reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?, - image_input_tokens: optional_unsigned(row, "image_input_tokens")?, - image_output_tokens: optional_unsigned(row, "image_output_tokens")?, - image_request_count: unsigned(row, "image_request_count")?, - image_request_failed_count: unsigned(row, "image_request_failed_count")?, - total_tokens: optional_unsigned(row, "total_tokens")?, - cost_coverage: coverage_from_row(row)?, - costs: Vec::new(), - last_used_at: get(row, "last_used_at")?, - }) -} - -pub(crate) fn provider_account_from_row( - row: &sqlx::postgres::PgRow, -) -> StoreResult { - Ok(ProviderAccountUsageObservation { - account_id: get(row, "id")?, - provider_kind: get(row, "provider_kind")?, - authentication_kind: get(row, "authentication_kind")?, - name: get(row, "name")?, - email: get(row, "email")?, - plan_type: get(row, "plan_type")?, - request_count: unsigned(row, "request_count")?, - success_count: unsigned(row, "success_count")?, - input_tokens: optional_unsigned(row, "input_tokens")?, - output_tokens: optional_unsigned(row, "output_tokens")?, - cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?, - cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?, - reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?, - image_input_tokens: optional_unsigned(row, "image_input_tokens")?, - image_output_tokens: optional_unsigned(row, "image_output_tokens")?, - image_request_count: unsigned(row, "image_request_count")?, - image_request_failed_count: unsigned(row, "image_request_failed_count")?, - total_tokens: optional_unsigned(row, "total_tokens")?, - cost_coverage: coverage_from_row(row)?, - costs: Vec::new(), - last_used_at: get(row, "last_used_at")?, - request_buckets: Vec::new(), - models: Vec::new(), - }) -} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs index 1bfec5925..195d4dad9 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs @@ -1,15 +1,7 @@ -//! 时间桶补齐、日历边界与共享费用覆盖映射 +//! 时间桶补齐与部署时区的日历边界 use super::super::*; -pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult { - Ok(CostCoverage { - provider_reported_count: unsigned(row, "provider_reported_count")?, - calculated_count: unsigned(row, "calculated_count")?, - unavailable_count: unsigned(row, "unavailable_count")?, - }) -} - pub(crate) fn fill_metric_gaps( range: ObservabilityRange, granularity: ObservationGranularity, diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs b/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs index 661b4a754..dd72026a0 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs @@ -69,38 +69,29 @@ left join model_request_observations mr on mr.id = oe.model_request_id where true"; pub(crate) async fn list_ops_errors( - connection: &mut PgConnection, + connection: &mut ObservabilityConnection, query: OpsErrorQuery, ) -> StoreResult { validate_ops_error_filter(&query.filter)?; let mut transaction = analytics::begin_snapshot(connection).await?; - let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?; let offset = observability_page_offset(query.current_page, query.page_size)?; - let mut statement = - QueryBuilder::::new("with candidate_page as materialized (select * from ("); - statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true"); - push_request_error_predicates(&mut statement, query.range, &query.filter); - statement.push(" union all "); - statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true"); - push_ops_event_predicates(&mut statement, query.range, &query.filter); - statement.push(") e order by occurred_at desc, stable_sort_id desc limit "); - statement.push_bind(i64::from(query.page_size.get())); - statement.push(" offset "); - statement.push_bind(offset); - statement.push(") select details.* from candidate_page page cross join lateral ("); - statement.push(REQUEST_ERROR_SELECT); - statement.push(" and page.source='model_request' and mr.id=page.event_id union all "); - statement.push(OPS_EVENT_SELECT); - statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc"); - let rows = statement - .build() - .fetch_all(&mut *transaction) - .await - .map_err(|source| postgres_unavailable("list ops errors", source))?; - let items = rows - .iter() - .map(ops_error_from_row) - .collect::>>()?; + let (total, page) = if query.current_page == 1 { + let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?; + ( + total, + (total > 0).then_some(TimePage { + range: query.range, + offset, + total: Some(total), + }), + ) + } else { + ops_time_page(&mut transaction, &query, offset).await? + }; + let items = match page { + Some(page) => list_ops_error_items_at(&mut transaction, &query, page).await?, + None => Vec::new(), + }; transaction .commit() .await @@ -113,6 +104,95 @@ pub(crate) async fn list_ops_errors( }) } +async fn list_ops_error_items_at( + connection: &mut PgConnection, + query: &OpsErrorQuery, + page: TimePage, +) -> StoreResult> { + let total = + i64::try_from(page.total.unwrap_or(0)).map_err(|_| invalid("Ops count overflows i64"))?; + let limit = i64::from(query.page_size.get()).min(total - page.offset); + let reverse_offset = total - page.offset - limit; + let ascending = reverse_offset < page.offset; + let offset = if ascending { + reverse_offset + } else { + page.offset + }; + let branch_limit = offset + limit; + let order = if ascending { " asc" } else { " desc" }; + let mut statement = + QueryBuilder::::new("with candidate_page as materialized (select * from (("); + statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true"); + push_request_error_predicates(&mut statement, page.range, &query.filter); + statement + .push(" order by mr.completed_at") + .push(order) + .push(", mr.id") + .push(order); + statement.push(" limit ").push_bind(branch_limit); + statement.push(") union all ("); + statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true"); + push_ops_event_predicates(&mut statement, page.range, &query.filter); + statement + .push(" order by oe.created_at") + .push(order) + .push(", oe.id") + .push(order); + statement.push(" limit ").push_bind(branch_limit); + statement + .push(")) e order by occurred_at") + .push(order) + .push(", stable_sort_id") + .push(order); + statement + .push(" limit ") + .push_bind(limit) + .push(" offset ") + .push_bind(offset); + statement.push(") select details.* from candidate_page page cross join lateral ("); + statement.push(REQUEST_ERROR_SELECT); + statement.push(" and page.source='model_request' and mr.id=page.event_id union all "); + statement.push(OPS_EVENT_SELECT); + statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc"); + let rows = statement + .build() + .fetch_all(connection) + .await + .map_err(|source| postgres_unavailable("list ops errors", source))?; + rows.iter().map(ops_error_from_row).collect() +} + +async fn ops_time_page( + connection: &mut PgConnection, + query: &OpsErrorQuery, + offset: i64, +) -> StoreResult<(u64, Option)> { + // 两个来源先各自精确分段，合并分钟计数后再定位同一个全局页码 + let mut statement = QueryBuilder::::new( + "select bucket_start, sum(request_count)::bigint as request_count from ( + select date_bin(interval '1 minute', mr.completed_at, timestamptz 'epoch') as bucket_start, + count(*)::bigint as request_count + from model_request_observations mr where true", + ); + push_request_error_predicates(&mut statement, query.range, &query.filter); + statement.push(" group by bucket_start union all "); + statement.push( + "select date_bin(interval '1 minute', oe.created_at, timestamptz 'epoch') as bucket_start, + count(*)::bigint as request_count + from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true", + ); + push_ops_event_predicates(&mut statement, query.range, &query.filter); + statement + .push(" group by bucket_start) counts group by bucket_start order by bucket_start desc"); + let rows = statement + .build() + .fetch_all(connection) + .await + .map_err(|source| postgres_unavailable("locate Ops page", source))?; + time_page_from_counts(&rows, query.range, offset, query.page_size) +} + pub(crate) async fn count_ops_errors( connection: &mut sqlx::PgConnection, range: ObservabilityRange, @@ -144,7 +224,6 @@ fn push_request_error_predicates( // 列表和总数共用此条件，避免流式响应中的错误因 outcome 被漏掉 statement.push(" and mr.error_kind is not null and mr.error_kind <> 'cancelled'"); push_range(statement, "mr.completed_at", range); - push_routing_groups(statement, &filter.routing_group_refs); for (column, value) in [ ("mr.client_api_key_ref", &filter.client_api_key_ref), ("mr.id", &filter.request_id), @@ -187,7 +266,6 @@ fn push_ops_event_predicates( filter: &OpsErrorFilter, ) { push_range(statement, "oe.created_at", range); - push_routing_groups(statement, &filter.routing_group_refs); for (column, value) in [ ("mr.client_api_key_ref", &filter.client_api_key_ref), ("oe.model_request_id", &filter.request_id), @@ -275,11 +353,3 @@ fn push_prefix_search(statement: &mut QueryBuilder, columns: &[&str], .push(" escape '\\'"); statement.push(")"); } - -fn push_routing_groups(statement: &mut QueryBuilder, groups: &[String]) { - if !groups.is_empty() { - statement.push(" and mr.routing_group_refs && "); - statement.push_bind(groups.to_vec()); - statement.push("::text[]"); - } -} diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs index 4c49d60be..c19f9688c 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -11,12 +11,6 @@ pub(crate) fn push_usage_filter( filter: &UsageRecordFilter, alias: &str, ) { - if !filter.routing_group_refs.is_empty() { - query - .push(format!(" and {alias}.routing_group_refs && ")) - .push_bind(filter.routing_group_refs.clone()) - .push("::text[]"); - } if let Some(value) = &filter.client_api_key_ref { query.push(format!(" and {alias}.client_api_key_ref = ")); query.push_bind(value.clone()); @@ -86,16 +80,14 @@ pub(crate) fn push_usage_filter( } if let Some(value) = &filter.search { let pattern = literal_prefix_pattern(value); - query.push(format!(" and ({alias}.id like ")); + query.push(format!(" and ({alias}.requested_model_id like ")); query.push_bind(pattern.clone()); for column in [ + "upstream_model_id", "client_api_key_ref", "provider_account_ref", "provider_account_email_snapshot", "provider_account_name_snapshot", - "requested_model_id", - "upstream_model_id", - "upstream_request_id", ] { query.push(format!(" escape '\\' or {alias}.{column} like ")); query.push_bind(pattern.clone()); @@ -174,29 +166,50 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str = from model_request_observations mr"; pub(crate) async fn list_usage_records( - connection: &mut PgConnection, + connection: &mut ObservabilityConnection, query: UsageRecordQuery, ) -> StoreResult { query.filter.validate()?; - let mut transaction = analytics::begin_snapshot(connection).await?; + let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?; // 第一页未填满时，当前快照中的行数已是精确总数，无需再次扫描计数 let (total, items) = if query.current_page == 1 { - let items = list_usage_record_items(&mut *transaction, &query, None).await?; + let items = list_usage_record_items(&mut *snapshot.tx, &query, None).await?; let total = if items.len() < usize::from(query.page_size.get()) { items.len() as u64 + } else if let Some(counts) = snapshot + .completed_request_counts(query.range, &query.filter) + .await? + { + counts.values().try_fold(0_u64, |sum, count| { + sum.checked_add(*count) + .ok_or_else(|| invalid("usage count overflows u64")) + })? } else { - count_usage_records(&mut *transaction, query.range, &query.filter).await? + count_usage_records(&mut *snapshot.tx, query.range, &query.filter).await? }; (total, items) } else { - let total = count_usage_records(&mut *transaction, query.range, &query.filter).await?; - let items = list_usage_record_items(&mut *transaction, &query, Some(total)).await?; + let (total, page) = if let Some(counts) = snapshot + .completed_request_counts(query.range, &query.filter) + .await? + { + time_page_from_segments( + counts.into_iter().rev(), + query.range, + observability_page_offset(query.current_page, query.page_size)?, + query.page_size, + TimeDelta::minutes(15), + )? + } else { + usage_time_page(&mut snapshot.tx, &query).await? + }; + let items = match page { + Some(page) => list_usage_record_items_at(&mut *snapshot.tx, &query, page).await?, + None => Vec::new(), + }; (total, items) }; - transaction - .commit() - .await - .map_err(|error| postgres_unavailable("complete usage page snapshot", error))?; + snapshot.finish().await?; Ok(UsageRecordPage { items, current_page: query.current_page, @@ -209,12 +222,25 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database = executor: E, query: &UsageRecordQuery, total: Option, +) -> StoreResult> { + let page = TimePage { + range: query.range, + offset: observability_page_offset(query.current_page, query.page_size)?, + total, + }; + list_usage_record_items_at(executor, query, page).await +} + +async fn list_usage_record_items_at<'e, E: sqlx::Executor<'e, Database = Postgres>>( + executor: E, + query: &UsageRecordQuery, + page: TimePage, ) -> StoreResult> { query.filter.validate()?; - let mut offset = observability_page_offset(query.current_page, query.page_size)?; + let mut offset = page.offset; let mut limit = i64::from(query.page_size.get()); let mut ascending = false; - if let Some(total) = total { + if let Some(total) = page.total { let total = i64::try_from(total).map_err(|_| invalid("usage count overflows i64"))?; if offset >= total { return Ok(Vec::new()); @@ -231,9 +257,9 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database = "with candidate_page as materialized (select mr.id, mr.started_at from model_request_observations mr", ); statement.push(" where mr.started_at >= "); - statement.push_bind(query.range.start); + statement.push_bind(page.range.start); statement.push(" and mr.started_at < "); - statement.push_bind(query.range.end); + statement.push_bind(page.range.end); push_completed_usage_fact_filter(&mut statement, "mr"); push_usage_filter(&mut statement, &query.filter, "mr"); statement.push(if ascending { @@ -261,6 +287,93 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database = Ok(items) } +// 页码保持全局精确排名，实际明细读取只落在覆盖该页的时间区间 +pub(crate) struct TimePage { + pub(crate) range: ObservabilityRange, + pub(crate) offset: i64, + pub(crate) total: Option, +} + +pub(crate) fn time_page_from_counts( + rows: &[sqlx::postgres::PgRow], + range: ObservabilityRange, + offset: i64, + page_size: ObservabilityPageSize, +) -> StoreResult<(u64, Option)> { + let counts = rows + .iter() + .map(|row| { + Ok(( + get(row, "bucket_start")?, + to_u64(get(row, "request_count")?)?, + )) + }) + .collect::>>()?; + time_page_from_segments(counts, range, offset, page_size, TimeDelta::minutes(1)) +} + +fn time_page_from_segments( + counts: impl IntoIterator, u64)>, + range: ObservabilityRange, + offset: i64, + page_size: ObservabilityPageSize, + width: TimeDelta, +) -> StoreResult<(u64, Option)> { + let offset = u64::try_from(offset).map_err(|_| invalid("negative page offset"))?; + let end = offset + .checked_add(u64::from(page_size.get())) + .ok_or_else(|| invalid("page range overflows u64"))?; + let mut total = 0_u64; + let mut page = None; + let mut selected_count = 0_u64; + for (bucket, count) in counts { + let next = total + .checked_add(count) + .ok_or_else(|| invalid("request count overflows u64"))?; + if next > offset && total < end { + let selected = page.get_or_insert_with(|| TimePage { + range: ObservabilityRange { + start: bucket.max(range.start), + end: (bucket + width).min(range.end), + }, + offset: (offset - total) as i64, + total: None, + }); + selected.range.start = bucket.max(range.start); + selected_count += count; + } + total = next; + } + if let Some(page) = &mut page { + page.total = Some(selected_count); + } + Ok((total, page)) +} + +async fn usage_time_page( + connection: &mut PgConnection, + query: &UsageRecordQuery, +) -> StoreResult<(u64, Option)> { + let offset = observability_page_offset(query.current_page, query.page_size)?; + let mut statement = QueryBuilder::::new( + "select date_bin(interval '1 minute', mr.started_at, timestamptz 'epoch') as bucket_start, + count(*)::bigint as request_count + from model_request_observations mr where mr.started_at >= ", + ); + statement.push_bind(query.range.start); + statement.push(" and mr.started_at < "); + statement.push_bind(query.range.end); + push_completed_usage_fact_filter(&mut statement, "mr"); + push_usage_filter(&mut statement, &query.filter, "mr"); + statement.push(" group by bucket_start order by bucket_start desc"); + let rows = statement + .build() + .fetch_all(connection) + .await + .map_err(|source| postgres_unavailable("locate usage page", source))?; + time_page_from_counts(&rows, query.range, offset, query.page_size) +} + pub(crate) async fn count_usage_records<'e, E: sqlx::Executor<'e, Database = Postgres>>( executor: E, range: ObservabilityRange, diff --git a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs index d8ee898cd..113ef50d2 100644 --- a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs +++ b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs @@ -17,12 +17,20 @@ use crate::{StoreBackend, StoreError, StoreResult, postgres_unavailable}; pub struct ObservabilityQueryBudget { slots: Arc, wait_timeout: Duration, + query_timeout: Duration, } /// 观测名额与实际 PostgreSQL 连接共同存活，取消后仍覆盖协议收尾和事务回滚 pub struct ObservabilityConnection { connection: PoolConnection, permit: Option, + query_timeout: Duration, +} + +impl ObservabilityConnection { + pub(crate) const fn query_timeout(&self) -> Duration { + self.query_timeout + } } impl Deref for ObservabilityConnection { @@ -53,12 +61,22 @@ impl Drop for ObservabilityConnection { impl ObservabilityQueryBudget { /// 预算表示观测类 SQL 最多可同时占用的连接数 - pub fn try_new(max_connections: u32, wait_timeout: Duration) -> StoreResult { - if max_connections == 0 || wait_timeout.is_zero() { + pub fn try_new( + max_connections: u32, + wait_timeout: Duration, + query_timeout: Duration, + ) -> StoreResult { + if max_connections == 0 + || wait_timeout.is_zero() + || query_timeout.is_zero() + || query_timeout.as_millis() > i32::MAX as u128 + { return Err(StoreError::InvalidData { source: None, entity: "observability query budget", - message: "requires a positive connection count and wait timeout".to_owned(), + message: + "requires positive connection and timeout budgets within PostgreSQL limits" + .to_owned(), }); } let permits = usize::try_from(max_connections).map_err(|_| StoreError::InvalidData { @@ -69,10 +87,16 @@ impl ObservabilityQueryBudget { Ok(Self { slots: Arc::new(Semaphore::new(permits)), wait_timeout, + query_timeout, }) } - /// 在共享槽位与十秒执行预算内完成整个观测操作 + #[must_use] + pub const fn query_timeout(&self) -> Duration { + self.query_timeout + } + + /// 在共享槽位与观测总预算内完成排队、连接获取和整个存储操作 /// /// # Errors /// @@ -87,14 +111,15 @@ impl ObservabilityQueryBudget { F: FnOnce(ObservabilityConnection) -> Q, Q: Future>, { - let permit = self.acquire(operation).await?; - tokio::time::timeout(Duration::from_secs(10), async { + tokio::time::timeout(self.query_timeout, async { + let permit = self.acquire(operation).await?; let connection = pool.acquire().await.map_err(|source| { postgres_unavailable("acquire observability connection", source) })?; query(ObservabilityConnection { connection, permit: Some(permit), + query_timeout: self.query_timeout, }) .await }) diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs index 25b09138f..72d16a526 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -102,109 +102,17 @@ impl PgAdminAccountStore { ObservabilityRange::new(window.range.start, window.range.end) .map_err(|error| admin_store_error(ENTITY, error))?; } - let keys = windows - .iter() - .map(|window| window.key.clone()) - .collect::>(); - let starts = windows - .iter() - .map(|window| window.range.start) - .collect::>(); - let ends = windows - .iter() - .map(|window| window.range.end) - .collect::>(); - let account_ids = windows - .iter() - .map(|window| window.account_id.clone()) - .collect::>(); - let rows = self - .query_budget + self.query_budget .run( &self.pool, "load account usage windows", |mut connection| async move { - sqlx::query(sqlx::AssertSqlSafe(account_usage_by_windows_sql())) - .bind(account_ids) - .bind(keys) - .bind(starts) - .bind(ends) - .fetch_all(&mut *connection) + crate::postgres::observability::account_usage_windows(&mut connection, windows) .await - .map_err(|source| { - postgres_unavailable("load provider account quota window usage", source) - }) }, ) .await - .map_err(|error| admin_store_error(ENTITY, error))?; - let mut usage_rows = Vec::with_capacity(windows.len()); - let mut costs_by_window = BTreeMap::<(String, String), Vec>::new(); - let mut model_costs = BTreeMap::<(String, String, String), Vec>::new(); - let mut models_by_window = BTreeMap::<(String, String), Vec>::new(); - for row in &rows { - let model_grouping = window_usage_value::(row, "model_grouping")?; - let currency_grouping = window_usage_value::(row, "currency_grouping")?; - match (model_grouping, currency_grouping) { - (1, 1) => usage_rows.push(row), - (1, 0) if window_usage_value::>(row, "cost_currency")?.is_some() => { - let (key, cost) = admin_account_usage_window_cost(row)?; - costs_by_window.entry(key).or_default().push(cost); - } - (0, 1) if window_usage_value::>(row, "model")?.is_some() => { - let ((account_id, window_key, _), usage) = - admin_account_usage_window_model(row)?; - models_by_window - .entry((account_id, window_key)) - .or_default() - .push(usage); - } - (0, 0) - if window_usage_value::>(row, "model")?.is_some() - && window_usage_value::>(row, "cost_currency")? - .is_some() => - { - let (key, cost) = admin_account_usage_window_model_cost(row)?; - model_costs.entry(key).or_default().push(cost); - } - (0 | 1, 0 | 1) => {} - _ => { - return Err(AdminStoreError::new( - AdminStoreErrorKind::Unavailable, - ENTITY, - "account usage window query returned an invalid grouping marker", - )); - } - } - } - let mut results = usage_rows - .into_iter() - .map(admin_account_usage_window) - .collect::>>()?; - for result in &mut results { - result.usage.costs = costs_by_window - .remove(&(result.account_id.clone(), result.key.clone())) - .unwrap_or_default(); - result.usage.models = models_by_window - .remove(&(result.account_id.clone(), result.key.clone())) - .unwrap_or_default(); - for model in &mut result.usage.models { - model.costs = model_costs - .remove(&( - result.account_id.clone(), - result.key.clone(), - model.model.clone(), - )) - .unwrap_or_default(); - } - result.usage.models.sort_by(|left, right| { - right - .request_count - .cmp(&left.request_count) - .then_with(|| left.model.cmp(&right.model)) - }); - } - Ok(results) + .map_err(|error| admin_store_error(ENTITY, error)) } async fn required_scope( @@ -376,8 +284,14 @@ impl AccountStore for PgAdminAccountStore { .filter(|(_, cooldown)| cooldown.is_active(now_system_time)) .map(|(account_id, _)| account_id.clone()) .collect::>(); - let page = - load_admin_account_page(&self.pool, &query, now, active_rate_limited_ids).await?; + let page = load_admin_account_page( + &self.pool, + &self.query_budget, + &query, + now, + active_rate_limited_ids, + ) + .await?; let item_ids = page .accounts .iter() diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs index aeadd01f9..4c0b3ebf8 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs @@ -12,6 +12,7 @@ pub(crate) struct AdminAccountPageRows { pub(crate) async fn load_admin_account_page( pool: &PgPool, + budget: &crate::postgres::ObservabilityQueryBudget, query: &AdminAccountListQuery, now: DateTime, active_rate_limited_ids: Vec, @@ -35,35 +36,20 @@ pub(crate) async fn load_admin_account_page( let page_size = i64::from(query.page_size.get()); let offset = i64::from(query.page - 1).saturating_mul(page_size); let order_by = admin_account_order(query.sort); - let usage_join = query.sort.is_some_and(|sort| { + let needs_usage = query.sort.is_some_and(|sort| { matches!( sort.field, AdminAccountSortField::Usage | AdminAccountSortField::LastUsedAt ) }); - let completed_usage = completed_usage_fact_predicate("mr"); - let usage_join = if usage_join { - format!( - "left join ( - select mr.provider_account_ref, - coalesce(sum(coalesce( - mr.total_tokens, - coalesce(mr.input_tokens, 0) + coalesce(mr.output_tokens, 0) - )), 0)::bigint as total_tokens, - max(mr.started_at) as last_used_at - from model_requests mr - where mr.started_at >= $2 - make_interval(days => settings.usage_retention_days::int) - and mr.started_at < $2 - and {completed_usage} - group by mr.provider_account_ref - ) usage on usage.provider_account_ref = a.id" - ) + let usage_join = if needs_usage { + "left join usage_values usage on usage.provider_account_ref = a.id" } else { - String::new() + "" }; let statement = format!( - "with account_statuses as ( + "with usage_values as (select * from unnest($10::text[],$11::bigint[],$12::timestamptz[]) as usage(provider_account_ref,total_tokens,last_used_at)), account_statuses as ( select a.id, a.provider_kind, case when not a.enabled then 'disabled' @@ -150,25 +136,45 @@ pub(crate) async fn load_admin_account_page( left join outbound_proxies p on p.id = a.outbound_proxy_id order by page.page_position" ); - // 动态片段只来自上面的封闭排序枚举与固定 usage predicate；所有请求值仍使用 bind - let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) - .bind(active_rate_limited_ids) - .bind(now) - .bind(provider_kind) - .bind(search_prefix) - .bind(status) - .bind(group_mode) - .bind(group_id) - .bind(page_size) - .bind(offset) - .fetch_all(pool) + // 排序事实与本页总数共享同一快照，避免一次查询混入不同源版本 + let rows = budget + .run( + pool, + "load admin account page", + |mut connection| async move { + let mut snapshot = + crate::postgres::observability::AnalyticsSnapshot::new_for_accounts( + &mut connection, + ) + .await?; + let (usage_ids, usage_tokens, usage_latest) = if needs_usage { + crate::postgres::observability::account_usage_sort_values(&mut snapshot, now) + .await? + } else { + (Vec::new(), Vec::new(), Vec::new()) + }; + let rows = sqlx::query(sqlx::AssertSqlSafe(statement)) + .bind(active_rate_limited_ids) + .bind(now) + .bind(provider_kind) + .bind(search_prefix) + .bind(status) + .bind(group_mode) + .bind(group_id) + .bind(page_size) + .bind(offset) + .bind(usage_ids) + .bind(usage_tokens) + .bind(usage_latest) + .fetch_all(&mut *snapshot.tx) + .await + .map_err(|source| postgres_unavailable("load admin account page", source))?; + snapshot.finish().await?; + Ok(rows) + }, + ) .await - .map_err(|source| { - admin_store_error( - ENTITY, - postgres_unavailable("load admin account page", source), - ) - })?; + .map_err(|error| admin_store_error(ENTITY, error))?; let metadata = rows.first().ok_or_else(|| { AdminStoreError::new( AdminStoreErrorKind::Unavailable, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs index f608c4109..23e6e3482 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs @@ -105,164 +105,3 @@ pub(crate) fn admin_account_usage(usage: ProviderAccountUsageObservation) -> Acc .collect(), } } - -pub(crate) fn admin_account_usage_window( - row: &sqlx::postgres::PgRow, -) -> AdminStoreResult { - let provider_reported_count = window_usage_count(row, "provider_reported_count")?; - let calculated_count = window_usage_count(row, "calculated_count")?; - let unavailable_count = window_usage_count(row, "unavailable_count")?; - Ok(AccountUsageWindowResult { - account_id: window_usage_value(row, "account_id")?, - key: window_usage_value(row, "window_key")?, - usage: AccountUsage { - account_id: window_usage_value(row, "account_id")?, - request_count: window_usage_count(row, "request_count")?, - success_count: window_usage_count(row, "success_count")?, - input_tokens: optional_window_usage_count(row, "input_tokens")?, - output_tokens: optional_window_usage_count(row, "output_tokens")?, - cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?, - cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?, - reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?, - image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?, - image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?, - image_request_count: window_usage_count(row, "image_request_count")?, - image_request_failed_count: window_usage_count(row, "image_request_failed_count")?, - total_tokens: optional_window_usage_count(row, "total_tokens")?, - cost_coverage: AdminCostCoverage { - provider_reported_count, - calculated_count, - partial_count: 0, - unavailable_count, - not_billable_count: 0, - }, - costs: Vec::new(), - last_used_at: window_usage_value(row, "last_used_at")?, - request_buckets: Vec::new(), - models: Vec::new(), - }, - }) -} - -pub(crate) fn admin_account_usage_window_model( - row: &sqlx::postgres::PgRow, -) -> AdminStoreResult<((String, String, String), AccountModelUsage)> { - let account_id: String = window_usage_value(row, "account_id")?; - let window_key: String = window_usage_value(row, "window_key")?; - let model: String = window_usage_value(row, "model")?; - let provider_reported_count = window_usage_count(row, "provider_reported_count")?; - let calculated_count = window_usage_count(row, "calculated_count")?; - let unavailable_count = window_usage_count(row, "unavailable_count")?; - Ok(( - (account_id, window_key, model.clone()), - AccountModelUsage { - model, - request_count: window_usage_count(row, "request_count")?, - success_count: window_usage_count(row, "success_count")?, - input_tokens: optional_window_usage_count(row, "input_tokens")?, - output_tokens: optional_window_usage_count(row, "output_tokens")?, - cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?, - cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?, - reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?, - image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?, - image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?, - image_request_count: window_usage_count(row, "image_request_count")?, - image_request_failed_count: window_usage_count(row, "image_request_failed_count")?, - total_tokens: optional_window_usage_count(row, "total_tokens")?, - cost_coverage: AdminCostCoverage { - provider_reported_count, - calculated_count, - partial_count: 0, - unavailable_count, - not_billable_count: 0, - }, - costs: Vec::new(), - last_used_at: window_usage_value(row, "last_used_at")?, - }, - )) -} - -pub(crate) fn admin_account_usage_window_model_cost( - row: &sqlx::postgres::PgRow, -) -> AdminStoreResult<((String, String, String), AccountCost)> { - let key = ( - window_usage_value(row, "account_id")?, - window_usage_value(row, "window_key")?, - window_usage_value(row, "model")?, - ); - let amount = window_usage_value::(row, "amount")?; - let amount = AdminDecimalAmount::from_str(&amount).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "persisted account model cost is invalid", - ) - })?; - Ok(( - key, - AccountCost { - currency: window_usage_value(row, "cost_currency")?, - amount, - }, - )) -} - -pub(crate) fn admin_account_usage_window_cost( - row: &sqlx::postgres::PgRow, -) -> AdminStoreResult<((String, String), AccountCost)> { - let key = ( - window_usage_value(row, "account_id")?, - window_usage_value(row, "window_key")?, - ); - let amount = window_usage_value::(row, "amount")?; - let amount = AdminDecimalAmount::from_str(&amount).map_err(|_| { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - "persisted account cost is invalid", - ) - })?; - Ok(( - key, - AccountCost { - currency: window_usage_value(row, "cost_currency")?, - amount, - }, - )) -} - -pub(crate) fn window_usage_count( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> AdminStoreResult { - let value = window_usage_value::(row, column)?; - u64::try_from(value).map_err(|_| invalid_window_usage(column)) -} - -pub(crate) fn optional_window_usage_count( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> AdminStoreResult> { - window_usage_value::>(row, column)? - .map(|value| u64::try_from(value).map_err(|_| invalid_window_usage(column))) - .transpose() -} - -pub(crate) fn window_usage_value( - row: &sqlx::postgres::PgRow, - column: &'static str, -) -> AdminStoreResult -where - for<'row> T: sqlx::Decode<'row, Postgres> + sqlx::Type, -{ - row.try_get(column) - .map_err(|_| invalid_window_usage(column)) -} - -pub(crate) fn invalid_window_usage(column: &'static str) -> AdminStoreError { - AdminStoreError::new( - AdminStoreErrorKind::Invalid, - ENTITY, - format!("invalid quota window usage column: {column}"), - ) -} diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs index 7bd4c8d4b..79bb1e4de 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs @@ -1,6 +1,6 @@ //! 明文 `provider_accounts` 与凭证 revision CAS 的唯一 PostgreSQL owner -use std::{collections::BTreeSet, fmt, str::FromStr}; +use std::{collections::BTreeSet, fmt}; use async_trait::async_trait; use chrono::{DateTime, TimeDelta, Utc}; @@ -9,17 +9,14 @@ use gateway_admin::{ MutationContext, Revision as AdminRevision, account_groups::AccountGroupRef, accounts::{ - AccountCost, AccountGroupFilter, AccountImportSettings, - AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage, - AccountPageItem, AccountRecord, AccountSort as AdminAccountSort, + AccountGroupFilter, AccountImportSettings, AccountListQuery as AdminAccountListQuery, + AccountPage, AccountPageItem, AccountRecord, AccountSort as AdminAccountSort, AccountSortField as AdminAccountSortField, AccountSummary, AccountUpdateResult, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult, BatchUpdateAccounts, DeleteAccounts, SortDirection as AdminSortDirection, UpdateAccount, }, - observability::{ - CostCoverage as AdminCostCoverage, DecimalAmount as AdminDecimalAmount, TimeRange, - }, + observability::TimeRange, provider_credentials::{ AuthorizationCommit, AuthorizationCredentialCommit, CredentialDetails, CredentialImportCommit, CredentialImportResult, CredentialMutationResult, @@ -54,7 +51,6 @@ use super::{ AdminAuditEvent, ControlPlaneRepository, ObservabilityRange, PgControlPlaneRepository, PgObservabilityRepository, ProviderAccountUsageObservation, ProviderAccountUsageQuery, append_admin_audit_event_in_transaction, bump_config_revision_in_transaction, - completed_usage_fact_predicate, }; mod admin_adapter; diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs index b8810e04c..a59fb184c 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs @@ -1,9 +1,6 @@ //! 预测只读采样：先聚合数值、再读取有界的 Provider 文档，不扫描模型排行维度 -use gateway_admin::model::quota_forecast_sampling::{ - MAX_FORECAST_HISTORY_POINTS, QuotaForecastHistory, QuotaForecastHistoryPoint, - QuotaForecastUsage, -}; +use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistory; use super::*; @@ -23,124 +20,21 @@ pub(super) async fn load_history( "forecast range exceeds supported quota windows", )); } - let rows = budget + budget .run( pool, "load quota forecast history", |mut connection| async move { - sqlx::query(sqlx::AssertSqlSafe(history_sql())) - .bind(&window.account_id) - .bind(range.start) - .bind(range.end) - .bind(MAX_FORECAST_HISTORY_POINTS as i32) - .fetch_all(&mut *connection) - .await - .map_err(|source| postgres_unavailable("load quota forecast history", source)) + let mut snapshot = + super::super::observability::AnalyticsSnapshot::new_for_accounts( + &mut connection, + ) + .await?; + let history = snapshot.forecast_history(&window.account_id, range).await?; + snapshot.finish().await?; + Ok(history) }, ) .await - .map_err(|error| admin_store_error(ENTITY, error))?; - let mut history = QuotaForecastHistory::default(); - for row in &rows { - let usage = QuotaForecastUsage { - request_count: window_usage_count(row, "request_count")?, - tokens: window_usage_count(row, "tokens")?, - input_tokens: window_usage_count(row, "input_tokens")?, - output_tokens: window_usage_count(row, "output_tokens")?, - cached_input_tokens: window_usage_count(row, "cached_input_tokens")?, - missing_token_count: window_usage_count(row, "missing_token_count")?, - known_cost_count: window_usage_count(row, "known_cost_count")?, - unavailable_cost_count: window_usage_count(row, "unavailable_cost_count")?, - usd: window_usage_value(row, "usd")?, - excluded_request_count: window_usage_count(row, "excluded_request_count")?, - }; - if window_usage_value::(row, "is_total")? { - history.usage = usage; - history.pending_request_count = window_usage_count(row, "pending_count")?; - } else if let Some(document) = window_usage_value::< - Option>>, - >(row, "document")? - { - history.points.push(QuotaForecastHistoryPoint { - started_at: window_usage_value(row, "started_at")?, - completed_at: window_usage_value(row, "completed_at")?, - usage, - provider_observation: ProviderDocument::new(OpaqueProviderData::new(document.0)), - }); - } - } - Ok(history) -} - -fn history_sql() -> String { - let completed_usage = completed_usage_fact_predicate("mr"); - // 一个语句共享 MVCC 快照 - // 用 RANGE 帧让相同完成时间的点拥有相同累计值， - // 避免并发请求的任意行顺序制造不同分子；未完成请求只进入待决计数 - format!( - "with scoped as materialized ( - select mr.id, mr.started_at, mr.completed_at, - mr.completed_at <= $3 and mr.outcome <> 'running' as settled, - coalesce(({completed_usage}), false) as included, - mr.provider_observation_json is not null as has_document, - mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.total_tokens, - mr.cost_amount, mr.cost_currency, - coalesce(mr.cost_source in ('calculated', 'provider_reported') - and mr.cost_amount is not null and mr.cost_currency = 'USD', false) as known_cost - from model_requests mr - where mr.provider_account_ref = $1 - and mr.started_at >= $2 and mr.started_at < $3 - ), facts as materialized ( - select *, - (included and settled)::integer as request_count, - case when included and settled then coalesce(total_tokens, - coalesce(input_tokens, 0) + coalesce(output_tokens, 0)) else 0 end as tokens, - case when included and settled then coalesce(input_tokens, 0) else 0 end as inputs, - case when included and settled then coalesce(output_tokens, 0) else 0 end as outputs, - case when included and settled then coalesce(cached_input_tokens, 0) else 0 end as cached, - (included and settled and total_tokens is null - and (input_tokens is null or output_tokens is null))::integer as missing_tokens, - (included and settled and known_cost)::integer as known_costs, - (included and settled and not known_cost)::integer as missing_costs, - case when included and settled and known_cost then cost_amount else 0 end as usd, - (not included and settled)::integer as excluded - from scoped - ), cumulative as ( - select id, started_at, completed_at, has_document, - least($4 - 1, floor(extract(epoch from (completed_at - $2)) - / greatest(1, extract(epoch from ($3::timestamptz - $2::timestamptz)) / $4))) as bucket, - sum(request_count) over w as request_count, - sum(tokens) over w as tokens, - sum(inputs) over w as input_tokens, - sum(outputs) over w as output_tokens, - sum(cached) over w as cached_input_tokens, - sum(missing_tokens) over w as missing_token_count, - sum(known_costs) over w as known_cost_count, - sum(missing_costs) over w as unavailable_cost_count, - sum(usd) over w as usd, - sum(excluded) over w as excluded_request_count - from facts where settled - window w as (order by completed_at range between unbounded preceding and current row) - ), selected as ( - select distinct on (bucket) * from cumulative - where has_document order by bucket, completed_at desc, id desc - ) - select false as is_total, s.started_at, s.completed_at, - s.request_count::bigint, s.tokens::bigint, s.input_tokens::bigint, - s.output_tokens::bigint, s.cached_input_tokens::bigint, s.missing_token_count::bigint, - s.known_cost_count::bigint, s.unavailable_cost_count::bigint, - s.usd::double precision, s.excluded_request_count::bigint, - 0::bigint as pending_count, mr.provider_observation_json as document - from selected s join model_requests mr on mr.id = s.id - union all - select true, $3, $3, - coalesce(sum(request_count), 0)::bigint, coalesce(sum(tokens), 0)::bigint, - coalesce(sum(inputs), 0)::bigint, coalesce(sum(outputs), 0)::bigint, - coalesce(sum(cached), 0)::bigint, coalesce(sum(missing_tokens), 0)::bigint, - coalesce(sum(known_costs), 0)::bigint, coalesce(sum(missing_costs), 0)::bigint, - coalesce(sum(usd), 0)::double precision, coalesce(sum(excluded), 0)::bigint, - count(*) filter (where settled is not true), null::jsonb - from facts - order by completed_at, is_total" - ) + .map_err(|error| admin_store_error(ENTITY, error)) } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs index 6d030d133..e1d5991e3 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -9,76 +9,6 @@ pub(crate) const QUOTA_MAX_BYTES: usize = 128 * 1024; pub(crate) const MAX_ADMIN_IMPORT_BATCH: usize = 200; pub(crate) const ADMIN_USAGE_CHUNK_SIZE: usize = 200; -/// 四种窗口投影通过一组 `GROUPING SETS` 从同一批匹配请求派生 -/// `grouping_* = 1` -/// 表示对应维度未参与该行聚合，调用方据此区分窗口、模型与成本行 -pub(crate) fn account_usage_by_windows_sql() -> String { - let completed_usage = completed_usage_fact_predicate("mr"); - format!( - "with requested_windows as ( - select * - from unnest($1::text[], $2::text[], $3::timestamptz[], $4::timestamptz[]) - as requested(account_id, window_key, window_start, window_end) - ), matched as ( - select requested.account_id, - requested.window_key, - mr.id as request_id, - coalesce(mr.upstream_model_id, mr.requested_model_id) as model, - mr.outcome, mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, - mr.cache_write_input_tokens, mr.reasoning_output_tokens, - mr.image_input_tokens, mr.image_output_tokens, - mr.image_generation_succeeded, mr.total_tokens, - mr.cost_source, mr.cost_amount, mr.cost_currency, mr.started_at - from requested_windows requested - left join model_requests mr - on mr.provider_account_ref = requested.account_id - and mr.started_at >= requested.window_start - and mr.started_at < requested.window_end - and {completed_usage} - ) - select account_id, - window_key, - model, - cost_currency, - grouping(model)::integer as model_grouping, - grouping(cost_currency)::integer as currency_grouping, - count(request_id)::bigint as request_count, - count(request_id) filter (where outcome = 'succeeded')::bigint as success_count, - sum(input_tokens)::bigint as input_tokens, - sum(output_tokens)::bigint as output_tokens, - sum(cached_input_tokens)::bigint as cached_input_tokens, - sum(cache_write_input_tokens)::bigint as cache_write_input_tokens, - sum(reasoning_output_tokens)::bigint as reasoning_output_tokens, - sum(image_input_tokens)::bigint as image_input_tokens, - sum(image_output_tokens)::bigint as image_output_tokens, - count(request_id) filter (where image_generation_succeeded is true)::bigint - as image_request_count, - count(request_id) filter (where image_generation_succeeded is false)::bigint - as image_request_failed_count, - coalesce(sum(coalesce( - total_tokens, - coalesce(input_tokens, 0) + coalesce(output_tokens, 0) - )), 0)::bigint as total_tokens, - count(request_id) filter (where cost_source = 'provider_reported')::bigint - as provider_reported_count, - count(request_id) filter (where cost_source = 'calculated')::bigint - as calculated_count, - count(request_id) filter (where cost_source = 'unavailable')::bigint - as unavailable_count, - max(started_at) as last_used_at, - sum(cost_amount)::text as amount - from matched - group by grouping sets ( - (account_id, window_key), - (account_id, window_key, cost_currency), - (account_id, window_key, model), - (account_id, window_key, model, cost_currency) - ) - order by account_id, window_key, model_grouping desc, currency_grouping desc, - model nulls last, cost_currency nulls last" - ) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProviderAccountAdminScope { pub provider_kind: String, diff --git a/backend/crates/gateway-store/tests/config.rs b/backend/crates/gateway-store/tests/config.rs index 83e81890b..0808162fa 100644 --- a/backend/crates/gateway-store/tests/config.rs +++ b/backend/crates/gateway-store/tests/config.rs @@ -27,6 +27,7 @@ fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits() assert_eq!(pool, StorePoolConfig::default()); assert_eq!(pool.max_connections, 20); assert_eq!(pool.acquire_timeout_seconds, 5); + assert_eq!(pool.observability_timeout_seconds, 120); assert_eq!(pool.observability_max_connections(), 16); assert_eq!( StorePoolConfig { @@ -87,3 +88,24 @@ fn valid_config() -> StoreConfig { })) .expect("test Store configuration") } + +#[test] +fn store_config_accepts_custom_observability_timeout_and_rejects_invalid_values() { + let root = tempfile::tempdir().unwrap(); + for seconds in [1_u64, 47, 180] { + let pool: StorePoolConfig = serde_json::from_value(serde_json::json!({ + "observability_timeout_seconds": seconds, + })) + .unwrap(); + assert_eq!(pool.observability_timeout_seconds, seconds); + } + for seconds in [0_u64, u64::MAX] { + let mut config: serde_json::Value = serde_json::json!({ + "database": { "url": "postgres://codex_proxy@127.0.0.1:5432/codex_proxy", "password": PASSWORD }, + "redis": { "url": "redis://127.0.0.1:6379/", "password": PASSWORD }, + }); + config["pool"] = serde_json::json!({"observability_timeout_seconds": seconds}); + let mut config: StoreConfig = serde_json::from_value(config).unwrap(); + assert!(config.resolve_and_validate(root.path()).is_err()); + } +} diff --git a/backend/crates/gateway-store/tests/postgres/account_groups.rs b/backend/crates/gateway-store/tests/postgres/account_groups.rs index 546018589..22ac26b06 100644 --- a/backend/crates/gateway-store/tests/postgres/account_groups.rs +++ b/backend/crates/gateway-store/tests/postgres/account_groups.rs @@ -18,7 +18,9 @@ use gateway_core::{ policy::{ClientApiKeyId, RateLimits}, routing::AccountGroupId, }; -use gateway_store::postgres::{PgAccountGroupRepository, PgAdminClientKeyStore}; +use gateway_store::postgres::{ + ObservabilityQueryBudget, PgAccountGroupRepository, PgAdminClientKeyStore, +}; use super::TestDatabase; @@ -75,7 +77,11 @@ async fn groups_aggregate_cross_provider_members_and_key_bindings_without_multip .await .expect("set account concurrency override"); - let groups = PgAccountGroupRepository::new(database.pool.clone()); + let groups = PgAccountGroupRepository::new( + database.pool.clone(), + ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10)) + .unwrap(), + ); let keys = PgAdminClientKeyStore::new(database.pool.clone()); let mixed_group = group_id(MIXED_GROUP); let empty_group = group_id(EMPTY_GROUP); @@ -315,7 +321,11 @@ async fn group_options_should_avoid_aggregate_tables_and_page_filtered_empty_gro .await .expect("seed account group option"); } - let groups = PgAccountGroupRepository::new(database.pool.clone()); + let groups = PgAccountGroupRepository::new( + database.pool.clone(), + ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10)) + .unwrap(), + ); let mut locked_tables = database.pool.begin().await.expect("begin table lock"); sqlx::query( @@ -383,7 +393,11 @@ async fn group_costs_should_include_statusless_websocket_but_reject_statusless_h "Statusless Account", ) .await; - let groups = PgAccountGroupRepository::new(database.pool.clone()); + let groups = PgAccountGroupRepository::new( + database.pool.clone(), + ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10)) + .unwrap(), + ); groups .create_account_group( NewAccountGroup { @@ -576,7 +590,11 @@ async fn fast_mode_group_updates_preserve_omitted_values_and_publish_snapshot_fa let Some(database) = TestDatabase::create("fast_mode_group").await else { return; }; - let repository = PgAccountGroupRepository::new(database.pool.clone()); + let repository = PgAccountGroupRepository::new( + database.pool.clone(), + ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10)) + .unwrap(), + ); let id = group_id(MIXED_GROUP); repository .create_account_group( @@ -632,3 +650,112 @@ async fn fast_mode_group_updates_preserve_omitted_values_and_publish_snapshot_fa } database.close().await; } + +#[tokio::test] +async fn group_costs_follow_current_members_and_pending_source_replacements() { + let Some(database) = TestDatabase::create("group_cost_current_members").await else { + return; + }; + seed_account( + &database.pool, + "acct_group_move", + "openai", + "Movable Account", + ) + .await; + let budget = + ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10)) + .unwrap(); + let groups = PgAccountGroupRepository::new(database.pool.clone(), budget.clone()); + for id in [MIXED_GROUP, EMPTY_GROUP] { + groups + .create_account_group( + NewAccountGroup { + fast_mode: FastMode::Default, + id: group_id(id), + name: id.to_owned(), + description: None, + color: group_color("#2563EBFF"), + }, + &context("create-group-move-fixture"), + ) + .await + .unwrap(); + } + assign_accounts(&database.pool, MIXED_GROUP, &["acct_group_move"]).await; + seed_group_cost_snapshot( + &database.pool, + "req_group_move", + "acct_group_move", + &[MIXED_GROUP], + "3.25", + ) + .await; + sqlx::query( + "update model_requests set started_at=now()-interval '2 hours' where id='req_group_move'", + ) + .execute(&database.pool) + .await + .unwrap(); + let worker = + gateway_store::postgres::PgRequestAnalyticsRepository::new(database.pool.clone(), budget); + let mut ready = false; + for _ in 0..100 { + worker.maintain(1000).await.unwrap(); + let status = worker.status().await.unwrap(); + if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 { + ready = true; + break; + } + } + assert!(ready, "group fixture must publish rollups"); + let costs = load_retained_group_costs(&groups).await; + assert_eq!(costs[MIXED_GROUP], "3.25"); + assert_eq!(costs[EMPTY_GROUP], "0"); + + sqlx::query("update account_group_accounts set account_group_id=$1 where provider_account_id='acct_group_move'") + .bind(EMPTY_GROUP).execute(&database.pool).await.unwrap(); + let costs = load_retained_group_costs(&groups).await; + assert_eq!( + costs[MIXED_GROUP], "0", + "moving an account reattributes retained usage" + ); + assert_eq!(costs[EMPTY_GROUP], "3.25"); + + // 未再次运行 Worker，当前源费用和模型仍必须原子替换旧汇总贡献 + sqlx::query("update model_requests set cost_amount=7.5,upstream_model_id='updated-model' where id='req_group_move'") + .execute(&database.pool).await.unwrap(); + let costs = load_retained_group_costs(&groups).await; + assert_eq!(costs[EMPTY_GROUP], "7.5"); + sqlx::query("delete from model_requests where id='req_group_move'") + .execute(&database.pool) + .await + .unwrap(); + let costs = load_retained_group_costs(&groups).await; + assert_eq!( + costs[EMPTY_GROUP], "0", + "pending deletion must withdraw the old projected amount" + ); + database.close().await; +} + +async fn load_retained_group_costs(groups: &PgAccountGroupRepository) -> BTreeMap { + groups + .list_account_groups(AccountGroupListQuery { + page: 1, + page_size: PageSize::new(20).unwrap(), + search: None, + enabled: None, + }) + .await + .unwrap() + .items + .into_iter() + .map(|group| { + ( + group.id.to_string(), + group.usage.retained_total_usd.to_string(), + ) + }) + .collect() +} diff --git a/backend/crates/gateway-store/tests/postgres/analytics.rs b/backend/crates/gateway-store/tests/postgres/analytics.rs index a7649a6d2..58031e786 100644 --- a/backend/crates/gateway-store/tests/postgres/analytics.rs +++ b/backend/crates/gateway-store/tests/postgres/analytics.rs @@ -21,6 +21,212 @@ pub(super) async fn seed(pool: &sqlx::PgPool, id: &str, start: DateTime, gr .bind(id).bind(start).bind(observation).execute(pool).await.expect("seed analytic request"); } +#[tokio::test] +async fn provider_counts_match_for_rollups_boundaries_and_filtered_facts() { + let Some(database) = TestDatabase::create("analytics_provider_counts").await else { + return; + }; + let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + for id in [ + "provider_counts_ok", + "provider_counts_fail", + "provider_counts_unrouted", + ] { + seed(&database.pool, id, start + TimeDelta::minutes(5), &[]).await; + } + sqlx::query("update model_requests set provider_kind='openai',provider_account_ref='account_provider_counts',upstream_transport='http',attempt_count=case when id='provider_counts_ok' then 2 else 1 end where id in ('provider_counts_ok','provider_counts_fail')") + .execute(&database.pool).await.unwrap(); + sqlx::query("update model_requests set outcome='failed',client_status_code=502,downstream_committed_at=null where id='provider_counts_fail'") + .execute(&database.pool).await.unwrap(); + drain_analytics(&database.pool).await; + let repo = observability_repository(&database.pool); + let full = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap(); + let edge = + ObservabilityRange::new(start + TimeDelta::minutes(4), start + TimeDelta::minutes(6)) + .unwrap(); + // 同一批请求分别走完整桶、首尾事实与筛选回退，供应商计数不能随读取路径变化 + for (range, filter) in [ + (full, UsageRecordFilter::default()), + (edge, UsageRecordFilter::default()), + ( + full, + UsageRecordFilter { + search: Some("model_".to_owned()), + ..Default::default() + }, + ), + ] { + let summary = repo.usage_summary(range, filter.clone()).await.unwrap(); + assert_eq!(summary.requests.request_count, 3); + let providers = summary + .providers + .into_iter() + .map(|provider| { + ( + provider.provider_kind, + ( + provider.request_count, + provider.attempt_count, + provider.failure_count, + provider.total_tokens, + ), + ) + }) + .collect::>(); + assert_eq!( + providers, + std::collections::BTreeMap::from([ + ("openai".to_owned(), (2, 3, 1, 120)), + ("unrouted".to_owned(), (1, 0, 0, 120)), + ]) + ); + let diagnostic = repo + .usage_diagnostics(range, filter, DiagnosticDimension::Provider, 100) + .await + .unwrap(); + assert_eq!(diagnostic.total_request_count, 3); + let openai = diagnostic + .items + .iter() + .find(|item| item.key == "openai") + .unwrap(); + assert_eq!( + ( + openai.request_count, + openai.success_count, + openai.failure_count, + openai.attempt_count, + openai.total_tokens + ), + (2, 1, 1, 3, 120) + ); + assert_eq!(openai.average_latency_ms, Some(100)); + // 汇总桶沿用直方图近似值，诊断与完整指标读取必须给出相同分位数 + assert_eq!( + openai.latency_p95_ms, + summary + .requests + .latency_percentiles + .p95_ms + .map(|value| value.as_f64().round() as u64) + ); + assert_eq!( + openai.first_token_p95_ms, + summary + .requests + .first_token_latency_percentiles + .p95_ms + .map(|value| value.as_f64().round() as u64) + ); + assert_eq!(openai.retry_count, 1); + assert_eq!(openai.retried_request_count, 1); + assert_eq!(openai.non_completion_count, 0); + assert_eq!(openai.cost_coverage.unavailable_count, 1); + } + database.close().await; +} + +#[tokio::test] +async fn insights_share_bucket_states_without_merging_exact_percentiles() { + let Some(database) = TestDatabase::create("analytics_shared_insights").await else { + return; + }; + let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + for (index, latency) in [10_i64, 1000, 2000, 3000].into_iter().enumerate() { + let id = format!("insights_{index}"); + let minute = if index == 0 { 1 } else { 16 }; + seed( + &database.pool, + &id, + start + TimeDelta::minutes(minute), + &["insights_group"], + ) + .await; + sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',5)) where id=$1") + .bind(id).bind(latency).execute(&database.pool).await.unwrap(); + } + drain_analytics(&database.pool).await; + let repo = observability_repository(&database.pool); + let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap(); + for filter in [ + UsageRecordFilter::default(), + UsageRecordFilter { + search: Some("model_".to_owned()), + ..Default::default() + }, + ] { + let expected = repo.usage_summary(range, filter.clone()).await.unwrap(); + if filter.search.is_some() { + assert_eq!( + expected + .requests + .latency_percentiles + .p50_ms + .unwrap() + .as_f64(), + 1500.0 + ); + } + for granularity in [ + gateway_admin::model::observability::Granularity::FifteenMinutes, + gateway_admin::model::observability::Granularity::Hour, + ] { + let observation = repo + .usage_insights_observation(range, filter.clone(), granularity) + .await + .unwrap(); + assert_eq!(observation.overview.requests, expected.requests); + assert_eq!(observation.overview.providers, expected.providers); + assert_eq!( + observation.overview.analytics.percentiles_approximate, + expected.analytics.percentiles_approximate + ); + assert_eq!( + observation + .trend + .iter() + .map(|point| point.metrics.request_count) + .sum::(), + 4 + ); + assert!(observation.costs.iter().all(|point| point.standard_amount.is_none() && point.no_cache_amount.is_none())); + } + } + database.close().await; +} + +#[tokio::test] +async fn prefix_search_ignores_request_ids_and_matches_snapshot_names() { + let Some(database) = TestDatabase::create("analytics_search_prefix").await else { + return; + }; + let start = Utc::now() - TimeDelta::hours(1); + for id in ["literal_1", "literalX1", "snapshot_only"] { + seed(&database.pool, id, start, &[]).await; + } + sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{request,clientApiKeyNameSnapshot}','\"literal_name\"'::jsonb) where id='snapshot_only'") + .execute(&database.pool).await.unwrap(); + drain_analytics(&database.pool).await; + let result = observability_repository(&database.pool) + .usage_summary( + ObservabilityRange::new(start - TimeDelta::seconds(1), start + TimeDelta::seconds(1)) + .unwrap(), + UsageRecordFilter { + search: Some("literal_".to_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(result.requests.request_count, 1); + assert_eq!(result.requests.total_tokens, 120); + database.close().await; +} + #[tokio::test] async fn opaque_identifiers_preserve_long_values_and_response_or_counts_each_request_once() { let Some(database) = TestDatabase::create("analytics_opaque_identifiers").await else { @@ -169,7 +375,7 @@ async fn rolled_back_source_changes_and_late_commits_do_not_lose_queue_work() { } #[tokio::test] -async fn group_or_filter_counts_each_historical_request_once_and_rebuilds_dirty_buckets() { +async fn dirty_buckets_replace_each_historical_request_contribution_once() { let Some(database) = TestDatabase::create("analytics_groups").await else { return; }; @@ -194,13 +400,8 @@ async fn group_or_filter_counts_each_historical_request_once_and_rebuilds_dirty_ let repo = observability_repository(&database.pool); let range = ObservabilityRange::new(start - TimeDelta::minutes(1), start + TimeDelta::hours(2)) .unwrap(); - let filter = UsageRecordFilter { - routing_group_refs: vec!["group_a".into(), "group_b".into()], - ..Default::default() - }; - let result = repo.usage_summary(range, filter).await.unwrap(); + let result = repo.usage_summary(range, Default::default()).await.unwrap(); assert_eq!(result.requests.request_count, 2); - assert!(!result.analytics.percentiles_approximate); // 已投影但桶待重建时整个桶用事实替换，不能叠加旧桶贡献 sqlx::query("update request_analytics_facts set total_tokens=333 where id='req_group_a'") .execute(&database.pool) @@ -453,6 +654,9 @@ async fn interrupted_quarters_restart_changed_versions_and_publish_one_consisten .execute(&database.pool) .await .unwrap(); + // 独立重建请求改变版本，验证旧前缀废弃后仍可恢复，普通源变更不再重启整桶 + sqlx::query("update request_analytics_pending_buckets set change_version=nextval('request_analytics_change_version')") + .execute(&database.pool).await.unwrap(); drop(worker); let worker = PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget()); @@ -487,6 +691,8 @@ async fn source_updates_do_not_wait_for_blocked_bucket_rebuild() { .execute(&database.pool) .await .unwrap(); + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),now())") + .bind(start).execute(&database.pool).await.unwrap(); let mut blocker = sqlx::PgConnection::connect_with(&database.pool.connect_options()) .await .unwrap(); @@ -677,7 +883,7 @@ async fn clean_aligned_statistics_do_not_access_facts_and_dirty_edges_are_disjoi .execute(&mut *transaction) .await .unwrap(); - let plan:serde_json::Value=sqlx::query_scalar("explain (analyze,buffers,format json) select fact.id from unnest($1::timestamptz[],$2::timestamptz[]) span(start_at,end_at) cross join lateral (select mr.id from request_analytics_facts mr where mr.started_at>=span.start_at and mr.started_at=span.start_at and mr.started_at= $1 and started_at < $2") + .bind(range.start).bind(range.end).fetch_one(&database.pool).await.unwrap(); + assert_eq!(result.requests.request_count, expected.0 as u64); + assert_eq!(result.requests.total_tokens, expected.1 as u64); + assert_eq!( + result.requests.latency_percentiles.p95_ms.unwrap().as_f64(), + expected.2 + ); + assert!(!result.analytics.percentiles_approximate); + // 同一个热点桶处于重建阶段时，不得使用旧汇总或只读取前十万条 + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),now())") + .bind(start).execute(&database.pool).await.unwrap(); + let dirty = repo.usage_summary(range, Default::default()).await.unwrap(); + assert_eq!(dirty.requests.request_count, expected.0 as u64); + assert_eq!(dirty.requests.total_tokens, expected.1 as u64); + assert_eq!(dirty.requests.latency_count, expected.0 as u64); + assert!(dirty.analytics.percentiles_approximate); + let ranked = repo + .usage_diagnostics(range, Default::default(), DiagnosticDimension::ApiKey, 1) + .await + .unwrap(); + assert_eq!(ranked.total_request_count, expected.0 as u64); + assert_eq!(ranked.items[0].request_count, expected.0 as u64); + assert_eq!(ranked.items[0].costs[0].amount.to_string(), expected.3); + database.close().await; +} + +#[tokio::test] +async fn high_cardinality_summary_merges_all_rows_before_returning_model_groups() { + let Some(database) = TestDatabase::create("analytics_sql_summary_aggregation").await else { + return; + }; + drain_analytics(&database.pool).await; + let start = DateTime::parse_from_rfc3339("2026-09-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let versions: (i32,i32) = sqlx::query_as("select schema_version,histogram_version from request_analytics_build_state where singleton=true") + .fetch_one(&database.pool).await.unwrap(); + // 150 个模型乘以 744 小时超过旧汇总行上限，输出仍只有 150 个模型组 + sqlx::query("insert into request_analytics_hourly(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) select $1::timestamptz+hour*interval '1 hour','provider_model',array['openai','model_'||lpad(model::text,3,'0'),'model_'||lpad(model::text,3,'0')],'succeeded',$2,$3,jsonb_build_object('request_count',model,'total_tokens',model*120),jsonb_build_object('USD',(model::numeric*0.0000000001)::text),'{}' from generate_series(0,743) hour cross join generate_series(1,150) model") + .bind(start).bind(versions.0).bind(versions.1).execute(&database.pool).await.unwrap(); + let result = observability_repository(&database.pool) + .usage_diagnostics( + ObservabilityRange::new(start, start + TimeDelta::days(31)).unwrap(), + UsageRecordFilter::default(), + DiagnosticDimension::Model, + 2, + ) + .await + .unwrap(); + assert_eq!(result.total_request_count, 744 * 150 * 151 / 2); + assert_eq!(result.items.len(), 2); + assert_eq!(result.items[0].key, "model_150"); + assert_eq!(result.items[0].request_count, 744 * 150); + assert_eq!(result.items[0].total_tokens, 744 * 150 * 120); + assert_eq!(result.items[0].costs[0].amount.to_string(), "0.00001116"); + database.close().await; +} + +#[tokio::test] +async fn sql_edge_histograms_match_persisted_power_of_two_boundaries() { + let Some(database) = TestDatabase::create("analytics_sql_histogram_boundaries").await else { + return; + }; + let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + for (index, value) in [0_i64, 1, 2, 4, 8, 16, 31, 32, 33, 64, 128, 1024, 1_048_576] + .into_iter() + .enumerate() + { + let id = format!("sql_hist_{index}"); + seed( + &database.pool, + &id, + start + TimeDelta::seconds(index as i64), + &[], + ) + .await; + sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',0)) where id=$1") + .bind(id).bind(value).execute(&database.pool).await.unwrap(); + } + drain_analytics(&database.pool).await; + let repo = observability_repository(&database.pool); + let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap(); + let persisted = repo.usage_summary(range, Default::default()).await.unwrap(); + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values($1,now())") + .bind(start).execute(&database.pool).await.unwrap(); + let edge = repo.usage_summary(range, Default::default()).await.unwrap(); + assert_eq!(edge.requests, persisted.requests); + assert_eq!(edge.attempts, persisted.attempts); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/analytics_buckets.rs b/backend/crates/gateway-store/tests/postgres/analytics_buckets.rs index 3001879aa..d0049263b 100644 --- a/backend/crates/gateway-store/tests/postgres/analytics_buckets.rs +++ b/backend/crates/gateway-store/tests/postgres/analytics_buckets.rs @@ -42,6 +42,9 @@ async fn dirty_quarters_share_one_hour_rebuild_and_acknowledge_each_bucket() { sqlx::raw_sql("create table hourly_build_audit (bucket_start timestamptz); create function audit_hourly_build() returns trigger language plpgsql as $$begin if new.collection='global' then insert into hourly_build_audit values(new.bucket_start); end if; return new; end$$; create trigger audit_hourly_build after insert on request_analytics_hourly for each row execute function audit_hourly_build()") .execute(&database.pool).await.unwrap(); seed_hour(&database.pool, hour(), "coalesced").await; + drain_analytics(&database.pool).await; + sqlx::raw_sql("truncate hourly_build_audit; insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts") + .execute(&database.pool).await.unwrap(); let worker = PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget()); let mut acknowledged = 0; @@ -85,6 +88,9 @@ async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuil .execute(&database.pool) .await .unwrap(); + // 显式修复任务重建整桶，普通更新只维护当前贡献，不重新启动构建 + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts") + .execute(&database.pool).await.unwrap(); let worker = PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget()); let mut owner = None; @@ -100,10 +106,12 @@ async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuil let changed: String = sqlx::query_scalar("select id from model_requests where date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC') <> $1 order by id limit 1") .bind(owner).fetch_one(&database.pool).await.unwrap(); sqlx::query("update model_requests set total_tokens=9999 where id=$1") - .bind(changed) + .bind(&changed) .execute(&database.pool) .await .unwrap(); + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from model_requests where id=$1 on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version')") + .bind(changed).execute(&database.pool).await.unwrap(); worker.maintain(2).await.unwrap(); let phase: i16 = sqlx::query_scalar( "select build_phase from request_analytics_pending_buckets where bucket_start=$1", @@ -174,3 +182,174 @@ async fn waiting_hours_cannot_fill_all_slots_and_starve_their_unbuilt_quarters() assert_eq!(result.requests.total_tokens, 18 * 120); database.close().await; } + +#[tokio::test] +async fn continuous_updates_do_not_restart_partial_quarter_and_hour_builds() { + let Some(database) = TestDatabase::create("analytics_continuous_stage_updates").await else { + return; + }; + for quarter in 0..2 { + for name in ["A", "a", "Z", "z", "Ä", "ä", "中", "é", "e", "_", "-", "2"] { + seed( + &database.pool, + &format!("prefix_{name}_{quarter}"), + hour() + TimeDelta::minutes(quarter * 15 + 1), + &[], + ) + .await; + } + } + drain_analytics(&database.pool).await; + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts") + .execute(&database.pool).await.unwrap(); + let mut quarter_prefix_seen = false; + let mut hour_prefix_seen = false; + let mut completed = false; + for round in 0..150_i64 { + // 同时间的大小写及非 ASCII ID、改变排序位置的账号维度同时覆盖两种前缀 + sqlx::query("update model_requests set total_tokens=$1,provider_account_ref=$2 where id in ('prefix_A_0','prefix_ä_1')") + .bind(200+round).bind(if round%2==0 {"account_Ä"}else{"account_Z"}) + .execute(&database.pool).await.unwrap(); + let worker = + PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget()); + worker.maintain(2).await.unwrap(); + let phases: (bool,bool,i64) = sqlx::query_as("select coalesce(bool_or(build_phase=1 and build_id is not null),false),coalesce(bool_or(build_phase=4 and build_collection is not null),false),count(*) from request_analytics_pending_buckets") + .fetch_one(&database.pool).await.unwrap(); + quarter_prefix_seen |= phases.0; + hour_prefix_seen |= phases.1; + if phases.2 == 0 { + completed = true; + break; + } + } + assert!( + quarter_prefix_seen && hour_prefix_seen, + "both persisted prefixes must be exercised" + ); + assert!( + completed, + "ongoing updates cannot repeatedly reset a partial rebuild" + ); + drain_analytics(&database.pool).await; + let expected: (i64, i64) = + sqlx::query_as("select count(*),sum(total_tokens)::bigint from model_requests") + .fetch_one(&database.pool) + .await + .unwrap(); + let actual = observability_repository(&database.pool) + .usage_summary( + ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(), + Default::default(), + ) + .await + .unwrap(); + assert_eq!(actual.requests.request_count, expected.0 as u64); + assert_eq!(actual.requests.total_tokens, expected.1 as u64); + database.close().await; +} + +#[tokio::test] +async fn incremental_deletions_repair_extrema_last_usage_and_zero_currency_presence() { + let Some(database) = TestDatabase::create("analytics_incremental_extrema").await else { + return; + }; + for number in 1..=4_i64 { + let id = format!("extrema_{number}"); + seed( + &database.pool, + &id, + hour() + TimeDelta::minutes(number), + &[], + ) + .await; + sqlx::query("update model_requests set provider_account_ref='extrema_account',request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',$2::bigint)),cost_source=case when $2<=2 then 'provider_reported' else 'unavailable' end,cost_currency=case $2 when 1 then 'EUR' when 2 then 'USD' end,cost_amount=case $2 when 1 then 1::numeric when 2 then 0::numeric end where id=$1") + .bind(id).bind(number).execute(&database.pool).await.unwrap(); + } + drain_analytics(&database.pool).await; + sqlx::query("delete from model_requests where id in ('extrema_1','extrema_4')") + .execute(&database.pool) + .await + .unwrap(); + drain_analytics(&database.pool).await; + let row: (i64,f64,f64,DateTime,serde_json::Value) = sqlx::query_as("select (counts->>'request_count')::bigint,(distributions->'latency_ms'->>'minimum')::double precision,(distributions->'latency_ms'->>'maximum')::double precision,last_used_at,costs from request_analytics_hourly where collection='global'") + .fetch_one(&database.pool).await.unwrap(); + assert_eq!( + (row.0, row.1, row.2, row.3), + (2, 2.0, 3.0, hour() + TimeDelta::minutes(3)) + ); + assert_eq!(row.4.as_object().unwrap().len(), 1); + assert_eq!( + row.4["USD"] + .as_str() + .unwrap() + .parse::() + .unwrap(), + gateway_core::metering::Decimal::ZERO + ); + sqlx::query("update model_requests set cost_source='unavailable',cost_currency=null,cost_amount=null where id='extrema_2'") + .execute(&database.pool).await.unwrap(); + drain_analytics(&database.pool).await; + let costs: serde_json::Value = + sqlx::query_scalar("select costs from request_analytics_hourly where collection='global'") + .fetch_one(&database.pool) + .await + .unwrap(); + assert_eq!(costs, serde_json::json!({})); + sqlx::query("select enqueue_request_analytics('extrema_3',$1)") + .bind(hour() + TimeDelta::minutes(3)) + .execute(&database.pool) + .await + .unwrap(); + drain_analytics(&database.pool).await; + let actual = observability_repository(&database.pool) + .usage_summary( + ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(), + Default::default(), + ) + .await + .unwrap(); + assert_eq!(actual.requests.request_count, 2); + assert_eq!(actual.requests.total_tokens, 240); + assert!(actual.attempts.costs.is_empty()); + database.close().await; +} + +#[tokio::test] +async fn incremental_summary_write_failure_rolls_back_facts_and_preserves_queue() { + let Some(database) = TestDatabase::create("analytics_incremental_rollback").await else { + return; + }; + seed( + &database.pool, + "atomic_incremental", + hour() + TimeDelta::minutes(1), + &[], + ) + .await; + drain_analytics(&database.pool).await; + sqlx::raw_sql("create function reject_incremental_summary() returns trigger language plpgsql as $$begin raise exception 'test summary write failure'; end$$; create trigger reject_incremental_summary before insert or update on request_analytics_15m for each row execute function reject_incremental_summary()") + .execute(&database.pool).await.unwrap(); + sqlx::query("update model_requests set total_tokens=240 where id='atomic_incremental'") + .execute(&database.pool) + .await + .unwrap(); + let worker = + PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget()); + assert!(worker.maintain(10).await.is_err()); + let state:(i64,i64,i64)=sqlx::query_as("select (select total_tokens from request_analytics_facts where id='atomic_incremental'),(select (counts->>'total_tokens')::bigint from request_analytics_15m where collection='global'),(select count(*) from request_analytics_pending_requests)") + .fetch_one(&database.pool).await.unwrap(); + assert_eq!(state, (120, 120, 1)); + sqlx::raw_sql("drop trigger reject_incremental_summary on request_analytics_15m; drop function reject_incremental_summary()") + .execute(&database.pool).await.unwrap(); + drain_analytics(&database.pool).await; + let actual = observability_repository(&database.pool) + .usage_summary( + ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(), + Default::default(), + ) + .await + .unwrap(); + assert_eq!(actual.requests.total_tokens, 240); + assert_eq!(actual.requests.request_count, 1); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/mod.rs b/backend/crates/gateway-store/tests/postgres/mod.rs index 500050d03..e2f07126b 100644 --- a/backend/crates/gateway-store/tests/postgres/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/mod.rs @@ -49,7 +49,7 @@ pub(super) struct TestDatabase { } pub(super) fn observability_query_budget() -> ObservabilityQueryBudget { - ObservabilityQueryBudget::try_new(4, Duration::from_secs(1)) + ObservabilityQueryBudget::try_new(4, Duration::from_secs(1), Duration::from_secs(10)) .expect("valid test observability query budget") } diff --git a/backend/crates/gateway-store/tests/postgres/observability/analytics/accounts.rs b/backend/crates/gateway-store/tests/postgres/observability/analytics/accounts.rs new file mode 100644 index 000000000..ea3c5ce60 --- /dev/null +++ b/backend/crates/gateway-store/tests/postgres/observability/analytics/accounts.rs @@ -0,0 +1,112 @@ +//! 十万条以上账号与 Key 用量的完整性、未知值和汇总读取验收 + +use std::time::Duration; + +use chrono::{DateTime, TimeDelta, Utc}; +use gateway_store::postgres::{ + ObservabilityQueryBudget, ObservabilityRange, PgObservabilityRepository, + PgRequestAnalyticsRepository, ProviderAccountUsageQuery, UsageRecordFilter, +}; +use sqlx::Connection; + +use super::super::super::{TestDatabase, analytics::seed}; + +#[tokio::test] +async fn large_account_and_key_usage_keep_exact_totals_and_read_clean_rollups() { + let Some(database) = TestDatabase::create("large_account_usage").await else { + return; + }; + let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + sqlx::query("insert into provider_accounts(id,provider_kind,name,authentication_kind,provider_credentials_json,credential_revision,has_refresh_token,credential_observed_at,created_at,updated_at) values('account_large','openai','large','oauth','{}',1,false,$1,$1,$1)") + .bind(start).execute(&database.pool).await.unwrap(); + seed( + &database.pool, + "large_template", + start + TimeDelta::minutes(1), + &[], + ) + .await; + sqlx::query("update model_requests set provider_kind='openai',provider_account_ref='account_large',cached_input_tokens=0,reasoning_output_tokens=null where id='large_template'") + .execute(&database.pool).await.unwrap(); + // 扩充真实源表并经过生产 Worker，确保回归覆盖账号接口而非只验证派生表 + sqlx::query("insert into model_requests select cloned.* from model_requests template cross join generate_series(1,100005) n cross join lateral jsonb_populate_record(null::model_requests,to_jsonb(template)||jsonb_build_object('id','account_large_'||n::text)) cloned where template.id='large_template'") + .execute(&database.pool).await.unwrap(); + let budget = + ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10)) + .unwrap(); + let worker = PgRequestAnalyticsRepository::new(database.pool.clone(), budget.clone()); + let mut ready = false; + for _ in 0..400 { + worker.maintain(1000).await.unwrap(); + let status = worker.status().await.unwrap(); + if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 { + ready = true; + break; + } + } + assert!( + ready, + "large source dataset must finish projection and rollups" + ); + let repo = PgObservabilityRepository::new(database.pool.clone(), None, budget, None); + let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap(); + let query = + ProviderAccountUsageQuery::for_accounts(range, vec!["account_large".to_owned()]).unwrap(); + let mut blocker = sqlx::PgConnection::connect_with(&database.pool.connect_options()) + .await + .unwrap(); + sqlx::raw_sql("begin;lock table request_analytics_facts in access exclusive mode") + .execute(&mut blocker) + .await + .unwrap(); + let account = tokio::time::timeout(Duration::from_secs(2), repo.provider_account_usage(query)) + .await + .expect("clean account usage must not read facts") + .unwrap() + .remove(0); + assert_eq!(account.request_count, 100006); + assert_eq!(account.total_tokens, Some(100006 * 120)); + assert_eq!(account.cached_input_tokens, Some(0)); + assert_eq!(account.reasoning_output_tokens, None); + assert_eq!(account.models.len(), 1); + assert_eq!(account.models[0].request_count, 100006); + assert_eq!(account.last_used_at, Some(start + TimeDelta::minutes(1))); + let filter = UsageRecordFilter { + client_api_key_ref: Some("key_analytics".to_owned()), + model: Some("model_a".to_owned()), + ..Default::default() + }; + let key = tokio::time::timeout(Duration::from_secs(2), repo.usage_summary(range, filter)) + .await + .expect("Key plus model must use joint rollups") + .unwrap(); + assert_eq!(key.requests.request_count, 100006); + assert_eq!(key.providers.len(), 1); + assert_eq!(key.providers[0].request_count, 100006); + sqlx::raw_sql("rollback") + .execute(&mut blocker) + .await + .unwrap(); + drop(blocker); + let partial = ObservabilityRange::new( + start + TimeDelta::seconds(30), + start + TimeDelta::minutes(2), + ) + .unwrap(); + let account = repo + .provider_account_usage( + ProviderAccountUsageQuery::for_accounts(partial, vec!["account_large".to_owned()]) + .unwrap(), + ) + .await + .unwrap() + .remove(0); + assert_eq!( + account.request_count, 100006, + "the boundary aggregation must not truncate a hot bucket" + ); + assert_eq!(account.total_tokens, Some(100006 * 120)); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/observability/analytics/mod.rs b/backend/crates/gateway-store/tests/postgres/observability/analytics/mod.rs new file mode 100644 index 000000000..0ae3a0338 --- /dev/null +++ b/backend/crates/gateway-store/tests/postgres/observability/analytics/mod.rs @@ -0,0 +1,3 @@ +//! 汇总读取、边界聚合与账号联合维度的真实数据库回归 + +mod accounts; diff --git a/backend/crates/gateway-store/tests/postgres/observability/mod.rs b/backend/crates/gateway-store/tests/postgres/observability/mod.rs index eb64126d1..82f240b06 100644 --- a/backend/crates/gateway-store/tests/postgres/observability/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/observability/mod.rs @@ -1,5 +1,6 @@ //! 观测存储测试入口，以及查询范围、过滤与端口合同测试 +mod analytics; mod projection; use std::{ @@ -441,7 +442,6 @@ async fn usage_list_and_diagnostics_preserve_account_snapshots_by_id() { .list_usage_records(admin_observability::UsageQuery { range, filter: admin_observability::UsageFilter { - routing_group_refs: Vec::new(), provider_account_ref: Some("acct_team".to_owned()), ..admin_observability::UsageFilter::default() }, @@ -504,7 +504,7 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() { .list_usage_records(UsageRecordQuery { range, filter: UsageRecordFilter { - search: Some("observe_success".to_owned()), + search: Some("rimary".to_owned()), ..UsageRecordFilter::default() }, current_page: 1, @@ -517,6 +517,51 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() { database.close().await; } +#[tokio::test] +async fn usage_search_should_ignore_request_ids_and_preserve_exact_filters() { + let Some(database) = TestDatabase::create("usage_search_without_request_ids").await else { + return; + }; + let now = Utc::now(); + seed_observability_facts(&database.pool, now) + .await + .expect("seed observability facts"); + let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1)) + .expect("observability range"); + + for search in [ + "req_observe", + "req_observe_success", + "upstream_req", + "upstream_req_success", + ] { + assert_usage_search_ids(&database.pool, range, search, &[]).await; + } + for filter in [ + UsageRecordFilter { + request_id: Some("req_observe_success".to_owned()), + ..Default::default() + }, + UsageRecordFilter { + upstream_request_id: Some("upstream_req_success".to_owned()), + ..Default::default() + }, + ] { + let page = observability_repository(&database.pool) + .list_usage_records(UsageRecordQuery { + range, + filter, + current_page: 1, + page_size: ObservabilityPageSize::new(10).expect("page size"), + }) + .await + .expect("exact request identity filter"); + assert_eq!(page.total, 1); + assert_eq!(page.items[0].id, "req_observe_success"); + } + database.close().await; +} + #[tokio::test] async fn usage_search_should_match_account_email_and_name_prefixes() { let Some(database) = TestDatabase::create("usage_account_prefix").await else { @@ -1687,7 +1732,6 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det .list_usage_records(admin_observability::UsageQuery { range, filter: admin_observability::UsageFilter { - routing_group_refs: Vec::new(), client_api_key_ref: Some("key_observe".to_owned()), request_id: Some("req_observe_success".to_owned()), provider_account_ref: Some("acct_observe".to_owned()), @@ -1700,7 +1744,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det attempt_index: Some(1), response_id: Some("resp_observe_success".to_owned()), upstream_request_id: Some("upstream_req_success".to_owned()), - search: Some("req_observe_success".to_owned()), + search: Some("upstream-model".to_owned()), }, current_page: 1, page_size: ObservabilityPageSize::new(10).expect("page size"), @@ -1716,7 +1760,6 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det .list_usage_records(admin_observability::UsageQuery { range, filter: admin_observability::UsageFilter { - routing_group_refs: Vec::new(), outcome: Some( admin_observability::RequestOutcome::new("provider_future_state") .expect("bounded other outcome"), diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs index a69c95267..fa9c0e370 100644 --- a/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs @@ -11,6 +11,7 @@ mod authorization; mod deletion; mod quota_forecast; mod timestamps; +mod usage_projection; use chrono::{TimeDelta, Utc}; use gateway_admin::{ diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts/quota_forecast.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts/quota_forecast.rs index 3efda91a0..025458f22 100644 --- a/backend/crates/gateway-store/tests/postgres/provider_accounts/quota_forecast.rs +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts/quota_forecast.rs @@ -9,6 +9,66 @@ use gateway_store::postgres::{ UsageRecordQuery, }; +#[tokio::test] +async fn quota_forecast_fractional_slots_keep_microsecond_edges_and_as_of_ties() { + let Some(database) = TestDatabase::create("forecast_fractional_slots").await else { + return; + }; + PgProviderAccountRepository::new(database.pool.clone()) + .insert_provider_account(account("acct_fraction", "user-fraction")) + .await + .unwrap(); + let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + let duration = MAX_FORECAST_HISTORY_POINTS as i64 * 1_000_000 + 64; + let end = start + TimeDelta::microseconds(duration); + for (id, completed) in [ + ("before_edge", 1_000_000), + ("after_edge", 1_000_001), + ("as_of_a", duration), + ("as_of_b", duration), + ("pending", duration + 1), + ] { + seed_model_request( + &database.pool, + ModelRequestSeed { + request_id: id, + account_id: "acct_fraction", + provider_kind: "openai", + model: "gpt-test", + total_tokens: 100, + cost_amount: "1", + started_at: start, + }, + ) + .await + .unwrap(); + sqlx::query("update model_requests set completed_at=$2,provider_observation_json='{}'::jsonb where id=$1") + .bind(id).bind(start+TimeDelta::microseconds(completed)).execute(&database.pool).await.unwrap(); + } + let history = admin_account_store(&database.pool) + .load_quota_forecast_history(&AccountUsageWindowQuery { + account_id: "acct_fraction".to_owned(), + key: "fraction".to_owned(), + range: TimeRange { start, end }, + }) + .await + .unwrap(); + assert_eq!( + history + .points + .iter() + .map(|point| point.usage.request_count) + .collect::>(), + vec![1, 2, 4] + ); + assert_eq!(history.usage.request_count, 4); + assert_eq!(history.usage.tokens, 400); + assert_eq!(history.pending_request_count, 1); + database.close().await; +} + #[tokio::test] async fn quota_forecast_excludes_openai_prewarm_but_preserves_inference_and_audit() { let Some(database) = TestDatabase::create("forecast_prewarm").await else { @@ -341,3 +401,219 @@ async fn quota_forecast_bounds_documents_without_sampling_away_token_totals() { assert_eq!(history.points.last().unwrap().usage.tokens, 25_600); database.close().await; } + +#[tokio::test] +async fn quota_forecast_carries_unsampled_usage_between_document_points() { + let Some(database) = TestDatabase::create("quota_forecast_bucket_prefix").await else { + return; + }; + PgProviderAccountRepository::new(database.pool.clone()) + .insert_provider_account(account("acct_prefix", "user-prefix")) + .await + .unwrap(); + let start = Utc::now() - TimeDelta::hours(6); + let end = start + TimeDelta::minutes(MAX_FORECAST_HISTORY_POINTS as i64); + for (id, second, document) in [ + ("before_doc", 10, false), + ("first_doc", 20, true), + ("tie_doc", 20, true), + ("after_doc", 40, false), + ("no_document_bucket", 70, false), + ("third_doc", 130, true), + ("after_last_doc", 150, false), + ("late", 180, true), + ] { + seed_model_request( + &database.pool, + ModelRequestSeed { + request_id: id, + account_id: "acct_prefix", + provider_kind: "openai", + model: "gpt-test", + total_tokens: 100, + cost_amount: "1", + started_at: start + TimeDelta::seconds(second), + }, + ) + .await + .unwrap(); + if document { + sqlx::query( + "update model_requests set provider_observation_json = '{}'::jsonb where id = $1", + ) + .bind(id) + .execute(&database.pool) + .await + .unwrap(); + } + } + sqlx::query("update model_requests set completed_at = $1 where id = 'late'") + .bind(end + TimeDelta::seconds(1)) + .execute(&database.pool) + .await + .unwrap(); + let history = admin_account_store(&database.pool) + .load_quota_forecast_history(&AccountUsageWindowQuery { + account_id: "acct_prefix".to_owned(), + key: "week".to_owned(), + range: TimeRange { start, end }, + }) + .await + .unwrap(); + assert_eq!(history.points.len(), 2); + assert_eq!(history.points[0].usage.request_count, 3); + assert_eq!(history.points[0].usage.tokens, 300); + assert_eq!(history.points[1].usage.request_count, 6); + assert_eq!(history.points[1].usage.tokens, 600); + assert_eq!(history.usage.request_count, 7); + assert_eq!(history.usage.tokens, 700); + assert_eq!(history.pending_request_count, 1); + database.close().await; +} + +#[tokio::test] +async fn quota_forecast_summaries_match_source_during_moves_deletes_and_rebuilds() { + use super::super::drain_analytics; + use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistory; + + fn assert_same(actual: &QuotaForecastHistory, expected: &QuotaForecastHistory) { + assert_eq!(actual.usage, expected.usage); + assert_eq!(actual.pending_request_count, expected.pending_request_count); + assert_eq!(actual.points.len(), expected.points.len()); + for (actual, expected) in actual.points.iter().zip(&expected.points) { + assert_eq!(actual.started_at, expected.started_at); + assert_eq!(actual.completed_at, expected.completed_at); + assert_eq!(actual.usage, expected.usage); + } + } + + let Some(database) = TestDatabase::create("forecast_completion_projection").await else { + return; + }; + let repository = PgProviderAccountRepository::new(database.pool.clone()); + for id in ["acct_forecast_projection", "acct_forecast_moved"] { + repository + .insert_provider_account(account(id, id)) + .await + .unwrap(); + } + let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + for index in 0..200 { + seed_model_request( + &database.pool, + ModelRequestSeed { + request_id: &format!("forecast_projection_{index}"), + account_id: "acct_forecast_projection", + provider_kind: "openai", + model: "gpt-test", + total_tokens: index + 10, + cost_amount: "0.125", + started_at: start + TimeDelta::minutes(index * 3), + }, + ) + .await + .unwrap(); + } + sqlx::query("update model_requests set provider_observation_json='{}'::jsonb where substring(id from '[0-9]+$')::int % 9=0") + .execute(&database.pool).await.unwrap(); + // 完成时间与开始时间跨越多个桶，包含未来完成、缺失 Token、未知费用与恢复记录 + sqlx::query("update model_requests set completed_at=completed_at+interval '90 minutes' where substring(id from '[0-9]+$')::int % 7=0") + .execute(&database.pool).await.unwrap(); + sqlx::query("update model_requests set total_tokens=null,input_tokens=null,cost_source='unavailable',cost_amount=null,cost_currency=null where id='forecast_projection_31'") + .execute(&database.pool).await.unwrap(); + sqlx::query( + "update model_requests set recovered_at=completed_at,recovery_request_id='forecast_projection_48',recovery_attempt_count=1,request_observation_json=request_observation_json || '{\"recovery\":{\"retryDelayMs\":0,\"totalLatencyMs\":1}}'::jsonb where id='forecast_projection_47'", + ) + .execute(&database.pool) + .await + .unwrap(); + sqlx::query("update model_requests set outcome='cancelled' where id='forecast_projection_63'") + .execute(&database.pool) + .await + .unwrap(); + sqlx::query("update model_requests set cost_currency='EUR' where id='forecast_projection_83'") + .execute(&database.pool) + .await + .unwrap(); + let store = admin_account_store(&database.pool); + let queries = [ + AccountUsageWindowQuery { + account_id: "acct_forecast_projection".to_owned(), + key: "complete".to_owned(), + range: TimeRange { + start, + end: start + TimeDelta::hours(10), + }, + }, + AccountUsageWindowQuery { + account_id: "acct_forecast_projection".to_owned(), + key: "edges".to_owned(), + range: TimeRange { + start: start + TimeDelta::microseconds(1), + end: start + TimeDelta::minutes(593) + TimeDelta::microseconds(1), + }, + }, + AccountUsageWindowQuery { + account_id: "acct_forecast_moved".to_owned(), + key: "moved".to_owned(), + range: TimeRange { + start, + end: start + TimeDelta::hours(10), + }, + }, + ]; + let mut source = Vec::new(); + for query in &queries { + source.push(store.load_quota_forecast_history(query).await.unwrap()); + } + drain_analytics(&database.pool).await; + for (query, expected) in queries.iter().zip(&source) { + assert_same( + &store.load_quota_forecast_history(query).await.unwrap(), + expected, + ); + } + sqlx::query("update model_requests set provider_account_ref='acct_forecast_moved',started_at=$1,completed_at=$1+interval '1 second',total_tokens=1234,cost_amount=2.5 where id='forecast_projection_72'") + .bind(start+TimeDelta::minutes(40)).execute(&database.pool).await.unwrap(); + sqlx::query("delete from model_requests where id='forecast_projection_47'") + .execute(&database.pool) + .await + .unwrap(); + sqlx::query("update model_requests set completed_at=$1 where id='forecast_projection_196'") + .bind(start + TimeDelta::minutes(590)) + .execute(&database.pool) + .await + .unwrap(); + let mut pending = Vec::new(); + for query in &queries { + pending.push(store.load_quota_forecast_history(query).await.unwrap()); + } + assert_eq!(pending[2].usage.request_count, 1); + assert_eq!(pending[2].usage.tokens, 1234); + drain_analytics(&database.pool).await; + for (query, expected) in queries.iter().zip(&pending) { + assert_same( + &store.load_quota_forecast_history(query).await.unwrap(), + expected, + ); + } + // 显式重建中的区间由窄事实接管，发布后仍与同一源结果一致 + sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts") + .execute(&database.pool).await.unwrap(); + for (query, expected) in queries.iter().zip(&pending) { + assert_same( + &store.load_quota_forecast_history(query).await.unwrap(), + expected, + ); + } + drain_analytics(&database.pool).await; + for (query, expected) in queries.iter().zip(&pending) { + assert_same( + &store.load_quota_forecast_history(query).await.unwrap(), + expected, + ); + } + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs new file mode 100644 index 000000000..a248a9394 --- /dev/null +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts/usage_projection.rs @@ -0,0 +1,150 @@ +//! 验证账号同步用量在汇总、待投影修改和删除之间保持源事实一致 + +use super::super::drain_analytics; +use super::*; + +#[tokio::test] +async fn quota_windows_reconcile_moved_requests_and_pending_deletions_without_waiting_for_worker() { + let Some(database) = TestDatabase::create("account_window_source_reconciliation").await else { + return; + }; + let repository = PgProviderAccountRepository::new(database.pool.clone()); + for id in ["acct_usage_before", "acct_usage_after"] { + repository + .insert_provider_account(account(id, id)) + .await + .unwrap(); + } + let start = chrono::DateTime::from_timestamp(Utc::now().timestamp().div_euclid(3600) * 3600, 0) + .unwrap() + - TimeDelta::hours(2); + for (id, offset, tokens, cost) in [("usage_stays", 5, 11, "0"), ("usage_moves", 40, 23, "2.5")] + { + seed_model_request( + &database.pool, + ModelRequestSeed { + request_id: id, + account_id: "acct_usage_before", + provider_kind: "openai", + model: "old-model", + total_tokens: tokens, + cost_amount: cost, + started_at: start + TimeDelta::minutes(offset), + }, + ) + .await + .unwrap(); + } + sqlx::query("update model_requests set cached_input_tokens=null,reasoning_output_tokens=null where id in ('usage_stays','usage_moves')") + .execute(&database.pool).await.unwrap(); + drain_analytics(&database.pool).await; + let windows = [ + AccountUsageWindowQuery { + account_id: "acct_usage_before".to_owned(), + key: "old".to_owned(), + range: TimeRange { + start, + end: start + TimeDelta::hours(1), + }, + }, + AccountUsageWindowQuery { + account_id: "acct_usage_after".to_owned(), + key: "new".to_owned(), + range: TimeRange { + start: start + TimeDelta::hours(1), + end: start + TimeDelta::hours(2), + }, + }, + ]; + let store = admin_account_store(&database.pool); + let before = store.load_account_usage_by_windows(&windows).await.unwrap(); + let initial = before.iter().find(|usage| usage.key == "old").unwrap(); + assert_eq!(initial.usage.total_tokens, Some(34)); + assert_eq!(initial.usage.cached_input_tokens, None); + assert_eq!(initial.usage.costs[0].amount.as_str(), "2.5"); + let ranked = store + .list_accounts( + AccountListQuery { + page: 1, + page_size: PageSize::new(1).unwrap(), + provider_kind: None, + group_filter: None, + search: None, + status: None, + sort: Some(AccountSort { + field: AccountSortField::Usage, + direction: SortDirection::Desc, + }), + }, + Default::default(), + ) + .await + .unwrap(); + assert_eq!(ranked.items[0].account.id, "acct_usage_before"); + + sqlx::query("update model_requests set provider_account_ref='acct_usage_after',upstream_model_id='new-model',total_tokens=null,input_tokens=7,output_tokens=8,cached_input_tokens=0,cost_amount=4.75,started_at=$1,completed_at=$1+interval '1 minute',downstream_committed_at=$1+interval '1 minute',recovery_lease_expires_at=$1+interval '2 minutes' where id='usage_moves'") + .bind(start+TimeDelta::minutes(70)).execute(&database.pool).await.unwrap(); + let changed = store.load_account_usage_by_windows(&windows).await.unwrap(); + let old = changed.iter().find(|usage| usage.key == "old").unwrap(); + let new = changed.iter().find(|usage| usage.key == "new").unwrap(); + assert_eq!(old.usage.request_count, 1); + assert_eq!(old.usage.total_tokens, Some(11)); + assert_eq!(old.usage.last_used_at, Some(start + TimeDelta::minutes(5))); + assert_eq!( + old.usage.costs[0].amount.as_str(), + "0", + "a real zero-cost request retains its currency" + ); + assert_eq!(new.usage.request_count, 1); + assert_eq!( + new.usage.total_tokens, + Some(15), + "quota windows retain component fallback for missing provider total" + ); + assert_eq!(new.usage.cached_input_tokens, Some(0)); + assert_eq!(new.usage.reasoning_output_tokens, None); + assert_eq!(new.usage.models[0].model, "new-model"); + assert_eq!(new.usage.costs[0].amount.as_str(), "4.75"); + for field in [AccountSortField::Usage, AccountSortField::LastUsedAt] { + let ranked = store + .list_accounts( + AccountListQuery { + page: 1, + page_size: PageSize::new(1).unwrap(), + provider_kind: None, + group_filter: None, + search: None, + status: None, + sort: Some(AccountSort { + field, + direction: SortDirection::Desc, + }), + }, + Default::default(), + ) + .await + .unwrap(); + assert_eq!(ranked.items[0].account.id, "acct_usage_after"); + } + + // 独立源表真值不经过同一汇总代码，防止仅比较两个相同错误的读取分支 + let truth:(i64,i64,String)=sqlx::query_as("select count(*)::bigint,sum(coalesce(total_tokens,input_tokens+output_tokens))::bigint,sum(cost_amount)::text from model_requests where provider_account_ref='acct_usage_after' and started_at >= $1 and started_at < $2") + .bind(windows[1].range.start).bind(windows[1].range.end).fetch_one(&database.pool).await.unwrap(); + assert_eq!(truth.0, new.usage.request_count as i64); + assert_eq!(truth.1, new.usage.total_tokens.unwrap() as i64); + assert_eq!( + truth.2.parse::().unwrap(), + new.usage.costs[0].amount + ); + sqlx::query("delete from model_requests where id='usage_stays'") + .execute(&database.pool) + .await + .unwrap(); + let deleted = store.load_account_usage_by_windows(&windows).await.unwrap(); + let old = deleted.iter().find(|usage| usage.key == "old").unwrap(); + assert_eq!(old.usage.request_count, 0); + assert_eq!(old.usage.last_used_at, None); + assert!(old.usage.models.is_empty()); + assert!(old.usage.costs.is_empty()); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/query_budget.rs b/backend/crates/gateway-store/tests/postgres/query_budget.rs index a2c98c2bf..49f2a2aca 100644 --- a/backend/crates/gateway-store/tests/postgres/query_budget.rs +++ b/backend/crates/gateway-store/tests/postgres/query_budget.rs @@ -19,7 +19,10 @@ async fn test_pool() -> Option { #[test] fn query_budget_rejects_an_empty_budget() { - assert!(ObservabilityQueryBudget::try_new(0, Duration::from_secs(1)).is_err()); + assert!( + ObservabilityQueryBudget::try_new(0, Duration::from_secs(1), Duration::from_secs(10)) + .is_err() + ); } #[tokio::test] @@ -27,7 +30,9 @@ async fn running_query_deadline_releases_the_slot() { let Some(pool) = test_pool().await else { return; }; - let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(10)).unwrap(); + let budget = + ObservabilityQueryBudget::try_new(1, Duration::from_millis(10), Duration::from_millis(250)) + .unwrap(); let error = budget .run(&pool, "never completes", |connection| async move { let result = std::future::pending::>().await; @@ -46,13 +51,77 @@ async fn running_query_deadline_releases_the_slot() { pool.close().await; } +#[tokio::test] +async fn queued_time_is_included_in_the_operation_deadline() { + let Some(pool) = test_pool().await else { + return; + }; + let budget = + ObservabilityQueryBudget::try_new(1, Duration::from_secs(2), Duration::from_secs(2)) + .unwrap(); + let (occupied_tx, occupied_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let occupied_budget = budget.clone(); + let occupied_pool = pool.clone(); + let occupied = tokio::spawn(async move { + occupied_budget + .run( + &occupied_pool, + "hold before queued deadline", + |connection| async move { + occupied_tx.send(()).unwrap(); + release_rx.await.unwrap(); + drop(connection); + Ok(()) + }, + ) + .await + .unwrap(); + }); + occupied_rx.await.unwrap(); + let started = tokio::time::Instant::now(); + let release = tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(2) / 5).await; + release_tx.send(()).unwrap(); + }); + let error = budget + .run( + &pool, + "queued operation deadline", + |connection| async move { + let result = std::future::pending::>().await; + drop(connection); + result + }, + ) + .await + .expect_err("queue and execution share one deadline"); + assert!( + matches!(error, StoreError::Unavailable { message, .. } if message.contains("execution budget")) + ); + assert!( + started.elapsed() < Duration::from_secs(2) + Duration::from_millis(200), + "acquiring a slot must not restart the operation budget" + ); + release.await.unwrap(); + occupied.await.unwrap(); + budget + .run(&pool, "after queued timeout", |_connection| async { + Ok(()) + }) + .await + .unwrap(); + pool.close().await; +} + #[tokio::test] async fn query_budget_times_out_without_exceeding_its_connection_limit() { let Some(pool) = test_pool().await else { return; }; - let budget = ObservabilityQueryBudget::try_new(4, Duration::from_millis(10)) - .expect("valid query budget"); + let budget = + ObservabilityQueryBudget::try_new(4, Duration::from_millis(10), Duration::from_secs(10)) + .expect("valid query budget"); let (started_tx, mut started_rx) = mpsc::unbounded_channel(); let mut releases = Vec::new(); let mut occupied = Vec::new(); @@ -110,7 +179,8 @@ async fn queued_query_runs_after_one_slot_is_released() { return; }; let budget = - ObservabilityQueryBudget::try_new(4, Duration::from_secs(1)).expect("valid query budget"); + ObservabilityQueryBudget::try_new(4, Duration::from_secs(1), Duration::from_secs(10)) + .expect("valid query budget"); let (started_tx, mut started_rx) = mpsc::unbounded_channel(); let mut releases = Vec::new(); let mut occupied = Vec::new(); @@ -183,7 +253,9 @@ async fn operation_timeout_keeps_the_slot_until_the_database_query_finishes() { let Some(pool) = test_pool().await else { return; }; - let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(100)).unwrap(); + let budget = + ObservabilityQueryBudget::try_new(1, Duration::from_millis(100), Duration::from_secs(2)) + .unwrap(); let (pid_tx, pid_rx) = oneshot::channel(); let error = budget .run( @@ -196,15 +268,18 @@ async fn operation_timeout_keeps_the_slot_until_the_database_query_finishes() { .map_err(database_error)?; pid_tx.send(pid).unwrap(); let mut tx = connection.begin().await.map_err(database_error)?; - sqlx::query("set local statement_timeout = '10s'") + sqlx::query("select set_config('statement_timeout',$1,true)") + .bind(format!("{}ms", Duration::from_secs(2).as_millis())) .execute(&mut *tx) .await .map_err(database_error)?; - sqlx::query("select pg_sleep(6)") + sqlx::query("select pg_sleep($1)") + .bind(Duration::from_secs(2).as_secs_f64() * 0.6) .execute(&mut *tx) .await .map_err(database_error)?; - sqlx::query("select pg_sleep(9)") + sqlx::query("select pg_sleep($1)") + .bind(Duration::from_secs(2).as_secs_f64() * 0.9) .execute(&mut *tx) .await .map_err(database_error)?; @@ -271,7 +346,9 @@ async fn dropping_an_operation_keeps_the_slot_through_database_rollback() { let Some(pool) = test_pool().await else { return; }; - let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(100)).unwrap(); + let budget = + ObservabilityQueryBudget::try_new(1, Duration::from_millis(100), Duration::from_secs(10)) + .unwrap(); let running_pool = pool.clone(); let running_budget = budget.clone(); let (pid_tx, pid_rx) = oneshot::channel(); diff --git a/backend/crates/gateway-store/tests/postgres/usage_paging.rs b/backend/crates/gateway-store/tests/postgres/usage_paging.rs index 87073e93b..53425fdfb 100644 --- a/backend/crates/gateway-store/tests/postgres/usage_paging.rs +++ b/backend/crates/gateway-store/tests/postgres/usage_paging.rs @@ -2,11 +2,97 @@ use chrono::{TimeDelta, Utc}; use gateway_store::postgres::{ - ObservabilityPageSize, ObservabilityRange, UsageRecordFilter, UsageRecordQuery, + ObservabilityPageSize, ObservabilityRange, OpsErrorFilter, OpsErrorQuery, UsageRecordFilter, + UsageRecordQuery, }; use super::{TestDatabase, analytics::seed, observability_repository}; +#[tokio::test] +async fn projected_page_counts_correct_pending_moves_deletions_and_model_or() { + let Some(database) = TestDatabase::create("usage_projected_pages").await else { + return; + }; + let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + for index in 0..40 { + seed( + &database.pool, + &format!("projected_{index:02}"), + start + TimeDelta::minutes(index), + &[], + ) + .await; + } + super::drain_analytics(&database.pool).await; + let repository = observability_repository(&database.pool); + for stage in 0..3 { + if stage == 1 { + sqlx::query("update model_requests set started_at=started_at+interval '30 minutes',recovery_lease_expires_at=recovery_lease_expires_at+interval '30 minutes',completed_at=completed_at+interval '30 minutes',requested_model_id='model_b',upstream_model_id='model_b' where id='projected_06'") + .execute(&database.pool).await.unwrap(); + sqlx::query("delete from model_requests where id='projected_17'") + .execute(&database.pool) + .await + .unwrap(); + sqlx::query("update model_requests set outcome='failed' where id='projected_21'") + .execute(&database.pool) + .await + .unwrap(); + seed( + &database.pool, + "projected_new", + start + TimeDelta::seconds(30), + &[], + ) + .await; + } else if stage == 2 { + super::drain_analytics(&database.pool).await; + } + for edge in [0, 31] { + let range = ObservabilityRange::new( + start + TimeDelta::seconds(edge), + start + TimeDelta::hours(1) - TimeDelta::seconds(edge), + ) + .unwrap(); + for model in [None, Some("model_a"), Some("model_b")] { + // 真值只读取实际源版本，不依赖被测汇总或分页辅助函数 + let expected:Vec = sqlx::query_scalar("select id from model_requests where started_at >= $1 and started_at < $2 and outcome='succeeded' and ($3::text is null or requested_model_id=$3 or upstream_model_id=$3) order by started_at desc,id desc") + .bind(range.start).bind(range.end).bind(model).fetch_all(&database.pool).await.unwrap(); + for current_page in [1, 3, 6, 8] { + let page = repository + .list_usage_records(UsageRecordQuery { + range, + filter: UsageRecordFilter { + client_api_key_ref: Some("key_analytics".to_owned()), + model: model.map(str::to_owned), + ..Default::default() + }, + current_page, + page_size: ObservabilityPageSize::new(7).unwrap(), + }) + .await + .unwrap(); + assert_eq!( + page.total, + expected.len() as u64, + "stage={stage}, edge={edge}, model={model:?}" + ); + assert_eq!( + page.items.iter().map(|row| &row.id).collect::>(), + expected + .iter() + .skip((current_page as usize - 1) * 7) + .take(7) + .collect::>() + ); + } + } + } + } + database.close().await; +} + #[tokio::test] async fn usage_pages_preserve_order_and_exact_totals_from_both_ends() { let Some(database) = TestDatabase::create("usage_bidirectional_pages").await else { @@ -70,3 +156,127 @@ async fn usage_pages_preserve_order_and_exact_totals_from_both_ends() { } database.close().await; } + +#[tokio::test] +async fn usage_pages_cross_minutes_and_keep_exact_counts_after_source_changes() { + let Some(database) = TestDatabase::create("usage_time_pages").await else { + return; + }; + let start = Utc::now() - TimeDelta::hours(1); + for index in 0..29 { + seed( + &database.pool, + &format!("cross_{index:02}"), + start + TimeDelta::seconds((index / 3) * 100), + &["group_page"], + ) + .await; + } + let repository = observability_repository(&database.pool); + let range = ObservabilityRange::new(start, start + TimeDelta::minutes(20)).unwrap(); + for removed in [None, Some("cross_16")] { + if let Some(id) = removed { + sqlx::query("delete from model_requests where id = $1") + .bind(id) + .execute(&database.pool) + .await + .unwrap(); + } + let expected = (0..29) + .rev() + .map(|index| format!("cross_{index:02}")) + .filter(|id| Some(id.as_str()) != removed) + .collect::>(); + for current_page in 1..=6 { + let page = repository + .list_usage_records(UsageRecordQuery { + range, + filter: UsageRecordFilter { + client_api_key_ref: Some("key_analytics".to_owned()), + search: Some("model_".to_owned()), + ..Default::default() + }, + current_page, + page_size: ObservabilityPageSize::new(7).unwrap(), + }) + .await + .unwrap(); + assert_eq!(page.total, expected.len() as u64); + assert_eq!( + page.items.iter().map(|item| &item.id).collect::>(), + expected + .iter() + .skip((current_page as usize - 1) * 7) + .take(7) + .collect::>(), + "page={current_page}, removed={removed:?}", + ); + } + } + database.close().await; +} + +#[tokio::test] +async fn ops_pages_merge_sources_across_minutes_with_stable_ties() { + let Some(database) = TestDatabase::create("ops_time_pages").await else { + return; + }; + let start = Utc::now() - TimeDelta::hours(1); + let mut expected = Vec::new(); + for index in 0..17 { + let id = format!("error_{index:02}"); + let at = start + TimeDelta::seconds((index / 3) * 100); + seed(&database.pool, &id, at, &[]).await; + sqlx::query("update model_requests set error_kind = 'upstream_error' where id = $1") + .bind(&id) + .execute(&database.pool) + .await + .unwrap(); + sqlx::query( + "insert into ops_events (id, model_request_id, attempt_index, level, component, + operation, failure_kind, message, created_at) + values ($1, $1, 1, 'error', 'routing', 'responses', 'upstream_error', 'fixture', $2)", + ) + .bind(&id) + .bind(at + TimeDelta::minutes(1)) + .execute(&database.pool) + .await + .unwrap(); + for source in ["model_request", "ops_event"] { + expected.push((at + TimeDelta::minutes(1), format!("{source}:{id}"))); + } + } + expected.sort_by(|a, b| b.cmp(a)); + let repository = observability_repository(&database.pool); + let range = ObservabilityRange::new(start, start + TimeDelta::minutes(20)).unwrap(); + for current_page in 1..=7 { + let page = repository + .list_ops_errors(OpsErrorQuery { + range, + filter: OpsErrorFilter { + client_api_key_ref: Some("key_analytics".to_owned()), + search: Some("error_".to_owned()), + ..Default::default() + }, + current_page, + page_size: ObservabilityPageSize::new(7).unwrap(), + }) + .await + .unwrap(); + assert_eq!(page.total, expected.len() as u64); + assert_eq!( + page.items + .iter() + .map(|item| &item.stable_sort_id) + .collect::>(), + expected + .iter() + .skip((current_page as usize - 1) * 7) + .take(7) + .map(|(_, id)| id) + .collect::>(), + "page={current_page}", + ); + } + database.close().await; +} diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 392662d06..26eea702e 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -27,3 +27,5 @@ d4b599066d7382a1c9cb6b3c003200046579e4fce3531f232cbd665a9d92c542 0026_codex_pri 8c9bbecd8315a26e928213a82a2ec651c35976e0643b82caef1d1bcfeee2916f 0027_request_observation_projection.sql 31d6668707186d7c19cdd507e20162a08c7fbb8ba41ff4a4439d0a5e08369cad 0028_request_analytics.sql 1923f14095dac5f808a530db002c37e17168afa0778bc7f3da768c818133eec9 0029_usage_indexes_and_identity_snapshots.sql +ab964fcdf56aa1f595fffaf02556aeebce5037f7bfa1ea6f4fef594112eaba6a 0030_analytics_usage_dimensions.sql +9d6e5254ee4a7aec40c4fc5f1904300228c57826941b962fa78cc6e550fa99f8 0031_analytics_forecast_completion.sql diff --git a/backend/migrations/0030_analytics_usage_dimensions.sql b/backend/migrations/0030_analytics_usage_dimensions.sql new file mode 100644 index 000000000..88167165d --- /dev/null +++ b/backend/migrations/0030_analytics_usage_dimensions.sql @@ -0,0 +1,33 @@ +-- 完整交付用量复用现有统计状态，最近使用时间保持可合并的独立极值 +alter table request_analytics_15m add column last_used_at timestamptz; +alter table request_analytics_hourly add column last_used_at timestamptz; + +-- 新指标只重建时间桶，不清空窄事实，也不重置已经完成的历史投影游标 +insert into request_analytics_pending_buckets(bucket_start, oldest_pending_at) +select distinct date_bin(interval '15 minutes', started_at, timestamptz '2000-01-01 UTC'), now() +from request_analytics_facts +on conflict (bucket_start) do update +set change_version = nextval('request_analytics_change_version'); + +-- 撤销最近一次模型贡献后按真实源定位极值，避免扫描整个账号历史 +create index model_requests_delivered_account_model on model_requests +(provider_account_ref,(coalesce(upstream_model_id,requested_model_id,'')),started_at desc,id desc) +where outcome='succeeded' and downstream_committed_at is not null + and (provider_kind is distinct from 'openai' or request_kind is distinct from 'prewarm') + and ((client_transport='websocket' and client_status_code is null) or client_status_code between 200 and 399) + and (requested_model_id is not null or upstream_model_id is not null or image_generation_requested + or input_tokens is not null or output_tokens is not null or cached_input_tokens is not null + or cache_write_input_tokens is not null or reasoning_output_tokens is not null + or image_input_tokens is not null or image_output_tokens is not null + or total_tokens is not null or cost_amount is not null); + +-- 预测只定位各采样桶最后一份 Provider 文档，宽 JSON 不进入索引 +create index model_requests_forecast_documents on model_requests +(provider_account_ref,completed_at desc,id desc) include(started_at) +where provider_observation_json is not null and outcome <> 'running'; + +-- 增量构建按字节序推进游标，索引排序必须与游标合同一致 +create index request_analytics_facts_build_cursor on request_analytics_facts +(started_at,id collate "C"); +create index request_analytics_15m_build_cursor on request_analytics_15m +(bucket_start,collection collate "C",dimensions collate "C",outcome collate "C"); diff --git a/backend/migrations/0031_analytics_forecast_completion.sql b/backend/migrations/0031_analytics_forecast_completion.sql new file mode 100644 index 000000000..b4901942b --- /dev/null +++ b/backend/migrations/0031_analytics_forecast_completion.sql @@ -0,0 +1,11 @@ +-- 完成时间边界只回读五分钟内的窄事实，Provider 文档继续使用源表采样索引 +create index request_analytics_facts_account_completion on request_analytics_facts +(provider_account_ref,completed_at,started_at) +where completed_at is not null and outcome <> 'running'; + +-- 新完成时间维度与既有统计共用可恢复构建，旧事实无需再次投影 +insert into request_analytics_pending_buckets(bucket_start, oldest_pending_at) +select distinct date_bin(interval '15 minutes', started_at, timestamptz '2000-01-01 UTC'), now() +from request_analytics_facts +on conflict (bucket_start) do update +set change_version = nextval('request_analytics_change_version'); diff --git a/deploy/README.md b/deploy/README.md index a9ccf12cb..24fc80cad 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -28,6 +28,10 @@ 配置加载会忽略未知字段，并在启动控制台提示字段名；不输出对应值。缺少必填字段时会指出缺项并停止启动， 已知字段的类型和取值仍需合法；可选字段省略时使用默认值 +`store.pool.observability_timeout_seconds` 默认 120 秒，控制观测查询与统计 Worker 的操作预算， +缓存等待和统计事务的 SQL 时限沿用该配置。修改后重启应用；连接排队仍受 +`store.pool.acquire_timeout_seconds` 约束，模型请求时限由 `api.request_timeout_seconds` 单独控制 + 后端从当前目录向上查找 `deploy/config.yaml`，相对数据、日志和静态资源路径以该文件所在目录解析 Compose 通过环境变量将监听和数据库地址设为容器内地址，并指定镜像中的静态资源目录 @@ -560,6 +564,10 @@ docker compose -f deploy/compose.yaml up -d --no-build --wait 升级后通过管理端版本接口或容器 image digest 确认运行实例的版本和 revision +管理端请求默认等待 120 秒。直接构建前端时可通过进程环境 `VITE_API_TIMEOUT_MS` 指定毫秒数， +例如 `VITE_API_TIMEOUT_MS=180000 pnpm --dir frontend build`，取值须为正整数。 +该值在构建时写入静态资源，修改后需要重新构建和部署前端。单个请求显式指定的时限仍优先 + 构建元数据通过一次性进程环境传入： ```bash diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index ee0f9dba0..c446955ca 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -53,6 +53,7 @@ store: # pool: # max_connections: 20 # acquire_timeout_seconds: 5 + # observability_timeout_seconds: 120 # 观测操作保护时限，不是性能目标 admin: # 不活动期限与从登录起计算的最长有效期，单位为分钟 diff --git a/docs/api.md b/docs/api.md index eadce70da..c3918646e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1248,8 +1248,8 @@ HTTP 请求头及新建 WS 的握手提示按当时的最终出站档位构造 `capacity.totalSlots` 为 `number | null`：`null` 表示可用成员中存在继承无限并发的账号，`0` 表示没有可用槽位。 分组容量只统计普通池，`capacity.usedSlots` 返回其实际在途数；Redis 不可用时为 `null` -分组费用按请求执行时实际服务账号的分组快照归属，不按 Client Key 绑定的分组分摊。 -账号属于多个组时，各组均包含该请求费用；之后调整账号分组不重写历史归属 +分组费用按当前组成员实际完成请求的费用汇总，不按 Client Key 绑定的分组分摊。 +账号属于多个组时，各组均包含该账号的请求费用；调整账号分组后，留存窗口内费用随当前成员关系重新汇总 ## 7. Client Key @@ -1830,8 +1830,11 @@ Dashboard 的 `accountUsage[]` 由后端提供 `usageWindow`、`metricLabel`、` Dashboard 默认 `period=today`，用量、诊断和错误查询默认 `period=7d`，范围参数遵循[页面时间合同](#页面时间合同)。 Dashboard 返回原始 `asOf` 及 `asOfDisplay`，作为当前数据的查询锚点 -用量查询可组合页码/游标、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、 -request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可取 `model`、`account`、 +请求明细列表可组合页码、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、 +request/response/upstream ID、outcome 与搜索文本。`search` 仅供明细及错误列表使用； +汇总、洞察和诊断不接受搜索参数，管理观测接口不接受路由分组筛选。页面搜索不改变顶部统计范围 + +诊断 `dimension` 可取 `model`、`account`、 `apiKey`、`provider`、`transport`、`failureClass`、`status`。诊断按请求量降序返回最多 100 项， 同请求量按维度标识稳定排列 @@ -1863,13 +1866,13 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告（流内 缺少终态声明时保留首次声明。这些值仅表示上游报告，不作为模型真实性证明，也不参与路由、 聚合或本地计价模型选择。没有保存上游模型报告的记录保持未知 -请求记录列表的 `search` 使用字面量前缀匹配，支持请求 ID、Client Key ID / 名称、 -账号 ID、账号邮箱与名称、请求 / 上游模型 ID、上游请求 ID。密钥名称不区分大小写，其他字段区分大小写。 -密钥名称按当前密钥记录检索，改名后使用新名称，删除后仍可按 Client Key ID 查询历史记录。 -账号邮箱与名称按请求记录的历史快照检索， -不随当前账号修改或删除而改变；`%`、`_` 和 `\` 均按普通字符处理，不作为搜索通配符 +请求记录列表的 `search` 使用字面量前缀匹配，支持 Client Key ID / 名称、 +账号 ID、账号邮箱与名称、请求 / 上游模型 ID。密钥名称不区分大小写，其他字段区分大小写。 +密钥名称、账号邮箱与名称按请求记录的历史快照检索， +不随当前密钥或账号修改、删除而改变；`%`、`_` 和 `\` 均按普通字符处理，不作为搜索通配符 -请求记录按模型执行 ID 或上游 ID 查询；只有入口 ID 时需结合时间与入口日志定位。 +请求记录的 `requestId`、`upstreamRequestId` 参数按模型执行 ID 或上游 ID 精确查询； +搜索框不匹配请求 ID，只有入口 ID 时需结合时间与入口日志定位。 请求记录和错误列表支持按密钥名称搜索，不支持密钥值或可见前缀搜索 ### 记录范围与统计口径 diff --git a/docs/architecture.md b/docs/architecture.md index e37d67328..bb7fae2cf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -865,14 +865,21 @@ OpenAI 订阅周期属于按需个人信息，不是额度事实。Admin 账号 ## 10. 观测与后台任务 -管理观测查询与统计 Worker 共用 PostgreSQL 连接预算，为请求执行保留连接容量。每项操作使用一个受预算约束的连接； +管理观测查询与统计 Worker 共用 PostgreSQL 连接预算，为请求执行保留连接容量。每项操作使用一个受预算约束的连接，排队计入总操作预算； +Store 声明配置后的读取时限，缓存沿用该值，统计事务只在事务内设置 SQL 时限，归还连接时不污染其他查询； 超时或调用方取消后，名额持续覆盖 SQLx 的协议收尾、事务回滚和连接归还，不能随查询 Future 提前释放。 -请求列表的精确总数与候选页共用数据库快照，按距首尾的较短方向读取，返回顺序和页码语义保持一致。 +请求列表的精确总数与候选页共用数据库快照，深页先按时间分段定位，再按距区间首尾的较短方向读取， +返回顺序和页码语义保持一致。 + +Store 从请求源事实维护窄事实及十五分钟、小时统计状态，账号用量与分组费用复用同一套已交付用量指标。 +完整时间桶在数据库内合并，边界与不能由汇总表达的筛选在数据库内精确聚合；向应用返回的状态按输出分组组织。 +计数、金额与分布状态的旧贡献撤销、新贡献合并和事实替换共用事务，构建期间只调整已扫描前缀， +不可相减的极值按受影响分组重算。同步账号用量查询在同一快照补齐待投影源版本，避免异步投影改变原有读数合同。 ### 额度预测 账号容量预测属于 Admin 的只读派生规则，不参与 quota 权威状态、调度或金额结算。Store 通过专用采样端口 -在同一 SQL 快照内返回截至观测时间的累计数值及有界历史 Provider 文档；对应 Provider 复用协议解析器解释 +在同一读事务快照内返回截至观测时间的累计数值及有界历史 Provider 文档；对应 Provider 复用协议解析器解释 文档，不在历史查询中刷新上游额度。观测通过 Core 原有 请求结算持久化，账号关联由实际执行上下文确定，窗口身份与归属必须匹配当前额度窗口。Admin 按中立的额度 事实选择近期进度段预测剩余量，再加本周期已记录用量形成周期总量。 @@ -890,6 +897,11 @@ OpenAI 订阅周期属于按需个人信息，不是额度事实。Admin 账号 - 重置后从新段重新累计，无法精确归属的用量不补算；只有账号在窗口开始前加入且无已知断点时，才允许以窗口累计量替代增量采样 - 历史点抽样不缩减区间累计用量，同一完成时间共用同一累计值 +预测数值复用统计投影的十五分钟开始时间桶，并以账号与五分钟完成时间桶分组。 +完整完成时间桶合并累计量，跨采样截止值的小桶、开始时间边界和重建区间读取窄事实； +同一快照撤销待投影旧版本并加入当前源版本。该维度保留恢复请求的原有预测口径， +不改变其他统计排除恢复请求的规则。Provider 文档仍按账号和完成时间索引读取，每个采样槽只保留最后一份 + ### 请求记录与交付 Provider 独占首事件、首字、首推理和首正文的观测，以及上游计时的协议解析，通过类型化响应观测交给 Core。 diff --git a/docs/plans/413-postgresql-capacity.md b/docs/plans/413-postgresql-capacity.md new file mode 100644 index 000000000..725bc1094 --- /dev/null +++ b/docs/plans/413-postgresql-capacity.md @@ -0,0 +1,81 @@ +# PostgreSQL 3100 万请求优化与验收 + +本报告对应 [Issue #413](https://github.com/zyycn/codex-proxy-rs/issues/413)，范围为大数据量下的统计、请求分页、账号用量、分组费用与额度预测 + +3100 万条源请求、窄事实和完整汇总已构建并独立对账。当前 HTTP 矩阵执行 167 次请求，包含 145 次合法业务请求、22 次预期参数或权限拒绝，未出现超时或非预期失败。按实际注册路由计，84 条已有通过记录、70 条未覆盖、6 条不适用；不宣称全接口或长期负载验收通过 + +## 问题与实现边界 + +原账号用量读取在命中事实超过 10 万条时拒绝查询；其他统计的事实回退与汇总读取也有固定行数门槛。仅放宽门槛会继续放大应用端逐行合并、深页 OFFSET 和预测窗口排序的成本 + +实现继续使用原生 PostgreSQL、现有窄事实、十五分钟／小时汇总和投影 Worker： + +- 账号与 Key 的必要模型联合维度进入共享汇总，完整桶和精确边界在数据库内聚合，不以命中数据量拒绝查询 +- 账号窗口、用量排序、分组费用和分页计数在同一读快照补齐待投影修改与删除，保留精确总数、金额及未知值语义 +- 请求深页通过时间分段定位，错误列表合并两类来源后保持稳定排序 +- 预测使用开始时间与完成时间二维汇总，累计量保留完整窗口，Provider 文档独立有界采样 +- 投影在同一事务撤销旧贡献并加入新贡献；普通更新维护构建前缀，不反复重启整个桶 +- 观测操作与前端默认保护时限为 120 秒，可分别配置；30 秒为优化方向，不能将提高超时时限当作查询提速 + +使用记录搜索支持密钥、账号与模型，不匹配请求 ID；请求 ID 的显式精确筛选仍保留。汇总、洞察和诊断不接受搜索参数，管理观测接口不接受路由分组筛选 + +当前合同与职责见 [API 参考](../api.md)、[系统架构](../architecture.md#10-观测与后台任务)与[部署配置](../../deploy/README.md)，本报告不重复字段清单 + +## 环境与样本 + +| 项目 | 实际条件 | +| --- | --- | +| 数据 | 31 天、3100 万条合成源请求；50 个账号、100 个 Key、20 个模型、10 个分组 | +| 时间范围 | `2026-09-12T04:45:00Z` 至 `2026-10-13T04:45:00Z` | +| 数据库 | PostgreSQL 18.6，容器限制 4 CPU、4 GiB 内存，持久磁盘 | +| 主要参数 | shared_buffers 512 MiB、work_mem 16 MiB、max_parallel_workers 8、max_parallel_workers_per_gather 2、JIT 开启 | +| 持久性 | fsync、full_page_writes、synchronous_commit 均开启 | +| 应用 | release 构建；核心轮次重启应用清空应用缓存，16 个查询客户端并发 | +| 其他轮次 | 剩余只读接口 4 并发；合成配置写入、鉴权及本机 Provider 场景串行执行 | +| 缓存边界 | 未清空数据库与宿主文件缓存，应用冷缓存不等于数据库冷缓存 | + +造数与离线汇总构建使用的临时资源配置已恢复，搜索索引试验已撤销。3100 万条容量基线不是生产 Worker 全量回填吞吐的测量结果；生产投影的正确性通过真实源表回归及样本对账验证 + +## HTTP 结果 + +以下为当前矩阵中的单次短样本，不能推断稳定的 P95／P99。并发样本包含资源竞争，不能与隔离单请求耗时直接当作前后优化对照 + +| 场景 | 耗时 | +| --- | ---: | +| 用量明细宽范围模型前缀搜索 | 32.63 秒 | +| 账号用量降序 | 20.50 秒 | +| 账号最近使用降序 | 20.39 秒 | +| 账号额度预测 | 19.94 秒 | +| Key 维度诊断 | 18.46 秒 | +| Dashboard 概览 | 12.30 秒 | +| 账号维度诊断 | 10.82 秒 | +| 用量洞察 | 6.20 秒 | +| 用量汇总 | 6.17 秒 | +| 用量中间页 | 3.60 秒 | +| 用量首／尾／超尾页 | 3.46／1.63／1.39 秒 | +| 错误列表第 6200 页 | 1.70 秒 | + +宽范围搜索命中大量明细，32.63 秒超过 30 秒优化期望，但未触发 120 秒保护时限。本次保留该性能局限，不再为此扩展索引或引入其他存储系统 + +正确性核对覆盖源请求总量、Token、费用、主要筛选组合、诊断分母及分页结果。移除的搜索与分组参数按预期返回 400，权限负例按预期返回拒绝；这些负例不计作合法容量查询通过 + +## 回归与自审范围 + +- 超过 10 万条真实源请求经过生产 Worker，验证账号／Key 汇总、热点边界和未知 Token；锁住窄事实表时，干净完整桶仍可读取 +- 待投影跨账号／模型／时间桶修改、删除、重复投递、写入失败回滚及构建前缀持续更新 +- 精确分页首中尾页、超尾页、跨时间段与同时间戳排序，两类错误来源合并 +- 预测微秒边界、同完成时间、截止时间、缺失费用、恢复请求、重建与待投影补差 +- 连接排队计入总预算，超时与取消后的事务回滚及连接归还仍持有查询名额 +- 分组费用沿用当前成员关系；账号窗口的 Token 兜底与普通统计的未知值口径分别保留 +- 投影与极值重算共用维度定义，已删除请求 ID 搜索捷径及淘汰的逐行读取映射 +- Rustfmt、相关 Store／Admin／API 回归、严格 Clippy、架构边界、迁移冻结清单、前端类型构建与 ESLint 对应实际最终工作区执行 + +页面仅调整搜索提示；按本次任务要求不提供截图 + +## 升级与验证局限 + +迁移 0030／0031 增加统计字段和索引，将既有时间桶排入可恢复重建，不清空窄事实或重置已完成的源投影游标。重建中的桶走事实回退，索引创建与桶重建仍需要额外 I/O、磁盘空间和时间；未完成生产环境在线升级的时延与负载验收 + +本次收尾聚焦 3100 万请求瓶颈，不再扩展外围插件、OAuth、备份和升级场景。数据库冷缓存、长期持续负载、生产 Worker 全量回填性能及未覆盖外部集成保留为验证局限；小规模生命周期回归不等于相同场景的 3100 万级持续写入验收 + +临时造数、HTTP 驱动、SQL 计划及逐请求记录保存在忽略跟踪的 `.runtime/capacity`，不进入工程提交。完整请求与路由结果为 `http-31m-latest.json`／`http-31m-latest.md`，原失败轮次分别保留，最终检查日志使用 `final-*` 前缀 diff --git a/frontend/env.d.ts b/frontend/env.d.ts index eec606cdc..e4e07997e 100644 --- a/frontend/env.d.ts +++ b/frontend/env.d.ts @@ -1,5 +1,9 @@ /// +interface ImportMetaEnv { + readonly VITE_API_TIMEOUT_MS?: string +} + declare module '@fontsource-variable/inter' declare module '@fontsource-variable/jetbrains-mono' diff --git a/frontend/src/api/constants.ts b/frontend/src/api/constants.ts index 8a2459a95..b0f5ea787 100644 --- a/frontend/src/api/constants.ts +++ b/frontend/src/api/constants.ts @@ -1,2 +1,6 @@ export const API_BASE_URL = import.meta.env.DEV ? '/dev' : '' -export const API_TIMEOUT_MS = 30_000 +const timeout = Number(import.meta.env.VITE_API_TIMEOUT_MS ?? 120_000) +if (!Number.isSafeInteger(timeout) || timeout <= 0 || timeout > 2_147_483_647) { + throw new Error('VITE_API_TIMEOUT_MS must be a positive 32-bit integer') +} +export const API_TIMEOUT_MS = timeout diff --git a/frontend/src/api/modules/usage.ts b/frontend/src/api/modules/usage.ts index f10ccc053..e1b71f503 100644 --- a/frontend/src/api/modules/usage.ts +++ b/frontend/src/api/modules/usage.ts @@ -490,7 +490,6 @@ interface UsageRangeQuery { provider?: string model?: string statusCode?: number - search?: string } interface PageQuery { @@ -498,7 +497,7 @@ interface PageQuery { pageSize: number } -type UsagePageQuery = UsageRangeQuery & PageQuery +type UsagePageQuery = UsageRangeQuery & PageQuery & { search?: string } type OpsErrorPageQuery = UsagePageQuery diff --git a/frontend/src/views/usage/components/UsageFilters.vue b/frontend/src/views/usage/components/UsageFilters.vue index fba2ed5d1..61c5903d6 100644 --- a/frontend/src/views/usage/components/UsageFilters.vue +++ b/frontend/src/views/usage/components/UsageFilters.vue @@ -18,7 +18,7 @@ const search = defineModel('search', { required: true })