perf(store): scale analytics and pagination for large request histories (#423)

Aggregate analytics in PostgreSQL, reconcile synchronous usage with pending source changes, and preserve exact pagination and forecast totals across large histories.

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-10-13 21:29:25 +08:00
committed by GitHub
co-authored by Zyy Codex
parent 5b80340135
commit 219888d5c2
64 changed files with 4740 additions and 1456 deletions
@@ -66,7 +66,7 @@ pub struct AccountGroupCapacity {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountGroupUsage {
pub today_usd: DecimalAmount,
/// 当前 usage retention 窗口内、按请求发生时 routing group 快照归属的累计成本
/// 当前 usage retention 窗口内、当前组成员实际完成请求的累计成本
pub retained_total_usd: DecimalAmount,
}
@@ -157,7 +157,6 @@ impl OtherRequestOutcome {
/// 用量记录过滤条件
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
pub struct UsageFilter {
pub routing_group_refs: Vec<String>,
pub client_api_key_ref: Option<String>,
pub request_id: Option<String>,
pub provider_account_ref: Option<String>,
@@ -211,7 +210,6 @@ impl DashboardQuery {
/// 运维错误过滤条件
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct OpsErrorFilter {
pub routing_group_refs: Vec<String>,
pub client_api_key_ref: Option<String>,
pub request_id: Option<String>,
pub provider_kind: Option<String>,
@@ -432,6 +432,9 @@ pub trait UsageBillingProjector: Send + Sync {
/// 用量、趋势、诊断与运维错误的只读能力
#[async_trait]
pub trait ObservabilityStore: Send + Sync {
/// 读取实现声明的总预算,缓存等待沿用同一配置,避免外层提前截断
fn query_timeout(&self) -> std::time::Duration;
/// 缓存必须与删除和后台投影提交的版本一致
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64>;
@@ -12,7 +12,6 @@ use tokio::{sync::Mutex, time::Instant};
const CAPACITY: usize = 64;
const TTL: Duration = Duration::from_secs(15);
const MAX_STALE: Duration = Duration::from_secs(60);
const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
type Slot<T> = Arc<Mutex<Option<Cached<T>>>>;
struct Cached<T> {
@@ -72,7 +71,7 @@ impl<K: Eq + Hash, T: AnalyticsValue> Cache<K, T> {
{
// 刷新失败后的同键等待也计入预算,避免串行重试把延迟放大为队列长度倍数
tokio::time::timeout(
REQUEST_TIMEOUT,
store.query_timeout(),
self.get_within_budget(key, store, ttl, load),
)
.await
@@ -174,12 +173,6 @@ fn with_age<T: AnalyticsValue>(mut value: T, stale: bool) -> T {
value
}
fn normalized(mut filter: UsageFilter) -> UsageFilter {
filter.routing_group_refs.sort_unstable();
filter.routing_group_refs.dedup();
filter
}
type UsageKey = (TimeRange, UsageFilter);
pub(crate) struct CachedObservabilityStore {
inner: Arc<dyn ObservabilityStore>,
@@ -202,6 +195,10 @@ impl CachedObservabilityStore {
#[async_trait]
impl ObservabilityStore for CachedObservabilityStore {
fn query_timeout(&self) -> Duration {
self.inner.query_timeout()
}
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
self.inner.analytics_cache_generation().await
}
@@ -241,7 +238,6 @@ impl ObservabilityStore for CachedObservabilityStore {
filter: UsageFilter,
granularity: Granularity,
) -> AdminStoreResult<UsageInsightsObservation> {
let filter = normalized(filter);
self.insights
.get(
((range, filter.clone()), granularity),
@@ -259,7 +255,6 @@ impl ObservabilityStore for CachedObservabilityStore {
range: TimeRange,
filter: UsageFilter,
) -> AdminStoreResult<UsageOverview> {
let filter = normalized(filter);
self.summaries
.get((range, filter.clone()), self.inner.as_ref(), TTL, || {
self.inner.usage_summary(range, filter)
@@ -273,7 +268,6 @@ impl ObservabilityStore for CachedObservabilityStore {
dimension: DiagnosticDimension,
limit: u16,
) -> AdminStoreResult<DiagnosticsObservation> {
let filter = normalized(filter);
self.diagnostics
.get(
((range, filter.clone()), dimension, limit),
@@ -747,6 +747,10 @@ impl ClientKeyStore for UnavailableStore {
#[async_trait]
impl ObservabilityStore for UnavailableStore {
fn query_timeout(&self) -> std::time::Duration {
std::time::Duration::from_secs(10)
}
async fn dashboard_summary(
&self,
_: DashboardQuery,
@@ -945,6 +945,10 @@ impl FixtureObservabilityStore {
#[async_trait]
impl ObservabilityStore for FixtureObservabilityStore {
fn query_timeout(&self) -> std::time::Duration {
std::time::Duration::from_secs(10)
}
async fn dashboard_summary(
&self,
query: DashboardQuery,
@@ -1406,23 +1410,23 @@ async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the
}
#[tokio::test]
async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_sets() {
async fn usage_summary_cache_should_isolate_key_identity_and_filters() {
let range = observation_range(Utc::now());
let store = Arc::new(FixtureObservabilityStore::new(range));
let services = observability_services(store.clone()).await;
let service = services.observability();
for (key, groups) in [
("key-a", vec!["deleted-b", "deleted-a", "deleted-b"]),
("key-a", vec!["deleted-a", "deleted-b"]),
("key-b", vec!["deleted-a", "deleted-b"]),
("key-a", vec!["deleted-a"]),
for (key, model) in [
("key-a", None),
("key-a", None),
("key-b", None),
("key-a", Some("model-a")),
] {
service
.usage_summary(
range,
UsageFilter {
client_api_key_ref: Some(key.to_owned()),
routing_group_refs: groups.into_iter().map(str::to_owned).collect(),
model: model.map(str::to_owned),
..Default::default()
},
)
@@ -1431,16 +1435,16 @@ async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_set
}
let calls = store.usage_summary_filters.lock().unwrap();
assert_eq!(calls.len(), 3);
assert_eq!(calls[0].routing_group_refs, ["deleted-a", "deleted-b"]);
assert_eq!(calls[0].client_api_key_ref.as_deref(), Some("key-a"));
assert_eq!(calls[1].client_api_key_ref.as_deref(), Some("key-b"));
assert_eq!(calls[2].routing_group_refs, ["deleted-a"]);
assert_eq!(calls[2].model.as_deref(), Some("model-a"));
}
#[tokio::test(start_paused = true)]
async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow_recovery() {
let range = observation_range(Utc::now());
let store = Arc::new(FixtureObservabilityStore::new(range));
store.set_dashboard_delay(StdDuration::from_secs(7));
store.set_dashboard_delay(store.query_timeout().mul_f64(0.7));
store.summary_fail.store(true, Ordering::SeqCst);
let services = observability_services(store.clone()).await;
let service = services.observability();
@@ -1453,7 +1457,7 @@ async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow
);
assert!(first.is_err() && second.is_err() && third.is_err() && fourth.is_err());
assert!(
started.elapsed() <= StdDuration::from_secs(10),
started.elapsed() <= store.query_timeout(),
"同键失败不能使后续请求逐个等待完整数据库超时: {:?}",
started.elapsed()
);
@@ -29,7 +29,6 @@ impl DashboardQuery {
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct UsageQuery {
pub routing_group_refs: Option<String>,
pub current_page: Option<u32>,
pub page_size: Option<u16>,
pub kind: Option<String>,
@@ -77,7 +76,6 @@ impl DetailQuery {
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct DiagnosticsQuery {
pub routing_group_refs: Option<String>,
pub dimension: Option<String>,
pub period: Option<String>,
pub as_of: Option<i64>,
@@ -86,7 +84,6 @@ pub struct DiagnosticsQuery {
pub provider: Option<String>,
pub model: Option<String>,
pub status_code: Option<i64>,
pub search: Option<String>,
}
impl DiagnosticsQuery {
@@ -100,7 +97,6 @@ impl DiagnosticsQuery {
#[derive(Clone, Default, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct OpsQuery {
pub routing_group_refs: Option<String>,
pub current_page: Option<u32>,
pub page_size: Option<u16>,
pub kind: Option<String>,
@@ -301,7 +297,6 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, Wi
})
});
Ok(domain::UsageFilter {
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?,
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
request_id: non_empty(query.request_id.clone()),
provider_account_ref: non_empty(query.account_id.clone()),
@@ -318,6 +313,15 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, Wi
})
}
pub(crate) fn analytics_filter(
query: &UsageQuery,
) -> Result<domain::UsageFilter, WireValidationError> {
if query.search.is_some() {
return Err(WireValidationError::new("search"));
}
usage_filter(query)
}
pub(crate) fn usage_command(
query: &UsageQuery,
timezone: gateway_core::time::DeploymentTimeZone,
@@ -361,7 +365,6 @@ pub(crate) fn ops_command(
timezone,
)?,
filter: domain::OpsErrorFilter {
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?,
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
request_id: non_empty(query.request_id.clone()),
provider_kind: non_empty(query.provider.clone()),
@@ -400,25 +403,3 @@ fn validate_pagination(
}
Ok((current_page, page_size))
}
/// 逗号分隔的稳定分组身份按集合归一化,不要求分组当前仍存在
pub(crate) fn routing_group_refs(value: Option<&str>) -> Result<Vec<String>, WireValidationError> {
let Some(value) = value else {
return Ok(Vec::new());
};
let mut refs = Vec::new();
for item in value.split(',') {
let item = item.trim();
if item.is_empty()
|| item.len() > 128
|| item.chars().any(char::is_control)
|| refs.len() >= 32
{
return Err(WireValidationError::new("routingGroupRefs"));
}
refs.push(item.to_owned());
}
refs.sort();
refs.dedup();
Ok(refs)
}
@@ -151,7 +151,7 @@ where
state.admin_services().timezone(),
)
.map_err(map_wire_error)?;
let filter = usage_filter(&query).map_err(map_wire_error)?;
let filter = analytics_filter(&query).map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
@@ -184,7 +184,7 @@ where
state.admin_services().timezone(),
)
.map_err(map_wire_error)?;
let filter = usage_filter(&query).map_err(map_wire_error)?;
let filter = analytics_filter(&query).map_err(map_wire_error)?;
let result = state
.admin_services()
.observability()
@@ -215,12 +215,9 @@ where
)
.map_err(map_wire_error)?;
let filter = domain::UsageFilter {
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())
.map_err(map_wire_error)?,
provider_kind: non_empty(query.provider),
model: non_empty(query.model),
status_code: parse_status(query.status_code).map_err(map_wire_error)?,
search: non_empty(query.search),
..domain::UsageFilter::default()
};
let result = state
@@ -1374,6 +1374,10 @@ impl AccountRuntimeStore for UnusedStore {
#[async_trait]
impl ObservabilityStore for UnusedStore {
fn query_timeout(&self) -> std::time::Duration {
std::time::Duration::from_secs(10)
}
async fn dashboard_summary(
&self,
query: DashboardQuery,
@@ -1245,7 +1245,7 @@ async fn calendar_query_rejects_an_anchor_outside_the_local_calendar() {
}
#[tokio::test]
async fn request_and_ops_queries_should_forward_deleted_group_id_sets() {
async fn observability_queries_reject_group_filters_and_analytics_search() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::Body,
@@ -1256,39 +1256,50 @@ async fn request_and_ops_queries_should_forward_deleted_group_id_sets() {
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let router = observability::router::<AdminTestState>().with_state(fixture.state());
for path in ["/api/admin/usage/records", "/api/admin/operations/errors"] {
for (path, parameter) in [
("/api/admin/usage/records", "routingGroupRefs=group-a"),
("/api/admin/operations/errors", "routingGroupRefs=group-a"),
(
"/api/admin/usage/records/summary",
"routingGroupRefs=group-a",
),
(
"/api/admin/usage/insights/overview",
"routingGroupRefs=group-a",
),
(
"/api/admin/usage/insights/diagnostics",
"routingGroupRefs=group-a",
),
("/api/admin/usage/records/summary", "search=request-prefix"),
(
"/api/admin/usage/insights/overview",
"search=request-prefix",
),
(
"/api/admin/usage/insights/diagnostics",
"search=request-prefix",
),
] {
let response = router
.clone()
.oneshot(
Request::builder()
.uri(format!(
"{path}?routingGroupRefs=deleted-b,deleted-a,deleted-b"
))
.uri(format!("{path}?{parameter}"))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_group_filter")
.header("x-request-id", "req_rejected_observability_filter")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
.await
.unwrap();
assert_eq!(
status,
StatusCode::OK,
"{path}: {}",
String::from_utf8_lossy(&body)
response.status(),
StatusCode::BAD_REQUEST,
"{path}?{parameter}"
);
}
let observations = fixture.observations.lock().unwrap();
assert_eq!(
observations.records[0].filter.routing_group_refs,
["deleted-a", "deleted-b"]
);
assert_eq!(
observations.errors[0].filter.routing_group_refs,
["deleted-a", "deleted-b"]
);
assert!(observations.records.is_empty());
assert!(observations.errors.is_empty());
}
+6 -2
View File
@@ -126,6 +126,7 @@ async fn connect(
let observability_query_budget = postgres::ObservabilityQueryBudget::try_new(
config.pool.observability_max_connections(),
config.pool.acquire_timeout(),
config.pool.observability_timeout(),
)?;
let redis_client = ::redis::Client::open(config.redis_url()?)
.map_err(|source| redis_unavailable("create Redis client", source))?;
@@ -180,8 +181,11 @@ async fn connect(
)),
admin_account_runtime,
Arc::new(
postgres::PgAccountGroupRepository::new(pool.clone())
.with_timezone(config.timezone),
postgres::PgAccountGroupRepository::new(
pool.clone(),
observability_query_budget.clone(),
)
.with_timezone(config.timezone),
),
Arc::new(postgres::PgProxyRepository::new(pool.clone())),
),
@@ -36,6 +36,7 @@ pub struct StoreConfig {
pub struct StorePoolConfig {
pub max_connections: u32,
pub acquire_timeout_seconds: u64,
pub observability_timeout_seconds: u64,
}
impl Default for StorePoolConfig {
@@ -43,6 +44,7 @@ impl Default for StorePoolConfig {
Self {
max_connections: 20,
acquire_timeout_seconds: 5,
observability_timeout_seconds: 120,
}
}
}
@@ -58,9 +60,22 @@ impl StorePoolConfig {
.to_owned(),
});
}
if self.observability_timeout_seconds == 0
|| self.observability_timeout().as_millis() > i32::MAX as u128
{
return Err(StoreError::InvalidData {
source: None,
entity: "store config",
message: "pool.observability_timeout_seconds must be positive and fit PostgreSQL statement_timeout".to_owned(),
});
}
Ok(())
}
pub(crate) const fn observability_timeout(self) -> Duration {
Duration::from_secs(self.observability_timeout_seconds)
}
/// 管理观测查询可并发占用的连接数;始终为数据面保留约 20% 的池容量
#[must_use]
pub const fn observability_max_connections(self) -> u32 {
@@ -31,9 +31,9 @@ use crate::{
};
use super::{
PgRuntimeSettingsRepository, RuntimeSettingsRepository, account_summary_from_row,
append_admin_audit_event_in_transaction, bump_config_revision_in_transaction,
completed_usage_fact_predicate,
ObservabilityQueryBudget, PgRuntimeSettingsRepository, RuntimeSettingsRepository,
account_summary_from_row, append_admin_audit_event_in_transaction,
bump_config_revision_in_transaction,
};
const ENTITY: &str = "account group";
@@ -43,13 +43,15 @@ const ENTITY: &str = "account group";
pub struct PgAccountGroupRepository {
timezone: gateway_core::time::DeploymentTimeZone,
pool: PgPool,
query_budget: ObservabilityQueryBudget,
}
impl PgAccountGroupRepository {
#[must_use]
pub fn new(pool: PgPool) -> Self {
pub fn new(pool: PgPool, query_budget: ObservabilityQueryBudget) -> Self {
Self {
pool,
query_budget,
timezone: Default::default(),
}
}
@@ -74,9 +76,14 @@ impl PgAccountGroupRepository {
.await
.map_err(|error| admin_store_error(ENTITY, error))?
.ok_or_else(|| not_found(id.as_str()))?;
let costs = group_costs(&self.pool, &[id.as_str().to_owned()], self.timezone)
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
let costs = group_costs(
&self.pool,
&self.query_budget,
&[id.as_str().to_owned()],
self.timezone,
)
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
if let Some(usage) = costs.get(id.as_str()) {
record.usage = usage.clone();
}
@@ -152,7 +159,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
.iter()
.map(|record| record.id.as_str().to_owned())
.collect::<Vec<_>>();
let costs = group_costs(&self.pool, &group_ids, self.timezone)
let costs = group_costs(&self.pool, &self.query_budget, &group_ids, self.timezone)
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
for record in &mut items {
@@ -601,71 +608,23 @@ fn group_ref(row: &sqlx::postgres::PgRow) -> StoreResult<AccountGroupRef> {
async fn group_costs(
pool: &PgPool,
budget: &ObservabilityQueryBudget,
group_ids: &[String],
timezone: gateway_core::time::DeploymentTimeZone,
) -> StoreResult<BTreeMap<String, AccountGroupUsage>> {
if group_ids.is_empty() {
return Ok(BTreeMap::new());
}
let completed_usage = completed_usage_fact_predicate("mr");
// 用量按实际完成请求的账号统计:账号属于多个分组时计入每个所属分组,
// 不再按 Client Key 绑定分组快照归属,避免多分组 Key 的费用重复出现在未承接请求的分组上
let statement = format!(
"with requested_groups(group_id) as (
select unnest($1::text[])
)
select requested_groups.group_id,
coalesce(sum(mr.cost_amount) filter (
where mr.started_at >= $2
), 0)::text as today_usd,
coalesce(sum(mr.cost_amount), 0)::text as retained_total_usd
from requested_groups
cross join runtime_settings settings
left join account_group_accounts membership
on membership.account_group_id = requested_groups.group_id
left join model_requests mr
on mr.provider_account_ref = membership.provider_account_id
and mr.started_at >= now() - make_interval(days => settings.usage_retention_days::int)
and {completed_usage}
and mr.cost_currency = 'USD'
and mr.cost_amount is not null
where settings.id = 1
group by requested_groups.group_id"
);
// 动态片段仅为共享的固定 usage-fact predicate;group IDs 仍使用 bind
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(group_ids)
.bind(
timezone
.day_start(chrono::Utc::now())
.ok_or_else(|| invalid("invalid business date"))?,
budget
.run(
pool,
"load account group costs",
|mut connection| async move {
super::observability::account_group_usage(&mut connection, group_ids, timezone)
.await
},
)
.fetch_all(pool)
.await
.map_err(|source| unavailable("load account group costs", source))?;
rows.into_iter()
.map(|row| {
let group_id: String = row
.try_get("group_id")
.map_err(|source| invalid("invalid group ID").with_source(source))?;
let today = row
.try_get::<String, _>("today_usd")
.map_err(|source| invalid("invalid today cost").with_source(source))?;
let retained_total = row
.try_get::<String, _>("retained_total_usd")
.map_err(|source| invalid("invalid retained total cost").with_source(source))?;
Ok((
group_id,
AccountGroupUsage {
today_usd: DecimalAmount::from_str(&today)
.map_err(|source| invalid("invalid today cost").with_source(source))?,
retained_total_usd: DecimalAmount::from_str(&retained_total).map_err(
|source| invalid("invalid retained total cost").with_source(source),
)?,
},
))
})
.collect()
}
fn count_value(row: &sqlx::postgres::PgRow, field: &str) -> StoreResult<u64> {
@@ -197,7 +197,7 @@ impl PgObservabilityRepository {
)
.await?;
let account_usage =
provider_account_usage(&mut snapshot.tx, account_usage_query).await?;
provider_account_usage(&mut snapshot, account_usage_query).await?;
let metadata = snapshot.metadata();
snapshot.finish().await?;
Ok((totals, trend, metadata, account_usage))
@@ -290,12 +290,8 @@ impl PgObservabilityRepository {
"load usage insights snapshot",
|mut connection| async move {
let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?;
let mut overview = snapshot.overview(range, &filter).await?;
let trend = snapshot
.trend(range, &filter, granularity, self.timezone, true)
.await?;
let costs = snapshot
.projected_costs(range, &filter, granularity, self.timezone)
let (mut overview, trend, costs) = snapshot
.insights(range, &filter, granularity, self.timezone)
.await?;
overview.analytics = snapshot.metadata();
snapshot.finish().await?;
@@ -318,12 +314,9 @@ impl PgObservabilityRepository {
&self.pool,
"load provider account usage",
|mut connection| async move {
let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?;
snapshot.require_visible(Some(query.range)).await?;
if let Some(range) = query.request_bucket_range {
snapshot.require_visible(Some(range)).await?;
}
let usage = provider_account_usage(&mut snapshot.tx, query).await?;
let mut snapshot =
analytics::AnalyticsSnapshot::new_for_accounts(&mut connection).await?;
let usage = provider_account_usage(&mut snapshot, query).await?;
snapshot.finish().await?;
Ok(usage)
},
@@ -407,6 +400,10 @@ impl PgObservabilityRepository {
#[async_trait]
impl AdminObservabilityStore for PgAdminObservabilityStore {
fn query_timeout(&self) -> std::time::Duration {
self.repository.query_budget.query_timeout()
}
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
self.repository
.query_budget
@@ -0,0 +1,655 @@
//! 账号用量复用汇总并补齐未投影源版本,保持同步查询的精确窗口口径
use super::query::push_summary_buckets;
use super::*;
use gateway_admin::model::{
account_groups::AccountGroupUsage,
accounts::{
AccountCost, AccountModelUsage, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult,
},
};
type AccountStates = BTreeMap<(String, String), MetricState>;
impl AnalyticsSnapshot<'_> {
pub(crate) async fn account_usage_states(
&mut self,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<AccountStates> {
self.account_states(range, account_ids, true).await
}
async fn account_states(
&mut self,
range: ObservabilityRange,
account_ids: &[String],
by_model: bool,
) -> StoreResult<AccountStates> {
if account_ids.is_empty() {
return Ok(BTreeMap::new());
}
// 原同步接口在首次回填期间仍可查询;只有完整覆盖的区间才允许用汇总替代源事实
if !self.projection_covers(range) {
return self
.account_fact_states("model_requests", range, account_ids, false, by_model)
.await;
}
let mut states = BTreeMap::new();
// 排序与分组费用只需要账号总量,不额外展开模型维度
let (collection, model) = if by_model {
(
"account_model",
"coalesce(nullif(dimensions[4],''),dimensions[3])",
)
} else {
("account", "''::text")
};
let (hours, quarters) = self.summary_buckets(range, true).await?;
for (table, seconds, buckets) in [
("request_analytics_hourly", 3600, hours),
("request_analytics_15m", 900, quarters),
] {
if buckets.is_empty() {
continue;
}
let mut query = QueryBuilder::<Postgres>::new(format!(
"with selected as materialized (select dimensions[1] as account_id,{model} as model,counts,costs,last_used_at from {table} a where collection='{collection}' and schema_version="
));
query
.push_bind(SCHEMA_VERSION)
.push(" and histogram_version=")
.push_bind(HISTOGRAM_VERSION)
.push(" and dimensions[1]=any(")
.push_bind(account_ids)
.push("::text[])");
push_summary_buckets(&mut query, &buckets, seconds);
let counters = usage_counter_names();
// 每行一次展开指标,避免对可能位于 TOAST 的同一 JSON 反复解码
query.push(format!(
"), totals as (select account_id,model,jsonb_build_object({}) as counts,max(last_used_at) as last_used_at from selected cross join lateral jsonb_to_record(counts) as metric({}) group by account_id,model),
currencies as (select account_id,model,c.key,sum(c.value::numeric)::text as amount from selected cross join lateral jsonb_each_text(costs) c where c.key not in ('standard_usd','no_cache_usd') group by account_id,model,c.key),
costs as (select account_id,model,jsonb_object_agg(key,amount) as costs from currencies group by account_id,model)
select totals.*,coalesce(costs.costs,'{{}}'::jsonb) as costs,'{{}}'::jsonb as distributions from totals left join costs using(account_id,model)",
counters.iter().map(|name| format!("'{name}',coalesce(sum(metric.{name}),0)")).collect::<Vec<_>>().join(","),
counters.iter().map(|name| format!("{name} numeric")).collect::<Vec<_>>().join(",")
));
let rows = query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| {
postgres_unavailable("aggregate account usage summaries", error)
})?;
merge_account_rows(&mut states, rows)?;
}
for (start, end) in self.uncovered_spans(range).await? {
let boundary = self
.account_fact_states(
"request_analytics_facts",
ObservabilityRange { start, end },
account_ids,
false,
by_model,
)
.await?;
merge_account_states(&mut states, boundary)?;
}
if !self.has_pending_requests().await? {
states.retain(|_, state| state.count("completed_request_count") > 0);
return Ok(states);
}
let old = self
.account_fact_states(
"request_analytics_facts",
range,
account_ids,
true,
by_model,
)
.await?;
let current = self
.account_fact_states("model_requests", range, account_ids, true, by_model)
.await?;
let mut repair = BTreeSet::new();
for (key, previous) in old {
let state = states.entry(key.clone()).or_default();
for (name, count) in &previous.counts {
let value = state.counts.entry(name.clone()).or_default();
*value = value
.checked_sub(*count)
.ok_or_else(|| invalid("account usage pending counter underflow"))?;
}
for (currency, amount) in &previous.costs {
let before = state
.costs
.get(currency)
.ok_or_else(|| invalid("account usage pending cost is missing"))?;
let before = before
.parse::<gateway_core::metering::Decimal>()
.map_err(|_| invalid("invalid account cost"))?;
let removed = amount
.parse::<gateway_core::metering::Decimal>()
.map_err(|_| invalid("invalid pending account cost"))?;
let difference = before
.scaled()
.checked_sub(removed.scaled())
.ok_or_else(|| invalid("account usage pending cost underflow"))?;
let value = gateway_core::metering::Decimal::from_scaled(difference)
.map_err(|_| invalid("invalid account cost difference"))?;
state.costs.insert(currency.clone(), value.canonical());
}
repair.insert(key);
}
merge_account_states(&mut states, current)?;
// 撤销最大值不能用减法;只对发生源版本替换的账号模型做索引定位
let completed = completed_usage_fact_predicate("mr");
for (account, model) in repair {
let state = states
.get_mut(&(account.clone(), model.clone()))
.ok_or_else(|| invalid("pending account state disappeared"))?;
if state.count("completed_request_count") == 0 {
*state = MetricState::default();
continue;
}
let statement = format!(
"select started_at from model_requests mr where provider_account_ref=$1 and ($2::text is null or coalesce(upstream_model_id,requested_model_id,'')=$2) and started_at >= $3 and started_at < $4 and {completed} order by started_at desc,id desc limit 1"
);
state.last_used_at = sqlx::query_scalar(sqlx::AssertSqlSafe(statement))
.bind(&account)
.bind(by_model.then_some(&model))
.bind(range.start)
.bind(range.end)
.fetch_optional(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("repair account latest usage", error))?;
let zero_currencies = state
.costs
.iter()
.filter(|(_, amount)| amount.as_str() == "0")
.map(|(currency, _)| currency.clone())
.collect::<Vec<_>>();
for currency in zero_currencies {
let statement = format!(
"select exists(select 1 from model_requests mr where provider_account_ref=$1 and ($2::text is null or coalesce(upstream_model_id,requested_model_id,'')=$2) and started_at >= $3 and started_at < $4 and {completed} and cost_currency=$5 and cost_amount is not null)"
);
let exists: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(statement))
.bind(&account)
.bind(by_model.then_some(&model))
.bind(range.start)
.bind(range.end)
.bind(&currency)
.fetch_one(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("repair account cost coverage", error))?;
if !exists {
state.costs.remove(&currency);
}
}
}
states.retain(|_, state| state.count("completed_request_count") > 0);
Ok(states)
}
pub(super) async fn has_pending_requests(&mut self) -> StoreResult<bool> {
if self.status.oldest_pending_at.is_none() {
return Ok(false);
}
sqlx::query_scalar("select exists(select 1 from request_analytics_pending_requests)")
.fetch_one(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("check pending account usage", error))
}
pub(super) fn projection_covers(&self, range: ObservabilityRange) -> bool {
self.status.schema_version == SCHEMA_VERSION as u32
&& self.status.histogram_version == HISTOGRAM_VERSION as u32
&& (!self.status.building
|| (self
.status
.coverage_start
.is_some_and(|start| range.start >= start)
&& self.status.coverage_end.is_some()))
}
pub(crate) async fn account_request_counts(
&mut self,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<BTreeMap<(String, i64), u64>> {
if account_ids.is_empty() {
return Ok(BTreeMap::new());
}
let mut counts = BTreeMap::new();
// 公开小时图采用 UTC 整点;非整刻内部窗口直接在源表按相对小时分组
if !self.projection_covers(range)
|| range.start.timestamp().rem_euclid(900) != 0
|| range.start.timestamp_subsec_nanos() != 0
{
let rows = self
.account_request_fact_counts(
"model_requests",
range,
range.start,
account_ids,
false,
)
.await?;
merge_request_counts(&mut counts, rows, false)?;
return Ok(counts);
}
let (hours, quarters) = self
.summary_buckets(range, range.start.timestamp().rem_euclid(3600) == 0)
.await?;
for (table, seconds, buckets) in [
("request_analytics_hourly", 3600, hours),
("request_analytics_15m", 900, quarters),
] {
if buckets.is_empty() {
continue;
}
let mut query = QueryBuilder::<Postgres>::new(
"select dimensions[1] as account_id,floor(extract(epoch from (bucket_start-",
);
query.push_bind(range.start).push("::timestamptz))/3600)::bigint as bucket_index,coalesce(sum((counts->>'completed_request_count')::bigint),0)::bigint as request_count from ")
.push(table).push(" a where collection='account' and schema_version=").push_bind(SCHEMA_VERSION)
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION).push(" and dimensions[1]=any(").push_bind(account_ids).push("::text[])");
push_summary_buckets(&mut query, &buckets, seconds);
query.push(" group by dimensions[1],bucket_index");
let rows = query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| {
postgres_unavailable("read account hourly request summaries", error)
})?;
merge_request_counts(&mut counts, rows, false)?;
}
for (start, end) in self.uncovered_spans(range).await? {
let rows = self
.account_request_fact_counts(
"request_analytics_facts",
ObservabilityRange { start, end },
range.start,
account_ids,
false,
)
.await?;
merge_request_counts(&mut counts, rows, false)?;
}
if self.has_pending_requests().await? {
let old = self
.account_request_fact_counts(
"request_analytics_facts",
range,
range.start,
account_ids,
true,
)
.await?;
merge_request_counts(&mut counts, old, true)?;
let current = self
.account_request_fact_counts(
"model_requests",
range,
range.start,
account_ids,
true,
)
.await?;
merge_request_counts(&mut counts, current, false)?;
}
Ok(counts)
}
async fn account_request_fact_counts(
&mut self,
table: &'static str,
range: ObservabilityRange,
origin: DateTime<Utc>,
account_ids: &[String],
pending_only: bool,
) -> StoreResult<Vec<sqlx::postgres::PgRow>> {
let mut query = QueryBuilder::<Postgres>::new(
"select mr.provider_account_ref as account_id,floor(extract(epoch from (mr.started_at-",
);
query.push_bind(origin).push("::timestamptz))/3600)::bigint as bucket_index,count(*)::bigint as request_count from ").push(table).push(" mr");
if pending_only {
query.push(" join request_analytics_pending_requests p on p.request_id=mr.id");
}
query
.push(" where mr.provider_account_ref=any(")
.push_bind(account_ids)
.push("::text[]) and mr.started_at >= ")
.push_bind(range.start)
.push(" and mr.started_at < ")
.push_bind(range.end);
push_completed_usage_fact_filter(&mut query, "mr");
query.push(" group by mr.provider_account_ref,bucket_index");
query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| {
postgres_unavailable("aggregate account hourly request boundaries", error)
})
}
async fn account_fact_states(
&mut self,
table: &'static str,
range: ObservabilityRange,
account_ids: &[String],
pending_only: bool,
by_model: bool,
) -> StoreResult<AccountStates> {
let mut query = QueryBuilder::<Postgres>::new(usage_fact_aggregate_sql(table, by_model));
if pending_only {
query.push(
" join request_analytics_pending_requests pending on pending.request_id=mr.id",
);
}
query
.push(" where mr.provider_account_ref=any(")
.push_bind(account_ids)
.push("::text[]) and mr.started_at >= ")
.push_bind(range.start)
.push(" and mr.started_at < ")
.push_bind(range.end);
push_completed_usage_fact_filter(&mut query, "mr");
query.push(" group by mr.provider_account_ref,mr.cost_currency");
if by_model {
query.push(",coalesce(mr.upstream_model_id,mr.requested_model_id,'')");
}
let rows = query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("aggregate account usage facts", error))?;
let mut states = BTreeMap::new();
merge_account_rows(&mut states, rows)?;
Ok(states)
}
}
fn merge_request_counts(
counts: &mut BTreeMap<(String, i64), u64>,
rows: Vec<sqlx::postgres::PgRow>,
subtract: bool,
) -> StoreResult<()> {
for row in rows {
let value = unsigned(&row, "request_count")?;
let total = counts
.entry((get(&row, "account_id")?, get(&row, "bucket_index")?))
.or_default();
*total = if subtract {
total.checked_sub(value)
} else {
total.checked_add(value)
}
.ok_or_else(|| invalid("invalid account request count delta"))?;
}
Ok(())
}
fn merge_account_rows(
states: &mut AccountStates,
rows: Vec<sqlx::postgres::PgRow>,
) -> StoreResult<()> {
for row in rows {
states
.entry((get(&row, "account_id")?, get(&row, "model")?))
.or_default()
.merge(&MetricState::decode(&row)?)?;
}
Ok(())
}
fn merge_account_states(states: &mut AccountStates, incoming: AccountStates) -> StoreResult<()> {
for (key, state) in incoming {
states.entry(key).or_default().merge(&state)?;
}
Ok(())
}
fn usage_counter_names() -> Vec<String> {
let mut names = [
"completed_request_count",
"image_request_count",
"image_request_failed_count",
"window_total_tokens",
"provider_reported_count",
"calculated_count",
"unavailable_count",
]
.map(str::to_owned)
.to_vec();
for name in super::metrics::TOKEN_METRICS {
names.push(name.to_owned());
names.push(format!("{name}_known_count"));
}
names
}
fn usage_fact_aggregate_sql(table: &str, by_model: bool) -> String {
let model = if by_model {
"coalesce(mr.upstream_model_id,mr.requested_model_id,'')"
} else {
"''::text"
};
let mut counters = vec![
"'completed_request_count',count(*)".to_owned(),
"'image_request_count',count(*) filter(where image_generation_succeeded is true)".to_owned(),
"'image_request_failed_count',count(*) filter(where image_generation_succeeded is false)".to_owned(),
"'window_total_tokens',coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)+coalesce(output_tokens,0))),0)".to_owned(),
];
for name in super::metrics::TOKEN_METRICS {
counters.push(format!(
"'{name}',coalesce(sum({name}),0),'{name}_known_count',count({name})"
));
}
for source in ["provider_reported", "calculated", "unavailable"] {
counters.push(format!(
"'{source}_count',count(*) filter(where cost_source='{source}')"
));
}
format!("select mr.provider_account_ref as account_id,{model} as model,
jsonb_build_object({}) as counts,case when mr.cost_currency is not null and sum(mr.cost_amount) is not null
then jsonb_build_object(mr.cost_currency,sum(mr.cost_amount)::text) else '{{}}'::jsonb end as costs,
'{{}}'::jsonb as distributions,max(mr.started_at) as last_used_at from {table} mr",counters.join(","))
}
pub(crate) async fn account_usage_windows(
connection: &mut ObservabilityConnection,
windows: &[AccountUsageWindowQuery],
) -> StoreResult<Vec<AccountUsageWindowResult>> {
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
// 相同窗口一次查询所有账号,避免列表逐账号重复读取同一套时间桶
let mut grouped = BTreeMap::<(DateTime<Utc>, DateTime<Utc>), BTreeSet<String>>::new();
for window in windows {
grouped
.entry((window.range.start, window.range.end))
.or_default()
.insert(window.account_id.clone());
}
let mut results = Vec::with_capacity(windows.len());
for ((start, end), accounts) in grouped {
let ids = accounts.into_iter().collect::<Vec<_>>();
let states = snapshot
.account_usage_states(ObservabilityRange { start, end }, &ids)
.await?;
for window in windows
.iter()
.filter(|window| window.range.start == start && window.range.end == end)
{
let mut total = MetricState::default();
let mut models = Vec::new();
for ((account, model), state) in &states {
if account != &window.account_id {
continue;
}
total.merge(state)?;
if !model.is_empty()
&& let Some(last_used_at) = state.last_used_at()
{
models.push(AccountModelUsage {
model: model.clone(),
request_count: state.count("completed_request_count"),
success_count: state.count("completed_request_count"),
input_tokens: state.optional_count("input_tokens"),
output_tokens: state.optional_count("output_tokens"),
cached_input_tokens: state.optional_count("cached_input_tokens"),
cache_write_input_tokens: state.optional_count("cache_write_input_tokens"),
reasoning_output_tokens: state.optional_count("reasoning_output_tokens"),
image_input_tokens: state.optional_count("image_input_tokens"),
image_output_tokens: state.optional_count("image_output_tokens"),
total_tokens: Some(state.count("window_total_tokens")),
image_request_count: state.count("image_request_count"),
image_request_failed_count: state.count("image_request_failed_count"),
cost_coverage: admin_cost_coverage(state.coverage()),
costs: account_costs(state)?,
last_used_at,
});
}
}
models.sort_by(|a, b| {
b.request_count
.cmp(&a.request_count)
.then(a.model.cmp(&b.model))
});
results.push(AccountUsageWindowResult {
account_id: window.account_id.clone(),
key: window.key.clone(),
usage: AccountUsage {
account_id: window.account_id.clone(),
request_count: total.count("completed_request_count"),
success_count: total.count("completed_request_count"),
input_tokens: total.optional_count("input_tokens"),
output_tokens: total.optional_count("output_tokens"),
cached_input_tokens: total.optional_count("cached_input_tokens"),
cache_write_input_tokens: total.optional_count("cache_write_input_tokens"),
reasoning_output_tokens: total.optional_count("reasoning_output_tokens"),
image_input_tokens: total.optional_count("image_input_tokens"),
image_output_tokens: total.optional_count("image_output_tokens"),
total_tokens: Some(total.count("window_total_tokens")),
image_request_count: total.count("image_request_count"),
image_request_failed_count: total.count("image_request_failed_count"),
cost_coverage: admin_cost_coverage(total.coverage()),
costs: account_costs(&total)?,
last_used_at: total.last_used_at(),
request_buckets: Vec::new(),
models,
},
});
}
}
snapshot.finish().await?;
results.sort_by(|a, b| a.account_id.cmp(&b.account_id).then(a.key.cmp(&b.key)));
Ok(results)
}
fn account_costs(state: &MetricState) -> StoreResult<Vec<AccountCost>> {
Ok(state
.costs()?
.into_iter()
.map(|cost| AccountCost {
currency: cost.currency,
amount: cost.amount,
})
.collect())
}
pub(crate) async fn account_usage_sort_values(
snapshot: &mut AnalyticsSnapshot<'_>,
now: DateTime<Utc>,
) -> StoreResult<(Vec<String>, Vec<i64>, Vec<Option<DateTime<Utc>>>)> {
let days: i32 =
sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1")
.fetch_one(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read account usage retention", error))?;
let ids: Vec<String> = sqlx::query_scalar("select id from provider_accounts")
.fetch_all(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read account usage sort identities", error))?;
let range = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?;
let mut totals = BTreeMap::<String, MetricState>::new();
for ((account, _), state) in snapshot.account_states(range, &ids, false).await? {
totals.entry(account).or_default().merge(&state)?;
}
let mut tokens = Vec::with_capacity(ids.len());
let mut latest = Vec::with_capacity(ids.len());
for id in &ids {
let state = totals.remove(id).unwrap_or_default();
tokens.push(
i64::try_from(state.count("window_total_tokens"))
.map_err(|_| invalid("account usage sort token overflow"))?,
);
latest.push(state.last_used_at());
}
Ok((ids, tokens, latest))
}
pub(crate) async fn account_group_usage(
connection: &mut ObservabilityConnection,
group_ids: &[String],
timezone: gateway_core::time::DeploymentTimeZone,
) -> StoreResult<BTreeMap<String, AccountGroupUsage>> {
if group_ids.is_empty() {
return Ok(BTreeMap::new());
}
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
let now = Utc::now();
let days: i32 =
sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1")
.fetch_one(&mut *snapshot.tx)
.await
.map_err(|error| postgres_unavailable("read group usage retention", error))?;
let memberships:Vec<(String,String)>=sqlx::query_as("select account_group_id,provider_account_id from account_group_accounts where account_group_id=any($1::text[])").bind(group_ids).fetch_all(&mut *snapshot.tx).await.map_err(|error|postgres_unavailable("read group usage membership",error))?;
let ids = memberships
.iter()
.map(|(_, account)| account.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let retained = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?;
let today = ObservabilityRange::new(
timezone
.day_start(now)
.ok_or_else(|| invalid("invalid business date"))?
.max(retained.start),
now,
)?;
let retained_states = snapshot.account_states(retained, &ids, false).await?;
let today_states = snapshot.account_states(today, &ids, false).await?;
let mut results = BTreeMap::new();
for group in group_ids {
let mut usage = AccountGroupUsage {
today_usd: "0".parse().map_err(|_| invalid("invalid zero cost"))?,
retained_total_usd: "0".parse().map_err(|_| invalid("invalid zero cost"))?,
};
for (_, account) in memberships.iter().filter(|(id, _)| id == group) {
for (states, total) in [
(&retained_states, &mut usage.retained_total_usd),
(&today_states, &mut usage.today_usd),
] {
for ((id, _), state) in states {
if id == account
&& let Some(amount) = state.costs.get("USD")
{
let amount = amount
.parse()
.map_err(|_| invalid("invalid group usage cost"))?;
*total = total
.checked_add(&amount)
.ok_or_else(|| invalid("group usage cost overflow"))?;
}
}
}
}
results.insert(group.clone(), usage);
}
snapshot.finish().await?;
Ok(results)
}
@@ -0,0 +1,410 @@
//! 在 PostgreSQL 内完成事实统计与汇总合并,结果规模只随输出分组增长
use super::*;
/// 输入 matching 提供 output_bucket、name 和完整事实列及 is_completed_usage
/// 精确分位数在整个结果组内计算,不把局部分位数合并成新的分位数
pub(super) fn push_fact_aggregation(
query: &mut QueryBuilder<Postgres>,
exact_percentiles: bool,
read_costs: bool,
) {
push_fact_aggregation_selected(query, exact_percentiles, read_costs, true, None, None);
}
pub(super) fn push_fact_aggregation_selected(
query: &mut QueryBuilder<Postgres>,
exact_percentiles: bool,
read_costs: bool,
read_distributions: bool,
counter_names: Option<&[&str]>,
distribution_names: Option<&[&str]>,
) {
let counters = selected_counter_expressions(counter_names);
query.push(", counted as (select output_bucket,name,max(started_at) filter(where is_completed_usage) as last_used_at,");
push_counter_object(query, &counters);
query.push(" as counts");
if exact_percentiles && read_distributions {
query.push(",");
push_exact_distributions(query, distribution_names);
}
query.push(" from matching group by output_bucket,name)");
if read_costs {
query.push(", cost_values as (select output_bucket,name,currency,sum(amount)::text as amount from matching cross join lateral (values (cost_currency,cost_amount),('standard_usd',standard_cost_amount),('no_cache_usd',no_cache_cost_amount)) cost(currency,amount) where is_completed_usage and currency is not null and amount is not null group by output_bucket,name,currency), merged_costs as (select output_bucket,name,jsonb_object_agg(currency,amount) as costs from cost_values group by output_bucket,name)");
}
if !read_distributions {
query.push(" select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,'{}'::jsonb as distributions,'{}'::jsonb as exact_percentiles,");
query.push(if read_costs {
"coalesce(k.costs,'{}'::jsonb)"
} else {
"'{}'::jsonb"
});
query.push(" as costs from counted c");
if read_costs {
query.push(" left join merged_costs k using(output_bucket,name)");
}
return;
}
if exact_percentiles {
query.push(" select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,c.distributions,c.exact_percentiles,");
query.push(if read_costs {
"coalesce(k.costs,'{}'::jsonb)"
} else {
"'{}'::jsonb"
});
query.push(" as costs from counted c");
if read_costs {
query.push(" left join merged_costs k using(output_bucket,name)");
}
return;
}
query.push(", samples as (select output_bucket,name,metric,value from matching cross join lateral (values ");
for (index, (name, expression)) in selected_sample_expressions(distribution_names)
.iter()
.enumerate()
{
if index > 0 {
query.push(",");
}
query.push(format!("('{name}',{expression})"));
}
query.push(") sample(metric,value) where value is not null), sample_stats as (select output_bucket,name,metric,count(*) as count,sum(value) as sum,min(value) as minimum,max(value) as maximum,count(*) filter(where value=0) as zero");
query.push(" from samples group by output_bucket,name,metric)");
// 对数桶定义与 Histogram::add 相同,零值单独保留,不对零取对数
query.push(", bin_counts as (select output_bucket,name,metric,floor(log(2::numeric,value::numeric)*16)::integer as bin,count(*) as count from samples where value>0 group by output_bucket,name,metric,bin), merged_bins as (select output_bucket,name,metric,jsonb_object_agg(bin,count) as bins from bin_counts group by output_bucket,name,metric)");
query.push(", merged_distributions as (select s.output_bucket,s.name,jsonb_object_agg(s.metric,jsonb_build_object('count',s.count,'sum',s.sum,'minimum',s.minimum,'maximum',s.maximum,'zero',s.zero,'bins',");
query.push("coalesce(b.bins,'{}'::jsonb)");
query.push(")) as distributions");
query.push(" from sample_stats s");
query.push(" left join merged_bins b using(output_bucket,name,metric)");
query.push(" group by s.output_bucket,s.name) select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,");
query.push(if read_costs {
"coalesce(k.costs,'{}'::jsonb)"
} else {
"'{}'::jsonb"
});
query.push(" as costs,coalesce(d.distributions,'{}'::jsonb) as distributions,");
query.push("'{}'::jsonb");
query.push(" as exact_percentiles from counted c left join merged_distributions d using(output_bucket,name)");
if read_costs {
query.push(" left join merged_costs k using(output_bucket,name)");
}
}
// 事实聚合与汇总读取共用指标清单,防止新增计数只在其中一条路径生效
fn counter_expressions() -> Vec<(String, String)> {
let mut counters = vec![
("request_count".to_owned(), "count(*)".to_owned()),
("completed_request_count".to_owned(), "count(*) filter(where is_completed_usage)".to_owned()),
("caller_error_count".to_owned(), "count(*) filter(where client_status_code between 400 and 499)".to_owned()),
("attempt_count".to_owned(), "coalesce(sum(attempt_count),0)".to_owned()),
("retry_count".to_owned(), "coalesce(sum(greatest(attempt_count-1,0)),0)".to_owned()),
("retried_request_count".to_owned(), "count(*) filter(where attempt_count>1)".to_owned()),
("capacity_sample_count".to_owned(), "count(capacity_total_slots)".to_owned()),
("window_total_tokens".to_owned(), "coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)::numeric+coalesce(output_tokens,0))) filter(where is_completed_usage),0)".to_owned()),
("cache_eligible_request_count".to_owned(), "count(*) filter(where is_completed_usage and input_tokens is not null)".to_owned()),
("cache_hit_request_count".to_owned(), "count(*) filter(where is_completed_usage and input_tokens is not null and cached_input_tokens>0)".to_owned()),
("image_request_count".to_owned(), "count(*) filter(where is_completed_usage and image_generation_succeeded)".to_owned()),
("image_request_failed_count".to_owned(), "count(*) filter(where is_completed_usage and not image_generation_succeeded)".to_owned()),
];
for (outcome, name) in [
("succeeded", "success"),
("failed", "failure"),
("cancelled", "cancelled"),
("incomplete", "incomplete"),
] {
counters.push((
format!("{name}_count"),
format!("count(*) filter(where outcome='{outcome}')"),
));
let extra = if outcome == "failed" {
"+coalesce(sum(ops_failure_count),0)"
} else {
""
};
counters.push((
format!("attempt_{name}_count"),
format!("count(*) filter(where attempt_count>0 and outcome='{outcome}'){extra}"),
));
}
for (name, predicate, source) in [
(
"rate_limited_count",
"(error_kind in ('rate_limited','quota_exhausted') or coalesce(upstream_status_code,client_status_code)=429)",
"ops_rate_limited_count",
),
(
"auth_failure_count",
"(error_kind in ('authentication','authorization','invalid_credential') or coalesce(upstream_status_code,client_status_code) in (401,403))",
"ops_auth_failure_count",
),
(
"provider_5xx_count",
"coalesce(upstream_status_code,client_status_code) between 500 and 599",
"ops_provider_5xx_count",
),
] {
counters.push((name.to_owned(), format!("count(*) filter(where attempt_count>0 and outcome='failed' and {predicate})+coalesce(sum({source}),0)")));
}
for name in metrics::TOKEN_METRICS {
counters.push((
name.to_owned(),
format!("coalesce(sum({name}) filter(where is_completed_usage),0)"),
));
counters.push((
format!("{name}_known_count"),
format!("count({name}) filter(where is_completed_usage)"),
));
}
for name in ["latency_ms", "first_token_ms"] {
counters.push((
format!("{name}_sum"),
format!("coalesce(sum({name}) filter(where is_completed_usage),0)"),
));
}
for (name, column) in [
("standard_usd", "standard_cost_amount"),
("no_cache_usd", "no_cache_cost_amount"),
] {
counters.push((
format!("{name}_known_count"),
format!("count({column}) filter(where is_completed_usage)"),
));
}
for source in ["provider_reported", "calculated", "unavailable"] {
counters.push((
format!("{source}_count"),
format!("count(*) filter(where is_completed_usage and cost_source='{source}')"),
));
}
counters
}
fn selected_counter_expressions(names: Option<&[&str]>) -> Vec<(String, String)> {
let mut counters = counter_expressions();
if let Some(names) = names {
counters.retain(|(name, _)| names.contains(&name.as_str()));
}
counters
}
fn push_exact_distributions(query: &mut QueryBuilder<Postgres>, names: Option<&[&str]>) {
let samples = selected_sample_expressions(names);
// 独立 ordered-set 聚合只排序实际样本,避免把每条事实展开为六条记录再排序
query.push("jsonb_build_object(");
for (index, (name, expression)) in samples.iter().enumerate() {
if index > 0 {
query.push(",");
}
query.push(format!("'{name}',jsonb_build_object('count',count({expression}),'sum',coalesce(sum({expression}),0),'minimum',min({expression}),'maximum',max({expression}),'zero',count(*) filter(where ({expression})=0),'bins','{{}}'::jsonb)"));
}
query.push(") as distributions,jsonb_build_object(");
for (index, (name, expression)) in samples.iter().enumerate() {
if index > 0 {
query.push(",");
}
query.push(format!("'{name}',coalesce(to_jsonb(percentile_cont(array[0.1,0.5,0.9,0.95,0.99]::double precision[]) within group(order by ({expression}))),'[]'::jsonb)"));
}
query.push(") as exact_percentiles");
}
fn sample_expressions() -> [(&'static str, &'static str); 6] {
[
(
"admission_decision_ms",
"admission_decision_ms::double precision",
),
(
"account_selection_wait_ms",
"account_selection_wait_ms::double precision",
),
(
"capacity",
"case when capacity_total_slots>0 then capacity_used_slots::double precision/capacity_total_slots::double precision*10000.0 end",
),
(
"latency_ms",
"case when is_completed_usage then latency_ms::double precision end",
),
(
"first_token_ms",
"case when is_completed_usage then first_token_ms::double precision end",
),
(
"throughput",
"case when is_completed_usage and output_tokens>0 and upstream_response_ms>0 then output_tokens::double precision*1000.0/upstream_response_ms::double precision end",
),
]
}
fn selected_sample_expressions(names: Option<&[&str]>) -> Vec<(&'static str, &'static str)> {
sample_expressions()
.into_iter()
.filter(|(name, _)| names.is_none_or(|names| names.contains(name)))
.collect()
}
fn push_counter_object(query: &mut QueryBuilder<Postgres>, counters: &[(String, String)]) {
// PostgreSQL 单个函数调用最多接收一百个参数,分块构造不限制指标数量
for (index, chunk) in counters.chunks(40).enumerate() {
if index > 0 {
query.push(" || ");
}
query.push("jsonb_build_object(");
for (position, (name, expression)) in chunk.iter().enumerate() {
if position > 0 {
query.push(",");
}
query.push(format!("'{name}',{expression}"));
}
query.push(")");
}
}
/// 输入 matching 的行来自互斥时间桶,合并后每个最终输出组只返回一行
pub(super) fn push_summary_aggregation(
query: &mut QueryBuilder<Postgres>,
distributions: bool,
costs: bool,
counter_names: Option<&[&str]>,
distribution_names: Option<&[&str]>,
) {
// 每个源汇总只展开一次固定指标,避免按指标键放大行数再排序
let counters = selected_counter_expressions(counter_names)
.into_iter()
.map(|(name, _)| {
let expression = format!("coalesce(sum(metric.{name}),0)");
(name, expression)
})
.collect::<Vec<_>>();
query.push(", grouped as (select output_bucket,name,max(last_used_at) as last_used_at,");
push_counter_object(query, &counters);
query.push(" as counts from matching cross join lateral jsonb_to_record(counts) as metric(");
query.push(
counters
.iter()
.map(|(name, _)| format!("{name} numeric"))
.collect::<Vec<_>>()
.join(","),
);
query.push(") group by output_bucket,name)");
if costs {
query.push(", cost_values as (select output_bucket,name,c.key,sum(c.value::numeric)::text as value from matching cross join lateral jsonb_each_text(costs) c group by output_bucket,name,c.key), merged_costs as (select output_bucket,name,jsonb_object_agg(key,value) as costs from cost_values group by output_bucket,name)");
}
if distributions {
// 先按最终输出组归并分布,再在每组内合并直方图键,避免全局展开后大排序
query.push(", samples as (select output_bucket,name,d.key as metric,d.value from matching cross join lateral jsonb_each(distributions) d");
if let Some(names) = distribution_names {
query
.push(" where d.key=any(")
.push_bind(
names
.iter()
.map(|name| (*name).to_owned())
.collect::<Vec<_>>(),
)
.push("::text[])");
}
query.push("), sample_stats as (select output_bucket,name,metric,sum((value->>'count')::numeric) as count,sum((value->>'sum')::double precision) as sum,min((value->>'minimum')::double precision) as minimum,max((value->>'maximum')::double precision) as maximum,sum((value->>'zero')::numeric) as zero,array_agg(value->'bins') as bins from samples group by output_bucket,name,metric), merged_distributions as (select s.output_bucket,s.name,jsonb_object_agg(s.metric,jsonb_build_object('count',s.count,'sum',s.sum,'minimum',s.minimum,'maximum',s.maximum,'zero',s.zero,'bins',coalesce(b.bins,'{}'::jsonb))) as distributions from sample_stats s cross join lateral(select jsonb_object_agg(key,count) as bins from(select b.key,sum(b.value::numeric) as count from unnest(s.bins) input(value) cross join lateral jsonb_each_text(input.value) b group by b.key) merged)b group by s.output_bucket,s.name)");
}
query.push(" select g.output_bucket as bucket_start,g.name,g.last_used_at,g.counts,");
query.push(if costs {
"coalesce(k.costs,'{}'::jsonb)"
} else {
"'{}'::jsonb"
});
query.push(" as costs,");
query.push(if distributions {
"coalesce(d.distributions,'{}'::jsonb)"
} else {
"'{}'::jsonb"
});
query.push(" as distributions from grouped g");
if costs {
query.push(" left join merged_costs k using(output_bucket,name)");
}
if distributions {
query.push(" left join merged_distributions d using(output_bucket,name)");
}
}
pub(super) fn push_output_bucket(
query: &mut QueryBuilder<Postgres>,
column: &str,
range: ObservabilityRange,
granularity: Option<ObservationGranularity>,
boundaries: Option<&[DateTime<Utc>]>,
) {
match granularity {
None => {
query.push_bind(range.start).push("::timestamptz");
}
Some(ObservationGranularity::Day) => {
// 使用有序阈值二分定位,保留部署时区及夏令时日界,不逐行扫描所有日界
let boundaries = boundaries.unwrap_or_default();
query
.push("(")
.push_bind(boundaries.to_vec())
.push(format!("::timestamptz[])[width_bucket({column},"))
.push_bind(boundaries.to_vec())
.push("::timestamptz[])]");
}
Some(granularity) => {
query
.push("date_bin(make_interval(secs => ")
.push_bind(granularity.seconds() as f64)
.push(format!("),{column},timestamptz '2000-01-01 UTC')"));
}
}
}
pub(super) fn fact_dimension_sql(dimension: Option<DiagnosticDimension>) -> &'static str {
match dimension {
None => "''::text",
Some(DiagnosticDimension::Provider) => "coalesce(nullif(mr.provider_kind,''),'unrouted')",
Some(DiagnosticDimension::Model) => {
"coalesce(nullif(mr.upstream_model_id,''),mr.requested_model_id,'')"
}
Some(DiagnosticDimension::Account) => {
"coalesce(nullif(mr.provider_account_ref,''),'unrouted')"
}
Some(DiagnosticDimension::ApiKey) => "coalesce(mr.client_api_key_ref,'')",
Some(DiagnosticDimension::Transport) => {
"coalesce(nullif(mr.upstream_transport,''),nullif(mr.client_transport,''),'unknown')"
}
Some(DiagnosticDimension::Failure) => "coalesce(mr.error_kind,'')",
Some(DiagnosticDimension::Status) => {
"coalesce(mr.upstream_status_code::text,mr.client_status_code::text,'')"
}
}
}
pub(super) fn summary_dimension_sql(
collection: &str,
dimension: Option<DiagnosticDimension>,
) -> String {
let offset = usize::from(matches!(collection, "account_model" | "key_model"));
match dimension {
None => "''::text".to_owned(),
Some(DiagnosticDimension::Provider) => {
format!("coalesce(nullif(dimensions[{}],''),'unrouted')", 1 + offset)
}
Some(DiagnosticDimension::Model) => format!(
"coalesce(nullif(dimensions[{}],''),dimensions[{}],'')",
3 + offset,
2 + offset
),
Some(DiagnosticDimension::Account) => {
"coalesce(nullif(dimensions[1],''),'unrouted')".to_owned()
}
Some(DiagnosticDimension::Transport) => {
"coalesce(nullif(dimensions[2],''),nullif(dimensions[1],''),'unknown')".to_owned()
}
Some(DiagnosticDimension::Status) => {
"coalesce(nullif(dimensions[2],''),dimensions[1],'')".to_owned()
}
Some(DiagnosticDimension::ApiKey | DiagnosticDimension::Failure) => {
"coalesce(dimensions[1],'')".to_owned()
}
}
}
@@ -49,13 +49,13 @@ pub(super) async fn rebuild_bucket_batch(
0
};
if !same_version {
// 处理中事实发生变化时清掉未发布的部分状态,不能把新旧快照增量相加
// 显式重建版本变化时清掉旧前缀,普通事实变化由增量维护更新前缀
sqlx::query("update request_analytics_pending_buckets set build_version=change_version,build_phase=0,build_started_at=null,build_id=null,build_collection=null,build_dimensions=null,build_outcome=null where bucket_start=$1")
.bind(bucket).execute(&mut **tx).await.map_err(|error|postgres_unavailable("restart changed analytics bucket",error))?;
}
let hour = bucket_for(bucket, 3600)?;
if phase < 2 {
// 小时分段期间不改写其输入;源事实仍可投影,版本变化使小时任务先退回等待
// 小时分段期间暂停其他 quarter 的完整重建,普通增量同步修正已扫描前缀
let owned:bool=sqlx::query_scalar("select exists(select 1 from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and bucket_start <> $2 and build_phase >= 3)")
.bind(hour).bind(bucket).fetch_one(&mut **tx).await.map_err(|error|postgres_unavailable("claim analytics hour stage",error))?;
if owned {
@@ -119,8 +119,9 @@ pub(super) async fn rebuild_bucket_batch(
let limit = i64::from(batch_size) + 1;
let mut groups = GroupStates::new();
if phase == 1 {
// 增量前缀判断使用 Rust 字节序,SQL 游标必须使用同一排序规则
let statement = format!(
"{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and mr.recovered_at is null and (mr.started_at,mr.id)>($2,$3) order by mr.started_at,mr.id limit $4",
"{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and (mr.started_at,mr.id collate \"C\")>($2,$3) order by mr.started_at,mr.id collate \"C\" limit $4",
fact_select()
);
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
@@ -139,11 +140,16 @@ pub(super) async fn rebuild_bucket_batch(
let outcome: String = get(fact, "outcome")?;
let mut state = MetricState::default();
state.add_row(fact)?;
let forecast = forecast::state(fact)?;
for (collection, dimensions) in collections(fact)? {
groups
.entry((collection.to_owned(), dimensions, outcome.clone()))
.or_default()
.merge(&state)?;
.merge(if collection == forecast::COLLECTION {
&forecast
} else {
&state
})?;
}
}
merge_staged_groups(tx, "request_analytics_15m", bucket, groups).await?;
@@ -155,7 +161,7 @@ pub(super) async fn rebuild_bucket_batch(
.bind(bucket).bind(get::<DateTime<Utc>>(last,"started_at")?).bind(get::<String>(last,"id")?).execute(&mut **tx).await.map_err(|error|postgres_unavailable("save analytics fact cursor",error))?;
}
} else if phase == 4 {
let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection,dimensions,outcome)>($4,$5,$6,$7) order by bucket_start,collection,dimensions,outcome limit $8")
let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions,last_used_at from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection collate \"C\",dimensions collate \"C\",outcome collate \"C\")>($4,$5,$6,$7) order by bucket_start,collection collate \"C\",dimensions collate \"C\",outcome collate \"C\" limit $8")
.bind(hour).bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).bind(get::<Option<DateTime<Utc>>>(&row,"build_started_at")?.unwrap_or(hour-TimeDelta::microseconds(1)))
.bind(get::<Option<String>>(&row,"build_collection")?.unwrap_or_default()).bind(get::<Option<Vec<String>>>(&row,"build_dimensions")?.unwrap_or_default()).bind(get::<Option<String>>(&row,"build_outcome")?.unwrap_or_default()).bind(limit)
.fetch_all(&mut **tx).await.map_err(|error|postgres_unavailable("read analytics hour segment",error))?;
@@ -198,7 +204,7 @@ async fn merge_staged_groups(
}
let keys=groups.keys().map(|(collection,dimensions,outcome)|serde_json::json!({"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::<Vec<_>>();
let statement = format!(
"select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1"
"select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions,stored.last_used_at from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1"
);
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(bucket)
@@ -220,37 +226,64 @@ async fn merge_staged_groups(
write_groups(tx, table, bucket, groups).await
}
async fn write_groups(
pub(super) async fn write_groups(
tx: &mut sqlx::Transaction<'_, Postgres>,
table: &str,
bucket: DateTime<Utc>,
groups: GroupStates,
) -> StoreResult<()> {
if groups.is_empty() {
return Ok(());
}
let mut statement = QueryBuilder::<Postgres>::new(format!(
"insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) "
));
let rows = groups
.into_iter()
.map(|((collection, dimensions, outcome), state)| {
let (counts, costs, distributions) = state.encode()?;
Ok((
write_bucket_groups(tx, table, [(bucket, groups)]).await
}
pub(super) async fn write_bucket_groups(
tx: &mut sqlx::Transaction<'_, Postgres>,
table: &str,
buckets: impl IntoIterator<Item = (DateTime<Utc>, GroupStates)>,
) -> StoreResult<()> {
let mut groups = buckets.into_iter().flat_map(|(bucket, groups)| {
groups
.into_iter()
.map(move |(key, state)| (bucket, key, state))
});
loop {
// 跨桶合并 SQL 往返,但编码缓冲只保留当前批次,避免回填将所有状态展开为 JSON
let rows = groups
.by_ref()
.take(1000)
.map(|(bucket, (collection, dimensions, outcome), state)| {
let (counts, costs, distributions) = state.encode()?;
Ok((
bucket,
collection,
dimensions,
outcome,
counts,
costs,
distributions,
state.last_used_at(),
))
})
.collect::<StoreResult<Vec<_>>>()?;
if rows.is_empty() {
return Ok(());
}
let mut statement = QueryBuilder::<Postgres>::new(format!(
"insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions,last_used_at) "
));
// 十个绑定参数限制单语句批量,避免高基数时间桶越过 PostgreSQL 参数上限
statement.push_values(
&rows,
|mut row,
(
bucket,
collection,
dimensions,
outcome,
counts,
costs,
distributions,
))
})
.collect::<StoreResult<Vec<_>>>()?;
// 九个绑定参数限制单语句批量,避免高基数时间桶越过 PostgreSQL 参数上限
for chunk in rows.chunks(1000) {
statement.push_values(
chunk,
|mut row, (collection, dimensions, outcome, counts, costs, distributions)| {
last_used_at,
)| {
row.push_bind(bucket)
.push_bind(collection)
.push_bind(dimensions)
@@ -259,16 +292,15 @@ async fn write_groups(
.push_bind(HISTOGRAM_VERSION)
.push_bind(counts)
.push_bind(costs)
.push_bind(distributions);
.push_bind(distributions)
.push_bind(last_used_at);
},
);
statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version");
statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,last_used_at=excluded.last_used_at,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version");
statement
.build()
.execute(&mut **tx)
.await
.map_err(|error| postgres_unavailable("write analytics bucket", error))?;
statement.reset();
}
Ok(())
}
@@ -0,0 +1,113 @@
//! 用完整刻钟计数和同快照源差量定位精确页码,避免每页扫描整个保留期
use super::query::{collection_for, push_collection_filter, push_summary_buckets};
use super::*;
enum ReadVersion {
Published,
Before,
Current,
}
impl AnalyticsSnapshot<'_> {
pub(crate) async fn completed_request_counts(
&mut self,
range: ObservabilityRange,
filter: &UsageRecordFilter,
) -> StoreResult<Option<BTreeMap<DateTime<Utc>, u64>>> {
let Some(collection) = collection_for(filter, None) else {
return Ok(None);
};
if !self.projection_covers(range) {
return Ok(None);
}
let mut counts = BTreeMap::new();
let (_, buckets) = self.summary_buckets(range, false).await?;
if !buckets.is_empty() {
let mut query = QueryBuilder::<Postgres>::new(
"select bucket_start,coalesce(sum((counts->>'completed_request_count')::bigint),0)::bigint as request_count from request_analytics_15m a where collection=",
);
query
.push_bind(collection)
.push(" and schema_version=")
.push_bind(SCHEMA_VERSION)
.push(" and histogram_version=")
.push_bind(HISTOGRAM_VERSION);
push_summary_buckets(&mut query, &buckets, 900);
push_collection_filter(&mut query, collection, filter);
query.push(" group by bucket_start");
for row in query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("read usage page counts", error))?
{
counts.insert(
get(&row, "bucket_start")?,
to_u64(get(&row, "request_count")?)?,
);
}
}
for (start, end) in self.uncovered_spans(range).await? {
self.merge_completed_counts(
&mut counts,
ObservabilityRange { start, end },
filter,
ReadVersion::Published,
)
.await?;
}
if self.has_pending_requests().await? {
// 先撤销已投影版本,再加入源表当前版本;移动、删除和未确认的重放均只计算一次
self.merge_completed_counts(&mut counts, range, filter, ReadVersion::Before)
.await?;
self.merge_completed_counts(&mut counts, range, filter, ReadVersion::Current)
.await?;
}
Ok(Some(counts))
}
async fn merge_completed_counts(
&mut self,
counts: &mut BTreeMap<DateTime<Utc>, u64>,
range: ObservabilityRange,
filter: &UsageRecordFilter,
version: ReadVersion,
) -> StoreResult<()> {
let table = if matches!(version, ReadVersion::Current) {
"model_request_observations"
} else {
"request_analytics_facts"
};
let mut query = QueryBuilder::<Postgres>::new(format!(
"select date_bin(interval '15 minutes',mr.started_at,timestamptz '2000-01-01 UTC') as bucket_start,count(*)::bigint as request_count from {table} mr"
));
if !matches!(version, ReadVersion::Published) {
query.push(" join request_analytics_pending_requests p on p.request_id=mr.id");
}
query
.push(" where mr.started_at >= ")
.push_bind(range.start)
.push(" and mr.started_at < ")
.push_bind(range.end);
push_completed_usage_fact_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by bucket_start");
let rows = query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("correct usage page counts", error))?;
for row in rows {
let value = to_u64(get(&row, "request_count")?)?;
let count = counts.entry(get(&row, "bucket_start")?).or_default();
*count = if matches!(version, ReadVersion::Before) {
count.checked_sub(value)
} else {
count.checked_add(value)
}
.ok_or_else(|| invalid("usage page count is inconsistent"))?;
}
Ok(())
}
}
@@ -0,0 +1,343 @@
//! 预测累计使用开始时间与完成时间的二维汇总,采样边界及待投影版本读取精确事实
use super::*;
use gateway_admin::model::provider_credentials::ProviderDocument;
use gateway_admin::model::quota_forecast_sampling::{
MAX_FORECAST_HISTORY_POINTS, QuotaForecastHistory, QuotaForecastHistoryPoint,
QuotaForecastUsage,
};
use gateway_core::account::OpaqueProviderData;
use sqlx::postgres::PgRow;
pub(super) const COLLECTION: &str = "account_completion";
const COMPLETION_SECONDS: i64 = 300;
const COUNTERS: [&str; 10] = [
"request_count",
"completed_request_count",
"window_total_tokens",
"input_tokens",
"output_tokens",
"cached_input_tokens",
"missing_token_count",
"known_cost_count",
"unavailable_cost_count",
"excluded_request_count",
];
pub(super) fn dimensions(row: &PgRow) -> StoreResult<Vec<String>> {
let completed: Option<DateTime<Utc>> = get(row, "completed_at")?;
let terminal = get::<String>(row, "outcome")? != "running";
Ok(vec![
get::<Option<String>>(row, "provider_account_ref")?.unwrap_or_default(),
completed
.filter(|_| terminal)
.map(|time| time.timestamp().div_euclid(COMPLETION_SECONDS).to_string())
.unwrap_or_default(),
])
}
pub(super) fn state(row: &PgRow) -> StoreResult<MetricState> {
let mut state = MetricState::default();
state.increment("request_count", 1)?;
state.add_cost("USD", "0")?;
if get::<Option<DateTime<Utc>>>(row, "completed_at")?.is_none()
|| get::<String>(row, "outcome")? == "running"
{
return Ok(state);
}
if !get::<bool>(row, "is_completed_usage")? {
state.increment("excluded_request_count", 1)?;
return Ok(state);
}
state.increment("completed_request_count", 1)?;
let input = optional_unsigned(row, "input_tokens")?;
let output = optional_unsigned(row, "output_tokens")?;
let total = optional_unsigned(row, "total_tokens")?;
state.increment(
"window_total_tokens",
total.unwrap_or(
input
.unwrap_or_default()
.checked_add(output.unwrap_or_default())
.ok_or_else(|| invalid("forecast token overflow"))?,
),
)?;
state.increment("input_tokens", input.unwrap_or_default())?;
state.increment("output_tokens", output.unwrap_or_default())?;
state.increment(
"cached_input_tokens",
optional_unsigned(row, "cached_input_tokens")?.unwrap_or_default(),
)?;
state.increment(
"missing_token_count",
u64::from(total.is_none() && (input.is_none() || output.is_none())),
)?;
let source: String = get(row, "cost_source")?;
let amount: Option<String> = get(row, "cost_amount_text")?;
let known = matches!(source.as_str(), "calculated" | "provider_reported")
&& get::<Option<String>>(row, "cost_currency")?.as_deref() == Some("USD")
&& amount.is_some();
state.increment("known_cost_count", u64::from(known))?;
state.increment("unavailable_cost_count", u64::from(!known))?;
if known {
state.add_cost(
"USD",
amount
.as_deref()
.ok_or_else(|| invalid("missing forecast amount"))?,
)?;
}
Ok(state)
}
impl AnalyticsSnapshot<'_> {
pub(crate) async fn forecast_history(
&mut self,
account: &str,
range: ObservabilityRange,
) -> StoreResult<QuotaForecastHistory> {
let documents = sqlx::query(DOCUMENTS)
.bind(account)
.bind(range.start)
.bind(range.end)
.bind(MAX_FORECAST_HISTORY_POINTS as i32)
.fetch_all(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("sample forecast documents", error))?;
let mut cutoffs = documents
.iter()
.map(|row| get(row, "completed_at"))
.collect::<StoreResult<Vec<DateTime<Utc>>>>()?;
// 最后一个状态专门代表整个窗口,文档恰好完成于窗口末端时复用同一截止值
if cutoffs.last() != Some(&range.end) {
cutoffs.push(range.end);
}
let boundaries = cutoffs
.iter()
.map(|time| time.timestamp().div_euclid(COMPLETION_SECONDS))
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let mut states = vec![MetricState::default(); cutoffs.len() + 1];
if self.projection_covers(range) {
let (_, quarters) = self.summary_buckets(range, false).await?;
if !quarters.is_empty() {
let sums = COUNTERS
.iter()
.map(|name| format!("'{name}',coalesce(sum((counts->>'{name}')::numeric),0)"))
.collect::<Vec<_>>()
.join(",");
let statement = format!(
"with selected as (select counts,costs,case when dimensions[2]='' then cardinality($3::timestamptz[]) else width_bucket(to_timestamp((dimensions[2]::bigint+1)*{COMPLETION_SECONDS})-interval '1 microsecond',$3::timestamptz[]) end as slot
from request_analytics_15m where collection='{COLLECTION}' and dimensions[1]=$1 and bucket_start=any($2::timestamptz[]) and schema_version=$5 and histogram_version=$6
and (dimensions[2]='' or not(nullif(dimensions[2],'')::bigint=any($4::bigint[]))))
select slot,jsonb_build_object({sums}) as counts,jsonb_build_object('USD',coalesce(sum((costs->>'USD')::numeric),0)::text) as costs,'{{}}'::jsonb as distributions,null::timestamptz as last_used_at from selected group by slot"
);
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(account)
.bind(&quarters)
.bind(&cutoffs)
.bind(&boundaries)
.bind(SCHEMA_VERSION)
.bind(HISTOGRAM_VERSION)
.fetch_all(&mut *self.tx)
.await
.map_err(|error| {
postgres_unavailable("read forecast completion summaries", error)
})?;
merge_rows(&mut states, rows, false)?;
// 只有跨过采样截止值的完成时间小桶需要回读,按完成时间索引逐段定位
let rows = self
.forecast_facts(
account,
range,
&cutoffs,
FactSelection::CompletionEdges {
quarters: &quarters,
boundaries: &boundaries,
},
)
.await?;
merge_rows(&mut states, rows, false)?;
}
for (start, end) in self.uncovered_spans(range).await? {
let rows = self
.forecast_facts(
account,
ObservabilityRange { start, end },
&cutoffs,
FactSelection::Projected,
)
.await?;
merge_rows(&mut states, rows, false)?;
}
if self.has_pending_requests().await? {
let old = self
.forecast_facts(account, range, &cutoffs, FactSelection::Previous)
.await?;
merge_rows(&mut states, old, true)?;
let current = self
.forecast_facts(account, range, &cutoffs, FactSelection::Current)
.await?;
merge_rows(&mut states, current, false)?;
}
} else {
let rows = self
.forecast_facts(account, range, &cutoffs, FactSelection::Source)
.await?;
merge_rows(&mut states, rows, false)?;
}
let pending_request_count = states
.last()
.map_or(0, |state| state.count("request_count"));
let mut cumulative = MetricState::default();
let mut usages = Vec::with_capacity(cutoffs.len());
for state in states.iter().take(cutoffs.len()) {
cumulative.merge(state)?;
usages.push(usage(&cumulative)?);
}
let mut points = Vec::with_capacity(documents.len());
for (row, usage) in documents.into_iter().zip(&usages) {
let document: sqlx::types::Json<serde_json::Map<String, serde_json::Value>> =
get(&row, "document")?;
points.push(QuotaForecastHistoryPoint {
started_at: get(&row, "started_at")?,
completed_at: get(&row, "completed_at")?,
usage: usage.clone(),
provider_observation: ProviderDocument::new(OpaqueProviderData::new(document.0)),
});
}
Ok(QuotaForecastHistory {
points,
usage: usage(&cumulative)?,
pending_request_count,
})
}
async fn forecast_facts(
&mut self,
account: &str,
range: ObservabilityRange,
cutoffs: &[DateTime<Utc>],
selection: FactSelection<'_>,
) -> StoreResult<Vec<PgRow>> {
let source = matches!(selection, FactSelection::Source | FactSelection::Current);
let table = if source {
"model_requests"
} else {
"request_analytics_facts"
};
let mut query = QueryBuilder::<Postgres>::new(
"with matching as (select mr.*,case when mr.completed_at is not null and mr.outcome <> 'running' then width_bucket(mr.completed_at-interval '1 microsecond',",
);
query.push_bind(cutoffs).push("::timestamptz[]) else ").push_bind(cutoffs.len() as i32)
.push(" end as slot,coalesce((").push(completed_usage_fact_predicate("mr"))
.push("),false) and mr.completed_at is not null and mr.outcome <> 'running' as included from ");
if let FactSelection::CompletionEdges { boundaries, .. } = selection {
// 保留按完成时间小桶的参数化索引扫描,避免展开后扫描整段账号事实再逐桶匹配
// OFFSET 0 不裁剪数据,只阻止此相关子查询被合并到外层连接
query.push("unnest(").push_bind(boundaries).push(format!("::bigint[]) edge(bucket) cross join lateral (select * from {table} f where f.provider_account_ref="))
.push_bind(account).push(format!(" and f.completed_at >= to_timestamp(edge.bucket*{COMPLETION_SECONDS}) and f.completed_at < to_timestamp((edge.bucket+1)*{COMPLETION_SECONDS}) and f.outcome <> 'running' offset 0) mr"));
} else {
query.push(format!("{table} mr"));
}
if matches!(selection, FactSelection::Previous | FactSelection::Current) {
query.push(" join request_analytics_pending_requests p on p.request_id=mr.id");
}
query
.push(" where mr.provider_account_ref=")
.push_bind(account)
.push(" and mr.started_at >= ")
.push_bind(range.start)
.push(" and mr.started_at < ")
.push_bind(range.end);
if let FactSelection::CompletionEdges { quarters, .. } = selection {
query.push(" and date_bin(interval '15 minutes',mr.started_at,timestamptz '2000-01-01 UTC')=any(").push_bind(quarters).push("::timestamptz[])");
}
// 多个累计指标复用同一分类结果,避免展开后重复判断完成用量并携带宽事实排序
// OFFSET 0 保留全部行,只让分类在聚合前计算一次
query.push("), classified as (select *,coalesce(cost_source in ('calculated','provider_reported') and cost_amount is not null and cost_currency='USD',false) as known_cost from matching offset 0)
select slot,jsonb_build_object(
'request_count',count(*),
'completed_request_count',count(*) filter(where included),
'window_total_tokens',coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)+coalesce(output_tokens,0))) filter(where included),0),
'input_tokens',coalesce(sum(input_tokens) filter(where included),0),
'output_tokens',coalesce(sum(output_tokens) filter(where included),0),
'cached_input_tokens',coalesce(sum(cached_input_tokens) filter(where included),0),
'missing_token_count',count(*) filter(where included and total_tokens is null and (input_tokens is null or output_tokens is null)),
'known_cost_count',count(*) filter(where included and known_cost),
'unavailable_cost_count',count(*) filter(where included and not known_cost),
'excluded_request_count',count(*) filter(where not included and completed_at is not null and outcome <> 'running')) as counts,
jsonb_build_object('USD',coalesce(sum(cost_amount) filter(where included and known_cost),0)::text) as costs,
'{}'::jsonb as distributions,null::timestamptz as last_used_at from classified group by slot");
query
.build()
.fetch_all(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("read exact forecast boundaries", error))
}
}
#[derive(Clone, Copy)]
enum FactSelection<'a> {
Source,
Projected,
Previous,
Current,
CompletionEdges {
quarters: &'a [DateTime<Utc>],
boundaries: &'a [i64],
},
}
fn merge_rows(states: &mut [MetricState], rows: Vec<PgRow>, subtract: bool) -> StoreResult<()> {
for row in rows {
let index = usize::try_from(get::<i32>(&row, "slot")?)
.map_err(|_| invalid("negative forecast slot"))?;
let target = states
.get_mut(index)
.ok_or_else(|| invalid("forecast slot out of range"))?;
let state = MetricState::decode(&row)?;
if subtract {
incremental::subtract(target, &state, &MetricState::default())?;
} else {
target.merge(&state)?;
}
}
Ok(())
}
fn usage(state: &MetricState) -> StoreResult<QuotaForecastUsage> {
Ok(QuotaForecastUsage {
request_count: state.count("completed_request_count"),
tokens: state.count("window_total_tokens"),
input_tokens: state.count("input_tokens"),
output_tokens: state.count("output_tokens"),
cached_input_tokens: state.count("cached_input_tokens"),
missing_token_count: state.count("missing_token_count"),
known_cost_count: state.count("known_cost_count"),
unavailable_cost_count: state.count("unavailable_cost_count"),
excluded_request_count: state.count("excluded_request_count"),
usd: state
.costs
.get("USD")
.map(|value| {
value
.parse::<f64>()
.map_err(|_| invalid("invalid forecast dollars"))
})
.transpose()?
.unwrap_or_default(),
})
}
// 微秒边界向上取整,最后一槽包含观测时刻;文档采样与累计量补差共用读事务
const DOCUMENTS: &str = "select mr.started_at,mr.completed_at,mr.provider_observation_json as document
from generate_series(0,$4-1) b(bucket) cross join lateral (
select mr.started_at,mr.completed_at,mr.provider_observation_json
from model_requests mr where mr.provider_account_ref=$1 and mr.started_at >= $2 and mr.started_at < $3
and mr.completed_at >= $2 + ceil(b.bucket*greatest(1,extract(epoch from ($3::timestamptz-$2::timestamptz))/$4)*1000000)::bigint*interval '1 microsecond'
and mr.completed_at < case when b.bucket=$4-1 then $3+interval '1 microsecond' else least($3+interval '1 microsecond',$2+ceil((b.bucket+1)*greatest(1,extract(epoch from ($3::timestamptz-$2::timestamptz))/$4)*1000000)::bigint*interval '1 microsecond') end
and mr.outcome <> 'running' and mr.provider_observation_json is not null
order by mr.completed_at desc,mr.id desc limit 1
) mr order by mr.completed_at";
@@ -14,8 +14,8 @@ pub(super) struct Histogram {
pub sum: f64,
pub minimum: Option<f64>,
pub maximum: Option<f64>,
zero: u64,
bins: BTreeMap<i32, u64>,
pub(super) zero: u64,
pub(super) bins: BTreeMap<i32, u64>,
}
impl Histogram {
@@ -0,0 +1,371 @@
//! 同事务撤销旧事实并合并新事实,构建中的统计只更新已经扫描的前缀
use super::buckets::write_bucket_groups;
use super::worker::collections;
use super::*;
use sqlx::postgres::PgRow;
type GroupKey = (DateTime<Utc>, String, Vec<String>, String);
#[derive(Default)]
struct Change {
previous: MetricState,
current: MetricState,
}
struct Progress {
phase: i16,
started_at: Option<DateTime<Utc>>,
id: Option<String>,
collection: Option<String>,
dimensions: Option<Vec<String>>,
outcome: Option<String>,
}
impl Progress {
fn contains_fact(&self, time: DateTime<Utc>, id: &str) -> bool {
match self.phase {
0 => false,
1 => self
.started_at
.zip(self.id.as_deref())
.is_some_and(|cursor| (time, id) <= cursor),
_ => true,
}
}
fn contains_quarter(&self, key: &GroupKey) -> bool {
if self.phase != 4 {
return false;
}
match (
&self.started_at,
&self.collection,
&self.dimensions,
&self.outcome,
) {
(Some(time), Some(collection), Some(dimensions), Some(outcome)) => {
(&key.0, &key.1, &key.2, &key.3) <= (time, collection, dimensions, outcome)
}
_ => false,
}
}
}
pub(super) async fn update_summaries(
tx: &mut sqlx::Transaction<'_, Postgres>,
previous: &[PgRow],
current: &[PgRow],
) -> StoreResult<()> {
let mut hours = BTreeSet::new();
for row in previous.iter().chain(current) {
hours.insert(bucket_for(get(row, "started_at")?, 3600)?);
}
if hours.is_empty() {
return Ok(());
}
let progress_rows = sqlx::query("select p.* from unnest($1::timestamptz[]) h(start_at) join request_analytics_pending_buckets p on p.bucket_start >= h.start_at and p.bucket_start < h.start_at+interval '1 hour'")
.bind(hours.into_iter().collect::<Vec<_>>()).fetch_all(&mut **tx).await
.map_err(|error|postgres_unavailable("read incremental analytics stages",error))?;
let mut progress = BTreeMap::new();
for row in progress_rows {
let phase =
if get::<Option<i64>>(&row, "build_version")? == Some(get(&row, "change_version")?) {
get(&row, "build_phase")?
} else {
0
};
progress.insert(
get::<DateTime<Utc>>(&row, "bucket_start")?,
Progress {
phase,
started_at: get(&row, "build_started_at")?,
id: get(&row, "build_id")?,
collection: get(&row, "build_collection")?,
dimensions: get(&row, "build_dimensions")?,
outcome: get(&row, "build_outcome")?,
},
);
}
// 两种粒度共用一次事实解码,仍分别检查构建前缀,不能把未发布季度并入小时
let mut changes = [BTreeMap::<GroupKey, Change>::new(), BTreeMap::new()];
let clean_hours = progress.is_empty();
for (rows, adding) in [(previous, false), (current, true)] {
for row in rows {
let time = get(row, "started_at")?;
let quarter = bucket_for(time, 900)?;
let hour = bucket_for(time, 3600)?;
let id: String = get(row, "id")?;
let outcome: String = get(row, "outcome")?;
let mut state = MetricState::default();
state.add_row(row)?;
let forecast = forecast::state(row)?;
let mut hour_pending = progress.range(hour..hour + TimeDelta::hours(1)).peekable();
let hour_ready = hour_pending.peek().is_none();
let hour_owner =
hour_pending.find_map(|(_, stage)| (stage.phase >= 3).then_some(stage));
for (collection, dimensions) in collections(row)? {
let key = (quarter, collection.to_owned(), dimensions, outcome.clone());
let included = [
progress
.get(&quarter)
.is_none_or(|p| p.contains_fact(time, &id)),
!clean_hours
&& (hour_ready
|| hour_owner.is_some_and(|stage| stage.contains_quarter(&key))),
];
let hourly_key = (hour, key.1.clone(), key.2.clone(), key.3.clone());
for (index, key) in [key, hourly_key].into_iter().enumerate() {
if !included[index] {
continue;
}
let change = changes[index].entry(key).or_default();
let state = if collection == forecast::COLLECTION {
&forecast
} else {
&state
};
if adding {
change.current.merge(state)?;
} else {
change.previous.merge(state)?;
}
}
}
}
}
if clean_hours {
// 无重建中的时间桶时,小时贡献可直接合并季度贡献,避免逐事实再合并一遍全部指标
let [quarters, hours] = &mut changes;
for ((quarter, collection, dimensions, outcome), contribution) in quarters.iter() {
let key = (
bucket_for(*quarter, 3600)?,
collection.clone(),
dimensions.clone(),
outcome.clone(),
);
let hour = hours.entry(key).or_default();
hour.previous.merge(&contribution.previous)?;
hour.current.merge(&contribution.current)?;
}
}
for (index, changes) in changes.into_iter().enumerate() {
apply_changes(tx, index == 1, &progress, changes).await?;
}
Ok(())
}
async fn apply_changes(
tx: &mut sqlx::Transaction<'_, Postgres>,
hourly: bool,
progress: &BTreeMap<DateTime<Utc>, Progress>,
changes: BTreeMap<GroupKey, Change>,
) -> StoreResult<()> {
if changes.is_empty() {
return Ok(());
}
let table = if hourly {
"request_analytics_hourly"
} else {
"request_analytics_15m"
};
let keys = changes.keys().map(|(bucket,collection,dimensions,outcome)|serde_json::json!({"bucket_start":bucket,"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::<Vec<_>>();
let statement = format!(
"select a.* from {table} a join jsonb_to_recordset($1::jsonb) k(bucket_start timestamptz,collection text,dimensions text[],outcome text) using(bucket_start,collection,dimensions,outcome)"
);
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(serde_json::Value::Array(keys))
.fetch_all(&mut **tx)
.await
.map_err(|error| postgres_unavailable("read incremental analytics groups", error))?;
let mut stored = BTreeMap::new();
for row in rows {
stored.insert(
(
get(&row, "bucket_start")?,
get(&row, "collection")?,
get(&row, "dimensions")?,
get(&row, "outcome")?,
),
MetricState::decode(&row)?,
);
}
let mut buckets =
BTreeMap::<DateTime<Utc>, BTreeMap<(String, Vec<String>, String), MetricState>>::new();
for (key, change) in changes {
if change.previous.encode()? == change.current.encode()?
&& change.previous.last_used_at() == change.current.last_used_at()
{
continue;
}
let mut state = stored.remove(&key).unwrap_or_default();
let needs_rebuild = subtract(&mut state, &change.previous, &change.current)?;
state.merge(&change.current)?;
if state.count("request_count") == 0 {
let statement = format!(
"delete from {table} where bucket_start=$1 and collection=$2 and dimensions=$3 and outcome=$4"
);
sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(key.0)
.bind(&key.1)
.bind(&key.2)
.bind(&key.3)
.execute(&mut **tx)
.await
.map_err(|error| postgres_unavailable("remove empty analytics group", error))?;
} else {
// 预测状态只有可加计数和美元总额,零金额无需修复币种存在性
if needs_rebuild && key.1 != forecast::COLLECTION {
state = rebuild_group(tx, hourly, progress, &key).await?;
}
buckets
.entry(key.0)
.or_default()
.insert((key.1, key.2, key.3), state);
}
}
write_bucket_groups(tx, table, buckets).await
}
/// 返回是否撤销了不能从可加状态推导的极值或币种存在性
pub(super) fn subtract(
state: &mut MetricState,
previous: &MetricState,
current: &MetricState,
) -> StoreResult<bool> {
if previous.count("request_count") == 0 {
return Ok(false);
}
// 新贡献仍持有旧极值时无需重读事实,例如只补齐 Token、费用或重放同一请求
let mut rebuild = previous.last_used_at.is_some()
&& previous.last_used_at == state.last_used_at
&& current.last_used_at < state.last_used_at;
for (name, value) in &previous.counts {
let count = state.counts.entry(name.clone()).or_default();
*count = count
.checked_sub(*value)
.ok_or_else(|| invalid("analytics counter underflow"))?;
}
for (name, removed) in &previous.distributions {
let distribution = state
.distributions
.get_mut(name)
.ok_or_else(|| invalid("missing analytics distribution"))?;
distribution.count = distribution
.count
.checked_sub(removed.count)
.ok_or_else(|| invalid("analytics distribution underflow"))?;
distribution.zero = distribution
.zero
.checked_sub(removed.zero)
.ok_or_else(|| invalid("analytics zero count underflow"))?;
distribution.sum -= removed.sum;
for (bin, value) in &removed.bins {
let count = distribution
.bins
.get_mut(bin)
.ok_or_else(|| invalid("missing analytics bin"))?;
*count = count
.checked_sub(*value)
.ok_or_else(|| invalid("analytics bin underflow"))?;
}
distribution.bins.retain(|_, count| *count > 0);
if distribution.count == 0 {
*distribution = Default::default();
} else {
let replacement = current.distributions.get(name);
rebuild |= distribution.minimum == removed.minimum
&& replacement
.and_then(|value| value.minimum)
.is_none_or(|value| Some(value) > distribution.minimum);
rebuild |= distribution.maximum == removed.maximum
&& replacement
.and_then(|value| value.maximum)
.is_none_or(|value| Some(value) < distribution.maximum);
rebuild |= !distribution.sum.is_finite() || distribution.sum < 0.0;
}
}
for (currency, amount) in &previous.costs {
let stored_amount = state
.costs
.get(currency)
.ok_or_else(|| invalid("missing analytics currency"))?;
let stored_amount: gateway_core::metering::Decimal = stored_amount
.parse()
.map_err(|_| invalid("invalid analytics amount"))?;
let removed: gateway_core::metering::Decimal = amount
.parse()
.map_err(|_| invalid("invalid analytics amount"))?;
let value = stored_amount
.scaled()
.checked_sub(removed.scaled())
.ok_or_else(|| invalid("analytics amount underflow"))?;
rebuild |= value == 0 && !current.costs.contains_key(currency);
state.costs.insert(
currency.clone(),
gateway_core::metering::Decimal::from_scaled(value)
.map_err(|_| invalid("analytics amount overflow"))?
.canonical(),
);
}
Ok(rebuild)
}
async fn rebuild_group(
tx: &mut sqlx::Transaction<'_, Postgres>,
hourly: bool,
progress: &BTreeMap<DateTime<Utc>, Progress>,
key: &GroupKey,
) -> StoreResult<MetricState> {
let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 });
let mut cursor = None;
if hourly {
if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4)
&& let Some(time) = owner.started_at
{
let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone());
end = if owner.contains_quarter(&candidate) {
time + TimeDelta::minutes(15)
} else {
time
};
}
} else if let Some(stage) = progress.get(&key.0)
&& stage.phase == 1
{
cursor = stage.started_at.zip(stage.id.as_deref());
}
let mut query = QueryBuilder::<Postgres>::new("with matching as materialized (select mr.*,");
query.push_bind(key.0).push("::timestamptz as output_bucket,''::text as name,coalesce((")
.push(completed_usage_fact_predicate("mr")).push("),false) as is_completed_usage from request_analytics_facts mr where mr.started_at >= ")
.push_bind(key.0).push(" and mr.started_at < ").push_bind(end)
.push(" and mr.recovered_at is null and mr.outcome=").push_bind(&key.3);
if let Some((time, id)) = cursor {
query
.push(" and (mr.started_at,mr.id collate \"C\") <= (")
.push_bind(time)
.push(",")
.push_bind(id)
.push(")");
}
let columns = worker::COLLECTION_COLUMNS
.iter()
.find_map(|(collection, columns)| (*collection == key.1).then_some(*columns))
.ok_or_else(|| invalid("unknown incremental analytics collection"))?;
for (column, value) in columns.iter().zip(&key.2) {
query
.push(format!(" and coalesce(mr.{column}::text,'')="))
.push_bind(value);
}
query.push(")");
aggregate::push_fact_aggregation(&mut query, false, true);
let row = query
.build()
.fetch_optional(&mut **tx)
.await
.map_err(|error| postgres_unavailable("rebuild changed analytics extrema", error))?;
row.as_ref()
.map(MetricState::decode)
.transpose()
.map(Option::unwrap_or_default)
}
@@ -5,13 +5,25 @@ use super::histogram::Histogram;
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgRow;
pub(super) const TOKEN_METRICS: [&str; 8] = [
"input_tokens",
"output_tokens",
"cached_input_tokens",
"cache_write_input_tokens",
"reasoning_output_tokens",
"total_tokens",
"image_input_tokens",
"image_output_tokens",
];
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub(super) struct MetricState {
counts: BTreeMap<String, u64>,
distributions: BTreeMap<String, Histogram>,
costs: BTreeMap<String, String>,
pub(crate) struct MetricState {
pub(super) counts: BTreeMap<String, u64>,
pub(super) distributions: BTreeMap<String, Histogram>,
pub(super) costs: BTreeMap<String, String>,
pub(super) last_used_at: Option<DateTime<Utc>>,
#[serde(skip)]
exact_samples: Option<BTreeMap<String, Vec<f64>>>,
exact_percentiles: Option<BTreeMap<String, Vec<Option<f64>>>>,
}
impl MetricState {
@@ -25,9 +37,23 @@ impl MetricState {
"distributions",
)?
.0,
exact_samples: None,
last_used_at: get(row, "last_used_at")?,
exact_percentiles: None,
})
}
pub(super) fn decode_aggregate(row: &PgRow, exact: bool) -> StoreResult<Self> {
let mut state = Self::decode(row)?;
if exact {
state.exact_percentiles = Some(
get::<sqlx::types::Json<BTreeMap<String, Vec<Option<f64>>>>>(
row,
"exact_percentiles",
)?
.0,
);
}
Ok(state)
}
pub fn encode(&self) -> StoreResult<(serde_json::Value, serde_json::Value, serde_json::Value)> {
Ok((
serde_json::to_value(&self.counts)
@@ -38,17 +64,24 @@ impl MetricState {
.map_err(|error| postgres_unavailable("encode analytics histograms", error))?,
))
}
pub fn exact() -> Self {
Self {
exact_samples: Some(BTreeMap::new()),
..Self::default()
}
}
pub fn count(&self, name: &str) -> u64 {
self.counts.get(name).copied().unwrap_or_default()
}
fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> {
let count = self.counts.entry(name.to_owned()).or_default();
pub fn optional_count(&self, name: &str) -> Option<u64> {
(self.count(&format!("{name}_known_count")) > 0).then(|| self.count(name))
}
pub fn last_used_at(&self) -> Option<DateTime<Utc>> {
self.last_used_at
}
pub(super) fn has_exact_percentiles(&self) -> bool {
self.exact_percentiles.is_some()
}
pub(super) fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> {
// 合并同一分组时键通常已经存在,避免每条事实都为已有指标分配字符串
let count = match self.counts.get_mut(name) {
Some(count) => count,
None => self.counts.entry(name.to_owned()).or_default(),
};
*count = count
.checked_add(amount)
.ok_or_else(|| invalid("analytics counter overflow"))?;
@@ -60,13 +93,10 @@ impl MetricState {
.entry(name.to_owned())
.or_default()
.add(sample)?;
if let Some(samples) = &mut self.exact_samples {
samples.entry(name.to_owned()).or_default().push(sample);
}
}
Ok(())
}
fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> {
pub(super) fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> {
let amount: DecimalAmount = amount
.parse()
.map_err(|_| invalid("analytics invalid cost"))?;
@@ -83,19 +113,28 @@ impl MetricState {
Ok(())
}
pub fn merge(&mut self, other: &Self) -> StoreResult<()> {
// 精确分位数只属于数据库已经完成分组的结果,不能把分位数再平均或合并
if self.count("request_count") > 0
&& other.count("request_count") > 0
&& (self.exact_percentiles.is_some() || other.exact_percentiles.is_some())
{
return Err(invalid(
"cannot merge finalized exact analytics percentiles",
));
}
if self.count("request_count") == 0 {
self.exact_percentiles.clone_from(&other.exact_percentiles);
}
self.last_used_at = self.last_used_at.max(other.last_used_at);
for (name, count) in &other.counts {
self.increment(name, *count)?;
}
for (name, distribution) in &other.distributions {
self.distributions
.entry(name.clone())
.or_default()
.merge(distribution)?;
}
if let (Some(ours), Some(theirs)) = (&mut self.exact_samples, &other.exact_samples) {
for (name, samples) in theirs {
ours.entry(name.clone()).or_default().extend(samples);
}
let stored = match self.distributions.get_mut(name) {
Some(stored) => stored,
None => self.distributions.entry(name.clone()).or_default(),
};
stored.merge(distribution)?;
}
for (currency, amount) in &other.costs {
self.add_cost(currency, amount)?;
@@ -185,18 +224,31 @@ impl MetricState {
self.sample("capacity", Some(used as f64 / total as f64 * 10_000.0))?;
}
if completed {
for name in [
"input_tokens",
"output_tokens",
"cached_input_tokens",
"cache_write_input_tokens",
"reasoning_output_tokens",
"total_tokens",
"image_input_tokens",
"image_output_tokens",
self.increment("completed_request_count", 1)?;
self.last_used_at = self.last_used_at.max(Some(get(row, "started_at")?));
for (value, name) in [
(Some(true), "image_request_count"),
(Some(false), "image_request_failed_count"),
] {
self.increment(name, optional_unsigned(row, name)?.unwrap_or_default())?;
if get::<Option<bool>>(row, "image_generation_succeeded")? == value {
self.increment(name, 1)?;
}
}
for name in TOKEN_METRICS {
if let Some(value) = optional_unsigned(row, name)? {
self.increment(name, value)?;
self.increment(&format!("{name}_known_count"), 1)?;
}
}
self.increment(
"window_total_tokens",
optional_unsigned(row, "total_tokens")?.unwrap_or(
optional_unsigned(row, "input_tokens")?
.unwrap_or_default()
.checked_add(optional_unsigned(row, "output_tokens")?.unwrap_or_default())
.ok_or_else(|| invalid("analytics token overflow"))?,
),
)?;
if get::<Option<i64>>(row, "input_tokens")?.is_some() {
self.increment("cache_eligible_request_count", 1)?;
if optional_unsigned(row, "cached_input_tokens")?.is_some_and(|value| value > 0) {
@@ -245,13 +297,11 @@ impl MetricState {
self.distributions.get(name).cloned().unwrap_or_default()
}
pub fn percentile(&self, name: &str, fraction: f64) -> Option<f64> {
if let Some(samples) = &self.exact_samples {
let mut samples = samples.get(name)?.clone();
samples.sort_by(f64::total_cmp);
let rank = samples.len().saturating_sub(1) as f64 * fraction;
let a = *samples.get(rank.floor() as usize)?;
let b = *samples.get(rank.ceil() as usize)?;
return Some(a + (b - a) * rank.fract());
if let Some(percentiles) = &self.exact_percentiles {
let index = [0.1, 0.5, 0.9, 0.95, 0.99]
.iter()
.position(|value| (*value - fraction).abs() < f64::EPSILON)?;
return percentiles.get(name)?.get(index).copied().flatten();
}
self.histogram(name).percentile(fraction)
}
@@ -1,7 +1,12 @@
//! 请求观测的持久化投影、可重建汇总与受预算约束的读取
pub(crate) mod accounts;
mod aggregate;
mod buckets;
mod counts;
mod forecast;
mod histogram;
mod incremental;
mod metrics;
mod query;
mod reset;
@@ -11,15 +16,15 @@ pub(crate) use query::*;
pub use worker::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
use super::*;
use metrics::MetricState;
pub(crate) use metrics::MetricState;
const SCHEMA_VERSION: i32 = 1;
const HISTOGRAM_VERSION: i32 = histogram::HISTOGRAM_VERSION;
const MAX_FACT_ROWS: usize = 100_000;
pub(super) async fn begin_snapshot(
connection: &mut PgConnection,
connection: &mut ObservabilityConnection,
) -> StoreResult<sqlx::Transaction<'_, Postgres>> {
let query_timeout = connection.query_timeout();
let mut transaction = connection
.begin()
.await
@@ -28,7 +33,8 @@ pub(super) async fn begin_snapshot(
.execute(&mut *transaction)
.await
.map_err(|error| postgres_unavailable("isolate analytics snapshot", error))?;
sqlx::query("set local statement_timeout = '10s'")
sqlx::query("select set_config('statement_timeout',$1,true)")
.bind(format!("{}ms", query_timeout.as_millis()))
.execute(&mut *transaction)
.await
.map_err(|error| postgres_unavailable("set analytics query deadline", error))?;
@@ -5,7 +5,7 @@ use futures::TryStreamExt;
pub(crate) struct AnalyticsSnapshot<'a> {
pub tx: sqlx::Transaction<'a, Postgres>,
status: AnalyticsStatus,
pub(super) status: AnalyticsStatus,
approximate: bool,
}
@@ -13,10 +13,25 @@ struct ReadSelection<'a> {
distributions: bool,
costs: bool,
keys: Option<&'a [String]>,
counters: Option<&'a [&'a str]>,
distribution_names: Option<&'a [&'a str]>,
}
impl<'a> AnalyticsSnapshot<'a> {
pub async fn new(connection: &'a mut PgConnection) -> StoreResult<Self> {
pub async fn new(connection: &'a mut ObservabilityConnection) -> StoreResult<Self> {
Self::begin(connection, true).await
}
pub(crate) async fn new_for_accounts(
connection: &'a mut ObservabilityConnection,
) -> StoreResult<Self> {
Self::begin(connection, false).await
}
async fn begin(
connection: &'a mut ObservabilityConnection,
require_current_schema: bool,
) -> StoreResult<Self> {
let mut tx = begin_snapshot(connection).await?;
// 时间跨度和分组选择性差异很大,通用计划会把长区间误估成短区间
// 仅当前只读事务使用实际参数规划,不改变连接池后续事务的计划策略
@@ -25,8 +40,9 @@ impl<'a> AnalyticsSnapshot<'a> {
.await
.map_err(|error| postgres_unavailable("plan analytics with actual bounds", error))?;
let status = worker::query_status(&mut tx).await?;
if status.schema_version != SCHEMA_VERSION as u32
|| status.histogram_version != HISTOGRAM_VERSION as u32
if require_current_schema
&& (status.schema_version != SCHEMA_VERSION as u32
|| status.histogram_version != HISTOGRAM_VERSION as u32)
{
return Err(unavailable(
"request analytics is building; retry after the background projection completes",
@@ -163,14 +179,35 @@ impl<'a> AnalyticsSnapshot<'a> {
.load(range, filter, None, None, Default::default(), true)
.await?;
let state = merge_states(states.into_values())?;
self.overview_from_state(range, filter, state).await
}
async fn overview_from_state(
&mut self,
range: ObservabilityRange,
filter: &UsageRecordFilter,
state: MetricState,
) -> StoreResult<UsageOverview> {
// Provider 列表只消费计数,不重复合并费用与分布
let provider_states = self
.load(
.load_selected(
range,
filter,
Some(DiagnosticDimension::Provider),
None,
Default::default(),
false,
ReadSelection {
distributions: false,
costs: false,
keys: None,
counters: Some(&[
"request_count",
"attempt_count",
"failure_count",
"total_tokens",
]),
distribution_names: None,
},
)
.await?;
let mut providers = provider_states
@@ -216,47 +253,48 @@ impl<'a> AnalyticsSnapshot<'a> {
distributions: true,
costs: include_costs,
keys: None,
counters: None,
distribution_names: None,
},
)
.await?;
let mut points = BTreeMap::new();
for ((bucket, _), state) in states {
points.insert(
bucket,
RequestMetricPoint {
bucket_start: bucket,
granularity,
metrics: state.request_metrics()?,
cost_coverage: state.coverage(),
costs: if include_costs {
state.costs()?
} else {
Vec::new()
},
},
);
}
fill_metric_gaps(range, granularity, points, timezone)
metric_points(range, granularity, timezone, &states, include_costs)
}
pub async fn projected_costs(
pub async fn insights(
&mut self,
range: ObservabilityRange,
filter: &UsageRecordFilter,
granularity: ObservationGranularity,
timezone: gateway_core::time::DeploymentTimeZone,
) -> StoreResult<Vec<admin_observability::UsageProjectedCostPoint>> {
self.load(range, filter, None, Some(granularity), timezone, false)
.await?
.into_iter()
) -> StoreResult<(
UsageOverview,
Vec<RequestMetricPoint>,
Vec<admin_observability::UsageProjectedCostPoint>,
)> {
let states = self
.load(range, filter, None, Some(granularity), timezone, true)
.await?;
let trend = metric_points(range, granularity, timezone, &states, true)?;
let costs = states
.iter()
.map(|((bucket_start, _), state)| {
Ok(admin_observability::UsageProjectedCostPoint {
bucket_start,
bucket_start: *bucket_start,
standard_amount: state.projected_cost("standard_usd")?,
no_cache_amount: state.projected_cost("no_cache_usd")?,
})
})
.collect()
.collect::<StoreResult<Vec<_>>>()?;
// 趋势已包含可合并的完整指标和情景费用,概览直接复用同一份状态
// 精确分位数不可跨桶合并,明细回退时仍按完整区间单独计算概览
let overview = if states.values().any(MetricState::has_exact_percentiles) {
self.overview(range, filter).await?
} else {
self.overview_from_state(range, filter, merge_states(states.into_values())?)
.await?
};
Ok((overview, trend, costs))
}
pub async fn diagnostics(
@@ -273,8 +311,10 @@ impl<'a> AnalyticsSnapshot<'a> {
let ranked = if matches!(
dimension,
DiagnosticDimension::Account | DiagnosticDimension::ApiKey
) && collection_for(filter, Some(dimension)).is_some()
{
) && matches!(
collection_for(filter, Some(dimension)),
Some("account" | "key")
) {
Some(
self.diagnostic_top_keys(range, filter, dimension, limit)
.await?,
@@ -294,6 +334,22 @@ impl<'a> AnalyticsSnapshot<'a> {
distributions: true,
costs: true,
keys: selected_keys,
counters: Some(&[
"request_count",
"success_count",
"failure_count",
"attempt_count",
"total_tokens",
"latency_ms_sum",
"cancelled_count",
"incomplete_count",
"retry_count",
"retried_request_count",
"provider_reported_count",
"calculated_count",
"unavailable_count",
]),
distribution_names: Some(&["latency_ms", "first_token_ms"]),
},
)
.await?;
@@ -415,16 +471,18 @@ impl<'a> AnalyticsSnapshot<'a> {
.collect::<Vec<_>>();
let mut query = QueryBuilder::<Postgres>::new("with edge_facts as materialized (");
if starts.is_empty() {
query.push("select null::text as key where false");
query.push("select null::text as key,0::bigint as request_count where false");
} else {
query.push("select edge.key from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ")
.push(format!("coalesce(mr.{column},'') as key from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at"));
query.push("select edge.key,sum(edge.request_count)::bigint as request_count from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ")
.push(format!("coalesce(mr.{column},'') as key,count(*)::bigint as request_count from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at"));
push_unrecovered_request_filter(&mut query, "mr");
push_analytics_filter(&mut query, filter, "mr");
push_diagnostic_dimension_filter(&mut query, dimension);
query.push(" limit 100001) edge limit 100001");
query.push(format!(
" group by coalesce(mr.{column},'')) edge group by edge.key"
));
}
query.push("), contributions as (select key,count(*)::bigint as request_count from edge_facts group by key");
query.push("), contributions as (select key,request_count from edge_facts");
for (table, column, seconds, buckets) in [
("request_analytics_daily_counts", "dimension", 86400, days),
("request_analytics_hourly", "dimensions[1]", 3600, hours),
@@ -446,7 +504,7 @@ impl<'a> AnalyticsSnapshot<'a> {
}
query.push(format!(" group by {column}"));
}
query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count,(select count(*) from edge_facts)::bigint as edge_count from grouped order by request_count desc,");
query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count from grouped order by request_count desc,");
if dimension == DiagnosticDimension::Account {
query.push("coalesce(nullif(key,''),'unrouted')");
} else {
@@ -458,12 +516,6 @@ impl<'a> AnalyticsSnapshot<'a> {
.fetch_all(&mut *self.tx)
.await
.map_err(|error| postgres_unavailable("rank full-range analytics groups", error))?;
if rows.first().is_some_and(|row| {
row.try_get::<i64, _>("edge_count")
.is_ok_and(|count| count > MAX_FACT_ROWS as i64)
}) {
return Err(unavailable("analytics fact row budget exceeded"));
}
let total = rows
.first()
.map(|row| unsigned(row, "total_request_count"))
@@ -476,18 +528,15 @@ impl<'a> AnalyticsSnapshot<'a> {
Ok((total, keys))
}
async fn summary_buckets(
pub(super) async fn summary_buckets(
&mut self,
range: ObservabilityRange,
use_hourly: bool,
) -> StoreResult<(Vec<DateTime<Utc>>, Vec<DateTime<Utc>>)> {
// 先按时间确定互斥的完整桶,空的十五分钟集合不读取汇总表
let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start limit 100001")
let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start")
.bind(range.start).bind(range.end).bind(use_hourly)
.fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("select disjoint analytics buckets",error))?;
if rows.len() > MAX_FACT_ROWS {
return Err(unavailable("analytics time bucket budget exceeded"));
}
let mut hours = Vec::new();
let mut quarters = Vec::new();
for row in rows {
@@ -502,7 +551,7 @@ impl<'a> AnalyticsSnapshot<'a> {
Ok((hours, quarters))
}
async fn uncovered_spans(
pub(super) async fn uncovered_spans(
&mut self,
range: ObservabilityRange,
) -> StoreResult<Vec<(DateTime<Utc>, DateTime<Utc>)>> {
@@ -522,11 +571,8 @@ impl<'a> AnalyticsSnapshot<'a> {
spans.push((right_start, range.end));
}
if complete_start < complete_end {
let dirty:Vec<DateTime<Utc>>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start limit 100001")
let dirty:Vec<DateTime<Utc>>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start")
.bind(complete_start).bind(complete_end).fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("plan bounded analytics fact spans",error))?;
if dirty.len() > MAX_FACT_ROWS {
return Err(unavailable("analytics dirty interval budget exceeded"));
}
spans.extend(
dirty
.into_iter()
@@ -534,7 +580,17 @@ impl<'a> AnalyticsSnapshot<'a> {
);
}
spans.sort_unstable();
Ok(spans)
let mut merged: Vec<(DateTime<Utc>, DateTime<Utc>)> = Vec::new();
for (start, end) in spans {
if let Some(previous) = merged.last_mut()
&& previous.1 == start
{
previous.1 = end;
} else {
merged.push((start, end));
}
}
Ok(merged)
}
async fn load(
@@ -556,6 +612,8 @@ impl<'a> AnalyticsSnapshot<'a> {
distributions: read_distributions,
costs: true,
keys: None,
counters: None,
distribution_names: None,
},
)
.await
@@ -574,6 +632,8 @@ impl<'a> AnalyticsSnapshot<'a> {
distributions: read_distributions,
costs: read_costs,
keys: selected_keys,
counters,
distribution_names,
} = selection;
filter.validate()?;
self.require_visible(Some(range)).await?;
@@ -610,90 +670,63 @@ impl<'a> AnalyticsSnapshot<'a> {
if buckets.is_empty() {
continue;
}
let (dimension_filter, either_filter, either_positions) = match collection {
"provider_model" => {
(filter.provider_kind.clone(), filter.model.clone(), (2, 3))
}
"account" => (filter.provider_account_ref.clone(), None, (1, 2)),
"key" => (filter.client_api_key_ref.clone(), None, (1, 2)),
"operation" => (filter.operation.clone(), None, (1, 2)),
"transport" => (None, filter.transport.clone(), (1, 2)),
"status" => (
None,
filter.status_code.map(|value| value.to_string()),
(1, 2),
),
_ => (None, None, (1, 2)),
let mut query =
QueryBuilder::<Postgres>::new("with matching as not materialized (select ");
aggregate::push_output_bucket(
&mut query,
"bucket_start",
range,
granularity,
boundaries.as_deref(),
);
let name = aggregate::summary_dimension_sql(collection, dimension);
let costs = if read_costs {
"costs"
} else {
"'{}'::jsonb as costs"
};
let distributions = if read_distributions {
"distributions"
} else {
"'{}'::jsonb as distributions"
};
let costs = if read_costs {
"costs"
} else {
"'{}'::jsonb as costs"
};
let provider_counts = dimension == Some(DiagnosticDimension::Provider)
&& granularity.is_none()
&& !read_distributions;
let mut query = QueryBuilder::<Postgres>::new(if provider_counts {
"select min(bucket_start) as bucket_start, array[dimensions[1]] as dimensions, ''::text as outcome, jsonb_build_object('request_count',sum((counts->>'request_count')::bigint),'attempt_count',sum((counts->>'attempt_count')::bigint),'failure_count',sum(coalesce((counts->>'failure_count')::bigint,0)),'total_tokens',sum(coalesce((counts->>'total_tokens')::bigint,0))) as counts, '{}'::jsonb as costs, '{}'::jsonb as distributions from ("
} else {
""
});
query.push(format!("select bucket_start, dimensions, outcome, counts, {costs}, {distributions} from {table} a where collection="))
query.push(format!(" as output_bucket,{name} as name,counts,{costs},{distributions},last_used_at from {table} a where collection="))
.push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION)
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION);
push_summary_buckets(&mut query, &buckets, seconds);
// 可选筛选仅在有值时生成谓词,保留维度与时间索引的选择空间
if let Some(outcome) = &filter.outcome {
query.push(" and outcome=").push_bind(outcome);
}
if let Some(value) = dimension_filter {
query.push(" and dimensions[1]=").push_bind(value);
}
if let Some(value) = either_filter {
query
.push(format!(" and (dimensions[{}]=", either_positions.0))
.push_bind(value.clone())
.push(format!(" or dimensions[{}]=", either_positions.1))
.push_bind(value)
.push(")");
}
push_collection_filter(&mut query, collection, filter);
if let Some(keys) = selected_keys {
query
.push(" and dimensions[1]=any(")
.push_bind(keys)
.push(")");
}
if provider_counts {
query.push(") matching group by dimensions[1]");
if matches!(
dimension,
Some(
DiagnosticDimension::Model
| DiagnosticDimension::Status
| DiagnosticDimension::Failure
)
) {
query.push(format!(" and ({name})<>''"));
}
query.push(" limit 100001");
query.push(")");
aggregate::push_summary_aggregation(
&mut query,
read_distributions,
read_costs,
counters,
distribution_names,
);
let mut rows = query.build().fetch(&mut *self.tx);
let mut count = 0;
while let Some(row) = rows
.try_next()
.await
.map_err(|error| postgres_unavailable("read analytics summaries", error))?
.map_err(|error| postgres_unavailable("merge analytics summaries", error))?
{
count += 1;
if count > MAX_FACT_ROWS {
return Err(unavailable("analytics aggregate row budget exceeded"));
}
let dimensions: Vec<String> = get(&row, "dimensions")?;
let name = dimension_name(collection, &dimensions, dimension);
let Some(name) = name else {
continue;
};
let bucket = output_bucket(
get(&row, "bucket_start")?,
range,
granularity,
boundaries.as_deref(),
)?;
let bucket = get(&row, "bucket_start")?;
let name = get(&row, "name")?;
let state = MetricState::decode(&row)?;
result.entry((bucket, name)).or_default().merge(&state)?;
}
@@ -707,28 +740,42 @@ impl<'a> AnalyticsSnapshot<'a> {
if spans.is_empty() {
return Ok(result);
}
// 单段直接绑定边界,让优化器看见真实跨度,避免把三十天误估成小范围 BitmapAnd
// 单段直接绑定边界,多个互斥区间用索引读取,所有统计都在数据库内完成
let single_span = spans.len() == 1;
let mut query = if single_span {
let mut query = QueryBuilder::<Postgres>::new(fact_select());
query
.push(" where mr.started_at >= ")
.push_bind(spans[0].0)
.push(" and mr.started_at < ")
.push_bind(spans[0].1);
query
} else {
let (starts, ends): (Vec<_>, Vec<_>) = spans.into_iter().unzip();
let mut query = QueryBuilder::<Postgres>::new("select fact.* from unnest(");
let mut query = QueryBuilder::<Postgres>::new("with matching as not materialized (");
if !single_span {
let (starts, ends): (Vec<_>, Vec<_>) = spans.iter().copied().unzip();
query
.push("select fact.* from unnest(")
.push_bind(starts)
.push("::timestamptz[],")
.push_bind(ends)
.push("::timestamptz[]) span(start_at,end_at) cross join lateral (")
.push(fact_select())
.push(" where mr.started_at >= span.start_at and mr.started_at < span.end_at");
.push("::timestamptz[]) span(start_at,end_at) cross join lateral (");
}
query.push(format!(
"select mr.*,coalesce(({}),false) as is_completed_usage,",
completed_usage_fact_predicate("mr")
));
aggregate::push_output_bucket(
&mut query,
"mr.started_at",
range,
granularity,
boundaries.as_deref(),
);
query.push(format!(
" as output_bucket,{} as name from request_analytics_facts mr where ",
aggregate::fact_dimension_sql(dimension)
));
if single_span {
query
};
.push("mr.started_at >= ")
.push_bind(spans[0].0)
.push(" and mr.started_at < ")
.push_bind(spans[0].1);
} else {
query.push("mr.started_at >= span.start_at and mr.started_at < span.end_at");
}
push_unrecovered_request_filter(&mut query, "mr");
push_analytics_filter(&mut query, filter, "mr");
if let Some(dimension) = dimension {
@@ -749,42 +796,40 @@ impl<'a> AnalyticsSnapshot<'a> {
}
query.push(")");
}
query.push(" limit 100001");
if !single_span {
query.push(") fact limit 100001");
// 保留逐区间的参数化索引扫描,避免展开 LATERAL 后全表读取再过滤
// OFFSET 0 不限制返回行数
query.push(" offset 0) fact");
}
query.push(")");
let exact = selected.is_none();
aggregate::push_fact_aggregation_selected(
&mut query,
exact,
read_costs,
read_distributions,
counters,
distribution_names,
);
let mut rows = query.build().fetch(&mut *self.tx);
let mut count = 0;
while let Some(row) = rows
.try_next()
.await
.map_err(|error| postgres_unavailable("read bounded analytics facts", error))?
.map_err(|error| postgres_unavailable("aggregate analytics facts", error))?
{
count += 1;
if count > MAX_FACT_ROWS {
return Err(unavailable("analytics fact row budget exceeded"));
let key = (get(&row, "bucket_start")?, get(&row, "name")?);
let state = MetricState::decode_aggregate(&row, exact)?;
if exact {
result.insert(key, state);
} else {
result.entry(key).or_default().merge(&state)?;
}
let bucket = output_bucket(
get(&row, "started_at")?,
range,
granularity,
boundaries.as_deref(),
)?;
let name = fact_dimension(&row, dimension)?;
let state = result.entry((bucket, name)).or_insert_with(|| {
if selected.is_some() {
MetricState::default()
} else {
MetricState::exact()
}
});
state.add_row(&row)?;
}
Ok(result)
}
}
fn push_summary_buckets(
pub(super) fn push_summary_buckets(
query: &mut QueryBuilder<Postgres>,
buckets: &[DateTime<Utc>],
seconds: i64,
@@ -810,6 +855,33 @@ fn push_summary_buckets(
}
}
fn metric_points(
range: ObservabilityRange,
granularity: ObservationGranularity,
timezone: gateway_core::time::DeploymentTimeZone,
states: &BTreeMap<(DateTime<Utc>, String), MetricState>,
include_costs: bool,
) -> StoreResult<Vec<RequestMetricPoint>> {
let mut points = BTreeMap::new();
for ((bucket, _), state) in states {
points.insert(
*bucket,
RequestMetricPoint {
bucket_start: *bucket,
granularity,
metrics: state.request_metrics()?,
cost_coverage: state.coverage(),
costs: if include_costs {
state.costs()?
} else {
Vec::new()
},
},
);
}
fill_metric_gaps(range, granularity, points, timezone)
}
fn merge_states(states: impl Iterator<Item = MetricState>) -> StoreResult<MetricState> {
let mut result: Option<MetricState> = None;
for state in states {
@@ -822,28 +894,7 @@ fn merge_states(states: impl Iterator<Item = MetricState>) -> StoreResult<Metric
Ok(result.unwrap_or_default())
}
fn output_bucket(
time: DateTime<Utc>,
range: ObservabilityRange,
granularity: Option<ObservationGranularity>,
boundaries: Option<&[DateTime<Utc>]>,
) -> StoreResult<DateTime<Utc>> {
match granularity {
None => Ok(range.start),
Some(ObservationGranularity::Day) => boundaries
.and_then(|values| {
values
.iter()
.rev()
.find(|boundary| **boundary <= time)
.copied()
})
.ok_or_else(|| invalid("missing analytics calendar boundary")),
Some(granularity) => bucket_for(time, granularity.seconds()),
}
}
fn collection_for(
pub(super) fn collection_for(
filter: &UsageRecordFilter,
dimension: Option<DiagnosticDimension>,
) -> Option<&'static str> {
@@ -852,7 +903,6 @@ fn collection_for(
|| filter.response_id.is_some()
|| filter.upstream_request_id.is_some()
|| filter.search.is_some()
|| !filter.routing_group_refs.is_empty()
{
return None;
}
@@ -885,85 +935,18 @@ fn collection_for(
DiagnosticDimension::Status => "status",
});
}
if required.len() > 1 {
if required.len() == 2 && required.contains("provider_model") && required.contains("account") {
Some("account_model")
} else if required.len() == 2 && required.contains("provider_model") && required.contains("key")
{
Some("key_model")
} else if required.len() > 1 {
None
} else {
Some(required.first().copied().unwrap_or("global"))
}
}
fn dimension_name(
collection: &str,
values: &[String],
dimension: Option<DiagnosticDimension>,
) -> Option<String> {
let Some(dimension) = dimension else {
return Some(String::new());
};
let value = |index| values.get(index).cloned().unwrap_or_default();
let coalesce = |preferred: usize, fallback: usize| {
let v = value(preferred);
if v.is_empty() { value(fallback) } else { v }
};
let name = match dimension {
DiagnosticDimension::Provider if collection == "provider_model" => {
let v = value(0);
if v.is_empty() {
"unrouted".to_owned()
} else {
v
}
}
DiagnosticDimension::Model => coalesce(2, 1),
DiagnosticDimension::Account => {
let v = value(0);
if v.is_empty() {
"unrouted".to_owned()
} else {
v
}
}
DiagnosticDimension::Transport => {
let v = coalesce(1, 0);
if v.is_empty() {
"unknown".to_owned()
} else {
v
}
}
DiagnosticDimension::Status => coalesce(1, 0),
_ => value(0),
};
if name.is_empty()
&& matches!(
dimension,
DiagnosticDimension::Model | DiagnosticDimension::Status | DiagnosticDimension::Failure
)
{
None
} else {
Some(name)
}
}
fn fact_dimension(
row: &sqlx::postgres::PgRow,
dimension: Option<DiagnosticDimension>,
) -> StoreResult<String> {
let Some(dimension) = dimension else {
return Ok(String::new());
};
let collection = collection_for(&UsageRecordFilter::default(), Some(dimension))
.ok_or_else(|| invalid("missing analytics dimension"))?;
let dimensions = worker::collections(row)?
.into_iter()
.find(|(name, _)| *name == collection)
.map(|(_, values)| values)
.ok_or_else(|| invalid("missing analytics dimension"))?;
dimension_name(collection, &dimensions, Some(dimension))
.ok_or_else(|| invalid("unexpected empty analytics dimension"))
}
pub(crate) fn push_analytics_filter(
query: &mut QueryBuilder<Postgres>,
filter: &UsageRecordFilter,
@@ -982,3 +965,53 @@ pub(crate) fn push_analytics_filter(
query.push(")");
}
}
pub(super) fn push_collection_filter(
query: &mut QueryBuilder<Postgres>,
collection: &str,
filter: &UsageRecordFilter,
) {
let (dimension_filter, either_filter, either_positions) = match collection {
"provider_model" => (filter.provider_kind.clone(), filter.model.clone(), (2, 3)),
"account_model" => (
filter.provider_account_ref.clone(),
filter.model.clone(),
(3, 4),
),
"key_model" => (
filter.client_api_key_ref.clone(),
filter.model.clone(),
(3, 4),
),
"account" => (filter.provider_account_ref.clone(), None, (1, 2)),
"key" => (filter.client_api_key_ref.clone(), None, (1, 2)),
"operation" => (filter.operation.clone(), None, (1, 2)),
"transport" => (None, filter.transport.clone(), (1, 2)),
"status" => (
None,
filter.status_code.map(|value| value.to_string()),
(1, 2),
),
_ => (None, None, (1, 2)),
};
// 可选筛选仅在有值时生成谓词,保留维度与时间索引的选择空间
if let Some(outcome) = &filter.outcome {
query.push(" and outcome=").push_bind(outcome);
}
if let Some(value) = dimension_filter {
query.push(" and dimensions[1]=").push_bind(value);
}
if matches!(collection, "account_model" | "key_model")
&& let Some(provider) = &filter.provider_kind
{
query.push(" and dimensions[2]=").push_bind(provider);
}
if let Some(value) = either_filter {
query
.push(format!(" and (dimensions[{}]=", either_positions.0))
.push_bind(value.clone())
.push(format!(" or dimensions[{}]=", either_positions.1))
.push_bind(value)
.push(")");
}
}
@@ -83,11 +83,11 @@ impl PgRequestAnalyticsRepository {
return Ok(AnalyticsMaintenance { reset_progress: true, ..Default::default() });
}
let mut result = AnalyticsMaintenance::default();
let rows = sqlx::query("select request_id, change_version, oldest_pending_at, previous_bucket from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
let rows = sqlx::query("select request_id, change_version from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
.bind(i64::from(batch_size)).fetch_all(&mut *transaction).await.map_err(|error| postgres_unavailable("load analytics request queue", error))?;
let mut jobs = Vec::with_capacity(rows.len());
for row in rows {
jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")?, pending_at: get(&row,"oldest_pending_at")?, previous_bucket: get(&row,"previous_bucket")? });
jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")? });
}
self.project_batch(&mut transaction, &jobs).await?;
result.projected_requests = jobs.len() as u64;
@@ -138,53 +138,36 @@ impl PgRequestAnalyticsRepository {
}
let ids = jobs.iter().map(|job| job.id.clone()).collect::<Vec<_>>();
let versions = jobs.iter().map(|job| job.version).collect::<Vec<_>>();
let pending = jobs
.iter()
.map(|job| (job.id.as_str(), job.pending_at))
.collect::<HashMap<_, _>>();
let mut buckets = BTreeMap::<DateTime<Utc>, DateTime<Utc>>::new();
let mut affect = |id: &str, start: DateTime<Utc>| -> StoreResult<()> {
let pending_at = pending
.get(id)
.copied()
.ok_or_else(|| invalid("analytics job disappeared"))?;
let bucket = bucket_for(start, 900)?;
buckets
.entry(bucket)
.and_modify(|old| *old = (*old).min(pending_at))
.or_insert(pending_at);
Ok(())
};
let rows = sqlx::query(
"delete from request_analytics_facts where id=any($1::text[]) returning id,started_at",
)
let previous = sqlx::query(sqlx::AssertSqlSafe(format!(
"{} where mr.id=any($1::text[])",
fact_select()
)))
.bind(&ids)
.fetch_all(&mut **tx)
.await
.map_err(|error| postgres_unavailable("replace analytics facts", error))?;
for row in rows {
affect(&get::<String>(&row, "id")?, get(&row, "started_at")?)?;
}
let rows = sqlx::query(PROJECT_REQUEST)
.map_err(|error| postgres_unavailable("read previous analytics contributions", error))?;
sqlx::query("delete from request_analytics_facts where id=any($1::text[])")
.bind(&ids)
.execute(&mut **tx)
.await
.map_err(|error| postgres_unavailable("replace analytics facts", error))?;
sqlx::query(PROJECT_REQUEST)
.bind(&ids)
.bind(versions)
.fetch_all(&mut **tx)
.execute(&mut **tx)
.await
.map_err(|error| postgres_unavailable("project analytics facts", error))?;
for row in rows {
affect(&get::<String>(&row, "id")?, get(&row, "started_at")?)?;
}
for job in jobs {
if let Some(bucket) = job.previous_bucket {
affect(&job.id, bucket)?;
}
}
self.project_billing_batch(tx, &ids).await?;
if !buckets.is_empty() {
let (starts, times): (Vec<_>, Vec<_>) = buckets.into_iter().unzip();
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select * from unnest($1::timestamptz[],$2::timestamptz[]) on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version'),oldest_pending_at=least(request_analytics_pending_buckets.oldest_pending_at,excluded.oldest_pending_at)")
.bind(starts).bind(times).execute(&mut **tx).await.map_err(|error| postgres_unavailable("enqueue analytics buckets",error))?;
}
let current = sqlx::query(sqlx::AssertSqlSafe(format!(
"{} where mr.id=any($1::text[])",
fact_select()
)))
.bind(&ids)
.fetch_all(&mut **tx)
.await
.map_err(|error| postgres_unavailable("read current analytics contributions", error))?;
// 事实、已发布汇总与构建前缀在同一事务中撤销和添加,重试不会重复计数
super::incremental::update_summaries(tx, &previous, &current).await?;
Ok(())
}
@@ -199,7 +182,7 @@ impl PgRequestAnalyticsRepository {
return Ok(0);
}
let mut query = QueryBuilder::<Postgres>::new(
"select id,started_at,nextval('request_analytics_change_version') as version from model_requests",
"select id,started_at,nextval('request_analytics_change_version') as version,exists(select 1 from request_analytics_facts f where f.id=model_requests.id) as projected from model_requests",
);
// 首批与后续批次使用两种明确的计划,避免可空游标的 OR 使通用计划退化为全索引过滤
if let Some(start) = get::<Option<DateTime<Utc>>>(&state, "backfill_cursor_started_at")? {
@@ -222,11 +205,12 @@ impl PgRequestAnalyticsRepository {
.map_err(|error| postgres_unavailable("read analytics backfill batch", error))?;
let mut jobs = Vec::with_capacity(rows.len());
for row in &rows {
if get::<bool>(row, "projected")? {
continue;
}
jobs.push(ProjectionJob {
id: get(row, "id")?,
version: get(row, "version")?,
pending_at: Utc::now(),
previous_bucket: None,
});
}
self.project_batch(tx, &jobs).await?;
@@ -312,49 +296,68 @@ async fn read_status(
})
}
// 投影与极值重算共用同一维度顺序,避免两条路径对集合身份作不同解释
pub(super) const COLLECTION_COLUMNS: &[(&str, &[&str])] = &[
("global", &[]),
(
"provider_model",
&["provider_kind", "requested_model_id", "upstream_model_id"],
),
("account", &["provider_account_ref"]),
("key", &["client_api_key_ref"]),
(
"account_model",
&[
"provider_account_ref",
"provider_kind",
"requested_model_id",
"upstream_model_id",
],
),
(
"key_model",
&[
"client_api_key_ref",
"provider_kind",
"requested_model_id",
"upstream_model_id",
],
),
("operation", &["operation"]),
("transport", &["client_transport", "upstream_transport"]),
("status", &["client_status_code", "upstream_status_code"]),
("failure", &["error_kind"]),
];
pub(super) fn collections(
row: &sqlx::postgres::PgRow,
) -> StoreResult<Vec<(&'static str, Vec<String>)>> {
let text =
|name| -> StoreResult<String> { Ok(get::<Option<String>>(row, name)?.unwrap_or_default()) };
let status = |name| -> StoreResult<String> {
Ok(get::<Option<i32>>(row, name)?
.map(|value| value.to_string())
.unwrap_or_default())
};
Ok(vec![
("global", vec![]),
(
"provider_model",
vec![
text("provider_kind")?,
text("requested_model_id")?,
text("upstream_model_id")?,
],
),
("account", vec![text("provider_account_ref")?]),
("key", vec![text("client_api_key_ref")?]),
("operation", vec![text("operation")?]),
(
"transport",
vec![text("client_transport")?, text("upstream_transport")?],
),
(
"status",
vec![
status("client_status_code")?,
status("upstream_status_code")?,
],
),
("failure", vec![text("error_kind")?]),
])
let mut result = vec![(forecast::COLLECTION, forecast::dimensions(row)?)];
// 预测保留原始完成状态,包括被恢复的请求;其他统计继续排除恢复记录
if get::<Option<DateTime<Utc>>>(row, "recovered_at")?.is_some() {
return Ok(result);
}
for &(collection, columns) in COLLECTION_COLUMNS {
let dimensions = columns
.iter()
.map(|column| {
if matches!(*column, "client_status_code" | "upstream_status_code") {
Ok(get::<Option<i32>>(row, column)?
.map(|value| value.to_string())
.unwrap_or_default())
} else {
Ok(get::<Option<String>>(row, column)?.unwrap_or_default())
}
})
.collect::<StoreResult<Vec<_>>>()?;
result.push((collection, dimensions));
}
Ok(result)
}
struct ProjectionJob {
id: String,
version: i64,
pending_at: DateTime<Utc>,
previous_bucket: Option<DateTime<Utc>>,
}
async fn publish_generation(
@@ -17,7 +17,6 @@ pub(crate) fn store_range(
pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter {
UsageRecordFilter {
routing_group_refs: filter.routing_group_refs,
client_api_key_ref: filter.client_api_key_ref,
request_id: filter.request_id,
provider_account_ref: filter.provider_account_ref,
@@ -31,6 +31,10 @@ mod queries;
mod query_budget;
pub use admin_adapter::*;
pub(crate) use analytics::AnalyticsSnapshot;
pub(crate) use analytics::accounts::{
account_group_usage, account_usage_sort_values, account_usage_windows,
};
pub use analytics::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
pub(crate) use mapping::*;
pub use model::*;
@@ -39,7 +39,6 @@ pub(crate) fn observability_page_offset(
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct UsageRecordFilter {
pub routing_group_refs: Vec<String>,
pub client_api_key_ref: Option<String>,
pub request_id: Option<String>,
pub provider_account_ref: Option<String>,
@@ -76,7 +75,6 @@ impl UsageRecordFilter {
] {
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
}
validate_account_ids_if_present(&self.routing_group_refs)?;
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
if self
.status_code
@@ -105,14 +103,6 @@ pub struct UsageRecordQuery {
pub use gateway_admin::model::observability::OpsErrorFilter;
pub(crate) fn validate_ops_error_filter(filter: &OpsErrorFilter) -> StoreResult<()> {
if filter.routing_group_refs.len() > 32
|| filter.routing_group_refs.iter().any(|value| {
value.is_empty() || value.len() > 128 || value.chars().any(char::is_control)
})
{
return Err(invalid("invalid routing group filter"));
}
for (value, field) in [
(
filter.client_api_key_ref.as_deref(),
@@ -384,10 +374,3 @@ pub struct DiagnosticObservation {
pub use gateway_admin::model::observability::OpsError as OpsErrorRecord;
pub use gateway_admin::model::observability::OpsErrorPage;
fn validate_account_ids_if_present(ids: &[String]) -> StoreResult<()> {
if ids.is_empty() {
return Ok(());
}
validate_account_ids(ids)
}
@@ -3,7 +3,7 @@
use super::super::*;
pub(crate) async fn provider_account_usage(
connection: &mut sqlx::PgConnection,
snapshot: &mut analytics::AnalyticsSnapshot<'_>,
query: ProviderAccountUsageQuery,
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
if let Some(account_ids) = &query.account_ids {
@@ -13,290 +13,133 @@ pub(crate) async fn provider_account_usage(
return Err(invalid("account usage limit must be between 1 and 200"));
}
// 账号卡片先确定有限候选,避免为只展示二十个账号聚合整个历史范围
let mut statement = QueryBuilder::<Postgres>::new(
"with candidate_accounts as materialized (select pa.* from provider_accounts pa",
);
// 账号卡片先确定有限候选,避免为少量账号聚合整个历史范围
let mut statement = QueryBuilder::<Postgres>::new("select pa.* from provider_accounts pa");
if let Some(account_ids) = &query.account_ids {
statement
.push(" where pa.id=any(")
.push_bind(account_ids.clone())
.push("::text[])");
} else {
statement.push(" join lateral (select mr.started_at from request_analytics_facts mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start)
statement.push(" join lateral (select mr.started_at from model_requests mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start)
.push(" and mr.started_at < ").push_bind(query.range.end);
push_completed_usage_fact_filter(&mut statement, "mr");
statement.push(" order by mr.started_at desc,mr.id desc limit 1) latest on true order by latest.started_at desc,pa.name,pa.id limit ").push_bind(i64::from(query.limit));
}
statement.push(
"), matched as materialized (
select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
pa.plan_type, mr.id as request_id,
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
mr.cost_currency, mr.outcome, mr.input_tokens, mr.output_tokens,
mr.cached_input_tokens, mr.cache_write_input_tokens, mr.reasoning_output_tokens,
mr.image_input_tokens, mr.image_output_tokens,
mr.image_generation_succeeded, mr.total_tokens, mr.cost_source,
mr.cost_amount, mr.started_at
from candidate_accounts pa
left join request_analytics_facts mr
on mr.provider_account_ref = pa.id and mr.started_at >= ",
);
statement.push_bind(query.range.start);
statement.push(" and mr.started_at < ");
statement.push_bind(query.range.end);
push_completed_usage_fact_filter(&mut statement, "mr");
if let Some(account_ids) = &query.account_ids {
statement.push(" where pa.id = any(");
statement.push_bind(account_ids.clone());
statement.push("::text[])");
}
statement.push(
" limit 100001), aggregated as (
select id, provider_kind, authentication_kind, name, email, plan_type,
model, cost_currency,
grouping(model)::integer as model_grouping,
grouping(cost_currency)::integer as currency_grouping,
count(request_id)::bigint as request_count,
count(request_id) filter (where outcome = 'succeeded')::bigint
as success_count,
sum(input_tokens)::bigint as input_tokens,
sum(output_tokens)::bigint as output_tokens,
sum(cached_input_tokens)::bigint as cached_input_tokens,
sum(cache_write_input_tokens)::bigint as cache_write_input_tokens,
sum(reasoning_output_tokens)::bigint as reasoning_output_tokens,
sum(image_input_tokens)::bigint as image_input_tokens,
sum(image_output_tokens)::bigint as image_output_tokens,
count(request_id) filter (where image_generation_succeeded is true)::bigint
as image_request_count,
count(request_id) filter (where image_generation_succeeded is false)::bigint
as image_request_failed_count,
sum(total_tokens)::bigint as total_tokens,
count(request_id) filter (where cost_source = 'provider_reported')::bigint
as provider_reported_count,
count(request_id) filter (where cost_source = 'calculated')::bigint
as calculated_count,
count(request_id) filter (where cost_source = 'unavailable')::bigint
as unavailable_count,
max(started_at) as last_used_at,
sum(cost_amount)::text as amount
from matched
group by id, provider_kind, authentication_kind, name, email, plan_type,
grouping sets ((), (cost_currency), (model), (model, cost_currency))
), selected_accounts as (
select id,
row_number() over (order by last_used_at desc nulls last, name, id)
as sort_position
from aggregated
where model_grouping = 1 and currency_grouping = 1",
);
if query.account_ids.is_none() {
statement.push(" and request_count > 0");
}
statement.push(" order by last_used_at desc nulls last, name, id limit ");
statement.push_bind(i64::from(query.limit));
statement.push(
")
select aggregated.*, (select count(*) from matched)::bigint as matched_count
from aggregated
join selected_accounts selected using (id)
order by selected.sort_position, model_grouping desc, currency_grouping desc,
model nulls last, cost_currency nulls last",
);
let rows = statement
.build()
.fetch_all(&mut *connection)
.fetch_all(&mut *snapshot.tx)
.await
.map_err(|source| postgres_unavailable("load provider account usage", source))?;
if let Some(row) = rows.first()
&& unsigned(row, "matched_count")? > 100_000
.map_err(|error| postgres_unavailable("select usage accounts", error))?;
let account_ids = rows
.iter()
.map(|row| get(row, "id"))
.collect::<StoreResult<Vec<String>>>()?;
let mut models_by_account = HashMap::<String, Vec<ProviderAccountModelUsageObservation>>::new();
let mut totals = BTreeMap::<String, analytics::MetricState>::new();
for ((account, model), state) in snapshot
.account_usage_states(query.range, &account_ids)
.await?
{
return Err(crate::StoreError::Unavailable {
backend: crate::StoreBackend::PostgreSql,
message: "account analytics fact row budget exceeded".to_owned(),
source: None,
});
}
let mut observations = Vec::with_capacity(usize::from(query.limit));
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
let mut models = HashMap::<String, Vec<ProviderAccountModelUsageObservation>>::new();
let mut model_costs = HashMap::<(String, String), Vec<CurrencyCostTotal>>::new();
for row in &rows {
let model_grouping: i32 = get(row, "model_grouping")?;
let currency_grouping: i32 = get(row, "currency_grouping")?;
match (model_grouping, currency_grouping) {
(1, 1) => observations.push(provider_account_from_row(row)?),
(1, 0) if get::<Option<String>>(row, "cost_currency")?.is_some() => {
costs
.entry(get(row, "id")?)
.or_default()
.push(cost_from_row(row)?);
}
(0, 1) if get::<Option<String>>(row, "model")?.is_some() => {
models
.entry(get(row, "id")?)
.or_default()
.push(provider_account_model_from_row(row)?);
}
(0, 0)
if get::<Option<String>>(row, "model")?.is_some()
&& get::<Option<String>>(row, "cost_currency")?.is_some() =>
{
model_costs
.entry((get(row, "id")?, get(row, "model")?))
.or_default()
.push(cost_from_row(row)?);
}
(0 | 1, 0 | 1) => {}
_ => {
return Err(crate::StoreError::Unavailable {
backend: crate::StoreBackend::PostgreSql,
message: "decode provider account usage grouping".to_owned(),
source: None,
});
}
totals.entry(account.clone()).or_default().merge(&state)?;
if !model.is_empty()
&& let Some(last_used_at) = state.last_used_at()
{
models_by_account.entry(account).or_default().push(
ProviderAccountModelUsageObservation {
model,
request_count: state.count("completed_request_count"),
success_count: state.count("completed_request_count"),
input_tokens: state.optional_count("input_tokens"),
output_tokens: state.optional_count("output_tokens"),
cached_input_tokens: state.optional_count("cached_input_tokens"),
cache_write_input_tokens: state.optional_count("cache_write_input_tokens"),
reasoning_output_tokens: state.optional_count("reasoning_output_tokens"),
image_input_tokens: state.optional_count("image_input_tokens"),
image_output_tokens: state.optional_count("image_output_tokens"),
total_tokens: state.optional_count("total_tokens"),
image_request_count: state.count("image_request_count"),
image_request_failed_count: state.count("image_request_failed_count"),
cost_coverage: state.coverage(),
costs: state.costs()?,
last_used_at,
},
);
}
}
if observations.is_empty() {
return Ok(observations);
}
let account_ids = observations
.iter()
.map(|item| item.account_id.clone())
.collect::<Vec<_>>();
let mut request_buckets = if let Some(range) = query.request_bucket_range {
provider_account_request_buckets(connection, range, &account_ids).await?
provider_account_request_buckets(snapshot, range, &account_ids).await?
} else {
HashMap::new()
};
for observation in &mut observations {
observation.costs = costs.remove(&observation.account_id).unwrap_or_default();
observation.request_buckets = request_buckets
.remove(&observation.account_id)
.unwrap_or_default();
observation.models = models.remove(&observation.account_id).unwrap_or_default();
for model in &mut observation.models {
model.costs = model_costs
.remove(&(observation.account_id.clone(), model.model.clone()))
.unwrap_or_default();
}
observation.models.sort_by(|left, right| {
right
.request_count
.cmp(&left.request_count)
.then_with(|| left.model.cmp(&right.model))
let mut observations = Vec::with_capacity(rows.len());
for row in rows {
let account_id: String = get(&row, "id")?;
let state = totals.remove(&account_id).unwrap_or_default();
let mut models = models_by_account.remove(&account_id).unwrap_or_default();
models.sort_by(|a, b| {
b.request_count
.cmp(&a.request_count)
.then(a.model.cmp(&b.model))
});
observations.push(ProviderAccountUsageObservation {
provider_kind: get(&row, "provider_kind")?,
authentication_kind: get(&row, "authentication_kind")?,
name: get(&row, "name")?,
email: get(&row, "email")?,
plan_type: get(&row, "plan_type")?,
request_count: state.count("completed_request_count"),
success_count: state.count("completed_request_count"),
input_tokens: state.optional_count("input_tokens"),
output_tokens: state.optional_count("output_tokens"),
cached_input_tokens: state.optional_count("cached_input_tokens"),
cache_write_input_tokens: state.optional_count("cache_write_input_tokens"),
reasoning_output_tokens: state.optional_count("reasoning_output_tokens"),
image_input_tokens: state.optional_count("image_input_tokens"),
image_output_tokens: state.optional_count("image_output_tokens"),
total_tokens: state.optional_count("total_tokens"),
image_request_count: state.count("image_request_count"),
image_request_failed_count: state.count("image_request_failed_count"),
cost_coverage: state.coverage(),
costs: state.costs()?,
last_used_at: state.last_used_at(),
request_buckets: request_buckets.remove(&account_id).unwrap_or_default(),
models,
account_id,
});
}
observations.sort_by(|a, b| {
b.last_used_at
.cmp(&a.last_used_at)
.then(a.name.cmp(&b.name))
.then(a.account_id.cmp(&b.account_id))
});
observations.truncate(usize::from(query.limit));
Ok(observations)
}
pub(crate) async fn provider_account_request_buckets(
connection: &mut sqlx::PgConnection,
snapshot: &mut analytics::AnalyticsSnapshot<'_>,
range: ObservabilityRange,
account_ids: &[String],
) -> StoreResult<HashMap<String, Vec<ProviderAccountRequestBucket>>> {
let completed_usage = completed_usage_fact_predicate("mr");
let statement = format!(
"select provider_account_ref,
floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index,
count(*)::bigint as request_count
from request_analytics_facts mr
where mr.provider_account_ref = any($2::text[])
and mr.started_at >= $1 and mr.started_at < $3
and {completed_usage}
group by mr.provider_account_ref, bucket_index
order by mr.provider_account_ref, bucket_index"
);
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(range.start)
.bind(account_ids)
.bind(range.end)
.fetch_all(&mut *connection)
.await
.map_err(|source| postgres_unavailable("load provider account request timeline", source))?;
let mut observed = HashMap::<String, BTreeMap<u64, u64>>::new();
for row in rows {
observed
.entry(get(&row, "provider_account_ref")?)
.or_default()
.insert(
unsigned(&row, "bucket_index")?,
unsigned(&row, "request_count")?,
);
}
let mut observed = snapshot.account_request_counts(range, account_ids).await?;
let mut timelines = HashMap::with_capacity(account_ids.len());
for account_id in account_ids {
let mut account_observed = observed.remove(account_id).unwrap_or_default();
let mut buckets = Vec::new();
for bucket_index in 0..ACCOUNT_USAGE_TIMELINE_HOURS {
let bucket_start = range
for account in account_ids {
let mut buckets = Vec::with_capacity(ACCOUNT_USAGE_TIMELINE_HOURS as usize);
for index in 0..ACCOUNT_USAGE_TIMELINE_HOURS {
let start = range
.start
.checked_add_signed(TimeDelta::hours(bucket_index))
.checked_add_signed(TimeDelta::hours(index))
.ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?;
buckets.push(ProviderAccountRequestBucket {
bucket_start,
request_count: account_observed
.remove(&(bucket_index as u64))
bucket_start: start,
request_count: observed
.remove(&(account.clone(), index))
.unwrap_or_default(),
});
}
timelines.insert(account_id.clone(), buckets);
timelines.insert(account.clone(), buckets);
}
Ok(timelines)
}
fn provider_account_model_from_row(
row: &sqlx::postgres::PgRow,
) -> StoreResult<ProviderAccountModelUsageObservation> {
Ok(ProviderAccountModelUsageObservation {
model: get(row, "model")?,
request_count: unsigned(row, "request_count")?,
success_count: unsigned(row, "success_count")?,
input_tokens: optional_unsigned(row, "input_tokens")?,
output_tokens: optional_unsigned(row, "output_tokens")?,
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
image_request_count: unsigned(row, "image_request_count")?,
image_request_failed_count: unsigned(row, "image_request_failed_count")?,
total_tokens: optional_unsigned(row, "total_tokens")?,
cost_coverage: coverage_from_row(row)?,
costs: Vec::new(),
last_used_at: get(row, "last_used_at")?,
})
}
pub(crate) fn provider_account_from_row(
row: &sqlx::postgres::PgRow,
) -> StoreResult<ProviderAccountUsageObservation> {
Ok(ProviderAccountUsageObservation {
account_id: get(row, "id")?,
provider_kind: get(row, "provider_kind")?,
authentication_kind: get(row, "authentication_kind")?,
name: get(row, "name")?,
email: get(row, "email")?,
plan_type: get(row, "plan_type")?,
request_count: unsigned(row, "request_count")?,
success_count: unsigned(row, "success_count")?,
input_tokens: optional_unsigned(row, "input_tokens")?,
output_tokens: optional_unsigned(row, "output_tokens")?,
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
image_request_count: unsigned(row, "image_request_count")?,
image_request_failed_count: unsigned(row, "image_request_failed_count")?,
total_tokens: optional_unsigned(row, "total_tokens")?,
cost_coverage: coverage_from_row(row)?,
costs: Vec::new(),
last_used_at: get(row, "last_used_at")?,
request_buckets: Vec::new(),
models: Vec::new(),
})
}
@@ -1,15 +1,7 @@
//! 时间桶补齐、日历边界与共享费用覆盖映射
//! 时间桶补齐与部署时区的日历边界
use super::super::*;
pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CostCoverage> {
Ok(CostCoverage {
provider_reported_count: unsigned(row, "provider_reported_count")?,
calculated_count: unsigned(row, "calculated_count")?,
unavailable_count: unsigned(row, "unavailable_count")?,
})
}
pub(crate) fn fill_metric_gaps(
range: ObservabilityRange,
granularity: ObservationGranularity,
@@ -69,38 +69,29 @@ left join model_request_observations mr on mr.id = oe.model_request_id
where true";
pub(crate) async fn list_ops_errors(
connection: &mut PgConnection,
connection: &mut ObservabilityConnection,
query: OpsErrorQuery,
) -> StoreResult<OpsErrorPage> {
validate_ops_error_filter(&query.filter)?;
let mut transaction = analytics::begin_snapshot(connection).await?;
let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?;
let offset = observability_page_offset(query.current_page, query.page_size)?;
let mut statement =
QueryBuilder::<Postgres>::new("with candidate_page as materialized (select * from (");
statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true");
push_request_error_predicates(&mut statement, query.range, &query.filter);
statement.push(" union all ");
statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true");
push_ops_event_predicates(&mut statement, query.range, &query.filter);
statement.push(") e order by occurred_at desc, stable_sort_id desc limit ");
statement.push_bind(i64::from(query.page_size.get()));
statement.push(" offset ");
statement.push_bind(offset);
statement.push(") select details.* from candidate_page page cross join lateral (");
statement.push(REQUEST_ERROR_SELECT);
statement.push(" and page.source='model_request' and mr.id=page.event_id union all ");
statement.push(OPS_EVENT_SELECT);
statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc");
let rows = statement
.build()
.fetch_all(&mut *transaction)
.await
.map_err(|source| postgres_unavailable("list ops errors", source))?;
let items = rows
.iter()
.map(ops_error_from_row)
.collect::<StoreResult<Vec<_>>>()?;
let (total, page) = if query.current_page == 1 {
let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?;
(
total,
(total > 0).then_some(TimePage {
range: query.range,
offset,
total: Some(total),
}),
)
} else {
ops_time_page(&mut transaction, &query, offset).await?
};
let items = match page {
Some(page) => list_ops_error_items_at(&mut transaction, &query, page).await?,
None => Vec::new(),
};
transaction
.commit()
.await
@@ -113,6 +104,95 @@ pub(crate) async fn list_ops_errors(
})
}
async fn list_ops_error_items_at(
connection: &mut PgConnection,
query: &OpsErrorQuery,
page: TimePage,
) -> StoreResult<Vec<OpsErrorRecord>> {
let total =
i64::try_from(page.total.unwrap_or(0)).map_err(|_| invalid("Ops count overflows i64"))?;
let limit = i64::from(query.page_size.get()).min(total - page.offset);
let reverse_offset = total - page.offset - limit;
let ascending = reverse_offset < page.offset;
let offset = if ascending {
reverse_offset
} else {
page.offset
};
let branch_limit = offset + limit;
let order = if ascending { " asc" } else { " desc" };
let mut statement =
QueryBuilder::<Postgres>::new("with candidate_page as materialized (select * from ((");
statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true");
push_request_error_predicates(&mut statement, page.range, &query.filter);
statement
.push(" order by mr.completed_at")
.push(order)
.push(", mr.id")
.push(order);
statement.push(" limit ").push_bind(branch_limit);
statement.push(") union all (");
statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true");
push_ops_event_predicates(&mut statement, page.range, &query.filter);
statement
.push(" order by oe.created_at")
.push(order)
.push(", oe.id")
.push(order);
statement.push(" limit ").push_bind(branch_limit);
statement
.push(")) e order by occurred_at")
.push(order)
.push(", stable_sort_id")
.push(order);
statement
.push(" limit ")
.push_bind(limit)
.push(" offset ")
.push_bind(offset);
statement.push(") select details.* from candidate_page page cross join lateral (");
statement.push(REQUEST_ERROR_SELECT);
statement.push(" and page.source='model_request' and mr.id=page.event_id union all ");
statement.push(OPS_EVENT_SELECT);
statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc");
let rows = statement
.build()
.fetch_all(connection)
.await
.map_err(|source| postgres_unavailable("list ops errors", source))?;
rows.iter().map(ops_error_from_row).collect()
}
async fn ops_time_page(
connection: &mut PgConnection,
query: &OpsErrorQuery,
offset: i64,
) -> StoreResult<(u64, Option<TimePage>)> {
// 两个来源先各自精确分段,合并分钟计数后再定位同一个全局页码
let mut statement = QueryBuilder::<Postgres>::new(
"select bucket_start, sum(request_count)::bigint as request_count from (
select date_bin(interval '1 minute', mr.completed_at, timestamptz 'epoch') as bucket_start,
count(*)::bigint as request_count
from model_request_observations mr where true",
);
push_request_error_predicates(&mut statement, query.range, &query.filter);
statement.push(" group by bucket_start union all ");
statement.push(
"select date_bin(interval '1 minute', oe.created_at, timestamptz 'epoch') as bucket_start,
count(*)::bigint as request_count
from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true",
);
push_ops_event_predicates(&mut statement, query.range, &query.filter);
statement
.push(" group by bucket_start) counts group by bucket_start order by bucket_start desc");
let rows = statement
.build()
.fetch_all(connection)
.await
.map_err(|source| postgres_unavailable("locate Ops page", source))?;
time_page_from_counts(&rows, query.range, offset, query.page_size)
}
pub(crate) async fn count_ops_errors(
connection: &mut sqlx::PgConnection,
range: ObservabilityRange,
@@ -144,7 +224,6 @@ fn push_request_error_predicates(
// 列表和总数共用此条件,避免流式响应中的错误因 outcome 被漏掉
statement.push(" and mr.error_kind is not null and mr.error_kind <> 'cancelled'");
push_range(statement, "mr.completed_at", range);
push_routing_groups(statement, &filter.routing_group_refs);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
("mr.id", &filter.request_id),
@@ -187,7 +266,6 @@ fn push_ops_event_predicates(
filter: &OpsErrorFilter,
) {
push_range(statement, "oe.created_at", range);
push_routing_groups(statement, &filter.routing_group_refs);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
("oe.model_request_id", &filter.request_id),
@@ -275,11 +353,3 @@ fn push_prefix_search(statement: &mut QueryBuilder<Postgres>, columns: &[&str],
.push(" escape '\\'");
statement.push(")");
}
fn push_routing_groups(statement: &mut QueryBuilder<Postgres>, groups: &[String]) {
if !groups.is_empty() {
statement.push(" and mr.routing_group_refs && ");
statement.push_bind(groups.to_vec());
statement.push("::text[]");
}
}
@@ -11,12 +11,6 @@ pub(crate) fn push_usage_filter(
filter: &UsageRecordFilter,
alias: &str,
) {
if !filter.routing_group_refs.is_empty() {
query
.push(format!(" and {alias}.routing_group_refs && "))
.push_bind(filter.routing_group_refs.clone())
.push("::text[]");
}
if let Some(value) = &filter.client_api_key_ref {
query.push(format!(" and {alias}.client_api_key_ref = "));
query.push_bind(value.clone());
@@ -86,16 +80,14 @@ pub(crate) fn push_usage_filter(
}
if let Some(value) = &filter.search {
let pattern = literal_prefix_pattern(value);
query.push(format!(" and ({alias}.id like "));
query.push(format!(" and ({alias}.requested_model_id like "));
query.push_bind(pattern.clone());
for column in [
"upstream_model_id",
"client_api_key_ref",
"provider_account_ref",
"provider_account_email_snapshot",
"provider_account_name_snapshot",
"requested_model_id",
"upstream_model_id",
"upstream_request_id",
] {
query.push(format!(" escape '\\' or {alias}.{column} like "));
query.push_bind(pattern.clone());
@@ -174,29 +166,50 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
from model_request_observations mr";
pub(crate) async fn list_usage_records(
connection: &mut PgConnection,
connection: &mut ObservabilityConnection,
query: UsageRecordQuery,
) -> StoreResult<UsageRecordPage> {
query.filter.validate()?;
let mut transaction = analytics::begin_snapshot(connection).await?;
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
// 第一页未填满时,当前快照中的行数已是精确总数,无需再次扫描计数
let (total, items) = if query.current_page == 1 {
let items = list_usage_record_items(&mut *transaction, &query, None).await?;
let items = list_usage_record_items(&mut *snapshot.tx, &query, None).await?;
let total = if items.len() < usize::from(query.page_size.get()) {
items.len() as u64
} else if let Some(counts) = snapshot
.completed_request_counts(query.range, &query.filter)
.await?
{
counts.values().try_fold(0_u64, |sum, count| {
sum.checked_add(*count)
.ok_or_else(|| invalid("usage count overflows u64"))
})?
} else {
count_usage_records(&mut *transaction, query.range, &query.filter).await?
count_usage_records(&mut *snapshot.tx, query.range, &query.filter).await?
};
(total, items)
} else {
let total = count_usage_records(&mut *transaction, query.range, &query.filter).await?;
let items = list_usage_record_items(&mut *transaction, &query, Some(total)).await?;
let (total, page) = if let Some(counts) = snapshot
.completed_request_counts(query.range, &query.filter)
.await?
{
time_page_from_segments(
counts.into_iter().rev(),
query.range,
observability_page_offset(query.current_page, query.page_size)?,
query.page_size,
TimeDelta::minutes(15),
)?
} else {
usage_time_page(&mut snapshot.tx, &query).await?
};
let items = match page {
Some(page) => list_usage_record_items_at(&mut *snapshot.tx, &query, page).await?,
None => Vec::new(),
};
(total, items)
};
transaction
.commit()
.await
.map_err(|error| postgres_unavailable("complete usage page snapshot", error))?;
snapshot.finish().await?;
Ok(UsageRecordPage {
items,
current_page: query.current_page,
@@ -209,12 +222,25 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database =
executor: E,
query: &UsageRecordQuery,
total: Option<u64>,
) -> StoreResult<Vec<UsageListRecord>> {
let page = TimePage {
range: query.range,
offset: observability_page_offset(query.current_page, query.page_size)?,
total,
};
list_usage_record_items_at(executor, query, page).await
}
async fn list_usage_record_items_at<'e, E: sqlx::Executor<'e, Database = Postgres>>(
executor: E,
query: &UsageRecordQuery,
page: TimePage,
) -> StoreResult<Vec<UsageListRecord>> {
query.filter.validate()?;
let mut offset = observability_page_offset(query.current_page, query.page_size)?;
let mut offset = page.offset;
let mut limit = i64::from(query.page_size.get());
let mut ascending = false;
if let Some(total) = total {
if let Some(total) = page.total {
let total = i64::try_from(total).map_err(|_| invalid("usage count overflows i64"))?;
if offset >= total {
return Ok(Vec::new());
@@ -231,9 +257,9 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database =
"with candidate_page as materialized (select mr.id, mr.started_at from model_request_observations mr",
);
statement.push(" where mr.started_at >= ");
statement.push_bind(query.range.start);
statement.push_bind(page.range.start);
statement.push(" and mr.started_at < ");
statement.push_bind(query.range.end);
statement.push_bind(page.range.end);
push_completed_usage_fact_filter(&mut statement, "mr");
push_usage_filter(&mut statement, &query.filter, "mr");
statement.push(if ascending {
@@ -261,6 +287,93 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database =
Ok(items)
}
// 页码保持全局精确排名,实际明细读取只落在覆盖该页的时间区间
pub(crate) struct TimePage {
pub(crate) range: ObservabilityRange,
pub(crate) offset: i64,
pub(crate) total: Option<u64>,
}
pub(crate) fn time_page_from_counts(
rows: &[sqlx::postgres::PgRow],
range: ObservabilityRange,
offset: i64,
page_size: ObservabilityPageSize,
) -> StoreResult<(u64, Option<TimePage>)> {
let counts = rows
.iter()
.map(|row| {
Ok((
get(row, "bucket_start")?,
to_u64(get(row, "request_count")?)?,
))
})
.collect::<StoreResult<Vec<_>>>()?;
time_page_from_segments(counts, range, offset, page_size, TimeDelta::minutes(1))
}
fn time_page_from_segments(
counts: impl IntoIterator<Item = (DateTime<Utc>, u64)>,
range: ObservabilityRange,
offset: i64,
page_size: ObservabilityPageSize,
width: TimeDelta,
) -> StoreResult<(u64, Option<TimePage>)> {
let offset = u64::try_from(offset).map_err(|_| invalid("negative page offset"))?;
let end = offset
.checked_add(u64::from(page_size.get()))
.ok_or_else(|| invalid("page range overflows u64"))?;
let mut total = 0_u64;
let mut page = None;
let mut selected_count = 0_u64;
for (bucket, count) in counts {
let next = total
.checked_add(count)
.ok_or_else(|| invalid("request count overflows u64"))?;
if next > offset && total < end {
let selected = page.get_or_insert_with(|| TimePage {
range: ObservabilityRange {
start: bucket.max(range.start),
end: (bucket + width).min(range.end),
},
offset: (offset - total) as i64,
total: None,
});
selected.range.start = bucket.max(range.start);
selected_count += count;
}
total = next;
}
if let Some(page) = &mut page {
page.total = Some(selected_count);
}
Ok((total, page))
}
async fn usage_time_page(
connection: &mut PgConnection,
query: &UsageRecordQuery,
) -> StoreResult<(u64, Option<TimePage>)> {
let offset = observability_page_offset(query.current_page, query.page_size)?;
let mut statement = QueryBuilder::<Postgres>::new(
"select date_bin(interval '1 minute', mr.started_at, timestamptz 'epoch') as bucket_start,
count(*)::bigint as request_count
from model_request_observations mr where mr.started_at >= ",
);
statement.push_bind(query.range.start);
statement.push(" and mr.started_at < ");
statement.push_bind(query.range.end);
push_completed_usage_fact_filter(&mut statement, "mr");
push_usage_filter(&mut statement, &query.filter, "mr");
statement.push(" group by bucket_start order by bucket_start desc");
let rows = statement
.build()
.fetch_all(connection)
.await
.map_err(|source| postgres_unavailable("locate usage page", source))?;
time_page_from_counts(&rows, query.range, offset, query.page_size)
}
pub(crate) async fn count_usage_records<'e, E: sqlx::Executor<'e, Database = Postgres>>(
executor: E,
range: ObservabilityRange,
@@ -17,12 +17,20 @@ use crate::{StoreBackend, StoreError, StoreResult, postgres_unavailable};
pub struct ObservabilityQueryBudget {
slots: Arc<Semaphore>,
wait_timeout: Duration,
query_timeout: Duration,
}
/// 观测名额与实际 PostgreSQL 连接共同存活,取消后仍覆盖协议收尾和事务回滚
pub struct ObservabilityConnection {
connection: PoolConnection<Postgres>,
permit: Option<OwnedSemaphorePermit>,
query_timeout: Duration,
}
impl ObservabilityConnection {
pub(crate) const fn query_timeout(&self) -> Duration {
self.query_timeout
}
}
impl Deref for ObservabilityConnection {
@@ -53,12 +61,22 @@ impl Drop for ObservabilityConnection {
impl ObservabilityQueryBudget {
/// 预算表示观测类 SQL 最多可同时占用的连接数
pub fn try_new(max_connections: u32, wait_timeout: Duration) -> StoreResult<Self> {
if max_connections == 0 || wait_timeout.is_zero() {
pub fn try_new(
max_connections: u32,
wait_timeout: Duration,
query_timeout: Duration,
) -> StoreResult<Self> {
if max_connections == 0
|| wait_timeout.is_zero()
|| query_timeout.is_zero()
|| query_timeout.as_millis() > i32::MAX as u128
{
return Err(StoreError::InvalidData {
source: None,
entity: "observability query budget",
message: "requires a positive connection count and wait timeout".to_owned(),
message:
"requires positive connection and timeout budgets within PostgreSQL limits"
.to_owned(),
});
}
let permits = usize::try_from(max_connections).map_err(|_| StoreError::InvalidData {
@@ -69,10 +87,16 @@ impl ObservabilityQueryBudget {
Ok(Self {
slots: Arc::new(Semaphore::new(permits)),
wait_timeout,
query_timeout,
})
}
/// 在共享槽位与十秒执行预算内完成整个观测操作
#[must_use]
pub const fn query_timeout(&self) -> Duration {
self.query_timeout
}
/// 在共享槽位与观测总预算内完成排队、连接获取和整个存储操作
///
/// # Errors
///
@@ -87,14 +111,15 @@ impl ObservabilityQueryBudget {
F: FnOnce(ObservabilityConnection) -> Q,
Q: Future<Output = StoreResult<T>>,
{
let permit = self.acquire(operation).await?;
tokio::time::timeout(Duration::from_secs(10), async {
tokio::time::timeout(self.query_timeout, async {
let permit = self.acquire(operation).await?;
let connection = pool.acquire().await.map_err(|source| {
postgres_unavailable("acquire observability connection", source)
})?;
query(ObservabilityConnection {
connection,
permit: Some(permit),
query_timeout: self.query_timeout,
})
.await
})
@@ -102,109 +102,17 @@ impl PgAdminAccountStore {
ObservabilityRange::new(window.range.start, window.range.end)
.map_err(|error| admin_store_error(ENTITY, error))?;
}
let keys = windows
.iter()
.map(|window| window.key.clone())
.collect::<Vec<_>>();
let starts = windows
.iter()
.map(|window| window.range.start)
.collect::<Vec<_>>();
let ends = windows
.iter()
.map(|window| window.range.end)
.collect::<Vec<_>>();
let account_ids = windows
.iter()
.map(|window| window.account_id.clone())
.collect::<Vec<_>>();
let rows = self
.query_budget
self.query_budget
.run(
&self.pool,
"load account usage windows",
|mut connection| async move {
sqlx::query(sqlx::AssertSqlSafe(account_usage_by_windows_sql()))
.bind(account_ids)
.bind(keys)
.bind(starts)
.bind(ends)
.fetch_all(&mut *connection)
crate::postgres::observability::account_usage_windows(&mut connection, windows)
.await
.map_err(|source| {
postgres_unavailable("load provider account quota window usage", source)
})
},
)
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
let mut usage_rows = Vec::with_capacity(windows.len());
let mut costs_by_window = BTreeMap::<(String, String), Vec<AccountCost>>::new();
let mut model_costs = BTreeMap::<(String, String, String), Vec<AccountCost>>::new();
let mut models_by_window = BTreeMap::<(String, String), Vec<AccountModelUsage>>::new();
for row in &rows {
let model_grouping = window_usage_value::<i32>(row, "model_grouping")?;
let currency_grouping = window_usage_value::<i32>(row, "currency_grouping")?;
match (model_grouping, currency_grouping) {
(1, 1) => usage_rows.push(row),
(1, 0) if window_usage_value::<Option<String>>(row, "cost_currency")?.is_some() => {
let (key, cost) = admin_account_usage_window_cost(row)?;
costs_by_window.entry(key).or_default().push(cost);
}
(0, 1) if window_usage_value::<Option<String>>(row, "model")?.is_some() => {
let ((account_id, window_key, _), usage) =
admin_account_usage_window_model(row)?;
models_by_window
.entry((account_id, window_key))
.or_default()
.push(usage);
}
(0, 0)
if window_usage_value::<Option<String>>(row, "model")?.is_some()
&& window_usage_value::<Option<String>>(row, "cost_currency")?
.is_some() =>
{
let (key, cost) = admin_account_usage_window_model_cost(row)?;
model_costs.entry(key).or_default().push(cost);
}
(0 | 1, 0 | 1) => {}
_ => {
return Err(AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
ENTITY,
"account usage window query returned an invalid grouping marker",
));
}
}
}
let mut results = usage_rows
.into_iter()
.map(admin_account_usage_window)
.collect::<AdminStoreResult<Vec<_>>>()?;
for result in &mut results {
result.usage.costs = costs_by_window
.remove(&(result.account_id.clone(), result.key.clone()))
.unwrap_or_default();
result.usage.models = models_by_window
.remove(&(result.account_id.clone(), result.key.clone()))
.unwrap_or_default();
for model in &mut result.usage.models {
model.costs = model_costs
.remove(&(
result.account_id.clone(),
result.key.clone(),
model.model.clone(),
))
.unwrap_or_default();
}
result.usage.models.sort_by(|left, right| {
right
.request_count
.cmp(&left.request_count)
.then_with(|| left.model.cmp(&right.model))
});
}
Ok(results)
.map_err(|error| admin_store_error(ENTITY, error))
}
async fn required_scope(
@@ -376,8 +284,14 @@ impl AccountStore for PgAdminAccountStore {
.filter(|(_, cooldown)| cooldown.is_active(now_system_time))
.map(|(account_id, _)| account_id.clone())
.collect::<Vec<_>>();
let page =
load_admin_account_page(&self.pool, &query, now, active_rate_limited_ids).await?;
let page = load_admin_account_page(
&self.pool,
&self.query_budget,
&query,
now,
active_rate_limited_ids,
)
.await?;
let item_ids = page
.accounts
.iter()
@@ -12,6 +12,7 @@ pub(crate) struct AdminAccountPageRows {
pub(crate) async fn load_admin_account_page(
pool: &PgPool,
budget: &crate::postgres::ObservabilityQueryBudget,
query: &AdminAccountListQuery,
now: DateTime<Utc>,
active_rate_limited_ids: Vec<String>,
@@ -35,35 +36,20 @@ pub(crate) async fn load_admin_account_page(
let page_size = i64::from(query.page_size.get());
let offset = i64::from(query.page - 1).saturating_mul(page_size);
let order_by = admin_account_order(query.sort);
let usage_join = query.sort.is_some_and(|sort| {
let needs_usage = query.sort.is_some_and(|sort| {
matches!(
sort.field,
AdminAccountSortField::Usage | AdminAccountSortField::LastUsedAt
)
});
let completed_usage = completed_usage_fact_predicate("mr");
let usage_join = if usage_join {
format!(
"left join (
select mr.provider_account_ref,
coalesce(sum(coalesce(
mr.total_tokens,
coalesce(mr.input_tokens, 0) + coalesce(mr.output_tokens, 0)
)), 0)::bigint as total_tokens,
max(mr.started_at) as last_used_at
from model_requests mr
where mr.started_at >= $2 - make_interval(days => settings.usage_retention_days::int)
and mr.started_at < $2
and {completed_usage}
group by mr.provider_account_ref
) usage on usage.provider_account_ref = a.id"
)
let usage_join = if needs_usage {
"left join usage_values usage on usage.provider_account_ref = a.id"
} else {
String::new()
""
};
let statement = format!(
"with account_statuses as (
"with usage_values as (select * from unnest($10::text[],$11::bigint[],$12::timestamptz[]) as usage(provider_account_ref,total_tokens,last_used_at)), account_statuses as (
select a.id, a.provider_kind,
case
when not a.enabled then 'disabled'
@@ -150,25 +136,45 @@ pub(crate) async fn load_admin_account_page(
left join outbound_proxies p on p.id = a.outbound_proxy_id
order by page.page_position"
);
// 动态片段只来自上面的封闭排序枚举与固定 usage predicate;所有请求值仍使用 bind
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(active_rate_limited_ids)
.bind(now)
.bind(provider_kind)
.bind(search_prefix)
.bind(status)
.bind(group_mode)
.bind(group_id)
.bind(page_size)
.bind(offset)
.fetch_all(pool)
// 排序事实与本页总数共享同一快照,避免一次查询混入不同源版本
let rows = budget
.run(
pool,
"load admin account page",
|mut connection| async move {
let mut snapshot =
crate::postgres::observability::AnalyticsSnapshot::new_for_accounts(
&mut connection,
)
.await?;
let (usage_ids, usage_tokens, usage_latest) = if needs_usage {
crate::postgres::observability::account_usage_sort_values(&mut snapshot, now)
.await?
} else {
(Vec::new(), Vec::new(), Vec::new())
};
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
.bind(active_rate_limited_ids)
.bind(now)
.bind(provider_kind)
.bind(search_prefix)
.bind(status)
.bind(group_mode)
.bind(group_id)
.bind(page_size)
.bind(offset)
.bind(usage_ids)
.bind(usage_tokens)
.bind(usage_latest)
.fetch_all(&mut *snapshot.tx)
.await
.map_err(|source| postgres_unavailable("load admin account page", source))?;
snapshot.finish().await?;
Ok(rows)
},
)
.await
.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("load admin account page", source),
)
})?;
.map_err(|error| admin_store_error(ENTITY, error))?;
let metadata = rows.first().ok_or_else(|| {
AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
@@ -105,164 +105,3 @@ pub(crate) fn admin_account_usage(usage: ProviderAccountUsageObservation) -> Acc
.collect(),
}
}
pub(crate) fn admin_account_usage_window(
row: &sqlx::postgres::PgRow,
) -> AdminStoreResult<AccountUsageWindowResult> {
let provider_reported_count = window_usage_count(row, "provider_reported_count")?;
let calculated_count = window_usage_count(row, "calculated_count")?;
let unavailable_count = window_usage_count(row, "unavailable_count")?;
Ok(AccountUsageWindowResult {
account_id: window_usage_value(row, "account_id")?,
key: window_usage_value(row, "window_key")?,
usage: AccountUsage {
account_id: window_usage_value(row, "account_id")?,
request_count: window_usage_count(row, "request_count")?,
success_count: window_usage_count(row, "success_count")?,
input_tokens: optional_window_usage_count(row, "input_tokens")?,
output_tokens: optional_window_usage_count(row, "output_tokens")?,
cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?,
cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?,
reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?,
image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?,
image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?,
image_request_count: window_usage_count(row, "image_request_count")?,
image_request_failed_count: window_usage_count(row, "image_request_failed_count")?,
total_tokens: optional_window_usage_count(row, "total_tokens")?,
cost_coverage: AdminCostCoverage {
provider_reported_count,
calculated_count,
partial_count: 0,
unavailable_count,
not_billable_count: 0,
},
costs: Vec::new(),
last_used_at: window_usage_value(row, "last_used_at")?,
request_buckets: Vec::new(),
models: Vec::new(),
},
})
}
pub(crate) fn admin_account_usage_window_model(
row: &sqlx::postgres::PgRow,
) -> AdminStoreResult<((String, String, String), AccountModelUsage)> {
let account_id: String = window_usage_value(row, "account_id")?;
let window_key: String = window_usage_value(row, "window_key")?;
let model: String = window_usage_value(row, "model")?;
let provider_reported_count = window_usage_count(row, "provider_reported_count")?;
let calculated_count = window_usage_count(row, "calculated_count")?;
let unavailable_count = window_usage_count(row, "unavailable_count")?;
Ok((
(account_id, window_key, model.clone()),
AccountModelUsage {
model,
request_count: window_usage_count(row, "request_count")?,
success_count: window_usage_count(row, "success_count")?,
input_tokens: optional_window_usage_count(row, "input_tokens")?,
output_tokens: optional_window_usage_count(row, "output_tokens")?,
cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?,
cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?,
reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?,
image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?,
image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?,
image_request_count: window_usage_count(row, "image_request_count")?,
image_request_failed_count: window_usage_count(row, "image_request_failed_count")?,
total_tokens: optional_window_usage_count(row, "total_tokens")?,
cost_coverage: AdminCostCoverage {
provider_reported_count,
calculated_count,
partial_count: 0,
unavailable_count,
not_billable_count: 0,
},
costs: Vec::new(),
last_used_at: window_usage_value(row, "last_used_at")?,
},
))
}
pub(crate) fn admin_account_usage_window_model_cost(
row: &sqlx::postgres::PgRow,
) -> AdminStoreResult<((String, String, String), AccountCost)> {
let key = (
window_usage_value(row, "account_id")?,
window_usage_value(row, "window_key")?,
window_usage_value(row, "model")?,
);
let amount = window_usage_value::<String>(row, "amount")?;
let amount = AdminDecimalAmount::from_str(&amount).map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
ENTITY,
"persisted account model cost is invalid",
)
})?;
Ok((
key,
AccountCost {
currency: window_usage_value(row, "cost_currency")?,
amount,
},
))
}
pub(crate) fn admin_account_usage_window_cost(
row: &sqlx::postgres::PgRow,
) -> AdminStoreResult<((String, String), AccountCost)> {
let key = (
window_usage_value(row, "account_id")?,
window_usage_value(row, "window_key")?,
);
let amount = window_usage_value::<String>(row, "amount")?;
let amount = AdminDecimalAmount::from_str(&amount).map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
ENTITY,
"persisted account cost is invalid",
)
})?;
Ok((
key,
AccountCost {
currency: window_usage_value(row, "cost_currency")?,
amount,
},
))
}
pub(crate) fn window_usage_count(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> AdminStoreResult<u64> {
let value = window_usage_value::<i64>(row, column)?;
u64::try_from(value).map_err(|_| invalid_window_usage(column))
}
pub(crate) fn optional_window_usage_count(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> AdminStoreResult<Option<u64>> {
window_usage_value::<Option<i64>>(row, column)?
.map(|value| u64::try_from(value).map_err(|_| invalid_window_usage(column)))
.transpose()
}
pub(crate) fn window_usage_value<T>(
row: &sqlx::postgres::PgRow,
column: &'static str,
) -> AdminStoreResult<T>
where
for<'row> T: sqlx::Decode<'row, Postgres> + sqlx::Type<Postgres>,
{
row.try_get(column)
.map_err(|_| invalid_window_usage(column))
}
pub(crate) fn invalid_window_usage(column: &'static str) -> AdminStoreError {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
ENTITY,
format!("invalid quota window usage column: {column}"),
)
}
@@ -1,6 +1,6 @@
//! 明文 `provider_accounts` 与凭证 revision CAS 的唯一 PostgreSQL owner
use std::{collections::BTreeSet, fmt, str::FromStr};
use std::{collections::BTreeSet, fmt};
use async_trait::async_trait;
use chrono::{DateTime, TimeDelta, Utc};
@@ -9,17 +9,14 @@ use gateway_admin::{
MutationContext, Revision as AdminRevision,
account_groups::AccountGroupRef,
accounts::{
AccountCost, AccountGroupFilter, AccountImportSettings,
AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage,
AccountPageItem, AccountRecord, AccountSort as AdminAccountSort,
AccountGroupFilter, AccountImportSettings, AccountListQuery as AdminAccountListQuery,
AccountPage, AccountPageItem, AccountRecord, AccountSort as AdminAccountSort,
AccountSortField as AdminAccountSortField, AccountSummary, AccountUpdateResult,
AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
BatchUpdateAccounts, DeleteAccounts, SortDirection as AdminSortDirection,
UpdateAccount,
},
observability::{
CostCoverage as AdminCostCoverage, DecimalAmount as AdminDecimalAmount, TimeRange,
},
observability::TimeRange,
provider_credentials::{
AuthorizationCommit, AuthorizationCredentialCommit, CredentialDetails,
CredentialImportCommit, CredentialImportResult, CredentialMutationResult,
@@ -54,7 +51,6 @@ use super::{
AdminAuditEvent, ControlPlaneRepository, ObservabilityRange, PgControlPlaneRepository,
PgObservabilityRepository, ProviderAccountUsageObservation, ProviderAccountUsageQuery,
append_admin_audit_event_in_transaction, bump_config_revision_in_transaction,
completed_usage_fact_predicate,
};
mod admin_adapter;
@@ -1,9 +1,6 @@
//! 预测只读采样:先聚合数值、再读取有界的 Provider 文档,不扫描模型排行维度
use gateway_admin::model::quota_forecast_sampling::{
MAX_FORECAST_HISTORY_POINTS, QuotaForecastHistory, QuotaForecastHistoryPoint,
QuotaForecastUsage,
};
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistory;
use super::*;
@@ -23,124 +20,21 @@ pub(super) async fn load_history(
"forecast range exceeds supported quota windows",
));
}
let rows = budget
budget
.run(
pool,
"load quota forecast history",
|mut connection| async move {
sqlx::query(sqlx::AssertSqlSafe(history_sql()))
.bind(&window.account_id)
.bind(range.start)
.bind(range.end)
.bind(MAX_FORECAST_HISTORY_POINTS as i32)
.fetch_all(&mut *connection)
.await
.map_err(|source| postgres_unavailable("load quota forecast history", source))
let mut snapshot =
super::super::observability::AnalyticsSnapshot::new_for_accounts(
&mut connection,
)
.await?;
let history = snapshot.forecast_history(&window.account_id, range).await?;
snapshot.finish().await?;
Ok(history)
},
)
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
let mut history = QuotaForecastHistory::default();
for row in &rows {
let usage = QuotaForecastUsage {
request_count: window_usage_count(row, "request_count")?,
tokens: window_usage_count(row, "tokens")?,
input_tokens: window_usage_count(row, "input_tokens")?,
output_tokens: window_usage_count(row, "output_tokens")?,
cached_input_tokens: window_usage_count(row, "cached_input_tokens")?,
missing_token_count: window_usage_count(row, "missing_token_count")?,
known_cost_count: window_usage_count(row, "known_cost_count")?,
unavailable_cost_count: window_usage_count(row, "unavailable_cost_count")?,
usd: window_usage_value(row, "usd")?,
excluded_request_count: window_usage_count(row, "excluded_request_count")?,
};
if window_usage_value::<bool>(row, "is_total")? {
history.usage = usage;
history.pending_request_count = window_usage_count(row, "pending_count")?;
} else if let Some(document) = window_usage_value::<
Option<sqlx::types::Json<serde_json::Map<String, serde_json::Value>>>,
>(row, "document")?
{
history.points.push(QuotaForecastHistoryPoint {
started_at: window_usage_value(row, "started_at")?,
completed_at: window_usage_value(row, "completed_at")?,
usage,
provider_observation: ProviderDocument::new(OpaqueProviderData::new(document.0)),
});
}
}
Ok(history)
}
fn history_sql() -> String {
let completed_usage = completed_usage_fact_predicate("mr");
// 一个语句共享 MVCC 快照
// 用 RANGE 帧让相同完成时间的点拥有相同累计值,
// 避免并发请求的任意行顺序制造不同分子;未完成请求只进入待决计数
format!(
"with scoped as materialized (
select mr.id, mr.started_at, mr.completed_at,
mr.completed_at <= $3 and mr.outcome <> 'running' as settled,
coalesce(({completed_usage}), false) as included,
mr.provider_observation_json is not null as has_document,
mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.total_tokens,
mr.cost_amount, mr.cost_currency,
coalesce(mr.cost_source in ('calculated', 'provider_reported')
and mr.cost_amount is not null and mr.cost_currency = 'USD', false) as known_cost
from model_requests mr
where mr.provider_account_ref = $1
and mr.started_at >= $2 and mr.started_at < $3
), facts as materialized (
select *,
(included and settled)::integer as request_count,
case when included and settled then coalesce(total_tokens,
coalesce(input_tokens, 0) + coalesce(output_tokens, 0)) else 0 end as tokens,
case when included and settled then coalesce(input_tokens, 0) else 0 end as inputs,
case when included and settled then coalesce(output_tokens, 0) else 0 end as outputs,
case when included and settled then coalesce(cached_input_tokens, 0) else 0 end as cached,
(included and settled and total_tokens is null
and (input_tokens is null or output_tokens is null))::integer as missing_tokens,
(included and settled and known_cost)::integer as known_costs,
(included and settled and not known_cost)::integer as missing_costs,
case when included and settled and known_cost then cost_amount else 0 end as usd,
(not included and settled)::integer as excluded
from scoped
), cumulative as (
select id, started_at, completed_at, has_document,
least($4 - 1, floor(extract(epoch from (completed_at - $2))
/ greatest(1, extract(epoch from ($3::timestamptz - $2::timestamptz)) / $4))) as bucket,
sum(request_count) over w as request_count,
sum(tokens) over w as tokens,
sum(inputs) over w as input_tokens,
sum(outputs) over w as output_tokens,
sum(cached) over w as cached_input_tokens,
sum(missing_tokens) over w as missing_token_count,
sum(known_costs) over w as known_cost_count,
sum(missing_costs) over w as unavailable_cost_count,
sum(usd) over w as usd,
sum(excluded) over w as excluded_request_count
from facts where settled
window w as (order by completed_at range between unbounded preceding and current row)
), selected as (
select distinct on (bucket) * from cumulative
where has_document order by bucket, completed_at desc, id desc
)
select false as is_total, s.started_at, s.completed_at,
s.request_count::bigint, s.tokens::bigint, s.input_tokens::bigint,
s.output_tokens::bigint, s.cached_input_tokens::bigint, s.missing_token_count::bigint,
s.known_cost_count::bigint, s.unavailable_cost_count::bigint,
s.usd::double precision, s.excluded_request_count::bigint,
0::bigint as pending_count, mr.provider_observation_json as document
from selected s join model_requests mr on mr.id = s.id
union all
select true, $3, $3,
coalesce(sum(request_count), 0)::bigint, coalesce(sum(tokens), 0)::bigint,
coalesce(sum(inputs), 0)::bigint, coalesce(sum(outputs), 0)::bigint,
coalesce(sum(cached), 0)::bigint, coalesce(sum(missing_tokens), 0)::bigint,
coalesce(sum(known_costs), 0)::bigint, coalesce(sum(missing_costs), 0)::bigint,
coalesce(sum(usd), 0)::double precision, coalesce(sum(excluded), 0)::bigint,
count(*) filter (where settled is not true), null::jsonb
from facts
order by completed_at, is_total"
)
.map_err(|error| admin_store_error(ENTITY, error))
}
@@ -9,76 +9,6 @@ pub(crate) const QUOTA_MAX_BYTES: usize = 128 * 1024;
pub(crate) const MAX_ADMIN_IMPORT_BATCH: usize = 200;
pub(crate) const ADMIN_USAGE_CHUNK_SIZE: usize = 200;
/// 四种窗口投影通过一组 `GROUPING SETS` 从同一批匹配请求派生
/// `grouping_* = 1`
/// 表示对应维度未参与该行聚合,调用方据此区分窗口、模型与成本行
pub(crate) fn account_usage_by_windows_sql() -> String {
let completed_usage = completed_usage_fact_predicate("mr");
format!(
"with requested_windows as (
select *
from unnest($1::text[], $2::text[], $3::timestamptz[], $4::timestamptz[])
as requested(account_id, window_key, window_start, window_end)
), matched as (
select requested.account_id,
requested.window_key,
mr.id as request_id,
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
mr.outcome, mr.input_tokens, mr.output_tokens, mr.cached_input_tokens,
mr.cache_write_input_tokens, mr.reasoning_output_tokens,
mr.image_input_tokens, mr.image_output_tokens,
mr.image_generation_succeeded, mr.total_tokens,
mr.cost_source, mr.cost_amount, mr.cost_currency, mr.started_at
from requested_windows requested
left join model_requests mr
on mr.provider_account_ref = requested.account_id
and mr.started_at >= requested.window_start
and mr.started_at < requested.window_end
and {completed_usage}
)
select account_id,
window_key,
model,
cost_currency,
grouping(model)::integer as model_grouping,
grouping(cost_currency)::integer as currency_grouping,
count(request_id)::bigint as request_count,
count(request_id) filter (where outcome = 'succeeded')::bigint as success_count,
sum(input_tokens)::bigint as input_tokens,
sum(output_tokens)::bigint as output_tokens,
sum(cached_input_tokens)::bigint as cached_input_tokens,
sum(cache_write_input_tokens)::bigint as cache_write_input_tokens,
sum(reasoning_output_tokens)::bigint as reasoning_output_tokens,
sum(image_input_tokens)::bigint as image_input_tokens,
sum(image_output_tokens)::bigint as image_output_tokens,
count(request_id) filter (where image_generation_succeeded is true)::bigint
as image_request_count,
count(request_id) filter (where image_generation_succeeded is false)::bigint
as image_request_failed_count,
coalesce(sum(coalesce(
total_tokens,
coalesce(input_tokens, 0) + coalesce(output_tokens, 0)
)), 0)::bigint as total_tokens,
count(request_id) filter (where cost_source = 'provider_reported')::bigint
as provider_reported_count,
count(request_id) filter (where cost_source = 'calculated')::bigint
as calculated_count,
count(request_id) filter (where cost_source = 'unavailable')::bigint
as unavailable_count,
max(started_at) as last_used_at,
sum(cost_amount)::text as amount
from matched
group by grouping sets (
(account_id, window_key),
(account_id, window_key, cost_currency),
(account_id, window_key, model),
(account_id, window_key, model, cost_currency)
)
order by account_id, window_key, model_grouping desc, currency_grouping desc,
model nulls last, cost_currency nulls last"
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountAdminScope {
pub provider_kind: String,
@@ -27,6 +27,7 @@ fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits()
assert_eq!(pool, StorePoolConfig::default());
assert_eq!(pool.max_connections, 20);
assert_eq!(pool.acquire_timeout_seconds, 5);
assert_eq!(pool.observability_timeout_seconds, 120);
assert_eq!(pool.observability_max_connections(), 16);
assert_eq!(
StorePoolConfig {
@@ -87,3 +88,24 @@ fn valid_config() -> StoreConfig {
}))
.expect("test Store configuration")
}
#[test]
fn store_config_accepts_custom_observability_timeout_and_rejects_invalid_values() {
let root = tempfile::tempdir().unwrap();
for seconds in [1_u64, 47, 180] {
let pool: StorePoolConfig = serde_json::from_value(serde_json::json!({
"observability_timeout_seconds": seconds,
}))
.unwrap();
assert_eq!(pool.observability_timeout_seconds, seconds);
}
for seconds in [0_u64, u64::MAX] {
let mut config: serde_json::Value = serde_json::json!({
"database": { "url": "postgres://codex_proxy@127.0.0.1:5432/codex_proxy", "password": PASSWORD },
"redis": { "url": "redis://127.0.0.1:6379/", "password": PASSWORD },
});
config["pool"] = serde_json::json!({"observability_timeout_seconds": seconds});
let mut config: StoreConfig = serde_json::from_value(config).unwrap();
assert!(config.resolve_and_validate(root.path()).is_err());
}
}
@@ -18,7 +18,9 @@ use gateway_core::{
policy::{ClientApiKeyId, RateLimits},
routing::AccountGroupId,
};
use gateway_store::postgres::{PgAccountGroupRepository, PgAdminClientKeyStore};
use gateway_store::postgres::{
ObservabilityQueryBudget, PgAccountGroupRepository, PgAdminClientKeyStore,
};
use super::TestDatabase;
@@ -75,7 +77,11 @@ async fn groups_aggregate_cross_provider_members_and_key_bindings_without_multip
.await
.expect("set account concurrency override");
let groups = PgAccountGroupRepository::new(database.pool.clone());
let groups = PgAccountGroupRepository::new(
database.pool.clone(),
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
.unwrap(),
);
let keys = PgAdminClientKeyStore::new(database.pool.clone());
let mixed_group = group_id(MIXED_GROUP);
let empty_group = group_id(EMPTY_GROUP);
@@ -315,7 +321,11 @@ async fn group_options_should_avoid_aggregate_tables_and_page_filtered_empty_gro
.await
.expect("seed account group option");
}
let groups = PgAccountGroupRepository::new(database.pool.clone());
let groups = PgAccountGroupRepository::new(
database.pool.clone(),
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
.unwrap(),
);
let mut locked_tables = database.pool.begin().await.expect("begin table lock");
sqlx::query(
@@ -383,7 +393,11 @@ async fn group_costs_should_include_statusless_websocket_but_reject_statusless_h
"Statusless Account",
)
.await;
let groups = PgAccountGroupRepository::new(database.pool.clone());
let groups = PgAccountGroupRepository::new(
database.pool.clone(),
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
.unwrap(),
);
groups
.create_account_group(
NewAccountGroup {
@@ -576,7 +590,11 @@ async fn fast_mode_group_updates_preserve_omitted_values_and_publish_snapshot_fa
let Some(database) = TestDatabase::create("fast_mode_group").await else {
return;
};
let repository = PgAccountGroupRepository::new(database.pool.clone());
let repository = PgAccountGroupRepository::new(
database.pool.clone(),
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
.unwrap(),
);
let id = group_id(MIXED_GROUP);
repository
.create_account_group(
@@ -632,3 +650,112 @@ async fn fast_mode_group_updates_preserve_omitted_values_and_publish_snapshot_fa
}
database.close().await;
}
#[tokio::test]
async fn group_costs_follow_current_members_and_pending_source_replacements() {
let Some(database) = TestDatabase::create("group_cost_current_members").await else {
return;
};
seed_account(
&database.pool,
"acct_group_move",
"openai",
"Movable Account",
)
.await;
let budget =
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
.unwrap();
let groups = PgAccountGroupRepository::new(database.pool.clone(), budget.clone());
for id in [MIXED_GROUP, EMPTY_GROUP] {
groups
.create_account_group(
NewAccountGroup {
fast_mode: FastMode::Default,
id: group_id(id),
name: id.to_owned(),
description: None,
color: group_color("#2563EBFF"),
},
&context("create-group-move-fixture"),
)
.await
.unwrap();
}
assign_accounts(&database.pool, MIXED_GROUP, &["acct_group_move"]).await;
seed_group_cost_snapshot(
&database.pool,
"req_group_move",
"acct_group_move",
&[MIXED_GROUP],
"3.25",
)
.await;
sqlx::query(
"update model_requests set started_at=now()-interval '2 hours' where id='req_group_move'",
)
.execute(&database.pool)
.await
.unwrap();
let worker =
gateway_store::postgres::PgRequestAnalyticsRepository::new(database.pool.clone(), budget);
let mut ready = false;
for _ in 0..100 {
worker.maintain(1000).await.unwrap();
let status = worker.status().await.unwrap();
if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 {
ready = true;
break;
}
}
assert!(ready, "group fixture must publish rollups");
let costs = load_retained_group_costs(&groups).await;
assert_eq!(costs[MIXED_GROUP], "3.25");
assert_eq!(costs[EMPTY_GROUP], "0");
sqlx::query("update account_group_accounts set account_group_id=$1 where provider_account_id='acct_group_move'")
.bind(EMPTY_GROUP).execute(&database.pool).await.unwrap();
let costs = load_retained_group_costs(&groups).await;
assert_eq!(
costs[MIXED_GROUP], "0",
"moving an account reattributes retained usage"
);
assert_eq!(costs[EMPTY_GROUP], "3.25");
// 未再次运行 Worker,当前源费用和模型仍必须原子替换旧汇总贡献
sqlx::query("update model_requests set cost_amount=7.5,upstream_model_id='updated-model' where id='req_group_move'")
.execute(&database.pool).await.unwrap();
let costs = load_retained_group_costs(&groups).await;
assert_eq!(costs[EMPTY_GROUP], "7.5");
sqlx::query("delete from model_requests where id='req_group_move'")
.execute(&database.pool)
.await
.unwrap();
let costs = load_retained_group_costs(&groups).await;
assert_eq!(
costs[EMPTY_GROUP], "0",
"pending deletion must withdraw the old projected amount"
);
database.close().await;
}
async fn load_retained_group_costs(groups: &PgAccountGroupRepository) -> BTreeMap<String, String> {
groups
.list_account_groups(AccountGroupListQuery {
page: 1,
page_size: PageSize::new(20).unwrap(),
search: None,
enabled: None,
})
.await
.unwrap()
.items
.into_iter()
.map(|group| {
(
group.id.to_string(),
group.usage.retained_total_usd.to_string(),
)
})
.collect()
}
@@ -21,6 +21,212 @@ pub(super) async fn seed(pool: &sqlx::PgPool, id: &str, start: DateTime<Utc>, gr
.bind(id).bind(start).bind(observation).execute(pool).await.expect("seed analytic request");
}
#[tokio::test]
async fn provider_counts_match_for_rollups_boundaries_and_filtered_facts() {
let Some(database) = TestDatabase::create("analytics_provider_counts").await else {
return;
};
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
for id in [
"provider_counts_ok",
"provider_counts_fail",
"provider_counts_unrouted",
] {
seed(&database.pool, id, start + TimeDelta::minutes(5), &[]).await;
}
sqlx::query("update model_requests set provider_kind='openai',provider_account_ref='account_provider_counts',upstream_transport='http',attempt_count=case when id='provider_counts_ok' then 2 else 1 end where id in ('provider_counts_ok','provider_counts_fail')")
.execute(&database.pool).await.unwrap();
sqlx::query("update model_requests set outcome='failed',client_status_code=502,downstream_committed_at=null where id='provider_counts_fail'")
.execute(&database.pool).await.unwrap();
drain_analytics(&database.pool).await;
let repo = observability_repository(&database.pool);
let full = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
let edge =
ObservabilityRange::new(start + TimeDelta::minutes(4), start + TimeDelta::minutes(6))
.unwrap();
// 同一批请求分别走完整桶、首尾事实与筛选回退,供应商计数不能随读取路径变化
for (range, filter) in [
(full, UsageRecordFilter::default()),
(edge, UsageRecordFilter::default()),
(
full,
UsageRecordFilter {
search: Some("model_".to_owned()),
..Default::default()
},
),
] {
let summary = repo.usage_summary(range, filter.clone()).await.unwrap();
assert_eq!(summary.requests.request_count, 3);
let providers = summary
.providers
.into_iter()
.map(|provider| {
(
provider.provider_kind,
(
provider.request_count,
provider.attempt_count,
provider.failure_count,
provider.total_tokens,
),
)
})
.collect::<std::collections::BTreeMap<_, _>>();
assert_eq!(
providers,
std::collections::BTreeMap::from([
("openai".to_owned(), (2, 3, 1, 120)),
("unrouted".to_owned(), (1, 0, 0, 120)),
])
);
let diagnostic = repo
.usage_diagnostics(range, filter, DiagnosticDimension::Provider, 100)
.await
.unwrap();
assert_eq!(diagnostic.total_request_count, 3);
let openai = diagnostic
.items
.iter()
.find(|item| item.key == "openai")
.unwrap();
assert_eq!(
(
openai.request_count,
openai.success_count,
openai.failure_count,
openai.attempt_count,
openai.total_tokens
),
(2, 1, 1, 3, 120)
);
assert_eq!(openai.average_latency_ms, Some(100));
// 汇总桶沿用直方图近似值,诊断与完整指标读取必须给出相同分位数
assert_eq!(
openai.latency_p95_ms,
summary
.requests
.latency_percentiles
.p95_ms
.map(|value| value.as_f64().round() as u64)
);
assert_eq!(
openai.first_token_p95_ms,
summary
.requests
.first_token_latency_percentiles
.p95_ms
.map(|value| value.as_f64().round() as u64)
);
assert_eq!(openai.retry_count, 1);
assert_eq!(openai.retried_request_count, 1);
assert_eq!(openai.non_completion_count, 0);
assert_eq!(openai.cost_coverage.unavailable_count, 1);
}
database.close().await;
}
#[tokio::test]
async fn insights_share_bucket_states_without_merging_exact_percentiles() {
let Some(database) = TestDatabase::create("analytics_shared_insights").await else {
return;
};
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
for (index, latency) in [10_i64, 1000, 2000, 3000].into_iter().enumerate() {
let id = format!("insights_{index}");
let minute = if index == 0 { 1 } else { 16 };
seed(
&database.pool,
&id,
start + TimeDelta::minutes(minute),
&["insights_group"],
)
.await;
sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',5)) where id=$1")
.bind(id).bind(latency).execute(&database.pool).await.unwrap();
}
drain_analytics(&database.pool).await;
let repo = observability_repository(&database.pool);
let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
for filter in [
UsageRecordFilter::default(),
UsageRecordFilter {
search: Some("model_".to_owned()),
..Default::default()
},
] {
let expected = repo.usage_summary(range, filter.clone()).await.unwrap();
if filter.search.is_some() {
assert_eq!(
expected
.requests
.latency_percentiles
.p50_ms
.unwrap()
.as_f64(),
1500.0
);
}
for granularity in [
gateway_admin::model::observability::Granularity::FifteenMinutes,
gateway_admin::model::observability::Granularity::Hour,
] {
let observation = repo
.usage_insights_observation(range, filter.clone(), granularity)
.await
.unwrap();
assert_eq!(observation.overview.requests, expected.requests);
assert_eq!(observation.overview.providers, expected.providers);
assert_eq!(
observation.overview.analytics.percentiles_approximate,
expected.analytics.percentiles_approximate
);
assert_eq!(
observation
.trend
.iter()
.map(|point| point.metrics.request_count)
.sum::<u64>(),
4
);
assert!(observation.costs.iter().all(|point| point.standard_amount.is_none() && point.no_cache_amount.is_none()));
}
}
database.close().await;
}
#[tokio::test]
async fn prefix_search_ignores_request_ids_and_matches_snapshot_names() {
let Some(database) = TestDatabase::create("analytics_search_prefix").await else {
return;
};
let start = Utc::now() - TimeDelta::hours(1);
for id in ["literal_1", "literalX1", "snapshot_only"] {
seed(&database.pool, id, start, &[]).await;
}
sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{request,clientApiKeyNameSnapshot}','\"literal_name\"'::jsonb) where id='snapshot_only'")
.execute(&database.pool).await.unwrap();
drain_analytics(&database.pool).await;
let result = observability_repository(&database.pool)
.usage_summary(
ObservabilityRange::new(start - TimeDelta::seconds(1), start + TimeDelta::seconds(1))
.unwrap(),
UsageRecordFilter {
search: Some("literal_".to_owned()),
..Default::default()
},
)
.await
.unwrap();
assert_eq!(result.requests.request_count, 1);
assert_eq!(result.requests.total_tokens, 120);
database.close().await;
}
#[tokio::test]
async fn opaque_identifiers_preserve_long_values_and_response_or_counts_each_request_once() {
let Some(database) = TestDatabase::create("analytics_opaque_identifiers").await else {
@@ -169,7 +375,7 @@ async fn rolled_back_source_changes_and_late_commits_do_not_lose_queue_work() {
}
#[tokio::test]
async fn group_or_filter_counts_each_historical_request_once_and_rebuilds_dirty_buckets() {
async fn dirty_buckets_replace_each_historical_request_contribution_once() {
let Some(database) = TestDatabase::create("analytics_groups").await else {
return;
};
@@ -194,13 +400,8 @@ async fn group_or_filter_counts_each_historical_request_once_and_rebuilds_dirty_
let repo = observability_repository(&database.pool);
let range = ObservabilityRange::new(start - TimeDelta::minutes(1), start + TimeDelta::hours(2))
.unwrap();
let filter = UsageRecordFilter {
routing_group_refs: vec!["group_a".into(), "group_b".into()],
..Default::default()
};
let result = repo.usage_summary(range, filter).await.unwrap();
let result = repo.usage_summary(range, Default::default()).await.unwrap();
assert_eq!(result.requests.request_count, 2);
assert!(!result.analytics.percentiles_approximate);
// 已投影但桶待重建时整个桶用事实替换,不能叠加旧桶贡献
sqlx::query("update request_analytics_facts set total_tokens=333 where id='req_group_a'")
.execute(&database.pool)
@@ -453,6 +654,9 @@ async fn interrupted_quarters_restart_changed_versions_and_publish_one_consisten
.execute(&database.pool)
.await
.unwrap();
// 独立重建请求改变版本,验证旧前缀废弃后仍可恢复,普通源变更不再重启整桶
sqlx::query("update request_analytics_pending_buckets set change_version=nextval('request_analytics_change_version')")
.execute(&database.pool).await.unwrap();
drop(worker);
let worker =
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
@@ -487,6 +691,8 @@ async fn source_updates_do_not_wait_for_blocked_bucket_rebuild() {
.execute(&database.pool)
.await
.unwrap();
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),now())")
.bind(start).execute(&database.pool).await.unwrap();
let mut blocker = sqlx::PgConnection::connect_with(&database.pool.connect_options())
.await
.unwrap();
@@ -677,7 +883,7 @@ async fn clean_aligned_statistics_do_not_access_facts_and_dirty_edges_are_disjoi
.execute(&mut *transaction)
.await
.unwrap();
let plan:serde_json::Value=sqlx::query_scalar("explain (analyze,buffers,format json) select fact.id from unnest($1::timestamptz[],$2::timestamptz[]) span(start_at,end_at) cross join lateral (select mr.id from request_analytics_facts mr where mr.started_at>=span.start_at and mr.started_at<span.end_at limit 100001) fact limit 100001")
let plan:serde_json::Value=sqlx::query_scalar("explain (analyze,buffers,format json) select fact.id from unnest($1::timestamptz[],$2::timestamptz[]) span(start_at,end_at) cross join lateral (select mr.id from request_analytics_facts mr where mr.started_at>=span.start_at and mr.started_at<span.end_at offset 0) fact")
.bind(vec![start+TimeDelta::minutes(5),start+TimeDelta::minutes(15)]).bind(vec![start+TimeDelta::minutes(15),start+TimeDelta::minutes(30)])
.fetch_one(&mut *transaction).await.unwrap();
let plan = plan.to_string();
@@ -728,7 +934,7 @@ async fn persisted_histograms_merge_long_tails_and_distinguish_empty_from_zero()
.usage_summary(
range,
UsageRecordFilter {
routing_group_refs: vec!["hist_group".into()],
search: Some("model_".to_owned()),
..Default::default()
},
)
@@ -1300,3 +1506,128 @@ async fn daily_rank_counts_follow_rebuilds_rollbacks_deletions_and_dirty_days()
assert_eq!(dirty.items[0].request_count, 2);
database.close().await;
}
#[tokio::test]
async fn combined_filters_and_dirty_edges_aggregate_over_one_hundred_thousand_facts() {
let Some(database) = TestDatabase::create("analytics_sql_fact_aggregation").await else {
return;
};
let start = DateTime::parse_from_rfc3339("2026-09-01T10:01:00Z")
.unwrap()
.with_timezone(&Utc);
seed(
&database.pool,
"aggregate_template",
start,
&["group_a", "group_b"],
)
.await;
drain_analytics(&database.pool).await;
// 派生事实扩充隔离查询的容量边界,请求模型和上游模型同时命中也只能计入一次
sqlx::query("insert into request_analytics_facts select cloned.* from request_analytics_facts facts cross join generate_series(1,100005) n cross join lateral jsonb_populate_record(null::request_analytics_facts,to_jsonb(facts)||jsonb_build_object('id','aggregate_'||n::text,'upstream_model_id','model_a','latency_ms',n%97,'first_token_ms',n%31,'cost_currency','USD','cost_amount',0.0000000001,'cost_source','provider_reported')) cloned where facts.id='aggregate_template'")
.execute(&database.pool).await.unwrap();
let range = ObservabilityRange::new(
start - TimeDelta::minutes(1),
start + TimeDelta::minutes(59),
)
.unwrap();
let filter = UsageRecordFilter {
transport: Some("http_sse".to_owned()),
model: Some("model_a".to_owned()),
..Default::default()
};
let repo = observability_repository(&database.pool);
let result = repo.usage_summary(range, filter).await.unwrap();
let expected: (i64, i64, f64, String) = sqlx::query_as("select count(*),sum(total_tokens)::bigint,percentile_cont(0.95) within group(order by latency_ms),trim_scale(sum(cost_amount))::text from request_analytics_facts where started_at >= $1 and started_at < $2")
.bind(range.start).bind(range.end).fetch_one(&database.pool).await.unwrap();
assert_eq!(result.requests.request_count, expected.0 as u64);
assert_eq!(result.requests.total_tokens, expected.1 as u64);
assert_eq!(
result.requests.latency_percentiles.p95_ms.unwrap().as_f64(),
expected.2
);
assert!(!result.analytics.percentiles_approximate);
// 同一个热点桶处于重建阶段时,不得使用旧汇总或只读取前十万条
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),now())")
.bind(start).execute(&database.pool).await.unwrap();
let dirty = repo.usage_summary(range, Default::default()).await.unwrap();
assert_eq!(dirty.requests.request_count, expected.0 as u64);
assert_eq!(dirty.requests.total_tokens, expected.1 as u64);
assert_eq!(dirty.requests.latency_count, expected.0 as u64);
assert!(dirty.analytics.percentiles_approximate);
let ranked = repo
.usage_diagnostics(range, Default::default(), DiagnosticDimension::ApiKey, 1)
.await
.unwrap();
assert_eq!(ranked.total_request_count, expected.0 as u64);
assert_eq!(ranked.items[0].request_count, expected.0 as u64);
assert_eq!(ranked.items[0].costs[0].amount.to_string(), expected.3);
database.close().await;
}
#[tokio::test]
async fn high_cardinality_summary_merges_all_rows_before_returning_model_groups() {
let Some(database) = TestDatabase::create("analytics_sql_summary_aggregation").await else {
return;
};
drain_analytics(&database.pool).await;
let start = DateTime::parse_from_rfc3339("2026-09-01T00:00:00Z")
.unwrap()
.with_timezone(&Utc);
let versions: (i32,i32) = sqlx::query_as("select schema_version,histogram_version from request_analytics_build_state where singleton=true")
.fetch_one(&database.pool).await.unwrap();
// 150 个模型乘以 744 小时超过旧汇总行上限,输出仍只有 150 个模型组
sqlx::query("insert into request_analytics_hourly(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) select $1::timestamptz+hour*interval '1 hour','provider_model',array['openai','model_'||lpad(model::text,3,'0'),'model_'||lpad(model::text,3,'0')],'succeeded',$2,$3,jsonb_build_object('request_count',model,'total_tokens',model*120),jsonb_build_object('USD',(model::numeric*0.0000000001)::text),'{}' from generate_series(0,743) hour cross join generate_series(1,150) model")
.bind(start).bind(versions.0).bind(versions.1).execute(&database.pool).await.unwrap();
let result = observability_repository(&database.pool)
.usage_diagnostics(
ObservabilityRange::new(start, start + TimeDelta::days(31)).unwrap(),
UsageRecordFilter::default(),
DiagnosticDimension::Model,
2,
)
.await
.unwrap();
assert_eq!(result.total_request_count, 744 * 150 * 151 / 2);
assert_eq!(result.items.len(), 2);
assert_eq!(result.items[0].key, "model_150");
assert_eq!(result.items[0].request_count, 744 * 150);
assert_eq!(result.items[0].total_tokens, 744 * 150 * 120);
assert_eq!(result.items[0].costs[0].amount.to_string(), "0.00001116");
database.close().await;
}
#[tokio::test]
async fn sql_edge_histograms_match_persisted_power_of_two_boundaries() {
let Some(database) = TestDatabase::create("analytics_sql_histogram_boundaries").await else {
return;
};
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
for (index, value) in [0_i64, 1, 2, 4, 8, 16, 31, 32, 33, 64, 128, 1024, 1_048_576]
.into_iter()
.enumerate()
{
let id = format!("sql_hist_{index}");
seed(
&database.pool,
&id,
start + TimeDelta::seconds(index as i64),
&[],
)
.await;
sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',0)) where id=$1")
.bind(id).bind(value).execute(&database.pool).await.unwrap();
}
drain_analytics(&database.pool).await;
let repo = observability_repository(&database.pool);
let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
let persisted = repo.usage_summary(range, Default::default()).await.unwrap();
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values($1,now())")
.bind(start).execute(&database.pool).await.unwrap();
let edge = repo.usage_summary(range, Default::default()).await.unwrap();
assert_eq!(edge.requests, persisted.requests);
assert_eq!(edge.attempts, persisted.attempts);
database.close().await;
}
@@ -42,6 +42,9 @@ async fn dirty_quarters_share_one_hour_rebuild_and_acknowledge_each_bucket() {
sqlx::raw_sql("create table hourly_build_audit (bucket_start timestamptz); create function audit_hourly_build() returns trigger language plpgsql as $$begin if new.collection='global' then insert into hourly_build_audit values(new.bucket_start); end if; return new; end$$; create trigger audit_hourly_build after insert on request_analytics_hourly for each row execute function audit_hourly_build()")
.execute(&database.pool).await.unwrap();
seed_hour(&database.pool, hour(), "coalesced").await;
drain_analytics(&database.pool).await;
sqlx::raw_sql("truncate hourly_build_audit; insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
.execute(&database.pool).await.unwrap();
let worker =
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
let mut acknowledged = 0;
@@ -85,6 +88,9 @@ async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuil
.execute(&database.pool)
.await
.unwrap();
// 显式修复任务重建整桶,普通更新只维护当前贡献,不重新启动构建
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
.execute(&database.pool).await.unwrap();
let worker =
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
let mut owner = None;
@@ -100,10 +106,12 @@ async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuil
let changed: String = sqlx::query_scalar("select id from model_requests where date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC') <> $1 order by id limit 1")
.bind(owner).fetch_one(&database.pool).await.unwrap();
sqlx::query("update model_requests set total_tokens=9999 where id=$1")
.bind(changed)
.bind(&changed)
.execute(&database.pool)
.await
.unwrap();
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from model_requests where id=$1 on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version')")
.bind(changed).execute(&database.pool).await.unwrap();
worker.maintain(2).await.unwrap();
let phase: i16 = sqlx::query_scalar(
"select build_phase from request_analytics_pending_buckets where bucket_start=$1",
@@ -174,3 +182,174 @@ async fn waiting_hours_cannot_fill_all_slots_and_starve_their_unbuilt_quarters()
assert_eq!(result.requests.total_tokens, 18 * 120);
database.close().await;
}
#[tokio::test]
async fn continuous_updates_do_not_restart_partial_quarter_and_hour_builds() {
let Some(database) = TestDatabase::create("analytics_continuous_stage_updates").await else {
return;
};
for quarter in 0..2 {
for name in ["A", "a", "Z", "z", "Ä", "ä", "中", "é", "e", "_", "-", "2"] {
seed(
&database.pool,
&format!("prefix_{name}_{quarter}"),
hour() + TimeDelta::minutes(quarter * 15 + 1),
&[],
)
.await;
}
}
drain_analytics(&database.pool).await;
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
.execute(&database.pool).await.unwrap();
let mut quarter_prefix_seen = false;
let mut hour_prefix_seen = false;
let mut completed = false;
for round in 0..150_i64 {
// 同时间的大小写及非 ASCII ID、改变排序位置的账号维度同时覆盖两种前缀
sqlx::query("update model_requests set total_tokens=$1,provider_account_ref=$2 where id in ('prefix_A_0','prefix_ä_1')")
.bind(200+round).bind(if round%2==0 {"account_Ä"}else{"account_Z"})
.execute(&database.pool).await.unwrap();
let worker =
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
worker.maintain(2).await.unwrap();
let phases: (bool,bool,i64) = sqlx::query_as("select coalesce(bool_or(build_phase=1 and build_id is not null),false),coalesce(bool_or(build_phase=4 and build_collection is not null),false),count(*) from request_analytics_pending_buckets")
.fetch_one(&database.pool).await.unwrap();
quarter_prefix_seen |= phases.0;
hour_prefix_seen |= phases.1;
if phases.2 == 0 {
completed = true;
break;
}
}
assert!(
quarter_prefix_seen && hour_prefix_seen,
"both persisted prefixes must be exercised"
);
assert!(
completed,
"ongoing updates cannot repeatedly reset a partial rebuild"
);
drain_analytics(&database.pool).await;
let expected: (i64, i64) =
sqlx::query_as("select count(*),sum(total_tokens)::bigint from model_requests")
.fetch_one(&database.pool)
.await
.unwrap();
let actual = observability_repository(&database.pool)
.usage_summary(
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
Default::default(),
)
.await
.unwrap();
assert_eq!(actual.requests.request_count, expected.0 as u64);
assert_eq!(actual.requests.total_tokens, expected.1 as u64);
database.close().await;
}
#[tokio::test]
async fn incremental_deletions_repair_extrema_last_usage_and_zero_currency_presence() {
let Some(database) = TestDatabase::create("analytics_incremental_extrema").await else {
return;
};
for number in 1..=4_i64 {
let id = format!("extrema_{number}");
seed(
&database.pool,
&id,
hour() + TimeDelta::minutes(number),
&[],
)
.await;
sqlx::query("update model_requests set provider_account_ref='extrema_account',request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',$2::bigint)),cost_source=case when $2<=2 then 'provider_reported' else 'unavailable' end,cost_currency=case $2 when 1 then 'EUR' when 2 then 'USD' end,cost_amount=case $2 when 1 then 1::numeric when 2 then 0::numeric end where id=$1")
.bind(id).bind(number).execute(&database.pool).await.unwrap();
}
drain_analytics(&database.pool).await;
sqlx::query("delete from model_requests where id in ('extrema_1','extrema_4')")
.execute(&database.pool)
.await
.unwrap();
drain_analytics(&database.pool).await;
let row: (i64,f64,f64,DateTime<Utc>,serde_json::Value) = sqlx::query_as("select (counts->>'request_count')::bigint,(distributions->'latency_ms'->>'minimum')::double precision,(distributions->'latency_ms'->>'maximum')::double precision,last_used_at,costs from request_analytics_hourly where collection='global'")
.fetch_one(&database.pool).await.unwrap();
assert_eq!(
(row.0, row.1, row.2, row.3),
(2, 2.0, 3.0, hour() + TimeDelta::minutes(3))
);
assert_eq!(row.4.as_object().unwrap().len(), 1);
assert_eq!(
row.4["USD"]
.as_str()
.unwrap()
.parse::<gateway_core::metering::Decimal>()
.unwrap(),
gateway_core::metering::Decimal::ZERO
);
sqlx::query("update model_requests set cost_source='unavailable',cost_currency=null,cost_amount=null where id='extrema_2'")
.execute(&database.pool).await.unwrap();
drain_analytics(&database.pool).await;
let costs: serde_json::Value =
sqlx::query_scalar("select costs from request_analytics_hourly where collection='global'")
.fetch_one(&database.pool)
.await
.unwrap();
assert_eq!(costs, serde_json::json!({}));
sqlx::query("select enqueue_request_analytics('extrema_3',$1)")
.bind(hour() + TimeDelta::minutes(3))
.execute(&database.pool)
.await
.unwrap();
drain_analytics(&database.pool).await;
let actual = observability_repository(&database.pool)
.usage_summary(
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
Default::default(),
)
.await
.unwrap();
assert_eq!(actual.requests.request_count, 2);
assert_eq!(actual.requests.total_tokens, 240);
assert!(actual.attempts.costs.is_empty());
database.close().await;
}
#[tokio::test]
async fn incremental_summary_write_failure_rolls_back_facts_and_preserves_queue() {
let Some(database) = TestDatabase::create("analytics_incremental_rollback").await else {
return;
};
seed(
&database.pool,
"atomic_incremental",
hour() + TimeDelta::minutes(1),
&[],
)
.await;
drain_analytics(&database.pool).await;
sqlx::raw_sql("create function reject_incremental_summary() returns trigger language plpgsql as $$begin raise exception 'test summary write failure'; end$$; create trigger reject_incremental_summary before insert or update on request_analytics_15m for each row execute function reject_incremental_summary()")
.execute(&database.pool).await.unwrap();
sqlx::query("update model_requests set total_tokens=240 where id='atomic_incremental'")
.execute(&database.pool)
.await
.unwrap();
let worker =
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
assert!(worker.maintain(10).await.is_err());
let state:(i64,i64,i64)=sqlx::query_as("select (select total_tokens from request_analytics_facts where id='atomic_incremental'),(select (counts->>'total_tokens')::bigint from request_analytics_15m where collection='global'),(select count(*) from request_analytics_pending_requests)")
.fetch_one(&database.pool).await.unwrap();
assert_eq!(state, (120, 120, 1));
sqlx::raw_sql("drop trigger reject_incremental_summary on request_analytics_15m; drop function reject_incremental_summary()")
.execute(&database.pool).await.unwrap();
drain_analytics(&database.pool).await;
let actual = observability_repository(&database.pool)
.usage_summary(
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
Default::default(),
)
.await
.unwrap();
assert_eq!(actual.requests.total_tokens, 240);
assert_eq!(actual.requests.request_count, 1);
database.close().await;
}
@@ -49,7 +49,7 @@ pub(super) struct TestDatabase {
}
pub(super) fn observability_query_budget() -> ObservabilityQueryBudget {
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1))
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1), Duration::from_secs(10))
.expect("valid test observability query budget")
}
@@ -0,0 +1,112 @@
//! 十万条以上账号与 Key 用量的完整性、未知值和汇总读取验收
use std::time::Duration;
use chrono::{DateTime, TimeDelta, Utc};
use gateway_store::postgres::{
ObservabilityQueryBudget, ObservabilityRange, PgObservabilityRepository,
PgRequestAnalyticsRepository, ProviderAccountUsageQuery, UsageRecordFilter,
};
use sqlx::Connection;
use super::super::super::{TestDatabase, analytics::seed};
#[tokio::test]
async fn large_account_and_key_usage_keep_exact_totals_and_read_clean_rollups() {
let Some(database) = TestDatabase::create("large_account_usage").await else {
return;
};
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
sqlx::query("insert into provider_accounts(id,provider_kind,name,authentication_kind,provider_credentials_json,credential_revision,has_refresh_token,credential_observed_at,created_at,updated_at) values('account_large','openai','large','oauth','{}',1,false,$1,$1,$1)")
.bind(start).execute(&database.pool).await.unwrap();
seed(
&database.pool,
"large_template",
start + TimeDelta::minutes(1),
&[],
)
.await;
sqlx::query("update model_requests set provider_kind='openai',provider_account_ref='account_large',cached_input_tokens=0,reasoning_output_tokens=null where id='large_template'")
.execute(&database.pool).await.unwrap();
// 扩充真实源表并经过生产 Worker,确保回归覆盖账号接口而非只验证派生表
sqlx::query("insert into model_requests select cloned.* from model_requests template cross join generate_series(1,100005) n cross join lateral jsonb_populate_record(null::model_requests,to_jsonb(template)||jsonb_build_object('id','account_large_'||n::text)) cloned where template.id='large_template'")
.execute(&database.pool).await.unwrap();
let budget =
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
.unwrap();
let worker = PgRequestAnalyticsRepository::new(database.pool.clone(), budget.clone());
let mut ready = false;
for _ in 0..400 {
worker.maintain(1000).await.unwrap();
let status = worker.status().await.unwrap();
if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 {
ready = true;
break;
}
}
assert!(
ready,
"large source dataset must finish projection and rollups"
);
let repo = PgObservabilityRepository::new(database.pool.clone(), None, budget, None);
let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
let query =
ProviderAccountUsageQuery::for_accounts(range, vec!["account_large".to_owned()]).unwrap();
let mut blocker = sqlx::PgConnection::connect_with(&database.pool.connect_options())
.await
.unwrap();
sqlx::raw_sql("begin;lock table request_analytics_facts in access exclusive mode")
.execute(&mut blocker)
.await
.unwrap();
let account = tokio::time::timeout(Duration::from_secs(2), repo.provider_account_usage(query))
.await
.expect("clean account usage must not read facts")
.unwrap()
.remove(0);
assert_eq!(account.request_count, 100006);
assert_eq!(account.total_tokens, Some(100006 * 120));
assert_eq!(account.cached_input_tokens, Some(0));
assert_eq!(account.reasoning_output_tokens, None);
assert_eq!(account.models.len(), 1);
assert_eq!(account.models[0].request_count, 100006);
assert_eq!(account.last_used_at, Some(start + TimeDelta::minutes(1)));
let filter = UsageRecordFilter {
client_api_key_ref: Some("key_analytics".to_owned()),
model: Some("model_a".to_owned()),
..Default::default()
};
let key = tokio::time::timeout(Duration::from_secs(2), repo.usage_summary(range, filter))
.await
.expect("Key plus model must use joint rollups")
.unwrap();
assert_eq!(key.requests.request_count, 100006);
assert_eq!(key.providers.len(), 1);
assert_eq!(key.providers[0].request_count, 100006);
sqlx::raw_sql("rollback")
.execute(&mut blocker)
.await
.unwrap();
drop(blocker);
let partial = ObservabilityRange::new(
start + TimeDelta::seconds(30),
start + TimeDelta::minutes(2),
)
.unwrap();
let account = repo
.provider_account_usage(
ProviderAccountUsageQuery::for_accounts(partial, vec!["account_large".to_owned()])
.unwrap(),
)
.await
.unwrap()
.remove(0);
assert_eq!(
account.request_count, 100006,
"the boundary aggregation must not truncate a hot bucket"
);
assert_eq!(account.total_tokens, Some(100006 * 120));
database.close().await;
}
@@ -0,0 +1,3 @@
//! 汇总读取、边界聚合与账号联合维度的真实数据库回归
mod accounts;
@@ -1,5 +1,6 @@
//! 观测存储测试入口,以及查询范围、过滤与端口合同测试
mod analytics;
mod projection;
use std::{
@@ -441,7 +442,6 @@ async fn usage_list_and_diagnostics_preserve_account_snapshots_by_id() {
.list_usage_records(admin_observability::UsageQuery {
range,
filter: admin_observability::UsageFilter {
routing_group_refs: Vec::new(),
provider_account_ref: Some("acct_team".to_owned()),
..admin_observability::UsageFilter::default()
},
@@ -504,7 +504,7 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
.list_usage_records(UsageRecordQuery {
range,
filter: UsageRecordFilter {
search: Some("observe_success".to_owned()),
search: Some("rimary".to_owned()),
..UsageRecordFilter::default()
},
current_page: 1,
@@ -517,6 +517,51 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
database.close().await;
}
#[tokio::test]
async fn usage_search_should_ignore_request_ids_and_preserve_exact_filters() {
let Some(database) = TestDatabase::create("usage_search_without_request_ids").await else {
return;
};
let now = Utc::now();
seed_observability_facts(&database.pool, now)
.await
.expect("seed observability facts");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
for search in [
"req_observe",
"req_observe_success",
"upstream_req",
"upstream_req_success",
] {
assert_usage_search_ids(&database.pool, range, search, &[]).await;
}
for filter in [
UsageRecordFilter {
request_id: Some("req_observe_success".to_owned()),
..Default::default()
},
UsageRecordFilter {
upstream_request_id: Some("upstream_req_success".to_owned()),
..Default::default()
},
] {
let page = observability_repository(&database.pool)
.list_usage_records(UsageRecordQuery {
range,
filter,
current_page: 1,
page_size: ObservabilityPageSize::new(10).expect("page size"),
})
.await
.expect("exact request identity filter");
assert_eq!(page.total, 1);
assert_eq!(page.items[0].id, "req_observe_success");
}
database.close().await;
}
#[tokio::test]
async fn usage_search_should_match_account_email_and_name_prefixes() {
let Some(database) = TestDatabase::create("usage_account_prefix").await else {
@@ -1687,7 +1732,6 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
.list_usage_records(admin_observability::UsageQuery {
range,
filter: admin_observability::UsageFilter {
routing_group_refs: Vec::new(),
client_api_key_ref: Some("key_observe".to_owned()),
request_id: Some("req_observe_success".to_owned()),
provider_account_ref: Some("acct_observe".to_owned()),
@@ -1700,7 +1744,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
attempt_index: Some(1),
response_id: Some("resp_observe_success".to_owned()),
upstream_request_id: Some("upstream_req_success".to_owned()),
search: Some("req_observe_success".to_owned()),
search: Some("upstream-model".to_owned()),
},
current_page: 1,
page_size: ObservabilityPageSize::new(10).expect("page size"),
@@ -1716,7 +1760,6 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
.list_usage_records(admin_observability::UsageQuery {
range,
filter: admin_observability::UsageFilter {
routing_group_refs: Vec::new(),
outcome: Some(
admin_observability::RequestOutcome::new("provider_future_state")
.expect("bounded other outcome"),
@@ -11,6 +11,7 @@ mod authorization;
mod deletion;
mod quota_forecast;
mod timestamps;
mod usage_projection;
use chrono::{TimeDelta, Utc};
use gateway_admin::{
@@ -9,6 +9,66 @@ use gateway_store::postgres::{
UsageRecordQuery,
};
#[tokio::test]
async fn quota_forecast_fractional_slots_keep_microsecond_edges_and_as_of_ties() {
let Some(database) = TestDatabase::create("forecast_fractional_slots").await else {
return;
};
PgProviderAccountRepository::new(database.pool.clone())
.insert_provider_account(account("acct_fraction", "user-fraction"))
.await
.unwrap();
let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
let duration = MAX_FORECAST_HISTORY_POINTS as i64 * 1_000_000 + 64;
let end = start + TimeDelta::microseconds(duration);
for (id, completed) in [
("before_edge", 1_000_000),
("after_edge", 1_000_001),
("as_of_a", duration),
("as_of_b", duration),
("pending", duration + 1),
] {
seed_model_request(
&database.pool,
ModelRequestSeed {
request_id: id,
account_id: "acct_fraction",
provider_kind: "openai",
model: "gpt-test",
total_tokens: 100,
cost_amount: "1",
started_at: start,
},
)
.await
.unwrap();
sqlx::query("update model_requests set completed_at=$2,provider_observation_json='{}'::jsonb where id=$1")
.bind(id).bind(start+TimeDelta::microseconds(completed)).execute(&database.pool).await.unwrap();
}
let history = admin_account_store(&database.pool)
.load_quota_forecast_history(&AccountUsageWindowQuery {
account_id: "acct_fraction".to_owned(),
key: "fraction".to_owned(),
range: TimeRange { start, end },
})
.await
.unwrap();
assert_eq!(
history
.points
.iter()
.map(|point| point.usage.request_count)
.collect::<Vec<_>>(),
vec![1, 2, 4]
);
assert_eq!(history.usage.request_count, 4);
assert_eq!(history.usage.tokens, 400);
assert_eq!(history.pending_request_count, 1);
database.close().await;
}
#[tokio::test]
async fn quota_forecast_excludes_openai_prewarm_but_preserves_inference_and_audit() {
let Some(database) = TestDatabase::create("forecast_prewarm").await else {
@@ -341,3 +401,219 @@ async fn quota_forecast_bounds_documents_without_sampling_away_token_totals() {
assert_eq!(history.points.last().unwrap().usage.tokens, 25_600);
database.close().await;
}
#[tokio::test]
async fn quota_forecast_carries_unsampled_usage_between_document_points() {
let Some(database) = TestDatabase::create("quota_forecast_bucket_prefix").await else {
return;
};
PgProviderAccountRepository::new(database.pool.clone())
.insert_provider_account(account("acct_prefix", "user-prefix"))
.await
.unwrap();
let start = Utc::now() - TimeDelta::hours(6);
let end = start + TimeDelta::minutes(MAX_FORECAST_HISTORY_POINTS as i64);
for (id, second, document) in [
("before_doc", 10, false),
("first_doc", 20, true),
("tie_doc", 20, true),
("after_doc", 40, false),
("no_document_bucket", 70, false),
("third_doc", 130, true),
("after_last_doc", 150, false),
("late", 180, true),
] {
seed_model_request(
&database.pool,
ModelRequestSeed {
request_id: id,
account_id: "acct_prefix",
provider_kind: "openai",
model: "gpt-test",
total_tokens: 100,
cost_amount: "1",
started_at: start + TimeDelta::seconds(second),
},
)
.await
.unwrap();
if document {
sqlx::query(
"update model_requests set provider_observation_json = '{}'::jsonb where id = $1",
)
.bind(id)
.execute(&database.pool)
.await
.unwrap();
}
}
sqlx::query("update model_requests set completed_at = $1 where id = 'late'")
.bind(end + TimeDelta::seconds(1))
.execute(&database.pool)
.await
.unwrap();
let history = admin_account_store(&database.pool)
.load_quota_forecast_history(&AccountUsageWindowQuery {
account_id: "acct_prefix".to_owned(),
key: "week".to_owned(),
range: TimeRange { start, end },
})
.await
.unwrap();
assert_eq!(history.points.len(), 2);
assert_eq!(history.points[0].usage.request_count, 3);
assert_eq!(history.points[0].usage.tokens, 300);
assert_eq!(history.points[1].usage.request_count, 6);
assert_eq!(history.points[1].usage.tokens, 600);
assert_eq!(history.usage.request_count, 7);
assert_eq!(history.usage.tokens, 700);
assert_eq!(history.pending_request_count, 1);
database.close().await;
}
#[tokio::test]
async fn quota_forecast_summaries_match_source_during_moves_deletes_and_rebuilds() {
use super::super::drain_analytics;
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistory;
fn assert_same(actual: &QuotaForecastHistory, expected: &QuotaForecastHistory) {
assert_eq!(actual.usage, expected.usage);
assert_eq!(actual.pending_request_count, expected.pending_request_count);
assert_eq!(actual.points.len(), expected.points.len());
for (actual, expected) in actual.points.iter().zip(&expected.points) {
assert_eq!(actual.started_at, expected.started_at);
assert_eq!(actual.completed_at, expected.completed_at);
assert_eq!(actual.usage, expected.usage);
}
}
let Some(database) = TestDatabase::create("forecast_completion_projection").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
for id in ["acct_forecast_projection", "acct_forecast_moved"] {
repository
.insert_provider_account(account(id, id))
.await
.unwrap();
}
let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T00:00:00Z")
.unwrap()
.with_timezone(&Utc);
for index in 0..200 {
seed_model_request(
&database.pool,
ModelRequestSeed {
request_id: &format!("forecast_projection_{index}"),
account_id: "acct_forecast_projection",
provider_kind: "openai",
model: "gpt-test",
total_tokens: index + 10,
cost_amount: "0.125",
started_at: start + TimeDelta::minutes(index * 3),
},
)
.await
.unwrap();
}
sqlx::query("update model_requests set provider_observation_json='{}'::jsonb where substring(id from '[0-9]+$')::int % 9=0")
.execute(&database.pool).await.unwrap();
// 完成时间与开始时间跨越多个桶,包含未来完成、缺失 Token、未知费用与恢复记录
sqlx::query("update model_requests set completed_at=completed_at+interval '90 minutes' where substring(id from '[0-9]+$')::int % 7=0")
.execute(&database.pool).await.unwrap();
sqlx::query("update model_requests set total_tokens=null,input_tokens=null,cost_source='unavailable',cost_amount=null,cost_currency=null where id='forecast_projection_31'")
.execute(&database.pool).await.unwrap();
sqlx::query(
"update model_requests set recovered_at=completed_at,recovery_request_id='forecast_projection_48',recovery_attempt_count=1,request_observation_json=request_observation_json || '{\"recovery\":{\"retryDelayMs\":0,\"totalLatencyMs\":1}}'::jsonb where id='forecast_projection_47'",
)
.execute(&database.pool)
.await
.unwrap();
sqlx::query("update model_requests set outcome='cancelled' where id='forecast_projection_63'")
.execute(&database.pool)
.await
.unwrap();
sqlx::query("update model_requests set cost_currency='EUR' where id='forecast_projection_83'")
.execute(&database.pool)
.await
.unwrap();
let store = admin_account_store(&database.pool);
let queries = [
AccountUsageWindowQuery {
account_id: "acct_forecast_projection".to_owned(),
key: "complete".to_owned(),
range: TimeRange {
start,
end: start + TimeDelta::hours(10),
},
},
AccountUsageWindowQuery {
account_id: "acct_forecast_projection".to_owned(),
key: "edges".to_owned(),
range: TimeRange {
start: start + TimeDelta::microseconds(1),
end: start + TimeDelta::minutes(593) + TimeDelta::microseconds(1),
},
},
AccountUsageWindowQuery {
account_id: "acct_forecast_moved".to_owned(),
key: "moved".to_owned(),
range: TimeRange {
start,
end: start + TimeDelta::hours(10),
},
},
];
let mut source = Vec::new();
for query in &queries {
source.push(store.load_quota_forecast_history(query).await.unwrap());
}
drain_analytics(&database.pool).await;
for (query, expected) in queries.iter().zip(&source) {
assert_same(
&store.load_quota_forecast_history(query).await.unwrap(),
expected,
);
}
sqlx::query("update model_requests set provider_account_ref='acct_forecast_moved',started_at=$1,completed_at=$1+interval '1 second',total_tokens=1234,cost_amount=2.5 where id='forecast_projection_72'")
.bind(start+TimeDelta::minutes(40)).execute(&database.pool).await.unwrap();
sqlx::query("delete from model_requests where id='forecast_projection_47'")
.execute(&database.pool)
.await
.unwrap();
sqlx::query("update model_requests set completed_at=$1 where id='forecast_projection_196'")
.bind(start + TimeDelta::minutes(590))
.execute(&database.pool)
.await
.unwrap();
let mut pending = Vec::new();
for query in &queries {
pending.push(store.load_quota_forecast_history(query).await.unwrap());
}
assert_eq!(pending[2].usage.request_count, 1);
assert_eq!(pending[2].usage.tokens, 1234);
drain_analytics(&database.pool).await;
for (query, expected) in queries.iter().zip(&pending) {
assert_same(
&store.load_quota_forecast_history(query).await.unwrap(),
expected,
);
}
// 显式重建中的区间由窄事实接管,发布后仍与同一源结果一致
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
.execute(&database.pool).await.unwrap();
for (query, expected) in queries.iter().zip(&pending) {
assert_same(
&store.load_quota_forecast_history(query).await.unwrap(),
expected,
);
}
drain_analytics(&database.pool).await;
for (query, expected) in queries.iter().zip(&pending) {
assert_same(
&store.load_quota_forecast_history(query).await.unwrap(),
expected,
);
}
database.close().await;
}
@@ -0,0 +1,150 @@
//! 验证账号同步用量在汇总、待投影修改和删除之间保持源事实一致
use super::super::drain_analytics;
use super::*;
#[tokio::test]
async fn quota_windows_reconcile_moved_requests_and_pending_deletions_without_waiting_for_worker() {
let Some(database) = TestDatabase::create("account_window_source_reconciliation").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
for id in ["acct_usage_before", "acct_usage_after"] {
repository
.insert_provider_account(account(id, id))
.await
.unwrap();
}
let start = chrono::DateTime::from_timestamp(Utc::now().timestamp().div_euclid(3600) * 3600, 0)
.unwrap()
- TimeDelta::hours(2);
for (id, offset, tokens, cost) in [("usage_stays", 5, 11, "0"), ("usage_moves", 40, 23, "2.5")]
{
seed_model_request(
&database.pool,
ModelRequestSeed {
request_id: id,
account_id: "acct_usage_before",
provider_kind: "openai",
model: "old-model",
total_tokens: tokens,
cost_amount: cost,
started_at: start + TimeDelta::minutes(offset),
},
)
.await
.unwrap();
}
sqlx::query("update model_requests set cached_input_tokens=null,reasoning_output_tokens=null where id in ('usage_stays','usage_moves')")
.execute(&database.pool).await.unwrap();
drain_analytics(&database.pool).await;
let windows = [
AccountUsageWindowQuery {
account_id: "acct_usage_before".to_owned(),
key: "old".to_owned(),
range: TimeRange {
start,
end: start + TimeDelta::hours(1),
},
},
AccountUsageWindowQuery {
account_id: "acct_usage_after".to_owned(),
key: "new".to_owned(),
range: TimeRange {
start: start + TimeDelta::hours(1),
end: start + TimeDelta::hours(2),
},
},
];
let store = admin_account_store(&database.pool);
let before = store.load_account_usage_by_windows(&windows).await.unwrap();
let initial = before.iter().find(|usage| usage.key == "old").unwrap();
assert_eq!(initial.usage.total_tokens, Some(34));
assert_eq!(initial.usage.cached_input_tokens, None);
assert_eq!(initial.usage.costs[0].amount.as_str(), "2.5");
let ranked = store
.list_accounts(
AccountListQuery {
page: 1,
page_size: PageSize::new(1).unwrap(),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: Some(AccountSort {
field: AccountSortField::Usage,
direction: SortDirection::Desc,
}),
},
Default::default(),
)
.await
.unwrap();
assert_eq!(ranked.items[0].account.id, "acct_usage_before");
sqlx::query("update model_requests set provider_account_ref='acct_usage_after',upstream_model_id='new-model',total_tokens=null,input_tokens=7,output_tokens=8,cached_input_tokens=0,cost_amount=4.75,started_at=$1,completed_at=$1+interval '1 minute',downstream_committed_at=$1+interval '1 minute',recovery_lease_expires_at=$1+interval '2 minutes' where id='usage_moves'")
.bind(start+TimeDelta::minutes(70)).execute(&database.pool).await.unwrap();
let changed = store.load_account_usage_by_windows(&windows).await.unwrap();
let old = changed.iter().find(|usage| usage.key == "old").unwrap();
let new = changed.iter().find(|usage| usage.key == "new").unwrap();
assert_eq!(old.usage.request_count, 1);
assert_eq!(old.usage.total_tokens, Some(11));
assert_eq!(old.usage.last_used_at, Some(start + TimeDelta::minutes(5)));
assert_eq!(
old.usage.costs[0].amount.as_str(),
"0",
"a real zero-cost request retains its currency"
);
assert_eq!(new.usage.request_count, 1);
assert_eq!(
new.usage.total_tokens,
Some(15),
"quota windows retain component fallback for missing provider total"
);
assert_eq!(new.usage.cached_input_tokens, Some(0));
assert_eq!(new.usage.reasoning_output_tokens, None);
assert_eq!(new.usage.models[0].model, "new-model");
assert_eq!(new.usage.costs[0].amount.as_str(), "4.75");
for field in [AccountSortField::Usage, AccountSortField::LastUsedAt] {
let ranked = store
.list_accounts(
AccountListQuery {
page: 1,
page_size: PageSize::new(1).unwrap(),
provider_kind: None,
group_filter: None,
search: None,
status: None,
sort: Some(AccountSort {
field,
direction: SortDirection::Desc,
}),
},
Default::default(),
)
.await
.unwrap();
assert_eq!(ranked.items[0].account.id, "acct_usage_after");
}
// 独立源表真值不经过同一汇总代码,防止仅比较两个相同错误的读取分支
let truth:(i64,i64,String)=sqlx::query_as("select count(*)::bigint,sum(coalesce(total_tokens,input_tokens+output_tokens))::bigint,sum(cost_amount)::text from model_requests where provider_account_ref='acct_usage_after' and started_at >= $1 and started_at < $2")
.bind(windows[1].range.start).bind(windows[1].range.end).fetch_one(&database.pool).await.unwrap();
assert_eq!(truth.0, new.usage.request_count as i64);
assert_eq!(truth.1, new.usage.total_tokens.unwrap() as i64);
assert_eq!(
truth.2.parse::<gateway_store::DecimalAmount>().unwrap(),
new.usage.costs[0].amount
);
sqlx::query("delete from model_requests where id='usage_stays'")
.execute(&database.pool)
.await
.unwrap();
let deleted = store.load_account_usage_by_windows(&windows).await.unwrap();
let old = deleted.iter().find(|usage| usage.key == "old").unwrap();
assert_eq!(old.usage.request_count, 0);
assert_eq!(old.usage.last_used_at, None);
assert!(old.usage.models.is_empty());
assert!(old.usage.costs.is_empty());
database.close().await;
}
@@ -19,7 +19,10 @@ async fn test_pool() -> Option<PgPool> {
#[test]
fn query_budget_rejects_an_empty_budget() {
assert!(ObservabilityQueryBudget::try_new(0, Duration::from_secs(1)).is_err());
assert!(
ObservabilityQueryBudget::try_new(0, Duration::from_secs(1), Duration::from_secs(10))
.is_err()
);
}
#[tokio::test]
@@ -27,7 +30,9 @@ async fn running_query_deadline_releases_the_slot() {
let Some(pool) = test_pool().await else {
return;
};
let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(10)).unwrap();
let budget =
ObservabilityQueryBudget::try_new(1, Duration::from_millis(10), Duration::from_millis(250))
.unwrap();
let error = budget
.run(&pool, "never completes", |connection| async move {
let result = std::future::pending::<Result<(), StoreError>>().await;
@@ -46,13 +51,77 @@ async fn running_query_deadline_releases_the_slot() {
pool.close().await;
}
#[tokio::test]
async fn queued_time_is_included_in_the_operation_deadline() {
let Some(pool) = test_pool().await else {
return;
};
let budget =
ObservabilityQueryBudget::try_new(1, Duration::from_secs(2), Duration::from_secs(2))
.unwrap();
let (occupied_tx, occupied_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let occupied_budget = budget.clone();
let occupied_pool = pool.clone();
let occupied = tokio::spawn(async move {
occupied_budget
.run(
&occupied_pool,
"hold before queued deadline",
|connection| async move {
occupied_tx.send(()).unwrap();
release_rx.await.unwrap();
drop(connection);
Ok(())
},
)
.await
.unwrap();
});
occupied_rx.await.unwrap();
let started = tokio::time::Instant::now();
let release = tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(2) / 5).await;
release_tx.send(()).unwrap();
});
let error = budget
.run(
&pool,
"queued operation deadline",
|connection| async move {
let result = std::future::pending::<Result<(), StoreError>>().await;
drop(connection);
result
},
)
.await
.expect_err("queue and execution share one deadline");
assert!(
matches!(error, StoreError::Unavailable { message, .. } if message.contains("execution budget"))
);
assert!(
started.elapsed() < Duration::from_secs(2) + Duration::from_millis(200),
"acquiring a slot must not restart the operation budget"
);
release.await.unwrap();
occupied.await.unwrap();
budget
.run(&pool, "after queued timeout", |_connection| async {
Ok(())
})
.await
.unwrap();
pool.close().await;
}
#[tokio::test]
async fn query_budget_times_out_without_exceeding_its_connection_limit() {
let Some(pool) = test_pool().await else {
return;
};
let budget = ObservabilityQueryBudget::try_new(4, Duration::from_millis(10))
.expect("valid query budget");
let budget =
ObservabilityQueryBudget::try_new(4, Duration::from_millis(10), Duration::from_secs(10))
.expect("valid query budget");
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let mut releases = Vec::new();
let mut occupied = Vec::new();
@@ -110,7 +179,8 @@ async fn queued_query_runs_after_one_slot_is_released() {
return;
};
let budget =
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1)).expect("valid query budget");
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1), Duration::from_secs(10))
.expect("valid query budget");
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
let mut releases = Vec::new();
let mut occupied = Vec::new();
@@ -183,7 +253,9 @@ async fn operation_timeout_keeps_the_slot_until_the_database_query_finishes() {
let Some(pool) = test_pool().await else {
return;
};
let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(100)).unwrap();
let budget =
ObservabilityQueryBudget::try_new(1, Duration::from_millis(100), Duration::from_secs(2))
.unwrap();
let (pid_tx, pid_rx) = oneshot::channel();
let error = budget
.run(
@@ -196,15 +268,18 @@ async fn operation_timeout_keeps_the_slot_until_the_database_query_finishes() {
.map_err(database_error)?;
pid_tx.send(pid).unwrap();
let mut tx = connection.begin().await.map_err(database_error)?;
sqlx::query("set local statement_timeout = '10s'")
sqlx::query("select set_config('statement_timeout',$1,true)")
.bind(format!("{}ms", Duration::from_secs(2).as_millis()))
.execute(&mut *tx)
.await
.map_err(database_error)?;
sqlx::query("select pg_sleep(6)")
sqlx::query("select pg_sleep($1)")
.bind(Duration::from_secs(2).as_secs_f64() * 0.6)
.execute(&mut *tx)
.await
.map_err(database_error)?;
sqlx::query("select pg_sleep(9)")
sqlx::query("select pg_sleep($1)")
.bind(Duration::from_secs(2).as_secs_f64() * 0.9)
.execute(&mut *tx)
.await
.map_err(database_error)?;
@@ -271,7 +346,9 @@ async fn dropping_an_operation_keeps_the_slot_through_database_rollback() {
let Some(pool) = test_pool().await else {
return;
};
let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(100)).unwrap();
let budget =
ObservabilityQueryBudget::try_new(1, Duration::from_millis(100), Duration::from_secs(10))
.unwrap();
let running_pool = pool.clone();
let running_budget = budget.clone();
let (pid_tx, pid_rx) = oneshot::channel();
@@ -2,11 +2,97 @@
use chrono::{TimeDelta, Utc};
use gateway_store::postgres::{
ObservabilityPageSize, ObservabilityRange, UsageRecordFilter, UsageRecordQuery,
ObservabilityPageSize, ObservabilityRange, OpsErrorFilter, OpsErrorQuery, UsageRecordFilter,
UsageRecordQuery,
};
use super::{TestDatabase, analytics::seed, observability_repository};
#[tokio::test]
async fn projected_page_counts_correct_pending_moves_deletions_and_model_or() {
let Some(database) = TestDatabase::create("usage_projected_pages").await else {
return;
};
let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
for index in 0..40 {
seed(
&database.pool,
&format!("projected_{index:02}"),
start + TimeDelta::minutes(index),
&[],
)
.await;
}
super::drain_analytics(&database.pool).await;
let repository = observability_repository(&database.pool);
for stage in 0..3 {
if stage == 1 {
sqlx::query("update model_requests set started_at=started_at+interval '30 minutes',recovery_lease_expires_at=recovery_lease_expires_at+interval '30 minutes',completed_at=completed_at+interval '30 minutes',requested_model_id='model_b',upstream_model_id='model_b' where id='projected_06'")
.execute(&database.pool).await.unwrap();
sqlx::query("delete from model_requests where id='projected_17'")
.execute(&database.pool)
.await
.unwrap();
sqlx::query("update model_requests set outcome='failed' where id='projected_21'")
.execute(&database.pool)
.await
.unwrap();
seed(
&database.pool,
"projected_new",
start + TimeDelta::seconds(30),
&[],
)
.await;
} else if stage == 2 {
super::drain_analytics(&database.pool).await;
}
for edge in [0, 31] {
let range = ObservabilityRange::new(
start + TimeDelta::seconds(edge),
start + TimeDelta::hours(1) - TimeDelta::seconds(edge),
)
.unwrap();
for model in [None, Some("model_a"), Some("model_b")] {
// 真值只读取实际源版本,不依赖被测汇总或分页辅助函数
let expected:Vec<String> = sqlx::query_scalar("select id from model_requests where started_at >= $1 and started_at < $2 and outcome='succeeded' and ($3::text is null or requested_model_id=$3 or upstream_model_id=$3) order by started_at desc,id desc")
.bind(range.start).bind(range.end).bind(model).fetch_all(&database.pool).await.unwrap();
for current_page in [1, 3, 6, 8] {
let page = repository
.list_usage_records(UsageRecordQuery {
range,
filter: UsageRecordFilter {
client_api_key_ref: Some("key_analytics".to_owned()),
model: model.map(str::to_owned),
..Default::default()
},
current_page,
page_size: ObservabilityPageSize::new(7).unwrap(),
})
.await
.unwrap();
assert_eq!(
page.total,
expected.len() as u64,
"stage={stage}, edge={edge}, model={model:?}"
);
assert_eq!(
page.items.iter().map(|row| &row.id).collect::<Vec<_>>(),
expected
.iter()
.skip((current_page as usize - 1) * 7)
.take(7)
.collect::<Vec<_>>()
);
}
}
}
}
database.close().await;
}
#[tokio::test]
async fn usage_pages_preserve_order_and_exact_totals_from_both_ends() {
let Some(database) = TestDatabase::create("usage_bidirectional_pages").await else {
@@ -70,3 +156,127 @@ async fn usage_pages_preserve_order_and_exact_totals_from_both_ends() {
}
database.close().await;
}
#[tokio::test]
async fn usage_pages_cross_minutes_and_keep_exact_counts_after_source_changes() {
let Some(database) = TestDatabase::create("usage_time_pages").await else {
return;
};
let start = Utc::now() - TimeDelta::hours(1);
for index in 0..29 {
seed(
&database.pool,
&format!("cross_{index:02}"),
start + TimeDelta::seconds((index / 3) * 100),
&["group_page"],
)
.await;
}
let repository = observability_repository(&database.pool);
let range = ObservabilityRange::new(start, start + TimeDelta::minutes(20)).unwrap();
for removed in [None, Some("cross_16")] {
if let Some(id) = removed {
sqlx::query("delete from model_requests where id = $1")
.bind(id)
.execute(&database.pool)
.await
.unwrap();
}
let expected = (0..29)
.rev()
.map(|index| format!("cross_{index:02}"))
.filter(|id| Some(id.as_str()) != removed)
.collect::<Vec<_>>();
for current_page in 1..=6 {
let page = repository
.list_usage_records(UsageRecordQuery {
range,
filter: UsageRecordFilter {
client_api_key_ref: Some("key_analytics".to_owned()),
search: Some("model_".to_owned()),
..Default::default()
},
current_page,
page_size: ObservabilityPageSize::new(7).unwrap(),
})
.await
.unwrap();
assert_eq!(page.total, expected.len() as u64);
assert_eq!(
page.items.iter().map(|item| &item.id).collect::<Vec<_>>(),
expected
.iter()
.skip((current_page as usize - 1) * 7)
.take(7)
.collect::<Vec<_>>(),
"page={current_page}, removed={removed:?}",
);
}
}
database.close().await;
}
#[tokio::test]
async fn ops_pages_merge_sources_across_minutes_with_stable_ties() {
let Some(database) = TestDatabase::create("ops_time_pages").await else {
return;
};
let start = Utc::now() - TimeDelta::hours(1);
let mut expected = Vec::new();
for index in 0..17 {
let id = format!("error_{index:02}");
let at = start + TimeDelta::seconds((index / 3) * 100);
seed(&database.pool, &id, at, &[]).await;
sqlx::query("update model_requests set error_kind = 'upstream_error' where id = $1")
.bind(&id)
.execute(&database.pool)
.await
.unwrap();
sqlx::query(
"insert into ops_events (id, model_request_id, attempt_index, level, component,
operation, failure_kind, message, created_at)
values ($1, $1, 1, 'error', 'routing', 'responses', 'upstream_error', 'fixture', $2)",
)
.bind(&id)
.bind(at + TimeDelta::minutes(1))
.execute(&database.pool)
.await
.unwrap();
for source in ["model_request", "ops_event"] {
expected.push((at + TimeDelta::minutes(1), format!("{source}:{id}")));
}
}
expected.sort_by(|a, b| b.cmp(a));
let repository = observability_repository(&database.pool);
let range = ObservabilityRange::new(start, start + TimeDelta::minutes(20)).unwrap();
for current_page in 1..=7 {
let page = repository
.list_ops_errors(OpsErrorQuery {
range,
filter: OpsErrorFilter {
client_api_key_ref: Some("key_analytics".to_owned()),
search: Some("error_".to_owned()),
..Default::default()
},
current_page,
page_size: ObservabilityPageSize::new(7).unwrap(),
})
.await
.unwrap();
assert_eq!(page.total, expected.len() as u64);
assert_eq!(
page.items
.iter()
.map(|item| &item.stable_sort_id)
.collect::<Vec<_>>(),
expected
.iter()
.skip((current_page as usize - 1) * 7)
.take(7)
.map(|(_, id)| id)
.collect::<Vec<_>>(),
"page={current_page}",
);
}
database.close().await;
}
+2
View File
@@ -27,3 +27,5 @@ d4b599066d7382a1c9cb6b3c003200046579e4fce3531f232cbd665a9d92c542 0026_codex_pri
8c9bbecd8315a26e928213a82a2ec651c35976e0643b82caef1d1bcfeee2916f 0027_request_observation_projection.sql
31d6668707186d7c19cdd507e20162a08c7fbb8ba41ff4a4439d0a5e08369cad 0028_request_analytics.sql
1923f14095dac5f808a530db002c37e17168afa0778bc7f3da768c818133eec9 0029_usage_indexes_and_identity_snapshots.sql
ab964fcdf56aa1f595fffaf02556aeebce5037f7bfa1ea6f4fef594112eaba6a 0030_analytics_usage_dimensions.sql
9d6e5254ee4a7aec40c4fc5f1904300228c57826941b962fa78cc6e550fa99f8 0031_analytics_forecast_completion.sql
@@ -0,0 +1,33 @@
-- 完整交付用量复用现有统计状态,最近使用时间保持可合并的独立极值
alter table request_analytics_15m add column last_used_at timestamptz;
alter table request_analytics_hourly add column last_used_at timestamptz;
-- 新指标只重建时间桶,不清空窄事实,也不重置已经完成的历史投影游标
insert into request_analytics_pending_buckets(bucket_start, oldest_pending_at)
select distinct date_bin(interval '15 minutes', started_at, timestamptz '2000-01-01 UTC'), now()
from request_analytics_facts
on conflict (bucket_start) do update
set change_version = nextval('request_analytics_change_version');
-- 撤销最近一次模型贡献后按真实源定位极值,避免扫描整个账号历史
create index model_requests_delivered_account_model on model_requests
(provider_account_ref,(coalesce(upstream_model_id,requested_model_id,'')),started_at desc,id desc)
where outcome='succeeded' and downstream_committed_at is not null
and (provider_kind is distinct from 'openai' or request_kind is distinct from 'prewarm')
and ((client_transport='websocket' and client_status_code is null) or client_status_code between 200 and 399)
and (requested_model_id is not null or upstream_model_id is not null or image_generation_requested
or input_tokens is not null or output_tokens is not null or cached_input_tokens is not null
or cache_write_input_tokens is not null or reasoning_output_tokens is not null
or image_input_tokens is not null or image_output_tokens is not null
or total_tokens is not null or cost_amount is not null);
-- 预测只定位各采样桶最后一份 Provider 文档,宽 JSON 不进入索引
create index model_requests_forecast_documents on model_requests
(provider_account_ref,completed_at desc,id desc) include(started_at)
where provider_observation_json is not null and outcome <> 'running';
-- 增量构建按字节序推进游标,索引排序必须与游标合同一致
create index request_analytics_facts_build_cursor on request_analytics_facts
(started_at,id collate "C");
create index request_analytics_15m_build_cursor on request_analytics_15m
(bucket_start,collection collate "C",dimensions collate "C",outcome collate "C");
@@ -0,0 +1,11 @@
-- 完成时间边界只回读五分钟内的窄事实,Provider 文档继续使用源表采样索引
create index request_analytics_facts_account_completion on request_analytics_facts
(provider_account_ref,completed_at,started_at)
where completed_at is not null and outcome <> 'running';
-- 新完成时间维度与既有统计共用可恢复构建,旧事实无需再次投影
insert into request_analytics_pending_buckets(bucket_start, oldest_pending_at)
select distinct date_bin(interval '15 minutes', started_at, timestamptz '2000-01-01 UTC'), now()
from request_analytics_facts
on conflict (bucket_start) do update
set change_version = nextval('request_analytics_change_version');
+8
View File
@@ -28,6 +28,10 @@
配置加载会忽略未知字段,并在启动控制台提示字段名;不输出对应值。缺少必填字段时会指出缺项并停止启动,
已知字段的类型和取值仍需合法;可选字段省略时使用默认值
`store.pool.observability_timeout_seconds` 默认 120 秒,控制观测查询与统计 Worker 的操作预算,
缓存等待和统计事务的 SQL 时限沿用该配置。修改后重启应用;连接排队仍受
`store.pool.acquire_timeout_seconds` 约束,模型请求时限由 `api.request_timeout_seconds` 单独控制
后端从当前目录向上查找 `deploy/config.yaml`,相对数据、日志和静态资源路径以该文件所在目录解析
Compose 通过环境变量将监听和数据库地址设为容器内地址,并指定镜像中的静态资源目录
@@ -560,6 +564,10 @@ docker compose -f deploy/compose.yaml up -d --no-build --wait
升级后通过管理端版本接口或容器 image digest 确认运行实例的版本和 revision
管理端请求默认等待 120 秒。直接构建前端时可通过进程环境 `VITE_API_TIMEOUT_MS` 指定毫秒数,
例如 `VITE_API_TIMEOUT_MS=180000 pnpm --dir frontend build`,取值须为正整数。
该值在构建时写入静态资源,修改后需要重新构建和部署前端。单个请求显式指定的时限仍优先
构建元数据通过一次性进程环境传入:
```bash
+1
View File
@@ -53,6 +53,7 @@ store:
# pool:
# max_connections: 20
# acquire_timeout_seconds: 5
# observability_timeout_seconds: 120 # 观测操作保护时限,不是性能目标
admin:
# 不活动期限与从登录起计算的最长有效期,单位为分钟
+13 -10
View File
@@ -1248,8 +1248,8 @@ HTTP 请求头及新建 WS 的握手提示按当时的最终出站档位构造
`capacity.totalSlots` 为 `number | null`:`null` 表示可用成员中存在继承无限并发的账号,`0` 表示没有可用槽位。
分组容量只统计普通池,`capacity.usedSlots` 返回其实际在途数;Redis 不可用时为 `null`
分组费用按请求执行时实际服务账号的分组快照归属,不按 Client Key 绑定的分组分摊。
账号属于多个组时,各组均包含该请求费用;之后调整账号分组不重写历史归属
分组费用按当前组成员实际完成请求的费用汇总,不按 Client Key 绑定的分组分摊。
账号属于多个组时,各组均包含该账号的请求费用;调整账号分组后,留存窗口内费用随当前成员关系重新汇总
## 7. Client Key
@@ -1830,8 +1830,11 @@ Dashboard 的 `accountUsage[]` 由后端提供 `usageWindow`、`metricLabel`、`
Dashboard 默认 `period=today`,用量、诊断和错误查询默认 `period=7d`,范围参数遵循[页面时间合同](#页面时间合同)。
Dashboard 返回原始 `asOf` 及 `asOfDisplay`,作为当前数据的查询锚点
用量查询可组合页码/游标、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、
request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可取 `model`、`account`、
请求明细列表可组合页码、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、
request/response/upstream ID、outcome 与搜索文本。`search` 仅供明细及错误列表使用;
汇总、洞察和诊断不接受搜索参数,管理观测接口不接受路由分组筛选。页面搜索不改变顶部统计范围
诊断 `dimension` 可取 `model`、`account`、
`apiKey`、`provider`、`transport`、`failureClass`、`status`。诊断按请求量降序返回最多 100 项,
同请求量按维度标识稳定排列
@@ -1863,13 +1866,13 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告(流内
缺少终态声明时保留首次声明。这些值仅表示上游报告,不作为模型真实性证明,也不参与路由、
聚合或本地计价模型选择。没有保存上游模型报告的记录保持未知
请求记录列表的 `search` 使用字面量前缀匹配,支持请求 ID、Client Key ID / 名称、
账号 ID、账号邮箱与名称、请求 / 上游模型 ID、上游请求 ID。密钥名称不区分大小写,其他字段区分大小写。
密钥名称按当前密钥记录检索,改名后使用新名称,删除后仍可按 Client Key ID 查询历史记录。
账号邮箱与名称按请求记录的历史快照检索,
不随当前账号修改或删除而改变;`%`、`_` 和 `\` 均按普通字符处理,不作为搜索通配符
请求记录列表的 `search` 使用字面量前缀匹配,支持 Client Key ID / 名称、
账号 ID、账号邮箱与名称、请求 / 上游模型 ID。密钥名称不区分大小写,其他字段区分大小写。
密钥名称、账号邮箱与名称按请求记录的历史快照检索,
不随当前密钥或账号修改、删除而改变;`%`、`_` 和 `\` 均按普通字符处理,不作为搜索通配符
请求记录按模型执行 ID 或上游 ID 查询;只有入口 ID 时需结合时间与入口日志定位。
请求记录的 `requestId`、`upstreamRequestId` 参数按模型执行 ID 或上游 ID 精确查询;
搜索框不匹配请求 ID,只有入口 ID 时需结合时间与入口日志定位。
请求记录和错误列表支持按密钥名称搜索,不支持密钥值或可见前缀搜索
### 记录范围与统计口径
+15 -3
View File
@@ -865,14 +865,21 @@ OpenAI 订阅周期属于按需个人信息,不是额度事实。Admin 账号
## 10. 观测与后台任务
管理观测查询与统计 Worker 共用 PostgreSQL 连接预算,为请求执行保留连接容量。每项操作使用一个受预算约束的连接;
管理观测查询与统计 Worker 共用 PostgreSQL 连接预算,为请求执行保留连接容量。每项操作使用一个受预算约束的连接,排队计入总操作预算;
Store 声明配置后的读取时限,缓存沿用该值,统计事务只在事务内设置 SQL 时限,归还连接时不污染其他查询;
超时或调用方取消后,名额持续覆盖 SQLx 的协议收尾、事务回滚和连接归还,不能随查询 Future 提前释放。
请求列表的精确总数与候选页共用数据库快照,按距首尾的较短方向读取,返回顺序和页码语义保持一致。
请求列表的精确总数与候选页共用数据库快照,深页先按时间分段定位,再按距区间首尾的较短方向读取,
返回顺序和页码语义保持一致。
Store 从请求源事实维护窄事实及十五分钟、小时统计状态,账号用量与分组费用复用同一套已交付用量指标。
完整时间桶在数据库内合并,边界与不能由汇总表达的筛选在数据库内精确聚合;向应用返回的状态按输出分组组织。
计数、金额与分布状态的旧贡献撤销、新贡献合并和事实替换共用事务,构建期间只调整已扫描前缀,
不可相减的极值按受影响分组重算。同步账号用量查询在同一快照补齐待投影源版本,避免异步投影改变原有读数合同。
### 额度预测
账号容量预测属于 Admin 的只读派生规则,不参与 quota 权威状态、调度或金额结算。Store 通过专用采样端口
在同一 SQL 快照内返回截至观测时间的累计数值及有界历史 Provider 文档;对应 Provider 复用协议解析器解释
在同一读事务快照内返回截至观测时间的累计数值及有界历史 Provider 文档;对应 Provider 复用协议解析器解释
文档,不在历史查询中刷新上游额度。观测通过 Core 原有
请求结算持久化,账号关联由实际执行上下文确定,窗口身份与归属必须匹配当前额度窗口。Admin 按中立的额度
事实选择近期进度段预测剩余量,再加本周期已记录用量形成周期总量。
@@ -890,6 +897,11 @@ OpenAI 订阅周期属于按需个人信息,不是额度事实。Admin 账号
- 重置后从新段重新累计,无法精确归属的用量不补算;只有账号在窗口开始前加入且无已知断点时,才允许以窗口累计量替代增量采样
- 历史点抽样不缩减区间累计用量,同一完成时间共用同一累计值
预测数值复用统计投影的十五分钟开始时间桶,并以账号与五分钟完成时间桶分组。
完整完成时间桶合并累计量,跨采样截止值的小桶、开始时间边界和重建区间读取窄事实;
同一快照撤销待投影旧版本并加入当前源版本。该维度保留恢复请求的原有预测口径,
不改变其他统计排除恢复请求的规则。Provider 文档仍按账号和完成时间索引读取,每个采样槽只保留最后一份
### 请求记录与交付
Provider 独占首事件、首字、首推理和首正文的观测,以及上游计时的协议解析,通过类型化响应观测交给 Core。
+81
View File
@@ -0,0 +1,81 @@
# PostgreSQL 3100 万请求优化与验收
本报告对应 [Issue #413](https://github.com/zyycn/codex-proxy-rs/issues/413),范围为大数据量下的统计、请求分页、账号用量、分组费用与额度预测
3100 万条源请求、窄事实和完整汇总已构建并独立对账。当前 HTTP 矩阵执行 167 次请求,包含 145 次合法业务请求、22 次预期参数或权限拒绝,未出现超时或非预期失败。按实际注册路由计,84 条已有通过记录、70 条未覆盖、6 条不适用;不宣称全接口或长期负载验收通过
## 问题与实现边界
原账号用量读取在命中事实超过 10 万条时拒绝查询;其他统计的事实回退与汇总读取也有固定行数门槛。仅放宽门槛会继续放大应用端逐行合并、深页 OFFSET 和预测窗口排序的成本
实现继续使用原生 PostgreSQL、现有窄事实、十五分钟/小时汇总和投影 Worker:
- 账号与 Key 的必要模型联合维度进入共享汇总,完整桶和精确边界在数据库内聚合,不以命中数据量拒绝查询
- 账号窗口、用量排序、分组费用和分页计数在同一读快照补齐待投影修改与删除,保留精确总数、金额及未知值语义
- 请求深页通过时间分段定位,错误列表合并两类来源后保持稳定排序
- 预测使用开始时间与完成时间二维汇总,累计量保留完整窗口,Provider 文档独立有界采样
- 投影在同一事务撤销旧贡献并加入新贡献;普通更新维护构建前缀,不反复重启整个桶
- 观测操作与前端默认保护时限为 120 秒,可分别配置;30 秒为优化方向,不能将提高超时时限当作查询提速
使用记录搜索支持密钥、账号与模型,不匹配请求 ID;请求 ID 的显式精确筛选仍保留。汇总、洞察和诊断不接受搜索参数,管理观测接口不接受路由分组筛选
当前合同与职责见 [API 参考](../api.md)、[系统架构](../architecture.md#10-观测与后台任务)与[部署配置](../../deploy/README.md),本报告不重复字段清单
## 环境与样本
| 项目 | 实际条件 |
| --- | --- |
| 数据 | 31 天、3100 万条合成源请求;50 个账号、100 个 Key、20 个模型、10 个分组 |
| 时间范围 | `2026-09-12T04:45:00Z` 至 `2026-10-13T04:45:00Z` |
| 数据库 | PostgreSQL 18.6,容器限制 4 CPU、4 GiB 内存,持久磁盘 |
| 主要参数 | shared_buffers 512 MiB、work_mem 16 MiB、max_parallel_workers 8、max_parallel_workers_per_gather 2、JIT 开启 |
| 持久性 | fsync、full_page_writes、synchronous_commit 均开启 |
| 应用 | release 构建;核心轮次重启应用清空应用缓存,16 个查询客户端并发 |
| 其他轮次 | 剩余只读接口 4 并发;合成配置写入、鉴权及本机 Provider 场景串行执行 |
| 缓存边界 | 未清空数据库与宿主文件缓存,应用冷缓存不等于数据库冷缓存 |
造数与离线汇总构建使用的临时资源配置已恢复,搜索索引试验已撤销。3100 万条容量基线不是生产 Worker 全量回填吞吐的测量结果;生产投影的正确性通过真实源表回归及样本对账验证
## HTTP 结果
以下为当前矩阵中的单次短样本,不能推断稳定的 P95/P99。并发样本包含资源竞争,不能与隔离单请求耗时直接当作前后优化对照
| 场景 | 耗时 |
| --- | ---: |
| 用量明细宽范围模型前缀搜索 | 32.63 秒 |
| 账号用量降序 | 20.50 秒 |
| 账号最近使用降序 | 20.39 秒 |
| 账号额度预测 | 19.94 秒 |
| Key 维度诊断 | 18.46 秒 |
| Dashboard 概览 | 12.30 秒 |
| 账号维度诊断 | 10.82 秒 |
| 用量洞察 | 6.20 秒 |
| 用量汇总 | 6.17 秒 |
| 用量中间页 | 3.60 秒 |
| 用量首/尾/超尾页 | 3.46/1.63/1.39 秒 |
| 错误列表第 6200 页 | 1.70 秒 |
宽范围搜索命中大量明细,32.63 秒超过 30 秒优化期望,但未触发 120 秒保护时限。本次保留该性能局限,不再为此扩展索引或引入其他存储系统
正确性核对覆盖源请求总量、Token、费用、主要筛选组合、诊断分母及分页结果。移除的搜索与分组参数按预期返回 400,权限负例按预期返回拒绝;这些负例不计作合法容量查询通过
## 回归与自审范围
- 超过 10 万条真实源请求经过生产 Worker,验证账号/Key 汇总、热点边界和未知 Token;锁住窄事实表时,干净完整桶仍可读取
- 待投影跨账号/模型/时间桶修改、删除、重复投递、写入失败回滚及构建前缀持续更新
- 精确分页首中尾页、超尾页、跨时间段与同时间戳排序,两类错误来源合并
- 预测微秒边界、同完成时间、截止时间、缺失费用、恢复请求、重建与待投影补差
- 连接排队计入总预算,超时与取消后的事务回滚及连接归还仍持有查询名额
- 分组费用沿用当前成员关系;账号窗口的 Token 兜底与普通统计的未知值口径分别保留
- 投影与极值重算共用维度定义,已删除请求 ID 搜索捷径及淘汰的逐行读取映射
- Rustfmt、相关 Store/Admin/API 回归、严格 Clippy、架构边界、迁移冻结清单、前端类型构建与 ESLint 对应实际最终工作区执行
页面仅调整搜索提示;按本次任务要求不提供截图
## 升级与验证局限
迁移 0030/0031 增加统计字段和索引,将既有时间桶排入可恢复重建,不清空窄事实或重置已完成的源投影游标。重建中的桶走事实回退,索引创建与桶重建仍需要额外 I/O、磁盘空间和时间;未完成生产环境在线升级的时延与负载验收
本次收尾聚焦 3100 万请求瓶颈,不再扩展外围插件、OAuth、备份和升级场景。数据库冷缓存、长期持续负载、生产 Worker 全量回填性能及未覆盖外部集成保留为验证局限;小规模生命周期回归不等于相同场景的 3100 万级持续写入验收
临时造数、HTTP 驱动、SQL 计划及逐请求记录保存在忽略跟踪的 `.runtime/capacity`,不进入工程提交。完整请求与路由结果为 `http-31m-latest.json`/`http-31m-latest.md`,原失败轮次分别保留,最终检查日志使用 `final-*` 前缀
+4
View File
@@ -1,5 +1,9 @@
/// <reference types="vite/client" />
interface ImportMetaEnv {
readonly VITE_API_TIMEOUT_MS?: string
}
declare module '@fontsource-variable/inter'
declare module '@fontsource-variable/jetbrains-mono'
+5 -1
View File
@@ -1,2 +1,6 @@
export const API_BASE_URL = import.meta.env.DEV ? '/dev' : ''
export const API_TIMEOUT_MS = 30_000
const timeout = Number(import.meta.env.VITE_API_TIMEOUT_MS ?? 120_000)
if (!Number.isSafeInteger(timeout) || timeout <= 0 || timeout > 2_147_483_647) {
throw new Error('VITE_API_TIMEOUT_MS must be a positive 32-bit integer')
}
export const API_TIMEOUT_MS = timeout
+1 -2
View File
@@ -490,7 +490,6 @@ interface UsageRangeQuery {
provider?: string
model?: string
statusCode?: number
search?: string
}
interface PageQuery {
@@ -498,7 +497,7 @@ interface PageQuery {
pageSize: number
}
type UsagePageQuery = UsageRangeQuery & PageQuery
type UsagePageQuery = UsageRangeQuery & PageQuery & { search?: string }
type OpsErrorPageQuery = UsagePageQuery
@@ -18,7 +18,7 @@ const search = defineModel<string>('search', { required: true })
<template>
<div class="flex w-full items-center gap-3" role="group" aria-label="使用记录筛选与操作">
<div class="min-w-0 flex-1 sm:w-96 sm:flex-none">
<ZInput v-model="search" placeholder="请求、密钥名称、账号或模型" aria-label="搜索使用记录:请求、密钥名称、账号或模型" class="w-full">
<ZInput v-model="search" placeholder="密钥名称、账号或模型" aria-label="搜索使用记录:密钥名称、账号或模型" class="w-full">
<template #prefix>
<Search class="size-4.5 text-cp-text-tertiary" />
</template>