mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
perf(store): scale analytics and pagination for large request histories (#423)
Aggregate analytics in PostgreSQL, reconcile synchronous usage with pending source changes, and preserve exact pagination and forecast totals across large histories. Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -66,7 +66,7 @@ pub struct AccountGroupCapacity {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupUsage {
|
||||
pub today_usd: DecimalAmount,
|
||||
/// 当前 usage retention 窗口内、按请求发生时 routing group 快照归属的累计成本
|
||||
/// 当前 usage retention 窗口内、当前组成员实际完成请求的累计成本
|
||||
pub retained_total_usd: DecimalAmount,
|
||||
}
|
||||
|
||||
|
||||
@@ -157,7 +157,6 @@ impl OtherRequestOutcome {
|
||||
/// 用量记录过滤条件
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
|
||||
pub struct UsageFilter {
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
@@ -211,7 +210,6 @@ impl DashboardQuery {
|
||||
/// 运维错误过滤条件
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct OpsErrorFilter {
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
|
||||
@@ -432,6 +432,9 @@ pub trait UsageBillingProjector: Send + Sync {
|
||||
/// 用量、趋势、诊断与运维错误的只读能力
|
||||
#[async_trait]
|
||||
pub trait ObservabilityStore: Send + Sync {
|
||||
/// 读取实现声明的总预算,缓存等待沿用同一配置,避免外层提前截断
|
||||
fn query_timeout(&self) -> std::time::Duration;
|
||||
|
||||
/// 缓存必须与删除和后台投影提交的版本一致
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64>;
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ use tokio::{sync::Mutex, time::Instant};
|
||||
const CAPACITY: usize = 64;
|
||||
const TTL: Duration = Duration::from_secs(15);
|
||||
const MAX_STALE: Duration = Duration::from_secs(60);
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
type Slot<T> = Arc<Mutex<Option<Cached<T>>>>;
|
||||
struct Cached<T> {
|
||||
@@ -72,7 +71,7 @@ impl<K: Eq + Hash, T: AnalyticsValue> Cache<K, T> {
|
||||
{
|
||||
// 刷新失败后的同键等待也计入预算,避免串行重试把延迟放大为队列长度倍数
|
||||
tokio::time::timeout(
|
||||
REQUEST_TIMEOUT,
|
||||
store.query_timeout(),
|
||||
self.get_within_budget(key, store, ttl, load),
|
||||
)
|
||||
.await
|
||||
@@ -174,12 +173,6 @@ fn with_age<T: AnalyticsValue>(mut value: T, stale: bool) -> T {
|
||||
value
|
||||
}
|
||||
|
||||
fn normalized(mut filter: UsageFilter) -> UsageFilter {
|
||||
filter.routing_group_refs.sort_unstable();
|
||||
filter.routing_group_refs.dedup();
|
||||
filter
|
||||
}
|
||||
|
||||
type UsageKey = (TimeRange, UsageFilter);
|
||||
pub(crate) struct CachedObservabilityStore {
|
||||
inner: Arc<dyn ObservabilityStore>,
|
||||
@@ -202,6 +195,10 @@ impl CachedObservabilityStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for CachedObservabilityStore {
|
||||
fn query_timeout(&self) -> Duration {
|
||||
self.inner.query_timeout()
|
||||
}
|
||||
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
self.inner.analytics_cache_generation().await
|
||||
}
|
||||
@@ -241,7 +238,6 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
let filter = normalized(filter);
|
||||
self.insights
|
||||
.get(
|
||||
((range, filter.clone()), granularity),
|
||||
@@ -259,7 +255,6 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
let filter = normalized(filter);
|
||||
self.summaries
|
||||
.get((range, filter.clone()), self.inner.as_ref(), TTL, || {
|
||||
self.inner.usage_summary(range, filter)
|
||||
@@ -273,7 +268,6 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
dimension: DiagnosticDimension,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<DiagnosticsObservation> {
|
||||
let filter = normalized(filter);
|
||||
self.diagnostics
|
||||
.get(
|
||||
((range, filter.clone()), dimension, limit),
|
||||
|
||||
@@ -747,6 +747,10 @@ impl ClientKeyStore for UnavailableStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnavailableStore {
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
_: DashboardQuery,
|
||||
|
||||
@@ -945,6 +945,10 @@ impl FixtureObservabilityStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for FixtureObservabilityStore {
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
query: DashboardQuery,
|
||||
@@ -1406,23 +1410,23 @@ async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_sets() {
|
||||
async fn usage_summary_cache_should_isolate_key_identity_and_filters() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
for (key, groups) in [
|
||||
("key-a", vec!["deleted-b", "deleted-a", "deleted-b"]),
|
||||
("key-a", vec!["deleted-a", "deleted-b"]),
|
||||
("key-b", vec!["deleted-a", "deleted-b"]),
|
||||
("key-a", vec!["deleted-a"]),
|
||||
for (key, model) in [
|
||||
("key-a", None),
|
||||
("key-a", None),
|
||||
("key-b", None),
|
||||
("key-a", Some("model-a")),
|
||||
] {
|
||||
service
|
||||
.usage_summary(
|
||||
range,
|
||||
UsageFilter {
|
||||
client_api_key_ref: Some(key.to_owned()),
|
||||
routing_group_refs: groups.into_iter().map(str::to_owned).collect(),
|
||||
model: model.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -1431,16 +1435,16 @@ async fn usage_summary_cache_should_isolate_key_identity_and_normalize_group_set
|
||||
}
|
||||
let calls = store.usage_summary_filters.lock().unwrap();
|
||||
assert_eq!(calls.len(), 3);
|
||||
assert_eq!(calls[0].routing_group_refs, ["deleted-a", "deleted-b"]);
|
||||
assert_eq!(calls[0].client_api_key_ref.as_deref(), Some("key-a"));
|
||||
assert_eq!(calls[1].client_api_key_ref.as_deref(), Some("key-b"));
|
||||
assert_eq!(calls[2].routing_group_refs, ["deleted-a"]);
|
||||
assert_eq!(calls[2].model.as_deref(), Some("model-a"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow_recovery() {
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store.set_dashboard_delay(StdDuration::from_secs(7));
|
||||
store.set_dashboard_delay(store.query_timeout().mul_f64(0.7));
|
||||
store.summary_fail.store(true, Ordering::SeqCst);
|
||||
let services = observability_services(store.clone()).await;
|
||||
let service = services.observability();
|
||||
@@ -1453,7 +1457,7 @@ async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow
|
||||
);
|
||||
assert!(first.is_err() && second.is_err() && third.is_err() && fourth.is_err());
|
||||
assert!(
|
||||
started.elapsed() <= StdDuration::from_secs(10),
|
||||
started.elapsed() <= store.query_timeout(),
|
||||
"同键失败不能使后续请求逐个等待完整数据库超时: {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
|
||||
@@ -29,7 +29,6 @@ impl DashboardQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub routing_group_refs: Option<String>,
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub kind: Option<String>,
|
||||
@@ -77,7 +76,6 @@ impl DetailQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DiagnosticsQuery {
|
||||
pub routing_group_refs: Option<String>,
|
||||
pub dimension: Option<String>,
|
||||
pub period: Option<String>,
|
||||
pub as_of: Option<i64>,
|
||||
@@ -86,7 +84,6 @@ pub struct DiagnosticsQuery {
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
impl DiagnosticsQuery {
|
||||
@@ -100,7 +97,6 @@ impl DiagnosticsQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct OpsQuery {
|
||||
pub routing_group_refs: Option<String>,
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub kind: Option<String>,
|
||||
@@ -301,7 +297,6 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, Wi
|
||||
})
|
||||
});
|
||||
Ok(domain::UsageFilter {
|
||||
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?,
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
@@ -318,6 +313,15 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, Wi
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn analytics_filter(
|
||||
query: &UsageQuery,
|
||||
) -> Result<domain::UsageFilter, WireValidationError> {
|
||||
if query.search.is_some() {
|
||||
return Err(WireValidationError::new("search"));
|
||||
}
|
||||
usage_filter(query)
|
||||
}
|
||||
|
||||
pub(crate) fn usage_command(
|
||||
query: &UsageQuery,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
@@ -361,7 +365,6 @@ pub(crate) fn ops_command(
|
||||
timezone,
|
||||
)?,
|
||||
filter: domain::OpsErrorFilter {
|
||||
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())?,
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
@@ -400,25 +403,3 @@ fn validate_pagination(
|
||||
}
|
||||
Ok((current_page, page_size))
|
||||
}
|
||||
|
||||
/// 逗号分隔的稳定分组身份按集合归一化,不要求分组当前仍存在
|
||||
pub(crate) fn routing_group_refs(value: Option<&str>) -> Result<Vec<String>, WireValidationError> {
|
||||
let Some(value) = value else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let mut refs = Vec::new();
|
||||
for item in value.split(',') {
|
||||
let item = item.trim();
|
||||
if item.is_empty()
|
||||
|| item.len() > 128
|
||||
|| item.chars().any(char::is_control)
|
||||
|| refs.len() >= 32
|
||||
{
|
||||
return Err(WireValidationError::new("routingGroupRefs"));
|
||||
}
|
||||
refs.push(item.to_owned());
|
||||
}
|
||||
refs.sort();
|
||||
refs.dedup();
|
||||
Ok(refs)
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ where
|
||||
state.admin_services().timezone(),
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = usage_filter(&query).map_err(map_wire_error)?;
|
||||
let filter = analytics_filter(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
@@ -184,7 +184,7 @@ where
|
||||
state.admin_services().timezone(),
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = usage_filter(&query).map_err(map_wire_error)?;
|
||||
let filter = analytics_filter(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
@@ -215,12 +215,9 @@ where
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
routing_group_refs: routing_group_refs(query.routing_group_refs.as_deref())
|
||||
.map_err(map_wire_error)?,
|
||||
provider_kind: non_empty(query.provider),
|
||||
model: non_empty(query.model),
|
||||
status_code: parse_status(query.status_code).map_err(map_wire_error)?,
|
||||
search: non_empty(query.search),
|
||||
..domain::UsageFilter::default()
|
||||
};
|
||||
let result = state
|
||||
|
||||
@@ -1374,6 +1374,10 @@ impl AccountRuntimeStore for UnusedStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
query: DashboardQuery,
|
||||
|
||||
@@ -1245,7 +1245,7 @@ async fn calendar_query_rejects_an_anchor_outside_the_local_calendar() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_and_ops_queries_should_forward_deleted_group_id_sets() {
|
||||
async fn observability_queries_reject_group_filters_and_analytics_search() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
use axum::{
|
||||
body::Body,
|
||||
@@ -1256,39 +1256,50 @@ async fn request_and_ops_queries_should_forward_deleted_group_id_sets() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let router = observability::router::<AdminTestState>().with_state(fixture.state());
|
||||
for path in ["/api/admin/usage/records", "/api/admin/operations/errors"] {
|
||||
for (path, parameter) in [
|
||||
("/api/admin/usage/records", "routingGroupRefs=group-a"),
|
||||
("/api/admin/operations/errors", "routingGroupRefs=group-a"),
|
||||
(
|
||||
"/api/admin/usage/records/summary",
|
||||
"routingGroupRefs=group-a",
|
||||
),
|
||||
(
|
||||
"/api/admin/usage/insights/overview",
|
||||
"routingGroupRefs=group-a",
|
||||
),
|
||||
(
|
||||
"/api/admin/usage/insights/diagnostics",
|
||||
"routingGroupRefs=group-a",
|
||||
),
|
||||
("/api/admin/usage/records/summary", "search=request-prefix"),
|
||||
(
|
||||
"/api/admin/usage/insights/overview",
|
||||
"search=request-prefix",
|
||||
),
|
||||
(
|
||||
"/api/admin/usage/insights/diagnostics",
|
||||
"search=request-prefix",
|
||||
),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!(
|
||||
"{path}?routingGroupRefs=deleted-b,deleted-a,deleted-b"
|
||||
))
|
||||
.uri(format!("{path}?{parameter}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_group_filter")
|
||||
.header("x-request-id", "req_rejected_observability_filter")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"{path}: {}",
|
||||
String::from_utf8_lossy(&body)
|
||||
response.status(),
|
||||
StatusCode::BAD_REQUEST,
|
||||
"{path}?{parameter}"
|
||||
);
|
||||
}
|
||||
let observations = fixture.observations.lock().unwrap();
|
||||
assert_eq!(
|
||||
observations.records[0].filter.routing_group_refs,
|
||||
["deleted-a", "deleted-b"]
|
||||
);
|
||||
assert_eq!(
|
||||
observations.errors[0].filter.routing_group_refs,
|
||||
["deleted-a", "deleted-b"]
|
||||
);
|
||||
assert!(observations.records.is_empty());
|
||||
assert!(observations.errors.is_empty());
|
||||
}
|
||||
|
||||
@@ -126,6 +126,7 @@ async fn connect(
|
||||
let observability_query_budget = postgres::ObservabilityQueryBudget::try_new(
|
||||
config.pool.observability_max_connections(),
|
||||
config.pool.acquire_timeout(),
|
||||
config.pool.observability_timeout(),
|
||||
)?;
|
||||
let redis_client = ::redis::Client::open(config.redis_url()?)
|
||||
.map_err(|source| redis_unavailable("create Redis client", source))?;
|
||||
@@ -180,8 +181,11 @@ async fn connect(
|
||||
)),
|
||||
admin_account_runtime,
|
||||
Arc::new(
|
||||
postgres::PgAccountGroupRepository::new(pool.clone())
|
||||
.with_timezone(config.timezone),
|
||||
postgres::PgAccountGroupRepository::new(
|
||||
pool.clone(),
|
||||
observability_query_budget.clone(),
|
||||
)
|
||||
.with_timezone(config.timezone),
|
||||
),
|
||||
Arc::new(postgres::PgProxyRepository::new(pool.clone())),
|
||||
),
|
||||
|
||||
@@ -36,6 +36,7 @@ pub struct StoreConfig {
|
||||
pub struct StorePoolConfig {
|
||||
pub max_connections: u32,
|
||||
pub acquire_timeout_seconds: u64,
|
||||
pub observability_timeout_seconds: u64,
|
||||
}
|
||||
|
||||
impl Default for StorePoolConfig {
|
||||
@@ -43,6 +44,7 @@ impl Default for StorePoolConfig {
|
||||
Self {
|
||||
max_connections: 20,
|
||||
acquire_timeout_seconds: 5,
|
||||
observability_timeout_seconds: 120,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,9 +60,22 @@ impl StorePoolConfig {
|
||||
.to_owned(),
|
||||
});
|
||||
}
|
||||
if self.observability_timeout_seconds == 0
|
||||
|| self.observability_timeout().as_millis() > i32::MAX as u128
|
||||
{
|
||||
return Err(StoreError::InvalidData {
|
||||
source: None,
|
||||
entity: "store config",
|
||||
message: "pool.observability_timeout_seconds must be positive and fit PostgreSQL statement_timeout".to_owned(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) const fn observability_timeout(self) -> Duration {
|
||||
Duration::from_secs(self.observability_timeout_seconds)
|
||||
}
|
||||
|
||||
/// 管理观测查询可并发占用的连接数;始终为数据面保留约 20% 的池容量
|
||||
#[must_use]
|
||||
pub const fn observability_max_connections(self) -> u32 {
|
||||
|
||||
@@ -31,9 +31,9 @@ use crate::{
|
||||
};
|
||||
|
||||
use super::{
|
||||
PgRuntimeSettingsRepository, RuntimeSettingsRepository, account_summary_from_row,
|
||||
append_admin_audit_event_in_transaction, bump_config_revision_in_transaction,
|
||||
completed_usage_fact_predicate,
|
||||
ObservabilityQueryBudget, PgRuntimeSettingsRepository, RuntimeSettingsRepository,
|
||||
account_summary_from_row, append_admin_audit_event_in_transaction,
|
||||
bump_config_revision_in_transaction,
|
||||
};
|
||||
|
||||
const ENTITY: &str = "account group";
|
||||
@@ -43,13 +43,15 @@ const ENTITY: &str = "account group";
|
||||
pub struct PgAccountGroupRepository {
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
pool: PgPool,
|
||||
query_budget: ObservabilityQueryBudget,
|
||||
}
|
||||
|
||||
impl PgAccountGroupRepository {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
pub fn new(pool: PgPool, query_budget: ObservabilityQueryBudget) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
query_budget,
|
||||
timezone: Default::default(),
|
||||
}
|
||||
}
|
||||
@@ -74,9 +76,14 @@ impl PgAccountGroupRepository {
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?
|
||||
.ok_or_else(|| not_found(id.as_str()))?;
|
||||
let costs = group_costs(&self.pool, &[id.as_str().to_owned()], self.timezone)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let costs = group_costs(
|
||||
&self.pool,
|
||||
&self.query_budget,
|
||||
&[id.as_str().to_owned()],
|
||||
self.timezone,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
if let Some(usage) = costs.get(id.as_str()) {
|
||||
record.usage = usage.clone();
|
||||
}
|
||||
@@ -152,7 +159,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
|
||||
.iter()
|
||||
.map(|record| record.id.as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let costs = group_costs(&self.pool, &group_ids, self.timezone)
|
||||
let costs = group_costs(&self.pool, &self.query_budget, &group_ids, self.timezone)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
for record in &mut items {
|
||||
@@ -601,71 +608,23 @@ fn group_ref(row: &sqlx::postgres::PgRow) -> StoreResult<AccountGroupRef> {
|
||||
|
||||
async fn group_costs(
|
||||
pool: &PgPool,
|
||||
budget: &ObservabilityQueryBudget,
|
||||
group_ids: &[String],
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<BTreeMap<String, AccountGroupUsage>> {
|
||||
if group_ids.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
// 用量按实际完成请求的账号统计:账号属于多个分组时计入每个所属分组,
|
||||
// 不再按 Client Key 绑定分组快照归属,避免多分组 Key 的费用重复出现在未承接请求的分组上
|
||||
let statement = format!(
|
||||
"with requested_groups(group_id) as (
|
||||
select unnest($1::text[])
|
||||
)
|
||||
select requested_groups.group_id,
|
||||
coalesce(sum(mr.cost_amount) filter (
|
||||
where mr.started_at >= $2
|
||||
), 0)::text as today_usd,
|
||||
coalesce(sum(mr.cost_amount), 0)::text as retained_total_usd
|
||||
from requested_groups
|
||||
cross join runtime_settings settings
|
||||
left join account_group_accounts membership
|
||||
on membership.account_group_id = requested_groups.group_id
|
||||
left join model_requests mr
|
||||
on mr.provider_account_ref = membership.provider_account_id
|
||||
and mr.started_at >= now() - make_interval(days => settings.usage_retention_days::int)
|
||||
and {completed_usage}
|
||||
and mr.cost_currency = 'USD'
|
||||
and mr.cost_amount is not null
|
||||
where settings.id = 1
|
||||
group by requested_groups.group_id"
|
||||
);
|
||||
// 动态片段仅为共享的固定 usage-fact predicate;group IDs 仍使用 bind
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(group_ids)
|
||||
.bind(
|
||||
timezone
|
||||
.day_start(chrono::Utc::now())
|
||||
.ok_or_else(|| invalid("invalid business date"))?,
|
||||
budget
|
||||
.run(
|
||||
pool,
|
||||
"load account group costs",
|
||||
|mut connection| async move {
|
||||
super::observability::account_group_usage(&mut connection, group_ids, timezone)
|
||||
.await
|
||||
},
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|source| unavailable("load account group costs", source))?;
|
||||
rows.into_iter()
|
||||
.map(|row| {
|
||||
let group_id: String = row
|
||||
.try_get("group_id")
|
||||
.map_err(|source| invalid("invalid group ID").with_source(source))?;
|
||||
let today = row
|
||||
.try_get::<String, _>("today_usd")
|
||||
.map_err(|source| invalid("invalid today cost").with_source(source))?;
|
||||
let retained_total = row
|
||||
.try_get::<String, _>("retained_total_usd")
|
||||
.map_err(|source| invalid("invalid retained total cost").with_source(source))?;
|
||||
Ok((
|
||||
group_id,
|
||||
AccountGroupUsage {
|
||||
today_usd: DecimalAmount::from_str(&today)
|
||||
.map_err(|source| invalid("invalid today cost").with_source(source))?,
|
||||
retained_total_usd: DecimalAmount::from_str(&retained_total).map_err(
|
||||
|source| invalid("invalid retained total cost").with_source(source),
|
||||
)?,
|
||||
},
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn count_value(row: &sqlx::postgres::PgRow, field: &str) -> StoreResult<u64> {
|
||||
|
||||
@@ -197,7 +197,7 @@ impl PgObservabilityRepository {
|
||||
)
|
||||
.await?;
|
||||
let account_usage =
|
||||
provider_account_usage(&mut snapshot.tx, account_usage_query).await?;
|
||||
provider_account_usage(&mut snapshot, account_usage_query).await?;
|
||||
let metadata = snapshot.metadata();
|
||||
snapshot.finish().await?;
|
||||
Ok((totals, trend, metadata, account_usage))
|
||||
@@ -290,12 +290,8 @@ impl PgObservabilityRepository {
|
||||
"load usage insights snapshot",
|
||||
|mut connection| async move {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?;
|
||||
let mut overview = snapshot.overview(range, &filter).await?;
|
||||
let trend = snapshot
|
||||
.trend(range, &filter, granularity, self.timezone, true)
|
||||
.await?;
|
||||
let costs = snapshot
|
||||
.projected_costs(range, &filter, granularity, self.timezone)
|
||||
let (mut overview, trend, costs) = snapshot
|
||||
.insights(range, &filter, granularity, self.timezone)
|
||||
.await?;
|
||||
overview.analytics = snapshot.metadata();
|
||||
snapshot.finish().await?;
|
||||
@@ -318,12 +314,9 @@ impl PgObservabilityRepository {
|
||||
&self.pool,
|
||||
"load provider account usage",
|
||||
|mut connection| async move {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?;
|
||||
snapshot.require_visible(Some(query.range)).await?;
|
||||
if let Some(range) = query.request_bucket_range {
|
||||
snapshot.require_visible(Some(range)).await?;
|
||||
}
|
||||
let usage = provider_account_usage(&mut snapshot.tx, query).await?;
|
||||
let mut snapshot =
|
||||
analytics::AnalyticsSnapshot::new_for_accounts(&mut connection).await?;
|
||||
let usage = provider_account_usage(&mut snapshot, query).await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(usage)
|
||||
},
|
||||
@@ -407,6 +400,10 @@ impl PgObservabilityRepository {
|
||||
|
||||
#[async_trait]
|
||||
impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
self.repository.query_budget.query_timeout()
|
||||
}
|
||||
|
||||
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
||||
self.repository
|
||||
.query_budget
|
||||
|
||||
@@ -0,0 +1,655 @@
|
||||
//! 账号用量复用汇总并补齐未投影源版本,保持同步查询的精确窗口口径
|
||||
|
||||
use super::query::push_summary_buckets;
|
||||
use super::*;
|
||||
use gateway_admin::model::{
|
||||
account_groups::AccountGroupUsage,
|
||||
accounts::{
|
||||
AccountCost, AccountModelUsage, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult,
|
||||
},
|
||||
};
|
||||
|
||||
type AccountStates = BTreeMap<(String, String), MetricState>;
|
||||
|
||||
impl AnalyticsSnapshot<'_> {
|
||||
pub(crate) async fn account_usage_states(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<AccountStates> {
|
||||
self.account_states(range, account_ids, true).await
|
||||
}
|
||||
|
||||
async fn account_states(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
by_model: bool,
|
||||
) -> StoreResult<AccountStates> {
|
||||
if account_ids.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
// 原同步接口在首次回填期间仍可查询;只有完整覆盖的区间才允许用汇总替代源事实
|
||||
if !self.projection_covers(range) {
|
||||
return self
|
||||
.account_fact_states("model_requests", range, account_ids, false, by_model)
|
||||
.await;
|
||||
}
|
||||
let mut states = BTreeMap::new();
|
||||
// 排序与分组费用只需要账号总量,不额外展开模型维度
|
||||
let (collection, model) = if by_model {
|
||||
(
|
||||
"account_model",
|
||||
"coalesce(nullif(dimensions[4],''),dimensions[3])",
|
||||
)
|
||||
} else {
|
||||
("account", "''::text")
|
||||
};
|
||||
let (hours, quarters) = self.summary_buckets(range, true).await?;
|
||||
for (table, seconds, buckets) in [
|
||||
("request_analytics_hourly", 3600, hours),
|
||||
("request_analytics_15m", 900, quarters),
|
||||
] {
|
||||
if buckets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"with selected as materialized (select dimensions[1] as account_id,{model} as model,counts,costs,last_used_at from {table} a where collection='{collection}' and schema_version="
|
||||
));
|
||||
query
|
||||
.push_bind(SCHEMA_VERSION)
|
||||
.push(" and histogram_version=")
|
||||
.push_bind(HISTOGRAM_VERSION)
|
||||
.push(" and dimensions[1]=any(")
|
||||
.push_bind(account_ids)
|
||||
.push("::text[])");
|
||||
push_summary_buckets(&mut query, &buckets, seconds);
|
||||
let counters = usage_counter_names();
|
||||
// 每行一次展开指标,避免对可能位于 TOAST 的同一 JSON 反复解码
|
||||
query.push(format!(
|
||||
"), totals as (select account_id,model,jsonb_build_object({}) as counts,max(last_used_at) as last_used_at from selected cross join lateral jsonb_to_record(counts) as metric({}) group by account_id,model),
|
||||
currencies as (select account_id,model,c.key,sum(c.value::numeric)::text as amount from selected cross join lateral jsonb_each_text(costs) c where c.key not in ('standard_usd','no_cache_usd') group by account_id,model,c.key),
|
||||
costs as (select account_id,model,jsonb_object_agg(key,amount) as costs from currencies group by account_id,model)
|
||||
select totals.*,coalesce(costs.costs,'{{}}'::jsonb) as costs,'{{}}'::jsonb as distributions from totals left join costs using(account_id,model)",
|
||||
counters.iter().map(|name| format!("'{name}',coalesce(sum(metric.{name}),0)")).collect::<Vec<_>>().join(","),
|
||||
counters.iter().map(|name| format!("{name} numeric")).collect::<Vec<_>>().join(",")
|
||||
));
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
postgres_unavailable("aggregate account usage summaries", error)
|
||||
})?;
|
||||
merge_account_rows(&mut states, rows)?;
|
||||
}
|
||||
for (start, end) in self.uncovered_spans(range).await? {
|
||||
let boundary = self
|
||||
.account_fact_states(
|
||||
"request_analytics_facts",
|
||||
ObservabilityRange { start, end },
|
||||
account_ids,
|
||||
false,
|
||||
by_model,
|
||||
)
|
||||
.await?;
|
||||
merge_account_states(&mut states, boundary)?;
|
||||
}
|
||||
if !self.has_pending_requests().await? {
|
||||
states.retain(|_, state| state.count("completed_request_count") > 0);
|
||||
return Ok(states);
|
||||
}
|
||||
let old = self
|
||||
.account_fact_states(
|
||||
"request_analytics_facts",
|
||||
range,
|
||||
account_ids,
|
||||
true,
|
||||
by_model,
|
||||
)
|
||||
.await?;
|
||||
let current = self
|
||||
.account_fact_states("model_requests", range, account_ids, true, by_model)
|
||||
.await?;
|
||||
let mut repair = BTreeSet::new();
|
||||
for (key, previous) in old {
|
||||
let state = states.entry(key.clone()).or_default();
|
||||
for (name, count) in &previous.counts {
|
||||
let value = state.counts.entry(name.clone()).or_default();
|
||||
*value = value
|
||||
.checked_sub(*count)
|
||||
.ok_or_else(|| invalid("account usage pending counter underflow"))?;
|
||||
}
|
||||
for (currency, amount) in &previous.costs {
|
||||
let before = state
|
||||
.costs
|
||||
.get(currency)
|
||||
.ok_or_else(|| invalid("account usage pending cost is missing"))?;
|
||||
let before = before
|
||||
.parse::<gateway_core::metering::Decimal>()
|
||||
.map_err(|_| invalid("invalid account cost"))?;
|
||||
let removed = amount
|
||||
.parse::<gateway_core::metering::Decimal>()
|
||||
.map_err(|_| invalid("invalid pending account cost"))?;
|
||||
let difference = before
|
||||
.scaled()
|
||||
.checked_sub(removed.scaled())
|
||||
.ok_or_else(|| invalid("account usage pending cost underflow"))?;
|
||||
let value = gateway_core::metering::Decimal::from_scaled(difference)
|
||||
.map_err(|_| invalid("invalid account cost difference"))?;
|
||||
state.costs.insert(currency.clone(), value.canonical());
|
||||
}
|
||||
repair.insert(key);
|
||||
}
|
||||
merge_account_states(&mut states, current)?;
|
||||
// 撤销最大值不能用减法;只对发生源版本替换的账号模型做索引定位
|
||||
let completed = completed_usage_fact_predicate("mr");
|
||||
for (account, model) in repair {
|
||||
let state = states
|
||||
.get_mut(&(account.clone(), model.clone()))
|
||||
.ok_or_else(|| invalid("pending account state disappeared"))?;
|
||||
if state.count("completed_request_count") == 0 {
|
||||
*state = MetricState::default();
|
||||
continue;
|
||||
}
|
||||
let statement = format!(
|
||||
"select started_at from model_requests mr where provider_account_ref=$1 and ($2::text is null or coalesce(upstream_model_id,requested_model_id,'')=$2) and started_at >= $3 and started_at < $4 and {completed} order by started_at desc,id desc limit 1"
|
||||
);
|
||||
state.last_used_at = sqlx::query_scalar(sqlx::AssertSqlSafe(statement))
|
||||
.bind(&account)
|
||||
.bind(by_model.then_some(&model))
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.fetch_optional(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("repair account latest usage", error))?;
|
||||
let zero_currencies = state
|
||||
.costs
|
||||
.iter()
|
||||
.filter(|(_, amount)| amount.as_str() == "0")
|
||||
.map(|(currency, _)| currency.clone())
|
||||
.collect::<Vec<_>>();
|
||||
for currency in zero_currencies {
|
||||
let statement = format!(
|
||||
"select exists(select 1 from model_requests mr where provider_account_ref=$1 and ($2::text is null or coalesce(upstream_model_id,requested_model_id,'')=$2) and started_at >= $3 and started_at < $4 and {completed} and cost_currency=$5 and cost_amount is not null)"
|
||||
);
|
||||
let exists: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(statement))
|
||||
.bind(&account)
|
||||
.bind(by_model.then_some(&model))
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.bind(¤cy)
|
||||
.fetch_one(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("repair account cost coverage", error))?;
|
||||
if !exists {
|
||||
state.costs.remove(¤cy);
|
||||
}
|
||||
}
|
||||
}
|
||||
states.retain(|_, state| state.count("completed_request_count") > 0);
|
||||
Ok(states)
|
||||
}
|
||||
|
||||
pub(super) async fn has_pending_requests(&mut self) -> StoreResult<bool> {
|
||||
if self.status.oldest_pending_at.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
sqlx::query_scalar("select exists(select 1 from request_analytics_pending_requests)")
|
||||
.fetch_one(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("check pending account usage", error))
|
||||
}
|
||||
|
||||
pub(super) fn projection_covers(&self, range: ObservabilityRange) -> bool {
|
||||
self.status.schema_version == SCHEMA_VERSION as u32
|
||||
&& self.status.histogram_version == HISTOGRAM_VERSION as u32
|
||||
&& (!self.status.building
|
||||
|| (self
|
||||
.status
|
||||
.coverage_start
|
||||
.is_some_and(|start| range.start >= start)
|
||||
&& self.status.coverage_end.is_some()))
|
||||
}
|
||||
|
||||
pub(crate) async fn account_request_counts(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<BTreeMap<(String, i64), u64>> {
|
||||
if account_ids.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
let mut counts = BTreeMap::new();
|
||||
// 公开小时图采用 UTC 整点;非整刻内部窗口直接在源表按相对小时分组
|
||||
if !self.projection_covers(range)
|
||||
|| range.start.timestamp().rem_euclid(900) != 0
|
||||
|| range.start.timestamp_subsec_nanos() != 0
|
||||
{
|
||||
let rows = self
|
||||
.account_request_fact_counts(
|
||||
"model_requests",
|
||||
range,
|
||||
range.start,
|
||||
account_ids,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
merge_request_counts(&mut counts, rows, false)?;
|
||||
return Ok(counts);
|
||||
}
|
||||
let (hours, quarters) = self
|
||||
.summary_buckets(range, range.start.timestamp().rem_euclid(3600) == 0)
|
||||
.await?;
|
||||
for (table, seconds, buckets) in [
|
||||
("request_analytics_hourly", 3600, hours),
|
||||
("request_analytics_15m", 900, quarters),
|
||||
] {
|
||||
if buckets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select dimensions[1] as account_id,floor(extract(epoch from (bucket_start-",
|
||||
);
|
||||
query.push_bind(range.start).push("::timestamptz))/3600)::bigint as bucket_index,coalesce(sum((counts->>'completed_request_count')::bigint),0)::bigint as request_count from ")
|
||||
.push(table).push(" a where collection='account' and schema_version=").push_bind(SCHEMA_VERSION)
|
||||
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION).push(" and dimensions[1]=any(").push_bind(account_ids).push("::text[])");
|
||||
push_summary_buckets(&mut query, &buckets, seconds);
|
||||
query.push(" group by dimensions[1],bucket_index");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
postgres_unavailable("read account hourly request summaries", error)
|
||||
})?;
|
||||
merge_request_counts(&mut counts, rows, false)?;
|
||||
}
|
||||
for (start, end) in self.uncovered_spans(range).await? {
|
||||
let rows = self
|
||||
.account_request_fact_counts(
|
||||
"request_analytics_facts",
|
||||
ObservabilityRange { start, end },
|
||||
range.start,
|
||||
account_ids,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
merge_request_counts(&mut counts, rows, false)?;
|
||||
}
|
||||
if self.has_pending_requests().await? {
|
||||
let old = self
|
||||
.account_request_fact_counts(
|
||||
"request_analytics_facts",
|
||||
range,
|
||||
range.start,
|
||||
account_ids,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
merge_request_counts(&mut counts, old, true)?;
|
||||
let current = self
|
||||
.account_request_fact_counts(
|
||||
"model_requests",
|
||||
range,
|
||||
range.start,
|
||||
account_ids,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
merge_request_counts(&mut counts, current, false)?;
|
||||
}
|
||||
Ok(counts)
|
||||
}
|
||||
|
||||
async fn account_request_fact_counts(
|
||||
&mut self,
|
||||
table: &'static str,
|
||||
range: ObservabilityRange,
|
||||
origin: DateTime<Utc>,
|
||||
account_ids: &[String],
|
||||
pending_only: bool,
|
||||
) -> StoreResult<Vec<sqlx::postgres::PgRow>> {
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select mr.provider_account_ref as account_id,floor(extract(epoch from (mr.started_at-",
|
||||
);
|
||||
query.push_bind(origin).push("::timestamptz))/3600)::bigint as bucket_index,count(*)::bigint as request_count from ").push(table).push(" mr");
|
||||
if pending_only {
|
||||
query.push(" join request_analytics_pending_requests p on p.request_id=mr.id");
|
||||
}
|
||||
query
|
||||
.push(" where mr.provider_account_ref=any(")
|
||||
.push_bind(account_ids)
|
||||
.push("::text[]) and mr.started_at >= ")
|
||||
.push_bind(range.start)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
query.push(" group by mr.provider_account_ref,bucket_index");
|
||||
query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
postgres_unavailable("aggregate account hourly request boundaries", error)
|
||||
})
|
||||
}
|
||||
|
||||
async fn account_fact_states(
|
||||
&mut self,
|
||||
table: &'static str,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
pending_only: bool,
|
||||
by_model: bool,
|
||||
) -> StoreResult<AccountStates> {
|
||||
let mut query = QueryBuilder::<Postgres>::new(usage_fact_aggregate_sql(table, by_model));
|
||||
if pending_only {
|
||||
query.push(
|
||||
" join request_analytics_pending_requests pending on pending.request_id=mr.id",
|
||||
);
|
||||
}
|
||||
query
|
||||
.push(" where mr.provider_account_ref=any(")
|
||||
.push_bind(account_ids)
|
||||
.push("::text[]) and mr.started_at >= ")
|
||||
.push_bind(range.start)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
query.push(" group by mr.provider_account_ref,mr.cost_currency");
|
||||
if by_model {
|
||||
query.push(",coalesce(mr.upstream_model_id,mr.requested_model_id,'')");
|
||||
}
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("aggregate account usage facts", error))?;
|
||||
let mut states = BTreeMap::new();
|
||||
merge_account_rows(&mut states, rows)?;
|
||||
Ok(states)
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_request_counts(
|
||||
counts: &mut BTreeMap<(String, i64), u64>,
|
||||
rows: Vec<sqlx::postgres::PgRow>,
|
||||
subtract: bool,
|
||||
) -> StoreResult<()> {
|
||||
for row in rows {
|
||||
let value = unsigned(&row, "request_count")?;
|
||||
let total = counts
|
||||
.entry((get(&row, "account_id")?, get(&row, "bucket_index")?))
|
||||
.or_default();
|
||||
*total = if subtract {
|
||||
total.checked_sub(value)
|
||||
} else {
|
||||
total.checked_add(value)
|
||||
}
|
||||
.ok_or_else(|| invalid("invalid account request count delta"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn merge_account_rows(
|
||||
states: &mut AccountStates,
|
||||
rows: Vec<sqlx::postgres::PgRow>,
|
||||
) -> StoreResult<()> {
|
||||
for row in rows {
|
||||
states
|
||||
.entry((get(&row, "account_id")?, get(&row, "model")?))
|
||||
.or_default()
|
||||
.merge(&MetricState::decode(&row)?)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn merge_account_states(states: &mut AccountStates, incoming: AccountStates) -> StoreResult<()> {
|
||||
for (key, state) in incoming {
|
||||
states.entry(key).or_default().merge(&state)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn usage_counter_names() -> Vec<String> {
|
||||
let mut names = [
|
||||
"completed_request_count",
|
||||
"image_request_count",
|
||||
"image_request_failed_count",
|
||||
"window_total_tokens",
|
||||
"provider_reported_count",
|
||||
"calculated_count",
|
||||
"unavailable_count",
|
||||
]
|
||||
.map(str::to_owned)
|
||||
.to_vec();
|
||||
for name in super::metrics::TOKEN_METRICS {
|
||||
names.push(name.to_owned());
|
||||
names.push(format!("{name}_known_count"));
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
fn usage_fact_aggregate_sql(table: &str, by_model: bool) -> String {
|
||||
let model = if by_model {
|
||||
"coalesce(mr.upstream_model_id,mr.requested_model_id,'')"
|
||||
} else {
|
||||
"''::text"
|
||||
};
|
||||
let mut counters = vec![
|
||||
"'completed_request_count',count(*)".to_owned(),
|
||||
"'image_request_count',count(*) filter(where image_generation_succeeded is true)".to_owned(),
|
||||
"'image_request_failed_count',count(*) filter(where image_generation_succeeded is false)".to_owned(),
|
||||
"'window_total_tokens',coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)+coalesce(output_tokens,0))),0)".to_owned(),
|
||||
];
|
||||
for name in super::metrics::TOKEN_METRICS {
|
||||
counters.push(format!(
|
||||
"'{name}',coalesce(sum({name}),0),'{name}_known_count',count({name})"
|
||||
));
|
||||
}
|
||||
for source in ["provider_reported", "calculated", "unavailable"] {
|
||||
counters.push(format!(
|
||||
"'{source}_count',count(*) filter(where cost_source='{source}')"
|
||||
));
|
||||
}
|
||||
format!("select mr.provider_account_ref as account_id,{model} as model,
|
||||
jsonb_build_object({}) as counts,case when mr.cost_currency is not null and sum(mr.cost_amount) is not null
|
||||
then jsonb_build_object(mr.cost_currency,sum(mr.cost_amount)::text) else '{{}}'::jsonb end as costs,
|
||||
'{{}}'::jsonb as distributions,max(mr.started_at) as last_used_at from {table} mr",counters.join(","))
|
||||
}
|
||||
|
||||
pub(crate) async fn account_usage_windows(
|
||||
connection: &mut ObservabilityConnection,
|
||||
windows: &[AccountUsageWindowQuery],
|
||||
) -> StoreResult<Vec<AccountUsageWindowResult>> {
|
||||
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
|
||||
// 相同窗口一次查询所有账号,避免列表逐账号重复读取同一套时间桶
|
||||
let mut grouped = BTreeMap::<(DateTime<Utc>, DateTime<Utc>), BTreeSet<String>>::new();
|
||||
for window in windows {
|
||||
grouped
|
||||
.entry((window.range.start, window.range.end))
|
||||
.or_default()
|
||||
.insert(window.account_id.clone());
|
||||
}
|
||||
let mut results = Vec::with_capacity(windows.len());
|
||||
for ((start, end), accounts) in grouped {
|
||||
let ids = accounts.into_iter().collect::<Vec<_>>();
|
||||
let states = snapshot
|
||||
.account_usage_states(ObservabilityRange { start, end }, &ids)
|
||||
.await?;
|
||||
for window in windows
|
||||
.iter()
|
||||
.filter(|window| window.range.start == start && window.range.end == end)
|
||||
{
|
||||
let mut total = MetricState::default();
|
||||
let mut models = Vec::new();
|
||||
for ((account, model), state) in &states {
|
||||
if account != &window.account_id {
|
||||
continue;
|
||||
}
|
||||
total.merge(state)?;
|
||||
if !model.is_empty()
|
||||
&& let Some(last_used_at) = state.last_used_at()
|
||||
{
|
||||
models.push(AccountModelUsage {
|
||||
model: model.clone(),
|
||||
request_count: state.count("completed_request_count"),
|
||||
success_count: state.count("completed_request_count"),
|
||||
input_tokens: state.optional_count("input_tokens"),
|
||||
output_tokens: state.optional_count("output_tokens"),
|
||||
cached_input_tokens: state.optional_count("cached_input_tokens"),
|
||||
cache_write_input_tokens: state.optional_count("cache_write_input_tokens"),
|
||||
reasoning_output_tokens: state.optional_count("reasoning_output_tokens"),
|
||||
image_input_tokens: state.optional_count("image_input_tokens"),
|
||||
image_output_tokens: state.optional_count("image_output_tokens"),
|
||||
total_tokens: Some(state.count("window_total_tokens")),
|
||||
image_request_count: state.count("image_request_count"),
|
||||
image_request_failed_count: state.count("image_request_failed_count"),
|
||||
cost_coverage: admin_cost_coverage(state.coverage()),
|
||||
costs: account_costs(state)?,
|
||||
last_used_at,
|
||||
});
|
||||
}
|
||||
}
|
||||
models.sort_by(|a, b| {
|
||||
b.request_count
|
||||
.cmp(&a.request_count)
|
||||
.then(a.model.cmp(&b.model))
|
||||
});
|
||||
results.push(AccountUsageWindowResult {
|
||||
account_id: window.account_id.clone(),
|
||||
key: window.key.clone(),
|
||||
usage: AccountUsage {
|
||||
account_id: window.account_id.clone(),
|
||||
request_count: total.count("completed_request_count"),
|
||||
success_count: total.count("completed_request_count"),
|
||||
input_tokens: total.optional_count("input_tokens"),
|
||||
output_tokens: total.optional_count("output_tokens"),
|
||||
cached_input_tokens: total.optional_count("cached_input_tokens"),
|
||||
cache_write_input_tokens: total.optional_count("cache_write_input_tokens"),
|
||||
reasoning_output_tokens: total.optional_count("reasoning_output_tokens"),
|
||||
image_input_tokens: total.optional_count("image_input_tokens"),
|
||||
image_output_tokens: total.optional_count("image_output_tokens"),
|
||||
total_tokens: Some(total.count("window_total_tokens")),
|
||||
image_request_count: total.count("image_request_count"),
|
||||
image_request_failed_count: total.count("image_request_failed_count"),
|
||||
cost_coverage: admin_cost_coverage(total.coverage()),
|
||||
costs: account_costs(&total)?,
|
||||
last_used_at: total.last_used_at(),
|
||||
request_buckets: Vec::new(),
|
||||
models,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
snapshot.finish().await?;
|
||||
results.sort_by(|a, b| a.account_id.cmp(&b.account_id).then(a.key.cmp(&b.key)));
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn account_costs(state: &MetricState) -> StoreResult<Vec<AccountCost>> {
|
||||
Ok(state
|
||||
.costs()?
|
||||
.into_iter()
|
||||
.map(|cost| AccountCost {
|
||||
currency: cost.currency,
|
||||
amount: cost.amount,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(crate) async fn account_usage_sort_values(
|
||||
snapshot: &mut AnalyticsSnapshot<'_>,
|
||||
now: DateTime<Utc>,
|
||||
) -> StoreResult<(Vec<String>, Vec<i64>, Vec<Option<DateTime<Utc>>>)> {
|
||||
let days: i32 =
|
||||
sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1")
|
||||
.fetch_one(&mut *snapshot.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read account usage retention", error))?;
|
||||
let ids: Vec<String> = sqlx::query_scalar("select id from provider_accounts")
|
||||
.fetch_all(&mut *snapshot.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read account usage sort identities", error))?;
|
||||
let range = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?;
|
||||
let mut totals = BTreeMap::<String, MetricState>::new();
|
||||
for ((account, _), state) in snapshot.account_states(range, &ids, false).await? {
|
||||
totals.entry(account).or_default().merge(&state)?;
|
||||
}
|
||||
let mut tokens = Vec::with_capacity(ids.len());
|
||||
let mut latest = Vec::with_capacity(ids.len());
|
||||
for id in &ids {
|
||||
let state = totals.remove(id).unwrap_or_default();
|
||||
tokens.push(
|
||||
i64::try_from(state.count("window_total_tokens"))
|
||||
.map_err(|_| invalid("account usage sort token overflow"))?,
|
||||
);
|
||||
latest.push(state.last_used_at());
|
||||
}
|
||||
Ok((ids, tokens, latest))
|
||||
}
|
||||
|
||||
pub(crate) async fn account_group_usage(
|
||||
connection: &mut ObservabilityConnection,
|
||||
group_ids: &[String],
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<BTreeMap<String, AccountGroupUsage>> {
|
||||
if group_ids.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
|
||||
let now = Utc::now();
|
||||
let days: i32 =
|
||||
sqlx::query_scalar("select usage_retention_days::integer from runtime_settings where id=1")
|
||||
.fetch_one(&mut *snapshot.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read group usage retention", error))?;
|
||||
let memberships:Vec<(String,String)>=sqlx::query_as("select account_group_id,provider_account_id from account_group_accounts where account_group_id=any($1::text[])").bind(group_ids).fetch_all(&mut *snapshot.tx).await.map_err(|error|postgres_unavailable("read group usage membership",error))?;
|
||||
let ids = memberships
|
||||
.iter()
|
||||
.map(|(_, account)| account.clone())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let retained = ObservabilityRange::new(now - TimeDelta::days(i64::from(days)), now)?;
|
||||
let today = ObservabilityRange::new(
|
||||
timezone
|
||||
.day_start(now)
|
||||
.ok_or_else(|| invalid("invalid business date"))?
|
||||
.max(retained.start),
|
||||
now,
|
||||
)?;
|
||||
let retained_states = snapshot.account_states(retained, &ids, false).await?;
|
||||
let today_states = snapshot.account_states(today, &ids, false).await?;
|
||||
let mut results = BTreeMap::new();
|
||||
for group in group_ids {
|
||||
let mut usage = AccountGroupUsage {
|
||||
today_usd: "0".parse().map_err(|_| invalid("invalid zero cost"))?,
|
||||
retained_total_usd: "0".parse().map_err(|_| invalid("invalid zero cost"))?,
|
||||
};
|
||||
for (_, account) in memberships.iter().filter(|(id, _)| id == group) {
|
||||
for (states, total) in [
|
||||
(&retained_states, &mut usage.retained_total_usd),
|
||||
(&today_states, &mut usage.today_usd),
|
||||
] {
|
||||
for ((id, _), state) in states {
|
||||
if id == account
|
||||
&& let Some(amount) = state.costs.get("USD")
|
||||
{
|
||||
let amount = amount
|
||||
.parse()
|
||||
.map_err(|_| invalid("invalid group usage cost"))?;
|
||||
*total = total
|
||||
.checked_add(&amount)
|
||||
.ok_or_else(|| invalid("group usage cost overflow"))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
results.insert(group.clone(), usage);
|
||||
}
|
||||
snapshot.finish().await?;
|
||||
Ok(results)
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
//! 在 PostgreSQL 内完成事实统计与汇总合并,结果规模只随输出分组增长
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 输入 matching 提供 output_bucket、name 和完整事实列及 is_completed_usage
|
||||
/// 精确分位数在整个结果组内计算,不把局部分位数合并成新的分位数
|
||||
pub(super) fn push_fact_aggregation(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
exact_percentiles: bool,
|
||||
read_costs: bool,
|
||||
) {
|
||||
push_fact_aggregation_selected(query, exact_percentiles, read_costs, true, None, None);
|
||||
}
|
||||
|
||||
pub(super) fn push_fact_aggregation_selected(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
exact_percentiles: bool,
|
||||
read_costs: bool,
|
||||
read_distributions: bool,
|
||||
counter_names: Option<&[&str]>,
|
||||
distribution_names: Option<&[&str]>,
|
||||
) {
|
||||
let counters = selected_counter_expressions(counter_names);
|
||||
query.push(", counted as (select output_bucket,name,max(started_at) filter(where is_completed_usage) as last_used_at,");
|
||||
push_counter_object(query, &counters);
|
||||
query.push(" as counts");
|
||||
if exact_percentiles && read_distributions {
|
||||
query.push(",");
|
||||
push_exact_distributions(query, distribution_names);
|
||||
}
|
||||
query.push(" from matching group by output_bucket,name)");
|
||||
if read_costs {
|
||||
query.push(", cost_values as (select output_bucket,name,currency,sum(amount)::text as amount from matching cross join lateral (values (cost_currency,cost_amount),('standard_usd',standard_cost_amount),('no_cache_usd',no_cache_cost_amount)) cost(currency,amount) where is_completed_usage and currency is not null and amount is not null group by output_bucket,name,currency), merged_costs as (select output_bucket,name,jsonb_object_agg(currency,amount) as costs from cost_values group by output_bucket,name)");
|
||||
}
|
||||
if !read_distributions {
|
||||
query.push(" select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,'{}'::jsonb as distributions,'{}'::jsonb as exact_percentiles,");
|
||||
query.push(if read_costs {
|
||||
"coalesce(k.costs,'{}'::jsonb)"
|
||||
} else {
|
||||
"'{}'::jsonb"
|
||||
});
|
||||
query.push(" as costs from counted c");
|
||||
if read_costs {
|
||||
query.push(" left join merged_costs k using(output_bucket,name)");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if exact_percentiles {
|
||||
query.push(" select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,c.distributions,c.exact_percentiles,");
|
||||
query.push(if read_costs {
|
||||
"coalesce(k.costs,'{}'::jsonb)"
|
||||
} else {
|
||||
"'{}'::jsonb"
|
||||
});
|
||||
query.push(" as costs from counted c");
|
||||
if read_costs {
|
||||
query.push(" left join merged_costs k using(output_bucket,name)");
|
||||
}
|
||||
return;
|
||||
}
|
||||
query.push(", samples as (select output_bucket,name,metric,value from matching cross join lateral (values ");
|
||||
for (index, (name, expression)) in selected_sample_expressions(distribution_names)
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
if index > 0 {
|
||||
query.push(",");
|
||||
}
|
||||
query.push(format!("('{name}',{expression})"));
|
||||
}
|
||||
query.push(") sample(metric,value) where value is not null), sample_stats as (select output_bucket,name,metric,count(*) as count,sum(value) as sum,min(value) as minimum,max(value) as maximum,count(*) filter(where value=0) as zero");
|
||||
query.push(" from samples group by output_bucket,name,metric)");
|
||||
// 对数桶定义与 Histogram::add 相同,零值单独保留,不对零取对数
|
||||
query.push(", bin_counts as (select output_bucket,name,metric,floor(log(2::numeric,value::numeric)*16)::integer as bin,count(*) as count from samples where value>0 group by output_bucket,name,metric,bin), merged_bins as (select output_bucket,name,metric,jsonb_object_agg(bin,count) as bins from bin_counts group by output_bucket,name,metric)");
|
||||
query.push(", merged_distributions as (select s.output_bucket,s.name,jsonb_object_agg(s.metric,jsonb_build_object('count',s.count,'sum',s.sum,'minimum',s.minimum,'maximum',s.maximum,'zero',s.zero,'bins',");
|
||||
query.push("coalesce(b.bins,'{}'::jsonb)");
|
||||
query.push(")) as distributions");
|
||||
query.push(" from sample_stats s");
|
||||
query.push(" left join merged_bins b using(output_bucket,name,metric)");
|
||||
query.push(" group by s.output_bucket,s.name) select c.output_bucket as bucket_start,c.name,c.counts,c.last_used_at,");
|
||||
query.push(if read_costs {
|
||||
"coalesce(k.costs,'{}'::jsonb)"
|
||||
} else {
|
||||
"'{}'::jsonb"
|
||||
});
|
||||
query.push(" as costs,coalesce(d.distributions,'{}'::jsonb) as distributions,");
|
||||
query.push("'{}'::jsonb");
|
||||
query.push(" as exact_percentiles from counted c left join merged_distributions d using(output_bucket,name)");
|
||||
if read_costs {
|
||||
query.push(" left join merged_costs k using(output_bucket,name)");
|
||||
}
|
||||
}
|
||||
|
||||
// 事实聚合与汇总读取共用指标清单,防止新增计数只在其中一条路径生效
|
||||
fn counter_expressions() -> Vec<(String, String)> {
|
||||
let mut counters = vec![
|
||||
("request_count".to_owned(), "count(*)".to_owned()),
|
||||
("completed_request_count".to_owned(), "count(*) filter(where is_completed_usage)".to_owned()),
|
||||
("caller_error_count".to_owned(), "count(*) filter(where client_status_code between 400 and 499)".to_owned()),
|
||||
("attempt_count".to_owned(), "coalesce(sum(attempt_count),0)".to_owned()),
|
||||
("retry_count".to_owned(), "coalesce(sum(greatest(attempt_count-1,0)),0)".to_owned()),
|
||||
("retried_request_count".to_owned(), "count(*) filter(where attempt_count>1)".to_owned()),
|
||||
("capacity_sample_count".to_owned(), "count(capacity_total_slots)".to_owned()),
|
||||
("window_total_tokens".to_owned(), "coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)::numeric+coalesce(output_tokens,0))) filter(where is_completed_usage),0)".to_owned()),
|
||||
("cache_eligible_request_count".to_owned(), "count(*) filter(where is_completed_usage and input_tokens is not null)".to_owned()),
|
||||
("cache_hit_request_count".to_owned(), "count(*) filter(where is_completed_usage and input_tokens is not null and cached_input_tokens>0)".to_owned()),
|
||||
("image_request_count".to_owned(), "count(*) filter(where is_completed_usage and image_generation_succeeded)".to_owned()),
|
||||
("image_request_failed_count".to_owned(), "count(*) filter(where is_completed_usage and not image_generation_succeeded)".to_owned()),
|
||||
];
|
||||
for (outcome, name) in [
|
||||
("succeeded", "success"),
|
||||
("failed", "failure"),
|
||||
("cancelled", "cancelled"),
|
||||
("incomplete", "incomplete"),
|
||||
] {
|
||||
counters.push((
|
||||
format!("{name}_count"),
|
||||
format!("count(*) filter(where outcome='{outcome}')"),
|
||||
));
|
||||
let extra = if outcome == "failed" {
|
||||
"+coalesce(sum(ops_failure_count),0)"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
counters.push((
|
||||
format!("attempt_{name}_count"),
|
||||
format!("count(*) filter(where attempt_count>0 and outcome='{outcome}'){extra}"),
|
||||
));
|
||||
}
|
||||
for (name, predicate, source) in [
|
||||
(
|
||||
"rate_limited_count",
|
||||
"(error_kind in ('rate_limited','quota_exhausted') or coalesce(upstream_status_code,client_status_code)=429)",
|
||||
"ops_rate_limited_count",
|
||||
),
|
||||
(
|
||||
"auth_failure_count",
|
||||
"(error_kind in ('authentication','authorization','invalid_credential') or coalesce(upstream_status_code,client_status_code) in (401,403))",
|
||||
"ops_auth_failure_count",
|
||||
),
|
||||
(
|
||||
"provider_5xx_count",
|
||||
"coalesce(upstream_status_code,client_status_code) between 500 and 599",
|
||||
"ops_provider_5xx_count",
|
||||
),
|
||||
] {
|
||||
counters.push((name.to_owned(), format!("count(*) filter(where attempt_count>0 and outcome='failed' and {predicate})+coalesce(sum({source}),0)")));
|
||||
}
|
||||
for name in metrics::TOKEN_METRICS {
|
||||
counters.push((
|
||||
name.to_owned(),
|
||||
format!("coalesce(sum({name}) filter(where is_completed_usage),0)"),
|
||||
));
|
||||
counters.push((
|
||||
format!("{name}_known_count"),
|
||||
format!("count({name}) filter(where is_completed_usage)"),
|
||||
));
|
||||
}
|
||||
for name in ["latency_ms", "first_token_ms"] {
|
||||
counters.push((
|
||||
format!("{name}_sum"),
|
||||
format!("coalesce(sum({name}) filter(where is_completed_usage),0)"),
|
||||
));
|
||||
}
|
||||
for (name, column) in [
|
||||
("standard_usd", "standard_cost_amount"),
|
||||
("no_cache_usd", "no_cache_cost_amount"),
|
||||
] {
|
||||
counters.push((
|
||||
format!("{name}_known_count"),
|
||||
format!("count({column}) filter(where is_completed_usage)"),
|
||||
));
|
||||
}
|
||||
for source in ["provider_reported", "calculated", "unavailable"] {
|
||||
counters.push((
|
||||
format!("{source}_count"),
|
||||
format!("count(*) filter(where is_completed_usage and cost_source='{source}')"),
|
||||
));
|
||||
}
|
||||
counters
|
||||
}
|
||||
|
||||
fn selected_counter_expressions(names: Option<&[&str]>) -> Vec<(String, String)> {
|
||||
let mut counters = counter_expressions();
|
||||
if let Some(names) = names {
|
||||
counters.retain(|(name, _)| names.contains(&name.as_str()));
|
||||
}
|
||||
counters
|
||||
}
|
||||
|
||||
fn push_exact_distributions(query: &mut QueryBuilder<Postgres>, names: Option<&[&str]>) {
|
||||
let samples = selected_sample_expressions(names);
|
||||
// 独立 ordered-set 聚合只排序实际样本,避免把每条事实展开为六条记录再排序
|
||||
query.push("jsonb_build_object(");
|
||||
for (index, (name, expression)) in samples.iter().enumerate() {
|
||||
if index > 0 {
|
||||
query.push(",");
|
||||
}
|
||||
query.push(format!("'{name}',jsonb_build_object('count',count({expression}),'sum',coalesce(sum({expression}),0),'minimum',min({expression}),'maximum',max({expression}),'zero',count(*) filter(where ({expression})=0),'bins','{{}}'::jsonb)"));
|
||||
}
|
||||
query.push(") as distributions,jsonb_build_object(");
|
||||
for (index, (name, expression)) in samples.iter().enumerate() {
|
||||
if index > 0 {
|
||||
query.push(",");
|
||||
}
|
||||
query.push(format!("'{name}',coalesce(to_jsonb(percentile_cont(array[0.1,0.5,0.9,0.95,0.99]::double precision[]) within group(order by ({expression}))),'[]'::jsonb)"));
|
||||
}
|
||||
query.push(") as exact_percentiles");
|
||||
}
|
||||
|
||||
fn sample_expressions() -> [(&'static str, &'static str); 6] {
|
||||
[
|
||||
(
|
||||
"admission_decision_ms",
|
||||
"admission_decision_ms::double precision",
|
||||
),
|
||||
(
|
||||
"account_selection_wait_ms",
|
||||
"account_selection_wait_ms::double precision",
|
||||
),
|
||||
(
|
||||
"capacity",
|
||||
"case when capacity_total_slots>0 then capacity_used_slots::double precision/capacity_total_slots::double precision*10000.0 end",
|
||||
),
|
||||
(
|
||||
"latency_ms",
|
||||
"case when is_completed_usage then latency_ms::double precision end",
|
||||
),
|
||||
(
|
||||
"first_token_ms",
|
||||
"case when is_completed_usage then first_token_ms::double precision end",
|
||||
),
|
||||
(
|
||||
"throughput",
|
||||
"case when is_completed_usage and output_tokens>0 and upstream_response_ms>0 then output_tokens::double precision*1000.0/upstream_response_ms::double precision end",
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn selected_sample_expressions(names: Option<&[&str]>) -> Vec<(&'static str, &'static str)> {
|
||||
sample_expressions()
|
||||
.into_iter()
|
||||
.filter(|(name, _)| names.is_none_or(|names| names.contains(name)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn push_counter_object(query: &mut QueryBuilder<Postgres>, counters: &[(String, String)]) {
|
||||
// PostgreSQL 单个函数调用最多接收一百个参数,分块构造不限制指标数量
|
||||
for (index, chunk) in counters.chunks(40).enumerate() {
|
||||
if index > 0 {
|
||||
query.push(" || ");
|
||||
}
|
||||
query.push("jsonb_build_object(");
|
||||
for (position, (name, expression)) in chunk.iter().enumerate() {
|
||||
if position > 0 {
|
||||
query.push(",");
|
||||
}
|
||||
query.push(format!("'{name}',{expression}"));
|
||||
}
|
||||
query.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
/// 输入 matching 的行来自互斥时间桶,合并后每个最终输出组只返回一行
|
||||
pub(super) fn push_summary_aggregation(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
distributions: bool,
|
||||
costs: bool,
|
||||
counter_names: Option<&[&str]>,
|
||||
distribution_names: Option<&[&str]>,
|
||||
) {
|
||||
// 每个源汇总只展开一次固定指标,避免按指标键放大行数再排序
|
||||
let counters = selected_counter_expressions(counter_names)
|
||||
.into_iter()
|
||||
.map(|(name, _)| {
|
||||
let expression = format!("coalesce(sum(metric.{name}),0)");
|
||||
(name, expression)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
query.push(", grouped as (select output_bucket,name,max(last_used_at) as last_used_at,");
|
||||
push_counter_object(query, &counters);
|
||||
query.push(" as counts from matching cross join lateral jsonb_to_record(counts) as metric(");
|
||||
query.push(
|
||||
counters
|
||||
.iter()
|
||||
.map(|(name, _)| format!("{name} numeric"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
);
|
||||
query.push(") group by output_bucket,name)");
|
||||
if costs {
|
||||
query.push(", cost_values as (select output_bucket,name,c.key,sum(c.value::numeric)::text as value from matching cross join lateral jsonb_each_text(costs) c group by output_bucket,name,c.key), merged_costs as (select output_bucket,name,jsonb_object_agg(key,value) as costs from cost_values group by output_bucket,name)");
|
||||
}
|
||||
if distributions {
|
||||
// 先按最终输出组归并分布,再在每组内合并直方图键,避免全局展开后大排序
|
||||
query.push(", samples as (select output_bucket,name,d.key as metric,d.value from matching cross join lateral jsonb_each(distributions) d");
|
||||
if let Some(names) = distribution_names {
|
||||
query
|
||||
.push(" where d.key=any(")
|
||||
.push_bind(
|
||||
names
|
||||
.iter()
|
||||
.map(|name| (*name).to_owned())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.push("::text[])");
|
||||
}
|
||||
query.push("), sample_stats as (select output_bucket,name,metric,sum((value->>'count')::numeric) as count,sum((value->>'sum')::double precision) as sum,min((value->>'minimum')::double precision) as minimum,max((value->>'maximum')::double precision) as maximum,sum((value->>'zero')::numeric) as zero,array_agg(value->'bins') as bins from samples group by output_bucket,name,metric), merged_distributions as (select s.output_bucket,s.name,jsonb_object_agg(s.metric,jsonb_build_object('count',s.count,'sum',s.sum,'minimum',s.minimum,'maximum',s.maximum,'zero',s.zero,'bins',coalesce(b.bins,'{}'::jsonb))) as distributions from sample_stats s cross join lateral(select jsonb_object_agg(key,count) as bins from(select b.key,sum(b.value::numeric) as count from unnest(s.bins) input(value) cross join lateral jsonb_each_text(input.value) b group by b.key) merged)b group by s.output_bucket,s.name)");
|
||||
}
|
||||
query.push(" select g.output_bucket as bucket_start,g.name,g.last_used_at,g.counts,");
|
||||
query.push(if costs {
|
||||
"coalesce(k.costs,'{}'::jsonb)"
|
||||
} else {
|
||||
"'{}'::jsonb"
|
||||
});
|
||||
query.push(" as costs,");
|
||||
query.push(if distributions {
|
||||
"coalesce(d.distributions,'{}'::jsonb)"
|
||||
} else {
|
||||
"'{}'::jsonb"
|
||||
});
|
||||
query.push(" as distributions from grouped g");
|
||||
if costs {
|
||||
query.push(" left join merged_costs k using(output_bucket,name)");
|
||||
}
|
||||
if distributions {
|
||||
query.push(" left join merged_distributions d using(output_bucket,name)");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn push_output_bucket(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
column: &str,
|
||||
range: ObservabilityRange,
|
||||
granularity: Option<ObservationGranularity>,
|
||||
boundaries: Option<&[DateTime<Utc>]>,
|
||||
) {
|
||||
match granularity {
|
||||
None => {
|
||||
query.push_bind(range.start).push("::timestamptz");
|
||||
}
|
||||
Some(ObservationGranularity::Day) => {
|
||||
// 使用有序阈值二分定位,保留部署时区及夏令时日界,不逐行扫描所有日界
|
||||
let boundaries = boundaries.unwrap_or_default();
|
||||
query
|
||||
.push("(")
|
||||
.push_bind(boundaries.to_vec())
|
||||
.push(format!("::timestamptz[])[width_bucket({column},"))
|
||||
.push_bind(boundaries.to_vec())
|
||||
.push("::timestamptz[])]");
|
||||
}
|
||||
Some(granularity) => {
|
||||
query
|
||||
.push("date_bin(make_interval(secs => ")
|
||||
.push_bind(granularity.seconds() as f64)
|
||||
.push(format!("),{column},timestamptz '2000-01-01 UTC')"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fact_dimension_sql(dimension: Option<DiagnosticDimension>) -> &'static str {
|
||||
match dimension {
|
||||
None => "''::text",
|
||||
Some(DiagnosticDimension::Provider) => "coalesce(nullif(mr.provider_kind,''),'unrouted')",
|
||||
Some(DiagnosticDimension::Model) => {
|
||||
"coalesce(nullif(mr.upstream_model_id,''),mr.requested_model_id,'')"
|
||||
}
|
||||
Some(DiagnosticDimension::Account) => {
|
||||
"coalesce(nullif(mr.provider_account_ref,''),'unrouted')"
|
||||
}
|
||||
Some(DiagnosticDimension::ApiKey) => "coalesce(mr.client_api_key_ref,'')",
|
||||
Some(DiagnosticDimension::Transport) => {
|
||||
"coalesce(nullif(mr.upstream_transport,''),nullif(mr.client_transport,''),'unknown')"
|
||||
}
|
||||
Some(DiagnosticDimension::Failure) => "coalesce(mr.error_kind,'')",
|
||||
Some(DiagnosticDimension::Status) => {
|
||||
"coalesce(mr.upstream_status_code::text,mr.client_status_code::text,'')"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn summary_dimension_sql(
|
||||
collection: &str,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> String {
|
||||
let offset = usize::from(matches!(collection, "account_model" | "key_model"));
|
||||
match dimension {
|
||||
None => "''::text".to_owned(),
|
||||
Some(DiagnosticDimension::Provider) => {
|
||||
format!("coalesce(nullif(dimensions[{}],''),'unrouted')", 1 + offset)
|
||||
}
|
||||
Some(DiagnosticDimension::Model) => format!(
|
||||
"coalesce(nullif(dimensions[{}],''),dimensions[{}],'')",
|
||||
3 + offset,
|
||||
2 + offset
|
||||
),
|
||||
Some(DiagnosticDimension::Account) => {
|
||||
"coalesce(nullif(dimensions[1],''),'unrouted')".to_owned()
|
||||
}
|
||||
Some(DiagnosticDimension::Transport) => {
|
||||
"coalesce(nullif(dimensions[2],''),nullif(dimensions[1],''),'unknown')".to_owned()
|
||||
}
|
||||
Some(DiagnosticDimension::Status) => {
|
||||
"coalesce(nullif(dimensions[2],''),dimensions[1],'')".to_owned()
|
||||
}
|
||||
Some(DiagnosticDimension::ApiKey | DiagnosticDimension::Failure) => {
|
||||
"coalesce(dimensions[1],'')".to_owned()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,13 +49,13 @@ pub(super) async fn rebuild_bucket_batch(
|
||||
0
|
||||
};
|
||||
if !same_version {
|
||||
// 处理中事实发生变化时清掉未发布的部分状态,不能把新旧快照增量相加
|
||||
// 显式重建版本变化时清掉旧前缀,普通事实变化由增量维护更新前缀
|
||||
sqlx::query("update request_analytics_pending_buckets set build_version=change_version,build_phase=0,build_started_at=null,build_id=null,build_collection=null,build_dimensions=null,build_outcome=null where bucket_start=$1")
|
||||
.bind(bucket).execute(&mut **tx).await.map_err(|error|postgres_unavailable("restart changed analytics bucket",error))?;
|
||||
}
|
||||
let hour = bucket_for(bucket, 3600)?;
|
||||
if phase < 2 {
|
||||
// 小时分段期间不改写其输入;源事实仍可投影,版本变化使小时任务先退回等待
|
||||
// 小时分段期间暂停其他 quarter 的完整重建,普通增量同步修正已扫描前缀
|
||||
let owned:bool=sqlx::query_scalar("select exists(select 1 from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and bucket_start <> $2 and build_phase >= 3)")
|
||||
.bind(hour).bind(bucket).fetch_one(&mut **tx).await.map_err(|error|postgres_unavailable("claim analytics hour stage",error))?;
|
||||
if owned {
|
||||
@@ -119,8 +119,9 @@ pub(super) async fn rebuild_bucket_batch(
|
||||
let limit = i64::from(batch_size) + 1;
|
||||
let mut groups = GroupStates::new();
|
||||
if phase == 1 {
|
||||
// 增量前缀判断使用 Rust 字节序,SQL 游标必须使用同一排序规则
|
||||
let statement = format!(
|
||||
"{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and mr.recovered_at is null and (mr.started_at,mr.id)>($2,$3) order by mr.started_at,mr.id limit $4",
|
||||
"{} where mr.started_at >= $1 and mr.started_at < $1+interval '15 minutes' and (mr.started_at,mr.id collate \"C\")>($2,$3) order by mr.started_at,mr.id collate \"C\" limit $4",
|
||||
fact_select()
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
@@ -139,11 +140,16 @@ pub(super) async fn rebuild_bucket_batch(
|
||||
let outcome: String = get(fact, "outcome")?;
|
||||
let mut state = MetricState::default();
|
||||
state.add_row(fact)?;
|
||||
let forecast = forecast::state(fact)?;
|
||||
for (collection, dimensions) in collections(fact)? {
|
||||
groups
|
||||
.entry((collection.to_owned(), dimensions, outcome.clone()))
|
||||
.or_default()
|
||||
.merge(&state)?;
|
||||
.merge(if collection == forecast::COLLECTION {
|
||||
&forecast
|
||||
} else {
|
||||
&state
|
||||
})?;
|
||||
}
|
||||
}
|
||||
merge_staged_groups(tx, "request_analytics_15m", bucket, groups).await?;
|
||||
@@ -155,7 +161,7 @@ pub(super) async fn rebuild_bucket_batch(
|
||||
.bind(bucket).bind(get::<DateTime<Utc>>(last,"started_at")?).bind(get::<String>(last,"id")?).execute(&mut **tx).await.map_err(|error|postgres_unavailable("save analytics fact cursor",error))?;
|
||||
}
|
||||
} else if phase == 4 {
|
||||
let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection,dimensions,outcome)>($4,$5,$6,$7) order by bucket_start,collection,dimensions,outcome limit $8")
|
||||
let rows=sqlx::query("select bucket_start,collection,dimensions,outcome,counts,costs,distributions,last_used_at from request_analytics_15m where bucket_start >= $1 and bucket_start < $1+interval '1 hour' and schema_version=$2 and histogram_version=$3 and (bucket_start,collection collate \"C\",dimensions collate \"C\",outcome collate \"C\")>($4,$5,$6,$7) order by bucket_start,collection collate \"C\",dimensions collate \"C\",outcome collate \"C\" limit $8")
|
||||
.bind(hour).bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).bind(get::<Option<DateTime<Utc>>>(&row,"build_started_at")?.unwrap_or(hour-TimeDelta::microseconds(1)))
|
||||
.bind(get::<Option<String>>(&row,"build_collection")?.unwrap_or_default()).bind(get::<Option<Vec<String>>>(&row,"build_dimensions")?.unwrap_or_default()).bind(get::<Option<String>>(&row,"build_outcome")?.unwrap_or_default()).bind(limit)
|
||||
.fetch_all(&mut **tx).await.map_err(|error|postgres_unavailable("read analytics hour segment",error))?;
|
||||
@@ -198,7 +204,7 @@ async fn merge_staged_groups(
|
||||
}
|
||||
let keys=groups.keys().map(|(collection,dimensions,outcome)|serde_json::json!({"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::<Vec<_>>();
|
||||
let statement = format!(
|
||||
"select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1"
|
||||
"select stored.collection,stored.dimensions,stored.outcome,stored.counts,stored.costs,stored.distributions,stored.last_used_at from {table} stored join jsonb_to_recordset($2::jsonb) requested(collection text,dimensions text[],outcome text) using(collection,dimensions,outcome) where bucket_start=$1"
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(bucket)
|
||||
@@ -220,37 +226,64 @@ async fn merge_staged_groups(
|
||||
write_groups(tx, table, bucket, groups).await
|
||||
}
|
||||
|
||||
async fn write_groups(
|
||||
pub(super) async fn write_groups(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
table: &str,
|
||||
bucket: DateTime<Utc>,
|
||||
groups: GroupStates,
|
||||
) -> StoreResult<()> {
|
||||
if groups.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut statement = QueryBuilder::<Postgres>::new(format!(
|
||||
"insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) "
|
||||
));
|
||||
let rows = groups
|
||||
.into_iter()
|
||||
.map(|((collection, dimensions, outcome), state)| {
|
||||
let (counts, costs, distributions) = state.encode()?;
|
||||
Ok((
|
||||
write_bucket_groups(tx, table, [(bucket, groups)]).await
|
||||
}
|
||||
|
||||
pub(super) async fn write_bucket_groups(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
table: &str,
|
||||
buckets: impl IntoIterator<Item = (DateTime<Utc>, GroupStates)>,
|
||||
) -> StoreResult<()> {
|
||||
let mut groups = buckets.into_iter().flat_map(|(bucket, groups)| {
|
||||
groups
|
||||
.into_iter()
|
||||
.map(move |(key, state)| (bucket, key, state))
|
||||
});
|
||||
loop {
|
||||
// 跨桶合并 SQL 往返,但编码缓冲只保留当前批次,避免回填将所有状态展开为 JSON
|
||||
let rows = groups
|
||||
.by_ref()
|
||||
.take(1000)
|
||||
.map(|(bucket, (collection, dimensions, outcome), state)| {
|
||||
let (counts, costs, distributions) = state.encode()?;
|
||||
Ok((
|
||||
bucket,
|
||||
collection,
|
||||
dimensions,
|
||||
outcome,
|
||||
counts,
|
||||
costs,
|
||||
distributions,
|
||||
state.last_used_at(),
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut statement = QueryBuilder::<Postgres>::new(format!(
|
||||
"insert into {table}(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions,last_used_at) "
|
||||
));
|
||||
// 十个绑定参数限制单语句批量,避免高基数时间桶越过 PostgreSQL 参数上限
|
||||
statement.push_values(
|
||||
&rows,
|
||||
|mut row,
|
||||
(
|
||||
bucket,
|
||||
collection,
|
||||
dimensions,
|
||||
outcome,
|
||||
counts,
|
||||
costs,
|
||||
distributions,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
// 九个绑定参数限制单语句批量,避免高基数时间桶越过 PostgreSQL 参数上限
|
||||
for chunk in rows.chunks(1000) {
|
||||
statement.push_values(
|
||||
chunk,
|
||||
|mut row, (collection, dimensions, outcome, counts, costs, distributions)| {
|
||||
last_used_at,
|
||||
)| {
|
||||
row.push_bind(bucket)
|
||||
.push_bind(collection)
|
||||
.push_bind(dimensions)
|
||||
@@ -259,16 +292,15 @@ async fn write_groups(
|
||||
.push_bind(HISTOGRAM_VERSION)
|
||||
.push_bind(counts)
|
||||
.push_bind(costs)
|
||||
.push_bind(distributions);
|
||||
.push_bind(distributions)
|
||||
.push_bind(last_used_at);
|
||||
},
|
||||
);
|
||||
statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version");
|
||||
statement.push(" on conflict(bucket_start,collection,dimensions,outcome) do update set counts=excluded.counts,costs=excluded.costs,distributions=excluded.distributions,last_used_at=excluded.last_used_at,schema_version=excluded.schema_version,histogram_version=excluded.histogram_version");
|
||||
statement
|
||||
.build()
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("write analytics bucket", error))?;
|
||||
statement.reset();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
//! 用完整刻钟计数和同快照源差量定位精确页码,避免每页扫描整个保留期
|
||||
|
||||
use super::query::{collection_for, push_collection_filter, push_summary_buckets};
|
||||
use super::*;
|
||||
|
||||
enum ReadVersion {
|
||||
Published,
|
||||
Before,
|
||||
Current,
|
||||
}
|
||||
|
||||
impl AnalyticsSnapshot<'_> {
|
||||
pub(crate) async fn completed_request_counts(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Option<BTreeMap<DateTime<Utc>, u64>>> {
|
||||
let Some(collection) = collection_for(filter, None) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !self.projection_covers(range) {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut counts = BTreeMap::new();
|
||||
let (_, buckets) = self.summary_buckets(range, false).await?;
|
||||
if !buckets.is_empty() {
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select bucket_start,coalesce(sum((counts->>'completed_request_count')::bigint),0)::bigint as request_count from request_analytics_15m a where collection=",
|
||||
);
|
||||
query
|
||||
.push_bind(collection)
|
||||
.push(" and schema_version=")
|
||||
.push_bind(SCHEMA_VERSION)
|
||||
.push(" and histogram_version=")
|
||||
.push_bind(HISTOGRAM_VERSION);
|
||||
push_summary_buckets(&mut query, &buckets, 900);
|
||||
push_collection_filter(&mut query, collection, filter);
|
||||
query.push(" group by bucket_start");
|
||||
for row in query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read usage page counts", error))?
|
||||
{
|
||||
counts.insert(
|
||||
get(&row, "bucket_start")?,
|
||||
to_u64(get(&row, "request_count")?)?,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (start, end) in self.uncovered_spans(range).await? {
|
||||
self.merge_completed_counts(
|
||||
&mut counts,
|
||||
ObservabilityRange { start, end },
|
||||
filter,
|
||||
ReadVersion::Published,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if self.has_pending_requests().await? {
|
||||
// 先撤销已投影版本,再加入源表当前版本;移动、删除和未确认的重放均只计算一次
|
||||
self.merge_completed_counts(&mut counts, range, filter, ReadVersion::Before)
|
||||
.await?;
|
||||
self.merge_completed_counts(&mut counts, range, filter, ReadVersion::Current)
|
||||
.await?;
|
||||
}
|
||||
Ok(Some(counts))
|
||||
}
|
||||
|
||||
async fn merge_completed_counts(
|
||||
&mut self,
|
||||
counts: &mut BTreeMap<DateTime<Utc>, u64>,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
version: ReadVersion,
|
||||
) -> StoreResult<()> {
|
||||
let table = if matches!(version, ReadVersion::Current) {
|
||||
"model_request_observations"
|
||||
} else {
|
||||
"request_analytics_facts"
|
||||
};
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select date_bin(interval '15 minutes',mr.started_at,timestamptz '2000-01-01 UTC') as bucket_start,count(*)::bigint as request_count from {table} mr"
|
||||
));
|
||||
if !matches!(version, ReadVersion::Published) {
|
||||
query.push(" join request_analytics_pending_requests p on p.request_id=mr.id");
|
||||
}
|
||||
query
|
||||
.push(" where mr.started_at >= ")
|
||||
.push_bind(range.start)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("correct usage page counts", error))?;
|
||||
for row in rows {
|
||||
let value = to_u64(get(&row, "request_count")?)?;
|
||||
let count = counts.entry(get(&row, "bucket_start")?).or_default();
|
||||
*count = if matches!(version, ReadVersion::Before) {
|
||||
count.checked_sub(value)
|
||||
} else {
|
||||
count.checked_add(value)
|
||||
}
|
||||
.ok_or_else(|| invalid("usage page count is inconsistent"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
//! 预测累计使用开始时间与完成时间的二维汇总,采样边界及待投影版本读取精确事实
|
||||
|
||||
use super::*;
|
||||
use gateway_admin::model::provider_credentials::ProviderDocument;
|
||||
use gateway_admin::model::quota_forecast_sampling::{
|
||||
MAX_FORECAST_HISTORY_POINTS, QuotaForecastHistory, QuotaForecastHistoryPoint,
|
||||
QuotaForecastUsage,
|
||||
};
|
||||
use gateway_core::account::OpaqueProviderData;
|
||||
use sqlx::postgres::PgRow;
|
||||
|
||||
pub(super) const COLLECTION: &str = "account_completion";
|
||||
const COMPLETION_SECONDS: i64 = 300;
|
||||
const COUNTERS: [&str; 10] = [
|
||||
"request_count",
|
||||
"completed_request_count",
|
||||
"window_total_tokens",
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cached_input_tokens",
|
||||
"missing_token_count",
|
||||
"known_cost_count",
|
||||
"unavailable_cost_count",
|
||||
"excluded_request_count",
|
||||
];
|
||||
|
||||
pub(super) fn dimensions(row: &PgRow) -> StoreResult<Vec<String>> {
|
||||
let completed: Option<DateTime<Utc>> = get(row, "completed_at")?;
|
||||
let terminal = get::<String>(row, "outcome")? != "running";
|
||||
Ok(vec![
|
||||
get::<Option<String>>(row, "provider_account_ref")?.unwrap_or_default(),
|
||||
completed
|
||||
.filter(|_| terminal)
|
||||
.map(|time| time.timestamp().div_euclid(COMPLETION_SECONDS).to_string())
|
||||
.unwrap_or_default(),
|
||||
])
|
||||
}
|
||||
|
||||
pub(super) fn state(row: &PgRow) -> StoreResult<MetricState> {
|
||||
let mut state = MetricState::default();
|
||||
state.increment("request_count", 1)?;
|
||||
state.add_cost("USD", "0")?;
|
||||
if get::<Option<DateTime<Utc>>>(row, "completed_at")?.is_none()
|
||||
|| get::<String>(row, "outcome")? == "running"
|
||||
{
|
||||
return Ok(state);
|
||||
}
|
||||
if !get::<bool>(row, "is_completed_usage")? {
|
||||
state.increment("excluded_request_count", 1)?;
|
||||
return Ok(state);
|
||||
}
|
||||
state.increment("completed_request_count", 1)?;
|
||||
let input = optional_unsigned(row, "input_tokens")?;
|
||||
let output = optional_unsigned(row, "output_tokens")?;
|
||||
let total = optional_unsigned(row, "total_tokens")?;
|
||||
state.increment(
|
||||
"window_total_tokens",
|
||||
total.unwrap_or(
|
||||
input
|
||||
.unwrap_or_default()
|
||||
.checked_add(output.unwrap_or_default())
|
||||
.ok_or_else(|| invalid("forecast token overflow"))?,
|
||||
),
|
||||
)?;
|
||||
state.increment("input_tokens", input.unwrap_or_default())?;
|
||||
state.increment("output_tokens", output.unwrap_or_default())?;
|
||||
state.increment(
|
||||
"cached_input_tokens",
|
||||
optional_unsigned(row, "cached_input_tokens")?.unwrap_or_default(),
|
||||
)?;
|
||||
state.increment(
|
||||
"missing_token_count",
|
||||
u64::from(total.is_none() && (input.is_none() || output.is_none())),
|
||||
)?;
|
||||
let source: String = get(row, "cost_source")?;
|
||||
let amount: Option<String> = get(row, "cost_amount_text")?;
|
||||
let known = matches!(source.as_str(), "calculated" | "provider_reported")
|
||||
&& get::<Option<String>>(row, "cost_currency")?.as_deref() == Some("USD")
|
||||
&& amount.is_some();
|
||||
state.increment("known_cost_count", u64::from(known))?;
|
||||
state.increment("unavailable_cost_count", u64::from(!known))?;
|
||||
if known {
|
||||
state.add_cost(
|
||||
"USD",
|
||||
amount
|
||||
.as_deref()
|
||||
.ok_or_else(|| invalid("missing forecast amount"))?,
|
||||
)?;
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
impl AnalyticsSnapshot<'_> {
|
||||
pub(crate) async fn forecast_history(
|
||||
&mut self,
|
||||
account: &str,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<QuotaForecastHistory> {
|
||||
let documents = sqlx::query(DOCUMENTS)
|
||||
.bind(account)
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.bind(MAX_FORECAST_HISTORY_POINTS as i32)
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("sample forecast documents", error))?;
|
||||
let mut cutoffs = documents
|
||||
.iter()
|
||||
.map(|row| get(row, "completed_at"))
|
||||
.collect::<StoreResult<Vec<DateTime<Utc>>>>()?;
|
||||
// 最后一个状态专门代表整个窗口,文档恰好完成于窗口末端时复用同一截止值
|
||||
if cutoffs.last() != Some(&range.end) {
|
||||
cutoffs.push(range.end);
|
||||
}
|
||||
let boundaries = cutoffs
|
||||
.iter()
|
||||
.map(|time| time.timestamp().div_euclid(COMPLETION_SECONDS))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let mut states = vec![MetricState::default(); cutoffs.len() + 1];
|
||||
if self.projection_covers(range) {
|
||||
let (_, quarters) = self.summary_buckets(range, false).await?;
|
||||
if !quarters.is_empty() {
|
||||
let sums = COUNTERS
|
||||
.iter()
|
||||
.map(|name| format!("'{name}',coalesce(sum((counts->>'{name}')::numeric),0)"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let statement = format!(
|
||||
"with selected as (select counts,costs,case when dimensions[2]='' then cardinality($3::timestamptz[]) else width_bucket(to_timestamp((dimensions[2]::bigint+1)*{COMPLETION_SECONDS})-interval '1 microsecond',$3::timestamptz[]) end as slot
|
||||
from request_analytics_15m where collection='{COLLECTION}' and dimensions[1]=$1 and bucket_start=any($2::timestamptz[]) and schema_version=$5 and histogram_version=$6
|
||||
and (dimensions[2]='' or not(nullif(dimensions[2],'')::bigint=any($4::bigint[]))))
|
||||
select slot,jsonb_build_object({sums}) as counts,jsonb_build_object('USD',coalesce(sum((costs->>'USD')::numeric),0)::text) as costs,'{{}}'::jsonb as distributions,null::timestamptz as last_used_at from selected group by slot"
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(account)
|
||||
.bind(&quarters)
|
||||
.bind(&cutoffs)
|
||||
.bind(&boundaries)
|
||||
.bind(SCHEMA_VERSION)
|
||||
.bind(HISTOGRAM_VERSION)
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
postgres_unavailable("read forecast completion summaries", error)
|
||||
})?;
|
||||
merge_rows(&mut states, rows, false)?;
|
||||
// 只有跨过采样截止值的完成时间小桶需要回读,按完成时间索引逐段定位
|
||||
let rows = self
|
||||
.forecast_facts(
|
||||
account,
|
||||
range,
|
||||
&cutoffs,
|
||||
FactSelection::CompletionEdges {
|
||||
quarters: &quarters,
|
||||
boundaries: &boundaries,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
merge_rows(&mut states, rows, false)?;
|
||||
}
|
||||
for (start, end) in self.uncovered_spans(range).await? {
|
||||
let rows = self
|
||||
.forecast_facts(
|
||||
account,
|
||||
ObservabilityRange { start, end },
|
||||
&cutoffs,
|
||||
FactSelection::Projected,
|
||||
)
|
||||
.await?;
|
||||
merge_rows(&mut states, rows, false)?;
|
||||
}
|
||||
if self.has_pending_requests().await? {
|
||||
let old = self
|
||||
.forecast_facts(account, range, &cutoffs, FactSelection::Previous)
|
||||
.await?;
|
||||
merge_rows(&mut states, old, true)?;
|
||||
let current = self
|
||||
.forecast_facts(account, range, &cutoffs, FactSelection::Current)
|
||||
.await?;
|
||||
merge_rows(&mut states, current, false)?;
|
||||
}
|
||||
} else {
|
||||
let rows = self
|
||||
.forecast_facts(account, range, &cutoffs, FactSelection::Source)
|
||||
.await?;
|
||||
merge_rows(&mut states, rows, false)?;
|
||||
}
|
||||
let pending_request_count = states
|
||||
.last()
|
||||
.map_or(0, |state| state.count("request_count"));
|
||||
let mut cumulative = MetricState::default();
|
||||
let mut usages = Vec::with_capacity(cutoffs.len());
|
||||
for state in states.iter().take(cutoffs.len()) {
|
||||
cumulative.merge(state)?;
|
||||
usages.push(usage(&cumulative)?);
|
||||
}
|
||||
let mut points = Vec::with_capacity(documents.len());
|
||||
for (row, usage) in documents.into_iter().zip(&usages) {
|
||||
let document: sqlx::types::Json<serde_json::Map<String, serde_json::Value>> =
|
||||
get(&row, "document")?;
|
||||
points.push(QuotaForecastHistoryPoint {
|
||||
started_at: get(&row, "started_at")?,
|
||||
completed_at: get(&row, "completed_at")?,
|
||||
usage: usage.clone(),
|
||||
provider_observation: ProviderDocument::new(OpaqueProviderData::new(document.0)),
|
||||
});
|
||||
}
|
||||
Ok(QuotaForecastHistory {
|
||||
points,
|
||||
usage: usage(&cumulative)?,
|
||||
pending_request_count,
|
||||
})
|
||||
}
|
||||
|
||||
async fn forecast_facts(
|
||||
&mut self,
|
||||
account: &str,
|
||||
range: ObservabilityRange,
|
||||
cutoffs: &[DateTime<Utc>],
|
||||
selection: FactSelection<'_>,
|
||||
) -> StoreResult<Vec<PgRow>> {
|
||||
let source = matches!(selection, FactSelection::Source | FactSelection::Current);
|
||||
let table = if source {
|
||||
"model_requests"
|
||||
} else {
|
||||
"request_analytics_facts"
|
||||
};
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"with matching as (select mr.*,case when mr.completed_at is not null and mr.outcome <> 'running' then width_bucket(mr.completed_at-interval '1 microsecond',",
|
||||
);
|
||||
query.push_bind(cutoffs).push("::timestamptz[]) else ").push_bind(cutoffs.len() as i32)
|
||||
.push(" end as slot,coalesce((").push(completed_usage_fact_predicate("mr"))
|
||||
.push("),false) and mr.completed_at is not null and mr.outcome <> 'running' as included from ");
|
||||
if let FactSelection::CompletionEdges { boundaries, .. } = selection {
|
||||
// 保留按完成时间小桶的参数化索引扫描,避免展开后扫描整段账号事实再逐桶匹配
|
||||
// OFFSET 0 不裁剪数据,只阻止此相关子查询被合并到外层连接
|
||||
query.push("unnest(").push_bind(boundaries).push(format!("::bigint[]) edge(bucket) cross join lateral (select * from {table} f where f.provider_account_ref="))
|
||||
.push_bind(account).push(format!(" and f.completed_at >= to_timestamp(edge.bucket*{COMPLETION_SECONDS}) and f.completed_at < to_timestamp((edge.bucket+1)*{COMPLETION_SECONDS}) and f.outcome <> 'running' offset 0) mr"));
|
||||
} else {
|
||||
query.push(format!("{table} mr"));
|
||||
}
|
||||
if matches!(selection, FactSelection::Previous | FactSelection::Current) {
|
||||
query.push(" join request_analytics_pending_requests p on p.request_id=mr.id");
|
||||
}
|
||||
query
|
||||
.push(" where mr.provider_account_ref=")
|
||||
.push_bind(account)
|
||||
.push(" and mr.started_at >= ")
|
||||
.push_bind(range.start)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(range.end);
|
||||
if let FactSelection::CompletionEdges { quarters, .. } = selection {
|
||||
query.push(" and date_bin(interval '15 minutes',mr.started_at,timestamptz '2000-01-01 UTC')=any(").push_bind(quarters).push("::timestamptz[])");
|
||||
}
|
||||
// 多个累计指标复用同一分类结果,避免展开后重复判断完成用量并携带宽事实排序
|
||||
// OFFSET 0 保留全部行,只让分类在聚合前计算一次
|
||||
query.push("), classified as (select *,coalesce(cost_source in ('calculated','provider_reported') and cost_amount is not null and cost_currency='USD',false) as known_cost from matching offset 0)
|
||||
select slot,jsonb_build_object(
|
||||
'request_count',count(*),
|
||||
'completed_request_count',count(*) filter(where included),
|
||||
'window_total_tokens',coalesce(sum(coalesce(total_tokens,coalesce(input_tokens,0)+coalesce(output_tokens,0))) filter(where included),0),
|
||||
'input_tokens',coalesce(sum(input_tokens) filter(where included),0),
|
||||
'output_tokens',coalesce(sum(output_tokens) filter(where included),0),
|
||||
'cached_input_tokens',coalesce(sum(cached_input_tokens) filter(where included),0),
|
||||
'missing_token_count',count(*) filter(where included and total_tokens is null and (input_tokens is null or output_tokens is null)),
|
||||
'known_cost_count',count(*) filter(where included and known_cost),
|
||||
'unavailable_cost_count',count(*) filter(where included and not known_cost),
|
||||
'excluded_request_count',count(*) filter(where not included and completed_at is not null and outcome <> 'running')) as counts,
|
||||
jsonb_build_object('USD',coalesce(sum(cost_amount) filter(where included and known_cost),0)::text) as costs,
|
||||
'{}'::jsonb as distributions,null::timestamptz as last_used_at from classified group by slot");
|
||||
query
|
||||
.build()
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read exact forecast boundaries", error))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum FactSelection<'a> {
|
||||
Source,
|
||||
Projected,
|
||||
Previous,
|
||||
Current,
|
||||
CompletionEdges {
|
||||
quarters: &'a [DateTime<Utc>],
|
||||
boundaries: &'a [i64],
|
||||
},
|
||||
}
|
||||
|
||||
fn merge_rows(states: &mut [MetricState], rows: Vec<PgRow>, subtract: bool) -> StoreResult<()> {
|
||||
for row in rows {
|
||||
let index = usize::try_from(get::<i32>(&row, "slot")?)
|
||||
.map_err(|_| invalid("negative forecast slot"))?;
|
||||
let target = states
|
||||
.get_mut(index)
|
||||
.ok_or_else(|| invalid("forecast slot out of range"))?;
|
||||
let state = MetricState::decode(&row)?;
|
||||
if subtract {
|
||||
incremental::subtract(target, &state, &MetricState::default())?;
|
||||
} else {
|
||||
target.merge(&state)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn usage(state: &MetricState) -> StoreResult<QuotaForecastUsage> {
|
||||
Ok(QuotaForecastUsage {
|
||||
request_count: state.count("completed_request_count"),
|
||||
tokens: state.count("window_total_tokens"),
|
||||
input_tokens: state.count("input_tokens"),
|
||||
output_tokens: state.count("output_tokens"),
|
||||
cached_input_tokens: state.count("cached_input_tokens"),
|
||||
missing_token_count: state.count("missing_token_count"),
|
||||
known_cost_count: state.count("known_cost_count"),
|
||||
unavailable_cost_count: state.count("unavailable_cost_count"),
|
||||
excluded_request_count: state.count("excluded_request_count"),
|
||||
usd: state
|
||||
.costs
|
||||
.get("USD")
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<f64>()
|
||||
.map_err(|_| invalid("invalid forecast dollars"))
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
// 微秒边界向上取整,最后一槽包含观测时刻;文档采样与累计量补差共用读事务
|
||||
const DOCUMENTS: &str = "select mr.started_at,mr.completed_at,mr.provider_observation_json as document
|
||||
from generate_series(0,$4-1) b(bucket) cross join lateral (
|
||||
select mr.started_at,mr.completed_at,mr.provider_observation_json
|
||||
from model_requests mr where mr.provider_account_ref=$1 and mr.started_at >= $2 and mr.started_at < $3
|
||||
and mr.completed_at >= $2 + ceil(b.bucket*greatest(1,extract(epoch from ($3::timestamptz-$2::timestamptz))/$4)*1000000)::bigint*interval '1 microsecond'
|
||||
and mr.completed_at < case when b.bucket=$4-1 then $3+interval '1 microsecond' else least($3+interval '1 microsecond',$2+ceil((b.bucket+1)*greatest(1,extract(epoch from ($3::timestamptz-$2::timestamptz))/$4)*1000000)::bigint*interval '1 microsecond') end
|
||||
and mr.outcome <> 'running' and mr.provider_observation_json is not null
|
||||
order by mr.completed_at desc,mr.id desc limit 1
|
||||
) mr order by mr.completed_at";
|
||||
@@ -14,8 +14,8 @@ pub(super) struct Histogram {
|
||||
pub sum: f64,
|
||||
pub minimum: Option<f64>,
|
||||
pub maximum: Option<f64>,
|
||||
zero: u64,
|
||||
bins: BTreeMap<i32, u64>,
|
||||
pub(super) zero: u64,
|
||||
pub(super) bins: BTreeMap<i32, u64>,
|
||||
}
|
||||
|
||||
impl Histogram {
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
//! 同事务撤销旧事实并合并新事实,构建中的统计只更新已经扫描的前缀
|
||||
|
||||
use super::buckets::write_bucket_groups;
|
||||
use super::worker::collections;
|
||||
use super::*;
|
||||
use sqlx::postgres::PgRow;
|
||||
|
||||
type GroupKey = (DateTime<Utc>, String, Vec<String>, String);
|
||||
|
||||
#[derive(Default)]
|
||||
struct Change {
|
||||
previous: MetricState,
|
||||
current: MetricState,
|
||||
}
|
||||
|
||||
struct Progress {
|
||||
phase: i16,
|
||||
started_at: Option<DateTime<Utc>>,
|
||||
id: Option<String>,
|
||||
collection: Option<String>,
|
||||
dimensions: Option<Vec<String>>,
|
||||
outcome: Option<String>,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
fn contains_fact(&self, time: DateTime<Utc>, id: &str) -> bool {
|
||||
match self.phase {
|
||||
0 => false,
|
||||
1 => self
|
||||
.started_at
|
||||
.zip(self.id.as_deref())
|
||||
.is_some_and(|cursor| (time, id) <= cursor),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn contains_quarter(&self, key: &GroupKey) -> bool {
|
||||
if self.phase != 4 {
|
||||
return false;
|
||||
}
|
||||
match (
|
||||
&self.started_at,
|
||||
&self.collection,
|
||||
&self.dimensions,
|
||||
&self.outcome,
|
||||
) {
|
||||
(Some(time), Some(collection), Some(dimensions), Some(outcome)) => {
|
||||
(&key.0, &key.1, &key.2, &key.3) <= (time, collection, dimensions, outcome)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn update_summaries(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
previous: &[PgRow],
|
||||
current: &[PgRow],
|
||||
) -> StoreResult<()> {
|
||||
let mut hours = BTreeSet::new();
|
||||
for row in previous.iter().chain(current) {
|
||||
hours.insert(bucket_for(get(row, "started_at")?, 3600)?);
|
||||
}
|
||||
if hours.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let progress_rows = sqlx::query("select p.* from unnest($1::timestamptz[]) h(start_at) join request_analytics_pending_buckets p on p.bucket_start >= h.start_at and p.bucket_start < h.start_at+interval '1 hour'")
|
||||
.bind(hours.into_iter().collect::<Vec<_>>()).fetch_all(&mut **tx).await
|
||||
.map_err(|error|postgres_unavailable("read incremental analytics stages",error))?;
|
||||
let mut progress = BTreeMap::new();
|
||||
for row in progress_rows {
|
||||
let phase =
|
||||
if get::<Option<i64>>(&row, "build_version")? == Some(get(&row, "change_version")?) {
|
||||
get(&row, "build_phase")?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
progress.insert(
|
||||
get::<DateTime<Utc>>(&row, "bucket_start")?,
|
||||
Progress {
|
||||
phase,
|
||||
started_at: get(&row, "build_started_at")?,
|
||||
id: get(&row, "build_id")?,
|
||||
collection: get(&row, "build_collection")?,
|
||||
dimensions: get(&row, "build_dimensions")?,
|
||||
outcome: get(&row, "build_outcome")?,
|
||||
},
|
||||
);
|
||||
}
|
||||
// 两种粒度共用一次事实解码,仍分别检查构建前缀,不能把未发布季度并入小时
|
||||
let mut changes = [BTreeMap::<GroupKey, Change>::new(), BTreeMap::new()];
|
||||
let clean_hours = progress.is_empty();
|
||||
for (rows, adding) in [(previous, false), (current, true)] {
|
||||
for row in rows {
|
||||
let time = get(row, "started_at")?;
|
||||
let quarter = bucket_for(time, 900)?;
|
||||
let hour = bucket_for(time, 3600)?;
|
||||
let id: String = get(row, "id")?;
|
||||
let outcome: String = get(row, "outcome")?;
|
||||
let mut state = MetricState::default();
|
||||
state.add_row(row)?;
|
||||
let forecast = forecast::state(row)?;
|
||||
let mut hour_pending = progress.range(hour..hour + TimeDelta::hours(1)).peekable();
|
||||
let hour_ready = hour_pending.peek().is_none();
|
||||
let hour_owner =
|
||||
hour_pending.find_map(|(_, stage)| (stage.phase >= 3).then_some(stage));
|
||||
for (collection, dimensions) in collections(row)? {
|
||||
let key = (quarter, collection.to_owned(), dimensions, outcome.clone());
|
||||
let included = [
|
||||
progress
|
||||
.get(&quarter)
|
||||
.is_none_or(|p| p.contains_fact(time, &id)),
|
||||
!clean_hours
|
||||
&& (hour_ready
|
||||
|| hour_owner.is_some_and(|stage| stage.contains_quarter(&key))),
|
||||
];
|
||||
let hourly_key = (hour, key.1.clone(), key.2.clone(), key.3.clone());
|
||||
for (index, key) in [key, hourly_key].into_iter().enumerate() {
|
||||
if !included[index] {
|
||||
continue;
|
||||
}
|
||||
let change = changes[index].entry(key).or_default();
|
||||
let state = if collection == forecast::COLLECTION {
|
||||
&forecast
|
||||
} else {
|
||||
&state
|
||||
};
|
||||
if adding {
|
||||
change.current.merge(state)?;
|
||||
} else {
|
||||
change.previous.merge(state)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if clean_hours {
|
||||
// 无重建中的时间桶时,小时贡献可直接合并季度贡献,避免逐事实再合并一遍全部指标
|
||||
let [quarters, hours] = &mut changes;
|
||||
for ((quarter, collection, dimensions, outcome), contribution) in quarters.iter() {
|
||||
let key = (
|
||||
bucket_for(*quarter, 3600)?,
|
||||
collection.clone(),
|
||||
dimensions.clone(),
|
||||
outcome.clone(),
|
||||
);
|
||||
let hour = hours.entry(key).or_default();
|
||||
hour.previous.merge(&contribution.previous)?;
|
||||
hour.current.merge(&contribution.current)?;
|
||||
}
|
||||
}
|
||||
for (index, changes) in changes.into_iter().enumerate() {
|
||||
apply_changes(tx, index == 1, &progress, changes).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_changes(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
hourly: bool,
|
||||
progress: &BTreeMap<DateTime<Utc>, Progress>,
|
||||
changes: BTreeMap<GroupKey, Change>,
|
||||
) -> StoreResult<()> {
|
||||
if changes.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let table = if hourly {
|
||||
"request_analytics_hourly"
|
||||
} else {
|
||||
"request_analytics_15m"
|
||||
};
|
||||
let keys = changes.keys().map(|(bucket,collection,dimensions,outcome)|serde_json::json!({"bucket_start":bucket,"collection":collection,"dimensions":dimensions,"outcome":outcome})).collect::<Vec<_>>();
|
||||
let statement = format!(
|
||||
"select a.* from {table} a join jsonb_to_recordset($1::jsonb) k(bucket_start timestamptz,collection text,dimensions text[],outcome text) using(bucket_start,collection,dimensions,outcome)"
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(serde_json::Value::Array(keys))
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read incremental analytics groups", error))?;
|
||||
let mut stored = BTreeMap::new();
|
||||
for row in rows {
|
||||
stored.insert(
|
||||
(
|
||||
get(&row, "bucket_start")?,
|
||||
get(&row, "collection")?,
|
||||
get(&row, "dimensions")?,
|
||||
get(&row, "outcome")?,
|
||||
),
|
||||
MetricState::decode(&row)?,
|
||||
);
|
||||
}
|
||||
let mut buckets =
|
||||
BTreeMap::<DateTime<Utc>, BTreeMap<(String, Vec<String>, String), MetricState>>::new();
|
||||
for (key, change) in changes {
|
||||
if change.previous.encode()? == change.current.encode()?
|
||||
&& change.previous.last_used_at() == change.current.last_used_at()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut state = stored.remove(&key).unwrap_or_default();
|
||||
let needs_rebuild = subtract(&mut state, &change.previous, &change.current)?;
|
||||
state.merge(&change.current)?;
|
||||
if state.count("request_count") == 0 {
|
||||
let statement = format!(
|
||||
"delete from {table} where bucket_start=$1 and collection=$2 and dimensions=$3 and outcome=$4"
|
||||
);
|
||||
sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(key.0)
|
||||
.bind(&key.1)
|
||||
.bind(&key.2)
|
||||
.bind(&key.3)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("remove empty analytics group", error))?;
|
||||
} else {
|
||||
// 预测状态只有可加计数和美元总额,零金额无需修复币种存在性
|
||||
if needs_rebuild && key.1 != forecast::COLLECTION {
|
||||
state = rebuild_group(tx, hourly, progress, &key).await?;
|
||||
}
|
||||
buckets
|
||||
.entry(key.0)
|
||||
.or_default()
|
||||
.insert((key.1, key.2, key.3), state);
|
||||
}
|
||||
}
|
||||
write_bucket_groups(tx, table, buckets).await
|
||||
}
|
||||
|
||||
/// 返回是否撤销了不能从可加状态推导的极值或币种存在性
|
||||
pub(super) fn subtract(
|
||||
state: &mut MetricState,
|
||||
previous: &MetricState,
|
||||
current: &MetricState,
|
||||
) -> StoreResult<bool> {
|
||||
if previous.count("request_count") == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
// 新贡献仍持有旧极值时无需重读事实,例如只补齐 Token、费用或重放同一请求
|
||||
let mut rebuild = previous.last_used_at.is_some()
|
||||
&& previous.last_used_at == state.last_used_at
|
||||
&& current.last_used_at < state.last_used_at;
|
||||
for (name, value) in &previous.counts {
|
||||
let count = state.counts.entry(name.clone()).or_default();
|
||||
*count = count
|
||||
.checked_sub(*value)
|
||||
.ok_or_else(|| invalid("analytics counter underflow"))?;
|
||||
}
|
||||
for (name, removed) in &previous.distributions {
|
||||
let distribution = state
|
||||
.distributions
|
||||
.get_mut(name)
|
||||
.ok_or_else(|| invalid("missing analytics distribution"))?;
|
||||
distribution.count = distribution
|
||||
.count
|
||||
.checked_sub(removed.count)
|
||||
.ok_or_else(|| invalid("analytics distribution underflow"))?;
|
||||
distribution.zero = distribution
|
||||
.zero
|
||||
.checked_sub(removed.zero)
|
||||
.ok_or_else(|| invalid("analytics zero count underflow"))?;
|
||||
distribution.sum -= removed.sum;
|
||||
for (bin, value) in &removed.bins {
|
||||
let count = distribution
|
||||
.bins
|
||||
.get_mut(bin)
|
||||
.ok_or_else(|| invalid("missing analytics bin"))?;
|
||||
*count = count
|
||||
.checked_sub(*value)
|
||||
.ok_or_else(|| invalid("analytics bin underflow"))?;
|
||||
}
|
||||
distribution.bins.retain(|_, count| *count > 0);
|
||||
if distribution.count == 0 {
|
||||
*distribution = Default::default();
|
||||
} else {
|
||||
let replacement = current.distributions.get(name);
|
||||
rebuild |= distribution.minimum == removed.minimum
|
||||
&& replacement
|
||||
.and_then(|value| value.minimum)
|
||||
.is_none_or(|value| Some(value) > distribution.minimum);
|
||||
rebuild |= distribution.maximum == removed.maximum
|
||||
&& replacement
|
||||
.and_then(|value| value.maximum)
|
||||
.is_none_or(|value| Some(value) < distribution.maximum);
|
||||
rebuild |= !distribution.sum.is_finite() || distribution.sum < 0.0;
|
||||
}
|
||||
}
|
||||
for (currency, amount) in &previous.costs {
|
||||
let stored_amount = state
|
||||
.costs
|
||||
.get(currency)
|
||||
.ok_or_else(|| invalid("missing analytics currency"))?;
|
||||
let stored_amount: gateway_core::metering::Decimal = stored_amount
|
||||
.parse()
|
||||
.map_err(|_| invalid("invalid analytics amount"))?;
|
||||
let removed: gateway_core::metering::Decimal = amount
|
||||
.parse()
|
||||
.map_err(|_| invalid("invalid analytics amount"))?;
|
||||
let value = stored_amount
|
||||
.scaled()
|
||||
.checked_sub(removed.scaled())
|
||||
.ok_or_else(|| invalid("analytics amount underflow"))?;
|
||||
rebuild |= value == 0 && !current.costs.contains_key(currency);
|
||||
state.costs.insert(
|
||||
currency.clone(),
|
||||
gateway_core::metering::Decimal::from_scaled(value)
|
||||
.map_err(|_| invalid("analytics amount overflow"))?
|
||||
.canonical(),
|
||||
);
|
||||
}
|
||||
Ok(rebuild)
|
||||
}
|
||||
|
||||
async fn rebuild_group(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
hourly: bool,
|
||||
progress: &BTreeMap<DateTime<Utc>, Progress>,
|
||||
key: &GroupKey,
|
||||
) -> StoreResult<MetricState> {
|
||||
let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
let mut cursor = None;
|
||||
if hourly {
|
||||
if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4)
|
||||
&& let Some(time) = owner.started_at
|
||||
{
|
||||
let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone());
|
||||
end = if owner.contains_quarter(&candidate) {
|
||||
time + TimeDelta::minutes(15)
|
||||
} else {
|
||||
time
|
||||
};
|
||||
}
|
||||
} else if let Some(stage) = progress.get(&key.0)
|
||||
&& stage.phase == 1
|
||||
{
|
||||
cursor = stage.started_at.zip(stage.id.as_deref());
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new("with matching as materialized (select mr.*,");
|
||||
query.push_bind(key.0).push("::timestamptz as output_bucket,''::text as name,coalesce((")
|
||||
.push(completed_usage_fact_predicate("mr")).push("),false) as is_completed_usage from request_analytics_facts mr where mr.started_at >= ")
|
||||
.push_bind(key.0).push(" and mr.started_at < ").push_bind(end)
|
||||
.push(" and mr.recovered_at is null and mr.outcome=").push_bind(&key.3);
|
||||
if let Some((time, id)) = cursor {
|
||||
query
|
||||
.push(" and (mr.started_at,mr.id collate \"C\") <= (")
|
||||
.push_bind(time)
|
||||
.push(",")
|
||||
.push_bind(id)
|
||||
.push(")");
|
||||
}
|
||||
let columns = worker::COLLECTION_COLUMNS
|
||||
.iter()
|
||||
.find_map(|(collection, columns)| (*collection == key.1).then_some(*columns))
|
||||
.ok_or_else(|| invalid("unknown incremental analytics collection"))?;
|
||||
for (column, value) in columns.iter().zip(&key.2) {
|
||||
query
|
||||
.push(format!(" and coalesce(mr.{column}::text,'')="))
|
||||
.push_bind(value);
|
||||
}
|
||||
query.push(")");
|
||||
aggregate::push_fact_aggregation(&mut query, false, true);
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("rebuild changed analytics extrema", error))?;
|
||||
row.as_ref()
|
||||
.map(MetricState::decode)
|
||||
.transpose()
|
||||
.map(Option::unwrap_or_default)
|
||||
}
|
||||
@@ -5,13 +5,25 @@ use super::histogram::Histogram;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::postgres::PgRow;
|
||||
|
||||
pub(super) const TOKEN_METRICS: [&str; 8] = [
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cached_input_tokens",
|
||||
"cache_write_input_tokens",
|
||||
"reasoning_output_tokens",
|
||||
"total_tokens",
|
||||
"image_input_tokens",
|
||||
"image_output_tokens",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub(super) struct MetricState {
|
||||
counts: BTreeMap<String, u64>,
|
||||
distributions: BTreeMap<String, Histogram>,
|
||||
costs: BTreeMap<String, String>,
|
||||
pub(crate) struct MetricState {
|
||||
pub(super) counts: BTreeMap<String, u64>,
|
||||
pub(super) distributions: BTreeMap<String, Histogram>,
|
||||
pub(super) costs: BTreeMap<String, String>,
|
||||
pub(super) last_used_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip)]
|
||||
exact_samples: Option<BTreeMap<String, Vec<f64>>>,
|
||||
exact_percentiles: Option<BTreeMap<String, Vec<Option<f64>>>>,
|
||||
}
|
||||
|
||||
impl MetricState {
|
||||
@@ -25,9 +37,23 @@ impl MetricState {
|
||||
"distributions",
|
||||
)?
|
||||
.0,
|
||||
exact_samples: None,
|
||||
last_used_at: get(row, "last_used_at")?,
|
||||
exact_percentiles: None,
|
||||
})
|
||||
}
|
||||
pub(super) fn decode_aggregate(row: &PgRow, exact: bool) -> StoreResult<Self> {
|
||||
let mut state = Self::decode(row)?;
|
||||
if exact {
|
||||
state.exact_percentiles = Some(
|
||||
get::<sqlx::types::Json<BTreeMap<String, Vec<Option<f64>>>>>(
|
||||
row,
|
||||
"exact_percentiles",
|
||||
)?
|
||||
.0,
|
||||
);
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
pub fn encode(&self) -> StoreResult<(serde_json::Value, serde_json::Value, serde_json::Value)> {
|
||||
Ok((
|
||||
serde_json::to_value(&self.counts)
|
||||
@@ -38,17 +64,24 @@ impl MetricState {
|
||||
.map_err(|error| postgres_unavailable("encode analytics histograms", error))?,
|
||||
))
|
||||
}
|
||||
pub fn exact() -> Self {
|
||||
Self {
|
||||
exact_samples: Some(BTreeMap::new()),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
pub fn count(&self, name: &str) -> u64 {
|
||||
self.counts.get(name).copied().unwrap_or_default()
|
||||
}
|
||||
fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> {
|
||||
let count = self.counts.entry(name.to_owned()).or_default();
|
||||
pub fn optional_count(&self, name: &str) -> Option<u64> {
|
||||
(self.count(&format!("{name}_known_count")) > 0).then(|| self.count(name))
|
||||
}
|
||||
pub fn last_used_at(&self) -> Option<DateTime<Utc>> {
|
||||
self.last_used_at
|
||||
}
|
||||
pub(super) fn has_exact_percentiles(&self) -> bool {
|
||||
self.exact_percentiles.is_some()
|
||||
}
|
||||
pub(super) fn increment(&mut self, name: &str, amount: u64) -> StoreResult<()> {
|
||||
// 合并同一分组时键通常已经存在,避免每条事实都为已有指标分配字符串
|
||||
let count = match self.counts.get_mut(name) {
|
||||
Some(count) => count,
|
||||
None => self.counts.entry(name.to_owned()).or_default(),
|
||||
};
|
||||
*count = count
|
||||
.checked_add(amount)
|
||||
.ok_or_else(|| invalid("analytics counter overflow"))?;
|
||||
@@ -60,13 +93,10 @@ impl MetricState {
|
||||
.entry(name.to_owned())
|
||||
.or_default()
|
||||
.add(sample)?;
|
||||
if let Some(samples) = &mut self.exact_samples {
|
||||
samples.entry(name.to_owned()).or_default().push(sample);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> {
|
||||
pub(super) fn add_cost(&mut self, currency: &str, amount: &str) -> StoreResult<()> {
|
||||
let amount: DecimalAmount = amount
|
||||
.parse()
|
||||
.map_err(|_| invalid("analytics invalid cost"))?;
|
||||
@@ -83,19 +113,28 @@ impl MetricState {
|
||||
Ok(())
|
||||
}
|
||||
pub fn merge(&mut self, other: &Self) -> StoreResult<()> {
|
||||
// 精确分位数只属于数据库已经完成分组的结果,不能把分位数再平均或合并
|
||||
if self.count("request_count") > 0
|
||||
&& other.count("request_count") > 0
|
||||
&& (self.exact_percentiles.is_some() || other.exact_percentiles.is_some())
|
||||
{
|
||||
return Err(invalid(
|
||||
"cannot merge finalized exact analytics percentiles",
|
||||
));
|
||||
}
|
||||
if self.count("request_count") == 0 {
|
||||
self.exact_percentiles.clone_from(&other.exact_percentiles);
|
||||
}
|
||||
self.last_used_at = self.last_used_at.max(other.last_used_at);
|
||||
for (name, count) in &other.counts {
|
||||
self.increment(name, *count)?;
|
||||
}
|
||||
for (name, distribution) in &other.distributions {
|
||||
self.distributions
|
||||
.entry(name.clone())
|
||||
.or_default()
|
||||
.merge(distribution)?;
|
||||
}
|
||||
if let (Some(ours), Some(theirs)) = (&mut self.exact_samples, &other.exact_samples) {
|
||||
for (name, samples) in theirs {
|
||||
ours.entry(name.clone()).or_default().extend(samples);
|
||||
}
|
||||
let stored = match self.distributions.get_mut(name) {
|
||||
Some(stored) => stored,
|
||||
None => self.distributions.entry(name.clone()).or_default(),
|
||||
};
|
||||
stored.merge(distribution)?;
|
||||
}
|
||||
for (currency, amount) in &other.costs {
|
||||
self.add_cost(currency, amount)?;
|
||||
@@ -185,18 +224,31 @@ impl MetricState {
|
||||
self.sample("capacity", Some(used as f64 / total as f64 * 10_000.0))?;
|
||||
}
|
||||
if completed {
|
||||
for name in [
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cached_input_tokens",
|
||||
"cache_write_input_tokens",
|
||||
"reasoning_output_tokens",
|
||||
"total_tokens",
|
||||
"image_input_tokens",
|
||||
"image_output_tokens",
|
||||
self.increment("completed_request_count", 1)?;
|
||||
self.last_used_at = self.last_used_at.max(Some(get(row, "started_at")?));
|
||||
for (value, name) in [
|
||||
(Some(true), "image_request_count"),
|
||||
(Some(false), "image_request_failed_count"),
|
||||
] {
|
||||
self.increment(name, optional_unsigned(row, name)?.unwrap_or_default())?;
|
||||
if get::<Option<bool>>(row, "image_generation_succeeded")? == value {
|
||||
self.increment(name, 1)?;
|
||||
}
|
||||
}
|
||||
for name in TOKEN_METRICS {
|
||||
if let Some(value) = optional_unsigned(row, name)? {
|
||||
self.increment(name, value)?;
|
||||
self.increment(&format!("{name}_known_count"), 1)?;
|
||||
}
|
||||
}
|
||||
self.increment(
|
||||
"window_total_tokens",
|
||||
optional_unsigned(row, "total_tokens")?.unwrap_or(
|
||||
optional_unsigned(row, "input_tokens")?
|
||||
.unwrap_or_default()
|
||||
.checked_add(optional_unsigned(row, "output_tokens")?.unwrap_or_default())
|
||||
.ok_or_else(|| invalid("analytics token overflow"))?,
|
||||
),
|
||||
)?;
|
||||
if get::<Option<i64>>(row, "input_tokens")?.is_some() {
|
||||
self.increment("cache_eligible_request_count", 1)?;
|
||||
if optional_unsigned(row, "cached_input_tokens")?.is_some_and(|value| value > 0) {
|
||||
@@ -245,13 +297,11 @@ impl MetricState {
|
||||
self.distributions.get(name).cloned().unwrap_or_default()
|
||||
}
|
||||
pub fn percentile(&self, name: &str, fraction: f64) -> Option<f64> {
|
||||
if let Some(samples) = &self.exact_samples {
|
||||
let mut samples = samples.get(name)?.clone();
|
||||
samples.sort_by(f64::total_cmp);
|
||||
let rank = samples.len().saturating_sub(1) as f64 * fraction;
|
||||
let a = *samples.get(rank.floor() as usize)?;
|
||||
let b = *samples.get(rank.ceil() as usize)?;
|
||||
return Some(a + (b - a) * rank.fract());
|
||||
if let Some(percentiles) = &self.exact_percentiles {
|
||||
let index = [0.1, 0.5, 0.9, 0.95, 0.99]
|
||||
.iter()
|
||||
.position(|value| (*value - fraction).abs() < f64::EPSILON)?;
|
||||
return percentiles.get(name)?.get(index).copied().flatten();
|
||||
}
|
||||
self.histogram(name).percentile(fraction)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
//! 请求观测的持久化投影、可重建汇总与受预算约束的读取
|
||||
|
||||
pub(crate) mod accounts;
|
||||
mod aggregate;
|
||||
mod buckets;
|
||||
mod counts;
|
||||
mod forecast;
|
||||
mod histogram;
|
||||
mod incremental;
|
||||
mod metrics;
|
||||
mod query;
|
||||
mod reset;
|
||||
@@ -11,15 +16,15 @@ pub(crate) use query::*;
|
||||
pub use worker::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
|
||||
|
||||
use super::*;
|
||||
use metrics::MetricState;
|
||||
pub(crate) use metrics::MetricState;
|
||||
|
||||
const SCHEMA_VERSION: i32 = 1;
|
||||
const HISTOGRAM_VERSION: i32 = histogram::HISTOGRAM_VERSION;
|
||||
const MAX_FACT_ROWS: usize = 100_000;
|
||||
|
||||
pub(super) async fn begin_snapshot(
|
||||
connection: &mut PgConnection,
|
||||
connection: &mut ObservabilityConnection,
|
||||
) -> StoreResult<sqlx::Transaction<'_, Postgres>> {
|
||||
let query_timeout = connection.query_timeout();
|
||||
let mut transaction = connection
|
||||
.begin()
|
||||
.await
|
||||
@@ -28,7 +33,8 @@ pub(super) async fn begin_snapshot(
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("isolate analytics snapshot", error))?;
|
||||
sqlx::query("set local statement_timeout = '10s'")
|
||||
sqlx::query("select set_config('statement_timeout',$1,true)")
|
||||
.bind(format!("{}ms", query_timeout.as_millis()))
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("set analytics query deadline", error))?;
|
||||
|
||||
@@ -5,7 +5,7 @@ use futures::TryStreamExt;
|
||||
|
||||
pub(crate) struct AnalyticsSnapshot<'a> {
|
||||
pub tx: sqlx::Transaction<'a, Postgres>,
|
||||
status: AnalyticsStatus,
|
||||
pub(super) status: AnalyticsStatus,
|
||||
approximate: bool,
|
||||
}
|
||||
|
||||
@@ -13,10 +13,25 @@ struct ReadSelection<'a> {
|
||||
distributions: bool,
|
||||
costs: bool,
|
||||
keys: Option<&'a [String]>,
|
||||
counters: Option<&'a [&'a str]>,
|
||||
distribution_names: Option<&'a [&'a str]>,
|
||||
}
|
||||
|
||||
impl<'a> AnalyticsSnapshot<'a> {
|
||||
pub async fn new(connection: &'a mut PgConnection) -> StoreResult<Self> {
|
||||
pub async fn new(connection: &'a mut ObservabilityConnection) -> StoreResult<Self> {
|
||||
Self::begin(connection, true).await
|
||||
}
|
||||
|
||||
pub(crate) async fn new_for_accounts(
|
||||
connection: &'a mut ObservabilityConnection,
|
||||
) -> StoreResult<Self> {
|
||||
Self::begin(connection, false).await
|
||||
}
|
||||
|
||||
async fn begin(
|
||||
connection: &'a mut ObservabilityConnection,
|
||||
require_current_schema: bool,
|
||||
) -> StoreResult<Self> {
|
||||
let mut tx = begin_snapshot(connection).await?;
|
||||
// 时间跨度和分组选择性差异很大,通用计划会把长区间误估成短区间
|
||||
// 仅当前只读事务使用实际参数规划,不改变连接池后续事务的计划策略
|
||||
@@ -25,8 +40,9 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("plan analytics with actual bounds", error))?;
|
||||
let status = worker::query_status(&mut tx).await?;
|
||||
if status.schema_version != SCHEMA_VERSION as u32
|
||||
|| status.histogram_version != HISTOGRAM_VERSION as u32
|
||||
if require_current_schema
|
||||
&& (status.schema_version != SCHEMA_VERSION as u32
|
||||
|| status.histogram_version != HISTOGRAM_VERSION as u32)
|
||||
{
|
||||
return Err(unavailable(
|
||||
"request analytics is building; retry after the background projection completes",
|
||||
@@ -163,14 +179,35 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
.load(range, filter, None, None, Default::default(), true)
|
||||
.await?;
|
||||
let state = merge_states(states.into_values())?;
|
||||
self.overview_from_state(range, filter, state).await
|
||||
}
|
||||
|
||||
async fn overview_from_state(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
state: MetricState,
|
||||
) -> StoreResult<UsageOverview> {
|
||||
// Provider 列表只消费计数,不重复合并费用与分布
|
||||
let provider_states = self
|
||||
.load(
|
||||
.load_selected(
|
||||
range,
|
||||
filter,
|
||||
Some(DiagnosticDimension::Provider),
|
||||
None,
|
||||
Default::default(),
|
||||
false,
|
||||
ReadSelection {
|
||||
distributions: false,
|
||||
costs: false,
|
||||
keys: None,
|
||||
counters: Some(&[
|
||||
"request_count",
|
||||
"attempt_count",
|
||||
"failure_count",
|
||||
"total_tokens",
|
||||
]),
|
||||
distribution_names: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let mut providers = provider_states
|
||||
@@ -216,47 +253,48 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
distributions: true,
|
||||
costs: include_costs,
|
||||
keys: None,
|
||||
counters: None,
|
||||
distribution_names: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let mut points = BTreeMap::new();
|
||||
for ((bucket, _), state) in states {
|
||||
points.insert(
|
||||
bucket,
|
||||
RequestMetricPoint {
|
||||
bucket_start: bucket,
|
||||
granularity,
|
||||
metrics: state.request_metrics()?,
|
||||
cost_coverage: state.coverage(),
|
||||
costs: if include_costs {
|
||||
state.costs()?
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
fill_metric_gaps(range, granularity, points, timezone)
|
||||
metric_points(range, granularity, timezone, &states, include_costs)
|
||||
}
|
||||
|
||||
pub async fn projected_costs(
|
||||
pub async fn insights(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> StoreResult<Vec<admin_observability::UsageProjectedCostPoint>> {
|
||||
self.load(range, filter, None, Some(granularity), timezone, false)
|
||||
.await?
|
||||
.into_iter()
|
||||
) -> StoreResult<(
|
||||
UsageOverview,
|
||||
Vec<RequestMetricPoint>,
|
||||
Vec<admin_observability::UsageProjectedCostPoint>,
|
||||
)> {
|
||||
let states = self
|
||||
.load(range, filter, None, Some(granularity), timezone, true)
|
||||
.await?;
|
||||
let trend = metric_points(range, granularity, timezone, &states, true)?;
|
||||
let costs = states
|
||||
.iter()
|
||||
.map(|((bucket_start, _), state)| {
|
||||
Ok(admin_observability::UsageProjectedCostPoint {
|
||||
bucket_start,
|
||||
bucket_start: *bucket_start,
|
||||
standard_amount: state.projected_cost("standard_usd")?,
|
||||
no_cache_amount: state.projected_cost("no_cache_usd")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
// 趋势已包含可合并的完整指标和情景费用,概览直接复用同一份状态
|
||||
// 精确分位数不可跨桶合并,明细回退时仍按完整区间单独计算概览
|
||||
let overview = if states.values().any(MetricState::has_exact_percentiles) {
|
||||
self.overview(range, filter).await?
|
||||
} else {
|
||||
self.overview_from_state(range, filter, merge_states(states.into_values())?)
|
||||
.await?
|
||||
};
|
||||
Ok((overview, trend, costs))
|
||||
}
|
||||
|
||||
pub async fn diagnostics(
|
||||
@@ -273,8 +311,10 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
let ranked = if matches!(
|
||||
dimension,
|
||||
DiagnosticDimension::Account | DiagnosticDimension::ApiKey
|
||||
) && collection_for(filter, Some(dimension)).is_some()
|
||||
{
|
||||
) && matches!(
|
||||
collection_for(filter, Some(dimension)),
|
||||
Some("account" | "key")
|
||||
) {
|
||||
Some(
|
||||
self.diagnostic_top_keys(range, filter, dimension, limit)
|
||||
.await?,
|
||||
@@ -294,6 +334,22 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
distributions: true,
|
||||
costs: true,
|
||||
keys: selected_keys,
|
||||
counters: Some(&[
|
||||
"request_count",
|
||||
"success_count",
|
||||
"failure_count",
|
||||
"attempt_count",
|
||||
"total_tokens",
|
||||
"latency_ms_sum",
|
||||
"cancelled_count",
|
||||
"incomplete_count",
|
||||
"retry_count",
|
||||
"retried_request_count",
|
||||
"provider_reported_count",
|
||||
"calculated_count",
|
||||
"unavailable_count",
|
||||
]),
|
||||
distribution_names: Some(&["latency_ms", "first_token_ms"]),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
@@ -415,16 +471,18 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
.collect::<Vec<_>>();
|
||||
let mut query = QueryBuilder::<Postgres>::new("with edge_facts as materialized (");
|
||||
if starts.is_empty() {
|
||||
query.push("select null::text as key where false");
|
||||
query.push("select null::text as key,0::bigint as request_count where false");
|
||||
} else {
|
||||
query.push("select edge.key from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ")
|
||||
.push(format!("coalesce(mr.{column},'') as key from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at"));
|
||||
query.push("select edge.key,sum(edge.request_count)::bigint as request_count from unnest(").push_bind(starts).push("::timestamptz[],").push_bind(ends).push("::timestamptz[]) span(start_at,end_at) cross join lateral (select ")
|
||||
.push(format!("coalesce(mr.{column},'') as key,count(*)::bigint as request_count from request_analytics_facts mr where mr.started_at >= span.start_at and mr.started_at < span.end_at"));
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_analytics_filter(&mut query, filter, "mr");
|
||||
push_diagnostic_dimension_filter(&mut query, dimension);
|
||||
query.push(" limit 100001) edge limit 100001");
|
||||
query.push(format!(
|
||||
" group by coalesce(mr.{column},'')) edge group by edge.key"
|
||||
));
|
||||
}
|
||||
query.push("), contributions as (select key,count(*)::bigint as request_count from edge_facts group by key");
|
||||
query.push("), contributions as (select key,request_count from edge_facts");
|
||||
for (table, column, seconds, buckets) in [
|
||||
("request_analytics_daily_counts", "dimension", 86400, days),
|
||||
("request_analytics_hourly", "dimensions[1]", 3600, hours),
|
||||
@@ -446,7 +504,7 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
}
|
||||
query.push(format!(" group by {column}"));
|
||||
}
|
||||
query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count,(select count(*) from edge_facts)::bigint as edge_count from grouped order by request_count desc,");
|
||||
query.push("), grouped as (select key,sum(request_count)::bigint as request_count from contributions group by key) select key,request_count,sum(request_count) over()::bigint as total_request_count from grouped order by request_count desc,");
|
||||
if dimension == DiagnosticDimension::Account {
|
||||
query.push("coalesce(nullif(key,''),'unrouted')");
|
||||
} else {
|
||||
@@ -458,12 +516,6 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
.fetch_all(&mut *self.tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("rank full-range analytics groups", error))?;
|
||||
if rows.first().is_some_and(|row| {
|
||||
row.try_get::<i64, _>("edge_count")
|
||||
.is_ok_and(|count| count > MAX_FACT_ROWS as i64)
|
||||
}) {
|
||||
return Err(unavailable("analytics fact row budget exceeded"));
|
||||
}
|
||||
let total = rows
|
||||
.first()
|
||||
.map(|row| unsigned(row, "total_request_count"))
|
||||
@@ -476,18 +528,15 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
Ok((total, keys))
|
||||
}
|
||||
|
||||
async fn summary_buckets(
|
||||
pub(super) async fn summary_buckets(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
use_hourly: bool,
|
||||
) -> StoreResult<(Vec<DateTime<Utc>>, Vec<DateTime<Utc>>)> {
|
||||
// 先按时间确定互斥的完整桶,空的十五分钟集合不读取汇总表
|
||||
let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start limit 100001")
|
||||
let rows = sqlx::query("with quarters as materialized (select q.bucket_start,date_bin(interval '1 hour',q.bucket_start,timestamptz '2000-01-01 UTC') as hour_start from generate_series(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),$2::timestamptz-interval '15 minutes',interval '15 minutes') q(bucket_start) where q.bucket_start >= $1 and not exists(select 1 from request_analytics_pending_buckets p where p.bucket_start=q.bucket_start)), hours as (select hour_start from quarters where $3 group by hour_start having count(*)=4) select q.bucket_start,h.hour_start from quarters q left join hours h on h.hour_start=q.hour_start order by q.bucket_start")
|
||||
.bind(range.start).bind(range.end).bind(use_hourly)
|
||||
.fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("select disjoint analytics buckets",error))?;
|
||||
if rows.len() > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics time bucket budget exceeded"));
|
||||
}
|
||||
let mut hours = Vec::new();
|
||||
let mut quarters = Vec::new();
|
||||
for row in rows {
|
||||
@@ -502,7 +551,7 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
Ok((hours, quarters))
|
||||
}
|
||||
|
||||
async fn uncovered_spans(
|
||||
pub(super) async fn uncovered_spans(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<Vec<(DateTime<Utc>, DateTime<Utc>)>> {
|
||||
@@ -522,11 +571,8 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
spans.push((right_start, range.end));
|
||||
}
|
||||
if complete_start < complete_end {
|
||||
let dirty:Vec<DateTime<Utc>>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start limit 100001")
|
||||
let dirty:Vec<DateTime<Utc>>=sqlx::query_scalar("select bucket_start from request_analytics_pending_buckets where bucket_start >= $1 and bucket_start < $2 order by bucket_start")
|
||||
.bind(complete_start).bind(complete_end).fetch_all(&mut *self.tx).await.map_err(|error|postgres_unavailable("plan bounded analytics fact spans",error))?;
|
||||
if dirty.len() > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics dirty interval budget exceeded"));
|
||||
}
|
||||
spans.extend(
|
||||
dirty
|
||||
.into_iter()
|
||||
@@ -534,7 +580,17 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
);
|
||||
}
|
||||
spans.sort_unstable();
|
||||
Ok(spans)
|
||||
let mut merged: Vec<(DateTime<Utc>, DateTime<Utc>)> = Vec::new();
|
||||
for (start, end) in spans {
|
||||
if let Some(previous) = merged.last_mut()
|
||||
&& previous.1 == start
|
||||
{
|
||||
previous.1 = end;
|
||||
} else {
|
||||
merged.push((start, end));
|
||||
}
|
||||
}
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
async fn load(
|
||||
@@ -556,6 +612,8 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
distributions: read_distributions,
|
||||
costs: true,
|
||||
keys: None,
|
||||
counters: None,
|
||||
distribution_names: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -574,6 +632,8 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
distributions: read_distributions,
|
||||
costs: read_costs,
|
||||
keys: selected_keys,
|
||||
counters,
|
||||
distribution_names,
|
||||
} = selection;
|
||||
filter.validate()?;
|
||||
self.require_visible(Some(range)).await?;
|
||||
@@ -610,90 +670,63 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
if buckets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (dimension_filter, either_filter, either_positions) = match collection {
|
||||
"provider_model" => {
|
||||
(filter.provider_kind.clone(), filter.model.clone(), (2, 3))
|
||||
}
|
||||
"account" => (filter.provider_account_ref.clone(), None, (1, 2)),
|
||||
"key" => (filter.client_api_key_ref.clone(), None, (1, 2)),
|
||||
"operation" => (filter.operation.clone(), None, (1, 2)),
|
||||
"transport" => (None, filter.transport.clone(), (1, 2)),
|
||||
"status" => (
|
||||
None,
|
||||
filter.status_code.map(|value| value.to_string()),
|
||||
(1, 2),
|
||||
),
|
||||
_ => (None, None, (1, 2)),
|
||||
let mut query =
|
||||
QueryBuilder::<Postgres>::new("with matching as not materialized (select ");
|
||||
aggregate::push_output_bucket(
|
||||
&mut query,
|
||||
"bucket_start",
|
||||
range,
|
||||
granularity,
|
||||
boundaries.as_deref(),
|
||||
);
|
||||
let name = aggregate::summary_dimension_sql(collection, dimension);
|
||||
let costs = if read_costs {
|
||||
"costs"
|
||||
} else {
|
||||
"'{}'::jsonb as costs"
|
||||
};
|
||||
let distributions = if read_distributions {
|
||||
"distributions"
|
||||
} else {
|
||||
"'{}'::jsonb as distributions"
|
||||
};
|
||||
let costs = if read_costs {
|
||||
"costs"
|
||||
} else {
|
||||
"'{}'::jsonb as costs"
|
||||
};
|
||||
let provider_counts = dimension == Some(DiagnosticDimension::Provider)
|
||||
&& granularity.is_none()
|
||||
&& !read_distributions;
|
||||
let mut query = QueryBuilder::<Postgres>::new(if provider_counts {
|
||||
"select min(bucket_start) as bucket_start, array[dimensions[1]] as dimensions, ''::text as outcome, jsonb_build_object('request_count',sum((counts->>'request_count')::bigint),'attempt_count',sum((counts->>'attempt_count')::bigint),'failure_count',sum(coalesce((counts->>'failure_count')::bigint,0)),'total_tokens',sum(coalesce((counts->>'total_tokens')::bigint,0))) as counts, '{}'::jsonb as costs, '{}'::jsonb as distributions from ("
|
||||
} else {
|
||||
""
|
||||
});
|
||||
query.push(format!("select bucket_start, dimensions, outcome, counts, {costs}, {distributions} from {table} a where collection="))
|
||||
query.push(format!(" as output_bucket,{name} as name,counts,{costs},{distributions},last_used_at from {table} a where collection="))
|
||||
.push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION)
|
||||
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION);
|
||||
push_summary_buckets(&mut query, &buckets, seconds);
|
||||
// 可选筛选仅在有值时生成谓词,保留维度与时间索引的选择空间
|
||||
if let Some(outcome) = &filter.outcome {
|
||||
query.push(" and outcome=").push_bind(outcome);
|
||||
}
|
||||
if let Some(value) = dimension_filter {
|
||||
query.push(" and dimensions[1]=").push_bind(value);
|
||||
}
|
||||
if let Some(value) = either_filter {
|
||||
query
|
||||
.push(format!(" and (dimensions[{}]=", either_positions.0))
|
||||
.push_bind(value.clone())
|
||||
.push(format!(" or dimensions[{}]=", either_positions.1))
|
||||
.push_bind(value)
|
||||
.push(")");
|
||||
}
|
||||
push_collection_filter(&mut query, collection, filter);
|
||||
if let Some(keys) = selected_keys {
|
||||
query
|
||||
.push(" and dimensions[1]=any(")
|
||||
.push_bind(keys)
|
||||
.push(")");
|
||||
}
|
||||
if provider_counts {
|
||||
query.push(") matching group by dimensions[1]");
|
||||
if matches!(
|
||||
dimension,
|
||||
Some(
|
||||
DiagnosticDimension::Model
|
||||
| DiagnosticDimension::Status
|
||||
| DiagnosticDimension::Failure
|
||||
)
|
||||
) {
|
||||
query.push(format!(" and ({name})<>''"));
|
||||
}
|
||||
query.push(" limit 100001");
|
||||
query.push(")");
|
||||
aggregate::push_summary_aggregation(
|
||||
&mut query,
|
||||
read_distributions,
|
||||
read_costs,
|
||||
counters,
|
||||
distribution_names,
|
||||
);
|
||||
let mut rows = query.build().fetch(&mut *self.tx);
|
||||
let mut count = 0;
|
||||
while let Some(row) = rows
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read analytics summaries", error))?
|
||||
.map_err(|error| postgres_unavailable("merge analytics summaries", error))?
|
||||
{
|
||||
count += 1;
|
||||
if count > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics aggregate row budget exceeded"));
|
||||
}
|
||||
let dimensions: Vec<String> = get(&row, "dimensions")?;
|
||||
let name = dimension_name(collection, &dimensions, dimension);
|
||||
let Some(name) = name else {
|
||||
continue;
|
||||
};
|
||||
let bucket = output_bucket(
|
||||
get(&row, "bucket_start")?,
|
||||
range,
|
||||
granularity,
|
||||
boundaries.as_deref(),
|
||||
)?;
|
||||
let bucket = get(&row, "bucket_start")?;
|
||||
let name = get(&row, "name")?;
|
||||
let state = MetricState::decode(&row)?;
|
||||
result.entry((bucket, name)).or_default().merge(&state)?;
|
||||
}
|
||||
@@ -707,28 +740,42 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
if spans.is_empty() {
|
||||
return Ok(result);
|
||||
}
|
||||
// 单段直接绑定边界,让优化器看见真实跨度,避免把三十天误估成小范围 BitmapAnd
|
||||
// 单段直接绑定边界,多个互斥区间用索引读取,所有统计都在数据库内完成
|
||||
let single_span = spans.len() == 1;
|
||||
let mut query = if single_span {
|
||||
let mut query = QueryBuilder::<Postgres>::new(fact_select());
|
||||
query
|
||||
.push(" where mr.started_at >= ")
|
||||
.push_bind(spans[0].0)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(spans[0].1);
|
||||
query
|
||||
} else {
|
||||
let (starts, ends): (Vec<_>, Vec<_>) = spans.into_iter().unzip();
|
||||
let mut query = QueryBuilder::<Postgres>::new("select fact.* from unnest(");
|
||||
let mut query = QueryBuilder::<Postgres>::new("with matching as not materialized (");
|
||||
if !single_span {
|
||||
let (starts, ends): (Vec<_>, Vec<_>) = spans.iter().copied().unzip();
|
||||
query
|
||||
.push("select fact.* from unnest(")
|
||||
.push_bind(starts)
|
||||
.push("::timestamptz[],")
|
||||
.push_bind(ends)
|
||||
.push("::timestamptz[]) span(start_at,end_at) cross join lateral (")
|
||||
.push(fact_select())
|
||||
.push(" where mr.started_at >= span.start_at and mr.started_at < span.end_at");
|
||||
.push("::timestamptz[]) span(start_at,end_at) cross join lateral (");
|
||||
}
|
||||
query.push(format!(
|
||||
"select mr.*,coalesce(({}),false) as is_completed_usage,",
|
||||
completed_usage_fact_predicate("mr")
|
||||
));
|
||||
aggregate::push_output_bucket(
|
||||
&mut query,
|
||||
"mr.started_at",
|
||||
range,
|
||||
granularity,
|
||||
boundaries.as_deref(),
|
||||
);
|
||||
query.push(format!(
|
||||
" as output_bucket,{} as name from request_analytics_facts mr where ",
|
||||
aggregate::fact_dimension_sql(dimension)
|
||||
));
|
||||
if single_span {
|
||||
query
|
||||
};
|
||||
.push("mr.started_at >= ")
|
||||
.push_bind(spans[0].0)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(spans[0].1);
|
||||
} else {
|
||||
query.push("mr.started_at >= span.start_at and mr.started_at < span.end_at");
|
||||
}
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_analytics_filter(&mut query, filter, "mr");
|
||||
if let Some(dimension) = dimension {
|
||||
@@ -749,42 +796,40 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
}
|
||||
query.push(")");
|
||||
}
|
||||
query.push(" limit 100001");
|
||||
if !single_span {
|
||||
query.push(") fact limit 100001");
|
||||
// 保留逐区间的参数化索引扫描,避免展开 LATERAL 后全表读取再过滤
|
||||
// OFFSET 0 不限制返回行数
|
||||
query.push(" offset 0) fact");
|
||||
}
|
||||
query.push(")");
|
||||
let exact = selected.is_none();
|
||||
aggregate::push_fact_aggregation_selected(
|
||||
&mut query,
|
||||
exact,
|
||||
read_costs,
|
||||
read_distributions,
|
||||
counters,
|
||||
distribution_names,
|
||||
);
|
||||
let mut rows = query.build().fetch(&mut *self.tx);
|
||||
let mut count = 0;
|
||||
while let Some(row) = rows
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read bounded analytics facts", error))?
|
||||
.map_err(|error| postgres_unavailable("aggregate analytics facts", error))?
|
||||
{
|
||||
count += 1;
|
||||
if count > MAX_FACT_ROWS {
|
||||
return Err(unavailable("analytics fact row budget exceeded"));
|
||||
let key = (get(&row, "bucket_start")?, get(&row, "name")?);
|
||||
let state = MetricState::decode_aggregate(&row, exact)?;
|
||||
if exact {
|
||||
result.insert(key, state);
|
||||
} else {
|
||||
result.entry(key).or_default().merge(&state)?;
|
||||
}
|
||||
let bucket = output_bucket(
|
||||
get(&row, "started_at")?,
|
||||
range,
|
||||
granularity,
|
||||
boundaries.as_deref(),
|
||||
)?;
|
||||
let name = fact_dimension(&row, dimension)?;
|
||||
let state = result.entry((bucket, name)).or_insert_with(|| {
|
||||
if selected.is_some() {
|
||||
MetricState::default()
|
||||
} else {
|
||||
MetricState::exact()
|
||||
}
|
||||
});
|
||||
state.add_row(&row)?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
fn push_summary_buckets(
|
||||
pub(super) fn push_summary_buckets(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
buckets: &[DateTime<Utc>],
|
||||
seconds: i64,
|
||||
@@ -810,6 +855,33 @@ fn push_summary_buckets(
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_points(
|
||||
range: ObservabilityRange,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
states: &BTreeMap<(DateTime<Utc>, String), MetricState>,
|
||||
include_costs: bool,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
let mut points = BTreeMap::new();
|
||||
for ((bucket, _), state) in states {
|
||||
points.insert(
|
||||
*bucket,
|
||||
RequestMetricPoint {
|
||||
bucket_start: *bucket,
|
||||
granularity,
|
||||
metrics: state.request_metrics()?,
|
||||
cost_coverage: state.coverage(),
|
||||
costs: if include_costs {
|
||||
state.costs()?
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
fill_metric_gaps(range, granularity, points, timezone)
|
||||
}
|
||||
|
||||
fn merge_states(states: impl Iterator<Item = MetricState>) -> StoreResult<MetricState> {
|
||||
let mut result: Option<MetricState> = None;
|
||||
for state in states {
|
||||
@@ -822,28 +894,7 @@ fn merge_states(states: impl Iterator<Item = MetricState>) -> StoreResult<Metric
|
||||
Ok(result.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn output_bucket(
|
||||
time: DateTime<Utc>,
|
||||
range: ObservabilityRange,
|
||||
granularity: Option<ObservationGranularity>,
|
||||
boundaries: Option<&[DateTime<Utc>]>,
|
||||
) -> StoreResult<DateTime<Utc>> {
|
||||
match granularity {
|
||||
None => Ok(range.start),
|
||||
Some(ObservationGranularity::Day) => boundaries
|
||||
.and_then(|values| {
|
||||
values
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|boundary| **boundary <= time)
|
||||
.copied()
|
||||
})
|
||||
.ok_or_else(|| invalid("missing analytics calendar boundary")),
|
||||
Some(granularity) => bucket_for(time, granularity.seconds()),
|
||||
}
|
||||
}
|
||||
|
||||
fn collection_for(
|
||||
pub(super) fn collection_for(
|
||||
filter: &UsageRecordFilter,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> Option<&'static str> {
|
||||
@@ -852,7 +903,6 @@ fn collection_for(
|
||||
|| filter.response_id.is_some()
|
||||
|| filter.upstream_request_id.is_some()
|
||||
|| filter.search.is_some()
|
||||
|| !filter.routing_group_refs.is_empty()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -885,85 +935,18 @@ fn collection_for(
|
||||
DiagnosticDimension::Status => "status",
|
||||
});
|
||||
}
|
||||
if required.len() > 1 {
|
||||
if required.len() == 2 && required.contains("provider_model") && required.contains("account") {
|
||||
Some("account_model")
|
||||
} else if required.len() == 2 && required.contains("provider_model") && required.contains("key")
|
||||
{
|
||||
Some("key_model")
|
||||
} else if required.len() > 1 {
|
||||
None
|
||||
} else {
|
||||
Some(required.first().copied().unwrap_or("global"))
|
||||
}
|
||||
}
|
||||
|
||||
fn dimension_name(
|
||||
collection: &str,
|
||||
values: &[String],
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> Option<String> {
|
||||
let Some(dimension) = dimension else {
|
||||
return Some(String::new());
|
||||
};
|
||||
let value = |index| values.get(index).cloned().unwrap_or_default();
|
||||
let coalesce = |preferred: usize, fallback: usize| {
|
||||
let v = value(preferred);
|
||||
if v.is_empty() { value(fallback) } else { v }
|
||||
};
|
||||
let name = match dimension {
|
||||
DiagnosticDimension::Provider if collection == "provider_model" => {
|
||||
let v = value(0);
|
||||
if v.is_empty() {
|
||||
"unrouted".to_owned()
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
DiagnosticDimension::Model => coalesce(2, 1),
|
||||
DiagnosticDimension::Account => {
|
||||
let v = value(0);
|
||||
if v.is_empty() {
|
||||
"unrouted".to_owned()
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
DiagnosticDimension::Transport => {
|
||||
let v = coalesce(1, 0);
|
||||
if v.is_empty() {
|
||||
"unknown".to_owned()
|
||||
} else {
|
||||
v
|
||||
}
|
||||
}
|
||||
DiagnosticDimension::Status => coalesce(1, 0),
|
||||
_ => value(0),
|
||||
};
|
||||
if name.is_empty()
|
||||
&& matches!(
|
||||
dimension,
|
||||
DiagnosticDimension::Model | DiagnosticDimension::Status | DiagnosticDimension::Failure
|
||||
)
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(name)
|
||||
}
|
||||
}
|
||||
|
||||
fn fact_dimension(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
dimension: Option<DiagnosticDimension>,
|
||||
) -> StoreResult<String> {
|
||||
let Some(dimension) = dimension else {
|
||||
return Ok(String::new());
|
||||
};
|
||||
let collection = collection_for(&UsageRecordFilter::default(), Some(dimension))
|
||||
.ok_or_else(|| invalid("missing analytics dimension"))?;
|
||||
let dimensions = worker::collections(row)?
|
||||
.into_iter()
|
||||
.find(|(name, _)| *name == collection)
|
||||
.map(|(_, values)| values)
|
||||
.ok_or_else(|| invalid("missing analytics dimension"))?;
|
||||
dimension_name(collection, &dimensions, Some(dimension))
|
||||
.ok_or_else(|| invalid("unexpected empty analytics dimension"))
|
||||
}
|
||||
|
||||
pub(crate) fn push_analytics_filter(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
filter: &UsageRecordFilter,
|
||||
@@ -982,3 +965,53 @@ pub(crate) fn push_analytics_filter(
|
||||
query.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn push_collection_filter(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
collection: &str,
|
||||
filter: &UsageRecordFilter,
|
||||
) {
|
||||
let (dimension_filter, either_filter, either_positions) = match collection {
|
||||
"provider_model" => (filter.provider_kind.clone(), filter.model.clone(), (2, 3)),
|
||||
"account_model" => (
|
||||
filter.provider_account_ref.clone(),
|
||||
filter.model.clone(),
|
||||
(3, 4),
|
||||
),
|
||||
"key_model" => (
|
||||
filter.client_api_key_ref.clone(),
|
||||
filter.model.clone(),
|
||||
(3, 4),
|
||||
),
|
||||
"account" => (filter.provider_account_ref.clone(), None, (1, 2)),
|
||||
"key" => (filter.client_api_key_ref.clone(), None, (1, 2)),
|
||||
"operation" => (filter.operation.clone(), None, (1, 2)),
|
||||
"transport" => (None, filter.transport.clone(), (1, 2)),
|
||||
"status" => (
|
||||
None,
|
||||
filter.status_code.map(|value| value.to_string()),
|
||||
(1, 2),
|
||||
),
|
||||
_ => (None, None, (1, 2)),
|
||||
};
|
||||
// 可选筛选仅在有值时生成谓词,保留维度与时间索引的选择空间
|
||||
if let Some(outcome) = &filter.outcome {
|
||||
query.push(" and outcome=").push_bind(outcome);
|
||||
}
|
||||
if let Some(value) = dimension_filter {
|
||||
query.push(" and dimensions[1]=").push_bind(value);
|
||||
}
|
||||
if matches!(collection, "account_model" | "key_model")
|
||||
&& let Some(provider) = &filter.provider_kind
|
||||
{
|
||||
query.push(" and dimensions[2]=").push_bind(provider);
|
||||
}
|
||||
if let Some(value) = either_filter {
|
||||
query
|
||||
.push(format!(" and (dimensions[{}]=", either_positions.0))
|
||||
.push_bind(value.clone())
|
||||
.push(format!(" or dimensions[{}]=", either_positions.1))
|
||||
.push_bind(value)
|
||||
.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,11 +83,11 @@ impl PgRequestAnalyticsRepository {
|
||||
return Ok(AnalyticsMaintenance { reset_progress: true, ..Default::default() });
|
||||
}
|
||||
let mut result = AnalyticsMaintenance::default();
|
||||
let rows = sqlx::query("select request_id, change_version, oldest_pending_at, previous_bucket from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
|
||||
let rows = sqlx::query("select request_id, change_version from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
|
||||
.bind(i64::from(batch_size)).fetch_all(&mut *transaction).await.map_err(|error| postgres_unavailable("load analytics request queue", error))?;
|
||||
let mut jobs = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")?, pending_at: get(&row,"oldest_pending_at")?, previous_bucket: get(&row,"previous_bucket")? });
|
||||
jobs.push(ProjectionJob { id: get(&row,"request_id")?, version: get(&row,"change_version")? });
|
||||
}
|
||||
self.project_batch(&mut transaction, &jobs).await?;
|
||||
result.projected_requests = jobs.len() as u64;
|
||||
@@ -138,53 +138,36 @@ impl PgRequestAnalyticsRepository {
|
||||
}
|
||||
let ids = jobs.iter().map(|job| job.id.clone()).collect::<Vec<_>>();
|
||||
let versions = jobs.iter().map(|job| job.version).collect::<Vec<_>>();
|
||||
let pending = jobs
|
||||
.iter()
|
||||
.map(|job| (job.id.as_str(), job.pending_at))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut buckets = BTreeMap::<DateTime<Utc>, DateTime<Utc>>::new();
|
||||
let mut affect = |id: &str, start: DateTime<Utc>| -> StoreResult<()> {
|
||||
let pending_at = pending
|
||||
.get(id)
|
||||
.copied()
|
||||
.ok_or_else(|| invalid("analytics job disappeared"))?;
|
||||
let bucket = bucket_for(start, 900)?;
|
||||
buckets
|
||||
.entry(bucket)
|
||||
.and_modify(|old| *old = (*old).min(pending_at))
|
||||
.or_insert(pending_at);
|
||||
Ok(())
|
||||
};
|
||||
let rows = sqlx::query(
|
||||
"delete from request_analytics_facts where id=any($1::text[]) returning id,started_at",
|
||||
)
|
||||
let previous = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"{} where mr.id=any($1::text[])",
|
||||
fact_select()
|
||||
)))
|
||||
.bind(&ids)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("replace analytics facts", error))?;
|
||||
for row in rows {
|
||||
affect(&get::<String>(&row, "id")?, get(&row, "started_at")?)?;
|
||||
}
|
||||
let rows = sqlx::query(PROJECT_REQUEST)
|
||||
.map_err(|error| postgres_unavailable("read previous analytics contributions", error))?;
|
||||
sqlx::query("delete from request_analytics_facts where id=any($1::text[])")
|
||||
.bind(&ids)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("replace analytics facts", error))?;
|
||||
sqlx::query(PROJECT_REQUEST)
|
||||
.bind(&ids)
|
||||
.bind(versions)
|
||||
.fetch_all(&mut **tx)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("project analytics facts", error))?;
|
||||
for row in rows {
|
||||
affect(&get::<String>(&row, "id")?, get(&row, "started_at")?)?;
|
||||
}
|
||||
for job in jobs {
|
||||
if let Some(bucket) = job.previous_bucket {
|
||||
affect(&job.id, bucket)?;
|
||||
}
|
||||
}
|
||||
self.project_billing_batch(tx, &ids).await?;
|
||||
if !buckets.is_empty() {
|
||||
let (starts, times): (Vec<_>, Vec<_>) = buckets.into_iter().unzip();
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select * from unnest($1::timestamptz[],$2::timestamptz[]) on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version'),oldest_pending_at=least(request_analytics_pending_buckets.oldest_pending_at,excluded.oldest_pending_at)")
|
||||
.bind(starts).bind(times).execute(&mut **tx).await.map_err(|error| postgres_unavailable("enqueue analytics buckets",error))?;
|
||||
}
|
||||
let current = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"{} where mr.id=any($1::text[])",
|
||||
fact_select()
|
||||
)))
|
||||
.bind(&ids)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("read current analytics contributions", error))?;
|
||||
// 事实、已发布汇总与构建前缀在同一事务中撤销和添加,重试不会重复计数
|
||||
super::incremental::update_summaries(tx, &previous, ¤t).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -199,7 +182,7 @@ impl PgRequestAnalyticsRepository {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select id,started_at,nextval('request_analytics_change_version') as version from model_requests",
|
||||
"select id,started_at,nextval('request_analytics_change_version') as version,exists(select 1 from request_analytics_facts f where f.id=model_requests.id) as projected from model_requests",
|
||||
);
|
||||
// 首批与后续批次使用两种明确的计划,避免可空游标的 OR 使通用计划退化为全索引过滤
|
||||
if let Some(start) = get::<Option<DateTime<Utc>>>(&state, "backfill_cursor_started_at")? {
|
||||
@@ -222,11 +205,12 @@ impl PgRequestAnalyticsRepository {
|
||||
.map_err(|error| postgres_unavailable("read analytics backfill batch", error))?;
|
||||
let mut jobs = Vec::with_capacity(rows.len());
|
||||
for row in &rows {
|
||||
if get::<bool>(row, "projected")? {
|
||||
continue;
|
||||
}
|
||||
jobs.push(ProjectionJob {
|
||||
id: get(row, "id")?,
|
||||
version: get(row, "version")?,
|
||||
pending_at: Utc::now(),
|
||||
previous_bucket: None,
|
||||
});
|
||||
}
|
||||
self.project_batch(tx, &jobs).await?;
|
||||
@@ -312,49 +296,68 @@ async fn read_status(
|
||||
})
|
||||
}
|
||||
|
||||
// 投影与极值重算共用同一维度顺序,避免两条路径对集合身份作不同解释
|
||||
pub(super) const COLLECTION_COLUMNS: &[(&str, &[&str])] = &[
|
||||
("global", &[]),
|
||||
(
|
||||
"provider_model",
|
||||
&["provider_kind", "requested_model_id", "upstream_model_id"],
|
||||
),
|
||||
("account", &["provider_account_ref"]),
|
||||
("key", &["client_api_key_ref"]),
|
||||
(
|
||||
"account_model",
|
||||
&[
|
||||
"provider_account_ref",
|
||||
"provider_kind",
|
||||
"requested_model_id",
|
||||
"upstream_model_id",
|
||||
],
|
||||
),
|
||||
(
|
||||
"key_model",
|
||||
&[
|
||||
"client_api_key_ref",
|
||||
"provider_kind",
|
||||
"requested_model_id",
|
||||
"upstream_model_id",
|
||||
],
|
||||
),
|
||||
("operation", &["operation"]),
|
||||
("transport", &["client_transport", "upstream_transport"]),
|
||||
("status", &["client_status_code", "upstream_status_code"]),
|
||||
("failure", &["error_kind"]),
|
||||
];
|
||||
|
||||
pub(super) fn collections(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> StoreResult<Vec<(&'static str, Vec<String>)>> {
|
||||
let text =
|
||||
|name| -> StoreResult<String> { Ok(get::<Option<String>>(row, name)?.unwrap_or_default()) };
|
||||
let status = |name| -> StoreResult<String> {
|
||||
Ok(get::<Option<i32>>(row, name)?
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_default())
|
||||
};
|
||||
Ok(vec![
|
||||
("global", vec![]),
|
||||
(
|
||||
"provider_model",
|
||||
vec![
|
||||
text("provider_kind")?,
|
||||
text("requested_model_id")?,
|
||||
text("upstream_model_id")?,
|
||||
],
|
||||
),
|
||||
("account", vec![text("provider_account_ref")?]),
|
||||
("key", vec![text("client_api_key_ref")?]),
|
||||
("operation", vec![text("operation")?]),
|
||||
(
|
||||
"transport",
|
||||
vec![text("client_transport")?, text("upstream_transport")?],
|
||||
),
|
||||
(
|
||||
"status",
|
||||
vec![
|
||||
status("client_status_code")?,
|
||||
status("upstream_status_code")?,
|
||||
],
|
||||
),
|
||||
("failure", vec![text("error_kind")?]),
|
||||
])
|
||||
let mut result = vec![(forecast::COLLECTION, forecast::dimensions(row)?)];
|
||||
// 预测保留原始完成状态,包括被恢复的请求;其他统计继续排除恢复记录
|
||||
if get::<Option<DateTime<Utc>>>(row, "recovered_at")?.is_some() {
|
||||
return Ok(result);
|
||||
}
|
||||
for &(collection, columns) in COLLECTION_COLUMNS {
|
||||
let dimensions = columns
|
||||
.iter()
|
||||
.map(|column| {
|
||||
if matches!(*column, "client_status_code" | "upstream_status_code") {
|
||||
Ok(get::<Option<i32>>(row, column)?
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_default())
|
||||
} else {
|
||||
Ok(get::<Option<String>>(row, column)?.unwrap_or_default())
|
||||
}
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
result.push((collection, dimensions));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
struct ProjectionJob {
|
||||
id: String,
|
||||
version: i64,
|
||||
pending_at: DateTime<Utc>,
|
||||
previous_bucket: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
async fn publish_generation(
|
||||
|
||||
@@ -17,7 +17,6 @@ pub(crate) fn store_range(
|
||||
|
||||
pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter {
|
||||
UsageRecordFilter {
|
||||
routing_group_refs: filter.routing_group_refs,
|
||||
client_api_key_ref: filter.client_api_key_ref,
|
||||
request_id: filter.request_id,
|
||||
provider_account_ref: filter.provider_account_ref,
|
||||
|
||||
@@ -31,6 +31,10 @@ mod queries;
|
||||
mod query_budget;
|
||||
|
||||
pub use admin_adapter::*;
|
||||
pub(crate) use analytics::AnalyticsSnapshot;
|
||||
pub(crate) use analytics::accounts::{
|
||||
account_group_usage, account_usage_sort_values, account_usage_windows,
|
||||
};
|
||||
pub use analytics::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
|
||||
pub(crate) use mapping::*;
|
||||
pub use model::*;
|
||||
|
||||
@@ -39,7 +39,6 @@ pub(crate) fn observability_page_offset(
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct UsageRecordFilter {
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
@@ -76,7 +75,6 @@ impl UsageRecordFilter {
|
||||
] {
|
||||
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
|
||||
}
|
||||
validate_account_ids_if_present(&self.routing_group_refs)?;
|
||||
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
|
||||
if self
|
||||
.status_code
|
||||
@@ -105,14 +103,6 @@ pub struct UsageRecordQuery {
|
||||
pub use gateway_admin::model::observability::OpsErrorFilter;
|
||||
|
||||
pub(crate) fn validate_ops_error_filter(filter: &OpsErrorFilter) -> StoreResult<()> {
|
||||
if filter.routing_group_refs.len() > 32
|
||||
|| filter.routing_group_refs.iter().any(|value| {
|
||||
value.is_empty() || value.len() > 128 || value.chars().any(char::is_control)
|
||||
})
|
||||
{
|
||||
return Err(invalid("invalid routing group filter"));
|
||||
}
|
||||
|
||||
for (value, field) in [
|
||||
(
|
||||
filter.client_api_key_ref.as_deref(),
|
||||
@@ -384,10 +374,3 @@ pub struct DiagnosticObservation {
|
||||
pub use gateway_admin::model::observability::OpsError as OpsErrorRecord;
|
||||
|
||||
pub use gateway_admin::model::observability::OpsErrorPage;
|
||||
|
||||
fn validate_account_ids_if_present(ids: &[String]) -> StoreResult<()> {
|
||||
if ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
validate_account_ids(ids)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) async fn provider_account_usage(
|
||||
connection: &mut sqlx::PgConnection,
|
||||
snapshot: &mut analytics::AnalyticsSnapshot<'_>,
|
||||
query: ProviderAccountUsageQuery,
|
||||
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
@@ -13,290 +13,133 @@ pub(crate) async fn provider_account_usage(
|
||||
return Err(invalid("account usage limit must be between 1 and 200"));
|
||||
}
|
||||
|
||||
// 账号卡片先确定有限候选,避免为只展示二十个账号聚合整个历史范围
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"with candidate_accounts as materialized (select pa.* from provider_accounts pa",
|
||||
);
|
||||
// 账号卡片先确定有限候选,避免为少量账号聚合整个历史范围
|
||||
let mut statement = QueryBuilder::<Postgres>::new("select pa.* from provider_accounts pa");
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
statement
|
||||
.push(" where pa.id=any(")
|
||||
.push_bind(account_ids.clone())
|
||||
.push("::text[])");
|
||||
} else {
|
||||
statement.push(" join lateral (select mr.started_at from request_analytics_facts mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start)
|
||||
statement.push(" join lateral (select mr.started_at from model_requests mr where mr.provider_account_ref=pa.id and mr.started_at >= ").push_bind(query.range.start)
|
||||
.push(" and mr.started_at < ").push_bind(query.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
statement.push(" order by mr.started_at desc,mr.id desc limit 1) latest on true order by latest.started_at desc,pa.name,pa.id limit ").push_bind(i64::from(query.limit));
|
||||
}
|
||||
statement.push(
|
||||
"), matched as materialized (
|
||||
select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
|
||||
pa.plan_type, mr.id as request_id,
|
||||
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
|
||||
mr.cost_currency, mr.outcome, mr.input_tokens, mr.output_tokens,
|
||||
mr.cached_input_tokens, mr.cache_write_input_tokens, mr.reasoning_output_tokens,
|
||||
mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.image_generation_succeeded, mr.total_tokens, mr.cost_source,
|
||||
mr.cost_amount, mr.started_at
|
||||
from candidate_accounts pa
|
||||
left join request_analytics_facts mr
|
||||
on mr.provider_account_ref = pa.id and mr.started_at >= ",
|
||||
);
|
||||
statement.push_bind(query.range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(query.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
statement.push(" where pa.id = any(");
|
||||
statement.push_bind(account_ids.clone());
|
||||
statement.push("::text[])");
|
||||
}
|
||||
statement.push(
|
||||
" limit 100001), aggregated as (
|
||||
select id, provider_kind, authentication_kind, name, email, plan_type,
|
||||
model, cost_currency,
|
||||
grouping(model)::integer as model_grouping,
|
||||
grouping(cost_currency)::integer as currency_grouping,
|
||||
count(request_id)::bigint as request_count,
|
||||
count(request_id) filter (where outcome = 'succeeded')::bigint
|
||||
as success_count,
|
||||
sum(input_tokens)::bigint as input_tokens,
|
||||
sum(output_tokens)::bigint as output_tokens,
|
||||
sum(cached_input_tokens)::bigint as cached_input_tokens,
|
||||
sum(cache_write_input_tokens)::bigint as cache_write_input_tokens,
|
||||
sum(reasoning_output_tokens)::bigint as reasoning_output_tokens,
|
||||
sum(image_input_tokens)::bigint as image_input_tokens,
|
||||
sum(image_output_tokens)::bigint as image_output_tokens,
|
||||
count(request_id) filter (where image_generation_succeeded is true)::bigint
|
||||
as image_request_count,
|
||||
count(request_id) filter (where image_generation_succeeded is false)::bigint
|
||||
as image_request_failed_count,
|
||||
sum(total_tokens)::bigint as total_tokens,
|
||||
count(request_id) filter (where cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(request_id) filter (where cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(request_id) filter (where cost_source = 'unavailable')::bigint
|
||||
as unavailable_count,
|
||||
max(started_at) as last_used_at,
|
||||
sum(cost_amount)::text as amount
|
||||
from matched
|
||||
group by id, provider_kind, authentication_kind, name, email, plan_type,
|
||||
grouping sets ((), (cost_currency), (model), (model, cost_currency))
|
||||
), selected_accounts as (
|
||||
select id,
|
||||
row_number() over (order by last_used_at desc nulls last, name, id)
|
||||
as sort_position
|
||||
from aggregated
|
||||
where model_grouping = 1 and currency_grouping = 1",
|
||||
);
|
||||
if query.account_ids.is_none() {
|
||||
statement.push(" and request_count > 0");
|
||||
}
|
||||
statement.push(" order by last_used_at desc nulls last, name, id limit ");
|
||||
statement.push_bind(i64::from(query.limit));
|
||||
statement.push(
|
||||
")
|
||||
select aggregated.*, (select count(*) from matched)::bigint as matched_count
|
||||
from aggregated
|
||||
join selected_accounts selected using (id)
|
||||
order by selected.sort_position, model_grouping desc, currency_grouping desc,
|
||||
model nulls last, cost_currency nulls last",
|
||||
);
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(&mut *connection)
|
||||
.fetch_all(&mut *snapshot.tx)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load provider account usage", source))?;
|
||||
if let Some(row) = rows.first()
|
||||
&& unsigned(row, "matched_count")? > 100_000
|
||||
.map_err(|error| postgres_unavailable("select usage accounts", error))?;
|
||||
let account_ids = rows
|
||||
.iter()
|
||||
.map(|row| get(row, "id"))
|
||||
.collect::<StoreResult<Vec<String>>>()?;
|
||||
let mut models_by_account = HashMap::<String, Vec<ProviderAccountModelUsageObservation>>::new();
|
||||
let mut totals = BTreeMap::<String, analytics::MetricState>::new();
|
||||
for ((account, model), state) in snapshot
|
||||
.account_usage_states(query.range, &account_ids)
|
||||
.await?
|
||||
{
|
||||
return Err(crate::StoreError::Unavailable {
|
||||
backend: crate::StoreBackend::PostgreSql,
|
||||
message: "account analytics fact row budget exceeded".to_owned(),
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
|
||||
let mut observations = Vec::with_capacity(usize::from(query.limit));
|
||||
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
|
||||
let mut models = HashMap::<String, Vec<ProviderAccountModelUsageObservation>>::new();
|
||||
let mut model_costs = HashMap::<(String, String), Vec<CurrencyCostTotal>>::new();
|
||||
for row in &rows {
|
||||
let model_grouping: i32 = get(row, "model_grouping")?;
|
||||
let currency_grouping: i32 = get(row, "currency_grouping")?;
|
||||
match (model_grouping, currency_grouping) {
|
||||
(1, 1) => observations.push(provider_account_from_row(row)?),
|
||||
(1, 0) if get::<Option<String>>(row, "cost_currency")?.is_some() => {
|
||||
costs
|
||||
.entry(get(row, "id")?)
|
||||
.or_default()
|
||||
.push(cost_from_row(row)?);
|
||||
}
|
||||
(0, 1) if get::<Option<String>>(row, "model")?.is_some() => {
|
||||
models
|
||||
.entry(get(row, "id")?)
|
||||
.or_default()
|
||||
.push(provider_account_model_from_row(row)?);
|
||||
}
|
||||
(0, 0)
|
||||
if get::<Option<String>>(row, "model")?.is_some()
|
||||
&& get::<Option<String>>(row, "cost_currency")?.is_some() =>
|
||||
{
|
||||
model_costs
|
||||
.entry((get(row, "id")?, get(row, "model")?))
|
||||
.or_default()
|
||||
.push(cost_from_row(row)?);
|
||||
}
|
||||
(0 | 1, 0 | 1) => {}
|
||||
_ => {
|
||||
return Err(crate::StoreError::Unavailable {
|
||||
backend: crate::StoreBackend::PostgreSql,
|
||||
message: "decode provider account usage grouping".to_owned(),
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
totals.entry(account.clone()).or_default().merge(&state)?;
|
||||
if !model.is_empty()
|
||||
&& let Some(last_used_at) = state.last_used_at()
|
||||
{
|
||||
models_by_account.entry(account).or_default().push(
|
||||
ProviderAccountModelUsageObservation {
|
||||
model,
|
||||
request_count: state.count("completed_request_count"),
|
||||
success_count: state.count("completed_request_count"),
|
||||
input_tokens: state.optional_count("input_tokens"),
|
||||
output_tokens: state.optional_count("output_tokens"),
|
||||
cached_input_tokens: state.optional_count("cached_input_tokens"),
|
||||
cache_write_input_tokens: state.optional_count("cache_write_input_tokens"),
|
||||
reasoning_output_tokens: state.optional_count("reasoning_output_tokens"),
|
||||
image_input_tokens: state.optional_count("image_input_tokens"),
|
||||
image_output_tokens: state.optional_count("image_output_tokens"),
|
||||
total_tokens: state.optional_count("total_tokens"),
|
||||
image_request_count: state.count("image_request_count"),
|
||||
image_request_failed_count: state.count("image_request_failed_count"),
|
||||
cost_coverage: state.coverage(),
|
||||
costs: state.costs()?,
|
||||
last_used_at,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
if observations.is_empty() {
|
||||
return Ok(observations);
|
||||
}
|
||||
let account_ids = observations
|
||||
.iter()
|
||||
.map(|item| item.account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let mut request_buckets = if let Some(range) = query.request_bucket_range {
|
||||
provider_account_request_buckets(connection, range, &account_ids).await?
|
||||
provider_account_request_buckets(snapshot, range, &account_ids).await?
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
for observation in &mut observations {
|
||||
observation.costs = costs.remove(&observation.account_id).unwrap_or_default();
|
||||
observation.request_buckets = request_buckets
|
||||
.remove(&observation.account_id)
|
||||
.unwrap_or_default();
|
||||
observation.models = models.remove(&observation.account_id).unwrap_or_default();
|
||||
for model in &mut observation.models {
|
||||
model.costs = model_costs
|
||||
.remove(&(observation.account_id.clone(), model.model.clone()))
|
||||
.unwrap_or_default();
|
||||
}
|
||||
observation.models.sort_by(|left, right| {
|
||||
right
|
||||
.request_count
|
||||
.cmp(&left.request_count)
|
||||
.then_with(|| left.model.cmp(&right.model))
|
||||
let mut observations = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let account_id: String = get(&row, "id")?;
|
||||
let state = totals.remove(&account_id).unwrap_or_default();
|
||||
let mut models = models_by_account.remove(&account_id).unwrap_or_default();
|
||||
models.sort_by(|a, b| {
|
||||
b.request_count
|
||||
.cmp(&a.request_count)
|
||||
.then(a.model.cmp(&b.model))
|
||||
});
|
||||
observations.push(ProviderAccountUsageObservation {
|
||||
provider_kind: get(&row, "provider_kind")?,
|
||||
authentication_kind: get(&row, "authentication_kind")?,
|
||||
name: get(&row, "name")?,
|
||||
email: get(&row, "email")?,
|
||||
plan_type: get(&row, "plan_type")?,
|
||||
request_count: state.count("completed_request_count"),
|
||||
success_count: state.count("completed_request_count"),
|
||||
input_tokens: state.optional_count("input_tokens"),
|
||||
output_tokens: state.optional_count("output_tokens"),
|
||||
cached_input_tokens: state.optional_count("cached_input_tokens"),
|
||||
cache_write_input_tokens: state.optional_count("cache_write_input_tokens"),
|
||||
reasoning_output_tokens: state.optional_count("reasoning_output_tokens"),
|
||||
image_input_tokens: state.optional_count("image_input_tokens"),
|
||||
image_output_tokens: state.optional_count("image_output_tokens"),
|
||||
total_tokens: state.optional_count("total_tokens"),
|
||||
image_request_count: state.count("image_request_count"),
|
||||
image_request_failed_count: state.count("image_request_failed_count"),
|
||||
cost_coverage: state.coverage(),
|
||||
costs: state.costs()?,
|
||||
last_used_at: state.last_used_at(),
|
||||
request_buckets: request_buckets.remove(&account_id).unwrap_or_default(),
|
||||
models,
|
||||
account_id,
|
||||
});
|
||||
}
|
||||
observations.sort_by(|a, b| {
|
||||
b.last_used_at
|
||||
.cmp(&a.last_used_at)
|
||||
.then(a.name.cmp(&b.name))
|
||||
.then(a.account_id.cmp(&b.account_id))
|
||||
});
|
||||
observations.truncate(usize::from(query.limit));
|
||||
Ok(observations)
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_account_request_buckets(
|
||||
connection: &mut sqlx::PgConnection,
|
||||
snapshot: &mut analytics::AnalyticsSnapshot<'_>,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, Vec<ProviderAccountRequestBucket>>> {
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
let statement = format!(
|
||||
"select provider_account_ref,
|
||||
floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index,
|
||||
count(*)::bigint as request_count
|
||||
from request_analytics_facts mr
|
||||
where mr.provider_account_ref = any($2::text[])
|
||||
and mr.started_at >= $1 and mr.started_at < $3
|
||||
and {completed_usage}
|
||||
group by mr.provider_account_ref, bucket_index
|
||||
order by mr.provider_account_ref, bucket_index"
|
||||
);
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(range.start)
|
||||
.bind(account_ids)
|
||||
.bind(range.end)
|
||||
.fetch_all(&mut *connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load provider account request timeline", source))?;
|
||||
|
||||
let mut observed = HashMap::<String, BTreeMap<u64, u64>>::new();
|
||||
for row in rows {
|
||||
observed
|
||||
.entry(get(&row, "provider_account_ref")?)
|
||||
.or_default()
|
||||
.insert(
|
||||
unsigned(&row, "bucket_index")?,
|
||||
unsigned(&row, "request_count")?,
|
||||
);
|
||||
}
|
||||
|
||||
let mut observed = snapshot.account_request_counts(range, account_ids).await?;
|
||||
let mut timelines = HashMap::with_capacity(account_ids.len());
|
||||
for account_id in account_ids {
|
||||
let mut account_observed = observed.remove(account_id).unwrap_or_default();
|
||||
let mut buckets = Vec::new();
|
||||
for bucket_index in 0..ACCOUNT_USAGE_TIMELINE_HOURS {
|
||||
let bucket_start = range
|
||||
for account in account_ids {
|
||||
let mut buckets = Vec::with_capacity(ACCOUNT_USAGE_TIMELINE_HOURS as usize);
|
||||
for index in 0..ACCOUNT_USAGE_TIMELINE_HOURS {
|
||||
let start = range
|
||||
.start
|
||||
.checked_add_signed(TimeDelta::hours(bucket_index))
|
||||
.checked_add_signed(TimeDelta::hours(index))
|
||||
.ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?;
|
||||
buckets.push(ProviderAccountRequestBucket {
|
||||
bucket_start,
|
||||
request_count: account_observed
|
||||
.remove(&(bucket_index as u64))
|
||||
bucket_start: start,
|
||||
request_count: observed
|
||||
.remove(&(account.clone(), index))
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
timelines.insert(account_id.clone(), buckets);
|
||||
timelines.insert(account.clone(), buckets);
|
||||
}
|
||||
Ok(timelines)
|
||||
}
|
||||
|
||||
fn provider_account_model_from_row(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> StoreResult<ProviderAccountModelUsageObservation> {
|
||||
Ok(ProviderAccountModelUsageObservation {
|
||||
model: get(row, "model")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
image_request_count: unsigned(row, "image_request_count")?,
|
||||
image_request_failed_count: unsigned(row, "image_request_failed_count")?,
|
||||
total_tokens: optional_unsigned(row, "total_tokens")?,
|
||||
cost_coverage: coverage_from_row(row)?,
|
||||
costs: Vec::new(),
|
||||
last_used_at: get(row, "last_used_at")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn provider_account_from_row(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> StoreResult<ProviderAccountUsageObservation> {
|
||||
Ok(ProviderAccountUsageObservation {
|
||||
account_id: get(row, "id")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
authentication_kind: get(row, "authentication_kind")?,
|
||||
name: get(row, "name")?,
|
||||
email: get(row, "email")?,
|
||||
plan_type: get(row, "plan_type")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_input_tokens: optional_unsigned(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_unsigned(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_unsigned(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
image_request_count: unsigned(row, "image_request_count")?,
|
||||
image_request_failed_count: unsigned(row, "image_request_failed_count")?,
|
||||
total_tokens: optional_unsigned(row, "total_tokens")?,
|
||||
cost_coverage: coverage_from_row(row)?,
|
||||
costs: Vec::new(),
|
||||
last_used_at: get(row, "last_used_at")?,
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,15 +1,7 @@
|
||||
//! 时间桶补齐、日历边界与共享费用覆盖映射
|
||||
//! 时间桶补齐与部署时区的日历边界
|
||||
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CostCoverage> {
|
||||
Ok(CostCoverage {
|
||||
provider_reported_count: unsigned(row, "provider_reported_count")?,
|
||||
calculated_count: unsigned(row, "calculated_count")?,
|
||||
unavailable_count: unsigned(row, "unavailable_count")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn fill_metric_gaps(
|
||||
range: ObservabilityRange,
|
||||
granularity: ObservationGranularity,
|
||||
|
||||
@@ -69,38 +69,29 @@ left join model_request_observations mr on mr.id = oe.model_request_id
|
||||
where true";
|
||||
|
||||
pub(crate) async fn list_ops_errors(
|
||||
connection: &mut PgConnection,
|
||||
connection: &mut ObservabilityConnection,
|
||||
query: OpsErrorQuery,
|
||||
) -> StoreResult<OpsErrorPage> {
|
||||
validate_ops_error_filter(&query.filter)?;
|
||||
let mut transaction = analytics::begin_snapshot(connection).await?;
|
||||
let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?;
|
||||
let offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut statement =
|
||||
QueryBuilder::<Postgres>::new("with candidate_page as materialized (select * from (");
|
||||
statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true");
|
||||
push_request_error_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(" union all ");
|
||||
statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true");
|
||||
push_ops_event_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(") e order by occurred_at desc, stable_sort_id desc limit ");
|
||||
statement.push_bind(i64::from(query.page_size.get()));
|
||||
statement.push(" offset ");
|
||||
statement.push_bind(offset);
|
||||
statement.push(") select details.* from candidate_page page cross join lateral (");
|
||||
statement.push(REQUEST_ERROR_SELECT);
|
||||
statement.push(" and page.source='model_request' and mr.id=page.event_id union all ");
|
||||
statement.push(OPS_EVENT_SELECT);
|
||||
statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc");
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("list ops errors", source))?;
|
||||
let items = rows
|
||||
.iter()
|
||||
.map(ops_error_from_row)
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
let (total, page) = if query.current_page == 1 {
|
||||
let total = count_ops_errors(&mut transaction, query.range, &query.filter).await?;
|
||||
(
|
||||
total,
|
||||
(total > 0).then_some(TimePage {
|
||||
range: query.range,
|
||||
offset,
|
||||
total: Some(total),
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
ops_time_page(&mut transaction, &query, offset).await?
|
||||
};
|
||||
let items = match page {
|
||||
Some(page) => list_ops_error_items_at(&mut transaction, &query, page).await?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
@@ -113,6 +104,95 @@ pub(crate) async fn list_ops_errors(
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_ops_error_items_at(
|
||||
connection: &mut PgConnection,
|
||||
query: &OpsErrorQuery,
|
||||
page: TimePage,
|
||||
) -> StoreResult<Vec<OpsErrorRecord>> {
|
||||
let total =
|
||||
i64::try_from(page.total.unwrap_or(0)).map_err(|_| invalid("Ops count overflows i64"))?;
|
||||
let limit = i64::from(query.page_size.get()).min(total - page.offset);
|
||||
let reverse_offset = total - page.offset - limit;
|
||||
let ascending = reverse_offset < page.offset;
|
||||
let offset = if ascending {
|
||||
reverse_offset
|
||||
} else {
|
||||
page.offset
|
||||
};
|
||||
let branch_limit = offset + limit;
|
||||
let order = if ascending { " asc" } else { " desc" };
|
||||
let mut statement =
|
||||
QueryBuilder::<Postgres>::new("with candidate_page as materialized (select * from ((");
|
||||
statement.push("select 'model_request'::text as source, mr.id as event_id, mr.completed_at as occurred_at, 'model_request:' || mr.id as stable_sort_id from model_request_observations mr where true");
|
||||
push_request_error_predicates(&mut statement, page.range, &query.filter);
|
||||
statement
|
||||
.push(" order by mr.completed_at")
|
||||
.push(order)
|
||||
.push(", mr.id")
|
||||
.push(order);
|
||||
statement.push(" limit ").push_bind(branch_limit);
|
||||
statement.push(") union all (");
|
||||
statement.push("select 'ops_event'::text as source, oe.id as event_id, oe.created_at as occurred_at, 'ops_event:' || oe.id as stable_sort_id from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true");
|
||||
push_ops_event_predicates(&mut statement, page.range, &query.filter);
|
||||
statement
|
||||
.push(" order by oe.created_at")
|
||||
.push(order)
|
||||
.push(", oe.id")
|
||||
.push(order);
|
||||
statement.push(" limit ").push_bind(branch_limit);
|
||||
statement
|
||||
.push(")) e order by occurred_at")
|
||||
.push(order)
|
||||
.push(", stable_sort_id")
|
||||
.push(order);
|
||||
statement
|
||||
.push(" limit ")
|
||||
.push_bind(limit)
|
||||
.push(" offset ")
|
||||
.push_bind(offset);
|
||||
statement.push(") select details.* from candidate_page page cross join lateral (");
|
||||
statement.push(REQUEST_ERROR_SELECT);
|
||||
statement.push(" and page.source='model_request' and mr.id=page.event_id union all ");
|
||||
statement.push(OPS_EVENT_SELECT);
|
||||
statement.push(" and page.source='ops_event' and oe.id=page.event_id) details order by page.occurred_at desc,page.stable_sort_id desc");
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("list ops errors", source))?;
|
||||
rows.iter().map(ops_error_from_row).collect()
|
||||
}
|
||||
|
||||
async fn ops_time_page(
|
||||
connection: &mut PgConnection,
|
||||
query: &OpsErrorQuery,
|
||||
offset: i64,
|
||||
) -> StoreResult<(u64, Option<TimePage>)> {
|
||||
// 两个来源先各自精确分段,合并分钟计数后再定位同一个全局页码
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"select bucket_start, sum(request_count)::bigint as request_count from (
|
||||
select date_bin(interval '1 minute', mr.completed_at, timestamptz 'epoch') as bucket_start,
|
||||
count(*)::bigint as request_count
|
||||
from model_request_observations mr where true",
|
||||
);
|
||||
push_request_error_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(" group by bucket_start union all ");
|
||||
statement.push(
|
||||
"select date_bin(interval '1 minute', oe.created_at, timestamptz 'epoch') as bucket_start,
|
||||
count(*)::bigint as request_count
|
||||
from ops_events oe left join model_request_observations mr on mr.id=oe.model_request_id where true",
|
||||
);
|
||||
push_ops_event_predicates(&mut statement, query.range, &query.filter);
|
||||
statement
|
||||
.push(" group by bucket_start) counts group by bucket_start order by bucket_start desc");
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("locate Ops page", source))?;
|
||||
time_page_from_counts(&rows, query.range, offset, query.page_size)
|
||||
}
|
||||
|
||||
pub(crate) async fn count_ops_errors(
|
||||
connection: &mut sqlx::PgConnection,
|
||||
range: ObservabilityRange,
|
||||
@@ -144,7 +224,6 @@ fn push_request_error_predicates(
|
||||
// 列表和总数共用此条件,避免流式响应中的错误因 outcome 被漏掉
|
||||
statement.push(" and mr.error_kind is not null and mr.error_kind <> 'cancelled'");
|
||||
push_range(statement, "mr.completed_at", range);
|
||||
push_routing_groups(statement, &filter.routing_group_refs);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
("mr.id", &filter.request_id),
|
||||
@@ -187,7 +266,6 @@ fn push_ops_event_predicates(
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
push_range(statement, "oe.created_at", range);
|
||||
push_routing_groups(statement, &filter.routing_group_refs);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
("oe.model_request_id", &filter.request_id),
|
||||
@@ -275,11 +353,3 @@ fn push_prefix_search(statement: &mut QueryBuilder<Postgres>, columns: &[&str],
|
||||
.push(" escape '\\'");
|
||||
statement.push(")");
|
||||
}
|
||||
|
||||
fn push_routing_groups(statement: &mut QueryBuilder<Postgres>, groups: &[String]) {
|
||||
if !groups.is_empty() {
|
||||
statement.push(" and mr.routing_group_refs && ");
|
||||
statement.push_bind(groups.to_vec());
|
||||
statement.push("::text[]");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,12 +11,6 @@ pub(crate) fn push_usage_filter(
|
||||
filter: &UsageRecordFilter,
|
||||
alias: &str,
|
||||
) {
|
||||
if !filter.routing_group_refs.is_empty() {
|
||||
query
|
||||
.push(format!(" and {alias}.routing_group_refs && "))
|
||||
.push_bind(filter.routing_group_refs.clone())
|
||||
.push("::text[]");
|
||||
}
|
||||
if let Some(value) = &filter.client_api_key_ref {
|
||||
query.push(format!(" and {alias}.client_api_key_ref = "));
|
||||
query.push_bind(value.clone());
|
||||
@@ -86,16 +80,14 @@ pub(crate) fn push_usage_filter(
|
||||
}
|
||||
if let Some(value) = &filter.search {
|
||||
let pattern = literal_prefix_pattern(value);
|
||||
query.push(format!(" and ({alias}.id like "));
|
||||
query.push(format!(" and ({alias}.requested_model_id like "));
|
||||
query.push_bind(pattern.clone());
|
||||
for column in [
|
||||
"upstream_model_id",
|
||||
"client_api_key_ref",
|
||||
"provider_account_ref",
|
||||
"provider_account_email_snapshot",
|
||||
"provider_account_name_snapshot",
|
||||
"requested_model_id",
|
||||
"upstream_model_id",
|
||||
"upstream_request_id",
|
||||
] {
|
||||
query.push(format!(" escape '\\' or {alias}.{column} like "));
|
||||
query.push_bind(pattern.clone());
|
||||
@@ -174,29 +166,50 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
from model_request_observations mr";
|
||||
|
||||
pub(crate) async fn list_usage_records(
|
||||
connection: &mut PgConnection,
|
||||
connection: &mut ObservabilityConnection,
|
||||
query: UsageRecordQuery,
|
||||
) -> StoreResult<UsageRecordPage> {
|
||||
query.filter.validate()?;
|
||||
let mut transaction = analytics::begin_snapshot(connection).await?;
|
||||
let mut snapshot = AnalyticsSnapshot::new_for_accounts(connection).await?;
|
||||
// 第一页未填满时,当前快照中的行数已是精确总数,无需再次扫描计数
|
||||
let (total, items) = if query.current_page == 1 {
|
||||
let items = list_usage_record_items(&mut *transaction, &query, None).await?;
|
||||
let items = list_usage_record_items(&mut *snapshot.tx, &query, None).await?;
|
||||
let total = if items.len() < usize::from(query.page_size.get()) {
|
||||
items.len() as u64
|
||||
} else if let Some(counts) = snapshot
|
||||
.completed_request_counts(query.range, &query.filter)
|
||||
.await?
|
||||
{
|
||||
counts.values().try_fold(0_u64, |sum, count| {
|
||||
sum.checked_add(*count)
|
||||
.ok_or_else(|| invalid("usage count overflows u64"))
|
||||
})?
|
||||
} else {
|
||||
count_usage_records(&mut *transaction, query.range, &query.filter).await?
|
||||
count_usage_records(&mut *snapshot.tx, query.range, &query.filter).await?
|
||||
};
|
||||
(total, items)
|
||||
} else {
|
||||
let total = count_usage_records(&mut *transaction, query.range, &query.filter).await?;
|
||||
let items = list_usage_record_items(&mut *transaction, &query, Some(total)).await?;
|
||||
let (total, page) = if let Some(counts) = snapshot
|
||||
.completed_request_counts(query.range, &query.filter)
|
||||
.await?
|
||||
{
|
||||
time_page_from_segments(
|
||||
counts.into_iter().rev(),
|
||||
query.range,
|
||||
observability_page_offset(query.current_page, query.page_size)?,
|
||||
query.page_size,
|
||||
TimeDelta::minutes(15),
|
||||
)?
|
||||
} else {
|
||||
usage_time_page(&mut snapshot.tx, &query).await?
|
||||
};
|
||||
let items = match page {
|
||||
Some(page) => list_usage_record_items_at(&mut *snapshot.tx, &query, page).await?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
(total, items)
|
||||
};
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("complete usage page snapshot", error))?;
|
||||
snapshot.finish().await?;
|
||||
Ok(UsageRecordPage {
|
||||
items,
|
||||
current_page: query.current_page,
|
||||
@@ -209,12 +222,25 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database =
|
||||
executor: E,
|
||||
query: &UsageRecordQuery,
|
||||
total: Option<u64>,
|
||||
) -> StoreResult<Vec<UsageListRecord>> {
|
||||
let page = TimePage {
|
||||
range: query.range,
|
||||
offset: observability_page_offset(query.current_page, query.page_size)?,
|
||||
total,
|
||||
};
|
||||
list_usage_record_items_at(executor, query, page).await
|
||||
}
|
||||
|
||||
async fn list_usage_record_items_at<'e, E: sqlx::Executor<'e, Database = Postgres>>(
|
||||
executor: E,
|
||||
query: &UsageRecordQuery,
|
||||
page: TimePage,
|
||||
) -> StoreResult<Vec<UsageListRecord>> {
|
||||
query.filter.validate()?;
|
||||
let mut offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut offset = page.offset;
|
||||
let mut limit = i64::from(query.page_size.get());
|
||||
let mut ascending = false;
|
||||
if let Some(total) = total {
|
||||
if let Some(total) = page.total {
|
||||
let total = i64::try_from(total).map_err(|_| invalid("usage count overflows i64"))?;
|
||||
if offset >= total {
|
||||
return Ok(Vec::new());
|
||||
@@ -231,9 +257,9 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database =
|
||||
"with candidate_page as materialized (select mr.id, mr.started_at from model_request_observations mr",
|
||||
);
|
||||
statement.push(" where mr.started_at >= ");
|
||||
statement.push_bind(query.range.start);
|
||||
statement.push_bind(page.range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(query.range.end);
|
||||
statement.push_bind(page.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
push_usage_filter(&mut statement, &query.filter, "mr");
|
||||
statement.push(if ascending {
|
||||
@@ -261,6 +287,93 @@ pub(crate) async fn list_usage_record_items<'e, E: sqlx::Executor<'e, Database =
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
// 页码保持全局精确排名,实际明细读取只落在覆盖该页的时间区间
|
||||
pub(crate) struct TimePage {
|
||||
pub(crate) range: ObservabilityRange,
|
||||
pub(crate) offset: i64,
|
||||
pub(crate) total: Option<u64>,
|
||||
}
|
||||
|
||||
pub(crate) fn time_page_from_counts(
|
||||
rows: &[sqlx::postgres::PgRow],
|
||||
range: ObservabilityRange,
|
||||
offset: i64,
|
||||
page_size: ObservabilityPageSize,
|
||||
) -> StoreResult<(u64, Option<TimePage>)> {
|
||||
let counts = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(row, "bucket_start")?,
|
||||
to_u64(get(row, "request_count")?)?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
time_page_from_segments(counts, range, offset, page_size, TimeDelta::minutes(1))
|
||||
}
|
||||
|
||||
fn time_page_from_segments(
|
||||
counts: impl IntoIterator<Item = (DateTime<Utc>, u64)>,
|
||||
range: ObservabilityRange,
|
||||
offset: i64,
|
||||
page_size: ObservabilityPageSize,
|
||||
width: TimeDelta,
|
||||
) -> StoreResult<(u64, Option<TimePage>)> {
|
||||
let offset = u64::try_from(offset).map_err(|_| invalid("negative page offset"))?;
|
||||
let end = offset
|
||||
.checked_add(u64::from(page_size.get()))
|
||||
.ok_or_else(|| invalid("page range overflows u64"))?;
|
||||
let mut total = 0_u64;
|
||||
let mut page = None;
|
||||
let mut selected_count = 0_u64;
|
||||
for (bucket, count) in counts {
|
||||
let next = total
|
||||
.checked_add(count)
|
||||
.ok_or_else(|| invalid("request count overflows u64"))?;
|
||||
if next > offset && total < end {
|
||||
let selected = page.get_or_insert_with(|| TimePage {
|
||||
range: ObservabilityRange {
|
||||
start: bucket.max(range.start),
|
||||
end: (bucket + width).min(range.end),
|
||||
},
|
||||
offset: (offset - total) as i64,
|
||||
total: None,
|
||||
});
|
||||
selected.range.start = bucket.max(range.start);
|
||||
selected_count += count;
|
||||
}
|
||||
total = next;
|
||||
}
|
||||
if let Some(page) = &mut page {
|
||||
page.total = Some(selected_count);
|
||||
}
|
||||
Ok((total, page))
|
||||
}
|
||||
|
||||
async fn usage_time_page(
|
||||
connection: &mut PgConnection,
|
||||
query: &UsageRecordQuery,
|
||||
) -> StoreResult<(u64, Option<TimePage>)> {
|
||||
let offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"select date_bin(interval '1 minute', mr.started_at, timestamptz 'epoch') as bucket_start,
|
||||
count(*)::bigint as request_count
|
||||
from model_request_observations mr where mr.started_at >= ",
|
||||
);
|
||||
statement.push_bind(query.range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(query.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
push_usage_filter(&mut statement, &query.filter, "mr");
|
||||
statement.push(" group by bucket_start order by bucket_start desc");
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("locate usage page", source))?;
|
||||
time_page_from_counts(&rows, query.range, offset, query.page_size)
|
||||
}
|
||||
|
||||
pub(crate) async fn count_usage_records<'e, E: sqlx::Executor<'e, Database = Postgres>>(
|
||||
executor: E,
|
||||
range: ObservabilityRange,
|
||||
|
||||
@@ -17,12 +17,20 @@ use crate::{StoreBackend, StoreError, StoreResult, postgres_unavailable};
|
||||
pub struct ObservabilityQueryBudget {
|
||||
slots: Arc<Semaphore>,
|
||||
wait_timeout: Duration,
|
||||
query_timeout: Duration,
|
||||
}
|
||||
|
||||
/// 观测名额与实际 PostgreSQL 连接共同存活,取消后仍覆盖协议收尾和事务回滚
|
||||
pub struct ObservabilityConnection {
|
||||
connection: PoolConnection<Postgres>,
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
query_timeout: Duration,
|
||||
}
|
||||
|
||||
impl ObservabilityConnection {
|
||||
pub(crate) const fn query_timeout(&self) -> Duration {
|
||||
self.query_timeout
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for ObservabilityConnection {
|
||||
@@ -53,12 +61,22 @@ impl Drop for ObservabilityConnection {
|
||||
|
||||
impl ObservabilityQueryBudget {
|
||||
/// 预算表示观测类 SQL 最多可同时占用的连接数
|
||||
pub fn try_new(max_connections: u32, wait_timeout: Duration) -> StoreResult<Self> {
|
||||
if max_connections == 0 || wait_timeout.is_zero() {
|
||||
pub fn try_new(
|
||||
max_connections: u32,
|
||||
wait_timeout: Duration,
|
||||
query_timeout: Duration,
|
||||
) -> StoreResult<Self> {
|
||||
if max_connections == 0
|
||||
|| wait_timeout.is_zero()
|
||||
|| query_timeout.is_zero()
|
||||
|| query_timeout.as_millis() > i32::MAX as u128
|
||||
{
|
||||
return Err(StoreError::InvalidData {
|
||||
source: None,
|
||||
entity: "observability query budget",
|
||||
message: "requires a positive connection count and wait timeout".to_owned(),
|
||||
message:
|
||||
"requires positive connection and timeout budgets within PostgreSQL limits"
|
||||
.to_owned(),
|
||||
});
|
||||
}
|
||||
let permits = usize::try_from(max_connections).map_err(|_| StoreError::InvalidData {
|
||||
@@ -69,10 +87,16 @@ impl ObservabilityQueryBudget {
|
||||
Ok(Self {
|
||||
slots: Arc::new(Semaphore::new(permits)),
|
||||
wait_timeout,
|
||||
query_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
/// 在共享槽位与十秒执行预算内完成整个观测操作
|
||||
#[must_use]
|
||||
pub const fn query_timeout(&self) -> Duration {
|
||||
self.query_timeout
|
||||
}
|
||||
|
||||
/// 在共享槽位与观测总预算内完成排队、连接获取和整个存储操作
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
@@ -87,14 +111,15 @@ impl ObservabilityQueryBudget {
|
||||
F: FnOnce(ObservabilityConnection) -> Q,
|
||||
Q: Future<Output = StoreResult<T>>,
|
||||
{
|
||||
let permit = self.acquire(operation).await?;
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::time::timeout(self.query_timeout, async {
|
||||
let permit = self.acquire(operation).await?;
|
||||
let connection = pool.acquire().await.map_err(|source| {
|
||||
postgres_unavailable("acquire observability connection", source)
|
||||
})?;
|
||||
query(ObservabilityConnection {
|
||||
connection,
|
||||
permit: Some(permit),
|
||||
query_timeout: self.query_timeout,
|
||||
})
|
||||
.await
|
||||
})
|
||||
|
||||
@@ -102,109 +102,17 @@ impl PgAdminAccountStore {
|
||||
ObservabilityRange::new(window.range.start, window.range.end)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
}
|
||||
let keys = windows
|
||||
.iter()
|
||||
.map(|window| window.key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let starts = windows
|
||||
.iter()
|
||||
.map(|window| window.range.start)
|
||||
.collect::<Vec<_>>();
|
||||
let ends = windows
|
||||
.iter()
|
||||
.map(|window| window.range.end)
|
||||
.collect::<Vec<_>>();
|
||||
let account_ids = windows
|
||||
.iter()
|
||||
.map(|window| window.account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let rows = self
|
||||
.query_budget
|
||||
self.query_budget
|
||||
.run(
|
||||
&self.pool,
|
||||
"load account usage windows",
|
||||
|mut connection| async move {
|
||||
sqlx::query(sqlx::AssertSqlSafe(account_usage_by_windows_sql()))
|
||||
.bind(account_ids)
|
||||
.bind(keys)
|
||||
.bind(starts)
|
||||
.bind(ends)
|
||||
.fetch_all(&mut *connection)
|
||||
crate::postgres::observability::account_usage_windows(&mut connection, windows)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
postgres_unavailable("load provider account quota window usage", source)
|
||||
})
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let mut usage_rows = Vec::with_capacity(windows.len());
|
||||
let mut costs_by_window = BTreeMap::<(String, String), Vec<AccountCost>>::new();
|
||||
let mut model_costs = BTreeMap::<(String, String, String), Vec<AccountCost>>::new();
|
||||
let mut models_by_window = BTreeMap::<(String, String), Vec<AccountModelUsage>>::new();
|
||||
for row in &rows {
|
||||
let model_grouping = window_usage_value::<i32>(row, "model_grouping")?;
|
||||
let currency_grouping = window_usage_value::<i32>(row, "currency_grouping")?;
|
||||
match (model_grouping, currency_grouping) {
|
||||
(1, 1) => usage_rows.push(row),
|
||||
(1, 0) if window_usage_value::<Option<String>>(row, "cost_currency")?.is_some() => {
|
||||
let (key, cost) = admin_account_usage_window_cost(row)?;
|
||||
costs_by_window.entry(key).or_default().push(cost);
|
||||
}
|
||||
(0, 1) if window_usage_value::<Option<String>>(row, "model")?.is_some() => {
|
||||
let ((account_id, window_key, _), usage) =
|
||||
admin_account_usage_window_model(row)?;
|
||||
models_by_window
|
||||
.entry((account_id, window_key))
|
||||
.or_default()
|
||||
.push(usage);
|
||||
}
|
||||
(0, 0)
|
||||
if window_usage_value::<Option<String>>(row, "model")?.is_some()
|
||||
&& window_usage_value::<Option<String>>(row, "cost_currency")?
|
||||
.is_some() =>
|
||||
{
|
||||
let (key, cost) = admin_account_usage_window_model_cost(row)?;
|
||||
model_costs.entry(key).or_default().push(cost);
|
||||
}
|
||||
(0 | 1, 0 | 1) => {}
|
||||
_ => {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
ENTITY,
|
||||
"account usage window query returned an invalid grouping marker",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut results = usage_rows
|
||||
.into_iter()
|
||||
.map(admin_account_usage_window)
|
||||
.collect::<AdminStoreResult<Vec<_>>>()?;
|
||||
for result in &mut results {
|
||||
result.usage.costs = costs_by_window
|
||||
.remove(&(result.account_id.clone(), result.key.clone()))
|
||||
.unwrap_or_default();
|
||||
result.usage.models = models_by_window
|
||||
.remove(&(result.account_id.clone(), result.key.clone()))
|
||||
.unwrap_or_default();
|
||||
for model in &mut result.usage.models {
|
||||
model.costs = model_costs
|
||||
.remove(&(
|
||||
result.account_id.clone(),
|
||||
result.key.clone(),
|
||||
model.model.clone(),
|
||||
))
|
||||
.unwrap_or_default();
|
||||
}
|
||||
result.usage.models.sort_by(|left, right| {
|
||||
right
|
||||
.request_count
|
||||
.cmp(&left.request_count)
|
||||
.then_with(|| left.model.cmp(&right.model))
|
||||
});
|
||||
}
|
||||
Ok(results)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))
|
||||
}
|
||||
|
||||
async fn required_scope(
|
||||
@@ -376,8 +284,14 @@ impl AccountStore for PgAdminAccountStore {
|
||||
.filter(|(_, cooldown)| cooldown.is_active(now_system_time))
|
||||
.map(|(account_id, _)| account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let page =
|
||||
load_admin_account_page(&self.pool, &query, now, active_rate_limited_ids).await?;
|
||||
let page = load_admin_account_page(
|
||||
&self.pool,
|
||||
&self.query_budget,
|
||||
&query,
|
||||
now,
|
||||
active_rate_limited_ids,
|
||||
)
|
||||
.await?;
|
||||
let item_ids = page
|
||||
.accounts
|
||||
.iter()
|
||||
|
||||
@@ -12,6 +12,7 @@ pub(crate) struct AdminAccountPageRows {
|
||||
|
||||
pub(crate) async fn load_admin_account_page(
|
||||
pool: &PgPool,
|
||||
budget: &crate::postgres::ObservabilityQueryBudget,
|
||||
query: &AdminAccountListQuery,
|
||||
now: DateTime<Utc>,
|
||||
active_rate_limited_ids: Vec<String>,
|
||||
@@ -35,35 +36,20 @@ pub(crate) async fn load_admin_account_page(
|
||||
let page_size = i64::from(query.page_size.get());
|
||||
let offset = i64::from(query.page - 1).saturating_mul(page_size);
|
||||
let order_by = admin_account_order(query.sort);
|
||||
let usage_join = query.sort.is_some_and(|sort| {
|
||||
let needs_usage = query.sort.is_some_and(|sort| {
|
||||
matches!(
|
||||
sort.field,
|
||||
AdminAccountSortField::Usage | AdminAccountSortField::LastUsedAt
|
||||
)
|
||||
});
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
let usage_join = if usage_join {
|
||||
format!(
|
||||
"left join (
|
||||
select mr.provider_account_ref,
|
||||
coalesce(sum(coalesce(
|
||||
mr.total_tokens,
|
||||
coalesce(mr.input_tokens, 0) + coalesce(mr.output_tokens, 0)
|
||||
)), 0)::bigint as total_tokens,
|
||||
max(mr.started_at) as last_used_at
|
||||
from model_requests mr
|
||||
where mr.started_at >= $2 - make_interval(days => settings.usage_retention_days::int)
|
||||
and mr.started_at < $2
|
||||
and {completed_usage}
|
||||
group by mr.provider_account_ref
|
||||
) usage on usage.provider_account_ref = a.id"
|
||||
)
|
||||
let usage_join = if needs_usage {
|
||||
"left join usage_values usage on usage.provider_account_ref = a.id"
|
||||
} else {
|
||||
String::new()
|
||||
""
|
||||
};
|
||||
|
||||
let statement = format!(
|
||||
"with account_statuses as (
|
||||
"with usage_values as (select * from unnest($10::text[],$11::bigint[],$12::timestamptz[]) as usage(provider_account_ref,total_tokens,last_used_at)), account_statuses as (
|
||||
select a.id, a.provider_kind,
|
||||
case
|
||||
when not a.enabled then 'disabled'
|
||||
@@ -150,25 +136,45 @@ pub(crate) async fn load_admin_account_page(
|
||||
left join outbound_proxies p on p.id = a.outbound_proxy_id
|
||||
order by page.page_position"
|
||||
);
|
||||
// 动态片段只来自上面的封闭排序枚举与固定 usage predicate;所有请求值仍使用 bind
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(active_rate_limited_ids)
|
||||
.bind(now)
|
||||
.bind(provider_kind)
|
||||
.bind(search_prefix)
|
||||
.bind(status)
|
||||
.bind(group_mode)
|
||||
.bind(group_id)
|
||||
.bind(page_size)
|
||||
.bind(offset)
|
||||
.fetch_all(pool)
|
||||
// 排序事实与本页总数共享同一快照,避免一次查询混入不同源版本
|
||||
let rows = budget
|
||||
.run(
|
||||
pool,
|
||||
"load admin account page",
|
||||
|mut connection| async move {
|
||||
let mut snapshot =
|
||||
crate::postgres::observability::AnalyticsSnapshot::new_for_accounts(
|
||||
&mut connection,
|
||||
)
|
||||
.await?;
|
||||
let (usage_ids, usage_tokens, usage_latest) = if needs_usage {
|
||||
crate::postgres::observability::account_usage_sort_values(&mut snapshot, now)
|
||||
.await?
|
||||
} else {
|
||||
(Vec::new(), Vec::new(), Vec::new())
|
||||
};
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(active_rate_limited_ids)
|
||||
.bind(now)
|
||||
.bind(provider_kind)
|
||||
.bind(search_prefix)
|
||||
.bind(status)
|
||||
.bind(group_mode)
|
||||
.bind(group_id)
|
||||
.bind(page_size)
|
||||
.bind(offset)
|
||||
.bind(usage_ids)
|
||||
.bind(usage_tokens)
|
||||
.bind(usage_latest)
|
||||
.fetch_all(&mut *snapshot.tx)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load admin account page", source))?;
|
||||
snapshot.finish().await?;
|
||||
Ok(rows)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
admin_store_error(
|
||||
ENTITY,
|
||||
postgres_unavailable("load admin account page", source),
|
||||
)
|
||||
})?;
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let metadata = rows.first().ok_or_else(|| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
|
||||
@@ -105,164 +105,3 @@ pub(crate) fn admin_account_usage(usage: ProviderAccountUsageObservation) -> Acc
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_usage_window(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> AdminStoreResult<AccountUsageWindowResult> {
|
||||
let provider_reported_count = window_usage_count(row, "provider_reported_count")?;
|
||||
let calculated_count = window_usage_count(row, "calculated_count")?;
|
||||
let unavailable_count = window_usage_count(row, "unavailable_count")?;
|
||||
Ok(AccountUsageWindowResult {
|
||||
account_id: window_usage_value(row, "account_id")?,
|
||||
key: window_usage_value(row, "window_key")?,
|
||||
usage: AccountUsage {
|
||||
account_id: window_usage_value(row, "account_id")?,
|
||||
request_count: window_usage_count(row, "request_count")?,
|
||||
success_count: window_usage_count(row, "success_count")?,
|
||||
input_tokens: optional_window_usage_count(row, "input_tokens")?,
|
||||
output_tokens: optional_window_usage_count(row, "output_tokens")?,
|
||||
cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?,
|
||||
image_request_count: window_usage_count(row, "image_request_count")?,
|
||||
image_request_failed_count: window_usage_count(row, "image_request_failed_count")?,
|
||||
total_tokens: optional_window_usage_count(row, "total_tokens")?,
|
||||
cost_coverage: AdminCostCoverage {
|
||||
provider_reported_count,
|
||||
calculated_count,
|
||||
partial_count: 0,
|
||||
unavailable_count,
|
||||
not_billable_count: 0,
|
||||
},
|
||||
costs: Vec::new(),
|
||||
last_used_at: window_usage_value(row, "last_used_at")?,
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_usage_window_model(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> AdminStoreResult<((String, String, String), AccountModelUsage)> {
|
||||
let account_id: String = window_usage_value(row, "account_id")?;
|
||||
let window_key: String = window_usage_value(row, "window_key")?;
|
||||
let model: String = window_usage_value(row, "model")?;
|
||||
let provider_reported_count = window_usage_count(row, "provider_reported_count")?;
|
||||
let calculated_count = window_usage_count(row, "calculated_count")?;
|
||||
let unavailable_count = window_usage_count(row, "unavailable_count")?;
|
||||
Ok((
|
||||
(account_id, window_key, model.clone()),
|
||||
AccountModelUsage {
|
||||
model,
|
||||
request_count: window_usage_count(row, "request_count")?,
|
||||
success_count: window_usage_count(row, "success_count")?,
|
||||
input_tokens: optional_window_usage_count(row, "input_tokens")?,
|
||||
output_tokens: optional_window_usage_count(row, "output_tokens")?,
|
||||
cached_input_tokens: optional_window_usage_count(row, "cached_input_tokens")?,
|
||||
cache_write_input_tokens: optional_window_usage_count(row, "cache_write_input_tokens")?,
|
||||
reasoning_output_tokens: optional_window_usage_count(row, "reasoning_output_tokens")?,
|
||||
image_input_tokens: optional_window_usage_count(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_window_usage_count(row, "image_output_tokens")?,
|
||||
image_request_count: window_usage_count(row, "image_request_count")?,
|
||||
image_request_failed_count: window_usage_count(row, "image_request_failed_count")?,
|
||||
total_tokens: optional_window_usage_count(row, "total_tokens")?,
|
||||
cost_coverage: AdminCostCoverage {
|
||||
provider_reported_count,
|
||||
calculated_count,
|
||||
partial_count: 0,
|
||||
unavailable_count,
|
||||
not_billable_count: 0,
|
||||
},
|
||||
costs: Vec::new(),
|
||||
last_used_at: window_usage_value(row, "last_used_at")?,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_usage_window_model_cost(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> AdminStoreResult<((String, String, String), AccountCost)> {
|
||||
let key = (
|
||||
window_usage_value(row, "account_id")?,
|
||||
window_usage_value(row, "window_key")?,
|
||||
window_usage_value(row, "model")?,
|
||||
);
|
||||
let amount = window_usage_value::<String>(row, "amount")?;
|
||||
let amount = AdminDecimalAmount::from_str(&amount).map_err(|_| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
ENTITY,
|
||||
"persisted account model cost is invalid",
|
||||
)
|
||||
})?;
|
||||
Ok((
|
||||
key,
|
||||
AccountCost {
|
||||
currency: window_usage_value(row, "cost_currency")?,
|
||||
amount,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_usage_window_cost(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> AdminStoreResult<((String, String), AccountCost)> {
|
||||
let key = (
|
||||
window_usage_value(row, "account_id")?,
|
||||
window_usage_value(row, "window_key")?,
|
||||
);
|
||||
let amount = window_usage_value::<String>(row, "amount")?;
|
||||
let amount = AdminDecimalAmount::from_str(&amount).map_err(|_| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
ENTITY,
|
||||
"persisted account cost is invalid",
|
||||
)
|
||||
})?;
|
||||
Ok((
|
||||
key,
|
||||
AccountCost {
|
||||
currency: window_usage_value(row, "cost_currency")?,
|
||||
amount,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn window_usage_count(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> AdminStoreResult<u64> {
|
||||
let value = window_usage_value::<i64>(row, column)?;
|
||||
u64::try_from(value).map_err(|_| invalid_window_usage(column))
|
||||
}
|
||||
|
||||
pub(crate) fn optional_window_usage_count(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> AdminStoreResult<Option<u64>> {
|
||||
window_usage_value::<Option<i64>>(row, column)?
|
||||
.map(|value| u64::try_from(value).map_err(|_| invalid_window_usage(column)))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn window_usage_value<T>(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> AdminStoreResult<T>
|
||||
where
|
||||
for<'row> T: sqlx::Decode<'row, Postgres> + sqlx::Type<Postgres>,
|
||||
{
|
||||
row.try_get(column)
|
||||
.map_err(|_| invalid_window_usage(column))
|
||||
}
|
||||
|
||||
pub(crate) fn invalid_window_usage(column: &'static str) -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
ENTITY,
|
||||
format!("invalid quota window usage column: {column}"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 明文 `provider_accounts` 与凭证 revision CAS 的唯一 PostgreSQL owner
|
||||
|
||||
use std::{collections::BTreeSet, fmt, str::FromStr};
|
||||
use std::{collections::BTreeSet, fmt};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
@@ -9,17 +9,14 @@ use gateway_admin::{
|
||||
MutationContext, Revision as AdminRevision,
|
||||
account_groups::AccountGroupRef,
|
||||
accounts::{
|
||||
AccountCost, AccountGroupFilter, AccountImportSettings,
|
||||
AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage,
|
||||
AccountPageItem, AccountRecord, AccountSort as AdminAccountSort,
|
||||
AccountGroupFilter, AccountImportSettings, AccountListQuery as AdminAccountListQuery,
|
||||
AccountPage, AccountPageItem, AccountRecord, AccountSort as AdminAccountSort,
|
||||
AccountSortField as AdminAccountSortField, AccountSummary, AccountUpdateResult,
|
||||
AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, DeleteAccounts, SortDirection as AdminSortDirection,
|
||||
UpdateAccount,
|
||||
},
|
||||
observability::{
|
||||
CostCoverage as AdminCostCoverage, DecimalAmount as AdminDecimalAmount, TimeRange,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationCredentialCommit, CredentialDetails,
|
||||
CredentialImportCommit, CredentialImportResult, CredentialMutationResult,
|
||||
@@ -54,7 +51,6 @@ use super::{
|
||||
AdminAuditEvent, ControlPlaneRepository, ObservabilityRange, PgControlPlaneRepository,
|
||||
PgObservabilityRepository, ProviderAccountUsageObservation, ProviderAccountUsageQuery,
|
||||
append_admin_audit_event_in_transaction, bump_config_revision_in_transaction,
|
||||
completed_usage_fact_predicate,
|
||||
};
|
||||
|
||||
mod admin_adapter;
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
//! 预测只读采样:先聚合数值、再读取有界的 Provider 文档,不扫描模型排行维度
|
||||
|
||||
use gateway_admin::model::quota_forecast_sampling::{
|
||||
MAX_FORECAST_HISTORY_POINTS, QuotaForecastHistory, QuotaForecastHistoryPoint,
|
||||
QuotaForecastUsage,
|
||||
};
|
||||
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistory;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -23,124 +20,21 @@ pub(super) async fn load_history(
|
||||
"forecast range exceeds supported quota windows",
|
||||
));
|
||||
}
|
||||
let rows = budget
|
||||
budget
|
||||
.run(
|
||||
pool,
|
||||
"load quota forecast history",
|
||||
|mut connection| async move {
|
||||
sqlx::query(sqlx::AssertSqlSafe(history_sql()))
|
||||
.bind(&window.account_id)
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.bind(MAX_FORECAST_HISTORY_POINTS as i32)
|
||||
.fetch_all(&mut *connection)
|
||||
.await
|
||||
.map_err(|source| postgres_unavailable("load quota forecast history", source))
|
||||
let mut snapshot =
|
||||
super::super::observability::AnalyticsSnapshot::new_for_accounts(
|
||||
&mut connection,
|
||||
)
|
||||
.await?;
|
||||
let history = snapshot.forecast_history(&window.account_id, range).await?;
|
||||
snapshot.finish().await?;
|
||||
Ok(history)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let mut history = QuotaForecastHistory::default();
|
||||
for row in &rows {
|
||||
let usage = QuotaForecastUsage {
|
||||
request_count: window_usage_count(row, "request_count")?,
|
||||
tokens: window_usage_count(row, "tokens")?,
|
||||
input_tokens: window_usage_count(row, "input_tokens")?,
|
||||
output_tokens: window_usage_count(row, "output_tokens")?,
|
||||
cached_input_tokens: window_usage_count(row, "cached_input_tokens")?,
|
||||
missing_token_count: window_usage_count(row, "missing_token_count")?,
|
||||
known_cost_count: window_usage_count(row, "known_cost_count")?,
|
||||
unavailable_cost_count: window_usage_count(row, "unavailable_cost_count")?,
|
||||
usd: window_usage_value(row, "usd")?,
|
||||
excluded_request_count: window_usage_count(row, "excluded_request_count")?,
|
||||
};
|
||||
if window_usage_value::<bool>(row, "is_total")? {
|
||||
history.usage = usage;
|
||||
history.pending_request_count = window_usage_count(row, "pending_count")?;
|
||||
} else if let Some(document) = window_usage_value::<
|
||||
Option<sqlx::types::Json<serde_json::Map<String, serde_json::Value>>>,
|
||||
>(row, "document")?
|
||||
{
|
||||
history.points.push(QuotaForecastHistoryPoint {
|
||||
started_at: window_usage_value(row, "started_at")?,
|
||||
completed_at: window_usage_value(row, "completed_at")?,
|
||||
usage,
|
||||
provider_observation: ProviderDocument::new(OpaqueProviderData::new(document.0)),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(history)
|
||||
}
|
||||
|
||||
fn history_sql() -> String {
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
// 一个语句共享 MVCC 快照
|
||||
// 用 RANGE 帧让相同完成时间的点拥有相同累计值,
|
||||
// 避免并发请求的任意行顺序制造不同分子;未完成请求只进入待决计数
|
||||
format!(
|
||||
"with scoped as materialized (
|
||||
select mr.id, mr.started_at, mr.completed_at,
|
||||
mr.completed_at <= $3 and mr.outcome <> 'running' as settled,
|
||||
coalesce(({completed_usage}), false) as included,
|
||||
mr.provider_observation_json is not null as has_document,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_input_tokens, mr.total_tokens,
|
||||
mr.cost_amount, mr.cost_currency,
|
||||
coalesce(mr.cost_source in ('calculated', 'provider_reported')
|
||||
and mr.cost_amount is not null and mr.cost_currency = 'USD', false) as known_cost
|
||||
from model_requests mr
|
||||
where mr.provider_account_ref = $1
|
||||
and mr.started_at >= $2 and mr.started_at < $3
|
||||
), facts as materialized (
|
||||
select *,
|
||||
(included and settled)::integer as request_count,
|
||||
case when included and settled then coalesce(total_tokens,
|
||||
coalesce(input_tokens, 0) + coalesce(output_tokens, 0)) else 0 end as tokens,
|
||||
case when included and settled then coalesce(input_tokens, 0) else 0 end as inputs,
|
||||
case when included and settled then coalesce(output_tokens, 0) else 0 end as outputs,
|
||||
case when included and settled then coalesce(cached_input_tokens, 0) else 0 end as cached,
|
||||
(included and settled and total_tokens is null
|
||||
and (input_tokens is null or output_tokens is null))::integer as missing_tokens,
|
||||
(included and settled and known_cost)::integer as known_costs,
|
||||
(included and settled and not known_cost)::integer as missing_costs,
|
||||
case when included and settled and known_cost then cost_amount else 0 end as usd,
|
||||
(not included and settled)::integer as excluded
|
||||
from scoped
|
||||
), cumulative as (
|
||||
select id, started_at, completed_at, has_document,
|
||||
least($4 - 1, floor(extract(epoch from (completed_at - $2))
|
||||
/ greatest(1, extract(epoch from ($3::timestamptz - $2::timestamptz)) / $4))) as bucket,
|
||||
sum(request_count) over w as request_count,
|
||||
sum(tokens) over w as tokens,
|
||||
sum(inputs) over w as input_tokens,
|
||||
sum(outputs) over w as output_tokens,
|
||||
sum(cached) over w as cached_input_tokens,
|
||||
sum(missing_tokens) over w as missing_token_count,
|
||||
sum(known_costs) over w as known_cost_count,
|
||||
sum(missing_costs) over w as unavailable_cost_count,
|
||||
sum(usd) over w as usd,
|
||||
sum(excluded) over w as excluded_request_count
|
||||
from facts where settled
|
||||
window w as (order by completed_at range between unbounded preceding and current row)
|
||||
), selected as (
|
||||
select distinct on (bucket) * from cumulative
|
||||
where has_document order by bucket, completed_at desc, id desc
|
||||
)
|
||||
select false as is_total, s.started_at, s.completed_at,
|
||||
s.request_count::bigint, s.tokens::bigint, s.input_tokens::bigint,
|
||||
s.output_tokens::bigint, s.cached_input_tokens::bigint, s.missing_token_count::bigint,
|
||||
s.known_cost_count::bigint, s.unavailable_cost_count::bigint,
|
||||
s.usd::double precision, s.excluded_request_count::bigint,
|
||||
0::bigint as pending_count, mr.provider_observation_json as document
|
||||
from selected s join model_requests mr on mr.id = s.id
|
||||
union all
|
||||
select true, $3, $3,
|
||||
coalesce(sum(request_count), 0)::bigint, coalesce(sum(tokens), 0)::bigint,
|
||||
coalesce(sum(inputs), 0)::bigint, coalesce(sum(outputs), 0)::bigint,
|
||||
coalesce(sum(cached), 0)::bigint, coalesce(sum(missing_tokens), 0)::bigint,
|
||||
coalesce(sum(known_costs), 0)::bigint, coalesce(sum(missing_costs), 0)::bigint,
|
||||
coalesce(sum(usd), 0)::double precision, coalesce(sum(excluded), 0)::bigint,
|
||||
count(*) filter (where settled is not true), null::jsonb
|
||||
from facts
|
||||
order by completed_at, is_total"
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))
|
||||
}
|
||||
|
||||
@@ -9,76 +9,6 @@ pub(crate) const QUOTA_MAX_BYTES: usize = 128 * 1024;
|
||||
pub(crate) const MAX_ADMIN_IMPORT_BATCH: usize = 200;
|
||||
pub(crate) const ADMIN_USAGE_CHUNK_SIZE: usize = 200;
|
||||
|
||||
/// 四种窗口投影通过一组 `GROUPING SETS` 从同一批匹配请求派生
|
||||
/// `grouping_* = 1`
|
||||
/// 表示对应维度未参与该行聚合,调用方据此区分窗口、模型与成本行
|
||||
pub(crate) fn account_usage_by_windows_sql() -> String {
|
||||
let completed_usage = completed_usage_fact_predicate("mr");
|
||||
format!(
|
||||
"with requested_windows as (
|
||||
select *
|
||||
from unnest($1::text[], $2::text[], $3::timestamptz[], $4::timestamptz[])
|
||||
as requested(account_id, window_key, window_start, window_end)
|
||||
), matched as (
|
||||
select requested.account_id,
|
||||
requested.window_key,
|
||||
mr.id as request_id,
|
||||
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
|
||||
mr.outcome, mr.input_tokens, mr.output_tokens, mr.cached_input_tokens,
|
||||
mr.cache_write_input_tokens, mr.reasoning_output_tokens,
|
||||
mr.image_input_tokens, mr.image_output_tokens,
|
||||
mr.image_generation_succeeded, mr.total_tokens,
|
||||
mr.cost_source, mr.cost_amount, mr.cost_currency, mr.started_at
|
||||
from requested_windows requested
|
||||
left join model_requests mr
|
||||
on mr.provider_account_ref = requested.account_id
|
||||
and mr.started_at >= requested.window_start
|
||||
and mr.started_at < requested.window_end
|
||||
and {completed_usage}
|
||||
)
|
||||
select account_id,
|
||||
window_key,
|
||||
model,
|
||||
cost_currency,
|
||||
grouping(model)::integer as model_grouping,
|
||||
grouping(cost_currency)::integer as currency_grouping,
|
||||
count(request_id)::bigint as request_count,
|
||||
count(request_id) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
sum(input_tokens)::bigint as input_tokens,
|
||||
sum(output_tokens)::bigint as output_tokens,
|
||||
sum(cached_input_tokens)::bigint as cached_input_tokens,
|
||||
sum(cache_write_input_tokens)::bigint as cache_write_input_tokens,
|
||||
sum(reasoning_output_tokens)::bigint as reasoning_output_tokens,
|
||||
sum(image_input_tokens)::bigint as image_input_tokens,
|
||||
sum(image_output_tokens)::bigint as image_output_tokens,
|
||||
count(request_id) filter (where image_generation_succeeded is true)::bigint
|
||||
as image_request_count,
|
||||
count(request_id) filter (where image_generation_succeeded is false)::bigint
|
||||
as image_request_failed_count,
|
||||
coalesce(sum(coalesce(
|
||||
total_tokens,
|
||||
coalesce(input_tokens, 0) + coalesce(output_tokens, 0)
|
||||
)), 0)::bigint as total_tokens,
|
||||
count(request_id) filter (where cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(request_id) filter (where cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(request_id) filter (where cost_source = 'unavailable')::bigint
|
||||
as unavailable_count,
|
||||
max(started_at) as last_used_at,
|
||||
sum(cost_amount)::text as amount
|
||||
from matched
|
||||
group by grouping sets (
|
||||
(account_id, window_key),
|
||||
(account_id, window_key, cost_currency),
|
||||
(account_id, window_key, model),
|
||||
(account_id, window_key, model, cost_currency)
|
||||
)
|
||||
order by account_id, window_key, model_grouping desc, currency_grouping desc,
|
||||
model nulls last, cost_currency nulls last"
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountAdminScope {
|
||||
pub provider_kind: String,
|
||||
|
||||
@@ -27,6 +27,7 @@ fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits()
|
||||
assert_eq!(pool, StorePoolConfig::default());
|
||||
assert_eq!(pool.max_connections, 20);
|
||||
assert_eq!(pool.acquire_timeout_seconds, 5);
|
||||
assert_eq!(pool.observability_timeout_seconds, 120);
|
||||
assert_eq!(pool.observability_max_connections(), 16);
|
||||
assert_eq!(
|
||||
StorePoolConfig {
|
||||
@@ -87,3 +88,24 @@ fn valid_config() -> StoreConfig {
|
||||
}))
|
||||
.expect("test Store configuration")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_config_accepts_custom_observability_timeout_and_rejects_invalid_values() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for seconds in [1_u64, 47, 180] {
|
||||
let pool: StorePoolConfig = serde_json::from_value(serde_json::json!({
|
||||
"observability_timeout_seconds": seconds,
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(pool.observability_timeout_seconds, seconds);
|
||||
}
|
||||
for seconds in [0_u64, u64::MAX] {
|
||||
let mut config: serde_json::Value = serde_json::json!({
|
||||
"database": { "url": "postgres://codex_proxy@127.0.0.1:5432/codex_proxy", "password": PASSWORD },
|
||||
"redis": { "url": "redis://127.0.0.1:6379/", "password": PASSWORD },
|
||||
});
|
||||
config["pool"] = serde_json::json!({"observability_timeout_seconds": seconds});
|
||||
let mut config: StoreConfig = serde_json::from_value(config).unwrap();
|
||||
assert!(config.resolve_and_validate(root.path()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,9 @@ use gateway_core::{
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::AccountGroupId,
|
||||
};
|
||||
use gateway_store::postgres::{PgAccountGroupRepository, PgAdminClientKeyStore};
|
||||
use gateway_store::postgres::{
|
||||
ObservabilityQueryBudget, PgAccountGroupRepository, PgAdminClientKeyStore,
|
||||
};
|
||||
|
||||
use super::TestDatabase;
|
||||
|
||||
@@ -75,7 +77,11 @@ async fn groups_aggregate_cross_provider_members_and_key_bindings_without_multip
|
||||
.await
|
||||
.expect("set account concurrency override");
|
||||
|
||||
let groups = PgAccountGroupRepository::new(database.pool.clone());
|
||||
let groups = PgAccountGroupRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
|
||||
.unwrap(),
|
||||
);
|
||||
let keys = PgAdminClientKeyStore::new(database.pool.clone());
|
||||
let mixed_group = group_id(MIXED_GROUP);
|
||||
let empty_group = group_id(EMPTY_GROUP);
|
||||
@@ -315,7 +321,11 @@ async fn group_options_should_avoid_aggregate_tables_and_page_filtered_empty_gro
|
||||
.await
|
||||
.expect("seed account group option");
|
||||
}
|
||||
let groups = PgAccountGroupRepository::new(database.pool.clone());
|
||||
let groups = PgAccountGroupRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let mut locked_tables = database.pool.begin().await.expect("begin table lock");
|
||||
sqlx::query(
|
||||
@@ -383,7 +393,11 @@ async fn group_costs_should_include_statusless_websocket_but_reject_statusless_h
|
||||
"Statusless Account",
|
||||
)
|
||||
.await;
|
||||
let groups = PgAccountGroupRepository::new(database.pool.clone());
|
||||
let groups = PgAccountGroupRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
|
||||
.unwrap(),
|
||||
);
|
||||
groups
|
||||
.create_account_group(
|
||||
NewAccountGroup {
|
||||
@@ -576,7 +590,11 @@ async fn fast_mode_group_updates_preserve_omitted_values_and_publish_snapshot_fa
|
||||
let Some(database) = TestDatabase::create("fast_mode_group").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgAccountGroupRepository::new(database.pool.clone());
|
||||
let repository = PgAccountGroupRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
|
||||
.unwrap(),
|
||||
);
|
||||
let id = group_id(MIXED_GROUP);
|
||||
repository
|
||||
.create_account_group(
|
||||
@@ -632,3 +650,112 @@ async fn fast_mode_group_updates_preserve_omitted_values_and_publish_snapshot_fa
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn group_costs_follow_current_members_and_pending_source_replacements() {
|
||||
let Some(database) = TestDatabase::create("group_cost_current_members").await else {
|
||||
return;
|
||||
};
|
||||
seed_account(
|
||||
&database.pool,
|
||||
"acct_group_move",
|
||||
"openai",
|
||||
"Movable Account",
|
||||
)
|
||||
.await;
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
|
||||
.unwrap();
|
||||
let groups = PgAccountGroupRepository::new(database.pool.clone(), budget.clone());
|
||||
for id in [MIXED_GROUP, EMPTY_GROUP] {
|
||||
groups
|
||||
.create_account_group(
|
||||
NewAccountGroup {
|
||||
fast_mode: FastMode::Default,
|
||||
id: group_id(id),
|
||||
name: id.to_owned(),
|
||||
description: None,
|
||||
color: group_color("#2563EBFF"),
|
||||
},
|
||||
&context("create-group-move-fixture"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
assign_accounts(&database.pool, MIXED_GROUP, &["acct_group_move"]).await;
|
||||
seed_group_cost_snapshot(
|
||||
&database.pool,
|
||||
"req_group_move",
|
||||
"acct_group_move",
|
||||
&[MIXED_GROUP],
|
||||
"3.25",
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"update model_requests set started_at=now()-interval '2 hours' where id='req_group_move'",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker =
|
||||
gateway_store::postgres::PgRequestAnalyticsRepository::new(database.pool.clone(), budget);
|
||||
let mut ready = false;
|
||||
for _ in 0..100 {
|
||||
worker.maintain(1000).await.unwrap();
|
||||
let status = worker.status().await.unwrap();
|
||||
if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 {
|
||||
ready = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(ready, "group fixture must publish rollups");
|
||||
let costs = load_retained_group_costs(&groups).await;
|
||||
assert_eq!(costs[MIXED_GROUP], "3.25");
|
||||
assert_eq!(costs[EMPTY_GROUP], "0");
|
||||
|
||||
sqlx::query("update account_group_accounts set account_group_id=$1 where provider_account_id='acct_group_move'")
|
||||
.bind(EMPTY_GROUP).execute(&database.pool).await.unwrap();
|
||||
let costs = load_retained_group_costs(&groups).await;
|
||||
assert_eq!(
|
||||
costs[MIXED_GROUP], "0",
|
||||
"moving an account reattributes retained usage"
|
||||
);
|
||||
assert_eq!(costs[EMPTY_GROUP], "3.25");
|
||||
|
||||
// 未再次运行 Worker,当前源费用和模型仍必须原子替换旧汇总贡献
|
||||
sqlx::query("update model_requests set cost_amount=7.5,upstream_model_id='updated-model' where id='req_group_move'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let costs = load_retained_group_costs(&groups).await;
|
||||
assert_eq!(costs[EMPTY_GROUP], "7.5");
|
||||
sqlx::query("delete from model_requests where id='req_group_move'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let costs = load_retained_group_costs(&groups).await;
|
||||
assert_eq!(
|
||||
costs[EMPTY_GROUP], "0",
|
||||
"pending deletion must withdraw the old projected amount"
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
async fn load_retained_group_costs(groups: &PgAccountGroupRepository) -> BTreeMap<String, String> {
|
||||
groups
|
||||
.list_account_groups(AccountGroupListQuery {
|
||||
page: 1,
|
||||
page_size: PageSize::new(20).unwrap(),
|
||||
search: None,
|
||||
enabled: None,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.items
|
||||
.into_iter()
|
||||
.map(|group| {
|
||||
(
|
||||
group.id.to_string(),
|
||||
group.usage.retained_total_usd.to_string(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -21,6 +21,212 @@ pub(super) async fn seed(pool: &sqlx::PgPool, id: &str, start: DateTime<Utc>, gr
|
||||
.bind(id).bind(start).bind(observation).execute(pool).await.expect("seed analytic request");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_counts_match_for_rollups_boundaries_and_filtered_facts() {
|
||||
let Some(database) = TestDatabase::create("analytics_provider_counts").await else {
|
||||
return;
|
||||
};
|
||||
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
for id in [
|
||||
"provider_counts_ok",
|
||||
"provider_counts_fail",
|
||||
"provider_counts_unrouted",
|
||||
] {
|
||||
seed(&database.pool, id, start + TimeDelta::minutes(5), &[]).await;
|
||||
}
|
||||
sqlx::query("update model_requests set provider_kind='openai',provider_account_ref='account_provider_counts',upstream_transport='http',attempt_count=case when id='provider_counts_ok' then 2 else 1 end where id in ('provider_counts_ok','provider_counts_fail')")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
sqlx::query("update model_requests set outcome='failed',client_status_code=502,downstream_committed_at=null where id='provider_counts_fail'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let repo = observability_repository(&database.pool);
|
||||
let full = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
|
||||
let edge =
|
||||
ObservabilityRange::new(start + TimeDelta::minutes(4), start + TimeDelta::minutes(6))
|
||||
.unwrap();
|
||||
// 同一批请求分别走完整桶、首尾事实与筛选回退,供应商计数不能随读取路径变化
|
||||
for (range, filter) in [
|
||||
(full, UsageRecordFilter::default()),
|
||||
(edge, UsageRecordFilter::default()),
|
||||
(
|
||||
full,
|
||||
UsageRecordFilter {
|
||||
search: Some("model_".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
] {
|
||||
let summary = repo.usage_summary(range, filter.clone()).await.unwrap();
|
||||
assert_eq!(summary.requests.request_count, 3);
|
||||
let providers = summary
|
||||
.providers
|
||||
.into_iter()
|
||||
.map(|provider| {
|
||||
(
|
||||
provider.provider_kind,
|
||||
(
|
||||
provider.request_count,
|
||||
provider.attempt_count,
|
||||
provider.failure_count,
|
||||
provider.total_tokens,
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
assert_eq!(
|
||||
providers,
|
||||
std::collections::BTreeMap::from([
|
||||
("openai".to_owned(), (2, 3, 1, 120)),
|
||||
("unrouted".to_owned(), (1, 0, 0, 120)),
|
||||
])
|
||||
);
|
||||
let diagnostic = repo
|
||||
.usage_diagnostics(range, filter, DiagnosticDimension::Provider, 100)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(diagnostic.total_request_count, 3);
|
||||
let openai = diagnostic
|
||||
.items
|
||||
.iter()
|
||||
.find(|item| item.key == "openai")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
openai.request_count,
|
||||
openai.success_count,
|
||||
openai.failure_count,
|
||||
openai.attempt_count,
|
||||
openai.total_tokens
|
||||
),
|
||||
(2, 1, 1, 3, 120)
|
||||
);
|
||||
assert_eq!(openai.average_latency_ms, Some(100));
|
||||
// 汇总桶沿用直方图近似值,诊断与完整指标读取必须给出相同分位数
|
||||
assert_eq!(
|
||||
openai.latency_p95_ms,
|
||||
summary
|
||||
.requests
|
||||
.latency_percentiles
|
||||
.p95_ms
|
||||
.map(|value| value.as_f64().round() as u64)
|
||||
);
|
||||
assert_eq!(
|
||||
openai.first_token_p95_ms,
|
||||
summary
|
||||
.requests
|
||||
.first_token_latency_percentiles
|
||||
.p95_ms
|
||||
.map(|value| value.as_f64().round() as u64)
|
||||
);
|
||||
assert_eq!(openai.retry_count, 1);
|
||||
assert_eq!(openai.retried_request_count, 1);
|
||||
assert_eq!(openai.non_completion_count, 0);
|
||||
assert_eq!(openai.cost_coverage.unavailable_count, 1);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn insights_share_bucket_states_without_merging_exact_percentiles() {
|
||||
let Some(database) = TestDatabase::create("analytics_shared_insights").await else {
|
||||
return;
|
||||
};
|
||||
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
for (index, latency) in [10_i64, 1000, 2000, 3000].into_iter().enumerate() {
|
||||
let id = format!("insights_{index}");
|
||||
let minute = if index == 0 { 1 } else { 16 };
|
||||
seed(
|
||||
&database.pool,
|
||||
&id,
|
||||
start + TimeDelta::minutes(minute),
|
||||
&["insights_group"],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',5)) where id=$1")
|
||||
.bind(id).bind(latency).execute(&database.pool).await.unwrap();
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
let repo = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
|
||||
for filter in [
|
||||
UsageRecordFilter::default(),
|
||||
UsageRecordFilter {
|
||||
search: Some("model_".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let expected = repo.usage_summary(range, filter.clone()).await.unwrap();
|
||||
if filter.search.is_some() {
|
||||
assert_eq!(
|
||||
expected
|
||||
.requests
|
||||
.latency_percentiles
|
||||
.p50_ms
|
||||
.unwrap()
|
||||
.as_f64(),
|
||||
1500.0
|
||||
);
|
||||
}
|
||||
for granularity in [
|
||||
gateway_admin::model::observability::Granularity::FifteenMinutes,
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
] {
|
||||
let observation = repo
|
||||
.usage_insights_observation(range, filter.clone(), granularity)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observation.overview.requests, expected.requests);
|
||||
assert_eq!(observation.overview.providers, expected.providers);
|
||||
assert_eq!(
|
||||
observation.overview.analytics.percentiles_approximate,
|
||||
expected.analytics.percentiles_approximate
|
||||
);
|
||||
assert_eq!(
|
||||
observation
|
||||
.trend
|
||||
.iter()
|
||||
.map(|point| point.metrics.request_count)
|
||||
.sum::<u64>(),
|
||||
4
|
||||
);
|
||||
assert!(observation.costs.iter().all(|point| point.standard_amount.is_none() && point.no_cache_amount.is_none()));
|
||||
}
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prefix_search_ignores_request_ids_and_matches_snapshot_names() {
|
||||
let Some(database) = TestDatabase::create("analytics_search_prefix").await else {
|
||||
return;
|
||||
};
|
||||
let start = Utc::now() - TimeDelta::hours(1);
|
||||
for id in ["literal_1", "literalX1", "snapshot_only"] {
|
||||
seed(&database.pool, id, start, &[]).await;
|
||||
}
|
||||
sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{request,clientApiKeyNameSnapshot}','\"literal_name\"'::jsonb) where id='snapshot_only'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let result = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(start - TimeDelta::seconds(1), start + TimeDelta::seconds(1))
|
||||
.unwrap(),
|
||||
UsageRecordFilter {
|
||||
search: Some("literal_".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.requests.request_count, 1);
|
||||
assert_eq!(result.requests.total_tokens, 120);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opaque_identifiers_preserve_long_values_and_response_or_counts_each_request_once() {
|
||||
let Some(database) = TestDatabase::create("analytics_opaque_identifiers").await else {
|
||||
@@ -169,7 +375,7 @@ async fn rolled_back_source_changes_and_late_commits_do_not_lose_queue_work() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn group_or_filter_counts_each_historical_request_once_and_rebuilds_dirty_buckets() {
|
||||
async fn dirty_buckets_replace_each_historical_request_contribution_once() {
|
||||
let Some(database) = TestDatabase::create("analytics_groups").await else {
|
||||
return;
|
||||
};
|
||||
@@ -194,13 +400,8 @@ async fn group_or_filter_counts_each_historical_request_once_and_rebuilds_dirty_
|
||||
let repo = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(start - TimeDelta::minutes(1), start + TimeDelta::hours(2))
|
||||
.unwrap();
|
||||
let filter = UsageRecordFilter {
|
||||
routing_group_refs: vec!["group_a".into(), "group_b".into()],
|
||||
..Default::default()
|
||||
};
|
||||
let result = repo.usage_summary(range, filter).await.unwrap();
|
||||
let result = repo.usage_summary(range, Default::default()).await.unwrap();
|
||||
assert_eq!(result.requests.request_count, 2);
|
||||
assert!(!result.analytics.percentiles_approximate);
|
||||
// 已投影但桶待重建时整个桶用事实替换,不能叠加旧桶贡献
|
||||
sqlx::query("update request_analytics_facts set total_tokens=333 where id='req_group_a'")
|
||||
.execute(&database.pool)
|
||||
@@ -453,6 +654,9 @@ async fn interrupted_quarters_restart_changed_versions_and_publish_one_consisten
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 独立重建请求改变版本,验证旧前缀废弃后仍可恢复,普通源变更不再重启整桶
|
||||
sqlx::query("update request_analytics_pending_buckets set change_version=nextval('request_analytics_change_version')")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drop(worker);
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
@@ -487,6 +691,8 @@ async fn source_updates_do_not_wait_for_blocked_bucket_rebuild() {
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),now())")
|
||||
.bind(start).execute(&database.pool).await.unwrap();
|
||||
let mut blocker = sqlx::PgConnection::connect_with(&database.pool.connect_options())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -677,7 +883,7 @@ async fn clean_aligned_statistics_do_not_access_facts_and_dirty_edges_are_disjoi
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.unwrap();
|
||||
let plan:serde_json::Value=sqlx::query_scalar("explain (analyze,buffers,format json) select fact.id from unnest($1::timestamptz[],$2::timestamptz[]) span(start_at,end_at) cross join lateral (select mr.id from request_analytics_facts mr where mr.started_at>=span.start_at and mr.started_at<span.end_at limit 100001) fact limit 100001")
|
||||
let plan:serde_json::Value=sqlx::query_scalar("explain (analyze,buffers,format json) select fact.id from unnest($1::timestamptz[],$2::timestamptz[]) span(start_at,end_at) cross join lateral (select mr.id from request_analytics_facts mr where mr.started_at>=span.start_at and mr.started_at<span.end_at offset 0) fact")
|
||||
.bind(vec![start+TimeDelta::minutes(5),start+TimeDelta::minutes(15)]).bind(vec![start+TimeDelta::minutes(15),start+TimeDelta::minutes(30)])
|
||||
.fetch_one(&mut *transaction).await.unwrap();
|
||||
let plan = plan.to_string();
|
||||
@@ -728,7 +934,7 @@ async fn persisted_histograms_merge_long_tails_and_distinguish_empty_from_zero()
|
||||
.usage_summary(
|
||||
range,
|
||||
UsageRecordFilter {
|
||||
routing_group_refs: vec!["hist_group".into()],
|
||||
search: Some("model_".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -1300,3 +1506,128 @@ async fn daily_rank_counts_follow_rebuilds_rollbacks_deletions_and_dirty_days()
|
||||
assert_eq!(dirty.items[0].request_count, 2);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn combined_filters_and_dirty_edges_aggregate_over_one_hundred_thousand_facts() {
|
||||
let Some(database) = TestDatabase::create("analytics_sql_fact_aggregation").await else {
|
||||
return;
|
||||
};
|
||||
let start = DateTime::parse_from_rfc3339("2026-09-01T10:01:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
seed(
|
||||
&database.pool,
|
||||
"aggregate_template",
|
||||
start,
|
||||
&["group_a", "group_b"],
|
||||
)
|
||||
.await;
|
||||
drain_analytics(&database.pool).await;
|
||||
// 派生事实扩充隔离查询的容量边界,请求模型和上游模型同时命中也只能计入一次
|
||||
sqlx::query("insert into request_analytics_facts select cloned.* from request_analytics_facts facts cross join generate_series(1,100005) n cross join lateral jsonb_populate_record(null::request_analytics_facts,to_jsonb(facts)||jsonb_build_object('id','aggregate_'||n::text,'upstream_model_id','model_a','latency_ms',n%97,'first_token_ms',n%31,'cost_currency','USD','cost_amount',0.0000000001,'cost_source','provider_reported')) cloned where facts.id='aggregate_template'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let range = ObservabilityRange::new(
|
||||
start - TimeDelta::minutes(1),
|
||||
start + TimeDelta::minutes(59),
|
||||
)
|
||||
.unwrap();
|
||||
let filter = UsageRecordFilter {
|
||||
transport: Some("http_sse".to_owned()),
|
||||
model: Some("model_a".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let repo = observability_repository(&database.pool);
|
||||
let result = repo.usage_summary(range, filter).await.unwrap();
|
||||
let expected: (i64, i64, f64, String) = sqlx::query_as("select count(*),sum(total_tokens)::bigint,percentile_cont(0.95) within group(order by latency_ms),trim_scale(sum(cost_amount))::text from request_analytics_facts where started_at >= $1 and started_at < $2")
|
||||
.bind(range.start).bind(range.end).fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(result.requests.request_count, expected.0 as u64);
|
||||
assert_eq!(result.requests.total_tokens, expected.1 as u64);
|
||||
assert_eq!(
|
||||
result.requests.latency_percentiles.p95_ms.unwrap().as_f64(),
|
||||
expected.2
|
||||
);
|
||||
assert!(!result.analytics.percentiles_approximate);
|
||||
// 同一个热点桶处于重建阶段时,不得使用旧汇总或只读取前十万条
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values(date_bin(interval '15 minutes',$1::timestamptz,timestamptz '2000-01-01 UTC'),now())")
|
||||
.bind(start).execute(&database.pool).await.unwrap();
|
||||
let dirty = repo.usage_summary(range, Default::default()).await.unwrap();
|
||||
assert_eq!(dirty.requests.request_count, expected.0 as u64);
|
||||
assert_eq!(dirty.requests.total_tokens, expected.1 as u64);
|
||||
assert_eq!(dirty.requests.latency_count, expected.0 as u64);
|
||||
assert!(dirty.analytics.percentiles_approximate);
|
||||
let ranked = repo
|
||||
.usage_diagnostics(range, Default::default(), DiagnosticDimension::ApiKey, 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ranked.total_request_count, expected.0 as u64);
|
||||
assert_eq!(ranked.items[0].request_count, expected.0 as u64);
|
||||
assert_eq!(ranked.items[0].costs[0].amount.to_string(), expected.3);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn high_cardinality_summary_merges_all_rows_before_returning_model_groups() {
|
||||
let Some(database) = TestDatabase::create("analytics_sql_summary_aggregation").await else {
|
||||
return;
|
||||
};
|
||||
drain_analytics(&database.pool).await;
|
||||
let start = DateTime::parse_from_rfc3339("2026-09-01T00:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
let versions: (i32,i32) = sqlx::query_as("select schema_version,histogram_version from request_analytics_build_state where singleton=true")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
// 150 个模型乘以 744 小时超过旧汇总行上限,输出仍只有 150 个模型组
|
||||
sqlx::query("insert into request_analytics_hourly(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) select $1::timestamptz+hour*interval '1 hour','provider_model',array['openai','model_'||lpad(model::text,3,'0'),'model_'||lpad(model::text,3,'0')],'succeeded',$2,$3,jsonb_build_object('request_count',model,'total_tokens',model*120),jsonb_build_object('USD',(model::numeric*0.0000000001)::text),'{}' from generate_series(0,743) hour cross join generate_series(1,150) model")
|
||||
.bind(start).bind(versions.0).bind(versions.1).execute(&database.pool).await.unwrap();
|
||||
let result = observability_repository(&database.pool)
|
||||
.usage_diagnostics(
|
||||
ObservabilityRange::new(start, start + TimeDelta::days(31)).unwrap(),
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::Model,
|
||||
2,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.total_request_count, 744 * 150 * 151 / 2);
|
||||
assert_eq!(result.items.len(), 2);
|
||||
assert_eq!(result.items[0].key, "model_150");
|
||||
assert_eq!(result.items[0].request_count, 744 * 150);
|
||||
assert_eq!(result.items[0].total_tokens, 744 * 150 * 120);
|
||||
assert_eq!(result.items[0].costs[0].amount.to_string(), "0.00001116");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sql_edge_histograms_match_persisted_power_of_two_boundaries() {
|
||||
let Some(database) = TestDatabase::create("analytics_sql_histogram_boundaries").await else {
|
||||
return;
|
||||
};
|
||||
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
for (index, value) in [0_i64, 1, 2, 4, 8, 16, 31, 32, 33, 64, 128, 1024, 1_048_576]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let id = format!("sql_hist_{index}");
|
||||
seed(
|
||||
&database.pool,
|
||||
&id,
|
||||
start + TimeDelta::seconds(index as i64),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("update model_requests set request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',0)) where id=$1")
|
||||
.bind(id).bind(value).execute(&database.pool).await.unwrap();
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
let repo = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
|
||||
let persisted = repo.usage_summary(range, Default::default()).await.unwrap();
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) values($1,now())")
|
||||
.bind(start).execute(&database.pool).await.unwrap();
|
||||
let edge = repo.usage_summary(range, Default::default()).await.unwrap();
|
||||
assert_eq!(edge.requests, persisted.requests);
|
||||
assert_eq!(edge.attempts, persisted.attempts);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -42,6 +42,9 @@ async fn dirty_quarters_share_one_hour_rebuild_and_acknowledge_each_bucket() {
|
||||
sqlx::raw_sql("create table hourly_build_audit (bucket_start timestamptz); create function audit_hourly_build() returns trigger language plpgsql as $$begin if new.collection='global' then insert into hourly_build_audit values(new.bucket_start); end if; return new; end$$; create trigger audit_hourly_build after insert on request_analytics_hourly for each row execute function audit_hourly_build()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
seed_hour(&database.pool, hour(), "coalesced").await;
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::raw_sql("truncate hourly_build_audit; insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
let mut acknowledged = 0;
|
||||
@@ -85,6 +88,9 @@ async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuil
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 显式修复任务重建整桶,普通更新只维护当前贡献,不重新启动构建
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
let mut owner = None;
|
||||
@@ -100,10 +106,12 @@ async fn changed_quarter_invalidates_a_partial_hour_before_its_inputs_are_rebuil
|
||||
let changed: String = sqlx::query_scalar("select id from model_requests where date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC') <> $1 order by id limit 1")
|
||||
.bind(owner).fetch_one(&database.pool).await.unwrap();
|
||||
sqlx::query("update model_requests set total_tokens=9999 where id=$1")
|
||||
.bind(changed)
|
||||
.bind(&changed)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from model_requests where id=$1 on conflict(bucket_start) do update set change_version=nextval('request_analytics_change_version')")
|
||||
.bind(changed).execute(&database.pool).await.unwrap();
|
||||
worker.maintain(2).await.unwrap();
|
||||
let phase: i16 = sqlx::query_scalar(
|
||||
"select build_phase from request_analytics_pending_buckets where bucket_start=$1",
|
||||
@@ -174,3 +182,174 @@ async fn waiting_hours_cannot_fill_all_slots_and_starve_their_unbuilt_quarters()
|
||||
assert_eq!(result.requests.total_tokens, 18 * 120);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuous_updates_do_not_restart_partial_quarter_and_hour_builds() {
|
||||
let Some(database) = TestDatabase::create("analytics_continuous_stage_updates").await else {
|
||||
return;
|
||||
};
|
||||
for quarter in 0..2 {
|
||||
for name in ["A", "a", "Z", "z", "Ä", "ä", "中", "é", "e", "_", "-", "2"] {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("prefix_{name}_{quarter}"),
|
||||
hour() + TimeDelta::minutes(quarter * 15 + 1),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let mut quarter_prefix_seen = false;
|
||||
let mut hour_prefix_seen = false;
|
||||
let mut completed = false;
|
||||
for round in 0..150_i64 {
|
||||
// 同时间的大小写及非 ASCII ID、改变排序位置的账号维度同时覆盖两种前缀
|
||||
sqlx::query("update model_requests set total_tokens=$1,provider_account_ref=$2 where id in ('prefix_A_0','prefix_ä_1')")
|
||||
.bind(200+round).bind(if round%2==0 {"account_Ä"}else{"account_Z"})
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
worker.maintain(2).await.unwrap();
|
||||
let phases: (bool,bool,i64) = sqlx::query_as("select coalesce(bool_or(build_phase=1 and build_id is not null),false),coalesce(bool_or(build_phase=4 and build_collection is not null),false),count(*) from request_analytics_pending_buckets")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
quarter_prefix_seen |= phases.0;
|
||||
hour_prefix_seen |= phases.1;
|
||||
if phases.2 == 0 {
|
||||
completed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
quarter_prefix_seen && hour_prefix_seen,
|
||||
"both persisted prefixes must be exercised"
|
||||
);
|
||||
assert!(
|
||||
completed,
|
||||
"ongoing updates cannot repeatedly reset a partial rebuild"
|
||||
);
|
||||
drain_analytics(&database.pool).await;
|
||||
let expected: (i64, i64) =
|
||||
sqlx::query_as("select count(*),sum(total_tokens)::bigint from model_requests")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let actual = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.request_count, expected.0 as u64);
|
||||
assert_eq!(actual.requests.total_tokens, expected.1 as u64);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_deletions_repair_extrema_last_usage_and_zero_currency_presence() {
|
||||
let Some(database) = TestDatabase::create("analytics_incremental_extrema").await else {
|
||||
return;
|
||||
};
|
||||
for number in 1..=4_i64 {
|
||||
let id = format!("extrema_{number}");
|
||||
seed(
|
||||
&database.pool,
|
||||
&id,
|
||||
hour() + TimeDelta::minutes(number),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("update model_requests set provider_account_ref='extrema_account',request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',$2::bigint)),cost_source=case when $2<=2 then 'provider_reported' else 'unavailable' end,cost_currency=case $2 when 1 then 'EUR' when 2 then 'USD' end,cost_amount=case $2 when 1 then 1::numeric when 2 then 0::numeric end where id=$1")
|
||||
.bind(id).bind(number).execute(&database.pool).await.unwrap();
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::query("delete from model_requests where id in ('extrema_1','extrema_4')")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let row: (i64,f64,f64,DateTime<Utc>,serde_json::Value) = sqlx::query_as("select (counts->>'request_count')::bigint,(distributions->'latency_ms'->>'minimum')::double precision,(distributions->'latency_ms'->>'maximum')::double precision,last_used_at,costs from request_analytics_hourly where collection='global'")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(
|
||||
(row.0, row.1, row.2, row.3),
|
||||
(2, 2.0, 3.0, hour() + TimeDelta::minutes(3))
|
||||
);
|
||||
assert_eq!(row.4.as_object().unwrap().len(), 1);
|
||||
assert_eq!(
|
||||
row.4["USD"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<gateway_core::metering::Decimal>()
|
||||
.unwrap(),
|
||||
gateway_core::metering::Decimal::ZERO
|
||||
);
|
||||
sqlx::query("update model_requests set cost_source='unavailable',cost_currency=null,cost_amount=null where id='extrema_2'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let costs: serde_json::Value =
|
||||
sqlx::query_scalar("select costs from request_analytics_hourly where collection='global'")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(costs, serde_json::json!({}));
|
||||
sqlx::query("select enqueue_request_analytics('extrema_3',$1)")
|
||||
.bind(hour() + TimeDelta::minutes(3))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let actual = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.request_count, 2);
|
||||
assert_eq!(actual.requests.total_tokens, 240);
|
||||
assert!(actual.attempts.costs.is_empty());
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_summary_write_failure_rolls_back_facts_and_preserves_queue() {
|
||||
let Some(database) = TestDatabase::create("analytics_incremental_rollback").await else {
|
||||
return;
|
||||
};
|
||||
seed(
|
||||
&database.pool,
|
||||
"atomic_incremental",
|
||||
hour() + TimeDelta::minutes(1),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::raw_sql("create function reject_incremental_summary() returns trigger language plpgsql as $$begin raise exception 'test summary write failure'; end$$; create trigger reject_incremental_summary before insert or update on request_analytics_15m for each row execute function reject_incremental_summary()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
sqlx::query("update model_requests set total_tokens=240 where id='atomic_incremental'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
assert!(worker.maintain(10).await.is_err());
|
||||
let state:(i64,i64,i64)=sqlx::query_as("select (select total_tokens from request_analytics_facts where id='atomic_incremental'),(select (counts->>'total_tokens')::bigint from request_analytics_15m where collection='global'),(select count(*) from request_analytics_pending_requests)")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(state, (120, 120, 1));
|
||||
sqlx::raw_sql("drop trigger reject_incremental_summary on request_analytics_15m; drop function reject_incremental_summary()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let actual = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(hour(), hour() + TimeDelta::hours(1)).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.total_tokens, 240);
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ pub(super) struct TestDatabase {
|
||||
}
|
||||
|
||||
pub(super) fn observability_query_budget() -> ObservabilityQueryBudget {
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1))
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1), Duration::from_secs(10))
|
||||
.expect("valid test observability query budget")
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
//! 十万条以上账号与 Key 用量的完整性、未知值和汇总读取验收
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{
|
||||
ObservabilityQueryBudget, ObservabilityRange, PgObservabilityRepository,
|
||||
PgRequestAnalyticsRepository, ProviderAccountUsageQuery, UsageRecordFilter,
|
||||
};
|
||||
use sqlx::Connection;
|
||||
|
||||
use super::super::super::{TestDatabase, analytics::seed};
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_account_and_key_usage_keep_exact_totals_and_read_clean_rollups() {
|
||||
let Some(database) = TestDatabase::create("large_account_usage").await else {
|
||||
return;
|
||||
};
|
||||
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
sqlx::query("insert into provider_accounts(id,provider_kind,name,authentication_kind,provider_credentials_json,credential_revision,has_refresh_token,credential_observed_at,created_at,updated_at) values('account_large','openai','large','oauth','{}',1,false,$1,$1,$1)")
|
||||
.bind(start).execute(&database.pool).await.unwrap();
|
||||
seed(
|
||||
&database.pool,
|
||||
"large_template",
|
||||
start + TimeDelta::minutes(1),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("update model_requests set provider_kind='openai',provider_account_ref='account_large',cached_input_tokens=0,reasoning_output_tokens=null where id='large_template'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
// 扩充真实源表并经过生产 Worker,确保回归覆盖账号接口而非只验证派生表
|
||||
sqlx::query("insert into model_requests select cloned.* from model_requests template cross join generate_series(1,100005) n cross join lateral jsonb_populate_record(null::model_requests,to_jsonb(template)||jsonb_build_object('id','account_large_'||n::text)) cloned where template.id='large_template'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(10), Duration::from_secs(10))
|
||||
.unwrap();
|
||||
let worker = PgRequestAnalyticsRepository::new(database.pool.clone(), budget.clone());
|
||||
let mut ready = false;
|
||||
for _ in 0..400 {
|
||||
worker.maintain(1000).await.unwrap();
|
||||
let status = worker.status().await.unwrap();
|
||||
if !status.building && status.pending_requests == 0 && status.pending_buckets == 0 {
|
||||
ready = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
ready,
|
||||
"large source dataset must finish projection and rollups"
|
||||
);
|
||||
let repo = PgObservabilityRepository::new(database.pool.clone(), None, budget, None);
|
||||
let range = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
|
||||
let query =
|
||||
ProviderAccountUsageQuery::for_accounts(range, vec!["account_large".to_owned()]).unwrap();
|
||||
let mut blocker = sqlx::PgConnection::connect_with(&database.pool.connect_options())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::raw_sql("begin;lock table request_analytics_facts in access exclusive mode")
|
||||
.execute(&mut blocker)
|
||||
.await
|
||||
.unwrap();
|
||||
let account = tokio::time::timeout(Duration::from_secs(2), repo.provider_account_usage(query))
|
||||
.await
|
||||
.expect("clean account usage must not read facts")
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
assert_eq!(account.request_count, 100006);
|
||||
assert_eq!(account.total_tokens, Some(100006 * 120));
|
||||
assert_eq!(account.cached_input_tokens, Some(0));
|
||||
assert_eq!(account.reasoning_output_tokens, None);
|
||||
assert_eq!(account.models.len(), 1);
|
||||
assert_eq!(account.models[0].request_count, 100006);
|
||||
assert_eq!(account.last_used_at, Some(start + TimeDelta::minutes(1)));
|
||||
let filter = UsageRecordFilter {
|
||||
client_api_key_ref: Some("key_analytics".to_owned()),
|
||||
model: Some("model_a".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let key = tokio::time::timeout(Duration::from_secs(2), repo.usage_summary(range, filter))
|
||||
.await
|
||||
.expect("Key plus model must use joint rollups")
|
||||
.unwrap();
|
||||
assert_eq!(key.requests.request_count, 100006);
|
||||
assert_eq!(key.providers.len(), 1);
|
||||
assert_eq!(key.providers[0].request_count, 100006);
|
||||
sqlx::raw_sql("rollback")
|
||||
.execute(&mut blocker)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(blocker);
|
||||
let partial = ObservabilityRange::new(
|
||||
start + TimeDelta::seconds(30),
|
||||
start + TimeDelta::minutes(2),
|
||||
)
|
||||
.unwrap();
|
||||
let account = repo
|
||||
.provider_account_usage(
|
||||
ProviderAccountUsageQuery::for_accounts(partial, vec!["account_large".to_owned()])
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
assert_eq!(
|
||||
account.request_count, 100006,
|
||||
"the boundary aggregation must not truncate a hot bucket"
|
||||
);
|
||||
assert_eq!(account.total_tokens, Some(100006 * 120));
|
||||
database.close().await;
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
//! 汇总读取、边界聚合与账号联合维度的真实数据库回归
|
||||
|
||||
mod accounts;
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 观测存储测试入口,以及查询范围、过滤与端口合同测试
|
||||
|
||||
mod analytics;
|
||||
mod projection;
|
||||
|
||||
use std::{
|
||||
@@ -441,7 +442,6 @@ async fn usage_list_and_diagnostics_preserve_account_snapshots_by_id() {
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
range,
|
||||
filter: admin_observability::UsageFilter {
|
||||
routing_group_refs: Vec::new(),
|
||||
provider_account_ref: Some("acct_team".to_owned()),
|
||||
..admin_observability::UsageFilter::default()
|
||||
},
|
||||
@@ -504,7 +504,7 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
search: Some("observe_success".to_owned()),
|
||||
search: Some("rimary".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
current_page: 1,
|
||||
@@ -517,6 +517,51 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_search_should_ignore_request_ids_and_preserve_exact_filters() {
|
||||
let Some(database) = TestDatabase::create("usage_search_without_request_ids").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now)
|
||||
.await
|
||||
.expect("seed observability facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
|
||||
for search in [
|
||||
"req_observe",
|
||||
"req_observe_success",
|
||||
"upstream_req",
|
||||
"upstream_req_success",
|
||||
] {
|
||||
assert_usage_search_ids(&database.pool, range, search, &[]).await;
|
||||
}
|
||||
for filter in [
|
||||
UsageRecordFilter {
|
||||
request_id: Some("req_observe_success".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
UsageRecordFilter {
|
||||
upstream_request_id: Some("upstream_req_success".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let page = observability_repository(&database.pool)
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("exact request identity filter");
|
||||
assert_eq!(page.total, 1);
|
||||
assert_eq!(page.items[0].id, "req_observe_success");
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_search_should_match_account_email_and_name_prefixes() {
|
||||
let Some(database) = TestDatabase::create("usage_account_prefix").await else {
|
||||
@@ -1687,7 +1732,6 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
range,
|
||||
filter: admin_observability::UsageFilter {
|
||||
routing_group_refs: Vec::new(),
|
||||
client_api_key_ref: Some("key_observe".to_owned()),
|
||||
request_id: Some("req_observe_success".to_owned()),
|
||||
provider_account_ref: Some("acct_observe".to_owned()),
|
||||
@@ -1700,7 +1744,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
attempt_index: Some(1),
|
||||
response_id: Some("resp_observe_success".to_owned()),
|
||||
upstream_request_id: Some("upstream_req_success".to_owned()),
|
||||
search: Some("req_observe_success".to_owned()),
|
||||
search: Some("upstream-model".to_owned()),
|
||||
},
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
@@ -1716,7 +1760,6 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
range,
|
||||
filter: admin_observability::UsageFilter {
|
||||
routing_group_refs: Vec::new(),
|
||||
outcome: Some(
|
||||
admin_observability::RequestOutcome::new("provider_future_state")
|
||||
.expect("bounded other outcome"),
|
||||
|
||||
@@ -11,6 +11,7 @@ mod authorization;
|
||||
mod deletion;
|
||||
mod quota_forecast;
|
||||
mod timestamps;
|
||||
mod usage_projection;
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_admin::{
|
||||
|
||||
@@ -9,6 +9,66 @@ use gateway_store::postgres::{
|
||||
UsageRecordQuery,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_forecast_fractional_slots_keep_microsecond_edges_and_as_of_ties() {
|
||||
let Some(database) = TestDatabase::create("forecast_fractional_slots").await else {
|
||||
return;
|
||||
};
|
||||
PgProviderAccountRepository::new(database.pool.clone())
|
||||
.insert_provider_account(account("acct_fraction", "user-fraction"))
|
||||
.await
|
||||
.unwrap();
|
||||
let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
let duration = MAX_FORECAST_HISTORY_POINTS as i64 * 1_000_000 + 64;
|
||||
let end = start + TimeDelta::microseconds(duration);
|
||||
for (id, completed) in [
|
||||
("before_edge", 1_000_000),
|
||||
("after_edge", 1_000_001),
|
||||
("as_of_a", duration),
|
||||
("as_of_b", duration),
|
||||
("pending", duration + 1),
|
||||
] {
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
ModelRequestSeed {
|
||||
request_id: id,
|
||||
account_id: "acct_fraction",
|
||||
provider_kind: "openai",
|
||||
model: "gpt-test",
|
||||
total_tokens: 100,
|
||||
cost_amount: "1",
|
||||
started_at: start,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update model_requests set completed_at=$2,provider_observation_json='{}'::jsonb where id=$1")
|
||||
.bind(id).bind(start+TimeDelta::microseconds(completed)).execute(&database.pool).await.unwrap();
|
||||
}
|
||||
let history = admin_account_store(&database.pool)
|
||||
.load_quota_forecast_history(&AccountUsageWindowQuery {
|
||||
account_id: "acct_fraction".to_owned(),
|
||||
key: "fraction".to_owned(),
|
||||
range: TimeRange { start, end },
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
history
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| point.usage.request_count)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![1, 2, 4]
|
||||
);
|
||||
assert_eq!(history.usage.request_count, 4);
|
||||
assert_eq!(history.usage.tokens, 400);
|
||||
assert_eq!(history.pending_request_count, 1);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_forecast_excludes_openai_prewarm_but_preserves_inference_and_audit() {
|
||||
let Some(database) = TestDatabase::create("forecast_prewarm").await else {
|
||||
@@ -341,3 +401,219 @@ async fn quota_forecast_bounds_documents_without_sampling_away_token_totals() {
|
||||
assert_eq!(history.points.last().unwrap().usage.tokens, 25_600);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_forecast_carries_unsampled_usage_between_document_points() {
|
||||
let Some(database) = TestDatabase::create("quota_forecast_bucket_prefix").await else {
|
||||
return;
|
||||
};
|
||||
PgProviderAccountRepository::new(database.pool.clone())
|
||||
.insert_provider_account(account("acct_prefix", "user-prefix"))
|
||||
.await
|
||||
.unwrap();
|
||||
let start = Utc::now() - TimeDelta::hours(6);
|
||||
let end = start + TimeDelta::minutes(MAX_FORECAST_HISTORY_POINTS as i64);
|
||||
for (id, second, document) in [
|
||||
("before_doc", 10, false),
|
||||
("first_doc", 20, true),
|
||||
("tie_doc", 20, true),
|
||||
("after_doc", 40, false),
|
||||
("no_document_bucket", 70, false),
|
||||
("third_doc", 130, true),
|
||||
("after_last_doc", 150, false),
|
||||
("late", 180, true),
|
||||
] {
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
ModelRequestSeed {
|
||||
request_id: id,
|
||||
account_id: "acct_prefix",
|
||||
provider_kind: "openai",
|
||||
model: "gpt-test",
|
||||
total_tokens: 100,
|
||||
cost_amount: "1",
|
||||
started_at: start + TimeDelta::seconds(second),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
if document {
|
||||
sqlx::query(
|
||||
"update model_requests set provider_observation_json = '{}'::jsonb where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
sqlx::query("update model_requests set completed_at = $1 where id = 'late'")
|
||||
.bind(end + TimeDelta::seconds(1))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let history = admin_account_store(&database.pool)
|
||||
.load_quota_forecast_history(&AccountUsageWindowQuery {
|
||||
account_id: "acct_prefix".to_owned(),
|
||||
key: "week".to_owned(),
|
||||
range: TimeRange { start, end },
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(history.points.len(), 2);
|
||||
assert_eq!(history.points[0].usage.request_count, 3);
|
||||
assert_eq!(history.points[0].usage.tokens, 300);
|
||||
assert_eq!(history.points[1].usage.request_count, 6);
|
||||
assert_eq!(history.points[1].usage.tokens, 600);
|
||||
assert_eq!(history.usage.request_count, 7);
|
||||
assert_eq!(history.usage.tokens, 700);
|
||||
assert_eq!(history.pending_request_count, 1);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_forecast_summaries_match_source_during_moves_deletes_and_rebuilds() {
|
||||
use super::super::drain_analytics;
|
||||
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistory;
|
||||
|
||||
fn assert_same(actual: &QuotaForecastHistory, expected: &QuotaForecastHistory) {
|
||||
assert_eq!(actual.usage, expected.usage);
|
||||
assert_eq!(actual.pending_request_count, expected.pending_request_count);
|
||||
assert_eq!(actual.points.len(), expected.points.len());
|
||||
for (actual, expected) in actual.points.iter().zip(&expected.points) {
|
||||
assert_eq!(actual.started_at, expected.started_at);
|
||||
assert_eq!(actual.completed_at, expected.completed_at);
|
||||
assert_eq!(actual.usage, expected.usage);
|
||||
}
|
||||
}
|
||||
|
||||
let Some(database) = TestDatabase::create("forecast_completion_projection").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
for id in ["acct_forecast_projection", "acct_forecast_moved"] {
|
||||
repository
|
||||
.insert_provider_account(account(id, id))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T00:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
for index in 0..200 {
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
ModelRequestSeed {
|
||||
request_id: &format!("forecast_projection_{index}"),
|
||||
account_id: "acct_forecast_projection",
|
||||
provider_kind: "openai",
|
||||
model: "gpt-test",
|
||||
total_tokens: index + 10,
|
||||
cost_amount: "0.125",
|
||||
started_at: start + TimeDelta::minutes(index * 3),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
sqlx::query("update model_requests set provider_observation_json='{}'::jsonb where substring(id from '[0-9]+$')::int % 9=0")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
// 完成时间与开始时间跨越多个桶,包含未来完成、缺失 Token、未知费用与恢复记录
|
||||
sqlx::query("update model_requests set completed_at=completed_at+interval '90 minutes' where substring(id from '[0-9]+$')::int % 7=0")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
sqlx::query("update model_requests set total_tokens=null,input_tokens=null,cost_source='unavailable',cost_amount=null,cost_currency=null where id='forecast_projection_31'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
sqlx::query(
|
||||
"update model_requests set recovered_at=completed_at,recovery_request_id='forecast_projection_48',recovery_attempt_count=1,request_observation_json=request_observation_json || '{\"recovery\":{\"retryDelayMs\":0,\"totalLatencyMs\":1}}'::jsonb where id='forecast_projection_47'",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update model_requests set outcome='cancelled' where id='forecast_projection_63'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update model_requests set cost_currency='EUR' where id='forecast_projection_83'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let store = admin_account_store(&database.pool);
|
||||
let queries = [
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_forecast_projection".to_owned(),
|
||||
key: "complete".to_owned(),
|
||||
range: TimeRange {
|
||||
start,
|
||||
end: start + TimeDelta::hours(10),
|
||||
},
|
||||
},
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_forecast_projection".to_owned(),
|
||||
key: "edges".to_owned(),
|
||||
range: TimeRange {
|
||||
start: start + TimeDelta::microseconds(1),
|
||||
end: start + TimeDelta::minutes(593) + TimeDelta::microseconds(1),
|
||||
},
|
||||
},
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_forecast_moved".to_owned(),
|
||||
key: "moved".to_owned(),
|
||||
range: TimeRange {
|
||||
start,
|
||||
end: start + TimeDelta::hours(10),
|
||||
},
|
||||
},
|
||||
];
|
||||
let mut source = Vec::new();
|
||||
for query in &queries {
|
||||
source.push(store.load_quota_forecast_history(query).await.unwrap());
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
for (query, expected) in queries.iter().zip(&source) {
|
||||
assert_same(
|
||||
&store.load_quota_forecast_history(query).await.unwrap(),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
sqlx::query("update model_requests set provider_account_ref='acct_forecast_moved',started_at=$1,completed_at=$1+interval '1 second',total_tokens=1234,cost_amount=2.5 where id='forecast_projection_72'")
|
||||
.bind(start+TimeDelta::minutes(40)).execute(&database.pool).await.unwrap();
|
||||
sqlx::query("delete from model_requests where id='forecast_projection_47'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update model_requests set completed_at=$1 where id='forecast_projection_196'")
|
||||
.bind(start + TimeDelta::minutes(590))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut pending = Vec::new();
|
||||
for query in &queries {
|
||||
pending.push(store.load_quota_forecast_history(query).await.unwrap());
|
||||
}
|
||||
assert_eq!(pending[2].usage.request_count, 1);
|
||||
assert_eq!(pending[2].usage.tokens, 1234);
|
||||
drain_analytics(&database.pool).await;
|
||||
for (query, expected) in queries.iter().zip(&pending) {
|
||||
assert_same(
|
||||
&store.load_quota_forecast_history(query).await.unwrap(),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
// 显式重建中的区间由窄事实接管,发布后仍与同一源结果一致
|
||||
sqlx::query("insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at) select distinct date_bin(interval '15 minutes',started_at,timestamptz '2000-01-01 UTC'),now() from request_analytics_facts")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
for (query, expected) in queries.iter().zip(&pending) {
|
||||
assert_same(
|
||||
&store.load_quota_forecast_history(query).await.unwrap(),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
for (query, expected) in queries.iter().zip(&pending) {
|
||||
assert_same(
|
||||
&store.load_quota_forecast_history(query).await.unwrap(),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
//! 验证账号同步用量在汇总、待投影修改和删除之间保持源事实一致
|
||||
|
||||
use super::super::drain_analytics;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_windows_reconcile_moved_requests_and_pending_deletions_without_waiting_for_worker() {
|
||||
let Some(database) = TestDatabase::create("account_window_source_reconciliation").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
for id in ["acct_usage_before", "acct_usage_after"] {
|
||||
repository
|
||||
.insert_provider_account(account(id, id))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let start = chrono::DateTime::from_timestamp(Utc::now().timestamp().div_euclid(3600) * 3600, 0)
|
||||
.unwrap()
|
||||
- TimeDelta::hours(2);
|
||||
for (id, offset, tokens, cost) in [("usage_stays", 5, 11, "0"), ("usage_moves", 40, 23, "2.5")]
|
||||
{
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
ModelRequestSeed {
|
||||
request_id: id,
|
||||
account_id: "acct_usage_before",
|
||||
provider_kind: "openai",
|
||||
model: "old-model",
|
||||
total_tokens: tokens,
|
||||
cost_amount: cost,
|
||||
started_at: start + TimeDelta::minutes(offset),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
sqlx::query("update model_requests set cached_input_tokens=null,reasoning_output_tokens=null where id in ('usage_stays','usage_moves')")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let windows = [
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_usage_before".to_owned(),
|
||||
key: "old".to_owned(),
|
||||
range: TimeRange {
|
||||
start,
|
||||
end: start + TimeDelta::hours(1),
|
||||
},
|
||||
},
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_usage_after".to_owned(),
|
||||
key: "new".to_owned(),
|
||||
range: TimeRange {
|
||||
start: start + TimeDelta::hours(1),
|
||||
end: start + TimeDelta::hours(2),
|
||||
},
|
||||
},
|
||||
];
|
||||
let store = admin_account_store(&database.pool);
|
||||
let before = store.load_account_usage_by_windows(&windows).await.unwrap();
|
||||
let initial = before.iter().find(|usage| usage.key == "old").unwrap();
|
||||
assert_eq!(initial.usage.total_tokens, Some(34));
|
||||
assert_eq!(initial.usage.cached_input_tokens, None);
|
||||
assert_eq!(initial.usage.costs[0].amount.as_str(), "2.5");
|
||||
let ranked = store
|
||||
.list_accounts(
|
||||
AccountListQuery {
|
||||
page: 1,
|
||||
page_size: PageSize::new(1).unwrap(),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: Some(AccountSort {
|
||||
field: AccountSortField::Usage,
|
||||
direction: SortDirection::Desc,
|
||||
}),
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ranked.items[0].account.id, "acct_usage_before");
|
||||
|
||||
sqlx::query("update model_requests set provider_account_ref='acct_usage_after',upstream_model_id='new-model',total_tokens=null,input_tokens=7,output_tokens=8,cached_input_tokens=0,cost_amount=4.75,started_at=$1,completed_at=$1+interval '1 minute',downstream_committed_at=$1+interval '1 minute',recovery_lease_expires_at=$1+interval '2 minutes' where id='usage_moves'")
|
||||
.bind(start+TimeDelta::minutes(70)).execute(&database.pool).await.unwrap();
|
||||
let changed = store.load_account_usage_by_windows(&windows).await.unwrap();
|
||||
let old = changed.iter().find(|usage| usage.key == "old").unwrap();
|
||||
let new = changed.iter().find(|usage| usage.key == "new").unwrap();
|
||||
assert_eq!(old.usage.request_count, 1);
|
||||
assert_eq!(old.usage.total_tokens, Some(11));
|
||||
assert_eq!(old.usage.last_used_at, Some(start + TimeDelta::minutes(5)));
|
||||
assert_eq!(
|
||||
old.usage.costs[0].amount.as_str(),
|
||||
"0",
|
||||
"a real zero-cost request retains its currency"
|
||||
);
|
||||
assert_eq!(new.usage.request_count, 1);
|
||||
assert_eq!(
|
||||
new.usage.total_tokens,
|
||||
Some(15),
|
||||
"quota windows retain component fallback for missing provider total"
|
||||
);
|
||||
assert_eq!(new.usage.cached_input_tokens, Some(0));
|
||||
assert_eq!(new.usage.reasoning_output_tokens, None);
|
||||
assert_eq!(new.usage.models[0].model, "new-model");
|
||||
assert_eq!(new.usage.costs[0].amount.as_str(), "4.75");
|
||||
for field in [AccountSortField::Usage, AccountSortField::LastUsedAt] {
|
||||
let ranked = store
|
||||
.list_accounts(
|
||||
AccountListQuery {
|
||||
page: 1,
|
||||
page_size: PageSize::new(1).unwrap(),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: Some(AccountSort {
|
||||
field,
|
||||
direction: SortDirection::Desc,
|
||||
}),
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ranked.items[0].account.id, "acct_usage_after");
|
||||
}
|
||||
|
||||
// 独立源表真值不经过同一汇总代码,防止仅比较两个相同错误的读取分支
|
||||
let truth:(i64,i64,String)=sqlx::query_as("select count(*)::bigint,sum(coalesce(total_tokens,input_tokens+output_tokens))::bigint,sum(cost_amount)::text from model_requests where provider_account_ref='acct_usage_after' and started_at >= $1 and started_at < $2")
|
||||
.bind(windows[1].range.start).bind(windows[1].range.end).fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(truth.0, new.usage.request_count as i64);
|
||||
assert_eq!(truth.1, new.usage.total_tokens.unwrap() as i64);
|
||||
assert_eq!(
|
||||
truth.2.parse::<gateway_store::DecimalAmount>().unwrap(),
|
||||
new.usage.costs[0].amount
|
||||
);
|
||||
sqlx::query("delete from model_requests where id='usage_stays'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let deleted = store.load_account_usage_by_windows(&windows).await.unwrap();
|
||||
let old = deleted.iter().find(|usage| usage.key == "old").unwrap();
|
||||
assert_eq!(old.usage.request_count, 0);
|
||||
assert_eq!(old.usage.last_used_at, None);
|
||||
assert!(old.usage.models.is_empty());
|
||||
assert!(old.usage.costs.is_empty());
|
||||
database.close().await;
|
||||
}
|
||||
@@ -19,7 +19,10 @@ async fn test_pool() -> Option<PgPool> {
|
||||
|
||||
#[test]
|
||||
fn query_budget_rejects_an_empty_budget() {
|
||||
assert!(ObservabilityQueryBudget::try_new(0, Duration::from_secs(1)).is_err());
|
||||
assert!(
|
||||
ObservabilityQueryBudget::try_new(0, Duration::from_secs(1), Duration::from_secs(10))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -27,7 +30,9 @@ async fn running_query_deadline_releases_the_slot() {
|
||||
let Some(pool) = test_pool().await else {
|
||||
return;
|
||||
};
|
||||
let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(10)).unwrap();
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(1, Duration::from_millis(10), Duration::from_millis(250))
|
||||
.unwrap();
|
||||
let error = budget
|
||||
.run(&pool, "never completes", |connection| async move {
|
||||
let result = std::future::pending::<Result<(), StoreError>>().await;
|
||||
@@ -46,13 +51,77 @@ async fn running_query_deadline_releases_the_slot() {
|
||||
pool.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_time_is_included_in_the_operation_deadline() {
|
||||
let Some(pool) = test_pool().await else {
|
||||
return;
|
||||
};
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(1, Duration::from_secs(2), Duration::from_secs(2))
|
||||
.unwrap();
|
||||
let (occupied_tx, occupied_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let occupied_budget = budget.clone();
|
||||
let occupied_pool = pool.clone();
|
||||
let occupied = tokio::spawn(async move {
|
||||
occupied_budget
|
||||
.run(
|
||||
&occupied_pool,
|
||||
"hold before queued deadline",
|
||||
|connection| async move {
|
||||
occupied_tx.send(()).unwrap();
|
||||
release_rx.await.unwrap();
|
||||
drop(connection);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
occupied_rx.await.unwrap();
|
||||
let started = tokio::time::Instant::now();
|
||||
let release = tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(2) / 5).await;
|
||||
release_tx.send(()).unwrap();
|
||||
});
|
||||
let error = budget
|
||||
.run(
|
||||
&pool,
|
||||
"queued operation deadline",
|
||||
|connection| async move {
|
||||
let result = std::future::pending::<Result<(), StoreError>>().await;
|
||||
drop(connection);
|
||||
result
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("queue and execution share one deadline");
|
||||
assert!(
|
||||
matches!(error, StoreError::Unavailable { message, .. } if message.contains("execution budget"))
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2) + Duration::from_millis(200),
|
||||
"acquiring a slot must not restart the operation budget"
|
||||
);
|
||||
release.await.unwrap();
|
||||
occupied.await.unwrap();
|
||||
budget
|
||||
.run(&pool, "after queued timeout", |_connection| async {
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
pool.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_budget_times_out_without_exceeding_its_connection_limit() {
|
||||
let Some(pool) = test_pool().await else {
|
||||
return;
|
||||
};
|
||||
let budget = ObservabilityQueryBudget::try_new(4, Duration::from_millis(10))
|
||||
.expect("valid query budget");
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_millis(10), Duration::from_secs(10))
|
||||
.expect("valid query budget");
|
||||
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
|
||||
let mut releases = Vec::new();
|
||||
let mut occupied = Vec::new();
|
||||
@@ -110,7 +179,8 @@ async fn queued_query_runs_after_one_slot_is_released() {
|
||||
return;
|
||||
};
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1)).expect("valid query budget");
|
||||
ObservabilityQueryBudget::try_new(4, Duration::from_secs(1), Duration::from_secs(10))
|
||||
.expect("valid query budget");
|
||||
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
|
||||
let mut releases = Vec::new();
|
||||
let mut occupied = Vec::new();
|
||||
@@ -183,7 +253,9 @@ async fn operation_timeout_keeps_the_slot_until_the_database_query_finishes() {
|
||||
let Some(pool) = test_pool().await else {
|
||||
return;
|
||||
};
|
||||
let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(100)).unwrap();
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(1, Duration::from_millis(100), Duration::from_secs(2))
|
||||
.unwrap();
|
||||
let (pid_tx, pid_rx) = oneshot::channel();
|
||||
let error = budget
|
||||
.run(
|
||||
@@ -196,15 +268,18 @@ async fn operation_timeout_keeps_the_slot_until_the_database_query_finishes() {
|
||||
.map_err(database_error)?;
|
||||
pid_tx.send(pid).unwrap();
|
||||
let mut tx = connection.begin().await.map_err(database_error)?;
|
||||
sqlx::query("set local statement_timeout = '10s'")
|
||||
sqlx::query("select set_config('statement_timeout',$1,true)")
|
||||
.bind(format!("{}ms", Duration::from_secs(2).as_millis()))
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(database_error)?;
|
||||
sqlx::query("select pg_sleep(6)")
|
||||
sqlx::query("select pg_sleep($1)")
|
||||
.bind(Duration::from_secs(2).as_secs_f64() * 0.6)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(database_error)?;
|
||||
sqlx::query("select pg_sleep(9)")
|
||||
sqlx::query("select pg_sleep($1)")
|
||||
.bind(Duration::from_secs(2).as_secs_f64() * 0.9)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(database_error)?;
|
||||
@@ -271,7 +346,9 @@ async fn dropping_an_operation_keeps_the_slot_through_database_rollback() {
|
||||
let Some(pool) = test_pool().await else {
|
||||
return;
|
||||
};
|
||||
let budget = ObservabilityQueryBudget::try_new(1, Duration::from_millis(100)).unwrap();
|
||||
let budget =
|
||||
ObservabilityQueryBudget::try_new(1, Duration::from_millis(100), Duration::from_secs(10))
|
||||
.unwrap();
|
||||
let running_pool = pool.clone();
|
||||
let running_budget = budget.clone();
|
||||
let (pid_tx, pid_rx) = oneshot::channel();
|
||||
|
||||
@@ -2,11 +2,97 @@
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_store::postgres::{
|
||||
ObservabilityPageSize, ObservabilityRange, UsageRecordFilter, UsageRecordQuery,
|
||||
ObservabilityPageSize, ObservabilityRange, OpsErrorFilter, OpsErrorQuery, UsageRecordFilter,
|
||||
UsageRecordQuery,
|
||||
};
|
||||
|
||||
use super::{TestDatabase, analytics::seed, observability_repository};
|
||||
|
||||
#[tokio::test]
|
||||
async fn projected_page_counts_correct_pending_moves_deletions_and_model_or() {
|
||||
let Some(database) = TestDatabase::create("usage_projected_pages").await else {
|
||||
return;
|
||||
};
|
||||
let start = chrono::DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc);
|
||||
for index in 0..40 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("projected_{index:02}"),
|
||||
start + TimeDelta::minutes(index),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
super::drain_analytics(&database.pool).await;
|
||||
let repository = observability_repository(&database.pool);
|
||||
for stage in 0..3 {
|
||||
if stage == 1 {
|
||||
sqlx::query("update model_requests set started_at=started_at+interval '30 minutes',recovery_lease_expires_at=recovery_lease_expires_at+interval '30 minutes',completed_at=completed_at+interval '30 minutes',requested_model_id='model_b',upstream_model_id='model_b' where id='projected_06'")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
sqlx::query("delete from model_requests where id='projected_17'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update model_requests set outcome='failed' where id='projected_21'")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
seed(
|
||||
&database.pool,
|
||||
"projected_new",
|
||||
start + TimeDelta::seconds(30),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
} else if stage == 2 {
|
||||
super::drain_analytics(&database.pool).await;
|
||||
}
|
||||
for edge in [0, 31] {
|
||||
let range = ObservabilityRange::new(
|
||||
start + TimeDelta::seconds(edge),
|
||||
start + TimeDelta::hours(1) - TimeDelta::seconds(edge),
|
||||
)
|
||||
.unwrap();
|
||||
for model in [None, Some("model_a"), Some("model_b")] {
|
||||
// 真值只读取实际源版本,不依赖被测汇总或分页辅助函数
|
||||
let expected:Vec<String> = sqlx::query_scalar("select id from model_requests where started_at >= $1 and started_at < $2 and outcome='succeeded' and ($3::text is null or requested_model_id=$3 or upstream_model_id=$3) order by started_at desc,id desc")
|
||||
.bind(range.start).bind(range.end).bind(model).fetch_all(&database.pool).await.unwrap();
|
||||
for current_page in [1, 3, 6, 8] {
|
||||
let page = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
client_api_key_ref: Some("key_analytics".to_owned()),
|
||||
model: model.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
current_page,
|
||||
page_size: ObservabilityPageSize::new(7).unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
page.total,
|
||||
expected.len() as u64,
|
||||
"stage={stage}, edge={edge}, model={model:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
page.items.iter().map(|row| &row.id).collect::<Vec<_>>(),
|
||||
expected
|
||||
.iter()
|
||||
.skip((current_page as usize - 1) * 7)
|
||||
.take(7)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_pages_preserve_order_and_exact_totals_from_both_ends() {
|
||||
let Some(database) = TestDatabase::create("usage_bidirectional_pages").await else {
|
||||
@@ -70,3 +156,127 @@ async fn usage_pages_preserve_order_and_exact_totals_from_both_ends() {
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_pages_cross_minutes_and_keep_exact_counts_after_source_changes() {
|
||||
let Some(database) = TestDatabase::create("usage_time_pages").await else {
|
||||
return;
|
||||
};
|
||||
let start = Utc::now() - TimeDelta::hours(1);
|
||||
for index in 0..29 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("cross_{index:02}"),
|
||||
start + TimeDelta::seconds((index / 3) * 100),
|
||||
&["group_page"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let repository = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(start, start + TimeDelta::minutes(20)).unwrap();
|
||||
for removed in [None, Some("cross_16")] {
|
||||
if let Some(id) = removed {
|
||||
sqlx::query("delete from model_requests where id = $1")
|
||||
.bind(id)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let expected = (0..29)
|
||||
.rev()
|
||||
.map(|index| format!("cross_{index:02}"))
|
||||
.filter(|id| Some(id.as_str()) != removed)
|
||||
.collect::<Vec<_>>();
|
||||
for current_page in 1..=6 {
|
||||
let page = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
client_api_key_ref: Some("key_analytics".to_owned()),
|
||||
search: Some("model_".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
current_page,
|
||||
page_size: ObservabilityPageSize::new(7).unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(page.total, expected.len() as u64);
|
||||
assert_eq!(
|
||||
page.items.iter().map(|item| &item.id).collect::<Vec<_>>(),
|
||||
expected
|
||||
.iter()
|
||||
.skip((current_page as usize - 1) * 7)
|
||||
.take(7)
|
||||
.collect::<Vec<_>>(),
|
||||
"page={current_page}, removed={removed:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_pages_merge_sources_across_minutes_with_stable_ties() {
|
||||
let Some(database) = TestDatabase::create("ops_time_pages").await else {
|
||||
return;
|
||||
};
|
||||
let start = Utc::now() - TimeDelta::hours(1);
|
||||
let mut expected = Vec::new();
|
||||
for index in 0..17 {
|
||||
let id = format!("error_{index:02}");
|
||||
let at = start + TimeDelta::seconds((index / 3) * 100);
|
||||
seed(&database.pool, &id, at, &[]).await;
|
||||
sqlx::query("update model_requests set error_kind = 'upstream_error' where id = $1")
|
||||
.bind(&id)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"insert into ops_events (id, model_request_id, attempt_index, level, component,
|
||||
operation, failure_kind, message, created_at)
|
||||
values ($1, $1, 1, 'error', 'routing', 'responses', 'upstream_error', 'fixture', $2)",
|
||||
)
|
||||
.bind(&id)
|
||||
.bind(at + TimeDelta::minutes(1))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for source in ["model_request", "ops_event"] {
|
||||
expected.push((at + TimeDelta::minutes(1), format!("{source}:{id}")));
|
||||
}
|
||||
}
|
||||
expected.sort_by(|a, b| b.cmp(a));
|
||||
let repository = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(start, start + TimeDelta::minutes(20)).unwrap();
|
||||
for current_page in 1..=7 {
|
||||
let page = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range,
|
||||
filter: OpsErrorFilter {
|
||||
client_api_key_ref: Some("key_analytics".to_owned()),
|
||||
search: Some("error_".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
current_page,
|
||||
page_size: ObservabilityPageSize::new(7).unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(page.total, expected.len() as u64);
|
||||
assert_eq!(
|
||||
page.items
|
||||
.iter()
|
||||
.map(|item| &item.stable_sort_id)
|
||||
.collect::<Vec<_>>(),
|
||||
expected
|
||||
.iter()
|
||||
.skip((current_page as usize - 1) * 7)
|
||||
.take(7)
|
||||
.map(|(_, id)| id)
|
||||
.collect::<Vec<_>>(),
|
||||
"page={current_page}",
|
||||
);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -27,3 +27,5 @@ d4b599066d7382a1c9cb6b3c003200046579e4fce3531f232cbd665a9d92c542 0026_codex_pri
|
||||
8c9bbecd8315a26e928213a82a2ec651c35976e0643b82caef1d1bcfeee2916f 0027_request_observation_projection.sql
|
||||
31d6668707186d7c19cdd507e20162a08c7fbb8ba41ff4a4439d0a5e08369cad 0028_request_analytics.sql
|
||||
1923f14095dac5f808a530db002c37e17168afa0778bc7f3da768c818133eec9 0029_usage_indexes_and_identity_snapshots.sql
|
||||
ab964fcdf56aa1f595fffaf02556aeebce5037f7bfa1ea6f4fef594112eaba6a 0030_analytics_usage_dimensions.sql
|
||||
9d6e5254ee4a7aec40c4fc5f1904300228c57826941b962fa78cc6e550fa99f8 0031_analytics_forecast_completion.sql
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
-- 完整交付用量复用现有统计状态,最近使用时间保持可合并的独立极值
|
||||
alter table request_analytics_15m add column last_used_at timestamptz;
|
||||
alter table request_analytics_hourly add column last_used_at timestamptz;
|
||||
|
||||
-- 新指标只重建时间桶,不清空窄事实,也不重置已经完成的历史投影游标
|
||||
insert into request_analytics_pending_buckets(bucket_start, oldest_pending_at)
|
||||
select distinct date_bin(interval '15 minutes', started_at, timestamptz '2000-01-01 UTC'), now()
|
||||
from request_analytics_facts
|
||||
on conflict (bucket_start) do update
|
||||
set change_version = nextval('request_analytics_change_version');
|
||||
|
||||
-- 撤销最近一次模型贡献后按真实源定位极值,避免扫描整个账号历史
|
||||
create index model_requests_delivered_account_model on model_requests
|
||||
(provider_account_ref,(coalesce(upstream_model_id,requested_model_id,'')),started_at desc,id desc)
|
||||
where outcome='succeeded' and downstream_committed_at is not null
|
||||
and (provider_kind is distinct from 'openai' or request_kind is distinct from 'prewarm')
|
||||
and ((client_transport='websocket' and client_status_code is null) or client_status_code between 200 and 399)
|
||||
and (requested_model_id is not null or upstream_model_id is not null or image_generation_requested
|
||||
or input_tokens is not null or output_tokens is not null or cached_input_tokens is not null
|
||||
or cache_write_input_tokens is not null or reasoning_output_tokens is not null
|
||||
or image_input_tokens is not null or image_output_tokens is not null
|
||||
or total_tokens is not null or cost_amount is not null);
|
||||
|
||||
-- 预测只定位各采样桶最后一份 Provider 文档,宽 JSON 不进入索引
|
||||
create index model_requests_forecast_documents on model_requests
|
||||
(provider_account_ref,completed_at desc,id desc) include(started_at)
|
||||
where provider_observation_json is not null and outcome <> 'running';
|
||||
|
||||
-- 增量构建按字节序推进游标,索引排序必须与游标合同一致
|
||||
create index request_analytics_facts_build_cursor on request_analytics_facts
|
||||
(started_at,id collate "C");
|
||||
create index request_analytics_15m_build_cursor on request_analytics_15m
|
||||
(bucket_start,collection collate "C",dimensions collate "C",outcome collate "C");
|
||||
@@ -0,0 +1,11 @@
|
||||
-- 完成时间边界只回读五分钟内的窄事实,Provider 文档继续使用源表采样索引
|
||||
create index request_analytics_facts_account_completion on request_analytics_facts
|
||||
(provider_account_ref,completed_at,started_at)
|
||||
where completed_at is not null and outcome <> 'running';
|
||||
|
||||
-- 新完成时间维度与既有统计共用可恢复构建,旧事实无需再次投影
|
||||
insert into request_analytics_pending_buckets(bucket_start, oldest_pending_at)
|
||||
select distinct date_bin(interval '15 minutes', started_at, timestamptz '2000-01-01 UTC'), now()
|
||||
from request_analytics_facts
|
||||
on conflict (bucket_start) do update
|
||||
set change_version = nextval('request_analytics_change_version');
|
||||
@@ -28,6 +28,10 @@
|
||||
配置加载会忽略未知字段,并在启动控制台提示字段名;不输出对应值。缺少必填字段时会指出缺项并停止启动,
|
||||
已知字段的类型和取值仍需合法;可选字段省略时使用默认值
|
||||
|
||||
`store.pool.observability_timeout_seconds` 默认 120 秒,控制观测查询与统计 Worker 的操作预算,
|
||||
缓存等待和统计事务的 SQL 时限沿用该配置。修改后重启应用;连接排队仍受
|
||||
`store.pool.acquire_timeout_seconds` 约束,模型请求时限由 `api.request_timeout_seconds` 单独控制
|
||||
|
||||
后端从当前目录向上查找 `deploy/config.yaml`,相对数据、日志和静态资源路径以该文件所在目录解析
|
||||
|
||||
Compose 通过环境变量将监听和数据库地址设为容器内地址,并指定镜像中的静态资源目录
|
||||
@@ -560,6 +564,10 @@ docker compose -f deploy/compose.yaml up -d --no-build --wait
|
||||
|
||||
升级后通过管理端版本接口或容器 image digest 确认运行实例的版本和 revision
|
||||
|
||||
管理端请求默认等待 120 秒。直接构建前端时可通过进程环境 `VITE_API_TIMEOUT_MS` 指定毫秒数,
|
||||
例如 `VITE_API_TIMEOUT_MS=180000 pnpm --dir frontend build`,取值须为正整数。
|
||||
该值在构建时写入静态资源,修改后需要重新构建和部署前端。单个请求显式指定的时限仍优先
|
||||
|
||||
构建元数据通过一次性进程环境传入:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -53,6 +53,7 @@ store:
|
||||
# pool:
|
||||
# max_connections: 20
|
||||
# acquire_timeout_seconds: 5
|
||||
# observability_timeout_seconds: 120 # 观测操作保护时限,不是性能目标
|
||||
|
||||
admin:
|
||||
# 不活动期限与从登录起计算的最长有效期,单位为分钟
|
||||
|
||||
+13
-10
@@ -1248,8 +1248,8 @@ HTTP 请求头及新建 WS 的握手提示按当时的最终出站档位构造
|
||||
`capacity.totalSlots` 为 `number | null`:`null` 表示可用成员中存在继承无限并发的账号,`0` 表示没有可用槽位。
|
||||
分组容量只统计普通池,`capacity.usedSlots` 返回其实际在途数;Redis 不可用时为 `null`
|
||||
|
||||
分组费用按请求执行时实际服务账号的分组快照归属,不按 Client Key 绑定的分组分摊。
|
||||
账号属于多个组时,各组均包含该请求费用;之后调整账号分组不重写历史归属
|
||||
分组费用按当前组成员实际完成请求的费用汇总,不按 Client Key 绑定的分组分摊。
|
||||
账号属于多个组时,各组均包含该账号的请求费用;调整账号分组后,留存窗口内费用随当前成员关系重新汇总
|
||||
|
||||
## 7. Client Key
|
||||
|
||||
@@ -1830,8 +1830,11 @@ Dashboard 的 `accountUsage[]` 由后端提供 `usageWindow`、`metricLabel`、`
|
||||
Dashboard 默认 `period=today`,用量、诊断和错误查询默认 `period=7d`,范围参数遵循[页面时间合同](#页面时间合同)。
|
||||
Dashboard 返回原始 `asOf` 及 `asOfDisplay`,作为当前数据的查询锚点
|
||||
|
||||
用量查询可组合页码/游标、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、
|
||||
request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可取 `model`、`account`、
|
||||
请求明细列表可组合页码、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、
|
||||
request/response/upstream ID、outcome 与搜索文本。`search` 仅供明细及错误列表使用;
|
||||
汇总、洞察和诊断不接受搜索参数,管理观测接口不接受路由分组筛选。页面搜索不改变顶部统计范围
|
||||
|
||||
诊断 `dimension` 可取 `model`、`account`、
|
||||
`apiKey`、`provider`、`transport`、`failureClass`、`status`。诊断按请求量降序返回最多 100 项,
|
||||
同请求量按维度标识稳定排列
|
||||
|
||||
@@ -1863,13 +1866,13 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告(流内
|
||||
缺少终态声明时保留首次声明。这些值仅表示上游报告,不作为模型真实性证明,也不参与路由、
|
||||
聚合或本地计价模型选择。没有保存上游模型报告的记录保持未知
|
||||
|
||||
请求记录列表的 `search` 使用字面量前缀匹配,支持请求 ID、Client Key ID / 名称、
|
||||
账号 ID、账号邮箱与名称、请求 / 上游模型 ID、上游请求 ID。密钥名称不区分大小写,其他字段区分大小写。
|
||||
密钥名称按当前密钥记录检索,改名后使用新名称,删除后仍可按 Client Key ID 查询历史记录。
|
||||
账号邮箱与名称按请求记录的历史快照检索,
|
||||
不随当前账号修改或删除而改变;`%`、`_` 和 `\` 均按普通字符处理,不作为搜索通配符
|
||||
请求记录列表的 `search` 使用字面量前缀匹配,支持 Client Key ID / 名称、
|
||||
账号 ID、账号邮箱与名称、请求 / 上游模型 ID。密钥名称不区分大小写,其他字段区分大小写。
|
||||
密钥名称、账号邮箱与名称按请求记录的历史快照检索,
|
||||
不随当前密钥或账号修改、删除而改变;`%`、`_` 和 `\` 均按普通字符处理,不作为搜索通配符
|
||||
|
||||
请求记录按模型执行 ID 或上游 ID 查询;只有入口 ID 时需结合时间与入口日志定位。
|
||||
请求记录的 `requestId`、`upstreamRequestId` 参数按模型执行 ID 或上游 ID 精确查询;
|
||||
搜索框不匹配请求 ID,只有入口 ID 时需结合时间与入口日志定位。
|
||||
请求记录和错误列表支持按密钥名称搜索,不支持密钥值或可见前缀搜索
|
||||
|
||||
### 记录范围与统计口径
|
||||
|
||||
+15
-3
@@ -865,14 +865,21 @@ OpenAI 订阅周期属于按需个人信息,不是额度事实。Admin 账号
|
||||
|
||||
## 10. 观测与后台任务
|
||||
|
||||
管理观测查询与统计 Worker 共用 PostgreSQL 连接预算,为请求执行保留连接容量。每项操作使用一个受预算约束的连接;
|
||||
管理观测查询与统计 Worker 共用 PostgreSQL 连接预算,为请求执行保留连接容量。每项操作使用一个受预算约束的连接,排队计入总操作预算;
|
||||
Store 声明配置后的读取时限,缓存沿用该值,统计事务只在事务内设置 SQL 时限,归还连接时不污染其他查询;
|
||||
超时或调用方取消后,名额持续覆盖 SQLx 的协议收尾、事务回滚和连接归还,不能随查询 Future 提前释放。
|
||||
请求列表的精确总数与候选页共用数据库快照,按距首尾的较短方向读取,返回顺序和页码语义保持一致。
|
||||
请求列表的精确总数与候选页共用数据库快照,深页先按时间分段定位,再按距区间首尾的较短方向读取,
|
||||
返回顺序和页码语义保持一致。
|
||||
|
||||
Store 从请求源事实维护窄事实及十五分钟、小时统计状态,账号用量与分组费用复用同一套已交付用量指标。
|
||||
完整时间桶在数据库内合并,边界与不能由汇总表达的筛选在数据库内精确聚合;向应用返回的状态按输出分组组织。
|
||||
计数、金额与分布状态的旧贡献撤销、新贡献合并和事实替换共用事务,构建期间只调整已扫描前缀,
|
||||
不可相减的极值按受影响分组重算。同步账号用量查询在同一快照补齐待投影源版本,避免异步投影改变原有读数合同。
|
||||
|
||||
### 额度预测
|
||||
|
||||
账号容量预测属于 Admin 的只读派生规则,不参与 quota 权威状态、调度或金额结算。Store 通过专用采样端口
|
||||
在同一 SQL 快照内返回截至观测时间的累计数值及有界历史 Provider 文档;对应 Provider 复用协议解析器解释
|
||||
在同一读事务快照内返回截至观测时间的累计数值及有界历史 Provider 文档;对应 Provider 复用协议解析器解释
|
||||
文档,不在历史查询中刷新上游额度。观测通过 Core 原有
|
||||
请求结算持久化,账号关联由实际执行上下文确定,窗口身份与归属必须匹配当前额度窗口。Admin 按中立的额度
|
||||
事实选择近期进度段预测剩余量,再加本周期已记录用量形成周期总量。
|
||||
@@ -890,6 +897,11 @@ OpenAI 订阅周期属于按需个人信息,不是额度事实。Admin 账号
|
||||
- 重置后从新段重新累计,无法精确归属的用量不补算;只有账号在窗口开始前加入且无已知断点时,才允许以窗口累计量替代增量采样
|
||||
- 历史点抽样不缩减区间累计用量,同一完成时间共用同一累计值
|
||||
|
||||
预测数值复用统计投影的十五分钟开始时间桶,并以账号与五分钟完成时间桶分组。
|
||||
完整完成时间桶合并累计量,跨采样截止值的小桶、开始时间边界和重建区间读取窄事实;
|
||||
同一快照撤销待投影旧版本并加入当前源版本。该维度保留恢复请求的原有预测口径,
|
||||
不改变其他统计排除恢复请求的规则。Provider 文档仍按账号和完成时间索引读取,每个采样槽只保留最后一份
|
||||
|
||||
### 请求记录与交付
|
||||
|
||||
Provider 独占首事件、首字、首推理和首正文的观测,以及上游计时的协议解析,通过类型化响应观测交给 Core。
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# PostgreSQL 3100 万请求优化与验收
|
||||
|
||||
本报告对应 [Issue #413](https://github.com/zyycn/codex-proxy-rs/issues/413),范围为大数据量下的统计、请求分页、账号用量、分组费用与额度预测
|
||||
|
||||
3100 万条源请求、窄事实和完整汇总已构建并独立对账。当前 HTTP 矩阵执行 167 次请求,包含 145 次合法业务请求、22 次预期参数或权限拒绝,未出现超时或非预期失败。按实际注册路由计,84 条已有通过记录、70 条未覆盖、6 条不适用;不宣称全接口或长期负载验收通过
|
||||
|
||||
## 问题与实现边界
|
||||
|
||||
原账号用量读取在命中事实超过 10 万条时拒绝查询;其他统计的事实回退与汇总读取也有固定行数门槛。仅放宽门槛会继续放大应用端逐行合并、深页 OFFSET 和预测窗口排序的成本
|
||||
|
||||
实现继续使用原生 PostgreSQL、现有窄事实、十五分钟/小时汇总和投影 Worker:
|
||||
|
||||
- 账号与 Key 的必要模型联合维度进入共享汇总,完整桶和精确边界在数据库内聚合,不以命中数据量拒绝查询
|
||||
- 账号窗口、用量排序、分组费用和分页计数在同一读快照补齐待投影修改与删除,保留精确总数、金额及未知值语义
|
||||
- 请求深页通过时间分段定位,错误列表合并两类来源后保持稳定排序
|
||||
- 预测使用开始时间与完成时间二维汇总,累计量保留完整窗口,Provider 文档独立有界采样
|
||||
- 投影在同一事务撤销旧贡献并加入新贡献;普通更新维护构建前缀,不反复重启整个桶
|
||||
- 观测操作与前端默认保护时限为 120 秒,可分别配置;30 秒为优化方向,不能将提高超时时限当作查询提速
|
||||
|
||||
使用记录搜索支持密钥、账号与模型,不匹配请求 ID;请求 ID 的显式精确筛选仍保留。汇总、洞察和诊断不接受搜索参数,管理观测接口不接受路由分组筛选
|
||||
|
||||
当前合同与职责见 [API 参考](../api.md)、[系统架构](../architecture.md#10-观测与后台任务)与[部署配置](../../deploy/README.md),本报告不重复字段清单
|
||||
|
||||
## 环境与样本
|
||||
|
||||
| 项目 | 实际条件 |
|
||||
| --- | --- |
|
||||
| 数据 | 31 天、3100 万条合成源请求;50 个账号、100 个 Key、20 个模型、10 个分组 |
|
||||
| 时间范围 | `2026-09-12T04:45:00Z` 至 `2026-10-13T04:45:00Z` |
|
||||
| 数据库 | PostgreSQL 18.6,容器限制 4 CPU、4 GiB 内存,持久磁盘 |
|
||||
| 主要参数 | shared_buffers 512 MiB、work_mem 16 MiB、max_parallel_workers 8、max_parallel_workers_per_gather 2、JIT 开启 |
|
||||
| 持久性 | fsync、full_page_writes、synchronous_commit 均开启 |
|
||||
| 应用 | release 构建;核心轮次重启应用清空应用缓存,16 个查询客户端并发 |
|
||||
| 其他轮次 | 剩余只读接口 4 并发;合成配置写入、鉴权及本机 Provider 场景串行执行 |
|
||||
| 缓存边界 | 未清空数据库与宿主文件缓存,应用冷缓存不等于数据库冷缓存 |
|
||||
|
||||
造数与离线汇总构建使用的临时资源配置已恢复,搜索索引试验已撤销。3100 万条容量基线不是生产 Worker 全量回填吞吐的测量结果;生产投影的正确性通过真实源表回归及样本对账验证
|
||||
|
||||
## HTTP 结果
|
||||
|
||||
以下为当前矩阵中的单次短样本,不能推断稳定的 P95/P99。并发样本包含资源竞争,不能与隔离单请求耗时直接当作前后优化对照
|
||||
|
||||
| 场景 | 耗时 |
|
||||
| --- | ---: |
|
||||
| 用量明细宽范围模型前缀搜索 | 32.63 秒 |
|
||||
| 账号用量降序 | 20.50 秒 |
|
||||
| 账号最近使用降序 | 20.39 秒 |
|
||||
| 账号额度预测 | 19.94 秒 |
|
||||
| Key 维度诊断 | 18.46 秒 |
|
||||
| Dashboard 概览 | 12.30 秒 |
|
||||
| 账号维度诊断 | 10.82 秒 |
|
||||
| 用量洞察 | 6.20 秒 |
|
||||
| 用量汇总 | 6.17 秒 |
|
||||
| 用量中间页 | 3.60 秒 |
|
||||
| 用量首/尾/超尾页 | 3.46/1.63/1.39 秒 |
|
||||
| 错误列表第 6200 页 | 1.70 秒 |
|
||||
|
||||
宽范围搜索命中大量明细,32.63 秒超过 30 秒优化期望,但未触发 120 秒保护时限。本次保留该性能局限,不再为此扩展索引或引入其他存储系统
|
||||
|
||||
正确性核对覆盖源请求总量、Token、费用、主要筛选组合、诊断分母及分页结果。移除的搜索与分组参数按预期返回 400,权限负例按预期返回拒绝;这些负例不计作合法容量查询通过
|
||||
|
||||
## 回归与自审范围
|
||||
|
||||
- 超过 10 万条真实源请求经过生产 Worker,验证账号/Key 汇总、热点边界和未知 Token;锁住窄事实表时,干净完整桶仍可读取
|
||||
- 待投影跨账号/模型/时间桶修改、删除、重复投递、写入失败回滚及构建前缀持续更新
|
||||
- 精确分页首中尾页、超尾页、跨时间段与同时间戳排序,两类错误来源合并
|
||||
- 预测微秒边界、同完成时间、截止时间、缺失费用、恢复请求、重建与待投影补差
|
||||
- 连接排队计入总预算,超时与取消后的事务回滚及连接归还仍持有查询名额
|
||||
- 分组费用沿用当前成员关系;账号窗口的 Token 兜底与普通统计的未知值口径分别保留
|
||||
- 投影与极值重算共用维度定义,已删除请求 ID 搜索捷径及淘汰的逐行读取映射
|
||||
- Rustfmt、相关 Store/Admin/API 回归、严格 Clippy、架构边界、迁移冻结清单、前端类型构建与 ESLint 对应实际最终工作区执行
|
||||
|
||||
页面仅调整搜索提示;按本次任务要求不提供截图
|
||||
|
||||
## 升级与验证局限
|
||||
|
||||
迁移 0030/0031 增加统计字段和索引,将既有时间桶排入可恢复重建,不清空窄事实或重置已完成的源投影游标。重建中的桶走事实回退,索引创建与桶重建仍需要额外 I/O、磁盘空间和时间;未完成生产环境在线升级的时延与负载验收
|
||||
|
||||
本次收尾聚焦 3100 万请求瓶颈,不再扩展外围插件、OAuth、备份和升级场景。数据库冷缓存、长期持续负载、生产 Worker 全量回填性能及未覆盖外部集成保留为验证局限;小规模生命周期回归不等于相同场景的 3100 万级持续写入验收
|
||||
|
||||
临时造数、HTTP 驱动、SQL 计划及逐请求记录保存在忽略跟踪的 `.runtime/capacity`,不进入工程提交。完整请求与路由结果为 `http-31m-latest.json`/`http-31m-latest.md`,原失败轮次分别保留,最终检查日志使用 `final-*` 前缀
|
||||
Vendored
+4
@@ -1,5 +1,9 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
interface ImportMetaEnv {
|
||||
readonly VITE_API_TIMEOUT_MS?: string
|
||||
}
|
||||
|
||||
declare module '@fontsource-variable/inter'
|
||||
declare module '@fontsource-variable/jetbrains-mono'
|
||||
|
||||
|
||||
@@ -1,2 +1,6 @@
|
||||
export const API_BASE_URL = import.meta.env.DEV ? '/dev' : ''
|
||||
export const API_TIMEOUT_MS = 30_000
|
||||
const timeout = Number(import.meta.env.VITE_API_TIMEOUT_MS ?? 120_000)
|
||||
if (!Number.isSafeInteger(timeout) || timeout <= 0 || timeout > 2_147_483_647) {
|
||||
throw new Error('VITE_API_TIMEOUT_MS must be a positive 32-bit integer')
|
||||
}
|
||||
export const API_TIMEOUT_MS = timeout
|
||||
|
||||
@@ -490,7 +490,6 @@ interface UsageRangeQuery {
|
||||
provider?: string
|
||||
model?: string
|
||||
statusCode?: number
|
||||
search?: string
|
||||
}
|
||||
|
||||
interface PageQuery {
|
||||
@@ -498,7 +497,7 @@ interface PageQuery {
|
||||
pageSize: number
|
||||
}
|
||||
|
||||
type UsagePageQuery = UsageRangeQuery & PageQuery
|
||||
type UsagePageQuery = UsageRangeQuery & PageQuery & { search?: string }
|
||||
|
||||
type OpsErrorPageQuery = UsagePageQuery
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ const search = defineModel<string>('search', { required: true })
|
||||
<template>
|
||||
<div class="flex w-full items-center gap-3" role="group" aria-label="使用记录筛选与操作">
|
||||
<div class="min-w-0 flex-1 sm:w-96 sm:flex-none">
|
||||
<ZInput v-model="search" placeholder="请求、密钥名称、账号或模型" aria-label="搜索使用记录:请求、密钥名称、账号或模型" class="w-full">
|
||||
<ZInput v-model="search" placeholder="密钥名称、账号或模型" aria-label="搜索使用记录:密钥名称、账号或模型" class="w-full">
|
||||
<template #prefix>
|
||||
<Search class="size-4.5 text-cp-text-tertiary" />
|
||||
</template>
|
||||
|
||||
Reference in New Issue
Block a user