feat(usage): group admin diagnostics by client key and model

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Yongzhao Chen
2026-09-22 21:33:39 +02:00
co-authored by Codex
parent 85b1c13c06
commit 1d91e7bc25
13 changed files with 105 additions and 13 deletions
@@ -158,6 +158,7 @@ pub struct OpsErrorQuery {
pub enum DiagnosticDimension {
Provider,
Model,
KeyModel,
Account,
ApiKey,
Transport,
@@ -1156,6 +1157,7 @@ pub struct DiagnosticsItem {
pub retry_rate: f64,
pub impact_score: f64,
pub estimated_cost: Option<DecimalAmount>,
pub cost_incomplete: bool,
pub attempt_count: u64,
pub total_tokens: u64,
}
@@ -385,6 +385,8 @@ impl ObservabilityService for DefaultObservabilityService {
retry_rate,
impact_score,
estimated_cost: usd_cost(&item.costs),
cost_incomplete: item.cost_coverage.partial_count > 0
|| item.cost_coverage.unavailable_count > 0,
attempt_count: item.attempt_count,
total_tokens: item.total_tokens,
}
@@ -1296,6 +1296,7 @@ pub(crate) fn diagnostics_view(
retry_rate: item.retry_rate,
impact_score: item.impact_score,
estimated_cost: item.estimated_cost.as_ref().map(ToString::to_string),
cost_incomplete: item.cost_incomplete,
attempt_count: item.attempt_count,
total_tokens: item.total_tokens,
})
@@ -160,6 +160,7 @@ impl TrendKind {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DiagnosticDimension {
Model,
KeyModel,
Account,
ApiKey,
Provider,
@@ -173,6 +174,7 @@ impl DiagnosticDimension {
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
match trimmed(value) {
None | Some("model") => Ok(Self::Model),
Some("keyModel") => Ok(Self::KeyModel),
Some("account") => Ok(Self::Account),
Some("apiKey" | "api_key") => Ok(Self::ApiKey),
Some("provider") => Ok(Self::Provider),
@@ -188,6 +190,7 @@ impl DiagnosticDimension {
pub const fn display_name(self) -> &'static str {
match self {
Self::Model => "model",
Self::KeyModel => "keyModel",
Self::Account => "account",
Self::ApiKey => "apiKey",
Self::Provider => "provider",
@@ -257,6 +260,7 @@ pub(crate) fn domain_diagnostic_dimension(
) -> domain::DiagnosticDimension {
match dimension {
DiagnosticDimension::Model => domain::DiagnosticDimension::Model,
DiagnosticDimension::KeyModel => domain::DiagnosticDimension::KeyModel,
DiagnosticDimension::Account => domain::DiagnosticDimension::Account,
DiagnosticDimension::ApiKey => domain::DiagnosticDimension::ApiKey,
DiagnosticDimension::Provider => domain::DiagnosticDimension::Provider,
@@ -760,6 +760,7 @@ pub struct DiagnosticItemView {
pub retry_rate: f64,
pub impact_score: f64,
pub estimated_cost: Option<String>,
pub cost_incomplete: bool,
pub attempt_count: u64,
pub total_tokens: u64,
}
@@ -61,6 +61,10 @@ fn diagnostics_query_should_keep_wire_dimension_name() {
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
let key_model: DiagnosticsQuery =
serde_json::from_value(json!({"dimension": "keyModel"})).unwrap();
assert_eq!(key_model.dimension().unwrap(), DiagnosticDimension::KeyModel);
assert_eq!(DiagnosticDimension::KeyModel.display_name(), "keyModel");
}
#[test]
@@ -79,6 +79,7 @@ pub(crate) const fn store_diagnostic_dimension(
match dimension {
admin_observability::DiagnosticDimension::Provider => DiagnosticDimension::Provider,
admin_observability::DiagnosticDimension::Model => DiagnosticDimension::Model,
admin_observability::DiagnosticDimension::KeyModel => DiagnosticDimension::KeyModel,
admin_observability::DiagnosticDimension::Account => DiagnosticDimension::Account,
admin_observability::DiagnosticDimension::ApiKey => DiagnosticDimension::ApiKey,
admin_observability::DiagnosticDimension::Transport => DiagnosticDimension::Transport,
@@ -185,6 +185,7 @@ pub struct OpsErrorQuery {
pub enum DiagnosticDimension {
Provider,
Model,
KeyModel,
Account,
ApiKey,
Transport,
@@ -5,6 +5,8 @@ use serde_json::Value;
const MODEL_DIAGNOSTIC_DIMENSION_SQL: &str =
"coalesce(mr.upstream_model_id, mr.requested_model_id)";
const KEY_MODEL_DIAGNOSTIC_DIMENSION_SQL: &str =
"json_build_array(mr.client_api_key_ref, coalesce(mr.upstream_model_id, mr.requested_model_id))::text";
pub(crate) fn push_usage_filter(
query: &mut QueryBuilder<Postgres>,
@@ -449,9 +451,9 @@ pub(crate) async fn usage_diagnostics(
where currency_grouping = 1
order by request_count desc, dimension_name limit ",
);
// 单个 Key 的模型用量页需要列出该 Key 在区间内的全部模型。
statement.push_bind(if dimension == DiagnosticDimension::Model
&& filter.client_api_key_ref.is_some()
// Key 用量页及管理端 Key×模型视图都需要保留区间内的全部组合。
statement.push_bind(if dimension == DiagnosticDimension::KeyModel
|| (dimension == DiagnosticDimension::Model && filter.client_api_key_ref.is_some())
{
i64::MAX
} else {
@@ -522,6 +524,28 @@ pub(crate) async fn usage_diagnostics(
)
.await?
}
DiagnosticDimension::KeyModel => {
let pairs = observations
.iter()
.map(|item| {
serde_json::from_str::<(String, String)>(&item.key)
.map_err(|_| postgres_unavailable("decode key/model diagnostic"))
})
.collect::<StoreResult<Vec<_>>>()?;
let names = diagnostic_api_key_display_names(
pool,
&pairs.iter().map(|(key, _)| key.clone()).collect::<Vec<_>>(),
)
.await?;
observations
.iter()
.zip(pairs)
.map(|(item, (key, model))| {
let name = names.get(&key).map_or(key.as_str(), String::as_str);
(item.key.clone(), format!("{name} → {model}"))
})
.collect()
}
_ => HashMap::new(),
};
for observation in &mut observations {
@@ -600,6 +624,7 @@ pub(crate) fn diagnostic_dimension_sql(dimension: DiagnosticDimension) -> &'stat
match dimension {
DiagnosticDimension::Provider => "coalesce(mr.provider_kind, 'unrouted')",
DiagnosticDimension::Model => MODEL_DIAGNOSTIC_DIMENSION_SQL,
DiagnosticDimension::KeyModel => KEY_MODEL_DIAGNOSTIC_DIMENSION_SQL,
DiagnosticDimension::Account => "coalesce(mr.provider_account_ref, 'unrouted')",
DiagnosticDimension::ApiKey => "mr.client_api_key_ref",
DiagnosticDimension::Transport => {
@@ -625,6 +650,11 @@ pub(crate) fn push_diagnostic_dimension_filter(
statement.push(MODEL_DIAGNOSTIC_DIMENSION_SQL);
statement.push(" is not null");
}
DiagnosticDimension::KeyModel => {
statement.push(" and mr.client_api_key_ref is not null and ");
statement.push(MODEL_DIAGNOSTIC_DIMENSION_SQL);
statement.push(" is not null");
}
DiagnosticDimension::Status => {
statement
.push(" and coalesce(mr.upstream_status_code, mr.client_status_code) is not null");
@@ -847,6 +847,11 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
seed_api_key(&database.pool, "key_diag", "My Key", started_at).await;
let store = PgExecutionStore::new(database.pool.clone());
for id in ["req_key_a", "req_key_b"] {
let upstream_model = if id == "req_key_a" {
"upstream-model"
} else {
"other-model"
};
let mut request = new_request(id, started_at);
request.client_api_key_id = Some("key_diag".to_owned());
request.client_api_key_ref = "key_diag".to_owned();
@@ -859,7 +864,7 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
provider_kind: "openai".to_owned(),
provider_account_id: None,
provider_account_ref: Some("acct_key".to_owned()),
upstream_model_id: Some("upstream-model".to_owned()),
upstream_model_id: Some(upstream_model.to_owned()),
upstream_transport: "http_sse".to_owned(),
http_version: None,
};
@@ -893,6 +898,27 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
assert_eq!(diagnostics[0].request_count, 2);
assert_eq!(diagnostics[0].latency_p95_ms, Some(1950));
let key_models = repository
.usage_diagnostics(
range_around(started_at),
UsageRecordFilter::default(),
DiagnosticDimension::KeyModel,
)
.await
.expect("key/model diagnostics");
assert_eq!(key_models.len(), 2);
let names = key_models
.iter()
.map(|item| {
let (key, model): (String, String) = serde_json::from_str(&item.key).unwrap();
assert_eq!(key, "key_diag");
assert_eq!(item.request_count, 1);
(model, item.name.clone())
})
.collect::<std::collections::BTreeMap<_, _>>();
assert_eq!(names["upstream-model"], "My Key → upstream-model");
assert_eq!(names["other-model"], "My Key → other-model");
sqlx::query("delete from client_api_keys where id = 'key_diag'")
.execute(&database.pool)
.await
+2 -1
View File
@@ -1337,7 +1337,8 @@ Dashboard 的 `capacityInfo.maxConcurrentPerAccount` 为默认账号并发上限
用量查询可组合页码/游标、时间范围、Provider、Client Key、账号、模型、route、transport、状态码、
request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可取 `model`、`account`、
`apiKey`、`provider`、`transport`、`failureClass`、`status`。
`apiKey`、`keyModel`、`provider`、`transport`、`failureClass`、`status`。`keyModel` 按 Client Key 与上游发送模型交叉汇总,
返回请求数、Token 和估算费用;费用不完整时 `costIncomplete` 为 `true`。
管理端请求列表及 Dashboard 最近请求中的 `accountNotes` 为账号当前备注,按内部账号 ID 关联。
备注不写入请求历史快照;无备注或账号已删除时返回 `null`,修改备注不改变历史请求的账号归属。
+1
View File
@@ -450,6 +450,7 @@ export interface UsageDiagnosticItem {
retryRate: number
impactScore: number
estimatedCost: string | null
costIncomplete: boolean
attemptCount: number
totalTokens: number
}
@@ -28,6 +28,7 @@ const dimension = defineModel('dimension', { type: String, required: true })
const dimensionOptions = [
{ label: '模型', value: 'model' },
{ label: '密钥×模型', value: 'keyModel' },
{ label: '账号', value: 'account' },
{ label: '密钥', value: 'apiKey' },
{ label: '上游', value: 'provider' },
@@ -69,18 +70,28 @@ const diagnosticColumns = defineTableColumns<DiagnosticDisplayItem>([
},
])
const keyModelColumns = defineTableColumns<DiagnosticDisplayItem>([
{ key: 'nameDisplay', label: '密钥 / 模型', kind: 'custom', size: 'xl' },
{ key: 'requestCount', label: '请求', kind: 'numeric', size: 'sm' },
{ key: 'totalTokens', label: 'TOKEN', kind: 'numeric', size: 'lg' },
{ key: 'estimatedCost', label: '估算费用 USD', kind: 'numeric', size: 'lg' },
])
const selectedDimensionLabel = computed(
() => dimensionOptions.find(option => option.value === dimension.value)?.label ?? '维度',
)
const resultDimension = computed(() => props.diagnostics.dimension || dimension.value)
const visibleColumns = computed(() => resultDimension.value === 'keyModel' ? keyModelColumns : diagnosticColumns)
const resultDimensionLabel = computed(
() => dimensionOptions.find(option => option.value === resultDimension.value)?.label ?? '维度',
)
const sortedItems = computed(() =>
[...props.diagnostics.items].sort(
(left, right) => right.impactScore - left.impactScore || right.requestCount - left.requestCount,
(left, right) => resultDimension.value === 'keyModel'
? right.requestCount - left.requestCount
: right.impactScore - left.impactScore || right.requestCount - left.requestCount,
),
)
@@ -100,7 +111,7 @@ function diagnosticNameDisplay(name: string) {
const raw = name.trim() || '未知'
const full
= resultDimension.value === 'transport' ? ({ websocket: 'WS', http_sse: 'SSE' }[raw] ?? raw) : raw
if (resultDimension.value !== 'model' && resultDimension.value !== 'account') {
if (resultDimension.value !== 'model' && resultDimension.value !== 'account' && resultDimension.value !== 'keyModel') {
return { primary: full, secondary: '', full }
}
@@ -116,8 +127,8 @@ function diagnosticNameDisplay(name: string) {
<template>
<BaseCard
as="article"
title="热点诊断"
:description="`按${selectedDimensionLabel}定位高影响请求`"
:title="resultDimension === 'keyModel' ? '密钥模型用量' : '热点诊断'"
:description="resultDimension === 'keyModel' ? '各密钥下每个模型的 Token 与估算费用' : `按${selectedDimensionLabel}定位高影响请求`"
class="h-105 min-h-105 max-h-105 min-w-0 w-full lg:h-full lg:min-h-90 lg:max-h-105"
>
<template #actions>
@@ -126,7 +137,7 @@ function diagnosticNameDisplay(name: string) {
label="诊断维度"
:options="dimensionOptions"
:disabled="loading"
class="w-full min-w-0 lg:w-80"
class="w-full min-w-0 lg:w-96"
/>
</template>
@@ -135,7 +146,7 @@ function diagnosticNameDisplay(name: string) {
v-if="hasData"
:key="resultDimension"
class="min-h-0 w-full xl:contain-[size]"
:columns="diagnosticColumns"
:columns="visibleColumns"
:rows="displayItems"
density="compact"
row-key="key"
@@ -229,8 +240,15 @@ function diagnosticNameDisplay(name: string) {
</template>
<template #estimatedCost="{ row }">
<span class="font-mono font-bold tabular-nums text-cp-green-text">
{{ formatUsd(row.estimatedCost) }}
<span class="grid justify-items-end gap-1 font-mono tabular-nums text-cp-green-text">
<strong>{{ formatUsd(row.estimatedCost) }}</strong>
<small v-if="row.costIncomplete" class="text-[10px] text-cp-text-tertiary">部分费用缺失</small>
</span>
</template>
<template #totalTokens="{ row }">
<span class="font-mono font-bold tabular-nums" :title="String(row.totalTokens)">
{{ formatCompactNumber(row.totalTokens) }}
</span>
</template>
</BaseTable>