starred/codex-proxy-rs
1
0
Fork 0
mirror of https://github.com/zyycn/codex-proxy-rs.git synced 2026-11-06 00:06:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(observability): unify request error statistics

Browse Source 
Count incomplete requests as errors, exclude recovered and cancelled requests from success rates, and preserve gaps for missing samples.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-11-05 21:56:12 +08:00
co-authored by Codex
parent c73e7126a2
commit 3e33e84a45
22 changed files with 286 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/crates/gateway-admin/src/model/observability.rs
+19 -7
View File
@@ -510,10 +510,20 @@ impl RequestHealth {
)
}
/// 账号与健康时间线使用百分数，洞察接口使用 0 到 1 的比例
#[must_use]
pub fn success_rate(self) -> Option<f64> {
self.success_ratio().map(|ratio| ratio * 100.0)
}
pub(crate) fn success_ratio(self) -> Option<f64> {
let total = self.eligible_requests();
(total > 0).then(|| self.success_requests as f64 / total as f64 * 100.0)
(total > 0).then(|| self.success_requests as f64 / total as f64)
}
pub(crate) fn error_ratio(self) -> Option<f64> {
let total = self.eligible_requests();
(total > 0).then(|| self.non_success_requests() as f64 / total as f64)
}
// 未完成保留交付终态，但上游失败、超时和进程中断都不能视为服务成功
@@ -1081,7 +1091,9 @@ pub struct DiagnosticObservation {
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
pub first_token_p95_ms: Option<u64>,
pub non_completion_count: u64,
pub cancelled_count: u64,
pub incomplete_count: u64,
pub caller_error_count: u64,
pub retry_count: u64,
pub retried_request_count: u64,
pub cost_coverage: CostCoverage,
@@ -1169,7 +1181,7 @@ pub struct TrendPoint {
pub bucket_start: DateTime<Utc>,
pub granularity: Granularity,
pub metrics: RequestMetrics,
pub service_failure_count: u64,
pub error_count: u64,
pub average_latency_ms: Option<u64>,
pub average_first_token_latency_ms: Option<u64>,
pub cached_token_rate: f64,
@@ -1187,7 +1199,7 @@ pub struct TrendSummary {
pub output_tokens: u64,
pub cached_input_tokens: u64,
pub total_tokens: u64,
pub service_failure_count: u64,
pub error_count: u64,
pub average_latency_ms: Option<u64>,
pub max_latency_ms: Option<u64>,
pub min_latency_ms: Option<u64>,
@@ -1351,7 +1363,7 @@ pub struct UsageInsightsHealthPoint {
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub error_rate: f64,
pub error_rate: Option<f64>,
}
/// 洞察健康汇总；失败数已排除调用方错误
@@ -1363,7 +1375,7 @@ pub struct UsageInsightsHealth {
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub success_rate: f64,
pub success_rate: Option<f64>,
pub completion_rate: f64,
pub points: Vec<UsageInsightsHealthPoint>,
}
@@ -1550,7 +1562,7 @@ pub struct DiagnosticsItem {
pub request_count: u64,
pub success_count: u64,
pub error_count: u64,
pub error_rate: f64,
pub error_rate: Option<f64>,
pub request_share: f64,
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
backend/crates/gateway-admin/src/use_case/observability/mod.rs
+22 -24
View File
@@ -356,9 +356,16 @@ impl ObservabilityService for DefaultObservabilityService {
.items
.into_iter()
.map(|item| {
let error_rate = rate_or_zero(item.failure_count, item.request_count);
let non_completion_rate =
rate_or_zero(item.non_completion_count, item.request_count);
let health = RequestHealth::from_counts(
item.success_count,
item.failure_count,
item.cancelled_count,
item.incomplete_count,
item.caller_error_count,
);
let non_completion_count =
item.cancelled_count.saturating_add(item.incomplete_count);
let non_completion_rate = rate_or_zero(non_completion_count, item.request_count);
let retry_rate = rate_or_zero(item.retried_request_count, item.request_count);
DiagnosticsItem {
account_upstream_peer: peers.get(&item.key).copied(),
@@ -374,8 +381,8 @@ impl ObservabilityService for DefaultObservabilityService {
account_plan_type: item.account_plan_type,
request_count: item.request_count,
success_count: item.success_count,
error_count: item.failure_count,
error_rate,
error_count: health.non_success_requests(),
error_rate: health.error_ratio(),
request_share: rate_or_zero(
item.request_count,
observation.total_request_count,
@@ -383,7 +390,7 @@ impl ObservabilityService for DefaultObservabilityService {
average_latency_ms: item.average_latency_ms,
latency_p95_ms: item.latency_p95_ms,
first_token_p95_ms: item.first_token_p95_ms,
non_completion_count: item.non_completion_count,
non_completion_count,
non_completion_rate,
retry_count: item.retry_count,
retry_rate,
@@ -576,7 +583,6 @@ fn build_usage_insights(
let requests = &overview.requests;
let health_window = RequestHealth::from_metrics(requests);
let failed_requests = health_window.failed_requests;
let health_requests = health_window.eligible_requests();
let health = UsageInsightsHealth {
total_requests: requests.request_count,
success_requests: requests.success_count,
@@ -584,7 +590,7 @@ fn build_usage_insights(
cancelled_requests: requests.cancelled_count,
incomplete_requests: requests.incomplete_count,
caller_error_requests: requests.caller_error_count,
success_rate: rate_or_zero(requests.success_count, health_requests),
success_rate: health_window.success_ratio(),
completion_rate: rate_or_zero(
requests
.success_count
@@ -603,10 +609,7 @@ fn build_usage_insights(
cancelled_requests: point.metrics.cancelled_count,
incomplete_requests: point.metrics.incomplete_count,
caller_error_requests: point.metrics.caller_error_count,
error_rate: rate_or_zero(
health_window.non_success_requests(),
health_window.eligible_requests(),
),
error_rate: health_window.error_ratio(),
}
})
.collect(),
@@ -1215,11 +1218,9 @@ fn trend(
}
}
let average_latency_ms = average(metrics.latency_sum_ms, metrics.latency_count);
let service_failure_count = RequestHealth::from_metrics(&metrics).failed_requests;
let success_rate = ratio(
metrics.success_count,
RequestHealth::from_metrics(&metrics).eligible_requests(),
);
let health = RequestHealth::from_metrics(&metrics);
let error_count = health.non_success_requests();
let success_rate = health.success_ratio();
let cache_hit_request_rate = ratio(
metrics.cache_hit_request_count,
metrics.cache_eligible_request_count,
@@ -1248,7 +1249,7 @@ fn trend(
output_tokens: metrics.output_tokens,
cached_input_tokens: metrics.cached_input_tokens,
total_tokens: metrics.total_tokens,
service_failure_count,
error_count,
average_latency_ms,
max_latency_ms: metrics.max_latency_ms,
min_latency_ms: metrics.min_latency_ms,
@@ -1267,11 +1268,11 @@ fn trend(
}
fn trend_point(point: RequestMetricPoint) -> TrendPoint {
let service_failure_count = RequestHealth::from_metrics(&point.metrics).failed_requests;
let health = RequestHealth::from_metrics(&point.metrics);
TrendPoint {
bucket_start: point.bucket_start,
granularity: point.granularity,
service_failure_count,
error_count: health.non_success_requests(),
average_latency_ms: average(point.metrics.latency_sum_ms, point.metrics.latency_count),
average_first_token_latency_ms: average(
point.metrics.first_token_latency_sum_ms,
@@ -1285,10 +1286,7 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint {
point.metrics.cache_hit_request_count,
point.metrics.cache_eligible_request_count,
),
success_rate: ratio(
point.metrics.success_count,
RequestHealth::from_metrics(&point.metrics).eligible_requests(),
),
success_rate: health.success_ratio(),
metrics: point.metrics,
cost_coverage: point.cost_coverage,
costs: point.costs,
backend/crates/gateway-admin/tests/use_case/observability/mod.rs
+66 -29
View File
@@ -364,21 +364,26 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
let services = observability_services(store.clone()).await;
let fixtures = [
(162, 0, 2, 0, 0, 98.780_487_804_878_05),
(297, 0, 2, 5, 0, 99.331_103_678_929_76),
(467, 1, 5, 0, 0, 98.731_501_057_082_45),
(95, 2, 3, 4, 1, 95.0),
(100, 0, 0, 0, 0, 100.0),
(162, 0, 2, 0, 0, 0, Some(98.780_487_804_878_05)),
(297, 0, 2, 5, 0, 0, Some(99.331_103_678_929_76)),
(467, 1, 5, 0, 0, 0, Some(98.731_501_057_082_45)),
(95, 2, 3, 4, 1, 7, Some(95.0)),
(100, 0, 0, 0, 0, 0, Some(100.0)),
(0, 0, 0, 0, 0, 0, None),
(0, 0, 0, 4, 3, 2, None),
(0, 1, 2, 0, 0, 0, Some(0.0)),
];
for (index, (success, failed, incomplete, cancelled, caller_error, expected_percent)) in
fixtures.into_iter().enumerate()
for (
index,
(success, failed, incomplete, cancelled, caller_error, running, expected_percent),
) in fixtures.into_iter().enumerate()
{
let range = observation_range(
now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2),
);
let metrics = RequestMetrics {
request_count: success + failed + incomplete + cancelled + caller_error,
request_count: success + failed + incomplete + cancelled + caller_error + running,
success_count: success,
failure_count: failed + caller_error,
incomplete_count: incomplete,
@@ -399,6 +404,27 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate
providers: Vec::new(),
});
let mut item = diagnostic(
"model",
success + failed + incomplete + cancelled + caller_error + running,
);
item.success_count = success;
item.failure_count = failed + caller_error;
item.incomplete_count = incomplete;
item.cancelled_count = cancelled;
item.caller_error_count = caller_error;
store.replace_diagnostics(DiagnosticsObservation {
analytics: Default::default(),
total_request_count: item.request_count,
items: vec![item],
});
let diagnostics = services
.observability()
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Model)
.await
.unwrap();
assert_eq!(diagnostics.items[0].error_count, failed + incomplete);
let dashboard = services
.observability()
.dashboard_summary(range, TrendKind::Usage)
@@ -410,23 +436,12 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate
.iter()
.find(|point| point.bucket_start == current_slot)
.expect("current health slot");
assert_eq!(point.status, HealthStatus::Stable);
assert_eq!(dashboard.trend.summary.error_count, failed + incomplete);
assert_eq!(dashboard.trend.points[0].error_count, failed + incomplete);
assert_eq!(point.failed_requests, failed);
assert_eq!(point.incomplete_requests, incomplete);
assert_eq!(point.cancelled_requests, cancelled);
assert_eq!(point.caller_error_requests, caller_error);
for reliability in [timeline.reliability_percent, point.reliability_percent] {
assert!((reliability.expect("health reliability") - expected_percent).abs() < 1e-10);
}
for success_rate in [
dashboard.trend.summary.success_rate,
dashboard.trend.points[0].success_rate,
] {
assert!(
(success_rate.expect("trend success rate") * 100.0 - expected_percent).abs()
< 1e-10
);
}
let insights = services
.observability()
@@ -435,11 +450,31 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate
.expect("usage insights");
assert_eq!(insights.health.failed_requests, failed);
assert_eq!(insights.health.incomplete_requests, incomplete);
assert!((insights.health.success_rate * 100.0 - expected_percent).abs() < 1e-10);
assert!(
(insights.health.points[0].error_rate * 100.0 - (100.0 - expected_percent)).abs()
< 1e-10
);
for actual in [
timeline.reliability_percent,
point.reliability_percent,
dashboard
.trend
.summary
.success_rate
.map(|rate| rate * 100.0),
dashboard.trend.points[0]
.success_rate
.map(|rate| rate * 100.0),
insights.health.success_rate.map(|rate| rate * 100.0),
insights.health.points[0]
.error_rate
.map(|rate| (1.0 - rate) * 100.0),
diagnostics.items[0]
.error_rate
.map(|rate| (1.0 - rate) * 100.0),
] {
match (actual, expected_percent) {
(Some(actual), Some(expected)) => assert!((actual - expected).abs() < 1e-10),
(None, None) => {}
_ => panic!("unexpected rate: {actual:?}, expected {expected_percent:?}"),
}
}
}
}
@@ -678,7 +713,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
assert_eq!(insights.health.total_requests, 10);
assert_eq!(insights.health.failed_requests, 2);
assert_eq!(insights.health.success_rate, 0.75);
assert_eq!(insights.health.success_rate, Some(0.75));
assert_eq!(insights.health.completion_rate, 1.0);
assert_eq!(insights.granularity, Granularity::FifteenMinutes);
assert_eq!(
@@ -833,7 +868,7 @@ async fn diagnostics_should_preserve_request_order_when_a_small_sample_fails() {
assert_eq!(result.items[1].key, "single_failure");
assert_eq!(result.items[0].error_count, 0);
assert_eq!(result.items[1].error_count, 1);
assert_eq!(result.items[1].error_rate, 1.0);
assert_eq!(result.items[1].error_rate, Some(1.0));
}
#[tokio::test]
@@ -1565,7 +1600,9 @@ fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) ->
fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
DiagnosticObservation {
first_token_p95_ms: None,
non_completion_count: 0,
cancelled_count: 0,
incomplete_count: 0,
caller_error_count: 0,
retry_count: 0,
retried_request_count: 0,
account_provider_kind: None,
backend/crates/gateway-api/src/admin/observability/presenter.rs
+3 -3
View File
@@ -701,8 +701,8 @@ pub(crate) fn trend_point_view(
cached_input_tokens_value: point.metrics.cached_input_tokens,
cache_hit_rate_value: point.cached_token_rate,
tokens_value: point.metrics.total_tokens,
errors: format_compact_number(point.service_failure_count),
errors_value: point.service_failure_count,
errors: format_compact_number(point.error_count),
errors_value: point.error_count,
latency_value: point.average_latency_ms,
success_rate: success_rate_value
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
@@ -765,7 +765,7 @@ pub(crate) fn trend_summary_view(
TrendKind::Errors => vec![
TrendSummaryView {
metric: TrendMetric::Errors,
value: format_compact_number(summary.service_failure_count),
value: format_compact_number(summary.error_count),
ratio: None,
},
TrendSummaryView {
backend/crates/gateway-api/src/admin/observability/wire.rs
+3 -3
View File
@@ -656,7 +656,7 @@ pub struct OverviewHealthPointView {
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub error_rate: f64,
pub error_rate: Option<f64>,
}
/// 洞察健康摘要
@@ -669,7 +669,7 @@ pub struct OverviewHealthView {
pub cancelled_requests: u64,
pub incomplete_requests: u64,
pub caller_error_requests: u64,
pub success_rate: f64,
pub success_rate: Option<f64>,
pub completion_rate: f64,
pub request_change_rate: Option<f64>,
pub success_rate_change: Option<f64>,
@@ -879,7 +879,7 @@ pub struct DiagnosticItemView {
pub request_count: u64,
pub success_count: u64,
pub error_count: u64,
pub error_rate: f64,
pub error_rate: Option<f64>,
pub request_share: f64,
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
backend/crates/gateway-api/tests/admin/observability/response.rs
+3 -1
View File
@@ -757,7 +757,9 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
average_latency_ms: Some(100),
latency_p95_ms: Some(3800),
first_token_p95_ms: Some(1200),
non_completion_count: 0,
cancelled_count: 0,
incomplete_count: 0,
caller_error_count: 0,
retry_count: 3,
retried_request_count: 1,
cost_coverage: CostCoverage::default(),
backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs
+9 -2
View File
@@ -141,7 +141,7 @@ fn counter_expressions() -> Vec<(String, String)> {
let mut counters = vec![
("request_count".to_owned(), "count(*)".to_owned()),
("completed_request_count".to_owned(), "count(*) filter(where is_completed_usage)".to_owned()),
("caller_error_count".to_owned(), "count(*) filter(where client_status_code between 400 and 499)".to_owned()),
("caller_error_count".to_owned(), "count(*) filter(where outcome='failed' and client_status_code between 400 and 499)".to_owned()),
("attempt_count".to_owned(), "coalesce(sum(attempt_count),0)".to_owned()),
("retry_count".to_owned(), "coalesce(sum(greatest(attempt_count-1,0)),0)".to_owned()),
("retried_request_count".to_owned(), "count(*) filter(where attempt_count>1)".to_owned()),
@@ -347,7 +347,14 @@ fn push_summary_aggregation_from(
let counters = selected_counter_expressions(counter_names)
.into_iter()
.map(|(name, _)| {
let expression = format!("coalesce(sum(metric.{name}),0)");
// 汇总按终态分行，取消请求的 499 不属于调用方失败
// 直接筛选已有桶计数，历史统计无需重建
let expression = if source == "matching" && name == "caller_error_count" {
"coalesce(sum(metric.caller_error_count) filter(where outcome='failed'),0)"
.to_owned()
} else {
format!("coalesce(sum(metric.{name}),0)")
};
(name, expression)
})
.collect::<Vec<_>>();
backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs
+1 -1
View File
@@ -156,7 +156,7 @@ impl MetricState {
}
}
let client_status: Option<i32> = get(row, "client_status_code")?;
if client_status.is_some_and(|value| (400..=499).contains(&value)) {
if outcome == "failed" && client_status.is_some_and(|value| (400..=499).contains(&value)) {
self.increment("caller_error_count", 1)?;
}
let attempts = u64::try_from(get::<i32>(row, "attempt_count")?)
backend/crates/gateway-store/src/postgres/observability/analytics/query.rs
+5 -5
View File
@@ -602,6 +602,7 @@ impl<'a> AnalyticsSnapshot<'a> {
"latency_ms_sum",
"cancelled_count",
"incomplete_count",
"caller_error_count",
"retry_count",
"retried_request_count",
"provider_reported_count",
@@ -650,10 +651,9 @@ impl<'a> AnalyticsSnapshot<'a> {
first_token_p95_ms: state
.percentile("first_token_ms", 0.95)
.map(|value| value.round() as u64),
non_completion_count: metrics
.cancelled_count
.checked_add(metrics.incomplete_count)
.ok_or_else(|| invalid("non-completion count overflow"))?,
cancelled_count: metrics.cancelled_count,
incomplete_count: metrics.incomplete_count,
caller_error_count: metrics.caller_error_count,
retry_count: state.count("retry_count"),
retried_request_count: state.count("retried_request_count"),
cost_coverage: state.coverage(),
@@ -1002,7 +1002,7 @@ impl<'a> AnalyticsSnapshot<'a> {
} else {
"'{}'::jsonb as distributions"
};
query.push(format!(" as output_bucket,{name} as name,counts,{costs},{distribution_sql},last_used_at from {table} a where collection="))
query.push(format!(" as output_bucket,{name} as name,outcome,counts,{costs},{distribution_sql},last_used_at from {table} a where collection="))
.push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION)
.push(" and histogram_version=").push_bind(HISTOGRAM_VERSION);
push_summary_buckets(&mut query, &buckets, seconds);
backend/crates/gateway-store/src/postgres/observability/mapping.rs
+3 -1
View File
@@ -290,7 +290,9 @@ fn admin_diagnostic_observation(
average_latency_ms: observation.average_latency_ms,
latency_p95_ms: observation.latency_p95_ms,
first_token_p95_ms: observation.first_token_p95_ms,
non_completion_count: observation.non_completion_count,
cancelled_count: observation.cancelled_count,
incomplete_count: observation.incomplete_count,
caller_error_count: observation.caller_error_count,
retry_count: observation.retry_count,
retried_request_count: observation.retried_request_count,
cost_coverage: admin_cost_coverage(observation.cost_coverage),
backend/crates/gateway-store/src/postgres/observability/model.rs
+3 -1
View File
@@ -388,7 +388,9 @@ pub struct DiagnosticObservation {
pub average_latency_ms: Option<u64>,
pub latency_p95_ms: Option<u64>,
pub first_token_p95_ms: Option<u64>,
pub non_completion_count: u64,
pub cancelled_count: u64,
pub incomplete_count: u64,
pub caller_error_count: u64,
pub retry_count: u64,
pub retried_request_count: u64,
pub cost_coverage: CostCoverage,
backend/crates/gateway-store/tests/postgres/analytics.rs
+97 -1
View File
@@ -372,12 +372,108 @@ async fn provider_counts_match_for_rollups_boundaries_and_filtered_facts() {
);
assert_eq!(openai.retry_count, 1);
assert_eq!(openai.retried_request_count, 1);
assert_eq!(openai.non_completion_count, 0);
assert_eq!(openai.cancelled_count + openai.incomplete_count, 0);
assert_eq!(openai.cost_coverage.unavailable_count, 1);
}
database.close().await;
}
#[tokio::test]
async fn health_counters_match_for_rollups_and_facts_without_counting_recovery_or_cancelled_499() {
let Some(database) = TestDatabase::create("analytics_health_counters").await else {
return;
};
let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
.unwrap()
.with_timezone(&Utc);
for (id, outcome, status) in [
("health_success", "succeeded", 200),
("health_failed", "failed", 502),
("health_incomplete", "incomplete", 200),
("health_caller", "failed", 400),
("health_cancelled", "cancelled", 499),
("health_recovered", "failed", 502),
] {
seed(&database.pool, id, start + TimeDelta::minutes(5), &[]).await;
sqlx::query("update model_requests set outcome=$2,client_status_code=$3 where id=$1")
.bind(id)
.bind(outcome)
.bind(status)
.execute(&database.pool)
.await
.unwrap();
}
sqlx::query("update model_requests set recovered_at=completed_at,recovery_request_id='health_success',recovery_attempt_count=1,request_observation_json=request_observation_json||jsonb_build_object('recovery',jsonb_build_object('retryDelayMs',0,'totalLatencyMs',1)) where id='health_recovered'")
.execute(&database.pool).await.unwrap();
drain_analytics(&database.pool).await;
// 模拟旧桶曾把取消的 499 计入调用方错误，无需重建也应得到相同结果
sqlx::raw_sql("update request_analytics_15m set counts=jsonb_set(counts,'{caller_error_count}','1') where outcome='cancelled'; update request_analytics_1h set counts=jsonb_set(counts,'{caller_error_count}','1') where outcome='cancelled'")
.execute(&database.pool).await.unwrap();
let repo = observability_repository(&database.pool);
let full = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap();
// 完整小时桶、十五分钟桶、边界事实与筛选回退复用同一批请求
for (range, filter) in [
(full, UsageRecordFilter::default()),
(
ObservabilityRange::new(start, start + TimeDelta::minutes(15)).unwrap(),
UsageRecordFilter::default(),
),
(
ObservabilityRange::new(start + TimeDelta::minutes(4), start + TimeDelta::minutes(6))
.unwrap(),
UsageRecordFilter::default(),
),
(
full,
UsageRecordFilter {
search: Some("model_".to_owned()),
..Default::default()
},
),
] {
let snapshot = repo
.usage_insights_observation(
range,
filter.clone(),
gateway_admin::model::observability::Granularity::Hour,
true,
)
.await
.unwrap();
let counts = snapshot.overview.requests;
assert_eq!(
(
counts.request_count,
counts.success_count,
counts.failure_count,
counts.incomplete_count,
counts.cancelled_count,
counts.caller_error_count
),
(5, 1, 2, 1, 1, 1)
);
let diagnostic = repo
.usage_diagnostics(range, filter, DiagnosticDimension::Model, 100)
.await
.unwrap();
assert_eq!(diagnostic.total_request_count, 5);
assert_eq!(diagnostic.items.len(), 1);
let counts = &diagnostic.items[0];
assert_eq!(
(
counts.request_count,
counts.success_count,
counts.failure_count,
counts.incomplete_count,
counts.cancelled_count,
counts.caller_error_count
),
(5, 1, 2, 1, 1, 1)
);
}
database.close().await;
}
#[tokio::test]
async fn insights_share_bucket_states_without_merging_exact_percentiles() {
let Some(database) = TestDatabase::create("analytics_shared_insights").await else {
docs/api.md
+5 -2
View File
@@ -2629,8 +2629,11 @@ OpenAI 推理强度优先取 `input` 中最后一条 `type: "configuration_updat
原生续接未再次更新时沿用已关联响应的配置，否则取请求顶层 `reasoning.effort`。
该字段是请求观测，不改写上游请求；网关未持有的续接历史无法用于推断档位
请求健康可用性与服务成功率按 `成功 /（成功 + 服务失败 + 未完成）` 计算
未完成保留独立计数，客户端取消与调用方错误不计入该分母，中间尝试失败不重复计为最终失败请求
首页趋势、请求健康与热点诊断统一口径：错误为服务失败与未完成之和，成功率为 `成功 /（成功 + 错误）`
已恢复记录、客户端取消与调用方错误不计入，进行中不参与比例；调用方错误仅指失败终态的 4xx，不含取消请求的 499
无有效样本时，成功率与错误率返回 `null`，趋势线保留断点
请求健康保留各终态的独立计数，热点诊断的 `errorCount`、`errorRate` 使用相同口径，中间尝试失败不重复计入
`completionRate` 表示成功或失败终态占总请求的比例，`nonCompletionCount`、`nonCompletionRate` 表示取消与未完成的数量及其占总请求的比例
健康时间线以 15 分钟分桶，可用性不低于 95% 且有效样本不少于 10 个时显示稳定
没有成功且服务失败与未完成合计不少于 3 个时显示不可达，其他不足 10 个有效样本的时段显示低样本
有效样本为零时显示无有效样本，尚未到达的时间格显示未来
frontend/src/api/modules/usage.ts
+3 -3
View File
@@ -257,7 +257,7 @@ export interface UsageOverviewHealthPoint {
cancelledRequests: number
incompleteRequests: number
callerErrorRequests: number
errorRate: number
errorRate: number | null
}
export interface UsageOverviewHealth {
@@ -267,7 +267,7 @@ export interface UsageOverviewHealth {
cancelledRequests: number
incompleteRequests: number
callerErrorRequests: number
successRate: number
successRate: number | null
completionRate: number
requestChangeRate: number | null
successRateChange: number | null
@@ -386,7 +386,7 @@ export interface UsageDiagnosticItem {
requestCount: number
successCount: number
errorCount: number
errorRate: number
errorRate: number | null
requestShare: number
averageLatencyMs: number | null
latencyP95Ms: number | null
frontend/src/i18n/locales/en/dashboard.json
+1 -1
View File
@@ -54,7 +54,7 @@
"firstTokenP95Ms": "TTFT P95",
"latencyP95Ms": "Latency P95",
"outputThroughputP50": "Output throughput P50",
"errors": "Errors",
"errors": "Error count",
"successRate": "Success rate",
"totalRequests": "Total requests",
"requestCount": "Requests",
frontend/src/i18n/locales/en/usage.json
+8 -7
View File
@@ -16,14 +16,15 @@
"serviceSuccessRate": "Service success",
"notCompleted": "Incomplete",
"requestVolume": "Requests",
"noServiceResult": "No service result",
"title": "Health",
"description": "Request outcomes",
"descriptionWithGranularity": "Request outcomes · {granularityText}",
"completionRate": "Completion",
"abnormalEnd": "Abnormal",
"loading": "Loading requests",
"empty": "No requests in this range"
"empty": "No requests in this range",
"errors": "Errors",
"noSamples": "No samples",
"completionRate": "Completion",
"abnormalEnd": "Abnormal"
},
"billing": {
"serviceLevel": "Service tier",
@@ -233,13 +234,13 @@
"diagnosticDimension": "Diagnostic dimension",
"noDiagnosticDataYet": "No diagnostic data yet",
"successful": "Successful: {count}",
"failed": "Failed: {count}",
"failureRate": "Failure rate: {rate}",
"retryRate": "Retry rate: {rate}",
"retries": "Retries: {count}",
"loading": "Loading diagnostics",
"empty": "No diagnostics in this range",
"download": "Download"
"download": "Download",
"errors": "Errors: {count}",
"errorRate": "Error rate: {rate}"
},
"cost": {
"savings": "Savings",
frontend/src/i18n/locales/zh-Hans/usage.json
+8 -7
View File
@@ -16,14 +16,15 @@
"serviceSuccessRate": "服务成功率",
"notCompleted": "未完成",
"requestVolume": "请求量",
"noServiceResult": "无服务结果",
"title": "请求健康",
"description": "请求状态分布",
"descriptionWithGranularity": "请求状态分布 · {granularityText}",
"completionRate": "完成率",
"abnormalEnd": "非正常结束",
"loading": "正在加载请求数据",
"empty": "当前范围暂无请求"
"empty": "当前范围暂无请求",
"errors": "错误",
"noSamples": "无样本",
"completionRate": "完成率",
"abnormalEnd": "非正常结束"
},
"billing": {
"serviceLevel": "服务档位",
@@ -233,13 +234,13 @@
"diagnosticDimension": "诊断维度",
"noDiagnosticDataYet": "暂无诊断数据",
"successful": "成功 {count}",
"failed": "失败 {count} 次",
"failureRate": "失败率 {rate}",
"retryRate": "重试率 {rate}",
"retries": "重试 {count}",
"loading": "正在加载诊断数据",
"empty": "当前范围暂无诊断数据",
"download": "下载"
"download": "下载",
"errors": "错误 {count} 次",
"errorRate": "错误率 {rate}"
},
"cost": {
"savings": "节省",
frontend/src/i18n/locales/zh-Hant/usage.json
+8 -7
View File
@@ -16,14 +16,15 @@
"serviceSuccessRate": "服務成功率",
"notCompleted": "未完成",
"requestVolume": "請求量",
"noServiceResult": "無服務結果",
"title": "請求健康",
"description": "請求狀態分佈",
"descriptionWithGranularity": "請求狀態分佈 · {granularityText}",
"completionRate": "完成率",
"abnormalEnd": "非正常結束",
"loading": "正在加載請求數據",
"empty": "當前範圍暫無請求"
"empty": "當前範圍暫無請求",
"errors": "錯誤",
"noSamples": "無樣本",
"completionRate": "完成率",
"abnormalEnd": "非正常結束"
},
"billing": {
"serviceLevel": "服務檔位",
@@ -233,13 +234,13 @@
"diagnosticDimension": "診斷維度",
"noDiagnosticDataYet": "暫無診斷數據",
"successful": "成功 {count}",
"failed": "失敗 {count} 次",
"failureRate": "失敗率 {rate}",
"retryRate": "重試率 {rate}",
"retries": "重試 {count}",
"loading": "正在加載診斷數據",
"empty": "當前範圍暫無診斷數據",
"download": "下載"
"download": "下載",
"errors": "錯誤 {count} 次",
"errorRate": "錯誤率 {rate}"
},
"cost": {
"savings": "節省",
frontend/src/views/dashboard/composables/useRequestTrendChart.ts
+4 -2
View File
@@ -254,7 +254,9 @@ export function useRequestTrendChart(options: {
case 'errors':
return point?.errors ?? value
case 'successRate':
return zeroAwareRateText(point, 'successRate', 'requestsValue') ?? value
return point?.successRateValue == null
? t('usage.health.noSamples')
: trendPointText(point, 'successRate') ?? value
case 'requests':
return point?.requests ?? value
default:
@@ -342,7 +344,7 @@ export function useRequestTrendChart(options: {
),
...gapLineSeries(
'successRate',
activeSeriesValues('successRateValue'),
seriesValues('successRateValue'),
trendColor('successRate'),
{ width: 2.4, yAxisIndex: 1 },
),
frontend/src/views/dashboard/presenter.ts
+2 -2
View File
@@ -115,8 +115,8 @@ export function dashboardTrendView(trend: DashboardTrendResponse | null) {
const summary = trend.summary.map(item => ({
...trendMetricView(item.metric),
value: trend.kind === 'errors' && item.ratio !== null
? item.ratio
value: trend.kind === 'errors' && item.metric === 'successRate'
? item.ratio ?? t('usage.health.noSamples')
: item.value,
}))
return { points: trend.points, summary }
frontend/src/views/usage/components/UsageDiagnosticCard.vue
+9 -8
View File
@@ -60,7 +60,7 @@ const resultDimensionLabel = computed(
)
const dimensionNameSizes: Record<string, TableColumnSize> = {
model: 'xl',
model: 'lg',
account: '2xl',
apiKey: 'md',
provider: 'lg',
@@ -90,25 +90,26 @@ const diagnosticColumns = computed(() => defineTableColumns<DiagnosticDisplayIte
key: 'requestCount',
label: t('usage.diagnostics.request'),
kind: 'numeric',
size: 'xs',
size: 'md',
},
{
key: 'errorCount',
label: t('common.status.failed'),
label: t('usage.health.errors'),
kind: 'numeric',
size: 'xs',
align: 'center',
size: 'md',
},
{
key: 'firstTokenP95Ms',
label: t('usage.diagnostics.performance'),
kind: 'numeric',
size: 'lg',
size: 'md',
},
{
key: 'estimatedCost',
label: t('usage.fields.cost'),
kind: 'numeric',
size: 'sm',
size: 'md',
},
]))
@@ -228,8 +229,8 @@ function diagnosticNameDisplay(name: string) {
<strong
class="font-mono font-bold tabular-nums"
:class="row.errorCount > 0 ? 'text-z-error-text' : 'text-z-text-quaternary'"
:aria-label="t('usage.diagnostics.failed', { count: formatCompactNumber(row.errorCount) })"
:title="t('usage.diagnostics.failureRate', { rate: formatPercent(row.errorRate) })"
:aria-label="t('usage.diagnostics.errors', { count: formatCompactNumber(row.errorCount) })"
:title="t('usage.diagnostics.errorRate', { rate: formatPercent(row.errorRate) })"
>
{{ formatCompactNumber(row.errorCount) }}
</strong>
frontend/src/views/usage/components/UsageHealthCard.vue
+4 -8
View File
@@ -10,7 +10,6 @@ import { computed } from 'vue'
import { useI18n } from 'vue-i18n'
import BaseChart from '@/components/charts/BaseChart.vue'
import { chartTooltipContent, tooltipIndex, tooltipRows } from '@/components/charts/tooltip'
import { healthEligibleRequestCount } from '@/components/usage/shared/health'
import { useChartPalette } from '@/composables/useChartPalette'
import { formatLocalizedCompactNumber as formatCompactNumber, formatPercent } from '@/utils/format'
@@ -132,12 +131,9 @@ function requestCount(point: HealthPoint) {
}
function serviceSuccessRate(point: HealthPoint) {
const total = healthEligibleRequestCount(point)
if (total > 0)
return 1 - point.errorRate
return requestCount(point) === 0
? 0
: null
return point.errorRate == null
? null
: 1 - point.errorRate
}
function outcomeSeries(name: string, data: number[], color: string) {
@@ -170,7 +166,7 @@ function formatTooltip(params: unknown) {
usageTooltipItem(
t('usage.health.serviceSuccessRate'),
successRate == null
? t('usage.health.noServiceResult')
? t('usage.health.noSamples')
: formatPercent(successRate),
theme.success,
),