From 3e33e84a45e740aaf15650c2ab26af5c3d3db88b Mon Sep 17 00:00:00 2001 From: Zyy Date: Thu, 5 Nov 2026 21:50:04 +0800 Subject: [PATCH] fix(observability): unify request error statistics Count incomplete requests as errors, exclude recovered and cancelled requests from success rates, and preserve gaps for missing samples. Co-authored-by: Codex --- .../gateway-admin/src/model/observability.rs | 26 +++-- .../src/use_case/observability/mod.rs | 46 +++++---- .../tests/use_case/observability/mod.rs | 95 ++++++++++++------ .../src/admin/observability/presenter.rs | 6 +- .../src/admin/observability/wire.rs | 6 +- .../tests/admin/observability/response.rs | 4 +- .../observability/analytics/aggregate.rs | 11 ++- .../observability/analytics/metrics.rs | 2 +- .../postgres/observability/analytics/query.rs | 10 +- .../src/postgres/observability/mapping.rs | 4 +- .../src/postgres/observability/model.rs | 4 +- .../gateway-store/tests/postgres/analytics.rs | 98 ++++++++++++++++++- docs/api.md | 7 +- frontend/src/api/modules/usage.ts | 6 +- frontend/src/i18n/locales/en/dashboard.json | 2 +- frontend/src/i18n/locales/en/usage.json | 15 +-- frontend/src/i18n/locales/zh-Hans/usage.json | 15 +-- frontend/src/i18n/locales/zh-Hant/usage.json | 15 +-- .../composables/useRequestTrendChart.ts | 6 +- frontend/src/views/dashboard/presenter.ts | 4 +- .../usage/components/UsageDiagnosticCard.vue | 17 ++-- .../usage/components/UsageHealthCard.vue | 12 +-- 22 files changed, 286 insertions(+), 125 deletions(-) diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index 42d3c5ddd..d7a5f1ef4 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -510,10 +510,20 @@ impl RequestHealth { ) } + /// 账号与健康时间线使用百分数，洞察接口使用 0 到 1 的比例 #[must_use] pub fn success_rate(self) -> Option { + self.success_ratio().map(|ratio| ratio * 100.0) + } + + pub(crate) fn success_ratio(self) -> Option { let total = self.eligible_requests(); - (total > 0).then(|| self.success_requests as f64 / total as f64 * 100.0) + (total > 0).then(|| self.success_requests as f64 / total as f64) + } + + pub(crate) fn error_ratio(self) -> Option { + let total = self.eligible_requests(); + (total > 0).then(|| self.non_success_requests() as f64 / total as f64) } // 未完成保留交付终态，但上游失败、超时和进程中断都不能视为服务成功 @@ -1081,7 +1091,9 @@ pub struct DiagnosticObservation { pub average_latency_ms: Option, pub latency_p95_ms: Option, pub first_token_p95_ms: Option, - pub non_completion_count: u64, + pub cancelled_count: u64, + pub incomplete_count: u64, + pub caller_error_count: u64, pub retry_count: u64, pub retried_request_count: u64, pub cost_coverage: CostCoverage, @@ -1169,7 +1181,7 @@ pub struct TrendPoint { pub bucket_start: DateTime, pub granularity: Granularity, pub metrics: RequestMetrics, - pub service_failure_count: u64, + pub error_count: u64, pub average_latency_ms: Option, pub average_first_token_latency_ms: Option, pub cached_token_rate: f64, @@ -1187,7 +1199,7 @@ pub struct TrendSummary { pub output_tokens: u64, pub cached_input_tokens: u64, pub total_tokens: u64, - pub service_failure_count: u64, + pub error_count: u64, pub average_latency_ms: Option, pub max_latency_ms: Option, pub min_latency_ms: Option, @@ -1351,7 +1363,7 @@ pub struct UsageInsightsHealthPoint { pub cancelled_requests: u64, pub incomplete_requests: u64, pub caller_error_requests: u64, - pub error_rate: f64, + pub error_rate: Option, } /// 洞察健康汇总；失败数已排除调用方错误 @@ -1363,7 +1375,7 @@ pub struct UsageInsightsHealth { pub cancelled_requests: u64, pub incomplete_requests: u64, pub caller_error_requests: u64, - pub success_rate: f64, + pub success_rate: Option, pub completion_rate: f64, pub points: Vec, } @@ -1550,7 +1562,7 @@ pub struct DiagnosticsItem { pub request_count: u64, pub success_count: u64, pub error_count: u64, - pub error_rate: f64, + pub error_rate: Option, pub request_share: f64, pub average_latency_ms: Option, pub latency_p95_ms: Option, diff --git a/backend/crates/gateway-admin/src/use_case/observability/mod.rs b/backend/crates/gateway-admin/src/use_case/observability/mod.rs index 190e9a3d4..6955135c6 100644 --- a/backend/crates/gateway-admin/src/use_case/observability/mod.rs +++ b/backend/crates/gateway-admin/src/use_case/observability/mod.rs @@ -356,9 +356,16 @@ impl ObservabilityService for DefaultObservabilityService { .items .into_iter() .map(|item| { - let error_rate = rate_or_zero(item.failure_count, item.request_count); - let non_completion_rate = - rate_or_zero(item.non_completion_count, item.request_count); + let health = RequestHealth::from_counts( + item.success_count, + item.failure_count, + item.cancelled_count, + item.incomplete_count, + item.caller_error_count, + ); + let non_completion_count = + item.cancelled_count.saturating_add(item.incomplete_count); + let non_completion_rate = rate_or_zero(non_completion_count, item.request_count); let retry_rate = rate_or_zero(item.retried_request_count, item.request_count); DiagnosticsItem { account_upstream_peer: peers.get(&item.key).copied(), @@ -374,8 +381,8 @@ impl ObservabilityService for DefaultObservabilityService { account_plan_type: item.account_plan_type, request_count: item.request_count, success_count: item.success_count, - error_count: item.failure_count, - error_rate, + error_count: health.non_success_requests(), + error_rate: health.error_ratio(), request_share: rate_or_zero( item.request_count, observation.total_request_count, @@ -383,7 +390,7 @@ impl ObservabilityService for DefaultObservabilityService { average_latency_ms: item.average_latency_ms, latency_p95_ms: item.latency_p95_ms, first_token_p95_ms: item.first_token_p95_ms, - non_completion_count: item.non_completion_count, + non_completion_count, non_completion_rate, retry_count: item.retry_count, retry_rate, @@ -576,7 +583,6 @@ fn build_usage_insights( let requests = &overview.requests; let health_window = RequestHealth::from_metrics(requests); let failed_requests = health_window.failed_requests; - let health_requests = health_window.eligible_requests(); let health = UsageInsightsHealth { total_requests: requests.request_count, success_requests: requests.success_count, @@ -584,7 +590,7 @@ fn build_usage_insights( cancelled_requests: requests.cancelled_count, incomplete_requests: requests.incomplete_count, caller_error_requests: requests.caller_error_count, - success_rate: rate_or_zero(requests.success_count, health_requests), + success_rate: health_window.success_ratio(), completion_rate: rate_or_zero( requests .success_count @@ -603,10 +609,7 @@ fn build_usage_insights( cancelled_requests: point.metrics.cancelled_count, incomplete_requests: point.metrics.incomplete_count, caller_error_requests: point.metrics.caller_error_count, - error_rate: rate_or_zero( - health_window.non_success_requests(), - health_window.eligible_requests(), - ), + error_rate: health_window.error_ratio(), } }) .collect(), @@ -1215,11 +1218,9 @@ fn trend( } } let average_latency_ms = average(metrics.latency_sum_ms, metrics.latency_count); - let service_failure_count = RequestHealth::from_metrics(&metrics).failed_requests; - let success_rate = ratio( - metrics.success_count, - RequestHealth::from_metrics(&metrics).eligible_requests(), - ); + let health = RequestHealth::from_metrics(&metrics); + let error_count = health.non_success_requests(); + let success_rate = health.success_ratio(); let cache_hit_request_rate = ratio( metrics.cache_hit_request_count, metrics.cache_eligible_request_count, @@ -1248,7 +1249,7 @@ fn trend( output_tokens: metrics.output_tokens, cached_input_tokens: metrics.cached_input_tokens, total_tokens: metrics.total_tokens, - service_failure_count, + error_count, average_latency_ms, max_latency_ms: metrics.max_latency_ms, min_latency_ms: metrics.min_latency_ms, @@ -1267,11 +1268,11 @@ fn trend( } fn trend_point(point: RequestMetricPoint) -> TrendPoint { - let service_failure_count = RequestHealth::from_metrics(&point.metrics).failed_requests; + let health = RequestHealth::from_metrics(&point.metrics); TrendPoint { bucket_start: point.bucket_start, granularity: point.granularity, - service_failure_count, + error_count: health.non_success_requests(), average_latency_ms: average(point.metrics.latency_sum_ms, point.metrics.latency_count), average_first_token_latency_ms: average( point.metrics.first_token_latency_sum_ms, @@ -1285,10 +1286,7 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint { point.metrics.cache_hit_request_count, point.metrics.cache_eligible_request_count, ), - success_rate: ratio( - point.metrics.success_count, - RequestHealth::from_metrics(&point.metrics).eligible_requests(), - ), + success_rate: health.success_ratio(), metrics: point.metrics, cost_coverage: point.cost_coverage, costs: point.costs, diff --git a/backend/crates/gateway-admin/tests/use_case/observability/mod.rs b/backend/crates/gateway-admin/tests/use_case/observability/mod.rs index f15fcce9e..349803cd4 100644 --- a/backend/crates/gateway-admin/tests/use_case/observability/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/observability/mod.rs @@ -364,21 +364,26 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate let store = Arc::new(FixtureObservabilityStore::new(observation_range(now))); let services = observability_services(store.clone()).await; let fixtures = [ - (162, 0, 2, 0, 0, 98.780_487_804_878_05), - (297, 0, 2, 5, 0, 99.331_103_678_929_76), - (467, 1, 5, 0, 0, 98.731_501_057_082_45), - (95, 2, 3, 4, 1, 95.0), - (100, 0, 0, 0, 0, 100.0), + (162, 0, 2, 0, 0, 0, Some(98.780_487_804_878_05)), + (297, 0, 2, 5, 0, 0, Some(99.331_103_678_929_76)), + (467, 1, 5, 0, 0, 0, Some(98.731_501_057_082_45)), + (95, 2, 3, 4, 1, 7, Some(95.0)), + (100, 0, 0, 0, 0, 0, Some(100.0)), + (0, 0, 0, 0, 0, 0, None), + (0, 0, 0, 4, 3, 2, None), + (0, 1, 2, 0, 0, 0, Some(0.0)), ]; - for (index, (success, failed, incomplete, cancelled, caller_error, expected_percent)) in - fixtures.into_iter().enumerate() + for ( + index, + (success, failed, incomplete, cancelled, caller_error, running, expected_percent), + ) in fixtures.into_iter().enumerate() { let range = observation_range( now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2), ); let metrics = RequestMetrics { - request_count: success + failed + incomplete + cancelled + caller_error, + request_count: success + failed + incomplete + cancelled + caller_error + running, success_count: success, failure_count: failed + caller_error, incomplete_count: incomplete, @@ -399,6 +404,27 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate providers: Vec::new(), }); + let mut item = diagnostic( + "model", + success + failed + incomplete + cancelled + caller_error + running, + ); + item.success_count = success; + item.failure_count = failed + caller_error; + item.incomplete_count = incomplete; + item.cancelled_count = cancelled; + item.caller_error_count = caller_error; + store.replace_diagnostics(DiagnosticsObservation { + analytics: Default::default(), + total_request_count: item.request_count, + items: vec![item], + }); + let diagnostics = services + .observability() + .diagnostics(range, UsageFilter::default(), DiagnosticDimension::Model) + .await + .unwrap(); + assert_eq!(diagnostics.items[0].error_count, failed + incomplete); + let dashboard = services .observability() .dashboard_summary(range, TrendKind::Usage) @@ -410,23 +436,12 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate .iter() .find(|point| point.bucket_start == current_slot) .expect("current health slot"); - assert_eq!(point.status, HealthStatus::Stable); + assert_eq!(dashboard.trend.summary.error_count, failed + incomplete); + assert_eq!(dashboard.trend.points[0].error_count, failed + incomplete); assert_eq!(point.failed_requests, failed); assert_eq!(point.incomplete_requests, incomplete); assert_eq!(point.cancelled_requests, cancelled); assert_eq!(point.caller_error_requests, caller_error); - for reliability in [timeline.reliability_percent, point.reliability_percent] { - assert!((reliability.expect("health reliability") - expected_percent).abs() < 1e-10); - } - for success_rate in [ - dashboard.trend.summary.success_rate, - dashboard.trend.points[0].success_rate, - ] { - assert!( - (success_rate.expect("trend success rate") * 100.0 - expected_percent).abs() - < 1e-10 - ); - } let insights = services .observability() @@ -435,11 +450,31 @@ async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate .expect("usage insights"); assert_eq!(insights.health.failed_requests, failed); assert_eq!(insights.health.incomplete_requests, incomplete); - assert!((insights.health.success_rate * 100.0 - expected_percent).abs() < 1e-10); - assert!( - (insights.health.points[0].error_rate * 100.0 - (100.0 - expected_percent)).abs() - < 1e-10 - ); + for actual in [ + timeline.reliability_percent, + point.reliability_percent, + dashboard + .trend + .summary + .success_rate + .map(|rate| rate * 100.0), + dashboard.trend.points[0] + .success_rate + .map(|rate| rate * 100.0), + insights.health.success_rate.map(|rate| rate * 100.0), + insights.health.points[0] + .error_rate + .map(|rate| (1.0 - rate) * 100.0), + diagnostics.items[0] + .error_rate + .map(|rate| (1.0 - rate) * 100.0), + ] { + match (actual, expected_percent) { + (Some(actual), Some(expected)) => assert!((actual - expected).abs() < 1e-10), + (None, None) => {} + _ => panic!("unexpected rate: {actual:?}, expected {expected_percent:?}"), + } + } } } @@ -678,7 +713,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s assert_eq!(insights.health.total_requests, 10); assert_eq!(insights.health.failed_requests, 2); - assert_eq!(insights.health.success_rate, 0.75); + assert_eq!(insights.health.success_rate, Some(0.75)); assert_eq!(insights.health.completion_rate, 1.0); assert_eq!(insights.granularity, Granularity::FifteenMinutes); assert_eq!( @@ -833,7 +868,7 @@ async fn diagnostics_should_preserve_request_order_when_a_small_sample_fails() { assert_eq!(result.items[1].key, "single_failure"); assert_eq!(result.items[0].error_count, 0); assert_eq!(result.items[1].error_count, 1); - assert_eq!(result.items[1].error_rate, 1.0); + assert_eq!(result.items[1].error_rate, Some(1.0)); } #[tokio::test] @@ -1565,7 +1600,9 @@ fn health_metric_point(bucket_start: DateTime, metrics: RequestMetrics) -> fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation { DiagnosticObservation { first_token_p95_ms: None, - non_completion_count: 0, + cancelled_count: 0, + incomplete_count: 0, + caller_error_count: 0, retry_count: 0, retried_request_count: 0, account_provider_kind: None, diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs index 70bd00d19..3efbc4012 100644 --- a/backend/crates/gateway-api/src/admin/observability/presenter.rs +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -701,8 +701,8 @@ pub(crate) fn trend_point_view( cached_input_tokens_value: point.metrics.cached_input_tokens, cache_hit_rate_value: point.cached_token_rate, tokens_value: point.metrics.total_tokens, - errors: format_compact_number(point.service_failure_count), - errors_value: point.service_failure_count, + errors: format_compact_number(point.error_count), + errors_value: point.error_count, latency_value: point.average_latency_ms, success_rate: success_rate_value .map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")), @@ -765,7 +765,7 @@ pub(crate) fn trend_summary_view( TrendKind::Errors => vec![ TrendSummaryView { metric: TrendMetric::Errors, - value: format_compact_number(summary.service_failure_count), + value: format_compact_number(summary.error_count), ratio: None, }, TrendSummaryView { diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs index daf93747c..09dc73979 100644 --- a/backend/crates/gateway-api/src/admin/observability/wire.rs +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -656,7 +656,7 @@ pub struct OverviewHealthPointView { pub cancelled_requests: u64, pub incomplete_requests: u64, pub caller_error_requests: u64, - pub error_rate: f64, + pub error_rate: Option, } /// 洞察健康摘要 @@ -669,7 +669,7 @@ pub struct OverviewHealthView { pub cancelled_requests: u64, pub incomplete_requests: u64, pub caller_error_requests: u64, - pub success_rate: f64, + pub success_rate: Option, pub completion_rate: f64, pub request_change_rate: Option, pub success_rate_change: Option, @@ -879,7 +879,7 @@ pub struct DiagnosticItemView { pub request_count: u64, pub success_count: u64, pub error_count: u64, - pub error_rate: f64, + pub error_rate: Option, pub request_share: f64, pub average_latency_ms: Option, pub latency_p95_ms: Option, diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index 081672cae..b1ad28373 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -757,7 +757,9 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() { average_latency_ms: Some(100), latency_p95_ms: Some(3800), first_token_p95_ms: Some(1200), - non_completion_count: 0, + cancelled_count: 0, + incomplete_count: 0, + caller_error_count: 0, retry_count: 3, retried_request_count: 1, cost_coverage: CostCoverage::default(), diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs index 4da5cbe13..301f94ed2 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/aggregate.rs @@ -141,7 +141,7 @@ fn counter_expressions() -> Vec<(String, String)> { let mut counters = vec![ ("request_count".to_owned(), "count(*)".to_owned()), ("completed_request_count".to_owned(), "count(*) filter(where is_completed_usage)".to_owned()), - ("caller_error_count".to_owned(), "count(*) filter(where client_status_code between 400 and 499)".to_owned()), + ("caller_error_count".to_owned(), "count(*) filter(where outcome='failed' and client_status_code between 400 and 499)".to_owned()), ("attempt_count".to_owned(), "coalesce(sum(attempt_count),0)".to_owned()), ("retry_count".to_owned(), "coalesce(sum(greatest(attempt_count-1,0)),0)".to_owned()), ("retried_request_count".to_owned(), "count(*) filter(where attempt_count>1)".to_owned()), @@ -347,7 +347,14 @@ fn push_summary_aggregation_from( let counters = selected_counter_expressions(counter_names) .into_iter() .map(|(name, _)| { - let expression = format!("coalesce(sum(metric.{name}),0)"); + // 汇总按终态分行，取消请求的 499 不属于调用方失败 + // 直接筛选已有桶计数，历史统计无需重建 + let expression = if source == "matching" && name == "caller_error_count" { + "coalesce(sum(metric.caller_error_count) filter(where outcome='failed'),0)" + .to_owned() + } else { + format!("coalesce(sum(metric.{name}),0)") + }; (name, expression) }) .collect::>(); diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs index 2098d0a6f..95322c411 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/metrics.rs @@ -156,7 +156,7 @@ impl MetricState { } } let client_status: Option = get(row, "client_status_code")?; - if client_status.is_some_and(|value| (400..=499).contains(&value)) { + if outcome == "failed" && client_status.is_some_and(|value| (400..=499).contains(&value)) { self.increment("caller_error_count", 1)?; } let attempts = u64::try_from(get::(row, "attempt_count")?) diff --git a/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs b/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs index de54dd4a7..f32487c34 100644 --- a/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs +++ b/backend/crates/gateway-store/src/postgres/observability/analytics/query.rs @@ -602,6 +602,7 @@ impl<'a> AnalyticsSnapshot<'a> { "latency_ms_sum", "cancelled_count", "incomplete_count", + "caller_error_count", "retry_count", "retried_request_count", "provider_reported_count", @@ -650,10 +651,9 @@ impl<'a> AnalyticsSnapshot<'a> { first_token_p95_ms: state .percentile("first_token_ms", 0.95) .map(|value| value.round() as u64), - non_completion_count: metrics - .cancelled_count - .checked_add(metrics.incomplete_count) - .ok_or_else(|| invalid("non-completion count overflow"))?, + cancelled_count: metrics.cancelled_count, + incomplete_count: metrics.incomplete_count, + caller_error_count: metrics.caller_error_count, retry_count: state.count("retry_count"), retried_request_count: state.count("retried_request_count"), cost_coverage: state.coverage(), @@ -1002,7 +1002,7 @@ impl<'a> AnalyticsSnapshot<'a> { } else { "'{}'::jsonb as distributions" }; - query.push(format!(" as output_bucket,{name} as name,counts,{costs},{distribution_sql},last_used_at from {table} a where collection=")) + query.push(format!(" as output_bucket,{name} as name,outcome,counts,{costs},{distribution_sql},last_used_at from {table} a where collection=")) .push_bind(collection).push(" and schema_version=").push_bind(SCHEMA_VERSION) .push(" and histogram_version=").push_bind(HISTOGRAM_VERSION); push_summary_buckets(&mut query, &buckets, seconds); diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index 2d627e8c1..c0fb97bb2 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -290,7 +290,9 @@ fn admin_diagnostic_observation( average_latency_ms: observation.average_latency_ms, latency_p95_ms: observation.latency_p95_ms, first_token_p95_ms: observation.first_token_p95_ms, - non_completion_count: observation.non_completion_count, + cancelled_count: observation.cancelled_count, + incomplete_count: observation.incomplete_count, + caller_error_count: observation.caller_error_count, retry_count: observation.retry_count, retried_request_count: observation.retried_request_count, cost_coverage: admin_cost_coverage(observation.cost_coverage), diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs index 25ab0aa30..25dc2a109 100644 --- a/backend/crates/gateway-store/src/postgres/observability/model.rs +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -388,7 +388,9 @@ pub struct DiagnosticObservation { pub average_latency_ms: Option, pub latency_p95_ms: Option, pub first_token_p95_ms: Option, - pub non_completion_count: u64, + pub cancelled_count: u64, + pub incomplete_count: u64, + pub caller_error_count: u64, pub retry_count: u64, pub retried_request_count: u64, pub cost_coverage: CostCoverage, diff --git a/backend/crates/gateway-store/tests/postgres/analytics.rs b/backend/crates/gateway-store/tests/postgres/analytics.rs index 034fbc76b..30a8b0b3b 100644 --- a/backend/crates/gateway-store/tests/postgres/analytics.rs +++ b/backend/crates/gateway-store/tests/postgres/analytics.rs @@ -372,12 +372,108 @@ async fn provider_counts_match_for_rollups_boundaries_and_filtered_facts() { ); assert_eq!(openai.retry_count, 1); assert_eq!(openai.retried_request_count, 1); - assert_eq!(openai.non_completion_count, 0); + assert_eq!(openai.cancelled_count + openai.incomplete_count, 0); assert_eq!(openai.cost_coverage.unavailable_count, 1); } database.close().await; } +#[tokio::test] +async fn health_counters_match_for_rollups_and_facts_without_counting_recovery_or_cancelled_499() { + let Some(database) = TestDatabase::create("analytics_health_counters").await else { + return; + }; + let start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z") + .unwrap() + .with_timezone(&Utc); + for (id, outcome, status) in [ + ("health_success", "succeeded", 200), + ("health_failed", "failed", 502), + ("health_incomplete", "incomplete", 200), + ("health_caller", "failed", 400), + ("health_cancelled", "cancelled", 499), + ("health_recovered", "failed", 502), + ] { + seed(&database.pool, id, start + TimeDelta::minutes(5), &[]).await; + sqlx::query("update model_requests set outcome=$2,client_status_code=$3 where id=$1") + .bind(id) + .bind(outcome) + .bind(status) + .execute(&database.pool) + .await + .unwrap(); + } + sqlx::query("update model_requests set recovered_at=completed_at,recovery_request_id='health_success',recovery_attempt_count=1,request_observation_json=request_observation_json||jsonb_build_object('recovery',jsonb_build_object('retryDelayMs',0,'totalLatencyMs',1)) where id='health_recovered'") + .execute(&database.pool).await.unwrap(); + drain_analytics(&database.pool).await; + // 模拟旧桶曾把取消的 499 计入调用方错误，无需重建也应得到相同结果 + sqlx::raw_sql("update request_analytics_15m set counts=jsonb_set(counts,'{caller_error_count}','1') where outcome='cancelled'; update request_analytics_1h set counts=jsonb_set(counts,'{caller_error_count}','1') where outcome='cancelled'") + .execute(&database.pool).await.unwrap(); + let repo = observability_repository(&database.pool); + let full = ObservabilityRange::new(start, start + TimeDelta::hours(1)).unwrap(); + // 完整小时桶、十五分钟桶、边界事实与筛选回退复用同一批请求 + for (range, filter) in [ + (full, UsageRecordFilter::default()), + ( + ObservabilityRange::new(start, start + TimeDelta::minutes(15)).unwrap(), + UsageRecordFilter::default(), + ), + ( + ObservabilityRange::new(start + TimeDelta::minutes(4), start + TimeDelta::minutes(6)) + .unwrap(), + UsageRecordFilter::default(), + ), + ( + full, + UsageRecordFilter { + search: Some("model_".to_owned()), + ..Default::default() + }, + ), + ] { + let snapshot = repo + .usage_insights_observation( + range, + filter.clone(), + gateway_admin::model::observability::Granularity::Hour, + true, + ) + .await + .unwrap(); + let counts = snapshot.overview.requests; + assert_eq!( + ( + counts.request_count, + counts.success_count, + counts.failure_count, + counts.incomplete_count, + counts.cancelled_count, + counts.caller_error_count + ), + (5, 1, 2, 1, 1, 1) + ); + let diagnostic = repo + .usage_diagnostics(range, filter, DiagnosticDimension::Model, 100) + .await + .unwrap(); + assert_eq!(diagnostic.total_request_count, 5); + assert_eq!(diagnostic.items.len(), 1); + let counts = &diagnostic.items[0]; + assert_eq!( + ( + counts.request_count, + counts.success_count, + counts.failure_count, + counts.incomplete_count, + counts.cancelled_count, + counts.caller_error_count + ), + (5, 1, 2, 1, 1, 1) + ); + } + database.close().await; +} + #[tokio::test] async fn insights_share_bucket_states_without_merging_exact_percentiles() { let Some(database) = TestDatabase::create("analytics_shared_insights").await else { diff --git a/docs/api.md b/docs/api.md index ef75621ef..e28b2db71 100644 --- a/docs/api.md +++ b/docs/api.md @@ -2629,8 +2629,11 @@ OpenAI 推理强度优先取 `input` 中最后一条 `type: "configuration_updat 原生续接未再次更新时沿用已关联响应的配置，否则取请求顶层 `reasoning.effort`。 该字段是请求观测，不改写上游请求；网关未持有的续接历史无法用于推断档位 -请求健康可用性与服务成功率按 `成功 /（成功 + 服务失败 + 未完成）` 计算 -未完成保留独立计数，客户端取消与调用方错误不计入该分母，中间尝试失败不重复计为最终失败请求 +首页趋势、请求健康与热点诊断统一口径：错误为服务失败与未完成之和，成功率为 `成功 /（成功 + 错误）` +已恢复记录、客户端取消与调用方错误不计入，进行中不参与比例；调用方错误仅指失败终态的 4xx，不含取消请求的 499 +无有效样本时，成功率与错误率返回 `null`，趋势线保留断点 +请求健康保留各终态的独立计数，热点诊断的 `errorCount`、`errorRate` 使用相同口径，中间尝试失败不重复计入 +`completionRate` 表示成功或失败终态占总请求的比例，`nonCompletionCount`、`nonCompletionRate` 表示取消与未完成的数量及其占总请求的比例 健康时间线以 15 分钟分桶，可用性不低于 95% 且有效样本不少于 10 个时显示稳定 没有成功且服务失败与未完成合计不少于 3 个时显示不可达，其他不足 10 个有效样本的时段显示低样本 有效样本为零时显示无有效样本，尚未到达的时间格显示未来 diff --git a/frontend/src/api/modules/usage.ts b/frontend/src/api/modules/usage.ts index 9e9980c7c..b2a29e507 100644 --- a/frontend/src/api/modules/usage.ts +++ b/frontend/src/api/modules/usage.ts @@ -257,7 +257,7 @@ export interface UsageOverviewHealthPoint { cancelledRequests: number incompleteRequests: number callerErrorRequests: number - errorRate: number + errorRate: number | null } export interface UsageOverviewHealth { @@ -267,7 +267,7 @@ export interface UsageOverviewHealth { cancelledRequests: number incompleteRequests: number callerErrorRequests: number - successRate: number + successRate: number | null completionRate: number requestChangeRate: number | null successRateChange: number | null @@ -386,7 +386,7 @@ export interface UsageDiagnosticItem { requestCount: number successCount: number errorCount: number - errorRate: number + errorRate: number | null requestShare: number averageLatencyMs: number | null latencyP95Ms: number | null diff --git a/frontend/src/i18n/locales/en/dashboard.json b/frontend/src/i18n/locales/en/dashboard.json index 0230cfdf5..7c768a587 100644 --- a/frontend/src/i18n/locales/en/dashboard.json +++ b/frontend/src/i18n/locales/en/dashboard.json @@ -54,7 +54,7 @@ "firstTokenP95Ms": "TTFT P95", "latencyP95Ms": "Latency P95", "outputThroughputP50": "Output throughput P50", - "errors": "Errors", + "errors": "Error count", "successRate": "Success rate", "totalRequests": "Total requests", "requestCount": "Requests", diff --git a/frontend/src/i18n/locales/en/usage.json b/frontend/src/i18n/locales/en/usage.json index ecabea90c..26624d4c0 100644 --- a/frontend/src/i18n/locales/en/usage.json +++ b/frontend/src/i18n/locales/en/usage.json @@ -16,14 +16,15 @@ "serviceSuccessRate": "Service success", "notCompleted": "Incomplete", "requestVolume": "Requests", - "noServiceResult": "No service result", "title": "Health", "description": "Request outcomes", "descriptionWithGranularity": "Request outcomes · {granularityText}", - "completionRate": "Completion", - "abnormalEnd": "Abnormal", "loading": "Loading requests", - "empty": "No requests in this range" + "empty": "No requests in this range", + "errors": "Errors", + "noSamples": "No samples", + "completionRate": "Completion", + "abnormalEnd": "Abnormal" }, "billing": { "serviceLevel": "Service tier", @@ -233,13 +234,13 @@ "diagnosticDimension": "Diagnostic dimension", "noDiagnosticDataYet": "No diagnostic data yet", "successful": "Successful: {count}", - "failed": "Failed: {count}", - "failureRate": "Failure rate: {rate}", "retryRate": "Retry rate: {rate}", "retries": "Retries: {count}", "loading": "Loading diagnostics", "empty": "No diagnostics in this range", - "download": "Download" + "download": "Download", + "errors": "Errors: {count}", + "errorRate": "Error rate: {rate}" }, "cost": { "savings": "Savings", diff --git a/frontend/src/i18n/locales/zh-Hans/usage.json b/frontend/src/i18n/locales/zh-Hans/usage.json index ba9cb9842..cf09f9dee 100644 --- a/frontend/src/i18n/locales/zh-Hans/usage.json +++ b/frontend/src/i18n/locales/zh-Hans/usage.json @@ -16,14 +16,15 @@ "serviceSuccessRate": "服务成功率", "notCompleted": "未完成", "requestVolume": "请求量", - "noServiceResult": "无服务结果", "title": "请求健康", "description": "请求状态分布", "descriptionWithGranularity": "请求状态分布 · {granularityText}", - "completionRate": "完成率", - "abnormalEnd": "非正常结束", "loading": "正在加载请求数据", - "empty": "当前范围暂无请求" + "empty": "当前范围暂无请求", + "errors": "错误", + "noSamples": "无样本", + "completionRate": "完成率", + "abnormalEnd": "非正常结束" }, "billing": { "serviceLevel": "服务档位", @@ -233,13 +234,13 @@ "diagnosticDimension": "诊断维度", "noDiagnosticDataYet": "暂无诊断数据", "successful": "成功 {count}", - "failed": "失败 {count} 次", - "failureRate": "失败率 {rate}", "retryRate": "重试率 {rate}", "retries": "重试 {count}", "loading": "正在加载诊断数据", "empty": "当前范围暂无诊断数据", - "download": "下载" + "download": "下载", + "errors": "错误 {count} 次", + "errorRate": "错误率 {rate}" }, "cost": { "savings": "节省", diff --git a/frontend/src/i18n/locales/zh-Hant/usage.json b/frontend/src/i18n/locales/zh-Hant/usage.json index 7322addc9..c27f498f4 100644 --- a/frontend/src/i18n/locales/zh-Hant/usage.json +++ b/frontend/src/i18n/locales/zh-Hant/usage.json @@ -16,14 +16,15 @@ "serviceSuccessRate": "服務成功率", "notCompleted": "未完成", "requestVolume": "請求量", - "noServiceResult": "無服務結果", "title": "請求健康", "description": "請求狀態分佈", "descriptionWithGranularity": "請求狀態分佈 · {granularityText}", - "completionRate": "完成率", - "abnormalEnd": "非正常結束", "loading": "正在加載請求數據", - "empty": "當前範圍暫無請求" + "empty": "當前範圍暫無請求", + "errors": "錯誤", + "noSamples": "無樣本", + "completionRate": "完成率", + "abnormalEnd": "非正常結束" }, "billing": { "serviceLevel": "服務檔位", @@ -233,13 +234,13 @@ "diagnosticDimension": "診斷維度", "noDiagnosticDataYet": "暫無診斷數據", "successful": "成功 {count}", - "failed": "失敗 {count} 次", - "failureRate": "失敗率 {rate}", "retryRate": "重試率 {rate}", "retries": "重試 {count}", "loading": "正在加載診斷數據", "empty": "當前範圍暫無診斷數據", - "download": "下載" + "download": "下載", + "errors": "錯誤 {count} 次", + "errorRate": "錯誤率 {rate}" }, "cost": { "savings": "節省", diff --git a/frontend/src/views/dashboard/composables/useRequestTrendChart.ts b/frontend/src/views/dashboard/composables/useRequestTrendChart.ts index 11cdef7a9..3e425fe37 100644 --- a/frontend/src/views/dashboard/composables/useRequestTrendChart.ts +++ b/frontend/src/views/dashboard/composables/useRequestTrendChart.ts @@ -254,7 +254,9 @@ export function useRequestTrendChart(options: { case 'errors': return point?.errors ?? value case 'successRate': - return zeroAwareRateText(point, 'successRate', 'requestsValue') ?? value + return point?.successRateValue == null + ? t('usage.health.noSamples') + : trendPointText(point, 'successRate') ?? value case 'requests': return point?.requests ?? value default: @@ -342,7 +344,7 @@ export function useRequestTrendChart(options: { ), ...gapLineSeries( 'successRate', - activeSeriesValues('successRateValue'), + seriesValues('successRateValue'), trendColor('successRate'), { width: 2.4, yAxisIndex: 1 }, ), diff --git a/frontend/src/views/dashboard/presenter.ts b/frontend/src/views/dashboard/presenter.ts index f244d132b..ac1971e9a 100644 --- a/frontend/src/views/dashboard/presenter.ts +++ b/frontend/src/views/dashboard/presenter.ts @@ -115,8 +115,8 @@ export function dashboardTrendView(trend: DashboardTrendResponse | null) { const summary = trend.summary.map(item => ({ ...trendMetricView(item.metric), - value: trend.kind === 'errors' && item.ratio !== null - ? item.ratio + value: trend.kind === 'errors' && item.metric === 'successRate' + ? item.ratio ?? t('usage.health.noSamples') : item.value, })) return { points: trend.points, summary } diff --git a/frontend/src/views/usage/components/UsageDiagnosticCard.vue b/frontend/src/views/usage/components/UsageDiagnosticCard.vue index db341ab65..e867cb2f2 100644 --- a/frontend/src/views/usage/components/UsageDiagnosticCard.vue +++ b/frontend/src/views/usage/components/UsageDiagnosticCard.vue @@ -60,7 +60,7 @@ const resultDimensionLabel = computed( ) const dimensionNameSizes: Record = { - model: 'xl', + model: 'lg', account: '2xl', apiKey: 'md', provider: 'lg', @@ -90,25 +90,26 @@ const diagnosticColumns = computed(() => defineTableColumns {{ formatCompactNumber(row.errorCount) }} diff --git a/frontend/src/views/usage/components/UsageHealthCard.vue b/frontend/src/views/usage/components/UsageHealthCard.vue index 5dc5f4646..8331cc056 100644 --- a/frontend/src/views/usage/components/UsageHealthCard.vue +++ b/frontend/src/views/usage/components/UsageHealthCard.vue @@ -10,7 +10,6 @@ import { computed } from 'vue' import { useI18n } from 'vue-i18n' import BaseChart from '@/components/charts/BaseChart.vue' import { chartTooltipContent, tooltipIndex, tooltipRows } from '@/components/charts/tooltip' -import { healthEligibleRequestCount } from '@/components/usage/shared/health' import { useChartPalette } from '@/composables/useChartPalette' import { formatLocalizedCompactNumber as formatCompactNumber, formatPercent } from '@/utils/format' @@ -132,12 +131,9 @@ function requestCount(point: HealthPoint) { } function serviceSuccessRate(point: HealthPoint) { - const total = healthEligibleRequestCount(point) - if (total > 0) - return 1 - point.errorRate - return requestCount(point) === 0 - ? 0 - : null + return point.errorRate == null + ? null + : 1 - point.errorRate } function outcomeSeries(name: string, data: number[], color: string) { @@ -170,7 +166,7 @@ function formatTooltip(params: unknown) { usageTooltipItem( t('usage.health.serviceSuccessRate'), successRate == null - ? t('usage.health.noServiceResult') + ? t('usage.health.noSamples') : formatPercent(successRate), theme.success, ),