diff --git a/backend/crates/gateway-admin/src/use_case/observability/mod.rs b/backend/crates/gateway-admin/src/use_case/observability/mod.rs index 3c861f02b..513b127f2 100644 --- a/backend/crates/gateway-admin/src/use_case/observability/mod.rs +++ b/backend/crates/gateway-admin/src/use_case/observability/mod.rs @@ -37,7 +37,7 @@ use super::{map_provider_error, map_store_error}; const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15; const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10; const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3; -const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0; +const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 95.0; mod store_cache; pub(crate) use store_cache::CachedObservabilityStore; @@ -379,8 +379,9 @@ fn build_usage_insights( } let requests = &overview.requests; - let failed_requests = service_failure_count(requests); - let health_requests = requests.success_count.saturating_add(failed_requests); + let health_window = HealthWindow::from_metrics(requests); + let failed_requests = health_window.failed_requests; + let health_requests = health_window.eligible_requests(); let health = UsageInsightsHealth { total_requests: requests.request_count, success_requests: requests.success_count, @@ -398,17 +399,19 @@ fn build_usage_insights( points: trend .iter() .map(|point| { - let failed_requests = service_failure_count(&point.metrics); - let health_requests = point.metrics.success_count.saturating_add(failed_requests); + let health_window = HealthWindow::from_metrics(&point.metrics); UsageInsightsHealthPoint { total_requests: point.metrics.request_count, bucket_start: point.bucket_start, success_requests: point.metrics.success_count, - failed_requests, + failed_requests: health_window.failed_requests, cancelled_requests: point.metrics.cancelled_count, incomplete_requests: point.metrics.incomplete_count, caller_error_requests: point.metrics.caller_error_count, - error_rate: rate_or_zero(failed_requests, health_requests), + error_rate: rate_or_zero( + health_window.non_success_requests(), + health_window.eligible_requests(), + ), } }) .collect(), @@ -956,7 +959,7 @@ fn trend( let service_failure_count = service_failure_count(&metrics); let success_rate = ratio( metrics.success_count, - metrics.success_count.saturating_add(service_failure_count), + HealthWindow::from_metrics(&metrics).eligible_requests(), ); let cache_hit_request_rate = ratio( metrics.cache_hit_request_count, @@ -1023,7 +1026,10 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint { point.metrics.cache_hit_request_count, point.metrics.cache_eligible_request_count, ), - success_rate: ratio(point.metrics.success_count, point.metrics.request_count), + success_rate: ratio( + point.metrics.success_count, + HealthWindow::from_metrics(&point.metrics).eligible_requests(), + ), metrics: point.metrics, cost_coverage: point.cost_coverage, costs: point.costs, @@ -1123,20 +1129,29 @@ struct HealthWindow { } impl HealthWindow { + fn from_metrics(metrics: &RequestMetrics) -> Self { + Self { + success_requests: metrics.success_count, + failed_requests: service_failure_count(metrics), + cancelled_requests: metrics.cancelled_count, + incomplete_requests: metrics.incomplete_count, + caller_error_requests: metrics.caller_error_count, + } + } + + // 未完成保留交付终态，但上游失败、超时和进程中断都不能视为服务成功 + fn non_success_requests(self) -> u64 { + self.failed_requests + .saturating_add(self.incomplete_requests) + } + + fn eligible_requests(self) -> u64 { + self.success_requests + .saturating_add(self.non_success_requests()) + } + fn add_metrics(&mut self, metrics: &RequestMetrics) { - self.success_requests = self.success_requests.saturating_add(metrics.success_count); - self.failed_requests = self - .failed_requests - .saturating_add(service_failure_count(metrics)); - self.cancelled_requests = self - .cancelled_requests - .saturating_add(metrics.cancelled_count); - self.incomplete_requests = self - .incomplete_requests - .saturating_add(metrics.incomplete_count); - self.caller_error_requests = self - .caller_error_requests - .saturating_add(metrics.caller_error_count); + self.add_window(Self::from_metrics(metrics)); } fn add_window(&mut self, other: Self) { @@ -1155,15 +1170,13 @@ impl HealthWindow { } fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus { - let eligible_requests = bucket - .success_requests - .saturating_add(bucket.failed_requests); + let eligible_requests = bucket.eligible_requests(); if is_future { HealthStatus::Future } else if eligible_requests == 0 { HealthStatus::NoData } else if bucket.success_requests == 0 - && bucket.failed_requests >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD + && bucket.non_success_requests() >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD { HealthStatus::Unavailable } else if eligible_requests < HEALTH_TIMELINE_MIN_SAMPLE_SIZE { @@ -1178,9 +1191,7 @@ fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus { } fn health_reliability(bucket: HealthWindow) -> Option { - let eligible_requests = bucket - .success_requests - .saturating_add(bucket.failed_requests); + let eligible_requests = bucket.eligible_requests(); (eligible_requests > 0) .then(|| bucket.success_requests as f64 / eligible_requests as f64 * 100.0) } diff --git a/backend/crates/gateway-admin/tests/use_case/observability/mod.rs b/backend/crates/gateway-admin/tests/use_case/observability/mod.rs index d450261d7..59fb44959 100644 --- a/backend/crates/gateway-admin/tests/use_case/observability/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/observability/mod.rs @@ -148,7 +148,7 @@ async fn dashboard_summary_should_derive_first_token_average_from_trend() { } #[tokio::test] -async fn health_timeline_should_match_legacy_status_precedence_and_thresholds() { +async fn health_timeline_should_apply_status_precedence_and_thresholds() { let now = Utc::now(); let current_slot = quarter_hour_start(now); let store = Arc::new(FixtureObservabilityStore::new(observation_range(now))); @@ -158,13 +158,28 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds() RequestMetrics { failure_count: 1, cancelled_count: 1, - incomplete_count: 1, caller_error_count: 1, ..RequestMetrics::default() }, HealthStatus::NoData, None, ), + ( + RequestMetrics { + incomplete_count: 1, + ..RequestMetrics::default() + }, + HealthStatus::LowSample, + Some(0.0), + ), + ( + RequestMetrics { + incomplete_count: 3, + ..RequestMetrics::default() + }, + HealthStatus::Unavailable, + Some(0.0), + ), ( RequestMetrics { failure_count: 3, @@ -184,21 +199,21 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds() ), ( RequestMetrics { - success_count: 98, - failure_count: 2, + success_count: 94, + failure_count: 6, ..RequestMetrics::default() }, HealthStatus::Unstable, - Some(98.0), + Some(94.0), ), ( RequestMetrics { - success_count: 99, - failure_count: 1, + success_count: 95, + incomplete_count: 5, ..RequestMetrics::default() }, HealthStatus::Stable, - Some(99.0), + Some(95.0), ), ]; @@ -224,6 +239,92 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds() } } +#[tokio::test] +async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate() { + let now = Utc::now(); + let current_slot = quarter_hour_start(now); + let store = Arc::new(FixtureObservabilityStore::new(observation_range(now))); + let services = observability_services(store.clone()).await; + let fixtures = [ + (162, 0, 2, 0, 0, 98.780_487_804_878_05), + (297, 0, 2, 5, 0, 99.331_103_678_929_76), + (467, 1, 5, 0, 0, 98.731_501_057_082_45), + (95, 2, 3, 4, 1, 95.0), + (100, 0, 0, 0, 0, 100.0), + ]; + + for (index, (success, failed, incomplete, cancelled, caller_error, expected_percent)) in + fixtures.into_iter().enumerate() + { + let range = observation_range( + now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2), + ); + let metrics = RequestMetrics { + request_count: success + failed + incomplete + cancelled + caller_error, + success_count: success, + failure_count: failed + caller_error, + incomplete_count: incomplete, + cancelled_count: cancelled, + caller_error_count: caller_error, + ..RequestMetrics::default() + }; + store.replace_trend(vec![health_metric_point(current_slot, metrics.clone())]); + store.replace_overview(UsageOverview { + analytics: Default::default(), + range, + requests: metrics, + // 中间尝试失败不会增加最终失败请求数 + attempts: AttemptMetrics { + failure_count: 2, + ..AttemptMetrics::default() + }, + providers: Vec::new(), + }); + + let dashboard = services + .observability() + .dashboard_summary(range, TrendKind::Usage) + .await + .expect("dashboard summary"); + let timeline = dashboard.health_timeline; + let point = timeline + .points + .iter() + .find(|point| point.bucket_start == current_slot) + .expect("current health slot"); + assert_eq!(point.status, HealthStatus::Stable); + assert_eq!(point.failed_requests, failed); + assert_eq!(point.incomplete_requests, incomplete); + assert_eq!(point.cancelled_requests, cancelled); + assert_eq!(point.caller_error_requests, caller_error); + for reliability in [timeline.reliability_percent, point.reliability_percent] { + assert!((reliability.expect("health reliability") - expected_percent).abs() < 1e-10); + } + for success_rate in [ + dashboard.trend.summary.success_rate, + dashboard.trend.points[0].success_rate, + ] { + assert!( + (success_rate.expect("trend success rate") * 100.0 - expected_percent).abs() + < 1e-10 + ); + } + + let insights = services + .observability() + .usage_insights(range, UsageFilter::default()) + .await + .expect("usage insights"); + assert_eq!(insights.health.failed_requests, failed); + assert_eq!(insights.health.incomplete_requests, incomplete); + assert!((insights.health.success_rate * 100.0 - expected_percent).abs() < 1e-10); + assert!( + (insights.health.points[0].error_rate * 100.0 - (100.0 - expected_percent)).abs() + < 1e-10 + ); + } +} + #[tokio::test] async fn dashboard_summary_should_project_rebuildable_runtime_slots() { let now = Utc::now(); diff --git a/docs/api.md b/docs/api.md index 527a53905..839032515 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1866,6 +1866,12 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告（流内 ### 记录范围与统计口径 +请求健康可用性与服务成功率按 `成功 /（成功 + 服务失败 + 未完成）` 计算 +未完成保留独立计数，客户端取消与调用方错误不计入该分母，中间尝试失败不重复计为最终失败请求 +健康时间线以 15 分钟分桶，可用性不低于 95% 且有效样本不少于 10 个时显示稳定 +没有成功且服务失败与未完成合计不少于 3 个时显示不可达，其他不足 10 个有效样本的时段显示低样本 +有效样本为零时显示无有效样本，尚未到达的时间格显示未来 + 已进入模型执行会话、但尚未登记执行尝试的失败请求也进入错误及详情查询， 包含无可用账号、准备失败、启动/准备超时与取消。此时 attempt 数为零，未确认的 Provider、账号及 上游传输为空；可用模型执行 ID 在“全部平台”下查询，不从路由候选推断实际调用平台。 diff --git a/frontend/src/components/usage/HealthTimelinePointPopover.vue b/frontend/src/components/usage/HealthTimelinePointPopover.vue index 5cbb6838a..33e33fadb 100644 --- a/frontend/src/components/usage/HealthTimelinePointPopover.vue +++ b/frontend/src/components/usage/HealthTimelinePointPopover.vue @@ -2,18 +2,16 @@ import type { DashboardHealthTimelinePoint } from '@/api' import { computed } from 'vue' -import { formatHealthCount, healthReliabilityValueClass, healthStatusMeta } from '@/components/usage/shared/health' +import { formatHealthCount, healthEligibleRequestCount, healthObservedRequestCount, healthServiceFailureCount, healthStatusMeta } from '@/components/usage/shared/health' const props = defineProps<{ point: DashboardHealthTimelinePoint }>() const status = computed(() => healthStatusMeta[props.point.status]) -const eligibleRequests = computed(() => props.point.successRequests + props.point.failedRequests) -const observedRequests = computed( - () => eligibleRequests.value + props.point.cancelledRequests + props.point.callerErrorRequests, -) -// 四类结果的标签/取值/配色单一来源；分段条与明细行都由此派生。 +const eligibleRequests = computed(() => healthEligibleRequestCount(props.point)) +const observedRequests = computed(() => healthObservedRequestCount(props.point)) +// 结果标签、取值与配色集中维护，分段条与明细行都由此派生 const outcomeMeta = computed(() => [ { label: '成功', @@ -23,7 +21,7 @@ const outcomeMeta = computed(() => [ }, { label: '服务失败', - value: props.point.failedRequests, + value: healthServiceFailureCount(props.point), dotClass: 'bg-cp-error', valueClass: 'text-cp-error-text', }, @@ -86,7 +84,7 @@ const metricItems = outcomeMeta

{{ point.reliabilityDisplay }} @@ -126,7 +124,7 @@ const metricItems = outcomeMeta

- 客户端取消与调用方错误单独记录，不计入可用性 + 取消与调用方错误不计入可用性

diff --git a/frontend/src/components/usage/RequestHealthTimelineCard.vue b/frontend/src/components/usage/RequestHealthTimelineCard.vue index cb26d362a..ee3e1f02e 100644 --- a/frontend/src/components/usage/RequestHealthTimelineCard.vue +++ b/frontend/src/components/usage/RequestHealthTimelineCard.vue @@ -6,7 +6,7 @@ import { useEventListener, usePreferredReducedMotion, useResizeObserver } from ' import { gsap } from 'gsap' import { computed, nextTick, onBeforeUnmount, shallowRef, useTemplateRef, watch } from 'vue' import HealthTimelinePointPopover from '@/components/usage/HealthTimelinePointPopover.vue' -import { formatHealthCount, healthLegend, healthReliabilityValueClass, healthStatusMeta } from '@/components/usage/shared/health' +import { formatHealthCount, healthEligibleRequestCount, healthLegend, healthObservedRequestCount, healthStatusMeta } from '@/components/usage/shared/health' const props = defineProps<{ timeline: DashboardHealthTimeline @@ -31,12 +31,8 @@ let wavedCellIndexes = new Set() let anchorXTo: ReturnType | undefined let anchorYTo: ReturnType | undefined -function observedRequests(point: DashboardHealthTimelinePoint) { - return point.successRequests + point.failedRequests + point.cancelledRequests + point.callerErrorRequests -} - function isInteractivePoint(point: DashboardHealthTimelinePoint) { - return point.status !== 'future' && observedRequests(point) > 0 + return point.status !== 'future' && healthObservedRequestCount(point) > 0 } function isActivePoint(point: DashboardHealthTimelinePoint) { @@ -222,7 +218,7 @@ function releasePointWave() { } function pointAccessibilityLabel(point: DashboardHealthTimelinePoint) { - const eligibleRequests = point.successRequests + point.failedRequests + const eligibleRequests = healthEligibleRequestCount(point) return `${point.time}，${healthStatusMeta[point.status].label}，有效请求 ${formatHealthCount(eligibleRequests)}，可用性 ${point.reliabilityDisplay}` } @@ -270,7 +266,7 @@ onBeforeUnmount(() => {
{{ timeline.reliabilityDisplay }} diff --git a/frontend/src/components/usage/shared/health.ts b/frontend/src/components/usage/shared/health.ts index e5ef3bd01..c0cc88a3b 100644 --- a/frontend/src/components/usage/shared/health.ts +++ b/frontend/src/components/usage/shared/health.ts @@ -1,10 +1,11 @@ -import type { DashboardHealthStatus } from '@/api' +import type { DashboardHealthStatus, DashboardHealthTimelinePoint } from '@/api' import { formatInteger } from '@/utils/format' interface DashboardHealthStatusMeta { label: string cellClass: string badgeClass: string + valueClass: string } export const healthLegend = [ @@ -20,47 +21,50 @@ export const healthStatusMeta: Record) { + return point.failedRequests + point.incompleteRequests +} - const reliability = (Math.max(0, successRequests) / eligibleRequests) * 100 - if (reliability >= 99.5) - return 'text-cp-success-text' - if (reliability >= 98) - return 'text-cp-cyan-text' - if (reliability >= 95) - return 'text-cp-warning-text' - return 'text-cp-error-text' +export function healthEligibleRequestCount(point: Pick) { + return point.successRequests + healthServiceFailureCount(point) +} + +export function healthObservedRequestCount(point: DashboardHealthTimelinePoint) { + return healthEligibleRequestCount(point) + point.cancelledRequests + point.callerErrorRequests } export function formatHealthCount(value: number) { diff --git a/frontend/src/views/usage/components/UsageHealthCard.vue b/frontend/src/views/usage/components/UsageHealthCard.vue index f8148054f..358e49e24 100644 --- a/frontend/src/views/usage/components/UsageHealthCard.vue +++ b/frontend/src/views/usage/components/UsageHealthCard.vue @@ -8,6 +8,7 @@ import { use } from 'echarts/core' import { computed } from 'vue' import BaseChart from '@/components/charts/BaseChart.vue' import { tooltipIndex, tooltipRows } from '@/components/charts/tooltip' +import { healthEligibleRequestCount } from '@/components/usage/shared/health' import { useChartPalette } from '@/composables/useChartPalette' import { formatLocalizedCompactNumber as formatCompactNumber, formatPercent } from '@/utils/format' @@ -113,9 +114,9 @@ function requestCount(point: HealthPoint) { } function serviceSuccessRate(point: HealthPoint) { - const total = Math.max(0, point.successRequests) + Math.max(0, point.failedRequests) + const total = healthEligibleRequestCount(point) if (total > 0) - return Math.max(0, point.successRequests ?? 0) / total + return 1 - point.errorRate return requestCount(point) === 0 ? 0 : null }