fix(observability): count incomplete requests in availability

Include incomplete final outcomes in service health metrics and lower the
stable threshold to 95%. Keep the health popover at four outcomes by
folding incomplete requests into the service failure display.

Use consistent service success rates across dashboard trends, health
timelines and usage insights. Cover the reported counts, threshold edges,
caller exclusions and final success after failed attempts in regressions.

Refs #406

Co-authored-by: Codex <noreply@openai.com>

backend/crates/gateway-admin/src/use_case/observability/mod.rs

@@ -37,7 +37,7 @@ use super::{map_provider_error, map_store_error};
 const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15;
 const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
 const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
-const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 99.0;
+const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 95.0;
 mod store_cache;
 pub(crate) use store_cache::CachedObservabilityStore;
 
@@ -379,8 +379,9 @@ fn build_usage_insights(
     }
 
     let requests = &overview.requests;
-    let failed_requests = service_failure_count(requests);
-    let health_requests = requests.success_count.saturating_add(failed_requests);
+    let health_window = HealthWindow::from_metrics(requests);
+    let failed_requests = health_window.failed_requests;
+    let health_requests = health_window.eligible_requests();
     let health = UsageInsightsHealth {
         total_requests: requests.request_count,
         success_requests: requests.success_count,
@@ -398,17 +399,19 @@ fn build_usage_insights(
         points: trend
             .iter()
             .map(|point| {
-                let failed_requests = service_failure_count(&point.metrics);
-                let health_requests = point.metrics.success_count.saturating_add(failed_requests);
+                let health_window = HealthWindow::from_metrics(&point.metrics);
                 UsageInsightsHealthPoint {
                     total_requests: point.metrics.request_count,
                     bucket_start: point.bucket_start,
                     success_requests: point.metrics.success_count,
-                    failed_requests,
+                    failed_requests: health_window.failed_requests,
                     cancelled_requests: point.metrics.cancelled_count,
                     incomplete_requests: point.metrics.incomplete_count,
                     caller_error_requests: point.metrics.caller_error_count,
-                    error_rate: rate_or_zero(failed_requests, health_requests),
+                    error_rate: rate_or_zero(
+                        health_window.non_success_requests(),
+                        health_window.eligible_requests(),
+                    ),
                 }
             })
             .collect(),
@@ -956,7 +959,7 @@ fn trend(
     let service_failure_count = service_failure_count(&metrics);
     let success_rate = ratio(
         metrics.success_count,
-        metrics.success_count.saturating_add(service_failure_count),
+        HealthWindow::from_metrics(&metrics).eligible_requests(),
     );
     let cache_hit_request_rate = ratio(
         metrics.cache_hit_request_count,
@@ -1023,7 +1026,10 @@ fn trend_point(point: RequestMetricPoint) -> TrendPoint {
             point.metrics.cache_hit_request_count,
             point.metrics.cache_eligible_request_count,
         ),
-        success_rate: ratio(point.metrics.success_count, point.metrics.request_count),
+        success_rate: ratio(
+            point.metrics.success_count,
+            HealthWindow::from_metrics(&point.metrics).eligible_requests(),
+        ),
         metrics: point.metrics,
         cost_coverage: point.cost_coverage,
         costs: point.costs,
@@ -1123,20 +1129,29 @@ struct HealthWindow {
 }
 
 impl HealthWindow {
+    fn from_metrics(metrics: &RequestMetrics) -> Self {
+        Self {
+            success_requests: metrics.success_count,
+            failed_requests: service_failure_count(metrics),
+            cancelled_requests: metrics.cancelled_count,
+            incomplete_requests: metrics.incomplete_count,
+            caller_error_requests: metrics.caller_error_count,
+        }
+    }
+
+    // 未完成保留交付终态，但上游失败、超时和进程中断都不能视为服务成功
+    fn non_success_requests(self) -> u64 {
+        self.failed_requests
+            .saturating_add(self.incomplete_requests)
+    }
+
+    fn eligible_requests(self) -> u64 {
+        self.success_requests
+            .saturating_add(self.non_success_requests())
+    }
+
     fn add_metrics(&mut self, metrics: &RequestMetrics) {
-        self.success_requests = self.success_requests.saturating_add(metrics.success_count);
-        self.failed_requests = self
-            .failed_requests
-            .saturating_add(service_failure_count(metrics));
-        self.cancelled_requests = self
-            .cancelled_requests
-            .saturating_add(metrics.cancelled_count);
-        self.incomplete_requests = self
-            .incomplete_requests
-            .saturating_add(metrics.incomplete_count);
-        self.caller_error_requests = self
-            .caller_error_requests
-            .saturating_add(metrics.caller_error_count);
+        self.add_window(Self::from_metrics(metrics));
     }
 
     fn add_window(&mut self, other: Self) {
@@ -1155,15 +1170,13 @@ impl HealthWindow {
 }
 
 fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus {
-    let eligible_requests = bucket
-        .success_requests
-        .saturating_add(bucket.failed_requests);
+    let eligible_requests = bucket.eligible_requests();
     if is_future {
         HealthStatus::Future
     } else if eligible_requests == 0 {
         HealthStatus::NoData
     } else if bucket.success_requests == 0
-        && bucket.failed_requests >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD
+        && bucket.non_success_requests() >= HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD
     {
         HealthStatus::Unavailable
     } else if eligible_requests < HEALTH_TIMELINE_MIN_SAMPLE_SIZE {
@@ -1178,9 +1191,7 @@ fn health_status(bucket: HealthWindow, is_future: bool) -> HealthStatus {
 }
 
 fn health_reliability(bucket: HealthWindow) -> Option<f64> {
-    let eligible_requests = bucket
-        .success_requests
-        .saturating_add(bucket.failed_requests);
+    let eligible_requests = bucket.eligible_requests();
     (eligible_requests > 0)
         .then(|| bucket.success_requests as f64 / eligible_requests as f64 * 100.0)
 }

backend/crates/gateway-admin/tests/use_case/observability/mod.rs

@@ -148,7 +148,7 @@ async fn dashboard_summary_should_derive_first_token_average_from_trend() {
 }
 
 #[tokio::test]
-async fn health_timeline_should_match_legacy_status_precedence_and_thresholds() {
+async fn health_timeline_should_apply_status_precedence_and_thresholds() {
     let now = Utc::now();
     let current_slot = quarter_hour_start(now);
     let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
@@ -158,13 +158,28 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds()
             RequestMetrics {
                 failure_count: 1,
                 cancelled_count: 1,
-                incomplete_count: 1,
                 caller_error_count: 1,
                 ..RequestMetrics::default()
             },
             HealthStatus::NoData,
             None,
         ),
+        (
+            RequestMetrics {
+                incomplete_count: 1,
+                ..RequestMetrics::default()
+            },
+            HealthStatus::LowSample,
+            Some(0.0),
+        ),
+        (
+            RequestMetrics {
+                incomplete_count: 3,
+                ..RequestMetrics::default()
+            },
+            HealthStatus::Unavailable,
+            Some(0.0),
+        ),
         (
             RequestMetrics {
                 failure_count: 3,
@@ -184,21 +199,21 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds()
         ),
         (
             RequestMetrics {
-                success_count: 98,
-                failure_count: 2,
+                success_count: 94,
+                failure_count: 6,
                 ..RequestMetrics::default()
             },
             HealthStatus::Unstable,
-            Some(98.0),
+            Some(94.0),
         ),
         (
             RequestMetrics {
-                success_count: 99,
-                failure_count: 1,
+                success_count: 95,
+                incomplete_count: 5,
                 ..RequestMetrics::default()
             },
             HealthStatus::Stable,
-            Some(99.0),
+            Some(95.0),
         ),
     ];
 
@@ -224,6 +239,92 @@ async fn health_timeline_should_match_legacy_status_precedence_and_thresholds()
     }
 }
 
+#[tokio::test]
+async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate() {
+    let now = Utc::now();
+    let current_slot = quarter_hour_start(now);
+    let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
+    let services = observability_services(store.clone()).await;
+    let fixtures = [
+        (162, 0, 2, 0, 0, 98.780_487_804_878_05),
+        (297, 0, 2, 5, 0, 99.331_103_678_929_76),
+        (467, 1, 5, 0, 0, 98.731_501_057_082_45),
+        (95, 2, 3, 4, 1, 95.0),
+        (100, 0, 0, 0, 0, 100.0),
+    ];
+
+    for (index, (success, failed, incomplete, cancelled, caller_error, expected_percent)) in
+        fixtures.into_iter().enumerate()
+    {
+        let range = observation_range(
+            now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2),
+        );
+        let metrics = RequestMetrics {
+            request_count: success + failed + incomplete + cancelled + caller_error,
+            success_count: success,
+            failure_count: failed + caller_error,
+            incomplete_count: incomplete,
+            cancelled_count: cancelled,
+            caller_error_count: caller_error,
+            ..RequestMetrics::default()
+        };
+        store.replace_trend(vec![health_metric_point(current_slot, metrics.clone())]);
+        store.replace_overview(UsageOverview {
+            analytics: Default::default(),
+            range,
+            requests: metrics,
+            // 中间尝试失败不会增加最终失败请求数
+            attempts: AttemptMetrics {
+                failure_count: 2,
+                ..AttemptMetrics::default()
+            },
+            providers: Vec::new(),
+        });
+
+        let dashboard = services
+            .observability()
+            .dashboard_summary(range, TrendKind::Usage)
+            .await
+            .expect("dashboard summary");
+        let timeline = dashboard.health_timeline;
+        let point = timeline
+            .points
+            .iter()
+            .find(|point| point.bucket_start == current_slot)
+            .expect("current health slot");
+        assert_eq!(point.status, HealthStatus::Stable);
+        assert_eq!(point.failed_requests, failed);
+        assert_eq!(point.incomplete_requests, incomplete);
+        assert_eq!(point.cancelled_requests, cancelled);
+        assert_eq!(point.caller_error_requests, caller_error);
+        for reliability in [timeline.reliability_percent, point.reliability_percent] {
+            assert!((reliability.expect("health reliability") - expected_percent).abs() < 1e-10);
+        }
+        for success_rate in [
+            dashboard.trend.summary.success_rate,
+            dashboard.trend.points[0].success_rate,
+        ] {
+            assert!(
+                (success_rate.expect("trend success rate") * 100.0 - expected_percent).abs()
+                    < 1e-10
+            );
+        }
+
+        let insights = services
+            .observability()
+            .usage_insights(range, UsageFilter::default())
+            .await
+            .expect("usage insights");
+        assert_eq!(insights.health.failed_requests, failed);
+        assert_eq!(insights.health.incomplete_requests, incomplete);
+        assert!((insights.health.success_rate * 100.0 - expected_percent).abs() < 1e-10);
+        assert!(
+            (insights.health.points[0].error_rate * 100.0 - (100.0 - expected_percent)).abs()
+                < 1e-10
+        );
+    }
+}
+
 #[tokio::test]
 async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
     let now = Utc::now();

docs/api.md

@@ -1866,6 +1866,12 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告（流内
 
 ### 记录范围与统计口径
 
+请求健康可用性与服务成功率按 `成功 /（成功 + 服务失败 + 未完成）` 计算
+未完成保留独立计数，客户端取消与调用方错误不计入该分母，中间尝试失败不重复计为最终失败请求
+健康时间线以 15 分钟分桶，可用性不低于 95% 且有效样本不少于 10 个时显示稳定
+没有成功且服务失败与未完成合计不少于 3 个时显示不可达，其他不足 10 个有效样本的时段显示低样本
+有效样本为零时显示无有效样本，尚未到达的时间格显示未来
+
 已进入模型执行会话、但尚未登记执行尝试的失败请求也进入错误及详情查询，
 包含无可用账号、准备失败、启动/准备超时与取消。此时 attempt 数为零，未确认的 Provider、账号及
 上游传输为空；可用模型执行 ID 在“全部平台”下查询，不从路由候选推断实际调用平台。

frontend/src/components/usage/HealthTimelinePointPopover.vue

@@ -2,18 +2,16 @@
 import type { DashboardHealthTimelinePoint } from '@/api'
 
 import { computed } from 'vue'
-import { formatHealthCount, healthReliabilityValueClass, healthStatusMeta } from '@/components/usage/shared/health'
+import { formatHealthCount, healthEligibleRequestCount, healthObservedRequestCount, healthServiceFailureCount, healthStatusMeta } from '@/components/usage/shared/health'
 
 const props = defineProps<{
   point: DashboardHealthTimelinePoint
 }>()
 
 const status = computed(() => healthStatusMeta[props.point.status])
-const eligibleRequests = computed(() => props.point.successRequests + props.point.failedRequests)
-const observedRequests = computed(
-  () => eligibleRequests.value + props.point.cancelledRequests + props.point.callerErrorRequests,
-)
-// 四类结果的标签/取值/配色单一来源；分段条与明细行都由此派生。
+const eligibleRequests = computed(() => healthEligibleRequestCount(props.point))
+const observedRequests = computed(() => healthObservedRequestCount(props.point))
+// 结果标签、取值与配色集中维护，分段条与明细行都由此派生
 const outcomeMeta = computed(() => [
   {
     label: '成功',
@@ -23,7 +21,7 @@ const outcomeMeta = computed(() => [
   },
   {
     label: '服务失败',
-    value: props.point.failedRequests,
+    value: healthServiceFailureCount(props.point),
     dotClass: 'bg-cp-error',
     valueClass: 'text-cp-error-text',
   },
@@ -86,7 +84,7 @@ const metricItems = outcomeMeta
           </p>
           <strong
             class="mt-2 block font-mono text-[25px] leading-none font-heavy tabular-nums"
-            :class="healthReliabilityValueClass(point.successRequests, point.failedRequests)"
+            :class="status.valueClass"
           >
             {{ point.reliabilityDisplay }}
           </strong>
@@ -126,7 +124,7 @@ const metricItems = outcomeMeta
       </div>
 
       <p class="m-0 border-t border-cp-split pt-3 text-[10px] leading-[1.45] font-emphasis text-cp-text-quaternary">
-        客户端取消与调用方错误单独记录，不计入可用性
+        取消与调用方错误不计入可用性
       </p>
     </div>
   </section>

frontend/src/components/usage/RequestHealthTimelineCard.vue

@@ -6,7 +6,7 @@ import { useEventListener, usePreferredReducedMotion, useResizeObserver } from '
 import { gsap } from 'gsap'
 import { computed, nextTick, onBeforeUnmount, shallowRef, useTemplateRef, watch } from 'vue'
 import HealthTimelinePointPopover from '@/components/usage/HealthTimelinePointPopover.vue'
-import { formatHealthCount, healthLegend, healthReliabilityValueClass, healthStatusMeta } from '@/components/usage/shared/health'
+import { formatHealthCount, healthEligibleRequestCount, healthLegend, healthObservedRequestCount, healthStatusMeta } from '@/components/usage/shared/health'
 
 const props = defineProps<{
   timeline: DashboardHealthTimeline
@@ -31,12 +31,8 @@ let wavedCellIndexes = new Set<number>()
 let anchorXTo: ReturnType<typeof gsap.quickTo> | undefined
 let anchorYTo: ReturnType<typeof gsap.quickTo> | undefined
 
-function observedRequests(point: DashboardHealthTimelinePoint) {
-  return point.successRequests + point.failedRequests + point.cancelledRequests + point.callerErrorRequests
-}
-
 function isInteractivePoint(point: DashboardHealthTimelinePoint) {
-  return point.status !== 'future' && observedRequests(point) > 0
+  return point.status !== 'future' && healthObservedRequestCount(point) > 0
 }
 
 function isActivePoint(point: DashboardHealthTimelinePoint) {
@@ -222,7 +218,7 @@ function releasePointWave() {
 }
 
 function pointAccessibilityLabel(point: DashboardHealthTimelinePoint) {
-  const eligibleRequests = point.successRequests + point.failedRequests
+  const eligibleRequests = healthEligibleRequestCount(point)
   return `${point.time}，${healthStatusMeta[point.status].label}，有效请求 ${formatHealthCount(eligibleRequests)}，可用性 ${point.reliabilityDisplay}`
 }
 
@@ -270,7 +266,7 @@ onBeforeUnmount(() => {
         <div class="flex shrink-0 items-center gap-3">
           <strong
             class="font-mono text-sm leading-none font-heavy tabular-nums"
-            :class="healthReliabilityValueClass(timeline.successRequests, timeline.failedRequests)"
+            :class="healthStatusMeta[timeline.status].valueClass"
           >
             {{ timeline.reliabilityDisplay }}
           </strong>

frontend/src/components/usage/shared/health.ts

@@ -1,10 +1,11 @@
-import type { DashboardHealthStatus } from '@/api'
+import type { DashboardHealthStatus, DashboardHealthTimelinePoint } from '@/api'
 import { formatInteger } from '@/utils/format'
 
 interface DashboardHealthStatusMeta {
   label: string
   cellClass: string
   badgeClass: string
+  valueClass: string
 }
 
 export const healthLegend = [
@@ -20,47 +21,50 @@ export const healthStatusMeta: Record<DashboardHealthStatus, DashboardHealthStat
     label: '未来',
     cellClass: 'bg-cp-bg-container-disabled opacity-60',
     badgeClass: 'bg-cp-fill-tertiary text-cp-text-quaternary',
+    valueClass: 'text-cp-text-quaternary',
   },
   no_data: {
     label: '无有效样本',
     cellClass: 'bg-cp-border',
     badgeClass: 'bg-cp-fill-tertiary text-cp-text-secondary',
+    valueClass: 'text-cp-text-quaternary',
   },
   unavailable: {
     label: '不可达',
     cellClass: 'bg-cp-error',
     badgeClass: 'bg-cp-error-container text-cp-error-on-container',
+    valueClass: 'text-cp-error-text',
   },
   unstable: {
     label: '不稳定',
     cellClass: 'bg-cp-warning',
     badgeClass: 'bg-cp-warning-container text-cp-warning-on-container',
+    valueClass: 'text-cp-warning-text',
   },
   low_sample: {
     label: '低样本',
     cellClass: 'bg-cp-cyan-solid',
     badgeClass: 'bg-cp-cyan-container text-cp-cyan-on-container',
+    valueClass: 'text-cp-cyan-text',
   },
   stable: {
     label: '稳定',
     cellClass: 'bg-cp-success',
     badgeClass: 'bg-cp-success-container text-cp-success-on-container',
+    valueClass: 'text-cp-success-text',
   },
 }
 
-export function healthReliabilityValueClass(successRequests: number, failedRequests: number) {
-  const eligibleRequests = Math.max(0, successRequests) + Math.max(0, failedRequests)
-  if (eligibleRequests === 0)
-    return 'text-cp-text-quaternary'
+export function healthServiceFailureCount(point: Pick<DashboardHealthTimelinePoint, 'failedRequests' | 'incompleteRequests'>) {
+  return point.failedRequests + point.incompleteRequests
+}
 
-  const reliability = (Math.max(0, successRequests) / eligibleRequests) * 100
-  if (reliability >= 99.5)
-    return 'text-cp-success-text'
-  if (reliability >= 98)
-    return 'text-cp-cyan-text'
-  if (reliability >= 95)
-    return 'text-cp-warning-text'
-  return 'text-cp-error-text'
+export function healthEligibleRequestCount(point: Pick<DashboardHealthTimelinePoint, 'successRequests' | 'failedRequests' | 'incompleteRequests'>) {
+  return point.successRequests + healthServiceFailureCount(point)
+}
+
+export function healthObservedRequestCount(point: DashboardHealthTimelinePoint) {
+  return healthEligibleRequestCount(point) + point.cancelledRequests + point.callerErrorRequests
 }
 
 export function formatHealthCount(value: number) {

frontend/src/views/usage/components/UsageHealthCard.vue

@@ -8,6 +8,7 @@ import { use } from 'echarts/core'
 import { computed } from 'vue'
 import BaseChart from '@/components/charts/BaseChart.vue'
 import { tooltipIndex, tooltipRows } from '@/components/charts/tooltip'
+import { healthEligibleRequestCount } from '@/components/usage/shared/health'
 import { useChartPalette } from '@/composables/useChartPalette'
 import { formatLocalizedCompactNumber as formatCompactNumber, formatPercent } from '@/utils/format'
 
@@ -113,9 +114,9 @@ function requestCount(point: HealthPoint) {
 }
 
 function serviceSuccessRate(point: HealthPoint) {
-  const total = Math.max(0, point.successRequests) + Math.max(0, point.failedRequests)
+  const total = healthEligibleRequestCount(point)
   if (total > 0)
-    return Math.max(0, point.successRequests ?? 0) / total
+    return 1 - point.errorRate
   return requestCount(point) === 0 ? 0 : null
 }