From d4855cca9c7b6992dd40dc8be27fbd35df1bcd1f Mon Sep 17 00:00:00 2001 From: Zyy Date: Sun, 30 Aug 2026 00:43:52 +0800 Subject: [PATCH] feat(observability): expose runtime pressure insights Co-authored-by: Codex --- .../gateway-admin/src/model/observability.rs | 51 ++++ .../src/use_case/observability.rs | 234 +++++++++++++++--- .../tests/use_case/observability.rs | 58 ++++- .../src/admin/observability/presenter.rs | 152 +++++++++++- .../src/admin/observability/wire.rs | 50 ++++ .../tests/admin/observability/response.rs | 18 ++ .../gateway-core/src/engine/coordinator.rs | 6 + .../gateway-core/src/engine/credential.rs | 67 +++++ .../gateway-core/src/engine/execution.rs | 10 +- backend/crates/gateway-core/src/engine/mod.rs | 7 + .../gateway-core/src/engine/provider.rs | 52 +++- .../gateway-core/tests/engine/coordinator.rs | 1 + backend/crates/gateway-store/src/lib.rs | 2 +- .../gateway-store/src/postgres/execution.rs | 74 +++++- .../crates/gateway-store/src/postgres/mod.rs | 2 +- .../src/postgres/observability/mapping.rs | 25 ++ .../src/postgres/observability/model.rs | 17 ++ .../observability/queries/dashboard.rs | 100 +++++++- .../postgres/observability/queries/usage.rs | 36 ++- .../gateway-store/tests/postgres/execution.rs | 9 + .../tests/postgres/observability.rs | 5 +- .../gateway-store/tests/postgres/snapshots.rs | 10 + .../openai/src/credential/selector.rs | 16 +- .../crates/providers/openai/src/provider.rs | 22 +- .../providers/xai/src/credential/selector.rs | 15 +- backend/crates/providers/xai/src/provider.rs | 21 +- .../providers/xai/src/transport/session.rs | 19 +- backend/migrations/.frozen-sha256 | 1 + .../0002_runtime_pressure_observability.sql | 20 ++ backend/migrations/README.md | 4 +- docs/architecture.md | 5 +- frontend/src/api/modules/dashboard.ts | 7 + frontend/src/api/modules/usage.ts | 40 +++ .../components/DashboardHeartbeat.vue | 7 +- .../components/HealthTimelinePointPopover.vue | 33 +-- .../views/dashboard/components/MetricCard.vue | 27 +- .../components/RequestHealthTimelineCard.vue | 57 +---- .../dashboard/components/RequestTrendCard.vue | 14 +- .../dashboard/components/WireProfileCard.vue | 78 ++---- .../dashboard/composables/useDashboard.ts | 8 +- .../composables/useRequestTrendChart.ts | 87 +++---- .../src/views/theme/fixtures/dashboard.ts | 7 + .../views/usage/components/UsageCostCard.vue | 134 ++++++---- .../usage/components/UsageDiagnosticCard.vue | 88 ++++--- .../usage/components/UsageHealthCard.vue | 105 +++++--- .../usage/components/UsageInsightsGrid.vue | 5 +- .../usage/components/UsagePerformanceCard.vue | 176 +++++++++---- .../usage/components/UsageSummaryCards.vue | 15 +- .../usage/composables/useUsageRecordsTable.ts | 18 ++ frontend/src/views/usage/utils/chart.ts | 10 + frontend/src/views/usage/utils/records.ts | 12 + 51 files changed, 1549 insertions(+), 488 deletions(-) create mode 100644 backend/migrations/0002_runtime_pressure_observability.sql diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index 70318a7d8..c36774fc2 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -402,6 +402,16 @@ pub struct RequestMetrics { pub max_latency_ms: Option, pub latency_percentiles: LatencyPercentiles, pub first_token_latency_percentiles: LatencyPercentiles, + pub admission_decision_count: u64, + pub admission_decision_percentiles: LatencyPercentiles, + pub account_selection_wait_count: u64, + pub account_selection_wait_percentiles: LatencyPercentiles, + pub output_throughput_p10: Option, + pub output_throughput_p50: Option, + pub output_throughput_p90: Option, + pub capacity_sample_count: u64, + pub capacity_utilization_avg_basis_points: Option, + pub capacity_utilization_p95_basis_points: Option, pub cache_eligible_request_count: u64, pub cache_hit_request_count: u64, } @@ -604,6 +614,10 @@ pub struct UsageRecord { pub first_token_ms: Option, pub provider_processing_ms: Option, pub latency_ms: Option, + pub admission_decision_ms: Option, + pub account_selection_wait_ms: Option, + pub capacity_used_slots: Option, + pub capacity_total_slots: Option, pub client_ip: Option, pub user_agent: Option, pub reasoning_effort: Option, @@ -708,6 +722,9 @@ pub struct DiagnosticObservation { pub total_tokens: u64, pub average_latency_ms: Option, pub latency_p95_ms: Option, + pub first_token_p95_ms: Option, + pub non_completion_count: u64, + pub retry_count: u64, pub cost_coverage: CostCoverage, pub costs: Vec, } @@ -787,6 +804,9 @@ pub struct TrendSummary { pub average_latency_ms: Option, pub max_latency_ms: Option, pub min_latency_ms: Option, + pub peak_first_token_p95_ms: Option, + pub peak_latency_p95_ms: Option, + pub minimum_output_throughput_p50: Option, pub success_rate: Option, pub cache_hit_request_rate: Option, pub costs: Vec, @@ -935,6 +955,7 @@ pub struct DashboardResult { /// 洞察健康时间点的语义结果。 #[derive(Debug, Clone, PartialEq)] pub struct UsageInsightsHealthPoint { + pub total_requests: u64, pub bucket_start: DateTime, pub success_requests: u64, pub failed_requests: u64, @@ -954,6 +975,7 @@ pub struct UsageInsightsHealth { pub incomplete_requests: u64, pub caller_error_requests: u64, pub success_rate: f64, + pub completion_rate: f64, pub points: Vec, } @@ -963,6 +985,13 @@ pub struct UsageInsightsPerformancePoint { pub bucket_start: DateTime, pub latency_percentiles: LatencyPercentiles, pub first_token_latency_percentiles: LatencyPercentiles, + pub admission_decision_percentiles: LatencyPercentiles, + pub account_selection_wait_percentiles: LatencyPercentiles, + pub output_throughput_p10: Option, + pub output_throughput_p50: Option, + pub output_throughput_p90: Option, + pub capacity_utilization_avg_basis_points: Option, + pub capacity_utilization_p95_basis_points: Option, } /// 洞察性能汇总与可观测覆盖率。 @@ -972,6 +1001,16 @@ pub struct UsageInsightsPerformance { pub first_token_latency_percentiles: LatencyPercentiles, pub latency_coverage: f64, pub first_token_coverage: f64, + pub admission_decision_percentiles: LatencyPercentiles, + pub account_selection_wait_percentiles: LatencyPercentiles, + pub admission_decision_coverage: f64, + pub account_selection_wait_coverage: f64, + pub output_throughput_p10: Option, + pub output_throughput_p50: Option, + pub output_throughput_p90: Option, + pub capacity_utilization_avg_basis_points: Option, + pub capacity_utilization_p95_basis_points: Option, + pub capacity_coverage: f64, pub points: Vec, } @@ -985,6 +1024,8 @@ pub struct UsageInsightsCostPoint { pub total_tokens: u64, pub estimated_cost: Option, pub standard_cost: Option, + pub no_cache_cost: Option, + pub cache_savings: Option, pub cached_token_rate: f64, pub cache_hit_request_rate: Option, } @@ -994,7 +1035,11 @@ pub struct UsageInsightsCostPoint { pub struct UsageInsightsCost { pub estimated_cost: Option, pub standard_cost: Option, + pub no_cache_cost: Option, + pub cache_savings: Option, + pub tier_premium: Option, pub cost_per_request: Option, + pub cost_per_successful_request: Option, pub tokens_per_request: f64, pub cached_token_rate: f64, pub cache_hit_request_rate: Option, @@ -1030,6 +1075,12 @@ pub struct DiagnosticsItem { pub request_share: f64, pub average_latency_ms: Option, pub latency_p95_ms: Option, + pub first_token_p95_ms: Option, + pub non_completion_count: u64, + pub non_completion_rate: f64, + pub retry_count: u64, + pub retry_rate: f64, + pub impact_score: f64, pub estimated_cost: Option, pub attempt_count: u64, pub total_tokens: u64, diff --git a/backend/crates/gateway-admin/src/use_case/observability.rs b/backend/crates/gateway-admin/src/use_case/observability.rs index 17c5afe70..e227fbf06 100644 --- a/backend/crates/gateway-admin/src/use_case/observability.rs +++ b/backend/crates/gateway-admin/src/use_case/observability.rs @@ -2,6 +2,7 @@ use std::{ collections::{BTreeMap, btree_map::Entry}, + str::FromStr, sync::Arc, }; @@ -246,26 +247,50 @@ impl ObservabilityService for DefaultObservabilityService { let total_requests = items.iter().fold(0_u64, |total, item| { total.saturating_add(item.request_count) }); - Ok(DiagnosticsResult { - dimension, - items: items - .into_iter() - .map(|item| DiagnosticsItem { + let mut items = items + .into_iter() + .map(|item| { + let error_rate = rate_or_zero(item.failure_count, item.request_count); + let non_completion_rate = + rate_or_zero(item.non_completion_count, item.request_count); + let retry_rate = rate_or_zero(item.retry_count, item.request_count); + let impact_score = diagnostic_impact_score( + item.request_count, + total_requests, + error_rate, + non_completion_rate, + retry_rate, + item.first_token_p95_ms, + ); + DiagnosticsItem { key: item.key, name: item.name, request_count: item.request_count, - success_count: item.request_count.saturating_sub(item.failure_count), + success_count: item.success_count, error_count: item.failure_count, - error_rate: rate_or_zero(item.failure_count, item.request_count), + error_rate, request_share: rate_or_zero(item.request_count, total_requests), average_latency_ms: item.average_latency_ms, latency_p95_ms: item.latency_p95_ms, + first_token_p95_ms: item.first_token_p95_ms, + non_completion_count: item.non_completion_count, + non_completion_rate, + retry_count: item.retry_count, + retry_rate, + impact_score, estimated_cost: usd_cost(&item.costs), attempt_count: item.attempt_count, total_tokens: item.total_tokens, - }) - .collect(), - }) + } + }) + .collect::>(); + items.sort_by(|left, right| { + right + .impact_score + .total_cmp(&left.impact_score) + .then_with(|| right.request_count.cmp(&left.request_count)) + }); + Ok(DiagnosticsResult { dimension, items }) } async fn ops_errors(&self, query: OpsErrorQuery) -> Result { @@ -279,7 +304,7 @@ impl ObservabilityService for DefaultObservabilityService { fn build_usage_insights( overview: UsageOverview, trend: Vec, - standard_costs: UsageStandardCosts, + standard_costs: UsageCostScenarios, ) -> Result { let granularity = trend.first().map_or( crate::model::observability::Granularity::FifteenMinutes, @@ -293,19 +318,26 @@ fn build_usage_insights( let failed_requests = service_failure_count(requests); let health_requests = requests.success_count.saturating_add(failed_requests); let health = UsageInsightsHealth { - total_requests: health_requests, + total_requests: requests.request_count, success_requests: requests.success_count, failed_requests, cancelled_requests: requests.cancelled_count, incomplete_requests: requests.incomplete_count, caller_error_requests: requests.caller_error_count, success_rate: rate_or_zero(requests.success_count, health_requests), + completion_rate: rate_or_zero( + requests + .success_count + .saturating_add(requests.failure_count), + requests.request_count, + ), points: trend .iter() .map(|point| { let failed_requests = service_failure_count(&point.metrics); let health_requests = point.metrics.success_count.saturating_add(failed_requests); UsageInsightsHealthPoint { + total_requests: point.metrics.request_count, bucket_start: point.bucket_start, success_requests: point.metrics.success_count, failed_requests, @@ -325,27 +357,74 @@ fn build_usage_insights( requests.first_token_latency_count, requests.request_count, ), + admission_decision_percentiles: requests.admission_decision_percentiles, + account_selection_wait_percentiles: requests.account_selection_wait_percentiles, + admission_decision_coverage: rate_or_zero( + requests.admission_decision_count, + requests.request_count, + ), + account_selection_wait_coverage: rate_or_zero( + requests.account_selection_wait_count, + requests.request_count, + ), + output_throughput_p10: requests.output_throughput_p10, + output_throughput_p50: requests.output_throughput_p50, + output_throughput_p90: requests.output_throughput_p90, + capacity_utilization_avg_basis_points: requests.capacity_utilization_avg_basis_points, + capacity_utilization_p95_basis_points: requests.capacity_utilization_p95_basis_points, + capacity_coverage: rate_or_zero(requests.capacity_sample_count, requests.request_count), points: trend .iter() .map(|point| UsageInsightsPerformancePoint { bucket_start: point.bucket_start, latency_percentiles: point.metrics.latency_percentiles, first_token_latency_percentiles: point.metrics.first_token_latency_percentiles, + admission_decision_percentiles: point.metrics.admission_decision_percentiles, + account_selection_wait_percentiles: point + .metrics + .account_selection_wait_percentiles, + output_throughput_p10: point.metrics.output_throughput_p10, + output_throughput_p50: point.metrics.output_throughput_p50, + output_throughput_p90: point.metrics.output_throughput_p90, + capacity_utilization_avg_basis_points: point + .metrics + .capacity_utilization_avg_basis_points, + capacity_utilization_p95_basis_points: point + .metrics + .capacity_utilization_p95_basis_points, }) .collect(), }; let estimated_cost = usd_cost(&overview.attempts.costs); - let UsageStandardCosts { - total: standard_cost, - by_bucket: standard_costs_by_bucket, + let UsageCostScenarios { + standard: + ScenarioCosts { + total: standard_cost, + by_bucket: standard_costs_by_bucket, + }, + no_cache: + ScenarioCosts { + total: no_cache_cost, + by_bucket: no_cache_costs_by_bucket, + }, } = standard_costs; + let cache_savings = amount_difference(no_cache_cost.as_ref(), estimated_cost.as_ref()); + let tier_premium = amount_difference(estimated_cost.as_ref(), standard_cost.as_ref()); + let cost_per_request = estimated_cost + .as_ref() + .and_then(|cost| cost.checked_div_u64(requests.request_count)); + let cost_per_successful_request = estimated_cost + .as_ref() + .and_then(|cost| cost.checked_div_u64(requests.success_count)); let cost = UsageInsightsCost { - cost_per_request: estimated_cost - .as_ref() - .and_then(|cost| cost.checked_div_u64(requests.request_count)), + cost_per_request, estimated_cost, standard_cost, + no_cache_cost, + cache_savings, + tier_premium, tokens_per_request: rate_or_zero(requests.total_tokens, requests.request_count), + cost_per_successful_request, cached_token_rate: rate_or_zero(requests.cached_tokens, requests.input_tokens), cache_hit_request_rate: ratio( requests.cache_hit_request_count, @@ -365,6 +444,11 @@ fn build_usage_insights( total_tokens: point.metrics.total_tokens, estimated_cost: usd_cost(&point.costs), standard_cost: standard_costs_by_bucket.get(&point.bucket_start).cloned(), + no_cache_cost: no_cache_costs_by_bucket.get(&point.bucket_start).cloned(), + cache_savings: amount_difference( + no_cache_costs_by_bucket.get(&point.bucket_start), + usd_cost(&point.costs).as_ref(), + ), cached_token_rate: rate_or_zero( point.metrics.cached_tokens, point.metrics.input_tokens, @@ -389,12 +473,12 @@ fn build_usage_insights( } #[derive(Debug, Default)] -struct UsageStandardCosts { +struct ScenarioCosts { total: Option, by_bucket: BTreeMap, DecimalAmount>, } -impl UsageStandardCosts { +impl ScenarioCosts { fn add( &mut self, bucket_start: DateTime, @@ -422,23 +506,29 @@ impl UsageStandardCosts { } } +#[derive(Debug, Default)] +struct UsageCostScenarios { + standard: ScenarioCosts, + no_cache: ScenarioCosts, +} + fn recover_standard_costs( providers: &ProviderAdminRegistry, facts: Vec, -) -> Result { - let mut standard_costs = UsageStandardCosts::default(); +) -> Result { + let mut scenarios = UsageCostScenarios::default(); for fact in facts { - let Ok(provider_kind) = ProviderKind::new(fact.provider_kind) else { + let Ok(provider_kind) = ProviderKind::new(fact.provider_kind.clone()) else { continue; }; let input = ProviderBillingInput { - upstream_model_id: fact.upstream_model_id, - service_tier: fact.service_tier, + upstream_model_id: fact.upstream_model_id.clone(), + service_tier: fact.service_tier.clone(), input_tokens: fact.input_tokens, output_tokens: fact.output_tokens, cached_tokens: fact.cached_tokens, cache_write_tokens: fact.cache_write_tokens, - total: fact.total, + total: fact.total.clone(), }; let breakdown = match providers.calculated_billing(&provider_kind, &input) { Ok(breakdown) => breakdown, @@ -455,9 +545,80 @@ fn recover_standard_costs( { continue; } - standard_costs.add(fact.bucket_start, breakdown.standard_amount.amount)?; + scenarios + .standard + .add(fact.bucket_start, breakdown.standard_amount.amount.clone())?; + if let Some(no_cache_cost) = no_cache_cost(&fact, &breakdown) { + scenarios.no_cache.add(fact.bucket_start, no_cache_cost)?; + } } - Ok(standard_costs) + Ok(scenarios) +} + +fn no_cache_cost( + fact: &UsageCalculatedBillingFact, + breakdown: &crate::model::observability::CalculatedBillingBreakdown, +) -> Option { + let input_tokens = fact.input_tokens?; + let cached_tokens = fact.cached_tokens.unwrap_or_default().min(input_tokens); + let cache_write_tokens = fact + .cache_write_tokens + .unwrap_or_default() + .min(input_tokens.saturating_sub(cached_tokens)); + let input_rate = decimal(&breakdown.input_price_per_million.amount)?; + let replaced_input_amount = input_rate + .scaled() + .checked_mul(u128::from(cached_tokens.saturating_add(cache_write_tokens)))? + .checked_div(1_000_000)?; + let total = decimal(&breakdown.total_amount.amount)? + .scaled() + .checked_sub(decimal(&breakdown.cache_read_amount.amount)?.scaled())? + .checked_sub(decimal(&breakdown.cache_write_amount.amount)?.scaled())? + .checked_add(replaced_input_amount)?; + DecimalAmount::from_str( + &gateway_core::accounting::Decimal::from_scaled(total) + .ok()? + .canonical(), + ) + .ok() +} + +fn amount_difference( + larger: Option<&DecimalAmount>, + smaller: Option<&DecimalAmount>, +) -> Option { + let difference = decimal(larger?)? + .scaled() + .checked_sub(decimal(smaller?)?.scaled())?; + DecimalAmount::from_str( + &gateway_core::accounting::Decimal::from_scaled(difference) + .ok()? + .canonical(), + ) + .ok() +} + +fn decimal(value: &DecimalAmount) -> Option { + value.as_str().parse().ok() +} + +fn diagnostic_impact_score( + request_count: u64, + total_requests: u64, + error_rate: f64, + non_completion_rate: f64, + retry_rate: f64, + first_token_p95_ms: Option, +) -> f64 { + let request_share = rate_or_zero(request_count, total_requests); + let slow_score = first_token_p95_ms + .map_or(0.0, |value| value as f64 / 30_000.0) + .min(1.0); + error_rate * 0.35 + + non_completion_rate * 0.25 + + retry_rate.min(1.0) * 0.20 + + request_share * 0.10 + + slow_score * 0.10 } fn usd_cost(costs: &[CurrencyCost]) -> Option { @@ -629,6 +790,20 @@ fn trend(kind: TrendKind, points: Vec) -> Result) -> Result, metrics: RequestMetrics) -> fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation { DiagnosticObservation { + first_token_p95_ms: None, + non_completion_count: 0, + retry_count: 0, key: name.to_owned(), name: name.to_owned(), request_count, diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs index b87ef4440..b1738940a 100644 --- a/backend/crates/gateway-api/src/admin/observability/presenter.rs +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -17,6 +17,21 @@ pub(crate) fn display_duration(value: Option) -> String { } } +fn display_duration_f64(value: Option) -> String { + let Some(value) = value.filter(|value| value.is_finite() && *value >= 0.0) else { + return "—".to_owned(); + }; + if value < 1_000.0 { + format!("{value:.0} ms") + } else if value < 60_000.0 { + format!("{:.2} s", value / 1_000.0) + } else if value < 3_600_000.0 { + format!("{:.1} min", value / 60_000.0) + } else { + format!("{:.1} h", value / 3_600_000.0) + } +} + pub(crate) fn display_rate(value: f64) -> String { if value.is_finite() { format!("{:.1}%", value * 100.0) @@ -25,6 +40,10 @@ pub(crate) fn display_rate(value: f64) -> String { } } +fn basis_points_rate(value: u64) -> f64 { + value as f64 / 10_000.0 +} + pub(crate) fn china_datetime(value: &DateTime) -> String { (*value + Duration::hours(8)) .format("%Y-%m-%d %H:%M:%S") @@ -267,6 +286,10 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView image_generation_requested: record.image_generation_requested, image_generation_succeeded: record.image_generation_succeeded, latency_details: UsageLatencyDetailsView { + admission_decision_ms: record.admission_decision_ms, + account_selection_wait_ms: record.account_selection_wait_ms, + capacity_used_slots: record.capacity_used_slots, + capacity_total_slots: record.capacity_total_slots, transport_decision_wait_ms: record.transport_decision_wait_ms, ws_connect_ms: record.connect_ms, upstream_headers_ms: record.headers_ms, @@ -514,6 +537,36 @@ pub(crate) fn trend_point_view(point: domain::TrendPoint) -> TrendPointView { success_rate: success_rate_value .map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")), success_rate_value, + first_token_p50_ms: point + .metrics + .first_token_latency_percentiles + .p50_ms + .map(|value| value.as_f64()), + first_token_p95_ms: point + .metrics + .first_token_latency_percentiles + .p95_ms + .map(|value| value.as_f64()), + latency_p95_ms: point + .metrics + .latency_percentiles + .p95_ms + .map(|value| value.as_f64()), + output_throughput_p50: point.metrics.output_throughput_p50, + admission_decision_p95_ms: point + .metrics + .admission_decision_percentiles + .p95_ms + .map(|value| value.as_f64()), + account_selection_wait_p95_ms: point + .metrics + .account_selection_wait_percentiles + .p95_ms + .map(|value| value.as_f64()), + capacity_utilization: point + .metrics + .capacity_utilization_p95_basis_points + .map(basis_points_rate), } } @@ -541,18 +594,20 @@ pub(crate) fn trend_summary_view( ], TrendKind::Latency => vec![ TrendSummaryView { - label: "平均".to_owned(), - value: display_duration(summary.average_latency_ms), + label: "首字 P95".to_owned(), + value: display_duration_f64(summary.peak_first_token_p95_ms), ratio: None, }, TrendSummaryView { - label: "最高".to_owned(), - value: display_duration(summary.max_latency_ms), + label: "总耗时 P95".to_owned(), + value: display_duration_f64(summary.peak_latency_p95_ms), ratio: None, }, TrendSummaryView { - label: "最低".to_owned(), - value: display_duration(summary.min_latency_ms), + label: "吞吐 P50".to_owned(), + value: summary + .minimum_output_throughput_p50 + .map_or_else(|| "—".to_owned(), |value| format!("{value} tok/s")), ratio: None, }, ], @@ -821,6 +876,7 @@ pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsig .map(|point| OverviewHealthPointView { bucket: point.bucket_start, label: china_label(point.bucket_start, "%m-%d %H:%M"), + total_requests: point.total_requests, success_requests: point.success_requests, failed_requests: point.failed_requests, cancelled_requests: point.cancelled_requests, @@ -851,6 +907,31 @@ pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsig .first_token_latency_percentiles .p99_ms .map(|value| value.as_f64()), + admission_decision_p50_ms: point + .admission_decision_percentiles + .p50_ms + .map(|value| value.as_f64()), + admission_decision_p95_ms: point + .admission_decision_percentiles + .p95_ms + .map(|value| value.as_f64()), + account_selection_wait_p50_ms: point + .account_selection_wait_percentiles + .p50_ms + .map(|value| value.as_f64()), + account_selection_wait_p95_ms: point + .account_selection_wait_percentiles + .p95_ms + .map(|value| value.as_f64()), + output_throughput_p10: point.output_throughput_p10, + output_throughput_p50: point.output_throughput_p50, + output_throughput_p90: point.output_throughput_p90, + capacity_utilization: point + .capacity_utilization_avg_basis_points + .map(basis_points_rate), + capacity_utilization_p95: point + .capacity_utilization_p95_basis_points + .map(basis_points_rate), }) .collect(); let cost_points = insights @@ -866,6 +947,8 @@ pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsig total_tokens: point.total_tokens, estimated_cost: point.estimated_cost.as_ref().map(ToString::to_string), standard_cost: point.standard_cost.as_ref().map(ToString::to_string), + no_cache_cost: point.no_cache_cost.as_ref().map(ToString::to_string), + cache_savings: point.cache_savings.as_ref().map(ToString::to_string), cached_token_rate: point.cached_token_rate, cache_hit_request_rate: point.cache_hit_request_rate, }) @@ -880,6 +963,7 @@ pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsig incomplete_requests: insights.health.incomplete_requests, caller_error_requests: insights.health.caller_error_requests, success_rate: insights.health.success_rate, + completion_rate: insights.health.completion_rate, request_change_rate: None, success_rate_change: None, points: health_points, @@ -915,8 +999,42 @@ pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsig .first_token_latency_percentiles .p99_ms .map(|value| value.as_f64()), + admission_decision_p50_ms: insights + .performance + .admission_decision_percentiles + .p50_ms + .map(|value| value.as_f64()), + admission_decision_p95_ms: insights + .performance + .admission_decision_percentiles + .p95_ms + .map(|value| value.as_f64()), + account_selection_wait_p50_ms: insights + .performance + .account_selection_wait_percentiles + .p50_ms + .map(|value| value.as_f64()), + account_selection_wait_p95_ms: insights + .performance + .account_selection_wait_percentiles + .p95_ms + .map(|value| value.as_f64()), + output_throughput_p10: insights.performance.output_throughput_p10, + output_throughput_p50: insights.performance.output_throughput_p50, + output_throughput_p90: insights.performance.output_throughput_p90, + capacity_utilization: insights + .performance + .capacity_utilization_avg_basis_points + .map(basis_points_rate), + capacity_utilization_p95: insights + .performance + .capacity_utilization_p95_basis_points + .map(basis_points_rate), latency_coverage: insights.performance.latency_coverage, first_token_coverage: insights.performance.first_token_coverage, + admission_decision_coverage: insights.performance.admission_decision_coverage, + account_selection_wait_coverage: insights.performance.account_selection_wait_coverage, + capacity_coverage: insights.performance.capacity_coverage, points: performance_points, }, cost: OverviewCostView { @@ -930,11 +1048,27 @@ pub(crate) fn usage_insights_view(insights: domain::UsageInsights) -> UsageInsig .standard_cost .as_ref() .map(ToString::to_string), + no_cache_cost: insights + .cost + .no_cache_cost + .as_ref() + .map(ToString::to_string), + cache_savings: insights + .cost + .cache_savings + .as_ref() + .map(ToString::to_string), + tier_premium: insights.cost.tier_premium.as_ref().map(ToString::to_string), cost_per_request: insights .cost .cost_per_request .as_ref() .map(ToString::to_string), + cost_per_successful_request: insights + .cost + .cost_per_successful_request + .as_ref() + .map(ToString::to_string), tokens_per_request: insights.cost.tokens_per_request, cached_token_rate: insights.cost.cached_token_rate, cache_hit_request_rate: insights.cost.cache_hit_request_rate, @@ -992,6 +1126,12 @@ pub(crate) fn diagnostics_view( request_share: item.request_share, average_latency_ms: item.average_latency_ms, latency_p95_ms: item.latency_p95_ms, + first_token_p95_ms: item.first_token_p95_ms, + non_completion_count: item.non_completion_count, + non_completion_rate: item.non_completion_rate, + retry_count: item.retry_count, + retry_rate: item.retry_rate, + impact_score: item.impact_score, estimated_cost: item.estimated_cost.as_ref().map(ToString::to_string), attempt_count: item.attempt_count, total_tokens: item.total_tokens, diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs index fd5a2b8e3..6eb89a5de 100644 --- a/backend/crates/gateway-api/src/admin/observability/wire.rs +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -157,6 +157,14 @@ pub struct WebSocketPoolMetadataView { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct UsageLatencyDetailsView { + #[serde(skip_serializing_if = "Option::is_none")] + pub admission_decision_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub account_selection_wait_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub capacity_used_slots: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub capacity_total_slots: Option, #[serde(skip_serializing_if = "Option::is_none")] pub transport_decision_wait_ms: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -256,6 +264,13 @@ pub struct TrendPointView { pub min_latency_value: Option, pub success_rate: String, pub success_rate_value: Option, + pub first_token_p50_ms: Option, + pub first_token_p95_ms: Option, + pub latency_p95_ms: Option, + pub output_throughput_p50: Option, + pub admission_decision_p95_ms: Option, + pub account_selection_wait_p95_ms: Option, + pub capacity_utilization: Option, } /// Dashboard 趋势摘要。 @@ -537,6 +552,7 @@ pub struct UsageSummaryView { pub struct OverviewHealthPointView { pub bucket: DateTime, pub label: String, + pub total_requests: u64, pub success_requests: u64, pub failed_requests: u64, pub cancelled_requests: u64, @@ -556,6 +572,7 @@ pub struct OverviewHealthView { pub incomplete_requests: u64, pub caller_error_requests: u64, pub success_rate: f64, + pub completion_rate: f64, pub request_change_rate: Option, pub success_rate_change: Option, pub points: Vec, @@ -573,6 +590,15 @@ pub struct OverviewPerformancePointView { pub first_token_p50_ms: Option, pub first_token_p95_ms: Option, pub first_token_p99_ms: Option, + pub admission_decision_p50_ms: Option, + pub admission_decision_p95_ms: Option, + pub account_selection_wait_p50_ms: Option, + pub account_selection_wait_p95_ms: Option, + pub output_throughput_p10: Option, + pub output_throughput_p50: Option, + pub output_throughput_p90: Option, + pub capacity_utilization: Option, + pub capacity_utilization_p95: Option, } /// 洞察性能摘要。 @@ -585,8 +611,20 @@ pub struct OverviewPerformanceView { pub first_token_p50_ms: Option, pub first_token_p95_ms: Option, pub first_token_p99_ms: Option, + pub admission_decision_p50_ms: Option, + pub admission_decision_p95_ms: Option, + pub account_selection_wait_p50_ms: Option, + pub account_selection_wait_p95_ms: Option, + pub output_throughput_p10: Option, + pub output_throughput_p50: Option, + pub output_throughput_p90: Option, + pub capacity_utilization: Option, + pub capacity_utilization_p95: Option, pub latency_coverage: f64, pub first_token_coverage: f64, + pub admission_decision_coverage: f64, + pub account_selection_wait_coverage: f64, + pub capacity_coverage: f64, pub points: Vec, } @@ -602,6 +640,8 @@ pub struct OverviewCostPointView { pub total_tokens: u64, pub estimated_cost: Option, pub standard_cost: Option, + pub no_cache_cost: Option, + pub cache_savings: Option, pub cached_token_rate: f64, pub cache_hit_request_rate: Option, } @@ -612,7 +652,11 @@ pub struct OverviewCostPointView { pub struct OverviewCostView { pub estimated_cost: Option, pub standard_cost: Option, + pub no_cache_cost: Option, + pub cache_savings: Option, + pub tier_premium: Option, pub cost_per_request: Option, + pub cost_per_successful_request: Option, pub tokens_per_request: f64, pub cached_token_rate: f64, pub cache_hit_request_rate: Option, @@ -661,6 +705,12 @@ pub struct DiagnosticItemView { pub request_share: f64, pub average_latency_ms: Option, pub latency_p95_ms: Option, + pub first_token_p95_ms: Option, + pub non_completion_count: u64, + pub non_completion_rate: f64, + pub retry_count: u64, + pub retry_rate: f64, + pub impact_score: f64, pub estimated_cost: Option, pub attempt_count: u64, pub total_tokens: u64, diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index 624fbff51..038f00d3b 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -110,6 +110,13 @@ fn trend_wire_should_serialize_kind_and_values_without_store_types() { min_latency_value: Some(1), success_rate: "100.0%".to_owned(), success_rate_value: Some(100.0), + first_token_p50_ms: Some(1.0), + first_token_p95_ms: Some(2.0), + latency_p95_ms: Some(3.0), + output_throughput_p50: Some(50), + admission_decision_p95_ms: Some(4.0), + account_selection_wait_p95_ms: Some(5.0), + capacity_utilization: Some(0.7), }], summary: vec![TrendSummaryView { label: "输入".to_owned(), @@ -636,6 +643,9 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() { total_tokens: 200, average_latency_ms: Some(100), latency_p95_ms: Some(3800), + first_token_p95_ms: Some(1200), + non_completion_count: 0, + retry_count: 0, cost_coverage: CostCoverage::default(), costs: Vec::new(), }); @@ -739,6 +749,10 @@ fn usage_record_with_account( first_token_ms: None, provider_processing_ms: None, latency_ms: None, + admission_decision_ms: None, + account_selection_wait_ms: None, + capacity_used_slots: None, + capacity_total_slots: None, client_ip: None, user_agent: None, reasoning_effort: None, @@ -890,6 +904,10 @@ async fn usage_route_should_expose_image_and_websocket_facts() { first_token_ms: Some(19), provider_processing_ms: Some(29), latency_ms: Some(31), + admission_decision_ms: Some(3), + account_selection_wait_ms: Some(5), + capacity_used_slots: Some(7), + capacity_total_slots: Some(10), client_ip: Some("127.0.0.1".to_owned()), user_agent: Some("usage-metadata-test".to_owned()), reasoning_effort: Some("max".to_owned()), diff --git a/backend/crates/gateway-core/src/engine/coordinator.rs b/backend/crates/gateway-core/src/engine/coordinator.rs index 58adf0234..00b647a1c 100644 --- a/backend/crates/gateway-core/src/engine/coordinator.rs +++ b/backend/crates/gateway-core/src/engine/coordinator.rs @@ -753,6 +753,8 @@ where } let metadata = stream.metadata().clone(); + let selection_observation = metadata.selection_observation(); + let capacity = selection_observation.and_then(|observation| observation.capacity()); if !self.account_selection.is_diagnostic() && !candidate .account_scope() @@ -830,6 +832,10 @@ where upstream_model_id: metadata.upstream_model().cloned(), upstream_transport: metadata.transport().as_str().to_owned(), http_version: None, + account_selection_wait_ms: selection_observation + .map(|observation| observation.account_selection_wait_ms()), + capacity_used_slots: capacity.map(|snapshot| snapshot.used_slots()), + capacity_total_slots: capacity.map(|snapshot| snapshot.total_slots()), }; if self.request_persisted { best_effort_store_write( diff --git a/backend/crates/gateway-core/src/engine/credential.rs b/backend/crates/gateway-core/src/engine/credential.rs index 8a9be1420..4a3893187 100644 --- a/backend/crates/gateway-core/src/engine/credential.rs +++ b/backend/crates/gateway-core/src/engine/credential.rs @@ -1525,6 +1525,36 @@ pub struct AccountCandidate { pub signals: AccountRuntimeSignals, } +/// 一次账号选择看到的可调度并发槽快照。 +/// +/// `used_slots` 包含刚刚获取成功的当前请求；快照只覆盖本次请求范围内、 +/// 模型可用且未被显式排除的账号池。 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AccountCapacitySnapshot { + used_slots: u64, + total_slots: u64, +} + +impl AccountCapacitySnapshot { + #[must_use] + pub const fn used_slots(self) -> u64 { + self.used_slots + } + + #[must_use] + pub const fn total_slots(self) -> u64 { + self.total_slots + } + + #[must_use] + pub fn with_acquired_request(self) -> Self { + Self { + used_slots: self.used_slots.saturating_add(1).min(self.total_slots), + total_slots: self.total_slots, + } + } +} + /// 一次账号选择使用的全局策略快照。 #[derive(Debug, Clone)] pub struct AccountSelectionContext { @@ -1600,6 +1630,43 @@ impl<'a> AccountSelection<'a> { pub struct AccountSelector; impl AccountSelector { + /// 汇总与本次调度约束一致的并发容量，供请求级观测使用。 + #[must_use] + pub fn capacity_snapshot( + &self, + candidates: &[AccountCandidate], + context: &AccountSelectionContext, + ) -> Option { + let (used_slots, total_slots) = candidates + .iter() + .filter(|candidate| { + !matches!( + self.scheduling_blocker(candidate, context), + Some( + AccountSchedulingBlocker::OutsideClientScope + | AccountSchedulingBlocker::LocalAvailability + | AccountSchedulingBlocker::Excluded + ) + ) + }) + .fold((0_u64, 0_u64), |(used, total), candidate| { + let capacity = u64::from( + candidate + .account + .effective_concurrency(context.policy.max_concurrent_per_account()) + .get(), + ); + ( + used.saturating_add(u64::from(candidate.signals.in_flight)), + total.saturating_add(capacity), + ) + }); + (total_slots > 0).then_some(AccountCapacitySnapshot { + used_slots: used_slots.min(total_slots), + total_slots, + }) + } + /// 从可调度账号中确定一个候选；这里只消费 Provider 已解析的额度投影。 #[must_use] pub fn select<'a>( diff --git a/backend/crates/gateway-core/src/engine/execution.rs b/backend/crates/gateway-core/src/engine/execution.rs index ecebca3a6..0b89ab7fd 100644 --- a/backend/crates/gateway-core/src/engine/execution.rs +++ b/backend/crates/gateway-core/src/engine/execution.rs @@ -5,7 +5,7 @@ use std::fmt; use std::net::IpAddr; use std::num::NonZeroU32; use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::{Duration, Instant, SystemTime}; use futures::{FutureExt, future::BoxFuture, pin_mut, select_biased}; use futures_timer::Delay; @@ -473,6 +473,7 @@ impl DefaultExecutionService { lease_ttl: MODEL_REQUEST_DEADLINE, limits: client.policy.limits(), }; + let admission_started_at = Instant::now(); match self .admissions .admit(admission_request) @@ -494,6 +495,7 @@ impl DefaultExecutionService { )); } } + let admission_decision_ms = duration_ms(admission_started_at.elapsed()); let admission = AdmissionLease { port: Arc::clone(&self.admissions), client_api_key_id: client.policy.key_id().clone(), @@ -525,6 +527,7 @@ impl DefaultExecutionService { subagent_kind: observation.subagent_kind, compact: observation.compact, image_generation_requested: operation.image_generation_requested(), + admission_decision_ms: Some(admission_decision_ms), started_at, deadline_at, }; @@ -666,6 +669,7 @@ impl DefaultExecutionService { subagent_kind: None, compact: false, image_generation_requested: false, + admission_decision_ms: None, started_at, deadline_at, }; @@ -1184,6 +1188,10 @@ fn constant_time_equal(left: &str, right: &str) -> bool { == 0 } +fn duration_ms(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + fn new_request_id() -> Result { ModelRequestId::new(format!("req_{}", Uuid::now_v7().simple())) .map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "failed to allocate request ID")) diff --git a/backend/crates/gateway-core/src/engine/mod.rs b/backend/crates/gateway-core/src/engine/mod.rs index 4645aecfa..4cf05544b 100644 --- a/backend/crates/gateway-core/src/engine/mod.rs +++ b/backend/crates/gateway-core/src/engine/mod.rs @@ -569,6 +569,8 @@ pub struct NewModelRequest { pub subagent_kind: Option, pub compact: bool, pub image_generation_requested: bool, + /// Client Key 准入判定的完整耗时；内部探测不经过该阶段。 + pub admission_decision_ms: Option, pub started_at: SystemTime, pub deadline_at: SystemTime, } @@ -585,6 +587,11 @@ pub struct AttemptRecord { pub upstream_model_id: Option, pub upstream_transport: String, pub http_version: Option, + /// 本 attempt 从进入 Provider 选择器到持有账号 lease 的等待。 + pub account_selection_wait_ms: Option, + /// 选择成功后（含当前请求）的请求级账号池容量快照。 + pub capacity_used_slots: Option, + pub capacity_total_slots: Option, } /// 需要解释换号的中间失败。 diff --git a/backend/crates/gateway-core/src/engine/provider.rs b/backend/crates/gateway-core/src/engine/provider.rs index 4825506ba..c42aa160c 100644 --- a/backend/crates/gateway-core/src/engine/provider.rs +++ b/backend/crates/gateway-core/src/engine/provider.rs @@ -11,7 +11,9 @@ use async_trait::async_trait; use futures::Stream; use thiserror::Error; -use crate::engine::credential::{AccountAttemptFeedback, AccountFeedbackStats, ProviderAccountId}; +use crate::engine::credential::{ + AccountAttemptFeedback, AccountCapacitySnapshot, AccountFeedbackStats, ProviderAccountId, +}; use crate::engine::{AttemptContext, UpstreamSendState}; use crate::error::{ IdentifierError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind, validate_text, @@ -57,6 +59,37 @@ pub struct ProviderCallMetadata { provider_account_id: ProviderAccountId, upstream_request_id: Option, transport: UpstreamTransport, + selection_observation: Option, +} + +/// Provider 账号选择阶段输出的中立运行压力事实。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderSelectionObservation { + account_selection_wait_ms: u64, + capacity: Option, +} + +impl ProviderSelectionObservation { + #[must_use] + pub const fn new( + account_selection_wait_ms: u64, + capacity: Option, + ) -> Self { + Self { + account_selection_wait_ms, + capacity, + } + } + + #[must_use] + pub const fn account_selection_wait_ms(self) -> u64 { + self.account_selection_wait_ms + } + + #[must_use] + pub const fn capacity(self) -> Option { + self.capacity + } } impl ProviderCallMetadata { @@ -74,6 +107,7 @@ impl ProviderCallMetadata { provider_account_id, upstream_request_id: None, transport, + selection_observation: None, } } @@ -90,6 +124,7 @@ impl ProviderCallMetadata { provider_account_id, upstream_request_id: None, transport, + selection_observation: None, } } @@ -100,6 +135,16 @@ impl ProviderCallMetadata { self } + /// 附加 Provider 选择账号时测得的等待与容量快照。 + #[must_use] + pub const fn with_selection_observation( + mut self, + observation: ProviderSelectionObservation, + ) -> Self { + self.selection_observation = Some(observation); + self + } + /// 返回 Provider。 #[must_use] pub const fn provider(&self) -> &ProviderKind { @@ -130,6 +175,11 @@ impl ProviderCallMetadata { &self.transport } + #[must_use] + pub const fn selection_observation(&self) -> Option { + self.selection_observation + } + /// 确认 metadata 没有替换请求计划中冻结的 Provider 候选。 #[must_use] pub fn confirms(&self, candidate: &ProviderCandidate) -> bool { diff --git a/backend/crates/gateway-core/tests/engine/coordinator.rs b/backend/crates/gateway-core/tests/engine/coordinator.rs index 52a4a44ea..3c82ae255 100644 --- a/backend/crates/gateway-core/tests/engine/coordinator.rs +++ b/backend/crates/gateway-core/tests/engine/coordinator.rs @@ -603,6 +603,7 @@ fn plan_with_policy( fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest { let client_key = ClientApiKeyId::new("key_client_1").expect("client key id"); NewModelRequest { + admission_decision_ms: None, id: ModelRequestId::new("req_core_1").expect("request id"), client_api_key_id: Some(client_key.clone()), client_api_key_ref: client_key, diff --git a/backend/crates/gateway-store/src/lib.rs b/backend/crates/gateway-store/src/lib.rs index afacfa48b..30f56a74f 100644 --- a/backend/crates/gateway-store/src/lib.rs +++ b/backend/crates/gateway-store/src/lib.rs @@ -1,6 +1,6 @@ //! 多 Provider 网关的 PostgreSQL 持久化与 Redis 协调 adapter。 //! -//! 业务规则与 port 由 `gateway-core` / `gateway-admin` 拥有。本 crate 只负责把当前十二张业务表 +//! 业务规则与 port 由 `gateway-core` / `gateway-admin` 拥有。本 crate 只负责把 PostgreSQL 业务表 //! 和可丢失 Redis 状态映射为明确的基础设施操作。 use std::sync::Arc; diff --git a/backend/crates/gateway-store/src/postgres/execution.rs b/backend/crates/gateway-store/src/postgres/execution.rs index de961fc44..68c5cef9f 100644 --- a/backend/crates/gateway-store/src/postgres/execution.rs +++ b/backend/crates/gateway-store/src/postgres/execution.rs @@ -100,6 +100,7 @@ pub struct NewModelRequest { pub subagent_kind: Option, pub compact: bool, pub image_generation_requested: bool, + pub admission_decision_ms: Option, pub started_at: DateTime, pub deadline_at: DateTime, } @@ -146,6 +147,9 @@ pub struct ModelRequestAttemptStart { pub upstream_model_id: Option, pub upstream_transport: String, pub http_version: Option, + pub account_selection_wait_ms: Option, + pub capacity_used_slots: Option, + pub capacity_total_slots: Option, } impl ModelRequestAttemptStart { @@ -163,6 +167,11 @@ impl ModelRequestAttemptStart { if self.attempt_count == 0 { return Err(invalid("attempt_count must be positive")); } + match (self.capacity_used_slots, self.capacity_total_slots) { + (None, None) => {} + (Some(used), Some(total)) if total > 0 && used <= total => {} + _ => return Err(invalid("capacity snapshot is invalid")), + } if self .provider_account_id .as_ref() @@ -379,11 +388,11 @@ impl ModelRequestRepository for PgExecutionStore { operation, endpoint, client_transport, requested_model_id, client_ip, user_agent, reasoning_effort, reasoning_preset, request_kind, subagent_kind, compact, - image_generation_requested, started_at, deadline_at + image_generation_requested, admission_decision_ms, started_at, deadline_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13::inet, $14, $15, - $16, $17, $18, $19, $20, $21, $22 + $16, $17, $18, $19, $20, $21, $22, $23 )", ) .bind(request.id) @@ -406,6 +415,10 @@ impl ModelRequestRepository for PgExecutionStore { .bind(request.subagent_kind) .bind(request.compact) .bind(request.image_generation_requested) + .bind(optional_i64( + request.admission_decision_ms, + "admission_decision_ms", + )?) .bind(request.started_at) .bind(request.deadline_at) .execute(&self.pool) @@ -431,21 +444,22 @@ impl ModelRequestRepository for PgExecutionStore { operation, endpoint, client_transport, requested_model_id, client_ip, user_agent, reasoning_effort, reasoning_preset, request_kind, subagent_kind, compact, - image_generation_requested, started_at, deadline_at, + image_generation_requested, admission_decision_ms, started_at, deadline_at, provider_kind, provider_account_id, provider_account_ref, provider_account_name_snapshot, provider_account_email_snapshot, provider_account_authentication_kind_snapshot, upstream_model_id, upstream_transport, http_version, - attempt_count, upstream_send_state + attempt_count, upstream_send_state, account_selection_wait_ms, + capacity_used_slots, capacity_total_slots ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13::inet, $14, $15, - $16, $17, $18, $19, $20, $21, $22, - $23, $24, $25, - (select name from provider_accounts where id = $24), - (select email from provider_accounts where id = $24), - (select authentication_kind from provider_accounts where id = $24), - $26, $27, $28, 1, 'not_sent' + $16, $17, $18, $19, $20, $21, $22, $23, + $24, $25, $26, + (select name from provider_accounts where id = $25), + (select email from provider_accounts where id = $25), + (select authentication_kind from provider_accounts where id = $25), + $27, $28, $29, 1, 'not_sent', $30, $31, $32 )", ) .bind(request.id) @@ -468,6 +482,10 @@ impl ModelRequestRepository for PgExecutionStore { .bind(request.subagent_kind) .bind(request.compact) .bind(request.image_generation_requested) + .bind(optional_i64( + request.admission_decision_ms, + "admission_decision_ms", + )?) .bind(request.started_at) .bind(request.deadline_at) .bind(attempt.provider_kind) @@ -476,6 +494,18 @@ impl ModelRequestRepository for PgExecutionStore { .bind(attempt.upstream_model_id) .bind(attempt.upstream_transport) .bind(attempt.http_version) + .bind(optional_i64( + attempt.account_selection_wait_ms, + "account_selection_wait_ms", + )?) + .bind(optional_i64( + attempt.capacity_used_slots, + "capacity_used_slots", + )?) + .bind(optional_i64( + attempt.capacity_total_slots, + "capacity_total_slots", + )?) .execute(&self.pool) .await .map_err(|_| postgres_unavailable("insert model request with first attempt"))?; @@ -507,7 +537,13 @@ impl ModelRequestRepository for PgExecutionStore { upstream_model_id = $5, upstream_transport = $6, http_version = $7, - attempt_count = $8 + attempt_count = $8, + account_selection_wait_ms = case + when $9::bigint is null then account_selection_wait_ms + else coalesce(account_selection_wait_ms, 0) + $9 + end, + capacity_used_slots = coalesce($10, capacity_used_slots), + capacity_total_slots = coalesce($11, capacity_total_slots) where id = $1 and outcome = 'running' and downstream_committed_at is null and $8 = attempt_count + 1 returning attempt_count", @@ -523,6 +559,18 @@ impl ModelRequestRepository for PgExecutionStore { i32::try_from(attempt.attempt_count) .map_err(|_| invalid("attempt_count is too large"))?, ) + .bind(optional_i64( + attempt.account_selection_wait_ms, + "account_selection_wait_ms", + )?) + .bind(optional_i64( + attempt.capacity_used_slots, + "capacity_used_slots", + )?) + .bind(optional_i64( + attempt.capacity_total_slots, + "capacity_total_slots", + )?) .fetch_optional(&self.pool) .await .map_err(|_| postgres_unavailable("begin model request attempt"))? @@ -1099,6 +1147,7 @@ fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest { subagent_kind: request.subagent_kind, compact: request.compact, image_generation_requested: request.image_generation_requested, + admission_decision_ms: request.admission_decision_ms, started_at: DateTime::::from(request.started_at), deadline_at: DateTime::::from(request.deadline_at), } @@ -1164,6 +1213,9 @@ fn attempt_start_row(attempt: CoreAttemptRecord) -> ModelRequestAttemptStart { .map(|model| model.as_str().to_owned()), upstream_transport: attempt.upstream_transport, http_version: attempt.http_version, + account_selection_wait_ms: attempt.account_selection_wait_ms, + capacity_used_slots: attempt.capacity_used_slots, + capacity_total_slots: attempt.capacity_total_slots, } } diff --git a/backend/crates/gateway-store/src/postgres/mod.rs b/backend/crates/gateway-store/src/postgres/mod.rs index b831bd22e..55569d2b1 100644 --- a/backend/crates/gateway-store/src/postgres/mod.rs +++ b/backend/crates/gateway-store/src/postgres/mod.rs @@ -1,4 +1,4 @@ -//! 当前十二张业务表的 PostgreSQL adapters。 +//! PostgreSQL 业务表的 adapters。 use async_trait::async_trait; use sqlx::{PgPool, postgres::PgPoolOptions}; diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index 6d4b4ecda..bb294b593 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -175,6 +175,20 @@ pub(crate) fn admin_request_metrics( first_token_latency_percentiles: admin_latency_percentiles( metrics.first_token_latency_percentiles, )?, + admission_decision_count: metrics.admission_decision_count, + admission_decision_percentiles: admin_latency_percentiles( + metrics.admission_decision_percentiles, + )?, + account_selection_wait_count: metrics.account_selection_wait_count, + account_selection_wait_percentiles: admin_latency_percentiles( + metrics.account_selection_wait_percentiles, + )?, + output_throughput_p10: metrics.output_throughput_p10, + output_throughput_p50: metrics.output_throughput_p50, + output_throughput_p90: metrics.output_throughput_p90, + capacity_sample_count: metrics.capacity_sample_count, + capacity_utilization_avg_basis_points: metrics.capacity_utilization_avg_basis_points, + capacity_utilization_p95_basis_points: metrics.capacity_utilization_p95_basis_points, cache_eligible_request_count: metrics.cache_eligible_request_count, cache_hit_request_count: metrics.cache_hit_request_count, }) @@ -489,6 +503,10 @@ pub(crate) fn admin_usage_record( first_token_ms: record.first_token_ms, provider_processing_ms: record.provider_processing_ms, latency_ms: record.latency_ms, + admission_decision_ms: record.admission_decision_ms, + account_selection_wait_ms: record.account_selection_wait_ms, + capacity_used_slots: record.capacity_used_slots, + capacity_total_slots: record.capacity_total_slots, client_ip: record.client_ip, user_agent: record.user_agent, reasoning_effort: record.reasoning_effort, @@ -607,6 +625,9 @@ pub(crate) fn admin_diagnostic_observation( total_tokens: observation.total_tokens, average_latency_ms: observation.average_latency_ms, latency_p95_ms: observation.latency_p95_ms, + first_token_p95_ms: observation.first_token_p95_ms, + non_completion_count: observation.non_completion_count, + retry_count: observation.retry_count, cost_coverage: admin_cost_coverage(observation.cost_coverage), costs: admin_currency_costs(observation.costs)?, }) @@ -717,6 +738,10 @@ pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult< first_token_ms: optional_unsigned(row, "first_token_ms")?, provider_processing_ms: optional_unsigned(row, "provider_processing_ms")?, latency_ms: optional_unsigned(row, "latency_ms")?, + admission_decision_ms: optional_unsigned(row, "admission_decision_ms")?, + account_selection_wait_ms: optional_unsigned(row, "account_selection_wait_ms")?, + capacity_used_slots: optional_unsigned(row, "capacity_used_slots")?, + capacity_total_slots: optional_unsigned(row, "capacity_total_slots")?, client_ip: get(row, "client_ip")?, user_agent: get(row, "user_agent")?, reasoning_effort: get(row, "reasoning_effort")?, diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs index a4de8cd06..b70bf7f3b 100644 --- a/backend/crates/gateway-store/src/postgres/observability/model.rs +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -255,6 +255,16 @@ pub struct RequestMetrics { pub cache_hit_request_count: u64, pub latency_percentiles: LatencyPercentiles, pub first_token_latency_percentiles: LatencyPercentiles, + pub admission_decision_count: u64, + pub admission_decision_percentiles: LatencyPercentiles, + pub account_selection_wait_count: u64, + pub account_selection_wait_percentiles: LatencyPercentiles, + pub output_throughput_p10: Option, + pub output_throughput_p50: Option, + pub output_throughput_p90: Option, + pub capacity_sample_count: u64, + pub capacity_utilization_avg_basis_points: Option, + pub capacity_utilization_p95_basis_points: Option, } impl RequestMetrics { @@ -549,6 +559,10 @@ pub struct UsageRecord { pub first_token_ms: Option, pub provider_processing_ms: Option, pub latency_ms: Option, + pub admission_decision_ms: Option, + pub account_selection_wait_ms: Option, + pub capacity_used_slots: Option, + pub capacity_total_slots: Option, pub client_ip: Option, pub user_agent: Option, pub reasoning_effort: Option, @@ -640,6 +654,9 @@ pub struct DiagnosticObservation { pub total_tokens: u64, pub average_latency_ms: Option, pub latency_p95_ms: Option, + pub first_token_p95_ms: Option, + pub non_completion_count: u64, + pub retry_count: u64, pub cost_coverage: CostCoverage, pub costs: Vec, } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs index a5de6fd28..dbf1f4cad 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs @@ -50,7 +50,44 @@ pub(crate) async fn request_metrics( percentile_cont(0.95) within group (order by first_token_ms) filter (where {fact}) as first_token_p95_ms, percentile_cont(0.99) within group (order by first_token_ms) - filter (where {fact}) as first_token_p99_ms + filter (where {fact}) as first_token_p99_ms, + count(admission_decision_ms)::bigint as admission_decision_count, + percentile_cont(0.50) within group (order by admission_decision_ms) + as admission_decision_p50_ms, + percentile_cont(0.95) within group (order by admission_decision_ms) + as admission_decision_p95_ms, + percentile_cont(0.99) within group (order by admission_decision_ms) + as admission_decision_p99_ms, + count(account_selection_wait_ms)::bigint as account_selection_wait_count, + percentile_cont(0.50) within group (order by account_selection_wait_ms) + as account_selection_wait_p50_ms, + percentile_cont(0.95) within group (order by account_selection_wait_ms) + as account_selection_wait_p95_ms, + percentile_cont(0.99) within group (order by account_selection_wait_ms) + as account_selection_wait_p99_ms, + round((percentile_cont(0.10) within group ( + order by output_tokens::double precision * 1000.0 + / greatest(latency_ms - first_token_ms, 1) + ) filter (where {fact} and output_tokens > 0 + and latency_ms > first_token_ms)))::bigint as output_throughput_p10, + round((percentile_cont(0.50) within group ( + order by output_tokens::double precision * 1000.0 + / greatest(latency_ms - first_token_ms, 1) + ) filter (where {fact} and output_tokens > 0 + and latency_ms > first_token_ms)))::bigint as output_throughput_p50, + round((percentile_cont(0.90) within group ( + order by output_tokens::double precision * 1000.0 + / greatest(latency_ms - first_token_ms, 1) + ) filter (where {fact} and output_tokens > 0 + and latency_ms > first_token_ms)))::bigint as output_throughput_p90, + count(capacity_total_slots)::bigint as capacity_sample_count, + round(avg(capacity_used_slots::double precision + / nullif(capacity_total_slots, 0)) * 10000)::bigint + as capacity_utilization_avg_basis_points, + round(percentile_cont(0.95) within group ( + order by capacity_used_slots::double precision + / nullif(capacity_total_slots, 0) + ) * 10000)::bigint as capacity_utilization_p95_basis_points from model_requests mr where mr.started_at >= " )); query.push_bind(range.start); @@ -145,6 +182,43 @@ pub(crate) async fn request_metric_series( filter (where {fact}) as first_token_p95_ms, percentile_cont(0.99) within group (order by first_token_ms) filter (where {fact}) as first_token_p99_ms, + count(admission_decision_ms)::bigint as admission_decision_count, + percentile_cont(0.50) within group (order by admission_decision_ms) + as admission_decision_p50_ms, + percentile_cont(0.95) within group (order by admission_decision_ms) + as admission_decision_p95_ms, + percentile_cont(0.99) within group (order by admission_decision_ms) + as admission_decision_p99_ms, + count(account_selection_wait_ms)::bigint as account_selection_wait_count, + percentile_cont(0.50) within group (order by account_selection_wait_ms) + as account_selection_wait_p50_ms, + percentile_cont(0.95) within group (order by account_selection_wait_ms) + as account_selection_wait_p95_ms, + percentile_cont(0.99) within group (order by account_selection_wait_ms) + as account_selection_wait_p99_ms, + round((percentile_cont(0.10) within group ( + order by output_tokens::double precision * 1000.0 + / greatest(latency_ms - first_token_ms, 1) + ) filter (where {fact} and output_tokens > 0 + and latency_ms > first_token_ms)))::bigint as output_throughput_p10, + round((percentile_cont(0.50) within group ( + order by output_tokens::double precision * 1000.0 + / greatest(latency_ms - first_token_ms, 1) + ) filter (where {fact} and output_tokens > 0 + and latency_ms > first_token_ms)))::bigint as output_throughput_p50, + round((percentile_cont(0.90) within group ( + order by output_tokens::double precision * 1000.0 + / greatest(latency_ms - first_token_ms, 1) + ) filter (where {fact} and output_tokens > 0 + and latency_ms > first_token_ms)))::bigint as output_throughput_p90, + count(capacity_total_slots)::bigint as capacity_sample_count, + round(avg(capacity_used_slots::double precision + / nullif(capacity_total_slots, 0)) * 10000)::bigint + as capacity_utilization_avg_basis_points, + round(percentile_cont(0.95) within group ( + order by capacity_used_slots::double precision + / nullif(capacity_total_slots, 0) + ) * 10000)::bigint as capacity_utilization_p95_basis_points, count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint as provider_reported_count, count(*) filter (where {fact} and cost_source = 'calculated')::bigint @@ -438,6 +512,30 @@ pub(crate) fn request_metrics_from_row(row: &sqlx::postgres::PgRow) -> StoreResu p95_ms: optional_percentile(row, "first_token_p95_ms")?, p99_ms: optional_percentile(row, "first_token_p99_ms")?, }, + admission_decision_count: unsigned(row, "admission_decision_count")?, + admission_decision_percentiles: LatencyPercentiles { + p50_ms: optional_percentile(row, "admission_decision_p50_ms")?, + p95_ms: optional_percentile(row, "admission_decision_p95_ms")?, + p99_ms: optional_percentile(row, "admission_decision_p99_ms")?, + }, + account_selection_wait_count: unsigned(row, "account_selection_wait_count")?, + account_selection_wait_percentiles: LatencyPercentiles { + p50_ms: optional_percentile(row, "account_selection_wait_p50_ms")?, + p95_ms: optional_percentile(row, "account_selection_wait_p95_ms")?, + p99_ms: optional_percentile(row, "account_selection_wait_p99_ms")?, + }, + output_throughput_p10: optional_unsigned(row, "output_throughput_p10")?, + output_throughput_p50: optional_unsigned(row, "output_throughput_p50")?, + output_throughput_p90: optional_unsigned(row, "output_throughput_p90")?, + capacity_sample_count: unsigned(row, "capacity_sample_count")?, + capacity_utilization_avg_basis_points: optional_unsigned( + row, + "capacity_utilization_avg_basis_points", + )?, + capacity_utilization_p95_basis_points: optional_unsigned( + row, + "capacity_utilization_p95_basis_points", + )?, }) } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs index 607297062..bd695e5ed 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -123,7 +123,9 @@ pub(crate) const USAGE_RECORD_SELECT: &str = mr.total_tokens, mr.cost_source, mr.cost_amount::text, mr.cost_currency, mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms, mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms, - mr.provider_processing_ms, mr.latency_ms, mr.client_ip::text as client_ip, + mr.provider_processing_ms, mr.latency_ms, mr.admission_decision_ms, + mr.account_selection_wait_ms, mr.capacity_used_slots, mr.capacity_total_slots, + mr.client_ip::text as client_ip, mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind, mr.subagent_kind, mr.compact, mr.image_generation_requested, mr.image_generation_succeeded, mr.started_at, mr.deadline_at, mr.completed_at @@ -351,6 +353,11 @@ pub(crate) async fn usage_diagnostics( round(avg(mr.latency_ms))::bigint as average_latency_ms, round(percentile_cont(0.95) within group (order by mr.latency_ms))::bigint as latency_p95_ms, + round(percentile_cont(0.95) within group (order by mr.first_token_ms))::bigint + as first_token_p95_ms, + count(*) filter (where mr.outcome in ('cancelled', 'incomplete'))::bigint + as non_completion_count, + coalesce(sum(greatest(mr.attempt_count - 1, 0)), 0)::bigint as retry_count, count(*) filter (where mr.cost_source = 'provider_reported')::bigint as provider_reported_count, count(*) filter (where mr.cost_source = 'calculated')::bigint @@ -362,7 +369,7 @@ pub(crate) async fn usage_diagnostics( statement.push_bind(range.start); statement.push(" and mr.started_at < "); statement.push_bind(range.end); - push_diagnostic_fact_filter(&mut statement, dimension); + push_diagnostic_dimension_filter(&mut statement, dimension); push_usage_filter(&mut statement, filter, "mr"); statement.push(" group by dimension_name order by request_count desc, dimension_name limit "); statement.push_bind(DIAGNOSTIC_LIMIT); @@ -384,6 +391,9 @@ pub(crate) async fn usage_diagnostics( total_tokens: unsigned(row, "total_tokens")?, average_latency_ms: optional_unsigned(row, "average_latency_ms")?, latency_p95_ms: optional_unsigned(row, "latency_p95_ms")?, + first_token_p95_ms: optional_unsigned(row, "first_token_p95_ms")?, + non_completion_count: unsigned(row, "non_completion_count")?, + retry_count: unsigned(row, "retry_count")?, cost_coverage: coverage_from_row(row)?, costs: Vec::new(), }) @@ -501,7 +511,8 @@ pub(crate) async fn diagnostic_costs( statement.push(" and mr.started_at < "); statement.push_bind(range.end); statement.push(" and mr.cost_amount is not null and mr.cost_currency is not null"); - push_diagnostic_fact_filter(&mut statement, dimension); + push_completed_usage_fact_filter(&mut statement, "mr"); + push_diagnostic_dimension_filter(&mut statement, dimension); push_usage_filter(&mut statement, filter, "mr"); statement.push( " group by dimension_name, mr.cost_currency order by dimension_name, mr.cost_currency", @@ -539,13 +550,22 @@ pub(crate) fn diagnostic_dimension_sql(dimension: DiagnosticDimension) -> &'stat } } -pub(crate) fn push_diagnostic_fact_filter( +pub(crate) fn push_diagnostic_dimension_filter( statement: &mut QueryBuilder, dimension: DiagnosticDimension, ) { - if dimension == DiagnosticDimension::Failure { - statement.push(" and mr.error_kind is not null"); - } else { - push_completed_usage_fact_filter(statement, "mr"); + match dimension { + DiagnosticDimension::Failure => { + statement.push(" and mr.error_kind is not null"); + } + DiagnosticDimension::Status => { + statement + .push(" and coalesce(mr.upstream_status_code, mr.client_status_code) is not null"); + } + DiagnosticDimension::Provider + | DiagnosticDimension::Model + | DiagnosticDimension::Account + | DiagnosticDimension::ApiKey + | DiagnosticDimension::Transport => {} } } diff --git a/backend/crates/gateway-store/tests/postgres/execution.rs b/backend/crates/gateway-store/tests/postgres/execution.rs index d9f7c0ef8..c9242dab9 100644 --- a/backend/crates/gateway-store/tests/postgres/execution.rs +++ b/backend/crates/gateway-store/tests/postgres/execution.rs @@ -20,6 +20,7 @@ fn postgres_execution_adapter_implements_core_port() { fn model_request_rejects_mismatched_client_key_live_id() { let started_at = Utc::now(); let request = NewModelRequest { + admission_decision_ms: None, id: "request-1".to_owned(), client_api_key_id: Some("key-live".to_owned()), client_api_key_ref: "key-history".to_owned(), @@ -70,6 +71,7 @@ async fn merged_model_less_first_attempt_should_match_sequential_semantics() { .await .expect("seed provider account"); let request = NewModelRequest { + admission_decision_ms: None, id: "req_merged".to_owned(), client_api_key_id: None, client_api_key_ref: "key_merged".to_owned(), @@ -94,6 +96,9 @@ async fn merged_model_less_first_attempt_should_match_sequential_semantics() { deadline_at: started_at + Duration::seconds(30), }; let attempt = ModelRequestAttemptStart { + account_selection_wait_ms: None, + capacity_used_slots: None, + capacity_total_slots: None, model_request_id: "req_merged".to_owned(), attempt_count: 1, provider_kind: "openai".to_owned(), @@ -146,6 +151,9 @@ async fn merged_model_less_first_attempt_should_match_sequential_semantics() { .expect("mark sent before retry"); let second = repository .begin_model_request_attempt(ModelRequestAttemptStart { + account_selection_wait_ms: None, + capacity_used_slots: None, + capacity_total_slots: None, model_request_id: "req_merged".to_owned(), attempt_count: 2, provider_kind: "openai".to_owned(), @@ -177,6 +185,7 @@ async fn model_request_persists_group_routing_snapshot_without_live_group_foreig let started_at = Utc::now(); repository .insert_model_request(NewModelRequest { + admission_decision_ms: None, id: "req_group_history".to_owned(), client_api_key_id: None, client_api_key_ref: "key_group_history".to_owned(), diff --git a/backend/crates/gateway-store/tests/postgres/observability.rs b/backend/crates/gateway-store/tests/postgres/observability.rs index c45d7c1a9..ab7e03326 100644 --- a/backend/crates/gateway-store/tests/postgres/observability.rs +++ b/backend/crates/gateway-store/tests/postgres/observability.rs @@ -983,7 +983,10 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact .expect("usage diagnostics"); assert_eq!(diagnostics[0].key, "acct_observe"); assert_eq!(diagnostics[0].name, "account@example.invalid"); - assert_eq!(diagnostics[0].request_count, 1); + assert_eq!(diagnostics[0].request_count, 3); + assert_eq!(diagnostics[0].success_count, 2); + assert_eq!(diagnostics[0].failure_count, 1); + assert_eq!(diagnostics[0].retry_count, 1); assert_eq!(diagnostics[0].costs[0].amount.as_str(), "1.25"); let errors = repository diff --git a/backend/crates/gateway-store/tests/postgres/snapshots.rs b/backend/crates/gateway-store/tests/postgres/snapshots.rs index 993dbde86..346441d71 100644 --- a/backend/crates/gateway-store/tests/postgres/snapshots.rs +++ b/backend/crates/gateway-store/tests/postgres/snapshots.rs @@ -431,6 +431,9 @@ async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() { .insert_model_request_with_first_attempt( new_request("req_snap_legacy", started_at + chrono::Duration::seconds(1)), ModelRequestAttemptStart { + account_selection_wait_ms: None, + capacity_used_slots: None, + capacity_total_slots: None, model_request_id: "req_snap_legacy".to_owned(), attempt_count: 1, provider_kind: "openai".to_owned(), @@ -542,6 +545,9 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() { request.client_api_key_id = Some("key_diag".to_owned()); request.client_api_key_ref = "key_diag".to_owned(); let attempt = ModelRequestAttemptStart { + account_selection_wait_ms: None, + capacity_used_slots: None, + capacity_total_slots: None, model_request_id: id.to_owned(), attempt_count: 1, provider_kind: "openai".to_owned(), @@ -656,6 +662,7 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() { fn new_request(id: &str, started_at: DateTime) -> NewModelRequest { NewModelRequest { + admission_decision_ms: None, id: id.to_owned(), client_api_key_id: None, client_api_key_ref: "key_snapshot".to_owned(), @@ -683,6 +690,9 @@ fn new_request(id: &str, started_at: DateTime) -> NewModelRequest { fn attempt(id: &str, count: u32, account_id: &str) -> ModelRequestAttemptStart { ModelRequestAttemptStart { + account_selection_wait_ms: None, + capacity_used_slots: None, + capacity_total_slots: None, model_request_id: id.to_owned(), attempt_count: count, provider_kind: "openai".to_owned(), diff --git a/backend/crates/providers/openai/src/credential/selector.rs b/backend/crates/providers/openai/src/credential/selector.rs index 6ffea2308..a84af2eff 100644 --- a/backend/crates/providers/openai/src/credential/selector.rs +++ b/backend/crates/providers/openai/src/credential/selector.rs @@ -6,10 +6,10 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use gateway_core::engine::credential::{ - AccountCandidate, AccountEligibilityPolicy, AccountErrorReason, AccountFeedbackStats, - AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, AccountSelector, - AccountStatus, CredentialState, PreferredAccountSelection, ProviderAccount, ProviderAccountId, - QuotaEvidence, + AccountCandidate, AccountCapacitySnapshot, AccountEligibilityPolicy, AccountErrorReason, + AccountFeedbackStats, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, + AccountSelector, AccountStatus, CredentialState, PreferredAccountSelection, ProviderAccount, + ProviderAccountId, QuotaEvidence, }; use gateway_core::engine::{AttemptContext, ContinuationAttempt}; use gateway_core::provider_ports::{ @@ -498,6 +498,7 @@ impl CodexCredentialSelector { }, account_scope: request.attempt.account_scope().cloned(), }; + let capacity = AccountSelector.capacity_snapshot(&candidates, &context); let Some(selection) = AccountSelector.select(&candidates, &context) else { return match shortest_retry { Some(retry_after) => Err(CredentialSelectionError::CapacityUnavailable { @@ -619,6 +620,7 @@ impl CodexCredentialSelector { allows_account_state_mutation, affinity_telemetry, affinity_expected_account_id, + capacity: capacity.map(AccountCapacitySnapshot::with_acquired_request), _guard: guard, }); } @@ -1224,6 +1226,7 @@ pub struct CodexCredentialLease { allows_account_state_mutation: bool, affinity_telemetry: AffinityTelemetry, affinity_expected_account_id: ProviderAccountId, + capacity: Option, _guard: Box, } @@ -1282,6 +1285,11 @@ impl CodexCredentialLease { self.affinity_telemetry.account_switch } + #[must_use] + pub const fn capacity_snapshot(&self) -> Option { + self.capacity + } + /// 成功反馈只能从选择时观察到的绑定迁移，避免迟到请求覆盖较新 winner。 #[must_use] pub(crate) const fn affinity_expected_account_id(&self) -> &ProviderAccountId { diff --git a/backend/crates/providers/openai/src/provider.rs b/backend/crates/providers/openai/src/provider.rs index f48633102..ece8ce8b9 100644 --- a/backend/crates/providers/openai/src/provider.rs +++ b/backend/crates/providers/openai/src/provider.rs @@ -12,8 +12,8 @@ use gateway_core::engine::continuation::{ContinuationBinding, NativeContinuation use gateway_core::engine::credential::{AccountFeedbackStats, ProviderAccount}; use gateway_core::engine::provider::{ EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration, - ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation, ProviderStream, - UpstreamTransport, + ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation, + ProviderSelectionObservation, ProviderStream, UpstreamTransport, }; use gateway_core::engine::{ AttemptContext, CancellationToken, ContinuationAttempt, UpstreamSendState, @@ -292,6 +292,7 @@ impl Provider for CodexProvider { let transport = selected_transport(&upstream_request); apply_transport(&mut upstream_request, transport); + let selection_started_at = Instant::now(); let lease = self .selector .select_with_cyber_policy( @@ -306,6 +307,8 @@ impl Provider for CodexProvider { ) .await .map_err(map_selection_error)?; + let account_selection_wait_ms = + u64::try_from(selection_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); let lease = Arc::new(lease); // 首字计时的起点：账号选择完成之后、上游建立之前。 let output_started_at = Instant::now(); @@ -390,7 +393,11 @@ impl Provider for CodexProvider { UpstreamTransport::new(transport_name(transport)).map_err(|_| { provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent) })?, - ); + ) + .with_selection_observation(ProviderSelectionObservation::new( + account_selection_wait_ms, + lease.capacity_snapshot(), + )); let response_store = upstream_request.store(); let session_capture = (!continuation_requested || previous_session.is_some()).then(|| OpenAiSessionCapture { @@ -447,6 +454,7 @@ impl CodexProvider { ), ImageRequestKind::Edit => (CODEX_IMAGE_EDITS_PATH, self.image_edits_url.clone()), }; + let selection_started_at = Instant::now(); let lease = self .selector .select_for_provider_endpoint(&SelectCodexProviderEndpointCredential { @@ -455,6 +463,8 @@ impl CodexProvider { }) .await .map_err(map_selection_error)?; + let account_selection_wait_ms = + u64::try_from(selection_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); let lease = Arc::new(lease); let allows_account_state_mutation = lease.allows_account_state_mutation(); let provider_kind = ProviderKind::new(PROVIDER_NAME) @@ -465,7 +475,11 @@ impl CodexProvider { UpstreamTransport::new(HTTP_JSON_TRANSPORT).map_err(|_| { provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent) })?, - ); + ) + .with_selection_observation(ProviderSelectionObservation::new( + account_selection_wait_ms, + lease.capacity_snapshot(), + )); let image_turn_id = image .payload() .context() diff --git a/backend/crates/providers/xai/src/credential/selector.rs b/backend/crates/providers/xai/src/credential/selector.rs index 3ce178324..4385ec554 100644 --- a/backend/crates/providers/xai/src/credential/selector.rs +++ b/backend/crates/providers/xai/src/credential/selector.rs @@ -6,9 +6,9 @@ use std::time::{Duration, SystemTime}; use chrono::Utc; use gateway_core::engine::credential::{ - AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats, AccountSelectionContext, - AccountSelector, CredentialState, ProviderAccount, ProviderAccountId, QuotaAccessState, - QuotaEvidence, QuotaState, + AccountCandidate, AccountCapacitySnapshot, AccountEligibilityPolicy, AccountFeedbackStats, + AccountSelectionContext, AccountSelector, CredentialState, ProviderAccount, ProviderAccountId, + QuotaAccessState, QuotaEvidence, QuotaState, }; use gateway_core::provider_ports::{ ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition, @@ -200,7 +200,11 @@ impl GrokAccountSessionSelector { }; let mut capacity_denied = false; let mut retry_after = None; - while let Some(selection) = AccountSelector.select(&candidates, &context) { + loop { + let capacity = AccountSelector.capacity_snapshot(&candidates, &context); + let Some(selection) = AccountSelector.select(&candidates, &context) else { + break; + }; let selected = selection.candidate(); let selected_id = selected.account.id().clone(); let selected_revision = selected.account.revision(); @@ -265,7 +269,8 @@ impl GrokAccountSessionSelector { binding, guard, ) - .map_err(|_| GrokSessionSelectorError::InvalidSession)?; + .map_err(|_| GrokSessionSelectorError::InvalidSession)? + .with_capacity_snapshot(capacity.map(AccountCapacitySnapshot::with_acquired_request)); return Ok(if allows_account_state_mutation { session } else { diff --git a/backend/crates/providers/xai/src/provider.rs b/backend/crates/providers/xai/src/provider.rs index 0387cafd0..0a64c3db0 100644 --- a/backend/crates/providers/xai/src/provider.rs +++ b/backend/crates/providers/xai/src/provider.rs @@ -13,8 +13,8 @@ use gateway_core::engine::credential::{ }; use gateway_core::engine::provider::{ EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration, - ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation, ProviderStream, - UpstreamTransport, + ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation, + ProviderSelectionObservation, ProviderStream, UpstreamTransport, }; use gateway_core::engine::{AttemptContext, ContinuationAttempt, UpstreamSendState}; use gateway_core::error::{ @@ -215,6 +215,7 @@ impl GrokBuildProvider { if let Some(previous) = previous_session.as_ref() { upstream_request.inherit_session(previous.session_id.as_deref()); } + let selection_started_at = Instant::now(); let selected = select_grok_session( self.selector.as_ref(), candidate, @@ -224,6 +225,8 @@ impl GrokBuildProvider { upstream_request.affinity().cloned(), ) .await?; + let account_selection_wait_ms = + u64::try_from(selection_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); // 首字计时的起点：账号选择完成之后、上游建立之前。 let output_started_at = Instant::now(); apply_continuation( @@ -278,7 +281,7 @@ impl GrokBuildProvider { }); let selected = Arc::new(selected); let allows_account_state_mutation = selected.allows_account_state_mutation(); - let metadata = provider_call_metadata(candidate, &selected)?; + let metadata = provider_call_metadata(candidate, &selected, account_selection_wait_ms)?; let events = cold_http_sse_stream( Arc::clone(&self.selector), Arc::clone(&self.transport), @@ -347,6 +350,7 @@ impl GrokBuildProvider { let upstream_session_id = inherited_session_id .clone() .or_else(|| explicit_replay_session_id.clone()); + let selection_started_at = Instant::now(); let selected = Arc::new( select_grok_session( self.selector.as_ref(), @@ -358,6 +362,8 @@ impl GrokBuildProvider { ) .await?, ); + let account_selection_wait_ms = + u64::try_from(selection_started_at.elapsed().as_millis()).unwrap_or(u64::MAX); let reasoning_replay_key = explicit_replay_session_id .as_deref() .or(inherited_session_id.as_deref()) @@ -369,7 +375,7 @@ impl GrokBuildProvider { ) }); let allows_account_state_mutation = selected.allows_account_state_mutation(); - let metadata = provider_call_metadata(candidate, &selected)?; + let metadata = provider_call_metadata(candidate, &selected, account_selection_wait_ms)?; let events = cold_compaction_http_sse_stream( Arc::clone(&self.selector), Arc::clone(&self.transport), @@ -451,13 +457,18 @@ async fn select_grok_session( fn provider_call_metadata( candidate: &ProviderCandidate, selected: &SelectedGrokSession, + account_selection_wait_ms: u64, ) -> Result { Ok(ProviderCallMetadata::new( ProviderKind::new(XAI_PROVIDER_NAME).map_err(|_| protocol_not_sent())?, candidate_upstream_model(candidate)?.clone(), selected.account_id().clone(), UpstreamTransport::new(HTTP_SSE_TRANSPORT).map_err(|_| protocol_not_sent())?, - )) + ) + .with_selection_observation(ProviderSelectionObservation::new( + account_selection_wait_ms, + selected.capacity_snapshot(), + ))) } fn candidate_upstream_model( diff --git a/backend/crates/providers/xai/src/transport/session.rs b/backend/crates/providers/xai/src/transport/session.rs index bd19805d8..f659d9cc0 100644 --- a/backend/crates/providers/xai/src/transport/session.rs +++ b/backend/crates/providers/xai/src/transport/session.rs @@ -6,7 +6,8 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use gateway_core::engine::credential::{ - AccountEligibilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId, + AccountCapacitySnapshot, AccountEligibilityPolicy, AccountSelectionPolicy, CredentialRevision, + ProviderAccountId, }; use gateway_core::policy::ClientApiKeyId; use gateway_core::routing::{FrozenAccountScope, UpstreamModelId}; @@ -89,6 +90,7 @@ pub struct SelectedGrokSession { email: Option, binding: GrokSessionBinding, allows_account_state_mutation: bool, + capacity: Option, _guard: Box, } @@ -123,6 +125,7 @@ impl SelectedGrokSession { email, binding, allows_account_state_mutation: true, + capacity: None, _guard: Box::new(guard), }) } @@ -134,6 +137,15 @@ impl SelectedGrokSession { self } + #[must_use] + pub(crate) const fn with_capacity_snapshot( + mut self, + capacity: Option, + ) -> Self { + self.capacity = capacity; + self + } + /// 返回选中的账号 ID。 #[must_use] pub const fn account_id(&self) -> &ProviderAccountId { @@ -174,6 +186,11 @@ impl SelectedGrokSession { pub(crate) const fn allows_account_state_mutation(&self) -> bool { self.allows_account_state_mutation } + + #[must_use] + pub(crate) const fn capacity_snapshot(&self) -> Option { + self.capacity + } } impl fmt::Debug for SelectedGrokSession { diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index a8d9121e4..f5c6e7efc 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -1 +1,2 @@ 60bcece284f2d6a2cd3e8dea67833dfc1c7da625cdd5f0b63884c725fa00a3d4 0001_initial.sql +93219760c733616e7bba9a8933c207b26752b2ebf7e6edfe1ef42ba227c50a7d 0002_runtime_pressure_observability.sql diff --git a/backend/migrations/0002_runtime_pressure_observability.sql b/backend/migrations/0002_runtime_pressure_observability.sql new file mode 100644 index 000000000..cf3392737 --- /dev/null +++ b/backend/migrations/0002_runtime_pressure_observability.sql @@ -0,0 +1,20 @@ +-- 请求热路径的准入判定、账号选择等待与账号池容量观测。 +alter table model_requests + add column admission_decision_ms bigint, + add column account_selection_wait_ms bigint, + add column capacity_used_slots bigint, + add column capacity_total_slots bigint; + +alter table model_requests + add constraint model_requests_runtime_pressure_ck check ( + (admission_decision_ms is null or admission_decision_ms >= 0) + and (account_selection_wait_ms is null or account_selection_wait_ms >= 0) + and ( + (capacity_used_slots is null and capacity_total_slots is null) + or ( + capacity_used_slots >= 0 + and capacity_total_slots > 0 + and capacity_used_slots <= capacity_total_slots + ) + ) + ); diff --git a/backend/migrations/README.md b/backend/migrations/README.md index dc6c41f2d..c55e6ab40 100644 --- a/backend/migrations/README.md +++ b/backend/migrations/README.md @@ -4,14 +4,12 @@ sqlx 在服务启动时（`serve` 监听之前）按编号顺序执行本目录 文件的 checksum 记入 `_sqlx_migrations`。此后每次启动都会重新校验：**已应用 迁移的文件字节与记录不一致时，服务直接拒绝启动**。 -当前大版本只有一份冻结迁移：`0001_initial.sql` 作为完整的新库基线。 - ## 冻结规则 - 同一大版本内，已合入 main 的迁移文件**永久冻结**，一个字节都不能改——包括注释和 空白。任何改动都会让该大版本的已部署实例在下次重启时因 checksum 失配而无法启动， 且没有自动恢复路径。 -- 同一大版本内的 schema 变更一律新增编号迁移（下一份从 `0003_...` 开始），永远不回改 +- 同一大版本内的 schema 变更一律新增编号迁移，永远不回改 旧文件。切换大版本时允许把已完成升级的 schema 折叠成新的 `0001` 基线，因为跨大版本 不支持在线升级，目标版本必须使用全新数据目录部署。 - `.frozen-sha256` 是冻结清单（`sha256sum` 格式）。CI 会校验：清单内文件 diff --git a/docs/architecture.md b/docs/architecture.md index b3f1656a5..5920f3d76 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -195,9 +195,8 @@ PostgreSQL 周期对账才是正确性基础。 `normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`。只有明确上游证据才能恢复或终态化账号， 本地时钟和不确定响应不能伪造事实。 -当前大版本的十二张业务表由 `0001_initial.sql` 建立；`0002_nullable_requested_model.sql` 允许 Images -等无模型端点把 `requested_model_id` 保存为 `NULL`。已应用迁移按字节冻结，后续 schema 变化只能新增 -编号迁移，详见 [迁移规则](../backend/migrations/README.md)。 +PostgreSQL schema 由迁移目录按编号管理。已应用迁移按字节冻结，后续 schema 变化只能新增编号迁移， +详见 [迁移规则](../backend/migrations/README.md)。 ## 9. Credential、额度与主动重置 diff --git a/frontend/src/api/modules/dashboard.ts b/frontend/src/api/modules/dashboard.ts index dc60e077b..7d9464617 100644 --- a/frontend/src/api/modules/dashboard.ts +++ b/frontend/src/api/modules/dashboard.ts @@ -30,6 +30,13 @@ export interface DashboardTrendPoint { minLatencyValue: number | null successRate: string successRateValue: number | null + firstTokenP50Ms: number | null + firstTokenP95Ms: number | null + latencyP95Ms: number | null + outputThroughputP50: number | null + admissionDecisionP95Ms: number | null + accountSelectionWaitP95Ms: number | null + capacityUtilization: number | null } export interface DashboardTrendSummaryItem { diff --git a/frontend/src/api/modules/usage.ts b/frontend/src/api/modules/usage.ts index 1e2430e09..9f7ede515 100644 --- a/frontend/src/api/modules/usage.ts +++ b/frontend/src/api/modules/usage.ts @@ -54,6 +54,10 @@ export interface UsageBilling { } export interface UsageLatencyDetails { + admissionDecisionMs?: number + accountSelectionWaitMs?: number + capacityUsedSlots?: number + capacityTotalSlots?: number transportDecisionWaitMs?: number wsConnectMs?: number upstreamHeadersMs?: number @@ -221,6 +225,7 @@ export interface UsageSummaryResponse { export interface UsageOverviewHealthPoint { bucket: string label: string + totalRequests: number successRequests: number failedRequests: number cancelledRequests: number @@ -237,6 +242,7 @@ export interface UsageOverviewHealth { incompleteRequests: number callerErrorRequests: number successRate: number + completionRate: number requestChangeRate: number | null successRateChange: number | null points: UsageOverviewHealthPoint[] @@ -251,6 +257,15 @@ export interface UsageOverviewPerformancePoint { firstTokenP50Ms: number | null firstTokenP95Ms: number | null firstTokenP99Ms: number | null + admissionDecisionP50Ms: number | null + admissionDecisionP95Ms: number | null + accountSelectionWaitP50Ms: number | null + accountSelectionWaitP95Ms: number | null + outputThroughputP10: number | null + outputThroughputP50: number | null + outputThroughputP90: number | null + capacityUtilization: number | null + capacityUtilizationP95: number | null } export interface UsageOverviewPerformance { @@ -260,8 +275,20 @@ export interface UsageOverviewPerformance { firstTokenP50Ms: number | null firstTokenP95Ms: number | null firstTokenP99Ms: number | null + admissionDecisionP50Ms: number | null + admissionDecisionP95Ms: number | null + accountSelectionWaitP50Ms: number | null + accountSelectionWaitP95Ms: number | null + outputThroughputP10: number | null + outputThroughputP50: number | null + outputThroughputP90: number | null + capacityUtilization: number | null + capacityUtilizationP95: number | null latencyCoverage: number firstTokenCoverage: number + admissionDecisionCoverage: number + accountSelectionWaitCoverage: number + capacityCoverage: number points: UsageOverviewPerformancePoint[] } @@ -274,6 +301,8 @@ export interface UsageOverviewCostPoint { totalTokens: number estimatedCost: string | null standardCost: string | null + noCacheCost: string | null + cacheSavings: string | null cachedTokenRate: number cacheHitRequestRate: number | null } @@ -281,7 +310,11 @@ export interface UsageOverviewCostPoint { export interface UsageOverviewCost { estimatedCost: string | null standardCost: string | null + noCacheCost: string | null + cacheSavings: string | null + tierPremium: string | null costPerRequest: string | null + costPerSuccessfulRequest: string | null tokensPerRequest: number cachedTokenRate: number cacheHitRequestRate: number | null @@ -290,6 +323,7 @@ export interface UsageOverviewCost { cachedTokens: number totalTokens: number points: UsageOverviewCostPoint[] + coverage: UsageCostCoverage } export interface UsageInsightsOverviewResponse { @@ -309,6 +343,12 @@ export interface UsageDiagnosticItem { requestShare: number averageLatencyMs: number | null latencyP95Ms: number | null + firstTokenP95Ms: number | null + nonCompletionCount: number + nonCompletionRate: number + retryCount: number + retryRate: number + impactScore: number estimatedCost: string | null attemptCount: number totalTokens: number diff --git a/frontend/src/views/dashboard/components/DashboardHeartbeat.vue b/frontend/src/views/dashboard/components/DashboardHeartbeat.vue index 5a273dd8a..92b6bcff1 100644 --- a/frontend/src/views/dashboard/components/DashboardHeartbeat.vue +++ b/frontend/src/views/dashboard/components/DashboardHeartbeat.vue @@ -67,13 +67,10 @@ onBeforeUnmount(() => { diff --git a/frontend/src/views/dashboard/components/HealthTimelinePointPopover.vue b/frontend/src/views/dashboard/components/HealthTimelinePointPopover.vue index d35a59556..5745da82e 100644 --- a/frontend/src/views/dashboard/components/HealthTimelinePointPopover.vue +++ b/frontend/src/views/dashboard/components/HealthTimelinePointPopover.vue @@ -2,12 +2,7 @@ import type { HealthTimelinePoint } from '../constants' import { computed } from 'vue' -import { - formatHealthCount, - healthReliabilityValueClass, - healthStatusMeta, - -} from '../constants' +import { formatHealthCount, healthReliabilityValueClass, healthStatusMeta } from '../constants' const props = defineProps<{ point: HealthTimelinePoint @@ -80,9 +75,7 @@ const metricItems = outcomeMeta {{ status.label }} - - 15 分钟 - + 15 分钟
@@ -99,21 +92,14 @@ const metricItems = outcomeMeta
- - 有效请求 - - + 有效请求 + {{ formatHealthCount(eligibleRequests) }}
- -

+

客户端取消与调用方错误单独记录，不计入可用性。

diff --git a/frontend/src/views/dashboard/components/MetricCard.vue b/frontend/src/views/dashboard/components/MetricCard.vue index dfd21cfdb..e4b73851a 100644 --- a/frontend/src/views/dashboard/components/MetricCard.vue +++ b/frontend/src/views/dashboard/components/MetricCard.vue @@ -75,7 +75,6 @@ const sparklineOption = computed(() => {
- - + + (() => {
- - {{ metric.details[0]?.label }} + + + {{ metric.details[0]?.label }} + (() => { {{ metric.details[0]?.value }} - - {{ metric.details[1]?.label }} + + + {{ metric.details[1]?.label }} + () let wavedCellIndexes = new Set() function observedRequests(point: HealthTimelinePoint) { - return ( - point.successRequests - + point.failedRequests - + point.cancelledRequests - + point.callerErrorRequests - ) + return point.successRequests + point.failedRequests + point.cancelledRequests + point.callerErrorRequests } function isInteractivePoint(point: HealthTimelinePoint) { @@ -79,9 +68,7 @@ function resetPointInteraction() { } function timelineButtons() { - return Array.from( - timelineGrid.value?.querySelectorAll('[data-health-timeline-point]') ?? [], - ) + return Array.from(timelineGrid.value?.querySelectorAll('[data-health-timeline-point]') ?? []) } function timelineCell(button?: HTMLButtonElement) { @@ -99,20 +86,13 @@ function animatePointWave(centerIndex: number) { const cellsByDistance: HTMLElement[][] = [[], [], []] for (let distance = 0; distance <= 2; distance += 1) { - const candidateIndexes - = distance === 0 ? [centerIndex] : [centerIndex - distance, centerIndex + distance] + const candidateIndexes = distance === 0 ? [centerIndex] : [centerIndex - distance, centerIndex + distance] for (const index of candidateIndexes) { const button = buttons[index] const point = points.value[index] const cell = timelineCell(button) - if ( - !button - || !point - || !cell - || button.offsetTop !== centerRow - || !isInteractivePoint(point) - ) { + if (!button || !point || !cell || button.offsetTop !== centerRow || !isInteractivePoint(point)) { continue } @@ -140,13 +120,7 @@ function animatePointWave(centerIndex: number) { wavedCellIndexes = nextCellIndexes } -function animateWaveCells( - cells: HTMLElement[], - y: number, - scaleY: number, - duration: number, - ease: string, -) { +function animateWaveCells(cells: HTMLElement[], y: number, scaleY: number, duration: number, ease: string) { if (cells.length === 0) return @@ -210,12 +184,7 @@ onBeforeUnmount(() => {