mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-20 08:01:46 +00:00
Rank quota scheduling strictly by configured reset windows and persist its settings independently from smart scheduling. Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
1919 lines
66 KiB
Rust
1919 lines
66 KiB
Rust
//! 验证管理观测查询的时间范围、健康投影与运行态聚合
|
|
|
|
use std::{
|
|
str::FromStr as _,
|
|
sync::{
|
|
Arc, Mutex,
|
|
atomic::{AtomicBool, AtomicUsize, Ordering},
|
|
},
|
|
time::Duration as StdDuration,
|
|
};
|
|
|
|
use async_trait::async_trait;
|
|
use chrono::{DateTime, Duration, Timelike as _, Utc};
|
|
|
|
use gateway_admin::{
|
|
AdminServices,
|
|
model::{
|
|
MutationContext, Revision,
|
|
observability::{
|
|
AccountPoolMetrics, AnalyticsState, AttemptMetrics, CostCoverage, CurrencyCost,
|
|
DashboardAccountUsage, DashboardObservation, DashboardQuery, DashboardRuntimeSlots,
|
|
DiagnosticDimension, DiagnosticObservation, DiagnosticsObservation, Granularity,
|
|
HealthStatus, LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
|
PercentileMilliseconds, RequestMetricPoint, RequestMetrics, TimeRange, TrendKind,
|
|
UsageBilling, UsageConcurrencyPoint, UsageDetail, UsageFilter,
|
|
UsageInsightsObservation, UsageListRecord, UsageOverview, UsagePage,
|
|
UsageProjectedCostPoint, UsageQuery, UsageSummaryObservation,
|
|
},
|
|
settings::{
|
|
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
|
RuntimeSettings,
|
|
},
|
|
},
|
|
ports::store::{AdminStoreResult, ObservabilityStore, SettingsStore},
|
|
};
|
|
|
|
mod errors;
|
|
|
|
#[test]
|
|
fn external_observability_range_accepts_exactly_366_days() {
|
|
let end = Utc::now();
|
|
let range = TimeRange::new(end - Duration::days(366), end)
|
|
.expect("366-day external range should be accepted");
|
|
assert_eq!(range.end, end);
|
|
}
|
|
|
|
#[test]
|
|
fn external_observability_range_rejects_over_366_days_and_reversed_range() {
|
|
let end = Utc::now();
|
|
assert!(TimeRange::new(end - Duration::days(367), end).is_err());
|
|
assert!(TimeRange::new(end, end).is_err());
|
|
assert!(TimeRange::new(end + Duration::seconds(1), end).is_err());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn health_timeline_should_keep_exactly_china_day_quarter_hour_slots() {
|
|
let now = Utc::now();
|
|
let day_start = gateway_core::time::DeploymentTimeZone::default()
|
|
.day_start(now)
|
|
.unwrap();
|
|
let current_slot = quarter_hour_start(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
store.replace_trend(vec![
|
|
health_metric_point(
|
|
day_start - Duration::minutes(15),
|
|
RequestMetrics {
|
|
success_count: 100,
|
|
..RequestMetrics::default()
|
|
},
|
|
),
|
|
health_metric_point(
|
|
current_slot,
|
|
RequestMetrics {
|
|
success_count: 2,
|
|
..RequestMetrics::default()
|
|
},
|
|
),
|
|
health_metric_point(
|
|
current_slot + Duration::minutes(15),
|
|
RequestMetrics {
|
|
success_count: 100,
|
|
..RequestMetrics::default()
|
|
},
|
|
),
|
|
]);
|
|
let services = observability_services(store).await;
|
|
|
|
let timeline = services
|
|
.observability()
|
|
.dashboard_summary(observation_range(now), TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard summary")
|
|
.health_timeline;
|
|
|
|
assert_eq!(timeline.points.len(), 96);
|
|
assert_eq!(
|
|
timeline.points.first().map(|point| point.bucket_start),
|
|
Some(day_start)
|
|
);
|
|
assert_eq!(
|
|
timeline.points.last().map(|point| point.bucket_start),
|
|
Some(day_start + Duration::minutes(95 * 15))
|
|
);
|
|
assert_eq!(timeline.success_requests, 2);
|
|
assert_eq!(
|
|
timeline
|
|
.points
|
|
.iter()
|
|
.find(|point| point.bucket_start == current_slot)
|
|
.map(|point| point.status),
|
|
Some(HealthStatus::LowSample)
|
|
);
|
|
assert!(timeline.points.iter().all(|point| {
|
|
point.bucket_start <= quarter_hour_start(Utc::now())
|
|
|| (point.status == HealthStatus::Future && point.success_requests == 0)
|
|
}));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_derive_first_token_average_from_trend() {
|
|
let now = Utc::now();
|
|
let current_slot = quarter_hour_start(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
store.replace_trend(vec![
|
|
health_metric_point(
|
|
current_slot - Duration::minutes(15),
|
|
RequestMetrics {
|
|
first_token_latency_sum_ms: 120,
|
|
first_token_latency_count: 2,
|
|
..RequestMetrics::default()
|
|
},
|
|
),
|
|
health_metric_point(
|
|
current_slot,
|
|
RequestMetrics {
|
|
first_token_latency_sum_ms: 180,
|
|
first_token_latency_count: 1,
|
|
..RequestMetrics::default()
|
|
},
|
|
),
|
|
]);
|
|
let services = observability_services(store).await;
|
|
|
|
let summary = services
|
|
.observability()
|
|
.dashboard_summary(observation_range(now), TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard summary");
|
|
|
|
assert_eq!(summary.average_first_token_latency_ms, Some(100));
|
|
}
|
|
|
|
#[test]
|
|
fn dashboard_comparison_should_follow_deployment_calendar_across_dst() {
|
|
let timezone = "America/New_York".parse().unwrap();
|
|
for (end, previous_start, previous_end, hours) in [
|
|
(
|
|
"2026-03-09T16:00:00Z",
|
|
"2026-03-08T05:00:00Z",
|
|
"2026-03-09T04:00:00Z",
|
|
23,
|
|
),
|
|
(
|
|
"2026-11-02T17:00:00Z",
|
|
"2026-11-01T04:00:00Z",
|
|
"2026-11-02T05:00:00Z",
|
|
25,
|
|
),
|
|
(
|
|
"2026-11-02T05:00:00Z",
|
|
"2026-11-01T04:00:00Z",
|
|
"2026-11-02T05:00:00Z",
|
|
25,
|
|
),
|
|
] {
|
|
let end = end.parse().unwrap();
|
|
let range = TimeRange::calendar_at(
|
|
gateway_admin::model::observability::CalendarPeriod::Today,
|
|
end,
|
|
timezone,
|
|
)
|
|
.unwrap();
|
|
let query = DashboardQuery::new(range, timezone).unwrap();
|
|
assert_eq!(query.range, range);
|
|
assert_eq!(
|
|
query.yesterday_range.start,
|
|
previous_start.parse::<DateTime<Utc>>().unwrap()
|
|
);
|
|
assert_eq!(
|
|
query.yesterday_range.end,
|
|
previous_end.parse::<DateTime<Utc>>().unwrap()
|
|
);
|
|
assert_eq!(
|
|
(query.yesterday_range.end - query.yesterday_range.start).num_hours(),
|
|
hours
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_compare_separate_yesterday_metrics_with_today_only_trend() {
|
|
let end: DateTime<Utc> = "2026-10-18T04:00:00Z".parse().unwrap();
|
|
let range = TimeRange::calendar_at(
|
|
gateway_admin::model::observability::CalendarPeriod::Today,
|
|
end,
|
|
Default::default(),
|
|
)
|
|
.unwrap();
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.replace_trend(vec![health_metric_point(
|
|
range.start,
|
|
RequestMetrics {
|
|
request_count: 2,
|
|
input_tokens: 100,
|
|
cached_input_tokens: 20,
|
|
total_tokens: 120,
|
|
..Default::default()
|
|
},
|
|
)]);
|
|
*store.yesterday.lock().unwrap() = RequestMetrics {
|
|
request_count: 4,
|
|
input_tokens: 400,
|
|
cached_input_tokens: 200,
|
|
total_tokens: 500,
|
|
..Default::default()
|
|
};
|
|
let result = observability_services(store.clone())
|
|
.await
|
|
.observability()
|
|
.dashboard_summary(range, TrendKind::Usage)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(result.today.request_count, 2);
|
|
assert_eq!(result.today.total_tokens, 120);
|
|
assert_eq!(result.today.observed_cached_token_rate, Some(0.2));
|
|
assert_eq!(result.yesterday.request_count, 4);
|
|
assert_eq!(result.yesterday.total_tokens, 500);
|
|
assert_eq!(result.yesterday.observed_cached_token_rate, Some(0.5));
|
|
assert_eq!(result.observation.range, range);
|
|
assert_eq!(result.observation.trend.len(), 1);
|
|
assert_eq!(result.observation.trend[0].bucket_start, range.start);
|
|
assert_eq!(
|
|
store
|
|
.summary_query
|
|
.lock()
|
|
.unwrap()
|
|
.as_ref()
|
|
.unwrap()
|
|
.yesterday_range
|
|
.end,
|
|
range.start
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_keep_missing_yesterday_cache_rate_unknown() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let result = observability_services(store)
|
|
.await
|
|
.observability()
|
|
.dashboard_summary(range, TrendKind::Usage)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(result.yesterday.request_count, 0);
|
|
assert_eq!(result.yesterday.total_tokens, 0);
|
|
assert_eq!(result.yesterday.observed_cached_token_rate, None);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn health_timeline_should_apply_status_precedence_and_thresholds() {
|
|
let now = Utc::now();
|
|
let current_slot = quarter_hour_start(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
let services = observability_services(store.clone()).await;
|
|
let fixtures = [
|
|
(
|
|
RequestMetrics {
|
|
failure_count: 1,
|
|
cancelled_count: 1,
|
|
caller_error_count: 1,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::NoData,
|
|
None,
|
|
),
|
|
(
|
|
RequestMetrics {
|
|
incomplete_count: 1,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::LowSample,
|
|
Some(0.0),
|
|
),
|
|
(
|
|
RequestMetrics {
|
|
incomplete_count: 3,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::Unavailable,
|
|
Some(0.0),
|
|
),
|
|
(
|
|
RequestMetrics {
|
|
failure_count: 3,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::Unavailable,
|
|
Some(0.0),
|
|
),
|
|
(
|
|
RequestMetrics {
|
|
success_count: 1,
|
|
failure_count: 1,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::LowSample,
|
|
Some(50.0),
|
|
),
|
|
(
|
|
RequestMetrics {
|
|
success_count: 94,
|
|
failure_count: 6,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::Unstable,
|
|
Some(94.0),
|
|
),
|
|
(
|
|
RequestMetrics {
|
|
success_count: 95,
|
|
incomplete_count: 5,
|
|
..RequestMetrics::default()
|
|
},
|
|
HealthStatus::Stable,
|
|
Some(95.0),
|
|
),
|
|
];
|
|
|
|
for (index, (metrics, expected_status, expected_reliability)) in
|
|
fixtures.into_iter().enumerate()
|
|
{
|
|
store.replace_trend(vec![health_metric_point(current_slot, metrics)]);
|
|
let range_end =
|
|
now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2);
|
|
let timeline = services
|
|
.observability()
|
|
.dashboard_summary(observation_range(range_end), TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard summary")
|
|
.health_timeline;
|
|
let point = timeline
|
|
.points
|
|
.iter()
|
|
.find(|point| point.bucket_start == current_slot)
|
|
.expect("current health slot");
|
|
assert_eq!(point.status, expected_status);
|
|
assert_eq!(point.reliability_percent, expected_reliability);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn health_views_should_include_incomplete_and_keep_final_outcomes_separate() {
|
|
let now = Utc::now();
|
|
let current_slot = quarter_hour_start(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
let services = observability_services(store.clone()).await;
|
|
let fixtures = [
|
|
(162, 0, 2, 0, 0, 0, Some(98.780_487_804_878_05)),
|
|
(297, 0, 2, 5, 0, 0, Some(99.331_103_678_929_76)),
|
|
(467, 1, 5, 0, 0, 0, Some(98.731_501_057_082_45)),
|
|
(95, 2, 3, 4, 1, 7, Some(95.0)),
|
|
(100, 0, 0, 0, 0, 0, Some(100.0)),
|
|
(0, 0, 0, 0, 0, 0, None),
|
|
(0, 0, 0, 4, 3, 2, None),
|
|
(0, 1, 2, 0, 0, 0, Some(0.0)),
|
|
];
|
|
|
|
for (
|
|
index,
|
|
(success, failed, incomplete, cancelled, caller_error, running, expected_percent),
|
|
) in fixtures.into_iter().enumerate()
|
|
{
|
|
let range = observation_range(
|
|
now + Duration::seconds(i64::try_from(index).expect("fixture index fits i64") * 2),
|
|
);
|
|
let metrics = RequestMetrics {
|
|
request_count: success + failed + incomplete + cancelled + caller_error + running,
|
|
success_count: success,
|
|
failure_count: failed + caller_error,
|
|
incomplete_count: incomplete,
|
|
cancelled_count: cancelled,
|
|
caller_error_count: caller_error,
|
|
..RequestMetrics::default()
|
|
};
|
|
store.replace_trend(vec![health_metric_point(current_slot, metrics.clone())]);
|
|
store.replace_overview(UsageOverview {
|
|
analytics: Default::default(),
|
|
range,
|
|
requests: metrics,
|
|
// 中间尝试失败不会增加最终失败请求数
|
|
attempts: AttemptMetrics {
|
|
failure_count: 2,
|
|
..AttemptMetrics::default()
|
|
},
|
|
providers: Vec::new(),
|
|
});
|
|
|
|
let mut item = diagnostic(
|
|
"model",
|
|
success + failed + incomplete + cancelled + caller_error + running,
|
|
);
|
|
item.success_count = success;
|
|
item.failure_count = failed + caller_error;
|
|
item.incomplete_count = incomplete;
|
|
item.cancelled_count = cancelled;
|
|
item.caller_error_count = caller_error;
|
|
store.replace_diagnostics(DiagnosticsObservation {
|
|
analytics: Default::default(),
|
|
total_request_count: item.request_count,
|
|
items: vec![item],
|
|
});
|
|
let diagnostics = services
|
|
.observability()
|
|
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Model)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(diagnostics.items[0].error_count, failed + incomplete);
|
|
|
|
let dashboard = services
|
|
.observability()
|
|
.dashboard_summary(range, TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard summary");
|
|
let timeline = dashboard.health_timeline;
|
|
let point = timeline
|
|
.points
|
|
.iter()
|
|
.find(|point| point.bucket_start == current_slot)
|
|
.expect("current health slot");
|
|
assert_eq!(dashboard.trend.summary.error_count, failed + incomplete);
|
|
assert_eq!(dashboard.trend.points[0].error_count, failed + incomplete);
|
|
assert_eq!(point.failed_requests, failed);
|
|
assert_eq!(point.incomplete_requests, incomplete);
|
|
assert_eq!(point.cancelled_requests, cancelled);
|
|
assert_eq!(point.caller_error_requests, caller_error);
|
|
|
|
let insights = services
|
|
.observability()
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.expect("usage insights");
|
|
assert_eq!(insights.health.failed_requests, failed);
|
|
assert_eq!(insights.health.incomplete_requests, incomplete);
|
|
for actual in [
|
|
timeline.reliability_percent,
|
|
point.reliability_percent,
|
|
dashboard
|
|
.trend
|
|
.summary
|
|
.success_rate
|
|
.map(|rate| rate * 100.0),
|
|
dashboard.trend.points[0]
|
|
.success_rate
|
|
.map(|rate| rate * 100.0),
|
|
insights.health.success_rate.map(|rate| rate * 100.0),
|
|
insights.health.points[0]
|
|
.error_rate
|
|
.map(|rate| (1.0 - rate) * 100.0),
|
|
diagnostics.items[0]
|
|
.error_rate
|
|
.map(|rate| (1.0 - rate) * 100.0),
|
|
] {
|
|
match (actual, expected_percent) {
|
|
(Some(actual), Some(expected)) => assert!((actual - expected).abs() < 1e-10),
|
|
(None, None) => {}
|
|
_ => panic!("unexpected rate: {actual:?}, expected {expected_percent:?}"),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
|
|
let now = Utc::now();
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
store.replace_runtime_slots(Some(DashboardRuntimeSlots {
|
|
inherited_versions: Vec::new(),
|
|
inherited_accounts: 2,
|
|
overridden_slots: 5,
|
|
used_slots: Some(2),
|
|
}));
|
|
let services = observability_services(store).await;
|
|
|
|
let capacity = services
|
|
.observability()
|
|
.dashboard_summary(observation_range(now), TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard summary")
|
|
.capacity;
|
|
|
|
assert_eq!(capacity.total_slots, Some(7));
|
|
assert_eq!(capacity.used_slots, Some(2));
|
|
assert_eq!(capacity.available_slots, Some(5));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_capacity_distinguishes_unlimited_inheritance_finite_overrides_and_empty_pool() {
|
|
for (inherited_accounts, overridden_slots, expected_total, expected_available) in [
|
|
(2, 5, None, None),
|
|
(0, 5, Some(5), Some(3)),
|
|
(0, 0, Some(0), Some(0)),
|
|
] {
|
|
let now = Utc::now();
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
store.replace_runtime_slots(Some(DashboardRuntimeSlots {
|
|
inherited_versions: Vec::new(),
|
|
inherited_accounts,
|
|
overridden_slots,
|
|
used_slots: Some(2),
|
|
}));
|
|
let services = super::AdminHarness::new()
|
|
.observability(store)
|
|
.settings(Arc::new(FixtureSettingsStore {
|
|
max_concurrent_per_account: 0,
|
|
}))
|
|
.provider(super::dashboard_profile_provider())
|
|
.build()
|
|
.await;
|
|
let capacity = services
|
|
.observability()
|
|
.dashboard_summary(observation_range(now), TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard")
|
|
.capacity;
|
|
assert_eq!(capacity.total_slots, expected_total);
|
|
assert_eq!(capacity.available_slots, expected_available);
|
|
assert_eq!(capacity.used_slots, Some(2));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_observe_current_accounts_for_a_historical_range() {
|
|
let historical_end = Utc::now() - Duration::hours(1);
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(
|
|
historical_end,
|
|
)));
|
|
let services = observability_services(store.clone()).await;
|
|
|
|
services
|
|
.observability()
|
|
.dashboard_summary(observation_range(historical_end), TrendKind::Usage)
|
|
.await
|
|
.expect("dashboard summary");
|
|
|
|
let summary_observed_at = *store.summary_observed_at.lock().unwrap();
|
|
assert!(summary_observed_at.is_some_and(|value| value > historical_end));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_keep_distinct_exact_ranges_in_one_short_window() {
|
|
let end_bucket = Utc::now().timestamp().div_euclid(2) * 2;
|
|
let first_end = DateTime::from_timestamp(end_bucket, 100_000_000).expect("first end");
|
|
let second_end = DateTime::from_timestamp(end_bucket, 900_000_000).expect("second end");
|
|
let first_range = TimeRange::new(
|
|
gateway_core::time::DeploymentTimeZone::default()
|
|
.day_start(first_end)
|
|
.unwrap(),
|
|
first_end,
|
|
)
|
|
.expect("first range");
|
|
let second_range = TimeRange::new(
|
|
gateway_core::time::DeploymentTimeZone::default()
|
|
.day_start(second_end)
|
|
.unwrap(),
|
|
second_end,
|
|
)
|
|
.expect("second range");
|
|
let store = Arc::new(FixtureObservabilityStore::new(first_range));
|
|
store.set_dashboard_delay(StdDuration::from_millis(20));
|
|
let services = observability_services(store.clone()).await;
|
|
let observability = services.observability();
|
|
|
|
let (first, second) = tokio::join!(
|
|
observability.dashboard_summary(first_range, TrendKind::Usage),
|
|
observability.dashboard_summary(second_range, TrendKind::Errors),
|
|
);
|
|
|
|
first.expect("first dashboard summary");
|
|
second.expect("second dashboard summary");
|
|
assert_eq!(store.dashboard_summary_calls(), 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_summary_should_reload_after_the_short_window_changes() {
|
|
let end_bucket = Utc::now().timestamp().div_euclid(2) * 2;
|
|
let first_end = DateTime::from_timestamp(end_bucket, 100_000_000).expect("first end");
|
|
let second_end = first_end + Duration::seconds(2);
|
|
let first_range = TimeRange::new(
|
|
gateway_core::time::DeploymentTimeZone::default()
|
|
.day_start(first_end)
|
|
.unwrap(),
|
|
first_end,
|
|
)
|
|
.expect("first range");
|
|
let second_range = TimeRange::new(
|
|
gateway_core::time::DeploymentTimeZone::default()
|
|
.day_start(second_end)
|
|
.unwrap(),
|
|
second_end,
|
|
)
|
|
.expect("second range");
|
|
let store = Arc::new(FixtureObservabilityStore::new(first_range));
|
|
let services = observability_services(store.clone()).await;
|
|
let observability = services.observability();
|
|
|
|
observability
|
|
.dashboard_summary(first_range, TrendKind::Usage)
|
|
.await
|
|
.expect("first dashboard summary");
|
|
observability
|
|
.dashboard_summary(second_range, TrendKind::Usage)
|
|
.await
|
|
.expect("second dashboard summary");
|
|
|
|
assert_eq!(store.dashboard_summary_calls(), 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn observability_services_should_calculate_usage_insights_and_diagnostic_shares() {
|
|
let now = Utc::now();
|
|
let range = observation_range(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let metrics = RequestMetrics {
|
|
request_count: 10,
|
|
success_count: 6,
|
|
failure_count: 4,
|
|
caller_error_count: 2,
|
|
input_tokens: 800,
|
|
output_tokens: 200,
|
|
total_tokens: 1_000,
|
|
latency_count: 5,
|
|
admission_decision_count: 10,
|
|
admission_decision_percentiles: LatencyPercentiles {
|
|
p50_ms: Some(PercentileMilliseconds::new(2.0).expect("admission P50")),
|
|
p95_ms: Some(PercentileMilliseconds::new(5.0).expect("admission P95")),
|
|
p99_ms: Some(PercentileMilliseconds::new(8.0).expect("admission P99")),
|
|
},
|
|
account_selection_wait_count: 8,
|
|
account_selection_wait_percentiles: LatencyPercentiles {
|
|
p50_ms: Some(PercentileMilliseconds::new(10.0).expect("selection P50")),
|
|
p95_ms: Some(PercentileMilliseconds::new(40.0).expect("selection P95")),
|
|
p99_ms: Some(PercentileMilliseconds::new(70.0).expect("selection P99")),
|
|
},
|
|
output_throughput_p10: Some(40),
|
|
output_throughput_p50: Some(50),
|
|
output_throughput_p90: Some(60),
|
|
capacity_sample_count: 8,
|
|
capacity_utilization_avg_basis_points: Some(6_500),
|
|
capacity_utilization_p95_basis_points: Some(8_000),
|
|
..RequestMetrics::default()
|
|
};
|
|
store.replace_overview(UsageOverview {
|
|
analytics: Default::default(),
|
|
range,
|
|
requests: metrics.clone(),
|
|
attempts: AttemptMetrics {
|
|
attempt_count: 12,
|
|
cost_coverage: CostCoverage {
|
|
calculated_count: 10,
|
|
..CostCoverage::default()
|
|
},
|
|
costs: vec![CurrencyCost {
|
|
currency: "USD".to_owned(),
|
|
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
|
.expect("USD cost"),
|
|
}],
|
|
..AttemptMetrics::default()
|
|
},
|
|
providers: Vec::new(),
|
|
});
|
|
store.replace_trend(vec![RequestMetricPoint {
|
|
bucket_start: quarter_hour_start(now),
|
|
granularity: Granularity::FifteenMinutes,
|
|
metrics,
|
|
cost_coverage: CostCoverage::default(),
|
|
costs: vec![CurrencyCost {
|
|
currency: "USD".to_owned(),
|
|
amount: gateway_admin::model::observability::DecimalAmount::from_str("0.25")
|
|
.expect("bucket USD cost"),
|
|
}],
|
|
}]);
|
|
store.replace_projected_costs(vec![UsageProjectedCostPoint {
|
|
bucket_start: quarter_hour_start(now),
|
|
standard_amount: Some("1".parse().expect("standard cost")),
|
|
no_cache_amount: Some("1.25".parse().expect("no-cache cost")),
|
|
}]);
|
|
store.replace_diagnostics(DiagnosticsObservation {
|
|
analytics: Default::default(),
|
|
total_request_count: 4,
|
|
items: vec![diagnostic("openai", 3), diagnostic("xai", 1)],
|
|
});
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let insights = services
|
|
.observability()
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.expect("usage insights");
|
|
let diagnostics = services
|
|
.observability()
|
|
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Provider)
|
|
.await
|
|
.expect("usage diagnostics");
|
|
|
|
assert_eq!(insights.health.total_requests, 10);
|
|
assert_eq!(insights.health.failed_requests, 2);
|
|
assert_eq!(insights.health.success_rate, Some(0.75));
|
|
assert_eq!(insights.health.completion_rate, 1.0);
|
|
assert_eq!(insights.granularity, Granularity::FifteenMinutes);
|
|
assert_eq!(
|
|
insights
|
|
.performance
|
|
.load
|
|
.admission_decision_p95_ms
|
|
.map(|value| value.as_f64()),
|
|
Some(5.0)
|
|
);
|
|
assert_eq!(
|
|
insights
|
|
.performance
|
|
.load
|
|
.account_selection_wait_p95_ms
|
|
.map(|value| value.as_f64()),
|
|
Some(40.0)
|
|
);
|
|
assert_eq!(
|
|
insights.performance.points[0].load,
|
|
insights.performance.load
|
|
);
|
|
assert_eq!(insights.performance.output_throughput_p50, Some(50));
|
|
assert_eq!(insights.cost.tokens_per_request, 100.0);
|
|
assert_eq!(
|
|
insights
|
|
.cost
|
|
.estimated_cost
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("1.25")
|
|
);
|
|
assert_eq!(
|
|
insights
|
|
.cost
|
|
.cost_per_request
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("0.125")
|
|
);
|
|
assert_eq!(
|
|
insights.cost.points[0]
|
|
.estimated_cost
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("0.25")
|
|
);
|
|
assert_eq!(
|
|
insights
|
|
.cost
|
|
.standard_cost
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("1")
|
|
);
|
|
assert_eq!(
|
|
insights.cost.points[0]
|
|
.standard_cost
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("1")
|
|
);
|
|
assert_eq!(
|
|
insights
|
|
.cost
|
|
.no_cache_cost
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("1.25")
|
|
);
|
|
assert_eq!(
|
|
insights
|
|
.cost
|
|
.tier_premium
|
|
.as_ref()
|
|
.map(gateway_admin::model::observability::DecimalAmount::as_str),
|
|
Some("0.25")
|
|
);
|
|
assert_eq!(diagnostics.items[0].request_share, 0.75);
|
|
assert_eq!(diagnostics.items[1].request_share, 0.25);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_insights_should_weight_concurrency_by_effective_bucket_duration() {
|
|
let bucket_start = DateTime::parse_from_rfc3339("2026-09-01T10:00:00Z")
|
|
.unwrap()
|
|
.with_timezone(&Utc);
|
|
let range = TimeRange::new(
|
|
bucket_start + Duration::minutes(5),
|
|
bucket_start + Duration::minutes(20),
|
|
)
|
|
.unwrap();
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.replace_trend(vec![
|
|
health_metric_point(bucket_start, RequestMetrics::default()),
|
|
health_metric_point(
|
|
bucket_start + Duration::minutes(15),
|
|
RequestMetrics::default(),
|
|
),
|
|
]);
|
|
*store.concurrency.lock().unwrap() = vec![
|
|
UsageConcurrencyPoint {
|
|
bucket_start,
|
|
average_concurrency: 1.0,
|
|
peak_concurrency: 1,
|
|
},
|
|
UsageConcurrencyPoint {
|
|
bucket_start: bucket_start + Duration::minutes(15),
|
|
average_concurrency: 3.0,
|
|
peak_concurrency: 4,
|
|
},
|
|
];
|
|
|
|
let insights = observability_services(store)
|
|
.await
|
|
.observability()
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!((insights.performance.load.average_concurrency - 5.0 / 3.0).abs() < 1e-10);
|
|
assert_eq!(insights.performance.load.peak_concurrency, 4);
|
|
assert_eq!(insights.performance.points[0].load.average_concurrency, 1.0);
|
|
assert_eq!(insights.performance.points[0].load.peak_concurrency, 1);
|
|
assert_eq!(insights.performance.points[1].load.average_concurrency, 3.0);
|
|
assert_eq!(insights.performance.points[1].load.peak_concurrency, 4);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_preserve_request_order_when_a_small_sample_fails() {
|
|
let now = Utc::now();
|
|
let range = observation_range(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let success = diagnostic("popular", 100);
|
|
let mut failure = diagnostic("single_failure", 1);
|
|
failure.success_count = 0;
|
|
failure.failure_count = 1;
|
|
store.replace_diagnostics(DiagnosticsObservation {
|
|
analytics: Default::default(),
|
|
total_request_count: 101,
|
|
items: vec![success, failure],
|
|
});
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let result = services
|
|
.observability()
|
|
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Model)
|
|
.await
|
|
.expect("diagnostics");
|
|
|
|
assert_eq!(result.items[0].key, "popular");
|
|
assert_eq!(result.items[1].key, "single_failure");
|
|
assert_eq!(result.items[0].error_count, 0);
|
|
assert_eq!(result.items[1].error_count, 1);
|
|
assert_eq!(result.items[1].error_rate, Some(1.0));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_use_the_total_before_truncating_groups() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.replace_diagnostics(DiagnosticsObservation {
|
|
analytics: Default::default(),
|
|
total_request_count: 200,
|
|
items: (0..100)
|
|
.map(|index| diagnostic(&format!("model-{index}"), 1))
|
|
.collect(),
|
|
});
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let result = services
|
|
.observability()
|
|
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Model)
|
|
.await
|
|
.expect("limited diagnostics");
|
|
|
|
assert!(result.items.iter().all(|item| item.request_share == 0.005));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_count_each_retried_request_once() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let mut item = diagnostic("model", 8);
|
|
item.attempt_count = 14;
|
|
item.retry_count = 6;
|
|
item.retried_request_count = 2;
|
|
store.replace_diagnostics(DiagnosticsObservation {
|
|
analytics: Default::default(),
|
|
total_request_count: 8,
|
|
items: vec![item],
|
|
});
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let result = services
|
|
.observability()
|
|
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Model)
|
|
.await
|
|
.expect("retry diagnostics");
|
|
|
|
assert_eq!(
|
|
(result.items[0].retry_count, result.items[0].retry_rate),
|
|
(6, 0.25)
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_preserve_empty_results() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let result = services
|
|
.observability()
|
|
.diagnostics(range, UsageFilter::default(), DiagnosticDimension::Account)
|
|
.await
|
|
.expect("empty diagnostics");
|
|
|
|
assert!(result.items.is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
|
let now = Utc::now();
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
let invalid_kind = "x".repeat(65);
|
|
store.replace_usage_records(vec![
|
|
total_record(
|
|
"request_invalid_kind",
|
|
Some(&invalid_kind),
|
|
"calculated",
|
|
now,
|
|
),
|
|
total_record(
|
|
"request_unregistered_kind",
|
|
Some("anthropic"),
|
|
"calculated",
|
|
now,
|
|
),
|
|
total_record("request_enriched", Some("openai"), "calculated", now),
|
|
]);
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let page = services
|
|
.observability()
|
|
.usage_records(usage_query(now))
|
|
.await
|
|
.expect("one bad record must not fail the whole usage list");
|
|
|
|
assert_eq!(page.items.len(), 3);
|
|
assert!(
|
|
matches!(page.items[0].billing, Some(UsageBilling::Total { .. })),
|
|
"invalid Provider kind keeps the stored total"
|
|
);
|
|
assert!(
|
|
matches!(page.items[1].billing, Some(UsageBilling::Total { .. })),
|
|
"unregistered Provider kind keeps the stored total"
|
|
);
|
|
assert!(
|
|
matches!(page.items[2].billing, Some(UsageBilling::Calculated(_))),
|
|
"healthy record is still enriched"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_records_should_enrich_provider_reported_totals_when_pricing_matches() {
|
|
let now = Utc::now();
|
|
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
|
store.replace_usage_records(vec![total_record(
|
|
"request_provider_reported",
|
|
Some("openai"),
|
|
"provider_reported",
|
|
now,
|
|
)]);
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
|
|
let page = services
|
|
.observability()
|
|
.usage_records(usage_query(now))
|
|
.await
|
|
.expect("usage records");
|
|
|
|
assert!(matches!(
|
|
page.items[0].billing,
|
|
Some(UsageBilling::Calculated(_))
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_insights_should_reject_partial_costs_when_insights_fail() {
|
|
let now = Utc::now();
|
|
let range = observation_range(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.replace_projected_costs(vec![UsageProjectedCostPoint {
|
|
bucket_start: quarter_hour_start(now),
|
|
standard_amount: Some("1".parse().expect("standard cost")),
|
|
no_cache_amount: Some("1.25".parse().expect("no-cache cost")),
|
|
}]);
|
|
store.insights_fail.store(true, Ordering::SeqCst);
|
|
let services = observability_services_with_calculated_billing(store).await;
|
|
assert!(
|
|
services
|
|
.observability()
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_should_preserve_provider_plan_names() {
|
|
let now = Utc::now();
|
|
let range = observation_range(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let plans = [("prolite", "ProLite"), ("pro", "Pro"), ("promax", "ProMax")];
|
|
*store.account_usage.lock().unwrap() = plans
|
|
.iter()
|
|
.map(|(plan, _)| DashboardAccountUsage {
|
|
upstream: None,
|
|
account_id: format!("account-{plan}"),
|
|
provider_kind: "openai".to_owned(),
|
|
authentication_kind: "oauth".to_owned(),
|
|
name: (*plan).to_owned(),
|
|
email: None,
|
|
notes: None,
|
|
plan_type: Some((*plan).to_owned()),
|
|
plan_type_display: None,
|
|
request_count: 0,
|
|
success_count: 0,
|
|
input_tokens: None,
|
|
output_tokens: None,
|
|
cached_input_tokens: None,
|
|
cache_write_input_tokens: None,
|
|
reasoning_output_tokens: None,
|
|
image_input_tokens: None,
|
|
image_output_tokens: None,
|
|
image_request_count: 0,
|
|
image_request_failed_count: 0,
|
|
total_tokens: None,
|
|
cost_coverage: CostCoverage::default(),
|
|
costs: Vec::new(),
|
|
last_used_at: None,
|
|
request_buckets: Vec::new(),
|
|
quota_used_percent: None,
|
|
quota_window: None,
|
|
models: Vec::new(),
|
|
})
|
|
.collect();
|
|
let provider =
|
|
super::accounts::FakeProviderAdmin::new("openai", Arc::new(Mutex::new(Vec::new())));
|
|
let services = super::AdminHarness::new()
|
|
.observability(store)
|
|
.settings(Arc::new(FixtureSettingsStore {
|
|
max_concurrent_per_account: 1,
|
|
}))
|
|
.provider(provider)
|
|
.build()
|
|
.await;
|
|
let result = services
|
|
.observability()
|
|
.dashboard_summary(range, TrendKind::Usage)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(result.observation.account_usage.len(), plans.len());
|
|
for (account, (raw, display)) in result.observation.account_usage.iter().zip(plans) {
|
|
assert_eq!(account.plan_type.as_deref(), Some(raw));
|
|
assert_eq!(account.plan_type_display.as_deref(), Some(display));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn dashboard_and_usage_insights_keep_concurrency_cache_entries_separate() {
|
|
for dashboard_first in [true, false] {
|
|
let now = Utc::now();
|
|
let range = observation_range(now);
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store
|
|
.concurrency
|
|
.lock()
|
|
.unwrap()
|
|
.push(UsageConcurrencyPoint {
|
|
bucket_start: range.start,
|
|
average_concurrency: 2.0,
|
|
peak_concurrency: 3,
|
|
});
|
|
let services = observability_services(store.clone()).await;
|
|
for dashboard in [
|
|
dashboard_first,
|
|
!dashboard_first,
|
|
dashboard_first,
|
|
!dashboard_first,
|
|
] {
|
|
if dashboard {
|
|
services
|
|
.observability()
|
|
.dashboard_trend(range, TrendKind::Usage)
|
|
.await
|
|
.unwrap();
|
|
} else {
|
|
let result = services
|
|
.observability()
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(result.performance.load.peak_concurrency, 3);
|
|
}
|
|
}
|
|
assert_eq!(
|
|
*store.concurrency_selections.lock().unwrap(),
|
|
vec![!dashboard_first, dashboard_first]
|
|
);
|
|
}
|
|
}
|
|
|
|
struct FixtureObservabilityStore {
|
|
yesterday: Mutex<RequestMetrics>,
|
|
trend: Mutex<Vec<RequestMetricPoint>>,
|
|
overview: Mutex<UsageOverview>,
|
|
projected_costs: Mutex<Vec<UsageProjectedCostPoint>>,
|
|
concurrency: Mutex<Vec<UsageConcurrencyPoint>>,
|
|
insights_fail: AtomicBool,
|
|
concurrency_selections: Mutex<Vec<bool>>,
|
|
metric_selections: Mutex<Vec<gateway_admin::model::observability::UsageInsightMetric>>,
|
|
summary_fail: AtomicBool,
|
|
diagnostics: Mutex<DiagnosticsObservation>,
|
|
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
|
summary_observed_at: Mutex<Option<DateTime<Utc>>>,
|
|
summary_query: Mutex<Option<DashboardQuery>>,
|
|
usage_granularities: Mutex<Vec<Granularity>>,
|
|
usage_records: Mutex<Vec<UsageListRecord>>,
|
|
account_usage: Mutex<Vec<DashboardAccountUsage>>,
|
|
dashboard_delay: Mutex<StdDuration>,
|
|
dashboard_summary_calls: AtomicUsize,
|
|
usage_summary_filters: Mutex<Vec<UsageFilter>>,
|
|
}
|
|
|
|
impl FixtureObservabilityStore {
|
|
fn new(range: TimeRange) -> Self {
|
|
Self {
|
|
yesterday: Mutex::new(RequestMetrics::default()),
|
|
trend: Mutex::new(Vec::new()),
|
|
overview: Mutex::new(UsageOverview {
|
|
analytics: Default::default(),
|
|
range,
|
|
requests: RequestMetrics::default(),
|
|
attempts: AttemptMetrics::default(),
|
|
providers: Vec::new(),
|
|
}),
|
|
projected_costs: Mutex::new(Vec::new()),
|
|
concurrency: Mutex::new(Vec::new()),
|
|
insights_fail: AtomicBool::new(false),
|
|
concurrency_selections: Mutex::new(Vec::new()),
|
|
metric_selections: Mutex::new(Vec::new()),
|
|
summary_fail: AtomicBool::new(false),
|
|
diagnostics: Mutex::new(DiagnosticsObservation::default()),
|
|
runtime_slots: Mutex::new(None),
|
|
summary_observed_at: Mutex::new(None),
|
|
summary_query: Mutex::new(None),
|
|
usage_granularities: Mutex::new(Vec::new()),
|
|
usage_records: Mutex::new(Vec::new()),
|
|
account_usage: Mutex::new(Vec::new()),
|
|
dashboard_delay: Mutex::new(StdDuration::ZERO),
|
|
dashboard_summary_calls: AtomicUsize::new(0),
|
|
usage_summary_filters: Mutex::new(Vec::new()),
|
|
}
|
|
}
|
|
|
|
fn set_dashboard_delay(&self, delay: StdDuration) {
|
|
*self.dashboard_delay.lock().expect("dashboard delay") = delay;
|
|
}
|
|
|
|
fn dashboard_summary_calls(&self) -> usize {
|
|
self.dashboard_summary_calls.load(Ordering::Relaxed)
|
|
}
|
|
|
|
fn replace_trend(&self, trend: Vec<RequestMetricPoint>) {
|
|
*self.trend.lock().expect("trend") = trend;
|
|
}
|
|
|
|
fn replace_overview(&self, overview: UsageOverview) {
|
|
*self.overview.lock().expect("overview") = overview;
|
|
}
|
|
|
|
fn replace_projected_costs(&self, facts: Vec<UsageProjectedCostPoint>) {
|
|
*self.projected_costs.lock().expect("projected costs") = facts;
|
|
}
|
|
|
|
fn replace_diagnostics(&self, diagnostics: DiagnosticsObservation) {
|
|
*self.diagnostics.lock().expect("diagnostics") = diagnostics;
|
|
}
|
|
|
|
fn replace_runtime_slots(&self, runtime_slots: Option<DashboardRuntimeSlots>) {
|
|
*self.runtime_slots.lock().expect("runtime slots") = runtime_slots;
|
|
}
|
|
|
|
fn replace_usage_records(&self, records: Vec<UsageListRecord>) {
|
|
*self.usage_records.lock().expect("usage records") = records;
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl ObservabilityStore for FixtureObservabilityStore {
|
|
async fn diagnostic_errors(
|
|
&self,
|
|
_target: &gateway_admin::model::diagnostics::DiagnosticTarget,
|
|
_limit: u16,
|
|
) -> AdminStoreResult<Vec<gateway_admin::model::observability::OpsError>> {
|
|
Err(super::unavailable("diagnostic errors"))
|
|
}
|
|
|
|
fn query_timeout(&self) -> std::time::Duration {
|
|
std::time::Duration::from_secs(10)
|
|
}
|
|
|
|
async fn dashboard_summary(
|
|
&self,
|
|
query: DashboardQuery,
|
|
observed_at: DateTime<Utc>,
|
|
) -> AdminStoreResult<DashboardObservation> {
|
|
self.dashboard_summary_calls.fetch_add(1, Ordering::Relaxed);
|
|
*self.summary_query.lock().unwrap() = Some(query.clone());
|
|
*self
|
|
.summary_observed_at
|
|
.lock()
|
|
.expect("summary observed at") = Some(observed_at);
|
|
let dashboard_delay = *self.dashboard_delay.lock().expect("dashboard delay");
|
|
if !dashboard_delay.is_zero() {
|
|
tokio::time::sleep(dashboard_delay).await;
|
|
}
|
|
Ok(DashboardObservation {
|
|
yesterday: self.yesterday.lock().unwrap().clone(),
|
|
analytics: self.overview.lock().expect("overview").analytics.clone(),
|
|
range: query.range,
|
|
totals: Default::default(),
|
|
provider_accounts: AccountPoolMetrics::default(),
|
|
runtime_slots: self.runtime_slots.lock().expect("runtime slots").clone(),
|
|
trend: self.trend.lock().expect("trend").clone(),
|
|
account_usage: self.account_usage.lock().expect("account usage").clone(),
|
|
recent_requests: Vec::new(),
|
|
})
|
|
}
|
|
|
|
async fn dashboard_trend(
|
|
&self,
|
|
_: TimeRange,
|
|
_: Granularity,
|
|
) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
|
Ok(self.trend.lock().expect("trend").clone())
|
|
}
|
|
|
|
async fn usage_trend(
|
|
&self,
|
|
_: TimeRange,
|
|
_: UsageFilter,
|
|
granularity: Granularity,
|
|
) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
|
self.usage_granularities.lock().unwrap().push(granularity);
|
|
Ok(self.trend.lock().expect("trend").clone())
|
|
}
|
|
|
|
async fn analytics_cache_generation(&self) -> AdminStoreResult<u64> {
|
|
Ok(self.overview.lock().expect("overview").analytics.generation)
|
|
}
|
|
|
|
async fn usage_insights_observation(
|
|
&self,
|
|
_: TimeRange,
|
|
_: UsageFilter,
|
|
granularity: Granularity,
|
|
include_concurrency: bool,
|
|
) -> AdminStoreResult<UsageInsightsObservation> {
|
|
self.usage_granularities.lock().unwrap().push(granularity);
|
|
self.concurrency_selections
|
|
.lock()
|
|
.unwrap()
|
|
.push(include_concurrency);
|
|
if self.insights_fail.load(Ordering::SeqCst) {
|
|
return Err(super::unavailable("projection read failed"));
|
|
}
|
|
Ok(UsageInsightsObservation {
|
|
overview: self.overview.lock().expect("overview").clone(),
|
|
trend: self.trend.lock().expect("trend").clone(),
|
|
costs: self
|
|
.projected_costs
|
|
.lock()
|
|
.expect("projected costs")
|
|
.clone(),
|
|
concurrency: if include_concurrency {
|
|
self.concurrency.lock().expect("concurrency").clone()
|
|
} else {
|
|
Vec::new()
|
|
},
|
|
})
|
|
}
|
|
|
|
async fn usage_metric_observation(
|
|
&self,
|
|
range: gateway_admin::model::observability::TimeRange,
|
|
filter: gateway_admin::model::observability::UsageFilter,
|
|
granularity: gateway_admin::model::observability::Granularity,
|
|
metric: gateway_admin::model::observability::UsageInsightMetric,
|
|
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
|
self.metric_selections.lock().unwrap().push(metric);
|
|
self.usage_insights_observation(
|
|
range,
|
|
filter,
|
|
granularity,
|
|
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
|
let items = self.usage_records.lock().expect("usage records").clone();
|
|
let total = u64::try_from(items.len()).unwrap_or(u64::MAX);
|
|
Ok(UsagePage {
|
|
items,
|
|
current_page: query.current_page,
|
|
page_size: query.page_size.get(),
|
|
total,
|
|
})
|
|
}
|
|
|
|
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
|
Err(super::unavailable("usage detail"))
|
|
}
|
|
|
|
async fn usage_summary(
|
|
&self,
|
|
range: TimeRange,
|
|
filter: UsageFilter,
|
|
) -> AdminStoreResult<UsageSummaryObservation> {
|
|
self.usage_summary_filters
|
|
.lock()
|
|
.expect("summary filters")
|
|
.push(filter);
|
|
let delay = *self.dashboard_delay.lock().expect("query delay");
|
|
if !delay.is_zero() {
|
|
tokio::time::sleep(delay).await;
|
|
}
|
|
if self.summary_fail.load(Ordering::SeqCst) {
|
|
return Err(super::unavailable("projection summary failed"));
|
|
}
|
|
let overview = self.overview.lock().expect("overview").clone();
|
|
Ok(UsageSummaryObservation {
|
|
range,
|
|
analytics: overview.analytics,
|
|
requests: (&overview.requests).into(),
|
|
attempts: overview.attempts,
|
|
})
|
|
}
|
|
|
|
async fn usage_diagnostics(
|
|
&self,
|
|
_: TimeRange,
|
|
_: UsageFilter,
|
|
_: DiagnosticDimension,
|
|
_: u16,
|
|
) -> AdminStoreResult<DiagnosticsObservation> {
|
|
Ok(self.diagnostics.lock().expect("diagnostics").clone())
|
|
}
|
|
|
|
async fn list_ops_errors(&self, _: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
|
Err(super::unavailable("ops errors"))
|
|
}
|
|
}
|
|
|
|
struct FixtureSettingsStore {
|
|
max_concurrent_per_account: u32,
|
|
}
|
|
|
|
#[async_trait]
|
|
impl SettingsStore for FixtureSettingsStore {
|
|
async fn load_pricing(&self) -> AdminStoreResult<gateway_admin::model::pricing::StoredPricing> {
|
|
Ok(Default::default())
|
|
}
|
|
async fn sync_pricing(
|
|
&self,
|
|
_: gateway_admin::model::pricing::PricingSyncChanges,
|
|
_: &MutationContext,
|
|
) -> AdminStoreResult<gateway_admin::model::Revision> {
|
|
panic!("unexpected pricing sync")
|
|
}
|
|
async fn update_pricing(
|
|
&self,
|
|
_: gateway_admin::model::pricing::UpdatePricing,
|
|
_: &MutationContext,
|
|
) -> AdminStoreResult<gateway_admin::model::Revision> {
|
|
panic!("unexpected pricing update")
|
|
}
|
|
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
|
Ok(RuntimeSettings {
|
|
request_profiles: Default::default(),
|
|
config_revision: Revision::new(1).expect("revision"),
|
|
model_mappings: Default::default(),
|
|
rotation_strategy: RotationStrategy::Smart,
|
|
updated_at: Utc::now(),
|
|
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
|
codex_privacy_policy: Default::default(),
|
|
request_location_enabled: false,
|
|
request_location: Default::default(),
|
|
refresh_margin_seconds: 300,
|
|
refresh_concurrency: 2,
|
|
max_concurrent_per_account: self.max_concurrent_per_account,
|
|
max_waiting_per_key: 0,
|
|
max_waiting_per_account: 0,
|
|
key_queue_timeout_seconds: 30,
|
|
account_queue_timeout_seconds: 30,
|
|
switch_account_on_queue_failure: false,
|
|
max_retry_after_seconds: 240,
|
|
openai_guardian_reserved_concurrency: 0,
|
|
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
|
max_account_rotations: 3,
|
|
openai_session_affinity_ttl_hours: 24,
|
|
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
|
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
|
quota_reset_window_weights: gateway_core::account::ResetWindowWeights::default(),
|
|
min_codex_desktop_version: None,
|
|
min_codex_cli_version: None,
|
|
usage_retention_days: 31,
|
|
ops_event_retention_days: 30,
|
|
audit_retention_days: 30,
|
|
account_error_protection: gateway_core::account::AccountErrorProtection::default(),
|
|
account_warmup_enabled: false,
|
|
account_warmup_cron: "0 8 * * *".to_owned(),
|
|
account_warmup_model: None,
|
|
reset_credit_lead_minutes: 10,
|
|
reset_credit_on_low_weekly_quota: false,
|
|
reset_credit_max_weekly_remaining_percent: None,
|
|
},
|
|
})
|
|
}
|
|
|
|
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
|
|
Err(super::unavailable("admin API key"))
|
|
}
|
|
|
|
async fn replace_runtime_settings(
|
|
&self,
|
|
_: ReplaceRuntimeSettings,
|
|
_: &MutationContext,
|
|
) -> AdminStoreResult<RuntimeSettings> {
|
|
Err(super::unavailable("settings"))
|
|
}
|
|
|
|
async fn replace_admin_api_key(
|
|
&self,
|
|
_: AdminApiKey,
|
|
_: &MutationContext,
|
|
) -> AdminStoreResult<AdminApiKeyMutation> {
|
|
Err(super::unavailable("admin API key"))
|
|
}
|
|
|
|
async fn delete_admin_api_key(
|
|
&self,
|
|
_: &MutationContext,
|
|
) -> AdminStoreResult<AdminApiKeyMutation> {
|
|
Err(super::unavailable("admin API key"))
|
|
}
|
|
}
|
|
|
|
async fn observability_services(store: Arc<FixtureObservabilityStore>) -> AdminServices {
|
|
super::AdminHarness::new()
|
|
.observability(store)
|
|
.settings(Arc::new(FixtureSettingsStore {
|
|
max_concurrent_per_account: 1,
|
|
}))
|
|
.provider(super::dashboard_profile_provider())
|
|
.build()
|
|
.await
|
|
}
|
|
|
|
async fn observability_services_with_calculated_billing(
|
|
store: Arc<FixtureObservabilityStore>,
|
|
) -> AdminServices {
|
|
super::AdminHarness::new()
|
|
.observability(store)
|
|
.settings(Arc::new(FixtureSettingsStore {
|
|
max_concurrent_per_account: 1,
|
|
}))
|
|
.provider(super::calculated_billing_provider())
|
|
.build()
|
|
.await
|
|
}
|
|
|
|
fn observation_range(end: DateTime<Utc>) -> TimeRange {
|
|
TimeRange::new(end - Duration::hours(24), end).expect("observation range")
|
|
}
|
|
|
|
fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
|
UsageQuery {
|
|
status: Default::default(),
|
|
range: observation_range(now),
|
|
filter: UsageFilter::default(),
|
|
current_page: 1,
|
|
page_size: ObservabilityPageSize::new(50).expect("page size"),
|
|
}
|
|
}
|
|
|
|
fn total_record(
|
|
id: &str,
|
|
provider_kind: Option<&str>,
|
|
source: &str,
|
|
now: DateTime<Utc>,
|
|
) -> UsageListRecord {
|
|
UsageListRecord {
|
|
provider_account_upstream_peer: None,
|
|
has_upstream_error: false,
|
|
error_code: None,
|
|
error_message: None,
|
|
error_kind: None,
|
|
websocket_close_code: None,
|
|
continuation_unavailable_reason: None,
|
|
upstream_connection_exit_reason: None,
|
|
recovered: false,
|
|
client_status_code: None,
|
|
upstream_status_code: None,
|
|
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
|
|
client_api_key_name: Some("Production".to_owned()),
|
|
id: id.to_owned(),
|
|
endpoint: "/v1/responses".to_owned(),
|
|
client_transport: "http_sse".to_owned(),
|
|
requested_model_id: Some("gpt-5.5".to_owned()),
|
|
provider_kind: provider_kind.map(str::to_owned),
|
|
provider_account_ref: None,
|
|
provider_account_name: None,
|
|
provider_account_email: None,
|
|
provider_account_notes: None,
|
|
provider_account_plan_type: None,
|
|
provider_account_plan_type_display: None,
|
|
provider_account_authentication_kind: None,
|
|
upstream_model_id: Some("gpt-5.5".to_owned()),
|
|
upstream_transport: None,
|
|
service_tier: None,
|
|
input_tokens: Some(800),
|
|
output_tokens: Some(200),
|
|
cached_input_tokens: Some(0),
|
|
cache_write_input_tokens: Some(0),
|
|
reasoning_output_tokens: Some(0),
|
|
image_input_tokens: None,
|
|
image_output_tokens: None,
|
|
total_tokens: Some(1_000),
|
|
cost_source: source.to_owned(),
|
|
cost_amount: None,
|
|
cost_currency: None,
|
|
billing: Some(UsageBilling::Total {
|
|
source: source.to_owned(),
|
|
total: CurrencyCost {
|
|
currency: "USD".to_owned(),
|
|
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
|
.expect("billing total"),
|
|
},
|
|
}),
|
|
transport_decision_wait_ms: None,
|
|
connect_ms: None,
|
|
headers_ms: None,
|
|
first_event_ms: None,
|
|
first_reasoning_ms: None,
|
|
first_text_ms: None,
|
|
first_token_ms: None,
|
|
provider_processing_ms: None,
|
|
upstream_response_ms: None,
|
|
upstream_api_overhead_ms: None,
|
|
upstream_engine_ms: None,
|
|
latency_ms: Some(100),
|
|
admission_decision_ms: None,
|
|
account_selection_wait_ms: None,
|
|
capacity_used_slots: None,
|
|
capacity_total_slots: None,
|
|
client_ip: None,
|
|
user_agent: None,
|
|
reasoning_effort: None,
|
|
reasoning_preset: None,
|
|
subagent_kind: None,
|
|
compact: false,
|
|
started_at: now,
|
|
}
|
|
}
|
|
|
|
fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) -> RequestMetricPoint {
|
|
RequestMetricPoint {
|
|
bucket_start,
|
|
granularity: Granularity::FifteenMinutes,
|
|
metrics,
|
|
cost_coverage: CostCoverage::default(),
|
|
costs: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
|
DiagnosticObservation {
|
|
first_token_p95_ms: None,
|
|
cancelled_count: 0,
|
|
incomplete_count: 0,
|
|
caller_error_count: 0,
|
|
retry_count: 0,
|
|
retried_request_count: 0,
|
|
account_provider_kind: None,
|
|
account_authentication_kind: None,
|
|
account_plan_type: None,
|
|
key: name.to_owned(),
|
|
name: name.to_owned(),
|
|
request_count,
|
|
success_count: request_count,
|
|
failure_count: 0,
|
|
attempt_count: request_count,
|
|
total_tokens: request_count.saturating_mul(100),
|
|
average_latency_ms: Some(100),
|
|
latency_p95_ms: Some(200),
|
|
cost_coverage: CostCoverage::default(),
|
|
costs: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn quarter_hour_start(value: DateTime<Utc>) -> DateTime<Utc> {
|
|
let elapsed = value.timestamp().rem_euclid(15 * 60);
|
|
value - Duration::seconds(elapsed) - Duration::nanoseconds(i64::from(value.nanosecond()))
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn admin_selects_dashboard_defaults_and_matching_usage_buckets_at_range_boundaries() {
|
|
let now = Utc::now();
|
|
for (duration, expected) in [
|
|
(Duration::days(2), Granularity::FifteenMinutes),
|
|
(Duration::days(2) + Duration::seconds(1), Granularity::Hour),
|
|
(Duration::days(31), Granularity::Hour),
|
|
(Duration::days(31) + Duration::seconds(1), Granularity::Day),
|
|
] {
|
|
let range = TimeRange::new(now - duration, now).unwrap();
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let services = observability_services(store.clone()).await;
|
|
services
|
|
.observability()
|
|
.dashboard_summary(range, TrendKind::Usage)
|
|
.await
|
|
.unwrap();
|
|
let query = store.summary_query.lock().unwrap().clone().unwrap();
|
|
assert_eq!(query.range, range);
|
|
assert_eq!(query.granularity, expected);
|
|
assert_eq!(query.account_limit, 4);
|
|
assert_eq!(query.recent_request_limit, 10);
|
|
assert_eq!(
|
|
query.recent_request_filter.outcome,
|
|
Some(gateway_admin::model::observability::RequestOutcome::Succeeded)
|
|
);
|
|
services
|
|
.observability()
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(*store.usage_granularities.lock().unwrap(), vec![expected]);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_summary_cache_should_coalesce_identical_queries_and_invalidate_deleted_generation() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.set_dashboard_delay(StdDuration::from_millis(20));
|
|
let services = observability_services(store.clone()).await;
|
|
let service = services.observability();
|
|
let filter = UsageFilter {
|
|
client_api_key_ref: Some("deleted-key".to_owned()),
|
|
..Default::default()
|
|
};
|
|
let (first, second) = tokio::join!(
|
|
service.usage_summary(range, filter.clone()),
|
|
service.usage_summary(range, filter.clone()),
|
|
);
|
|
first.expect("first summary");
|
|
second.expect("coalesced summary");
|
|
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 1);
|
|
{
|
|
let mut overview = store.overview.lock().unwrap();
|
|
overview.analytics.generation = 1;
|
|
overview.requests.request_count = 7;
|
|
}
|
|
let refreshed = service
|
|
.usage_summary(range, filter)
|
|
.await
|
|
.expect("generation refresh");
|
|
assert_eq!(refreshed.observation.analytics.generation, 1);
|
|
assert_eq!(refreshed.observation.requests.request_count, 7);
|
|
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 2);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the_first_load() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.set_dashboard_delay(StdDuration::from_millis(20));
|
|
let services = observability_services(store.clone()).await;
|
|
let service = services.observability();
|
|
let (first, second, ()) = tokio::join!(
|
|
service.usage_summary(range, UsageFilter::default()),
|
|
service.usage_summary(range, UsageFilter::default()),
|
|
async {
|
|
tokio::time::sleep(StdDuration::from_millis(5)).await;
|
|
store.overview.lock().unwrap().analytics.generation = 1;
|
|
},
|
|
);
|
|
assert_eq!(
|
|
first
|
|
.expect("first summary")
|
|
.observation
|
|
.analytics
|
|
.generation,
|
|
1
|
|
);
|
|
assert_eq!(
|
|
second
|
|
.expect("coalesced summary")
|
|
.observation
|
|
.analytics
|
|
.generation,
|
|
1
|
|
);
|
|
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_summary_cache_should_isolate_key_identity_and_filters() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let services = observability_services(store.clone()).await;
|
|
let service = services.observability();
|
|
for (key, model) in [
|
|
("key-a", None),
|
|
("key-a", None),
|
|
("key-b", None),
|
|
("key-a", Some("model-a")),
|
|
] {
|
|
service
|
|
.usage_summary(
|
|
range,
|
|
UsageFilter {
|
|
client_api_key_ref: Some(key.to_owned()),
|
|
model: model.map(str::to_owned),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
.expect("isolated summary");
|
|
}
|
|
let calls = store.usage_summary_filters.lock().unwrap();
|
|
assert_eq!(calls.len(), 3);
|
|
assert_eq!(calls[0].client_api_key_ref.as_deref(), Some("key-a"));
|
|
assert_eq!(calls[1].client_api_key_ref.as_deref(), Some("key-b"));
|
|
assert_eq!(calls[2].model.as_deref(), Some("model-a"));
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn usage_summary_cache_should_bound_waiting_after_failed_refresh_and_allow_recovery() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
store.set_dashboard_delay(store.query_timeout().mul_f64(0.7));
|
|
store.summary_fail.store(true, Ordering::SeqCst);
|
|
let services = observability_services(store.clone()).await;
|
|
let service = services.observability();
|
|
let started = tokio::time::Instant::now();
|
|
let (first, second, third, fourth) = tokio::join!(
|
|
service.usage_summary(range, UsageFilter::default()),
|
|
service.usage_summary(range, UsageFilter::default()),
|
|
service.usage_summary(range, UsageFilter::default()),
|
|
service.usage_summary(range, UsageFilter::default()),
|
|
);
|
|
assert!(first.is_err() && second.is_err() && third.is_err() && fourth.is_err());
|
|
assert!(
|
|
started.elapsed() <= store.query_timeout(),
|
|
"同键失败不能使后续请求逐个等待完整数据库超时: {:?}",
|
|
started.elapsed()
|
|
);
|
|
|
|
store.summary_fail.store(false, Ordering::SeqCst);
|
|
store.set_dashboard_delay(StdDuration::ZERO);
|
|
service
|
|
.usage_summary(range, UsageFilter::default())
|
|
.await
|
|
.expect("cancelled cache refresh releases its slot for recovery");
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn insights_cache_should_bound_stale_lifetime_without_renewing_failed_refreshes() {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
{
|
|
let mut overview = store.overview.lock().unwrap();
|
|
overview.analytics.state = AnalyticsState::Ready;
|
|
overview.analytics.generated_at = Utc::now();
|
|
overview.analytics.lag_seconds = 7;
|
|
overview.requests.request_count = 19;
|
|
}
|
|
let services = observability_services(store.clone()).await;
|
|
let service = services.observability();
|
|
service
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.unwrap();
|
|
store.insights_fail.store(true, Ordering::SeqCst);
|
|
tokio::time::advance(StdDuration::from_secs(16)).await;
|
|
let stale = service
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(stale.analytics.state, AnalyticsState::Stale);
|
|
assert!(stale.analytics.lag_seconds >= 7);
|
|
assert_eq!(stale.health.total_requests, 19);
|
|
tokio::time::advance(StdDuration::from_secs(45)).await;
|
|
assert!(
|
|
service
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn insights_cache_should_reject_deleted_building_and_old_generation_times() {
|
|
for reason in ["deleted", "building", "generated-too-long-ago"] {
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
{
|
|
let mut overview = store.overview.lock().unwrap();
|
|
overview.analytics.state = if reason == "building" {
|
|
AnalyticsState::Building
|
|
} else {
|
|
AnalyticsState::Ready
|
|
};
|
|
overview.analytics.generated_at = Utc::now()
|
|
- Duration::seconds(if reason == "generated-too-long-ago" {
|
|
61
|
|
} else {
|
|
0
|
|
});
|
|
}
|
|
let services = observability_services(store.clone()).await;
|
|
let service = services.observability();
|
|
service
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.unwrap();
|
|
store.insights_fail.store(true, Ordering::SeqCst);
|
|
if reason == "deleted" {
|
|
store.overview.lock().unwrap().analytics.generation += 1;
|
|
}
|
|
tokio::time::advance(StdDuration::from_secs(16)).await;
|
|
assert!(
|
|
service
|
|
.usage_insights(range, UsageFilter::default())
|
|
.await
|
|
.is_err(),
|
|
"{reason}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn usage_metric_cache_isolates_selections_and_filters() {
|
|
use gateway_admin::model::observability::UsageInsightMetric as Metric;
|
|
let range = observation_range(Utc::now());
|
|
let store = Arc::new(FixtureObservabilityStore::new(range));
|
|
let services = observability_services(store.clone()).await;
|
|
let metrics = [
|
|
Metric::Health,
|
|
Metric::Latency,
|
|
Metric::FirstToken,
|
|
Metric::Throughput,
|
|
Metric::Load,
|
|
Metric::Cost,
|
|
Metric::Savings,
|
|
Metric::Cache,
|
|
];
|
|
for _ in 0..2 {
|
|
for metric in metrics {
|
|
services
|
|
.observability()
|
|
.usage_metric(range, UsageFilter::default(), metric)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
}
|
|
assert_eq!(*store.metric_selections.lock().unwrap(), metrics);
|
|
assert_eq!(
|
|
*store.concurrency_selections.lock().unwrap(),
|
|
vec![false, false, false, false, true, false, false, false]
|
|
);
|
|
services
|
|
.observability()
|
|
.usage_metric(
|
|
range,
|
|
UsageFilter {
|
|
provider_kind: Some("openai".into()),
|
|
..Default::default()
|
|
},
|
|
Metric::Latency,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(store.metric_selections.lock().unwrap().len(), 9);
|
|
}
|