Files
codex-proxy-rs/backend/crates/gateway-api/tests/admin/observability/response.rs
T
dca9806961 feat(usage): filter request errors by origin and focus the detail table (#743)
* feat(usage): filter request errors by origin and focus the detail table

Co-authored-by: Codex <noreply@openai.com>

* fix(usage): preserve request columns and fill the error table

Keep reasoning effort separate and use the success table's column order. Let the error summary fill the space left by hidden columns and reuse the decoded upstream body when filtering saved errors.

Co-authored-by: Codex <noreply@openai.com>

---------

Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-11-19 13:40:05 +08:00

2205 lines
86 KiB
Rust

//! 验证观测响应的指标、展示投影与查询时间边界
use chrono::Utc;
use serde_json::json;
#[tokio::test]
async fn usage_summary_keeps_request_attempt_cost_and_average_latency_wire_fields() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Duration;
use gateway_admin::model::observability::{
AttemptMetrics, CostCoverage, CurrencyCost, RequestMetrics, TimeRange, UsageOverview,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
for (sum, count, display, raw) in [
(404, 3, "134 ms", Some(134)),
(0, 1, "0 ms", Some(0)),
(0, 0, "—", None),
] {
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
fixture.observations.lock().unwrap().summary = Some(UsageOverview {
analytics: gateway_admin::model::observability::AnalyticsMetadata {
generated_at: now,
..Default::default()
},
range: TimeRange::new(now - Duration::hours(1), now).unwrap(),
requests: RequestMetrics {
request_count: 5,
success_count: 4,
cancelled_count: 1,
caller_error_count: 1,
input_tokens: 400,
output_tokens: 80,
cached_input_tokens: 30,
cache_write_input_tokens: 7,
reasoning_output_tokens: 9,
total_tokens: 480,
latency_sum_ms: sum,
latency_count: count,
..Default::default()
},
attempts: AttemptMetrics {
attempt_count: 6,
success_count: 4,
failure_count: 1,
cancelled_count: 1,
rate_limited_count: 1,
cost_coverage: CostCoverage {
provider_reported_count: 3,
unavailable_count: 1,
..Default::default()
},
costs: vec![CurrencyCost {
currency: "USD".into(),
amount: "1.25".parse().unwrap(),
}],
..Default::default()
},
providers: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records/summary?provider=openai")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_summary_projection")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(status, StatusCode::OK, "{value}");
assert_eq!(
value["data"],
json!({
"analytics": value["data"]["analytics"],
"totalRequests": "5", "inputTokens": "400", "outputTokens": "80",
"cachedInputTokens": "30", "cacheWriteInputTokens": "7", "totalTokens": "480",
"averageLatencyMs": display,
"averageLatencyMsValue": raw,
"logicalRequests": {"requestCount":5,"successCount":4,"failureCount":0,"cancelledCount":1,"incompleteCount":0,"callerErrorCount":1,"inputTokens":400,"outputTokens":80,"cachedInputTokens":30,"cacheWriteInputTokens":7,"reasoningOutputTokens":9,"totalTokens":480},
"attempts": {"attemptCount":6,"successCount":4,"failureCount":1,"cancelledCount":1,"incompleteCount":0,"rateLimitedCount":1,"authFailureCount":0,"provider5xxCount":0,"costCoverage":{"known":3,"partial":0,"unknown":1,"notBillable":0},"costs":[{"currency":"USD","estimatedAmount":"1.25"}]}
})
);
let observations = fixture.observations.lock().unwrap();
assert_eq!(observations.summaries.len(), 1);
assert_eq!(
observations.summaries[0].1.provider_kind.as_deref(),
Some("openai")
);
}
}
#[tokio::test]
async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headline_count() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, DashboardObservation, TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
yesterday: Default::default(),
analytics: gateway_admin::model::observability::AnalyticsMetadata {
coverage_start: Some(now - Duration::hours(1)),
coverage_end: Some(now),
generated_at: now,
lag_seconds: 12,
state: gateway_admin::model::observability::AnalyticsState::Lagging,
percentiles_approximate: true,
schema_version: 1,
histogram_version: 1,
generation: 42,
},
runtime_slots: None,
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
totals: Default::default(),
provider_accounts: AccountPoolMetrics {
total: 1002,
normal: 943,
quota_paused: 0,
quota_exhausted: 52,
rate_limited: 0,
disabled: 3,
error: 4,
},
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_dashboard_account_summary")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("dashboard response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
assert_eq!(status, StatusCode::OK, "{value}");
assert_eq!(value["data"]["analytics"]["state"], "lagging");
assert_eq!(value["data"]["analytics"]["lagSeconds"], 12);
assert_eq!(value["data"]["analytics"]["percentilesApproximate"], true);
assert_eq!(value["data"]["analytics"]["schemaVersion"], 1);
assert_eq!(value["data"]["analytics"]["histogramVersion"], 1);
assert!(value["data"]["analytics"]["coverageStart"].is_string());
assert!(value["data"]["analytics"]["generatedAtDisplay"].is_string());
assert!(value["data"]["analytics"].get("generation").is_none());
assert_eq!(
(
&value["data"]["cards"]["credentials"]["totalValue"],
&value["data"]["cards"]["credentials"]["availableValue"],
&value["data"]["cards"]["credentials"]["unavailableValue"],
),
(
&serde_json::json!(1002),
&serde_json::json!(943),
&serde_json::json!(59)
),
);
assert_eq!(
(
&value["data"]["cards"]["credentials"]["total"],
&value["data"]["cards"]["credentials"]["available"],
&value["data"]["cards"]["credentials"]["unavailable"],
),
(
&serde_json::json!("1002"),
&serde_json::json!("943"),
&serde_json::json!("59")
),
);
}
#[tokio::test]
async fn dashboard_summary_should_keep_yesterday_comparison_separate_from_today_and_lifetime() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, DashboardObservation, DashboardTotals, RequestMetrics, TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
yesterday: RequestMetrics {
request_count: 7,
input_tokens: 400,
cached_input_tokens: 200,
total_tokens: 500,
..Default::default()
},
analytics: Default::default(),
runtime_slots: None,
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
totals: DashboardTotals {
request_count: 42,
input_tokens: 800,
cached_input_tokens: 200,
total_tokens: 840,
billing_usd: None,
},
provider_accounts: AccountPoolMetrics::default(),
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_dashboard_lifetime_totals")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("dashboard response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
assert_eq!(status, StatusCode::OK, "{value}");
let cards = &value["data"]["cards"];
assert_eq!(cards["traffic"]["todayRequestsValue"], 0);
assert_eq!(cards["traffic"]["yesterdayRequestsValue"], 7);
assert_eq!(cards["tokens"]["yesterdayTokensValue"], 500);
assert_eq!(cards["cache"]["yesterdayHitRateValue"], 0.5);
assert_eq!(
value["data"]["analytics"]["coverageStartDisplay"],
serde_json::Value::Null
);
assert_eq!(
value["data"]["analytics"]["coverageEndDisplay"],
serde_json::Value::Null
);
assert_eq!(
(
&value["data"]["cards"]["traffic"]["totalRequests"],
&value["data"]["cards"]["tokens"]["totalTokens"],
&value["data"]["cards"]["cache"]["totalHitRate"],
&value["data"]["cards"]["cache"]["totalCachedInputTokens"],
),
(
&serde_json::json!("42"),
&serde_json::json!("840"),
&serde_json::json!("25.0%"),
&serde_json::json!("200"),
),
);
}
#[tokio::test]
async fn dashboard_summary_should_default_to_current_deployment_day() {
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, DashboardObservation, TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
yesterday: Default::default(),
analytics: Default::default(),
runtime_slots: None,
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
totals: Default::default(),
provider_accounts: AccountPoolMetrics::default(),
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_dashboard_today_range")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let range = fixture
.dashboard_summary_range
.lock()
.expect("dashboard summary range")
.take()
.expect("recorded dashboard summary range");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
range.start,
gateway_core::time::DeploymentTimeZone::default()
.day_start(range.end)
.unwrap()
);
}
#[tokio::test]
async fn usage_detail_should_keep_attempt_snapshot_contract() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::{UsageAttempt, UsageDetail};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
let mut record = usage_record_with_account(
"req_detail",
"acct_snap_a",
"Snapshot Alpha",
"alpha@example.invalid",
"oauth",
now,
);
record.first_token_ms = Some(20);
record.upstream_response_ms = Some(1_000);
record.upstream_api_overhead_ms = Some(120.25);
record.upstream_engine_ms = Some(6400.0);
record.outcome = gateway_admin::model::observability::RequestOutcome::Failed;
record.error_kind = Some("continuation_recovery_required".to_owned());
record.provider_error_code = Some("previous_response_not_found".to_owned());
record.continuation_unavailable_reason = Some("reused_connection_lost".to_owned());
record.upstream_connection_exit_reason = Some("max_age_expired".to_owned());
record.recovered = true;
record.upstream_error = Some(gateway_admin::model::observability::UsageUpstreamError {
code: Some("previous_response_not_found".to_owned()),
error_type: Some("invalid_request_error".to_owned()),
message: Some("Previous response is unavailable.".to_owned()),
param: Some("previous_response_id".to_owned()),
close_reason: None,
last_event_type: None,
});
record.retry_after_ms = Some(206_000);
fixture
.usage_detail
.lock()
.expect("usage detail")
.replace(UsageDetail {
trace: None,
related_requests: Vec::new(),
request: record,
attempts: vec![UsageAttempt {
source: "ops_event".to_owned(),
id: "ops_detail".to_owned(),
attempt_index: 1,
component: "routing".to_owned(),
operation: "fallback".to_owned(),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some("acct_snap_b".to_owned()),
provider_account_name: Some("Snapshot Beta".to_owned()),
provider_account_email: Some("beta@example.invalid".to_owned()),
provider_account_authentication_kind: Some("api_key".to_owned()),
upstream_model_id: Some("upstream-b".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
upstream_send_state: Some("sent".to_owned()),
outcome: gateway_admin::model::observability::RequestOutcome::Failed,
downstream_committed: None,
status_code: Some(429),
provider_error_code: Some("rate_limit".to_owned()),
failure_kind: Some("rate_limited".to_owned()),
retry_after_ms: Some(1_000),
upstream_request_id: None,
latency_ms: Some(120),
message: Some("limited".to_owned()),
input_tokens: None,
output_tokens: None,
cached_input_tokens: None,
cache_write_input_tokens: None,
reasoning_output_tokens: None,
total_tokens: None,
cost_source: Some("unavailable".to_owned()),
cost_amount: None,
cost_currency: None,
occurred_at: now,
}],
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records/detail?id=req_detail")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_detail_snapshot")
.body(Body::empty())
.expect("usage detail request"),
)
.await
.expect("usage detail response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("usage detail body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage detail JSON");
assert_eq!(value["data"]["errorOrigin"], "upstream");
assert!(value["data"]["attempts"][0]["downstreamCommitted"].is_null());
assert_eq!(value["data"]["errorKind"], "continuation_recovery_required");
assert_eq!(value["data"]["errorCode"], "previous_response_not_found");
assert_eq!(
value["data"]["upstreamError"]["message"],
"Previous response is unavailable."
);
assert_eq!(
value["data"]["upstreamError"]["errorType"],
"invalid_request_error"
);
assert_eq!(
value["data"]["upstreamError"]["param"],
"previous_response_id"
);
assert_eq!(value["data"]["retryAfterMs"], 206_000);
assert_eq!(
value["data"]["continuationUnavailableReason"],
"reused_connection_lost"
);
assert_eq!(
value["data"]["upstreamConnectionExitReason"],
"max_age_expired"
);
assert_eq!(value["data"]["websocketCloseCode"], serde_json::Value::Null);
assert_eq!(value["data"]["recovered"], true);
assert_eq!(value["data"]["firstTokenLatencyMs"], 20);
assert_eq!(value["data"]["latencyDetails"]["upstreamResponseMs"], 1_000);
assert_eq!(
value["data"]["latencyDetails"]["upstreamApiOverheadMs"],
json!(120.25)
);
assert_eq!(
value["data"]["latencyDetails"]["upstreamEngineMs"],
json!(6400.0)
);
assert_eq!(
serde_json::json!({
"accountId": value["data"]["accountId"],
"accountName": value["data"]["accountName"],
"accountEmail": value["data"]["accountEmail"],
"authenticationKind": value["data"]["authenticationKind"],
"attemptCredentialName": value["data"]["attempts"][0]["credentialName"],
"attemptAccountId": value["data"]["attempts"][0]["accountId"],
"attemptAccountName": value["data"]["attempts"][0]["accountName"],
"attemptAccountEmail": value["data"]["attempts"][0]["accountEmail"],
"attemptAuthenticationKind": value["data"]["attempts"][0]["authenticationKind"],
}),
serde_json::json!({
"accountId": "acct_snap_a",
"accountName": "Snapshot Alpha",
"accountEmail": "alpha@example.invalid",
"authenticationKind": "oauth",
"attemptCredentialName": "Snapshot Beta",
"attemptAccountId": "acct_snap_b",
"attemptAccountName": "Snapshot Beta",
"attemptAccountEmail": "beta@example.invalid",
"attemptAuthenticationKind": "api_key",
})
);
}
#[tokio::test]
async fn zero_attempt_failure_detail_omits_trace_and_keeps_missing_upstream_facts() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use gateway_admin::model::observability::{RequestOutcome, UsageDetail};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let mut record = usage_record_with_account("req_early", "", "", "", "", Utc::now());
record.provider_kind = None;
record.provider_account_ref = None;
record.provider_account_name = None;
record.provider_account_email = None;
record.provider_account_authentication_kind = None;
record.upstream_model_id = None;
record.upstream_transport = None;
record.http_version = None;
record.attempt_count = 0;
record.upstream_send_state = "not_sent".to_owned();
record.downstream_committed_at = None;
record.outcome = RequestOutcome::Failed;
record.client_status_code = Some(503);
record.upstream_status_code = None;
record.error_kind = Some("no_available_provider".to_owned());
record.error_message = Some("no upstream account is available".to_owned());
record.input_tokens = None;
record.output_tokens = None;
record.cached_input_tokens = None;
record.cache_write_input_tokens = None;
record.reasoning_output_tokens = None;
record.image_input_tokens = None;
record.image_output_tokens = None;
record.total_tokens = None;
let trace = json!({
"events": [
{"stage": "attempt.started", "attemptIndex": 1},
{"stage": "attempt.failed", "attemptIndex": 1},
{"stage": "request.finished", "data": {"attempts": 0}},
]
});
fixture.usage_detail.lock().unwrap().replace(UsageDetail {
request: record,
attempts: Vec::new(),
trace: Some(trace.clone()),
related_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records/detail?id=req_early")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_zero_attempt_detail")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
let data = &value["data"];
assert_eq!(data["requestId"], "req_early");
assert_eq!(data["attemptCount"], 0);
assert_eq!(data["attempts"], json!([]));
// 明细仍沿用全局 best-effort 合同,不从预备 trace 推断完整尝试列表
assert_eq!(data["attemptsComplete"], false);
assert!(data.get("trace").is_none());
assert!(data.get("relatedRequests").is_none());
assert_eq!(data["logicalOutcome"], "failed");
assert_eq!(data["clientStatusCode"], 503);
assert_eq!(data["message"], "no upstream account is available");
assert_eq!(data["model"], "coding");
for field in [
"provider",
"accountId",
"upstreamModel",
"upstreamTransport",
"inputTokens",
"outputTokens",
] {
assert_eq!(data.get(field), Some(&serde_json::Value::Null), "{field}");
}
}
#[tokio::test]
async fn ops_errors_should_keep_account_label_and_authentication_contract() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
fixture
.ops_errors
.lock()
.expect("ops errors")
.push(ops_error_record());
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/operations/errors")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_ops_errors_snapshot")
.body(Body::empty())
.expect("ops errors request"),
)
.await
.expect("ops errors response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("ops errors body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("ops errors JSON");
assert_eq!(value["data"]["items"][0]["clientApiKeyName"], "Production");
assert_eq!(
serde_json::json!({
"provider": value["data"]["items"][0]["provider"],
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
"kind": value["data"]["items"][0]["kind"],
"accountId": value["data"]["items"][0]["accountId"],
"accountLabel": value["data"]["items"][0]["metadata"]["accountLabel"],
"accountPlanType": value["data"]["items"][0]["accountPlanType"],
"accountPlanTypeDisplay": value["data"]["items"][0]["accountPlanTypeDisplay"],
"clientStatusCode": value["data"]["items"][0]["clientStatusCode"],
"route": value["data"]["items"][0]["route"],
"requestedModel": value["data"]["items"][0]["requestedModel"],
"clientTransport": value["data"]["items"][0]["clientTransport"],
"clientIp": value["data"]["items"][0]["clientIp"],
"userAgent": value["data"]["items"][0]["userAgent"],
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
"upstreamSendState": value["data"]["items"][0]["upstreamSendState"],
"errorDetails": value["data"]["items"][0]["errorDetails"],
"continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"],
"upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"],
"upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"],
}),
serde_json::json!({
"provider": "openai",
"authenticationKind": "api_key",
"kind": "model_request",
"accountId": "acct_err",
"accountLabel": "err@example.invalid",
"accountPlanType": "pro",
"accountPlanTypeDisplay": "Pro",
"clientStatusCode": 502,
"route": "/v1/responses",
"requestedModel": "gpt-5.4",
"clientTransport": "http_sse",
"clientIp": "203.0.113.8",
"userAgent": "codex-cli/0.144.0",
"providerErrorCode": "upstream",
"upstreamSendState": "sent",
"errorDetails": serde_json::to_string_pretty(&json!({"error": {"code": "upstream", "message": "raw upstream marker"}})).unwrap(),
"continuationUnavailableReason": "reused_connection_lost",
"upstreamConnectionExitReason": "tcp_reset",
"upstreamConnectionAgeMs": 12000,
})
);
}
fn peer_summary(
discovered: bool,
observed: bool,
) -> gateway_admin::model::provider_credentials::ProviderQuota {
gateway_admin::model::provider_credentials::ProviderQuota {
upstream: Some(gateway_admin::model::upstream_info::ProviderUpstreamInfo {
max_concurrency: None,
requests_per_minute: None,
discovered,
observed_at: observed.then(chrono::Utc::now),
stale: true,
connection_revision: "test-connection".to_owned(),
last_error: None,
pool: None,
}),
..Default::default()
}
}
#[tokio::test]
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use gateway_admin::model::observability::{
CostCoverage, DiagnosticObservation, DiagnosticsObservation,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
for (authentication_kind, plan_type, plan_display, peer) in [
(Some("oauth"), Some("pro"), Some("Pro"), None),
(Some("api_key"), None, None, None),
(Some("api_key"), None, None, Some((true, true))),
(Some("api_key"), None, None, Some((true, false))),
(Some("api_key"), None, None, Some((false, true))),
(None, None, None, None),
] {
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
*fixture.provider_quota.lock().unwrap() = peer.map(|(d, o)| peer_summary(d, o));
*fixture.diagnostics.lock().expect("diagnostics") = DiagnosticsObservation {
analytics: Default::default(),
total_request_count: 4,
items: vec![DiagnosticObservation {
key: "acct_diag".to_owned(),
account_provider_kind: Some("openai".to_owned()),
account_authentication_kind: authentication_kind.map(str::to_owned),
account_plan_type: plan_type.map(str::to_owned),
name: "diag@example.invalid".to_owned(),
request_count: 2,
success_count: 2,
failure_count: 0,
attempt_count: 5,
total_tokens: 200,
average_latency_ms: Some(100),
latency_p95_ms: Some(3800),
first_token_p95_ms: Some(1200),
cancelled_count: 0,
incomplete_count: 0,
caller_error_count: 0,
retry_count: 3,
retried_request_count: 1,
cost_coverage: CostCoverage::default(),
costs: Vec::new(),
}],
};
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/insights/diagnostics?dimension=account")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_diagnostics_snapshot")
.body(Body::empty())
.expect("diagnostics request"),
)
.await
.expect("diagnostics response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("diagnostics body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
assert_eq!(
value["data"]["items"][0]["accountUpstreamPeer"],
json!(peer.map(|(d, o)| d && o))
);
assert_eq!(value["data"]["items"][0]["requestShare"], 0.5);
assert_eq!(value["data"]["items"][0]["retryCount"], 3);
assert_eq!(value["data"]["items"][0]["retryRate"], 0.5);
assert_eq!(
value["data"]["items"][0]["authenticationKind"],
serde_json::json!(authentication_kind)
);
assert_eq!(
value["data"]["items"][0]["accountPlanType"],
serde_json::json!(plan_type)
);
assert_eq!(
value["data"]["items"][0]["accountPlanTypeDisplay"],
serde_json::json!(plan_display)
);
assert_eq!(
(
&value["data"]["items"][0]["key"],
&value["data"]["items"][0]["name"],
&value["data"]["items"][0]["latencyP95Ms"],
&value["data"]["dimension"],
),
(
&serde_json::json!("acct_diag"),
&serde_json::json!("diag@example.invalid"),
&serde_json::json!(3800),
&serde_json::json!("account"),
)
);
}
}
pub(super) fn usage_record_with_account(
id: &str,
account_id: &str,
name: &str,
email: &str,
authentication_kind: &str,
started_at: chrono::DateTime<chrono::Utc>,
) -> gateway_admin::model::observability::UsageRecord {
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
UsageRecord {
upstream_error: None,
id: id.to_owned(),
client_api_key_ref: "key_detail".to_owned(),
config_revision: 1,
routing_scope: "all".to_owned(),
routing_group_refs: Vec::new(),
routing_group_names_snapshot: Vec::new(),
protocol: "openai".to_owned(),
operation: "responses".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "http_sse".to_owned(),
requested_model_id: Some("coding".to_owned()),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some(account_id.to_owned()),
provider_account_name: Some(name.to_owned()),
provider_account_email: Some(email.to_owned()),
provider_account_authentication_kind: Some(authentication_kind.to_owned()),
upstream_model_id: Some("upstream-model".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
http_version: Some("h2".to_owned()),
websocket_pool: None,
service_tier: None,
provider_metadata_json: None,
attempt_count: 1,
upstream_send_state: "sent".to_owned(),
downstream_committed_at: Some(started_at),
outcome: RequestOutcome::Succeeded,
client_status_code: Some(200),
upstream_status_code: Some(200),
client_response_id: None,
upstream_request_id: None,
upstream_response_id: None,
websocket_close_code: None,
continuation_unavailable_reason: None,
upstream_connection_exit_reason: None,
recovered: false,
error_kind: None,
provider_error_code: None,
error_message: None,
retry_after_ms: None,
input_tokens: Some(1),
output_tokens: Some(1),
cached_input_tokens: Some(0),
cache_write_input_tokens: Some(0),
reasoning_output_tokens: Some(0),
image_input_tokens: Some(0),
image_output_tokens: Some(0),
total_tokens: Some(2),
cost_source: "unavailable".to_owned(),
cost_amount: None,
cost_currency: None,
billing: None,
transport_decision_wait_ms: None,
connect_ms: None,
headers_ms: None,
first_event_ms: None,
first_reasoning_ms: None,
first_text_ms: None,
first_token_ms: None,
provider_processing_ms: None,
upstream_response_ms: None,
upstream_api_overhead_ms: None,
upstream_engine_ms: None,
latency_ms: None,
admission_decision_ms: None,
account_selection_wait_ms: None,
capacity_used_slots: None,
capacity_total_slots: None,
client_ip: None,
user_agent: None,
reasoning_effort: None,
reasoning_preset: None,
request_kind: None,
subagent_kind: None,
compact: false,
image_generation_requested: false,
image_generation_succeeded: None,
started_at,
deadline_at: started_at + chrono::Duration::seconds(30),
completed_at: Some(started_at),
}
}
#[tokio::test]
async fn usage_route_should_forward_record_text_filters() {
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records?model=%20coding%20&clientApiKeyName=%20Key%20%25%20&accountSearch=%20alice%40%20")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_other_outcome")
.body(Body::empty())
.expect("usage request"),
)
.await
.expect("usage response");
assert_eq!(response.status(), StatusCode::OK);
let observations = fixture.observations.lock().unwrap();
let filter = &observations.records.last().expect("usage query").filter;
assert_eq!(filter.client_api_key_name.as_deref(), Some("Key %"));
assert_eq!(filter.model_contains.as_deref(), Some("coding"));
assert_eq!(filter.provider_account_search.as_deref(), Some("alice@"));
assert!(filter.client_api_key_ref.is_none());
assert!(filter.provider_account_ref.is_none());
}
#[tokio::test]
async fn usage_route_projects_request_facts_and_billing() {
use std::str::FromStr as _;
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::{
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, UsageBilling, UsageListRecord,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let usd = |amount: &str| CurrencyCost {
currency: "USD".to_owned(),
amount: DecimalAmount::from_str(amount).expect("USD amount"),
};
fixture
.usage_records
.lock()
.expect("usage records")
.push(UsageListRecord {
provider_account_upstream_peer: None,
has_upstream_error: false,
error_code: None,
error_message: None,
error_kind: None,
websocket_close_code: None,
continuation_unavailable_reason: None,
upstream_connection_exit_reason: None,
recovered: false,
client_status_code: None,
upstream_status_code: None,
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
client_api_key_name: Some("Production".to_owned()),
id: "request_endpoint".to_owned(),
endpoint: "/v1/responses".to_owned(),
client_transport: "websocket".to_owned(),
requested_model_id: Some("grok-4.5".to_owned()),
provider_kind: Some("xai".to_owned()),
provider_account_ref: Some("acct_snapshot".to_owned()),
provider_account_name: Some("Snapshot Alpha".to_owned()),
provider_account_email: Some("alpha@example.invalid".to_owned()),
provider_account_notes: Some("Team workspace".to_owned()),
provider_account_plan_type: Some("supergrok_plus".to_owned()),
provider_account_plan_type_display: None,
provider_account_authentication_kind: Some("oauth".to_owned()),
upstream_model_id: Some("grok-4.5".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
service_tier: Some("default".to_owned()),
input_tokens: Some(1),
output_tokens: Some(1),
cached_input_tokens: Some(0),
cache_write_input_tokens: Some(0),
reasoning_output_tokens: Some(0),
image_input_tokens: Some(31),
image_output_tokens: Some(9),
total_tokens: Some(2),
cost_source: "unavailable".to_owned(),
cost_amount: None,
cost_currency: None,
billing: Some(UsageBilling::Calculated(Box::new(
CalculatedBillingBreakdown {
long_context_billing_applied: true,
custom_multiplier_bps: 10_000,
image: None,
input_amount: usd("0.03"),
output_amount: usd("0.07"),
cache_read_amount: usd("0.00"),
cache_write_amount: usd("0.00"),
standard_amount: usd("0.10"),
total_amount: usd("0.10"),
input_price_per_million: usd("10.0000"),
output_price_per_million: usd("60.0000"),
cache_read_price_per_million: usd("1.0000"),
cache_write_price_per_million: usd("12.5000"),
service_tier: Some("default".to_owned()),
multiplier_percent: 100,
},
))),
transport_decision_wait_ms: Some(4),
connect_ms: Some(5),
headers_ms: Some(7),
first_event_ms: Some(17),
first_reasoning_ms: Some(18),
first_text_ms: Some(19),
first_token_ms: Some(20),
provider_processing_ms: Some(21),
upstream_response_ms: Some(2_000),
upstream_api_overhead_ms: None,
upstream_engine_ms: None,
latency_ms: Some(31),
admission_decision_ms: Some(1),
account_selection_wait_ms: Some(2),
capacity_used_slots: Some(3),
capacity_total_slots: Some(8),
client_ip: Some("192.0.2.10".to_owned()),
user_agent: Some("codex-cli/1.0".to_owned()),
reasoning_effort: Some("max".to_owned()),
reasoning_preset: Some("ultra".to_owned()),
subagent_kind: Some("worker".to_owned()),
compact: true,
started_at: Utc::now(),
});
{
let mut records = fixture.usage_records.lock().expect("usage records");
for source in ["calculated", "provider_reported"] {
let mut image = records[0].clone();
image.id = format!("image_{source}");
image.endpoint = "/v1/images/generations".to_owned();
image.provider_kind = Some("openai".to_owned());
image.requested_model_id = Some("gpt-image-2".to_owned());
image.upstream_model_id = None;
image.billing = Some(UsageBilling::Total {
source: source.to_owned(),
total: usd("0.00696"),
});
records.push(image);
}
let mut failed = records[0].clone();
failed.id = "request_failed".to_owned();
failed.outcome = gateway_admin::model::observability::RequestOutcome::Failed;
failed.client_status_code = Some(503);
failed.upstream_status_code = Some(429);
failed.error_code = Some("server_error".to_owned());
failed.error_message = Some("The upstream service is temporarily unavailable.".to_owned());
records.push(failed);
let mut closed = records[0].clone();
closed.id = "request_ws_closed".to_owned();
closed.outcome = gateway_admin::model::observability::RequestOutcome::Incomplete;
closed.error_kind = Some("upstream_unavailable".to_owned());
closed.websocket_close_code = Some(1012);
closed.error_message =
Some("OpenAI WebSocket closed before a terminal response".to_owned());
records.push(closed);
}
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records?status=all")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_endpoint")
.body(Body::empty())
.expect("usage request"),
)
.await
.expect("usage response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("usage response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
assert!(value["data"]["items"][0]["errorCode"].is_null());
assert!(value["data"]["items"][0]["errorMessage"].is_null());
assert_eq!(value["data"]["items"][3]["errorCode"], "server_error");
assert_eq!(value["data"]["items"][3]["clientStatusCode"], 503);
assert_eq!(value["data"]["items"][3]["upstreamStatusCode"], 429);
assert_eq!(value["data"]["items"][3]["errorOrigin"], "upstream");
assert_eq!(value["data"]["items"][4]["errorOrigin"], "upstream");
assert_eq!(value["data"]["items"][0]["errorOrigin"], "unknown");
assert_eq!(
value["data"]["items"][4].get("upstreamStatusCode"),
Some(&serde_json::Value::Null)
);
assert_eq!(
value["data"]["items"][4].get("clientStatusCode"),
Some(&serde_json::Value::Null)
);
assert_eq!(value["data"]["items"][4]["logicalOutcome"], "incomplete");
assert_eq!(
value["data"]["items"][4]["errorKind"],
"upstream_unavailable"
);
assert_eq!(value["data"]["items"][4]["websocketCloseCode"], 1012);
assert_eq!(
value["data"]["items"][4]["errorCode"],
serde_json::Value::Null
);
assert_eq!(value["data"]["items"][4]["recovered"], false);
assert_eq!(
value["data"]["items"][3]["errorMessage"],
"The upstream service is temporarily unavailable."
);
assert_eq!(value["data"]["items"][0]["clientApiKeyName"], "Production");
assert_eq!(value["data"]["items"][0]["accountNotes"], "Team workspace");
assert_eq!(
value["data"]["items"][0]["accountPlanType"],
"supergrok_plus"
);
assert_eq!(
value["data"]["items"][0]["accountPlanTypeDisplay"],
"SupergrokPlus"
);
assert_eq!(
value["data"]["items"][0]["billing"]["longContextBillingApplied"],
true
);
assert_eq!(
value["data"]["items"][1]["billing"]["longContextBillingApplied"],
false
);
assert_eq!(
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
"$10 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
"$60 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
"$12.5 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["serviceTierDisplay"],
"Standard"
);
assert_eq!(
serde_json::json!({
"route": value["data"]["items"][0]["route"],
"serviceTier": value["data"]["items"][0]["serviceTier"],
"accountId": value["data"]["items"][0]["accountId"],
"accountName": value["data"]["items"][0]["accountName"],
"accountEmail": value["data"]["items"][0]["accountEmail"],
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
"requestedModel": value["data"]["items"][0]["requestedModel"],
"upstreamModel": value["data"]["items"][0]["upstreamModel"],
"reasoningEffort": value["data"]["items"][0]["reasoningEffort"],
"reasoningPreset": value["data"]["items"][0]["reasoningPreset"],
"subagentKind": value["data"]["items"][0]["subagentKind"],
"compact": value["data"]["items"][0]["compact"],
"clientTransport": value["data"]["items"][0]["clientTransport"],
"upstreamTransport": value["data"]["items"][0]["upstreamTransport"],
"latencyDetails": value["data"]["items"][0]["latencyDetails"],
"firstTokenLatencyMs": value["data"]["items"][0]["firstTokenLatencyMs"],
"latencyMs": value["data"]["items"][0]["latencyMs"],
"clientIp": value["data"]["items"][0]["clientIp"],
"userAgent": value["data"]["items"][0]["userAgent"],
}),
serde_json::json!({
"route": "/v1/responses",
"serviceTier": "default",
"accountId": "acct_snapshot",
"accountName": "Snapshot Alpha",
"accountEmail": "alpha@example.invalid",
"authenticationKind": "oauth",
"imageInputTokens": 31,
"imageOutputTokens": 9,
"requestedModel": "grok-4.5",
"upstreamModel": "grok-4.5",
"reasoningEffort": "max",
"reasoningPreset": "ultra",
"subagentKind": "worker",
"compact": true,
"clientTransport": "websocket",
"upstreamTransport": "http_sse",
"latencyDetails": {
"admissionDecisionMs": 1,
"accountSelectionWaitMs": 2,
"capacityUsedSlots": 3,
"capacityTotalSlots": 8,
"transportDecisionWaitMs": 4,
"wsConnectMs": 5,
"upstreamHeadersMs": 7,
"firstEventMs": 17,
"firstReasoningMs": 18,
"firstTextMs": 19,
"openaiProcessingMs": 21,
"upstreamResponseMs": 2000,
},
"firstTokenLatencyMs": 20,
"latencyMs": 31,
"clientIp": "192.0.2.10",
"userAgent": "codex-cli/1.0",
})
);
assert_eq!(
value["data"]["items"][1]["billing"]["totalAmountDisplay"],
"≈ $0.007"
);
assert_eq!(
value["data"]["items"][2]["billing"]["totalAmountDisplay"],
"$0.007"
);
// 互联标识随当前摘要变化,重复请求共享同一账号,既有请求快照保持原值
for peer in [
Some((true, true)),
Some((true, false)),
Some((false, true)),
None,
] {
*fixture.provider_quota.lock().unwrap() = peer.map(|(d, o)| peer_summary(d, o));
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/records?status=all")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_peer_badge")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
for item in value["data"]["items"].as_array().unwrap() {
assert_eq!(
item["accountUpstreamPeer"],
json!(peer.map(|(d, o)| d && o))
);
assert_eq!(item["accountName"], "Snapshot Alpha");
assert_eq!(item["accountPlanType"], "supergrok_plus");
}
}
}
#[tokio::test]
async fn calendar_queries_and_display_follow_deployment_timezone_with_one_anchor() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use gateway_admin::model::observability::{CalendarPeriod, DashboardObservation, TimeRange};
use gateway_api::admin::observability;
use gateway_core::time::DeploymentTimeZone;
use tower::ServiceExt as _;
for (name, anchor, start, display, slots) in [
(
"Asia/Shanghai",
"2026-10-01T00:00:00Z",
"2026-09-30T16:00:00Z",
"2026-10-01 08:00:00",
96,
),
(
"UTC",
"2026-10-01T00:00:00Z",
"2026-10-01T00:00:00Z",
"2026-10-01 00:00:00",
96,
),
(
"Asia/Kathmandu",
"2026-10-01T00:00:00Z",
"2026-09-30T18:15:00Z",
"2026-10-01 05:45:00",
96,
),
(
"America/New_York",
"2026-03-09T03:59:00Z",
"2026-03-08T05:00:00Z",
"2026-03-08 23:59:00",
92,
),
(
"America/New_York",
"2026-11-02T04:59:00Z",
"2026-11-01T04:00:00Z",
"2026-11-01 23:59:00",
100,
),
] {
let timezone: DeploymentTimeZone = name.parse().unwrap();
let end: chrono::DateTime<Utc> = anchor.parse().unwrap();
let range = TimeRange::calendar_at(CalendarPeriod::Today, end, timezone).unwrap();
let fixture = AdminTestFixture::with_timezone(timezone).await;
fixture.auth.insert_session("valid-session");
*fixture.dashboard_observation.lock().unwrap() = Some(DashboardObservation {
yesterday: Default::default(),
analytics: gateway_admin::model::observability::AnalyticsMetadata {
coverage_start: Some(range.start),
coverage_end: Some(end),
generated_at: end,
..Default::default()
},
runtime_slots: None,
range,
totals: Default::default(),
provider_accounts: Default::default(),
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let router = observability::router::<AdminTestState>().with_state(fixture.state());
let response = router
.clone()
.oneshot(
Request::builder()
.uri(format!(
"/api/admin/dashboard/summary?period=today&asOf={}",
end.timestamp_millis()
))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_timezone_calendar")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let bytes = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
assert_eq!(
status,
StatusCode::OK,
"{name}: {}",
String::from_utf8_lossy(&bytes)
);
let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
let queried = fixture.dashboard_summary_range.lock().unwrap().unwrap();
assert_eq!(
queried.start,
start.parse::<chrono::DateTime<Utc>>().unwrap()
);
assert_eq!(queried.end, end);
assert_eq!(value["data"]["asOf"], anchor);
assert_eq!(value["data"]["asOfDisplay"], display);
let analytics = &value["data"]["analytics"];
assert_eq!(analytics["coverageStart"], start);
assert_eq!(analytics["coverageEnd"], anchor);
assert_eq!(
analytics["coverageStartDisplay"],
format!("{} 00:00:00", &display[..10])
);
assert_eq!(analytics["coverageEndDisplay"], display);
assert_eq!(analytics["generatedAtDisplay"], display);
let points = value["data"]["healthTimeline"]["points"]
.as_array()
.unwrap();
assert_eq!(points.len(), slots);
let labels = points
.iter()
.map(|p| p["time"].as_str().unwrap())
.collect::<Vec<_>>();
assert!(labels.iter().all(|label| label.len() == 5));
let buckets = points
.iter()
.map(|p| p["bucketStart"].as_str().unwrap())
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(buckets.len(), slots, "fold buckets remain distinct");
if slots == 100 {
assert_eq!(labels.iter().filter(|label| **label == "01:00").count(), 2);
}
let invalid = router.oneshot(Request::builder()
.uri(format!("/api/admin/dashboard/summary?period=today&asOf={}&startTime=2026-01-01T00%3A00%3A00Z", end.timestamp_millis()))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_timezone_mixed_range")
.body(Body::empty()).unwrap()).await.unwrap();
assert_eq!(invalid.status(), StatusCode::BAD_REQUEST);
}
}
#[tokio::test]
async fn calendar_query_rejects_an_anchor_outside_the_local_calendar() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
let fixture = AdminTestFixture::with_timezone(Default::default()).await;
fixture.auth.insert_session("valid-session");
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri(format!(
"/api/admin/dashboard/summary?period=today&asOf={}",
chrono::DateTime::<chrono::Utc>::MAX_UTC.timestamp_millis()
))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_timezone_out_of_range")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert!(fixture.dashboard_summary_range.lock().unwrap().is_none());
}
#[tokio::test]
async fn usage_insights_maps_load_summary_and_points_to_camel_case() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Duration;
use gateway_admin::model::observability::{
Granularity, LatencyPercentiles, PercentileMilliseconds, RequestMetricPoint,
RequestMetrics, TimeRange, UsageConcurrencyPoint, UsageOverview,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
let range = TimeRange::new(now - Duration::minutes(15), now).unwrap();
let bucket = range.start;
{
let mut observations = fixture.observations.lock().unwrap();
observations.summary = Some(UsageOverview {
analytics: Default::default(),
range,
requests: RequestMetrics {
admission_decision_percentiles: LatencyPercentiles {
p95_ms: Some(PercentileMilliseconds::new(12.5).unwrap()),
..Default::default()
},
account_selection_wait_percentiles: LatencyPercentiles {
p95_ms: Some(PercentileMilliseconds::new(128.75).unwrap()),
..Default::default()
},
..Default::default()
},
attempts: Default::default(),
providers: Vec::new(),
});
observations.trend = Some(vec![RequestMetricPoint {
bucket_start: bucket,
granularity: Granularity::FifteenMinutes,
metrics: RequestMetrics {
admission_decision_percentiles: LatencyPercentiles {
p95_ms: Some(PercentileMilliseconds::new(2.5).unwrap()),
..Default::default()
},
..Default::default()
},
cost_coverage: Default::default(),
costs: Vec::new(),
}]);
observations.concurrency = vec![UsageConcurrencyPoint {
bucket_start: bucket,
average_concurrency: 1.25,
peak_concurrency: 3,
}];
}
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/usage/insights/overview")
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_insights_load")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(status, StatusCode::OK, "{value}");
let performance = &value["data"]["performance"];
assert_eq!(
performance["load"],
json!({
"averageConcurrency": 1.25,
"peakConcurrency": 3,
"admissionDecisionP95Ms": 12.5,
"accountSelectionWaitP95Ms": 128.75,
})
);
assert_eq!(
performance["points"][0]["load"],
json!({
"averageConcurrency": 1.25,
"peakConcurrency": 3,
"admissionDecisionP95Ms": 2.5,
"accountSelectionWaitP95Ms": null,
})
);
}
#[tokio::test]
async fn usage_card_routes_return_only_the_selected_metric_and_validate_options() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Duration;
use gateway_admin::model::observability::{
AttemptMetrics, CurrencyCost, Granularity, LatencyPercentiles, PercentileMilliseconds,
RequestMetricPoint, RequestMetrics, TimeRange, UsageConcurrencyPoint, UsageOverview,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
let range = TimeRange::new(now - Duration::minutes(15), now).unwrap();
let metrics = RequestMetrics {
request_count: 10,
success_count: 8,
failure_count: 2,
latency_percentiles: LatencyPercentiles {
p95_ms: Some(PercentileMilliseconds::new(43.5).unwrap()),
..Default::default()
},
first_token_latency_percentiles: LatencyPercentiles {
p95_ms: Some(PercentileMilliseconds::new(17.25).unwrap()),
..Default::default()
},
output_throughput_p50: Some(22),
..Default::default()
};
{
let mut observations = fixture.observations.lock().unwrap();
observations.summary = Some(UsageOverview {
analytics: Default::default(),
range,
requests: metrics.clone(),
attempts: AttemptMetrics {
costs: vec![CurrencyCost {
currency: "USD".into(),
amount: "12.5".parse().unwrap(),
}],
..Default::default()
},
providers: Vec::new(),
});
observations.trend = Some(vec![RequestMetricPoint {
bucket_start: range.start,
granularity: Granularity::FifteenMinutes,
metrics,
cost_coverage: Default::default(),
costs: Vec::new(),
}]);
observations.concurrency = vec![UsageConcurrencyPoint {
bucket_start: range.start,
average_concurrency: 1.25,
peak_concurrency: 3,
}];
}
let router = observability::router::<AdminTestState>().with_state(fixture.state());
for (path, metric, field, expected) in [
("health", "health", "totalRequests", json!(10)),
(
"performance?metric=latency",
"latency",
"p95Ms",
json!(43.5),
),
(
"performance?metric=firstToken",
"firstToken",
"p95Ms",
json!(17.25),
),
(
"performance?metric=throughput",
"throughput",
"p50",
json!(22),
),
(
"performance?metric=load",
"load",
"peakConcurrency",
json!(3),
),
("cost?metric=cost", "cost", "estimatedCost", json!("12.5")),
(
"cost?metric=savings",
"savings",
"estimatedCost",
json!("12.5"),
),
("cost?metric=cache", "cache", "estimatedCost", json!("12.5")),
] {
let response = router
.clone()
.oneshot(
Request::builder()
.uri(format!("/api/admin/usage/insights/{path}"))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_card_metric")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(status, StatusCode::OK, "{path}: {value}");
let result = &value["data"];
assert_eq!(result["metric"], metric);
assert_eq!(result.as_object().unwrap().len(), 4);
let data = &result["data"];
if metric == "health" {
assert_eq!(data[field], expected);
} else {
assert_eq!(data["summary"][field], expected, "{path}");
assert_eq!(data["points"][0]["requestCount"], 10);
assert_eq!(data.as_object().unwrap().len(), 2);
}
}
for path in [
"performance",
"cost",
"performance?metric=cost",
"cost?metric=load",
"health?metric=load",
"performance?metric=unknown",
"cost?metric=cache&status=error",
] {
let response = router
.clone()
.oneshot(
Request::builder()
.uri(format!("/api/admin/usage/insights/{path}"))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_usage_card_metric")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}");
}
}
#[tokio::test]
async fn observability_queries_reject_group_filters_and_analytics_search() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::Body,
http::{Request, StatusCode, header},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let router = observability::router::<AdminTestState>().with_state(fixture.state());
for (path, parameter) in [
("/api/admin/usage/records", "routingGroupRefs=group-a"),
("/api/admin/operations/errors", "routingGroupRefs=group-a"),
(
"/api/admin/usage/records/summary",
"routingGroupRefs=group-a",
),
(
"/api/admin/usage/insights/overview",
"routingGroupRefs=group-a",
),
(
"/api/admin/usage/insights/diagnostics",
"routingGroupRefs=group-a",
),
("/api/admin/usage/records/summary", "search=request-prefix"),
("/api/admin/usage/records/summary", "errorOrigin=proxy"),
("/api/admin/usage/insights/overview", "errorOrigin=proxy"),
(
"/api/admin/usage/insights/overview",
"search=request-prefix",
),
(
"/api/admin/usage/insights/diagnostics",
"search=request-prefix",
),
] {
let response = router
.clone()
.oneshot(
Request::builder()
.uri(format!("{path}?{parameter}"))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "req_rejected_observability_filter")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::BAD_REQUEST,
"{path}?{parameter}"
);
}
let observations = fixture.observations.lock().unwrap();
assert!(observations.records.is_empty());
assert!(observations.errors.is_empty());
}
pub(super) fn ops_error_record() -> gateway_admin::model::observability::OpsError {
use gateway_admin::model::observability::OpsError;
OpsError {
client_api_key_name: Some("Production".to_owned()),
source: "model_request".to_owned(),
event_id: "err_snapshot".to_owned(),
request_id: Some("req_err".to_owned()),
attempt_index: Some(1),
client_api_key_ref: Some("key_err".to_owned()),
component: "model_request".to_owned(),
operation: "responses".to_owned(),
protocol: Some("openai".to_owned()),
client_transport: Some("http_sse".to_owned()),
requested_model_id: Some("gpt-5.4".to_owned()),
service_tier: Some("priority".to_owned()),
endpoint: Some("/v1/responses".to_owned()),
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some("acct_err".to_owned()),
provider_account_name: None,
provider_account_email: Some("err@example.invalid".to_owned()),
provider_account_plan_type: Some("pro".to_owned()),
provider_account_plan_type_display: Some("Pro".to_owned()),
provider_account_authentication_kind: Some("api_key".to_owned()),
upstream_model_id: Some("upstream-err".to_owned()),
upstream_transport: Some("http_sse".to_owned()),
failure_kind: "upstream_error".to_owned(),
upstream_send_state: Some("sent".to_owned()),
client_status_code: Some(502),
upstream_status_code: Some(502),
provider_error_code: Some("upstream".to_owned()),
client_response_id: None,
upstream_request_id: None,
latency_ms: Some(90),
message: "snapshot error".to_owned(),
error_details: Some(
r#"{"error":{"code":"upstream","message":"raw upstream marker"}}"#.to_owned(),
),
client_ip: Some("203.0.113.8".to_owned()),
user_agent: Some("codex-cli/0.144.0".to_owned()),
reasoning_effort: Some("medium".to_owned()),
reasoning_preset: None,
request_kind: Some("root".to_owned()),
subagent_kind: None,
compact: Some(false),
continuation_affinity_hash: Some("affinity-hash".to_owned()),
continuation_previous_response_id_hash: Some("response-hash".to_owned()),
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
upstream_connection_id: Some("connection-id".to_owned()),
upstream_connection_exit_reason: Some("tcp_reset".to_owned()),
upstream_connection_age_ms: Some(12_000),
upstream_connection_idle_ms: Some(4_000),
recovery_request_id: None,
recovered_at: None,
recovery_attempt_count: 0,
recovery_retry_delay_ms: None,
recovery_total_latency_ms: None,
occurred_at: Utc::now(),
stable_sort_id: "model_request:req_err".to_owned(),
}
}
#[tokio::test]
async fn usage_record_query_accepts_error_sources_only_for_error_status() {
use crate::admin::{AdminTestFixture, AdminTestState};
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use gateway_admin::model::observability::{UsageErrorOrigin, UsageRecordStatus};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
for (query, expected) in [
("", Some(UsageRecordStatus::Success)),
("status=success", Some(UsageRecordStatus::Success)),
("status=error", Some(UsageRecordStatus::Error(None))),
("status=all", Some(UsageRecordStatus::All)),
(
"status=error&errorOrigin=proxy",
Some(UsageRecordStatus::Error(Some(UsageErrorOrigin::Proxy))),
),
(
"status=error&errorOrigin=upstream",
Some(UsageRecordStatus::Error(Some(UsageErrorOrigin::Upstream))),
),
(
"status=error&errorOrigin=connection",
Some(UsageRecordStatus::Error(Some(UsageErrorOrigin::Connection))),
),
(
"status=error&errorOrigin=unknown",
Some(UsageRecordStatus::Error(Some(UsageErrorOrigin::Unknown))),
),
("status=failed", None),
("errorOrigin=proxy", None),
("status=success&errorOrigin=proxy", None),
("status=all&errorOrigin=unknown", None),
("status=error&errorOrigin=other", None),
("status=error&errorOrigin=", None),
] {
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri(format!("/api/admin/usage/records?{query}"))
.header(header::COOKIE, "cpr_session=valid-session")
.header("x-request-id", "usage_status_fixture")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let actual = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
assert_eq!(
actual,
if expected.is_some() {
StatusCode::OK
} else {
StatusCode::BAD_REQUEST
},
"{query}: {}",
String::from_utf8_lossy(&body)
);
if let Some(status_value) = expected {
assert_eq!(
fixture
.observations
.lock()
.unwrap()
.records
.last()
.unwrap()
.status,
status_value
);
}
}
}
#[tokio::test]
async fn dashboard_summary_identifies_trend_metrics_and_account_usage_values() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Duration;
use gateway_admin::model::{
accounts::AccountUsage,
observability::{DashboardAccountUsage, DashboardObservation, TimeRange},
provider_credentials::{ProviderQuotaWindow, QuotaLocalUsageAttribution},
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture.dashboard_observation.lock().unwrap() = Some(DashboardObservation {
yesterday: Default::default(),
analytics: Default::default(),
runtime_slots: None,
range: TimeRange::new(now - Duration::hours(1), now).unwrap(),
totals: Default::default(),
provider_accounts: Default::default(),
trend: Vec::new(),
account_usage: [
(None, Some(2_000), Some(now)),
(None, Some(0), None),
(None, None, None),
(Some(9), Some(2_000), Some(now)),
(Some(0), Some(2_000), None),
]
.into_iter()
.enumerate()
.map(|(index, (requests, tokens, last_used_at))| {
let account_id = format!("acct_metric_{index}");
DashboardAccountUsage {
upstream: None,
account_id: account_id.clone(),
provider_kind: "openai".to_owned(),
authentication_kind: "oauth".to_owned(),
name: account_id.clone(),
email: None,
notes: None,
plan_type: None,
plan_type_display: None,
request_count: 10,
success_count: 10,
input_tokens: None,
output_tokens: None,
cached_input_tokens: None,
cache_write_input_tokens: None,
reasoning_output_tokens: None,
image_input_tokens: None,
image_output_tokens: None,
image_request_count: 0,
image_request_failed_count: 0,
total_tokens: tokens,
cost_coverage: Default::default(),
costs: Vec::new(),
last_used_at,
request_buckets: Vec::new(),
quota_used_percent: None,
quota_window: requests.map(|request_count| ProviderQuotaWindow {
upstream: None,
key: "rolling".to_owned(),
group: "shortTerm".to_owned(),
label: "滚动窗口".to_owned(),
limit_id: None,
limit_name: None,
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(86_400),
used_percent: None,
reset_at: None,
limit_reached: false,
local_usage: Some(AccountUsage {
account_id,
request_count,
success_count: request_count,
input_tokens: None,
output_tokens: None,
cached_input_tokens: None,
cache_write_input_tokens: None,
reasoning_output_tokens: None,
image_input_tokens: None,
image_output_tokens: None,
image_request_count: 0,
image_request_failed_count: 0,
total_tokens: None,
cost_coverage: Default::default(),
costs: Vec::new(),
last_used_at,
request_buckets: Vec::new(),
models: Vec::new(),
}),
provider_data: None,
}),
models: Vec::new(),
}
})
.collect(),
recent_requests: Vec::new(),
});
for (kind, metrics) in [
(
"usage",
["inputTokens", "outputTokens", "cachedInputTokens"],
),
(
"latency",
["firstTokenP95Ms", "latencyP95Ms", "outputThroughputP50"],
),
("errors", ["errors", "successRate", "requests"]),
] {
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri(format!("/api/admin/dashboard/summary?kind={kind}"))
.header(header::COOKIE, "cpr_session=valid-session")
.header(header::ACCEPT_LANGUAGE, "en")
.header("x-request-id", format!("req_trend_metrics_{kind}"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(status, StatusCode::OK, "{kind}: {value}");
let accounts = value["data"]["accountUsage"].as_array().unwrap();
assert_eq!(accounts.len(), 5);
for (account, (metric, raw, display, last_used_at)) in accounts.iter().zip([
("tokens", Some(2_000), "2K", Some(now)),
("tokens", Some(0), "0", None),
("tokens", None, "—", None),
("requests", Some(9), "9", Some(now)),
("requests", Some(0), "—", None),
]) {
assert_eq!(account["metric"], metric);
assert_eq!(account["metricValueRaw"], json!(raw));
assert_eq!(account["metricValue"], display);
assert_eq!(account["lastUsedAt"], json!(last_used_at));
}
let summary = value["data"]["trend"]["summary"].as_array().unwrap();
assert_eq!(summary.len(), metrics.len());
for (item, metric) in summary.iter().zip(metrics) {
assert_eq!(item["metric"], metric);
assert!(item["value"].is_string());
}
}
}
#[tokio::test]
async fn dashboard_relative_time_preserves_server_anchor_boundaries_and_deployment_timezone() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::{DateTime, Duration};
use gateway_admin::model::observability::{
DashboardAccountUsage, DashboardObservation, TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use crate::admin::{AdminTestFixture, AdminTestState};
let now = "2026-06-15T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
for (zone, future_display) in [
("UTC", "2026-06-15 00:00:01"),
("Asia/Shanghai", "2026-06-15 08:00:01"),
("Asia/Kathmandu", "2026-06-15 05:45:01"),
("America/New_York", "2026-06-14 20:00:01"),
] {
for (elapsed_ms, expected, legacy) in [
(None, json!(null), "—"),
(
Some(-1_000),
json!({ "kind": "absolute", "value": future_display }),
future_display,
),
(Some(-1), json!({ "kind": "just_now" }), "刚刚"),
(Some(0), json!({ "kind": "just_now" }), "刚刚"),
(Some(59_999), json!({ "kind": "just_now" }), "刚刚"),
(
Some(60_000),
json!({ "kind": "elapsed", "value": 1, "unit": "minute" }),
"1 分钟前",
),
(
Some(119_999),
json!({ "kind": "elapsed", "value": 1, "unit": "minute" }),
"1 分钟前",
),
(
Some(3_599_999),
json!({ "kind": "elapsed", "value": 59, "unit": "minute" }),
"59 分钟前",
),
(
Some(3_600_000),
json!({ "kind": "elapsed", "value": 1, "unit": "hour" }),
"1 小时前",
),
(
Some(86_399_999),
json!({ "kind": "elapsed", "value": 23, "unit": "hour" }),
"23 小时前",
),
(
Some(86_400_000),
json!({ "kind": "elapsed", "value": 1, "unit": "day" }),
"1 天前",
),
(
Some(34_560_000_000),
json!({ "kind": "elapsed", "value": 400, "unit": "day" }),
"400 天前",
),
] {
// 每个边界使用独立缓存,避免相同锚点复用上一轮观测数据
let fixture = AdminTestFixture::with_timezone(zone.parse().unwrap()).await;
fixture.auth.insert_session("valid-session");
let last_used_at = elapsed_ms.map(|value| now - Duration::milliseconds(value));
*fixture.dashboard_observation.lock().unwrap() = Some(DashboardObservation {
yesterday: Default::default(),
analytics: Default::default(),
runtime_slots: None,
range: TimeRange::new(now - Duration::hours(1), now).unwrap(),
totals: Default::default(),
provider_accounts: Default::default(),
trend: Vec::new(),
account_usage: vec![DashboardAccountUsage {
upstream: None,
account_id: "acct_relative_time".to_owned(),
provider_kind: "openai".to_owned(),
authentication_kind: "oauth".to_owned(),
name: "Relative time fixture".to_owned(),
email: None,
notes: None,
plan_type: None,
plan_type_display: None,
request_count: 1,
success_count: 1,
input_tokens: None,
output_tokens: None,
cached_input_tokens: None,
cache_write_input_tokens: None,
reasoning_output_tokens: None,
image_input_tokens: None,
image_output_tokens: None,
image_request_count: 0,
image_request_failed_count: 0,
total_tokens: None,
cost_coverage: Default::default(),
costs: Vec::new(),
last_used_at,
request_buckets: Vec::new(),
quota_used_percent: None,
quota_window: None,
models: Vec::new(),
}],
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri(format!(
"/api/admin/dashboard/summary?asOf={}",
now.timestamp_millis()
))
.header(header::COOKIE, "cpr_session=valid-session")
.header(header::ACCEPT_LANGUAGE, "en")
.header("x-request-id", "req_relative_time")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(status, StatusCode::OK, "{zone}: {value}");
let account = &value["data"]["accountUsage"][0];
assert_eq!(
account["lastUsedRelative"], expected,
"{zone}: {elapsed_ms:?}"
);
assert_eq!(account["lastUsed"], legacy, "{zone}: {elapsed_ms:?}");
assert_eq!(account["lastUsedAt"], json!(last_used_at));
}
}
}