mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-21 16:01:32 +00:00
Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
1496 lines
50 KiB
Rust
1496 lines
50 KiB
Rust
//! 验证账号删除后请求、尝试与运维事件仍保留各自历史快照
|
|
|
|
use gateway_admin::model::observability as admin_observability;
|
|
|
|
use chrono::{DateTime, TimeDelta, Utc};
|
|
use gateway_admin::ports::store::ObservabilityStore as _;
|
|
use gateway_store::postgres::{
|
|
DiagnosticDimension, ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest,
|
|
ObservabilityPageSize, ObservabilityRange, OpsErrorFilter, OpsErrorQuery, OpsEvent,
|
|
OpsEventLevel, OpsEventRepository, PgExecutionStore, PgOpsEventRepository,
|
|
ProviderAccountUsageQuery, UsageRecordFilter, UsageRecordQuery,
|
|
};
|
|
use sqlx::PgPool;
|
|
|
|
use super::{TestDatabase, observability_repository};
|
|
|
|
#[tokio::test]
|
|
async fn completed_usage_projections_should_accept_statusless_websocket_but_reject_statusless_http()
|
|
{
|
|
let Some(database) = TestDatabase::create("snapshot_statusless_websocket_usage").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_statusless_transport",
|
|
"Statusless transport",
|
|
None,
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
|
|
let mut websocket_request = new_request("req_statusless_websocket", started_at);
|
|
websocket_request.client_transport = "websocket".to_owned();
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
websocket_request,
|
|
attempt(
|
|
&database.pool,
|
|
"req_statusless_websocket",
|
|
1,
|
|
"acct_statusless_transport",
|
|
)
|
|
.await,
|
|
)
|
|
.await
|
|
.expect("insert statusless WebSocket request");
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_statusless_http", started_at),
|
|
attempt(
|
|
&database.pool,
|
|
"req_statusless_http",
|
|
1,
|
|
"acct_statusless_transport",
|
|
)
|
|
.await,
|
|
)
|
|
.await
|
|
.expect("insert statusless HTTP request");
|
|
finalize_request_without_client_status(&database.pool, "req_statusless_websocket", started_at)
|
|
.await;
|
|
finalize_request_without_client_status(&database.pool, "req_statusless_http", started_at).await;
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
let page = repository
|
|
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
|
.await
|
|
.expect("list statusless transport usage records");
|
|
let request_ids = page
|
|
.items
|
|
.iter()
|
|
.map(|record| record.id.as_str())
|
|
.collect::<Vec<_>>();
|
|
|
|
assert_eq!(
|
|
(page.total, request_ids),
|
|
(1, vec!["req_statusless_websocket"])
|
|
);
|
|
|
|
super::drain_analytics(&database.pool).await;
|
|
let overview = repository
|
|
.usage_summary(range_around(started_at), UsageRecordFilter::default())
|
|
.await
|
|
.expect("summarize statusless transport usage");
|
|
assert_eq!(
|
|
(
|
|
overview.requests.request_count,
|
|
overview.requests.success_count,
|
|
overview.requests.total_tokens,
|
|
overview.attempts.cost_coverage.provider_reported_count,
|
|
),
|
|
(2, 2, 10, 1),
|
|
);
|
|
|
|
let account_usage = repository
|
|
.provider_account_usage(
|
|
ProviderAccountUsageQuery::for_accounts(
|
|
range_around(started_at),
|
|
vec!["acct_statusless_transport".to_owned()],
|
|
)
|
|
.expect("statusless account usage query")
|
|
.with_hourly_request_buckets()
|
|
.expect("statusless account request timeline"),
|
|
)
|
|
.await
|
|
.expect("load statusless account usage");
|
|
assert_eq!(
|
|
(
|
|
account_usage[0].request_count,
|
|
account_usage[0].total_tokens,
|
|
account_usage[0].models[0].request_count,
|
|
account_usage[0]
|
|
.request_buckets
|
|
.iter()
|
|
.map(|bucket| bucket.request_count)
|
|
.sum::<u64>(),
|
|
account_usage[0].costs[0].amount.as_str(),
|
|
),
|
|
(1, Some(10), 1, 1, "1.25"),
|
|
);
|
|
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = repository
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Account,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("load statusless usage diagnostics")
|
|
.items;
|
|
assert_eq!(
|
|
(
|
|
diagnostics[0].request_count,
|
|
diagnostics[0].success_count,
|
|
diagnostics[0].total_tokens,
|
|
diagnostics[0].average_latency_ms,
|
|
diagnostics[0].cost_coverage.provider_reported_count,
|
|
diagnostics[0].costs[0].amount.as_str(),
|
|
),
|
|
(2, 2, 10, Some(500), 1, "1.25"),
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn request_snapshots_should_survive_account_deletion() {
|
|
let Some(database) = TestDatabase::create("snapshot_request_delete").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Snapshot Alpha",
|
|
Some("alpha@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_a", started_at),
|
|
attempt(&database.pool, "req_snap_a", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert request with first attempt");
|
|
finalize_request(&database.pool, "req_snap_a", started_at).await;
|
|
|
|
let before: (String, String) =
|
|
sqlx::query_as("select ctid::text, xmin::text from model_requests where id = 'req_snap_a'")
|
|
.fetch_one(&database.pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("delete from provider_accounts where id = 'acct_snap_a'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("delete provider account");
|
|
let after: (String, String) =
|
|
sqlx::query_as("select ctid::text, xmin::text from model_requests where id = 'req_snap_a'")
|
|
.fetch_one(&database.pool)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
after, before,
|
|
"deleting configuration must not rewrite history"
|
|
);
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
let page = repository
|
|
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
|
.await
|
|
.expect("list usage records after deletion");
|
|
let record = page
|
|
.items
|
|
.iter()
|
|
.find(|record| record.id == "req_snap_a")
|
|
.expect("request snapshot record");
|
|
assert_eq!(
|
|
(
|
|
record.provider_account_ref.as_deref(),
|
|
record.provider_account_name.as_deref(),
|
|
record.provider_account_email.as_deref(),
|
|
record.provider_account_authentication_kind.as_deref(),
|
|
),
|
|
(
|
|
Some("acct_snap_a"),
|
|
Some("Snapshot Alpha"),
|
|
Some("alpha@example.invalid"),
|
|
Some("oauth"),
|
|
)
|
|
);
|
|
|
|
let detail = repository
|
|
.usage_record_detail("req_snap_a")
|
|
.await
|
|
.expect("usage record detail after deletion");
|
|
assert_eq!(
|
|
(
|
|
detail.request.provider_account_name.as_deref(),
|
|
detail.request.provider_account_email.as_deref(),
|
|
detail
|
|
.request
|
|
.provider_account_authentication_kind
|
|
.as_deref(),
|
|
),
|
|
(
|
|
Some("Snapshot Alpha"),
|
|
Some("alpha@example.invalid"),
|
|
Some("oauth"),
|
|
)
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn attempts_should_keep_their_own_account_snapshots() {
|
|
let Some(database) = TestDatabase::create("snapshot_attempt").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Snapshot Alpha",
|
|
Some("alpha@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_b",
|
|
"Snapshot Beta",
|
|
Some("beta@example.invalid"),
|
|
"api_key",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_ab", started_at),
|
|
attempt(&database.pool, "req_snap_ab", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert first attempt");
|
|
PgOpsEventRepository::new(database.pool.clone())
|
|
.append_ops_event(OpsEvent {
|
|
id: "ops_snap_a".to_owned(),
|
|
model_request_id: Some("req_snap_ab".to_owned()),
|
|
attempt_index: Some(1),
|
|
level: OpsEventLevel::Warning,
|
|
component: "routing".to_owned(),
|
|
operation: "fallback".to_owned(),
|
|
provider_kind: Some("openai".to_owned()),
|
|
account_snapshot: Some(account_snapshot(&database.pool, "acct_snap_a").await),
|
|
provider_account_ref: Some("acct_snap_a".to_owned()),
|
|
upstream_model_id: Some("upstream-a".to_owned()),
|
|
failure_kind: "rate_limited".to_owned(),
|
|
upstream_send_state: Some("sent".to_owned()),
|
|
error_details: Some(
|
|
r#"{"error":{"code":"rate_limit","message":"raw marker"}}"#.to_owned(),
|
|
),
|
|
status_code: Some(429),
|
|
provider_error_code: Some("rate_limit".to_owned()),
|
|
retry_after_ms: Some(1_000),
|
|
upstream_request_id: None,
|
|
latency_ms: Some(120),
|
|
message: "first account was limited".to_owned(),
|
|
created_at: started_at + chrono::Duration::seconds(1),
|
|
})
|
|
.await
|
|
.expect("append intermediate failure");
|
|
store
|
|
.begin_model_request_attempt(attempt(&database.pool, "req_snap_ab", 2, "acct_snap_b").await)
|
|
.await
|
|
.expect("begin second attempt");
|
|
finalize_request(&database.pool, "req_snap_ab", started_at).await;
|
|
|
|
sqlx::query("delete from provider_accounts where id in ('acct_snap_a', 'acct_snap_b')")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("delete provider accounts");
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
let detail = repository
|
|
.usage_record_detail("req_snap_ab")
|
|
.await
|
|
.expect("usage detail with mixed attempts");
|
|
let intermediate = detail
|
|
.attempts
|
|
.iter()
|
|
.find(|attempt| attempt.source == "ops_event")
|
|
.expect("intermediate attempt");
|
|
assert_eq!(
|
|
(
|
|
intermediate.provider_account_name.as_deref(),
|
|
intermediate.provider_account_email.as_deref(),
|
|
intermediate.provider_account_authentication_kind.as_deref(),
|
|
),
|
|
(
|
|
Some("Snapshot Alpha"),
|
|
Some("alpha@example.invalid"),
|
|
Some("oauth"),
|
|
)
|
|
);
|
|
let final_attempt = detail
|
|
.attempts
|
|
.iter()
|
|
.find(|attempt| attempt.source == "model_request")
|
|
.expect("final attempt");
|
|
assert_eq!(
|
|
(
|
|
final_attempt.provider_account_name.as_deref(),
|
|
final_attempt.provider_account_email.as_deref(),
|
|
final_attempt
|
|
.provider_account_authentication_kind
|
|
.as_deref(),
|
|
),
|
|
(
|
|
Some("Snapshot Beta"),
|
|
Some("beta@example.invalid"),
|
|
Some("api_key"),
|
|
)
|
|
);
|
|
assert_eq!(
|
|
detail.request.provider_account_email.as_deref(),
|
|
Some("beta@example.invalid")
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deletion() {
|
|
let Some(database) = TestDatabase::create("snapshot_ops_errors").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Snapshot Alpha",
|
|
Some("alpha@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_c",
|
|
"Snapshot Charlie",
|
|
Some("charlie@example.invalid"),
|
|
"api_key",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_error", started_at),
|
|
attempt(&database.pool, "req_snap_error", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert failed request");
|
|
sqlx::query(
|
|
"update model_requests
|
|
set outcome = 'failed',
|
|
error_kind = 'upstream_error',
|
|
client_status_code = 502,
|
|
upstream_status_code = 502,
|
|
completed_at = $1,
|
|
error_details = $2,
|
|
request_observation_json = jsonb_strip_nulls(request_observation_json || jsonb_build_object(
|
|
'error', coalesce(request_observation_json #> '{error}', '{}'::jsonb) || jsonb_build_object(
|
|
'message', 'snapshot failure')))
|
|
where id = 'req_snap_error'",
|
|
)
|
|
.bind(started_at + chrono::Duration::seconds(2))
|
|
.bind(r#"{"error":{"message":"raw request failure"}}"#)
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("mark request failed");
|
|
PgOpsEventRepository::new(database.pool.clone())
|
|
.append_ops_event(OpsEvent {
|
|
id: "ops_snap_probe".to_owned(),
|
|
model_request_id: None,
|
|
attempt_index: None,
|
|
level: OpsEventLevel::Warning,
|
|
component: "account_probe".to_owned(),
|
|
operation: "connection_test".to_owned(),
|
|
provider_kind: Some("xai".to_owned()),
|
|
account_snapshot: Some(account_snapshot(&database.pool, "acct_snap_c").await),
|
|
provider_account_ref: Some("acct_snap_c".to_owned()),
|
|
upstream_model_id: Some("grok-test".to_owned()),
|
|
failure_kind: "auth_failed".to_owned(),
|
|
upstream_send_state: Some("not_sent".to_owned()),
|
|
error_details: Some(r#"{"error":"raw probe failure"}"#.to_owned()),
|
|
status_code: Some(401),
|
|
provider_error_code: Some("invalid_api_key".to_owned()),
|
|
retry_after_ms: None,
|
|
upstream_request_id: None,
|
|
latency_ms: Some(80),
|
|
message: "probe failed".to_owned(),
|
|
created_at: started_at + chrono::Duration::seconds(3),
|
|
})
|
|
.await
|
|
.expect("append probe failure");
|
|
|
|
sqlx::query("delete from provider_accounts where id in ('acct_snap_a', 'acct_snap_c')")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("delete provider accounts");
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
let errors = repository
|
|
.list_ops_errors(OpsErrorQuery {
|
|
range: range_around(started_at),
|
|
filter: OpsErrorFilter::default(),
|
|
current_page: 1,
|
|
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
|
})
|
|
.await
|
|
.expect("list ops errors after deletion");
|
|
let request_error = errors
|
|
.items
|
|
.iter()
|
|
.find(|error| error.source == "model_request")
|
|
.expect("request error");
|
|
assert_eq!(
|
|
(
|
|
request_error.provider_account_name.as_deref(),
|
|
request_error.provider_account_email.as_deref(),
|
|
request_error
|
|
.provider_account_authentication_kind
|
|
.as_deref(),
|
|
),
|
|
(
|
|
Some("Snapshot Alpha"),
|
|
Some("alpha@example.invalid"),
|
|
Some("oauth"),
|
|
)
|
|
);
|
|
assert_eq!(request_error.endpoint.as_deref(), Some("/v1/responses"));
|
|
assert_eq!(
|
|
request_error.error_details.as_deref(),
|
|
Some(r#"{"error":{"message":"raw request failure"}}"#)
|
|
);
|
|
let probe_error = errors
|
|
.items
|
|
.iter()
|
|
.find(|error| error.source == "ops_event")
|
|
.expect("probe error");
|
|
assert_eq!(
|
|
(
|
|
probe_error.provider_account_name.as_deref(),
|
|
probe_error.provider_account_email.as_deref(),
|
|
probe_error.provider_account_authentication_kind.as_deref(),
|
|
),
|
|
(
|
|
Some("Snapshot Charlie"),
|
|
Some("charlie@example.invalid"),
|
|
Some("api_key"),
|
|
)
|
|
);
|
|
assert_eq!(probe_error.endpoint, None);
|
|
assert_eq!(
|
|
probe_error.error_details.as_deref(),
|
|
Some(r#"{"error":"raw probe failure"}"#)
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
|
|
let Some(database) = TestDatabase::create("snapshot_diagnostics_same_email").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Alpha",
|
|
Some("same@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_b",
|
|
"Beta",
|
|
Some("same@example.invalid"),
|
|
"api_key",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_a", started_at),
|
|
attempt(&database.pool, "req_snap_a", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert account a request");
|
|
finalize_request(&database.pool, "req_snap_a", started_at).await;
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_b", started_at + chrono::Duration::seconds(1)),
|
|
attempt(&database.pool, "req_snap_b", 1, "acct_snap_b").await,
|
|
)
|
|
.await
|
|
.expect("insert account b request");
|
|
finalize_request(&database.pool, "req_snap_b", started_at).await;
|
|
|
|
sqlx::query("delete from provider_accounts where id in ('acct_snap_a', 'acct_snap_b')")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("delete provider accounts");
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = repository
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Account,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("account diagnostics")
|
|
.items;
|
|
assert_eq!(diagnostics.len(), 2);
|
|
assert_eq!(
|
|
diagnostics
|
|
.iter()
|
|
.map(|item| item.key.as_str())
|
|
.collect::<Vec<_>>(),
|
|
vec!["acct_snap_a", "acct_snap_b"]
|
|
);
|
|
assert!(
|
|
diagnostics
|
|
.iter()
|
|
.all(|item| item.name == "same@example.invalid")
|
|
);
|
|
assert!(diagnostics.iter().all(|item| item.request_count == 1));
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
|
|
let Some(database) = TestDatabase::create("snapshot_diagnostics_fallback").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_c",
|
|
"Snapshot Charlie",
|
|
None,
|
|
"api_key",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_c", started_at),
|
|
attempt(&database.pool, "req_snap_c", 1, "acct_snap_c").await,
|
|
)
|
|
.await
|
|
.expect("insert account c request");
|
|
finalize_request(&database.pool, "req_snap_c", started_at).await;
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_legacy", started_at + chrono::Duration::seconds(1)),
|
|
ModelRequestAttemptStart {
|
|
account_selection_wait_ms: None,
|
|
capacity_used_slots: None,
|
|
capacity_total_slots: None,
|
|
model_request_id: "req_snap_legacy".to_owned(),
|
|
attempt_count: 1,
|
|
provider_kind: "openai".to_owned(),
|
|
account_snapshot: None,
|
|
provider_account_ref: Some("acct_legacy_deleted".to_owned()),
|
|
upstream_model_id: Some("legacy-model".to_owned()),
|
|
upstream_transport: "http_sse".to_owned(),
|
|
http_version: None,
|
|
},
|
|
)
|
|
.await
|
|
.expect("insert legacy request without live account");
|
|
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = repository
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Account,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("account diagnostics")
|
|
.items;
|
|
let by_key = diagnostics
|
|
.iter()
|
|
.map(|item| (item.key.as_str(), item.name.as_str()))
|
|
.collect::<std::collections::HashMap<_, _>>();
|
|
assert_eq!(by_key.get("acct_snap_c"), Some(&"Snapshot Charlie"));
|
|
assert_eq!(
|
|
by_key.get("acct_legacy_deleted"),
|
|
Some(&"acct_legacy_deleted")
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_use_only_the_latest_historical_snapshot() {
|
|
let Some(database) = TestDatabase::create("snapshot_diagnostics_latest_email").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snapshot_history",
|
|
"Older Name",
|
|
Some("history@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snapshot_history_old", started_at),
|
|
attempt(
|
|
&database.pool,
|
|
"req_snapshot_history_old",
|
|
1,
|
|
"acct_snapshot_history",
|
|
)
|
|
.await,
|
|
)
|
|
.await
|
|
.expect("insert older account snapshot");
|
|
finalize_request(&database.pool, "req_snapshot_history_old", started_at).await;
|
|
sqlx::query(
|
|
"update provider_accounts
|
|
set name = 'Newer Name', email = null, updated_at = now()
|
|
where id = 'acct_snapshot_history'",
|
|
)
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("update account snapshot source");
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request(
|
|
"req_snapshot_history_new",
|
|
started_at + chrono::Duration::seconds(1),
|
|
),
|
|
attempt(
|
|
&database.pool,
|
|
"req_snapshot_history_new",
|
|
1,
|
|
"acct_snapshot_history",
|
|
)
|
|
.await,
|
|
)
|
|
.await
|
|
.expect("insert newer account snapshot");
|
|
finalize_request(&database.pool, "req_snapshot_history_new", started_at).await;
|
|
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = observability_repository(&database.pool)
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Account,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("account diagnostics")
|
|
.items;
|
|
|
|
assert_eq!(diagnostics.len(), 1);
|
|
assert_eq!(diagnostics[0].key, "acct_snapshot_history");
|
|
assert_eq!(diagnostics[0].name, "Newer Name");
|
|
// 最新快照缺失时保留稳定引用,不再让更早快照覆盖最新请求的未知值
|
|
sqlx::query("update model_requests set request_observation_json=request_observation_json #- '{account,name}' #- '{account,email}' where id='req_snapshot_history_new'")
|
|
.execute(&database.pool).await.expect("clear latest account snapshot");
|
|
super::drain_analytics(&database.pool).await;
|
|
let missing = observability_repository(&database.pool)
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Account,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("unknown latest snapshot remains unknown")
|
|
.items;
|
|
assert_eq!(missing[0].name, "acct_snapshot_history");
|
|
assert_eq!(diagnostics[0].request_count, 2);
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn failure_diagnostics_should_only_include_errored_requests() {
|
|
let Some(database) = TestDatabase::create("snapshot_failure_dimension").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Snapshot Alpha",
|
|
Some("alpha@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_ok", started_at),
|
|
attempt(&database.pool, "req_snap_ok", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert healthy request");
|
|
finalize_request(&database.pool, "req_snap_ok", started_at).await;
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_err", started_at + chrono::Duration::seconds(1)),
|
|
attempt(&database.pool, "req_snap_err", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert errored request");
|
|
sqlx::query(
|
|
"update model_requests
|
|
set outcome = 'failed',
|
|
error_kind = 'rate_limited',
|
|
client_status_code = 429,
|
|
upstream_status_code = 429,
|
|
completed_at = $2,
|
|
request_observation_json = jsonb_strip_nulls(request_observation_json || jsonb_build_object(
|
|
'error', coalesce(request_observation_json #> '{error}', '{}'::jsonb) || jsonb_build_object(
|
|
'message', 'upstream limited')))
|
|
where id = $1",
|
|
)
|
|
.bind("req_snap_err")
|
|
.bind(started_at + chrono::Duration::seconds(6))
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("mark request failed");
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = repository
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Failure,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("failure diagnostics")
|
|
.items;
|
|
assert_eq!(diagnostics.len(), 1);
|
|
assert_eq!(diagnostics[0].key, "rate_limited");
|
|
assert_eq!(diagnostics[0].request_count, 1);
|
|
assert_eq!(diagnostics[0].failure_count, 1);
|
|
assert_eq!(diagnostics[0].success_count, 0);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn model_diagnostics_should_exclude_provider_endpoints_without_a_model() {
|
|
let Some(database) = TestDatabase::create("snapshot_model_dimension").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Snapshot Alpha",
|
|
Some("alpha@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_model", started_at),
|
|
attempt(&database.pool, "req_snap_model", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert model request");
|
|
finalize_request(&database.pool, "req_snap_model", started_at).await;
|
|
|
|
let mut search_request = new_request(
|
|
"req_snap_provider_endpoint",
|
|
started_at + chrono::Duration::seconds(1),
|
|
);
|
|
search_request.operation = "search".to_owned();
|
|
search_request.endpoint = "/v1/alpha/search".to_owned();
|
|
search_request.client_transport = "http_json".to_owned();
|
|
search_request.requested_model_id = None;
|
|
let mut search_attempt = attempt(
|
|
&database.pool,
|
|
"req_snap_provider_endpoint",
|
|
1,
|
|
"acct_snap_a",
|
|
)
|
|
.await;
|
|
search_attempt.upstream_model_id = None;
|
|
search_attempt.upstream_transport = "http_json".to_owned();
|
|
store
|
|
.insert_model_request_with_first_attempt(search_request, search_attempt)
|
|
.await
|
|
.expect("insert provider endpoint request");
|
|
finalize_request(
|
|
&database.pool,
|
|
"req_snap_provider_endpoint",
|
|
started_at + chrono::Duration::seconds(1),
|
|
)
|
|
.await;
|
|
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = observability_repository(&database.pool)
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Model,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("model diagnostics")
|
|
.items;
|
|
|
|
assert_eq!(diagnostics.len(), 1);
|
|
assert_eq!(diagnostics[0].key, "upstream-model");
|
|
assert_eq!(diagnostics[0].request_count, 1);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tokenless_provider_endpoints_should_remain_in_health_but_not_usage_records() {
|
|
let Some(database) = TestDatabase::create("snapshot_usage_fact_boundary").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Snapshot Alpha",
|
|
Some("alpha@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
let mut search_request = new_request("req_snap_search", started_at);
|
|
search_request.operation = "search".to_owned();
|
|
search_request.endpoint = "/v1/alpha/search".to_owned();
|
|
search_request.client_transport = "http_json".to_owned();
|
|
search_request.requested_model_id = None;
|
|
let mut search_attempt = attempt(&database.pool, "req_snap_search", 1, "acct_snap_a").await;
|
|
search_attempt.upstream_model_id = None;
|
|
search_attempt.upstream_transport = "http_json".to_owned();
|
|
store
|
|
.insert_model_request_with_first_attempt(search_request, search_attempt)
|
|
.await
|
|
.expect("insert search request");
|
|
finalize_request(&database.pool, "req_snap_search", started_at).await;
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
let page = repository
|
|
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
|
.await
|
|
.expect("list usage records");
|
|
super::drain_analytics(&database.pool).await;
|
|
let overview = repository
|
|
.usage_summary(range_around(started_at), UsageRecordFilter::default())
|
|
.await
|
|
.expect("summarize request health");
|
|
|
|
assert_eq!(
|
|
(
|
|
page.total,
|
|
page.items.len(),
|
|
overview.requests.request_count
|
|
),
|
|
(0, 0, 1)
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_keep_filtered_totals_before_limiting_groups() {
|
|
let Some(database) = TestDatabase::create("diagnostic_totals").await else {
|
|
return;
|
|
};
|
|
let now = Utc::now();
|
|
seed_account(&database.pool, "acct_totals", "Totals", None, "oauth", now).await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
for index in 0..201 {
|
|
let id = format!("req_total_{index:03}");
|
|
let mut first_attempt = attempt(&database.pool, &id, 1, "acct_totals").await;
|
|
first_attempt.upstream_model_id = Some(format!("model_{:03}", index % 200));
|
|
store
|
|
.insert_model_request_with_first_attempt(new_request(&id, now), first_attempt)
|
|
.await
|
|
.expect("insert model group");
|
|
finalize_request(&database.pool, &id, now).await;
|
|
}
|
|
sqlx::query(
|
|
"update model_requests set cost_source = 'provider_reported', cost_amount = 1,
|
|
cost_currency = case when id = 'req_total_200' then 'EUR' else 'USD' end",
|
|
)
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("set currency groups");
|
|
let repository = observability_repository(&database.pool);
|
|
super::drain_analytics(&database.pool).await;
|
|
let result = repository
|
|
.usage_diagnostics(
|
|
range_around(now),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Model,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("limited groups");
|
|
assert_eq!(
|
|
(
|
|
result.total_request_count,
|
|
result.items.len(),
|
|
result
|
|
.items
|
|
.iter()
|
|
.map(|item| item.request_count)
|
|
.sum::<u64>()
|
|
),
|
|
(201, 100, 101)
|
|
);
|
|
assert_eq!(
|
|
(result.items[0].key.as_str(), result.items[0].costs.len()),
|
|
("model_000", 2)
|
|
);
|
|
|
|
super::drain_analytics(&database.pool).await;
|
|
let filtered = repository
|
|
.usage_diagnostics(
|
|
range_around(now),
|
|
UsageRecordFilter {
|
|
model: Some("model_000".to_owned()),
|
|
..UsageRecordFilter::default()
|
|
},
|
|
DiagnosticDimension::Model,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("filtered denominator");
|
|
assert_eq!((filtered.total_request_count, filtered.items.len()), (2, 1));
|
|
super::drain_analytics(&database.pool).await;
|
|
let empty = repository
|
|
.usage_diagnostics(
|
|
range_around(now),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::Failure,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("empty error dimension");
|
|
assert_eq!((empty.total_request_count, empty.items.len()), (0, 0));
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn diagnostics_should_distinguish_retry_attempts_from_retried_requests() {
|
|
let Some(database) = TestDatabase::create("diagnostic_retries").await else {
|
|
return;
|
|
};
|
|
let now = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_retries",
|
|
"Retries",
|
|
None,
|
|
"oauth",
|
|
now,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
for (index, attempts) in [0, 1, 1, 1, 1, 1, 3, 5].into_iter().enumerate() {
|
|
let id = format!("req_retry_{index}");
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request(&id, now),
|
|
attempt(&database.pool, &id, 1, "acct_retries").await,
|
|
)
|
|
.await
|
|
.expect("insert retry request");
|
|
finalize_request(&database.pool, &id, now).await;
|
|
sqlx::query("update model_requests set attempt_count = $2 where id = $1")
|
|
.bind(id)
|
|
.bind(attempts)
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("set attempts");
|
|
}
|
|
super::drain_analytics(&database.pool).await;
|
|
let result = super::admin_observability_store(&database.pool)
|
|
.usage_diagnostics(
|
|
admin_observability::TimeRange::new(
|
|
now - TimeDelta::hours(1),
|
|
now + TimeDelta::hours(1),
|
|
)
|
|
.expect("range"),
|
|
gateway_admin::model::observability::UsageFilter::default(),
|
|
gateway_admin::model::observability::DiagnosticDimension::Model,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("retry metrics through admin adapter");
|
|
assert_eq!(
|
|
(
|
|
result.total_request_count,
|
|
result.items[0].retry_count,
|
|
result.items[0].retried_request_count
|
|
),
|
|
(8, 6, 2)
|
|
);
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn api_key_diagnostics_preserve_historical_key_name_after_deletion() {
|
|
let Some(database) = TestDatabase::create("snapshot_api_key_dimension").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_api_key(&database.pool, "key_diag", "My Key", started_at).await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
for id in ["req_key_a", "req_key_b"] {
|
|
let mut request = new_request(id, started_at);
|
|
request.client_api_key_name_snapshot = Some("My Key".to_owned());
|
|
request.client_api_key_ref = "key_diag".to_owned();
|
|
let attempt = ModelRequestAttemptStart {
|
|
account_selection_wait_ms: None,
|
|
capacity_used_slots: None,
|
|
capacity_total_slots: None,
|
|
model_request_id: id.to_owned(),
|
|
attempt_count: 1,
|
|
provider_kind: "openai".to_owned(),
|
|
account_snapshot: None,
|
|
provider_account_ref: Some("acct_key".to_owned()),
|
|
upstream_model_id: Some("upstream-model".to_owned()),
|
|
upstream_transport: "http_sse".to_owned(),
|
|
http_version: None,
|
|
};
|
|
store
|
|
.insert_model_request_with_first_attempt(request, attempt)
|
|
.await
|
|
.expect("insert api key request");
|
|
finalize_request(&database.pool, id, started_at).await;
|
|
}
|
|
sqlx::query("update model_requests set request_observation_json = jsonb_strip_nulls(request_observation_json || jsonb_build_object(
|
|
'timings', coalesce(request_observation_json #> '{timings}', '{}'::jsonb) || jsonb_build_object(
|
|
'local', coalesce(request_observation_json #> '{timings,local}', '{}'::jsonb) || jsonb_build_object(
|
|
'latencyMs', 1000)))) where id = 'req_key_a'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("set request a latency");
|
|
sqlx::query("update model_requests set request_observation_json = jsonb_strip_nulls(request_observation_json || jsonb_build_object(
|
|
'timings', coalesce(request_observation_json #> '{timings}', '{}'::jsonb) || jsonb_build_object(
|
|
'local', coalesce(request_observation_json #> '{timings,local}', '{}'::jsonb) || jsonb_build_object(
|
|
'latencyMs', 2000)))) where id = 'req_key_b'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("set request b latency");
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
super::drain_analytics(&database.pool).await;
|
|
let result = repository
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::ApiKey,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("api key diagnostics");
|
|
let diagnostics = result.items;
|
|
assert_eq!(diagnostics.len(), 1);
|
|
assert_eq!(diagnostics[0].key, "key_diag");
|
|
assert_eq!(diagnostics[0].name, "My Key");
|
|
assert_eq!(diagnostics[0].request_count, 2);
|
|
if result.analytics.percentiles_approximate {
|
|
// 直方图沿连续样本秩插值,与精确 P95 的误差受对数桶宽约束
|
|
assert!(
|
|
diagnostics[0]
|
|
.latency_p95_ms
|
|
.is_some_and(|value| value.abs_diff(1950) <= 59)
|
|
);
|
|
} else {
|
|
assert_eq!(diagnostics[0].latency_p95_ms, Some(1950));
|
|
}
|
|
|
|
sqlx::query("delete from client_api_keys where id = 'key_diag'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("delete api key");
|
|
super::drain_analytics(&database.pool).await;
|
|
let diagnostics = repository
|
|
.usage_diagnostics(
|
|
range_around(started_at),
|
|
UsageRecordFilter::default(),
|
|
DiagnosticDimension::ApiKey,
|
|
100,
|
|
)
|
|
.await
|
|
.expect("api key diagnostics after deletion")
|
|
.items;
|
|
assert_eq!(diagnostics.len(), 1);
|
|
assert_eq!(diagnostics[0].key, "key_diag");
|
|
assert_eq!(diagnostics[0].name, "My Key");
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
|
let Some(database) = TestDatabase::create("snapshot_rename").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"acct_snap_a",
|
|
"Original Name",
|
|
Some("original@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
let store = PgExecutionStore::new(database.pool.clone());
|
|
store
|
|
.insert_model_request_with_first_attempt(
|
|
new_request("req_snap_a", started_at),
|
|
attempt(&database.pool, "req_snap_a", 1, "acct_snap_a").await,
|
|
)
|
|
.await
|
|
.expect("insert request with original snapshot");
|
|
finalize_request(&database.pool, "req_snap_a", started_at).await;
|
|
sqlx::query(
|
|
"update provider_accounts
|
|
set name = 'Renamed', email = 'renamed@example.invalid'
|
|
where id = 'acct_snap_a'",
|
|
)
|
|
.execute(&database.pool)
|
|
.await
|
|
.expect("rename provider account");
|
|
|
|
let repository = observability_repository(&database.pool);
|
|
let page = repository
|
|
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
|
.await
|
|
.expect("list usage records after rename");
|
|
let record = page
|
|
.items
|
|
.iter()
|
|
.find(|record| record.id == "req_snap_a")
|
|
.expect("historical request");
|
|
assert_eq!(
|
|
(
|
|
record.provider_account_name.as_deref(),
|
|
record.provider_account_email.as_deref(),
|
|
),
|
|
(Some("Original Name"), Some("original@example.invalid"))
|
|
);
|
|
|
|
database.close().await;
|
|
}
|
|
|
|
fn new_request(id: &str, started_at: DateTime<Utc>) -> NewModelRequest {
|
|
NewModelRequest {
|
|
admission_decision_ms: None,
|
|
id: id.to_owned(),
|
|
client_api_key_name_snapshot: None,
|
|
client_api_key_ref: "key_snapshot".to_owned(),
|
|
config_revision: 1,
|
|
protocol: "openai".to_owned(),
|
|
operation: "responses".to_owned(),
|
|
endpoint: "/v1/responses".to_owned(),
|
|
client_transport: "http_sse".to_owned(),
|
|
requested_model_id: Some("coding".to_owned()),
|
|
routing_scope: "all".to_owned(),
|
|
routing_group_refs: Vec::new(),
|
|
routing_group_names_snapshot: serde_json::json!([]),
|
|
client_ip: None,
|
|
user_agent: None,
|
|
reasoning_effort: None,
|
|
reasoning_preset: None,
|
|
request_kind: None,
|
|
subagent_kind: None,
|
|
compact: false,
|
|
continuation: Default::default(),
|
|
image_generation_requested: false,
|
|
started_at,
|
|
lease_expires_at: started_at + chrono::Duration::seconds(30),
|
|
}
|
|
}
|
|
|
|
async fn attempt(
|
|
pool: &sqlx::PgPool,
|
|
id: &str,
|
|
count: u32,
|
|
account_id: &str,
|
|
) -> ModelRequestAttemptStart {
|
|
ModelRequestAttemptStart {
|
|
account_selection_wait_ms: None,
|
|
capacity_used_slots: None,
|
|
capacity_total_slots: None,
|
|
model_request_id: id.to_owned(),
|
|
attempt_count: count,
|
|
provider_kind: "openai".to_owned(),
|
|
account_snapshot: Some(account_snapshot(pool, account_id).await),
|
|
provider_account_ref: Some(account_id.to_owned()),
|
|
upstream_model_id: Some("upstream-model".to_owned()),
|
|
upstream_transport: "http_sse".to_owned(),
|
|
http_version: None,
|
|
}
|
|
}
|
|
|
|
async fn account_snapshot(
|
|
pool: &sqlx::PgPool,
|
|
id: &str,
|
|
) -> gateway_core::account::AccountHistorySnapshot {
|
|
let (name, email, authentication_kind, plan_type_snapshot, notes_snapshot) = sqlx::query_as::<_, (String, Option<String>, String, Option<String>, Option<String>)>(
|
|
"select name, email, authentication_kind, plan_type, notes from provider_accounts where id = $1",
|
|
).bind(id).fetch_one(pool).await.unwrap();
|
|
gateway_core::account::AccountHistorySnapshot {
|
|
name,
|
|
email,
|
|
authentication_kind,
|
|
plan_type_snapshot,
|
|
notes_snapshot,
|
|
}
|
|
}
|
|
|
|
fn usage_query(started_at: DateTime<Utc>, filter: UsageRecordFilter) -> UsageRecordQuery {
|
|
UsageRecordQuery {
|
|
status: Default::default(),
|
|
range: range_around(started_at),
|
|
filter,
|
|
current_page: 1,
|
|
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
|
}
|
|
}
|
|
|
|
fn range_around(started_at: DateTime<Utc>) -> ObservabilityRange {
|
|
ObservabilityRange::new(
|
|
started_at - TimeDelta::hours(1),
|
|
started_at + TimeDelta::hours(1),
|
|
)
|
|
.expect("observability range")
|
|
}
|
|
|
|
async fn seed_account(
|
|
pool: &PgPool,
|
|
id: &str,
|
|
name: &str,
|
|
email: Option<&str>,
|
|
authentication_kind: &str,
|
|
now: DateTime<Utc>,
|
|
) {
|
|
sqlx::query(
|
|
"insert into provider_accounts (
|
|
id, provider_kind, name, email, upstream_user_id,
|
|
upstream_account_id, plan_type, authentication_kind,
|
|
provider_credentials_json, credential_revision,
|
|
has_refresh_token, access_token_expires_at, enabled, credential_state,
|
|
credential_observed_at, created_at, updated_at
|
|
) values (
|
|
$1, 'openai', $2, $3, 'user-' || $1, null, null, $4,
|
|
'{}'::jsonb, 1, false, $5 + interval '1 day', true, 'ready',
|
|
$5, $5, $5
|
|
)",
|
|
)
|
|
.bind(id)
|
|
.bind(name)
|
|
.bind(email)
|
|
.bind(authentication_kind)
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.expect("seed provider account");
|
|
}
|
|
|
|
async fn finalize_request(pool: &PgPool, id: &str, started_at: DateTime<Utc>) {
|
|
sqlx::query(
|
|
"update model_requests
|
|
set outcome = 'succeeded', client_status_code = 200,
|
|
upstream_status_code = 200, downstream_committed_at = $2, completed_at = $2
|
|
where id = $1",
|
|
)
|
|
.bind(id)
|
|
.bind(started_at + chrono::Duration::seconds(5))
|
|
.execute(pool)
|
|
.await
|
|
.expect("finalize request");
|
|
}
|
|
|
|
async fn finalize_request_without_client_status(
|
|
pool: &PgPool,
|
|
id: &str,
|
|
started_at: DateTime<Utc>,
|
|
) {
|
|
sqlx::query(
|
|
"update model_requests
|
|
set outcome = 'succeeded',
|
|
upstream_send_state = 'sent',
|
|
client_status_code = null,
|
|
upstream_status_code = 200,
|
|
input_tokens = 7,
|
|
output_tokens = 3,
|
|
total_tokens = 10,
|
|
cost_source = 'provider_reported',
|
|
cost_amount = 1.25,
|
|
cost_currency = 'USD',
|
|
downstream_committed_at = $2,
|
|
completed_at = $2,
|
|
request_observation_json = jsonb_strip_nulls(request_observation_json || jsonb_build_object(
|
|
'timings', coalesce(request_observation_json #> '{timings}', '{}'::jsonb) || jsonb_build_object(
|
|
'local', coalesce(request_observation_json #> '{timings,local}', '{}'::jsonb) || jsonb_build_object(
|
|
'firstTokenMs', 100,
|
|
'latencyMs', 500))))
|
|
where id = $1",
|
|
)
|
|
.bind(id)
|
|
.bind(started_at + chrono::Duration::seconds(5))
|
|
.execute(pool)
|
|
.await
|
|
.expect("finalize request without client status");
|
|
}
|
|
|
|
async fn seed_api_key(pool: &PgPool, id: &str, name: &str, now: DateTime<Utc>) {
|
|
sqlx::query(
|
|
"insert into client_api_keys (id, name, key, enabled, created_at, updated_at)
|
|
values ($1, $2, 'sk_' || $3, true, $4, $4)",
|
|
)
|
|
.bind(id)
|
|
.bind(name)
|
|
.bind("A".repeat(43))
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.expect("seed api key");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn queued_history_keeps_frozen_identity_after_configuration_deletion() {
|
|
let Some(database) = TestDatabase::create("queued_history_delete").await else {
|
|
return;
|
|
};
|
|
let started_at = Utc::now();
|
|
seed_account(
|
|
&database.pool,
|
|
"queued_account",
|
|
"Original Account",
|
|
Some("old@example.invalid"),
|
|
"oauth",
|
|
started_at,
|
|
)
|
|
.await;
|
|
seed_api_key(&database.pool, "queued_key", "Original Key", started_at).await;
|
|
sqlx::query("update provider_accounts set plan_type = 'pro', notes = 'Original note' where id = 'queued_account'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.unwrap();
|
|
let mut request = new_request("queued_history", started_at);
|
|
request.client_api_key_ref = "queued_key".to_owned();
|
|
request.client_api_key_name_snapshot = Some("Original Key".to_owned());
|
|
let first_attempt = attempt(&database.pool, "queued_history", 1, "queued_account").await;
|
|
sqlx::query("delete from provider_accounts where id = 'queued_account'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("delete from client_api_keys where id = 'queued_key'")
|
|
.execute(&database.pool)
|
|
.await
|
|
.unwrap();
|
|
PgExecutionStore::new(database.pool.clone())
|
|
.insert_model_request_with_first_attempt(request, first_attempt)
|
|
.await
|
|
.unwrap();
|
|
let historical: (String, String, String, String, String, String) = sqlx::query_as(
|
|
"select client_api_key_ref, client_api_key_name_snapshot, provider_account_ref, provider_account_name_snapshot, provider_account_plan_type_snapshot, provider_account_notes_snapshot from model_request_observations where id = 'queued_history'",
|
|
).fetch_one(&database.pool).await.unwrap();
|
|
assert_eq!(
|
|
historical,
|
|
(
|
|
"queued_key".to_owned(),
|
|
"Original Key".to_owned(),
|
|
"queued_account".to_owned(),
|
|
"Original Account".to_owned(),
|
|
"pro".to_owned(),
|
|
"Original note".to_owned()
|
|
)
|
|
);
|
|
finalize_request(&database.pool, "queued_history", started_at).await;
|
|
let repository = observability_repository(&database.pool);
|
|
for published in [false, true] {
|
|
if published {
|
|
super::drain_analytics(&database.pool).await;
|
|
}
|
|
for (key_name, account_search, expected) in [
|
|
(Some("Original Key"), None, 1),
|
|
(None, Some("Original Account"), 1),
|
|
(Some("Original Key"), Some("Original Account"), 1),
|
|
(Some("Original"), None, 1),
|
|
(Some("ginal K"), None, 1),
|
|
(None, Some("ginal Acc"), 1),
|
|
(None, Some("old@"), 1),
|
|
(Some("queued_key"), None, 0),
|
|
(None, Some("queued_account"), 0),
|
|
(Some("Original Account"), None, 0),
|
|
(None, Some("Original Key"), 0),
|
|
(Some("Original%"), None, 0),
|
|
] {
|
|
for status in [
|
|
admin_observability::UsageRecordStatus::Success,
|
|
admin_observability::UsageRecordStatus::All,
|
|
] {
|
|
let mut query = usage_query(
|
|
started_at,
|
|
UsageRecordFilter {
|
|
client_api_key_name: key_name.map(str::to_owned),
|
|
provider_account_search: account_search.map(str::to_owned),
|
|
..Default::default()
|
|
},
|
|
);
|
|
query.status = status;
|
|
let page = repository.list_usage_records(query).await.unwrap();
|
|
assert_eq!(
|
|
page.total, expected,
|
|
"published={published}, key={key_name:?}, account={account_search:?}, status={status:?}"
|
|
);
|
|
assert_eq!(page.items.len() as u64, expected);
|
|
if expected == 1 {
|
|
assert_eq!(page.items[0].id, "queued_history");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
database.close().await;
|
|
}
|