Compare commits

...
5 Commits
Author SHA1 Message Date
zyycnandCodex acf574129f chore: bump version to 3.0.28
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 02:26:35 +08:00
zyycnandCodex 6426e0d3de fix(accounts): align availability summaries
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 02:22:23 +08:00
zyycnandCodex eedc39b33a chore: bump version to 3.0.27
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 01:55:51 +08:00
zyycnandCodex 9672a2e000 fix(xai): prioritize structured quota signals
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 01:53:43 +08:00
zyycnandCodex 0fffd2c550 fix(accounts): converge account state and imports
Co-authored-by: Codex <noreply@openai.com>
2026-07-30 01:23:42 +08:00
29 changed files with 862 additions and 233 deletions
@@ -169,6 +169,8 @@ pub struct AccountPage {
}
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
///
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountSummary {
pub total: u64,
@@ -460,6 +460,8 @@ pub struct RequestMetricPoint {
}
/// 账号池容量统计。
///
/// 配额耗尽单列为可恢复状态;`unavailable` 只包含过期、禁用和封禁账号。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AccountPoolMetrics {
pub total: u64,
@@ -374,6 +374,7 @@ impl ProviderAdmin for FakeProviderAdmin {
pub(super) struct FakeAccountStore {
events: EventLog,
account: AccountRecord,
account_after_probe: Mutex<Option<AccountRecord>>,
fail_commit: Mutex<bool>,
audit_requests: Mutex<Vec<String>>,
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
@@ -388,6 +389,7 @@ impl FakeAccountStore {
Arc::new(Self {
events,
account,
account_after_probe: Mutex::new(None),
fail_commit: Mutex::new(false),
audit_requests: Mutex::new(Vec::new()),
quota_window_usage: Mutex::new(Vec::new()),
@@ -406,6 +408,13 @@ impl FakeAccountStore {
*self.quota_window_usage.lock().expect("quota window usage") = usage;
}
fn set_account_after_probe(&self, account: AccountRecord) {
*self
.account_after_probe
.lock()
.expect("account after probe") = Some(account);
}
fn record(&self, event: &'static str) {
self.events.lock().expect("store events").push(event);
}
@@ -445,7 +454,13 @@ impl AccountStore for FakeAccountStore {
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
self.record("store.load_account");
Ok((account_id == self.account.id).then(|| self.account.clone()))
let account = self
.account_after_probe
.lock()
.expect("account after probe")
.clone()
.unwrap_or_else(|| self.account.clone());
Ok((account_id == account.id.as_str()).then_some(account))
}
async fn load_account_usage(
@@ -802,6 +817,36 @@ async fn connection_test_should_probe_unavailable_account() {
));
}
#[tokio::test]
async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events);
let probe = Arc::new(FreeModelQuotaProbe {
store: Arc::clone(&store),
});
let services = accounts_service_with_probe(provider, store, probe).await;
let events = services
.accounts()
.test_connection(
ProviderAccountId::new("acct_test").expect("account ID"),
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
)
.await
.expect("connection test stream")
.collect::<Vec<_>>()
.await;
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
..
})
));
}
#[tokio::test]
async fn connection_test_should_preserve_disabled_account_status() {
let provider = FakeProviderAdmin::new("xai", events());
@@ -1234,6 +1279,30 @@ impl AccountProbe for FailingAccountProbe {
}
}
struct FreeModelQuotaProbe {
store: Arc<FakeAccountStore>,
}
impl AccountProbe for FreeModelQuotaProbe {
fn probe(
&self,
_: AccountProbeRequest,
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
Box::pin(async move {
let mut account = account_record("xai");
account.availability = AccountAvailability::Cooldown;
account.availability_reason = Some("usage_limit_exhausted".to_owned());
account.cooldown_until = Some(Utc::now() + TimeDelta::hours(24));
self.store.set_account_after_probe(account);
Err(GatewayError::new(
GatewayErrorKind::RateLimited,
"xAI free model quota is exhausted",
)
.into())
})
}
}
fn store_unavailable() -> AdminStoreError {
AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
@@ -1653,12 +1653,7 @@ fn format_token_price(cost: &domain::CurrencyCost) -> String {
if cost.currency != "USD" {
return format!("{} {} / 1M Token", cost.currency, cost.amount.as_str());
}
let value = cost
.amount
.as_str()
.parse::<f64>()
.map_or(0.0, |value| value);
format!("${value:.4} / 1M Token")
format!("${} / 1M Token", cost.amount.as_str())
}
fn format_service_tier(service_tier: Option<&str>) -> String {
@@ -2239,6 +2234,9 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
account_usage,
recent_requests,
} = observation;
// 概览卡只有可用和不可用两桶:额度耗尽可恢复,保留在可用;不可用只含终态账号。
let available_accounts =
dashboard_available_account_count(provider_accounts.total, provider_accounts.unavailable);
let mut account_usage_views = Vec::with_capacity(account_usage.len());
let mut credential_usage_views = Vec::with_capacity(account_usage.len());
for credential in account_usage {
@@ -2287,8 +2285,8 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
credentials: DashboardCredentialsCardView {
total: format_compact_number(provider_accounts.total),
total_value: provider_accounts.total,
available: format_compact_number(provider_accounts.active),
available_value: provider_accounts.active,
available: format_compact_number(available_accounts),
available_value: available_accounts,
unavailable: format_compact_number(provider_accounts.unavailable),
unavailable_value: provider_accounts.unavailable,
},
@@ -2344,6 +2342,10 @@ fn dashboard_view(result: domain::DashboardResult, kind: TrendKind) -> Dashboard
}
}
fn dashboard_available_account_count(total: u64, unavailable: u64) -> u64 {
total.saturating_sub(unavailable)
}
fn usage_summary_view(summary: domain::UsageSummary) -> UsageSummaryView {
let overview = summary.overview;
UsageSummaryView {
@@ -340,7 +340,7 @@ mod actions {
upstream_status: Some(429),
upstream_content_type: Some("application/json".to_owned()),
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
account_status: AccountStatus::Active,
account_status: AccountStatus::QuotaExhausted,
},
]
.map(|event| AccountConnectionTestEvent::from(event).data);
@@ -374,7 +374,7 @@ mod actions {
"upstreamStatus": 429,
"upstreamContentType": "application/json",
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
"accountStatus": "active"
"accountStatus": "quota_exhausted"
}),
]
);
+10 -1
View File
@@ -79,6 +79,7 @@ pub(super) struct AdminTestFixture {
pub auth: Arc<MemoryAuthStore>,
pub settings: Arc<MemorySettingsStore>,
pub usage_records: Arc<Mutex<Vec<UsageRecord>>>,
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
}
impl AdminTestFixture {
@@ -92,8 +93,10 @@ impl AdminTestFixture {
let settings = Arc::new(MemorySettingsStore::new(api_key));
let client_keys = Arc::new(MemoryClientKeyStore);
let usage_records = Arc::new(Mutex::new(Vec::new()));
let dashboard_observation = Arc::new(Mutex::new(None));
let unused = Arc::new(UnusedStore {
usage_records: Arc::clone(&usage_records),
dashboard_observation: Arc::clone(&dashboard_observation),
});
let stores = AdminStorePorts::new(
unused.clone(),
@@ -125,6 +128,7 @@ impl AdminTestFixture {
auth,
settings,
usage_records,
dashboard_observation,
}
}
@@ -392,6 +396,7 @@ impl ClientKeyStore for MemoryClientKeyStore {
struct UnusedStore {
usage_records: Arc<Mutex<Vec<UsageRecord>>>,
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
}
#[async_trait]
@@ -503,7 +508,11 @@ impl AccountStore for UnusedStore {
#[async_trait]
impl ObservabilityStore for UnusedStore {
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
Err(unavailable("dashboard"))
self.dashboard_observation
.lock()
.expect("dashboard observation")
.clone()
.ok_or_else(|| unavailable("dashboard"))
}
async fn dashboard_trend(&self, _: TimeRange) -> AdminStoreResult<Vec<RequestMetricPoint>> {
@@ -260,6 +260,80 @@ mod response {
}
}
#[tokio::test]
async fn dashboard_summary_should_count_quota_exhausted_accounts_as_available_in_headline_card() {
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::{Duration, Utc};
use gateway_admin::model::observability::{
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
use super::{AdminTestFixture, AdminTestState};
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let now = Utc::now();
*fixture
.dashboard_observation
.lock()
.expect("dashboard observation") = Some(DashboardObservation {
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
requests: RequestMetrics::default(),
attempts: AttemptMetrics::default(),
provider_accounts: AccountPoolMetrics {
total: 807,
enabled: 804,
unavailable: 7,
active: 15,
expired: 2,
quota_exhausted: 785,
refreshing: None,
disabled: 3,
banned: 2,
},
trend: Vec::new(),
account_usage: Vec::new(),
recent_requests: Vec::new(),
});
let response = observability::router::<AdminTestState>()
.with_state(fixture.state())
.oneshot(
Request::builder()
.uri("/api/admin/dashboard/summary")
.header(header::COOKIE, "cpr_admin_session=valid-session")
.header("x-request-id", "req_dashboard_account_summary")
.body(Body::empty())
.expect("dashboard request"),
)
.await
.expect("dashboard response");
let status = response.status();
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("dashboard response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
assert_eq!(status, StatusCode::OK, "{value}");
assert_eq!(
(
&value["data"]["cards"]["credentials"]["totalValue"],
&value["data"]["cards"]["credentials"]["availableValue"],
&value["data"]["cards"]["credentials"]["unavailableValue"],
),
(
&serde_json::json!(807),
&serde_json::json!(800),
&serde_json::json!(7)
),
);
}
#[tokio::test]
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
use axum::{
@@ -291,12 +365,17 @@ async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
#[tokio::test]
async fn usage_route_should_expose_image_and_websocket_facts() {
use std::str::FromStr as _;
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header},
};
use chrono::Utc;
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
use gateway_admin::model::observability::{
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, RequestOutcome, UsageBilling,
UsageRecord,
};
use gateway_api::admin::observability;
use tower::ServiceExt as _;
@@ -304,6 +383,10 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
let fixture = AdminTestFixture::new().await;
fixture.auth.insert_session("valid-session");
let usd = |amount: &str| CurrencyCost {
currency: "USD".to_owned(),
amount: DecimalAmount::from_str(amount).expect("USD amount"),
};
fixture
.usage_records
.lock()
@@ -359,7 +442,22 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
cost_source: "unavailable".to_owned(),
cost_amount: None,
cost_currency: None,
billing: None,
billing: Some(UsageBilling::Calculated(Box::new(
CalculatedBillingBreakdown {
input_amount: usd("0.03"),
output_amount: usd("0.07"),
cache_read_amount: usd("0.00"),
cache_write_amount: usd("0.00"),
standard_amount: usd("0.10"),
total_amount: usd("0.10"),
input_price_per_million: usd("10.0000"),
output_price_per_million: usd("60.0000"),
cache_read_price_per_million: usd("1.0000"),
cache_write_price_per_million: usd("12.5000"),
service_tier: Some("priority".to_owned()),
multiplier_percent: 100,
},
))),
transport_decision_wait_ms: Some(7),
connect_ms: Some(11),
headers_ms: Some(13),
@@ -400,6 +498,19 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
.expect("usage response body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
assert_eq!(
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
"$10 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
"$60 / 1M Token"
);
assert_eq!(
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
"$12.5 / 1M Token"
);
assert_eq!(
serde_json::json!({
"route": value["data"]["items"][0]["route"],
@@ -1610,43 +1610,26 @@ fn admin_account_summary(
accounts: &[ProviderAccountSummary],
now: DateTime<Utc>,
) -> AccountSummary {
let total = u64::try_from(accounts.len()).unwrap_or(u64::MAX);
let active = u64::try_from(
accounts
.iter()
.filter(|account| admin_account_status(account, now) == AdminAccountStatus::Active)
.count(),
)
.unwrap_or(u64::MAX);
let quota_exhausted = u64::try_from(
accounts
.iter()
.filter(|account| {
admin_account_status(account, now) == AdminAccountStatus::QuotaExhausted
})
.count(),
)
.unwrap_or(u64::MAX);
let unavailable = u64::try_from(
accounts
.iter()
.filter(|account| {
matches!(
admin_account_status(account, now),
AdminAccountStatus::Expired
| AdminAccountStatus::Disabled
| AdminAccountStatus::Banned
)
})
.count(),
)
.unwrap_or(u64::MAX);
AccountSummary {
total,
active,
quota_exhausted,
unavailable,
let mut summary = AccountSummary {
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
active: 0,
quota_exhausted: 0,
unavailable: 0,
};
for account in accounts {
match admin_account_status(account, now) {
AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1),
AdminAccountStatus::QuotaExhausted => {
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
}
AdminAccountStatus::Expired
| AdminAccountStatus::Disabled
| AdminAccountStatus::Banned => {
summary.unavailable = summary.unavailable.saturating_add(1);
}
}
}
summary
}
fn sort_admin_account_items(items: &mut [AdminAccountListItem], sort: Option<AdminAccountSort>) {
@@ -112,7 +112,7 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
}
#[tokio::test]
async fn dashboard_account_metrics_should_partition_available_and_unavailable_accounts() {
async fn dashboard_account_metrics_should_partition_account_statuses() {
let Some(database) = TestDatabase::create("dashboard_account_metrics").await else {
return;
};
@@ -134,7 +134,7 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
'rate_limited', $1 + interval '2 hours', $1, $1, $1),
('acct_usage_limit', 'openai', 'usage-limit', 'user-usage-limit', 'oauth',
('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
'usage_limit_exhausted', $1 + interval '2 hours', $1, $1, $1),
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
@@ -171,6 +171,10 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
),
(6, 5, 2, 3, 1, 1, 1, 1),
);
assert_eq!(
metrics.total,
metrics.active + metrics.quota_exhausted + metrics.unavailable
);
database.close().await;
}
@@ -130,10 +130,13 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
let mut invalid = account("acct_invalid", "user-invalid");
invalid.email = Some("invalid@example.invalid".to_owned());
invalid.availability = AccountAvailability::Invalid;
let mut disabled = account("acct_disabled", "user-disabled");
disabled.email = Some("disabled@example.invalid".to_owned());
disabled.enabled = false;
let mut quota_exhausted = account("acct_quota_exhausted", "user-quota-exhausted");
quota_exhausted.email = Some("quota-exhausted@example.invalid".to_owned());
quota_exhausted.availability = AccountAvailability::QuotaExhausted;
for account in [alpha, beta, charlie, invalid, quota_exhausted] {
for account in [alpha, beta, charlie, invalid, disabled, quota_exhausted] {
repository
.insert_provider_account(account)
.await
@@ -213,10 +216,17 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
.await
.expect("sort accounts by retained usage");
assert_eq!(usage_page.config_revision.get(), 1);
assert_eq!(usage_page.total, 5);
assert_eq!(usage_page.summary.total, 5);
assert_eq!(usage_page.total, 6);
assert_eq!(usage_page.summary.total, 6);
assert_eq!(usage_page.summary.active, 2);
assert_eq!(usage_page.summary.quota_exhausted, 1);
assert_eq!(usage_page.summary.unavailable, 3);
assert_eq!(
usage_page.summary.total,
usage_page.summary.active
+ usage_page.summary.quota_exhausted
+ usage_page.summary.unavailable
);
assert_eq!(
usage_page
.items
@@ -1104,6 +1114,103 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
database.close().await;
}
#[tokio::test]
async fn xai_free_quota_exhaustion_state_is_persisted() {
let Some(database) = TestDatabase::create("provider_account_xai_free_quota").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let account_id = ProviderAccountId::new("acct_xai_free_quota").expect("account ID");
let mut imported = account(account_id.as_str(), "user-xai-free-quota");
imported.provider_kind = "xai".to_owned();
repository
.insert_provider_account(imported)
.await
.expect("insert xAI account");
repository
.apply_state_change(AccountStateChange {
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::QuotaExhausted,
reason: Some("upstream_free_quota_exhausted".to_owned()),
cooldown_until: None,
observed_at: SystemTime::now(),
})
.await
.expect("persist xAI free quota exhaustion");
let current: (String, String, Option<String>) = sqlx::query_as(
"select provider_kind, availability, availability_reason
from provider_accounts where id = $1",
)
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("load persisted xAI quota state");
assert_eq!(
current,
(
"xai".to_owned(),
"quota_exhausted".to_owned(),
Some("upstream_free_quota_exhausted".to_owned()),
),
);
database.close().await;
}
#[tokio::test]
async fn xai_resettable_usage_limit_state_is_persisted() {
let Some(database) = TestDatabase::create("provider_account_xai_usage_limit").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let account_id = ProviderAccountId::new("acct_xai_usage_limit").expect("account ID");
let mut imported = account(account_id.as_str(), "user-xai-usage-limit");
imported.provider_kind = "xai".to_owned();
repository
.insert_provider_account(imported)
.await
.expect("insert xAI account");
let observed_at = SystemTime::now();
let cooldown_until = observed_at
.checked_add(Duration::from_secs(24 * 60 * 60))
.expect("cooldown deadline");
repository
.apply_state_change(AccountStateChange {
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Cooldown,
reason: Some("usage_limit_exhausted".to_owned()),
cooldown_until: Some(cooldown_until),
observed_at,
})
.await
.expect("persist xAI resettable usage limit");
let current: (String, String, Option<String>, bool) = sqlx::query_as(
"select provider_kind, availability, availability_reason, cooldown_until is not null
from provider_accounts where id = $1",
)
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("load persisted xAI usage limit state");
assert_eq!(
current,
(
"xai".to_owned(),
"cooldown".to_owned(),
Some("usage_limit_exhausted".to_owned()),
true,
),
);
database.close().await;
}
#[tokio::test]
async fn disabled_account_preserves_user_state_during_refresh_writes() {
let Some(database) = TestDatabase::create("provider_account_disabled_refresh").await else {
@@ -16,9 +16,16 @@ pub(crate) fn derive_codex_session_affinity_key(
.map(str::trim)
.filter(|value| !value.is_empty())
{
return ProviderSessionAffinityKey::try_new(conversation_id.to_owned()).ok();
if let Ok(key) = ProviderSessionAffinityKey::try_new(conversation_id.to_owned()) {
return Some(key);
}
return opaque_affinity_key("local-conversation", conversation_id);
}
let (domain, value) = derive_conversation_anchor(request)?;
opaque_affinity_key(domain, &value)
}
fn opaque_affinity_key(domain: &str, value: &str) -> Option<ProviderSessionAffinityKey> {
let mut hasher = Sha256::new();
hasher.update(b"codex-session-affinity-v1\0");
hasher.update(domain.as_bytes());
@@ -161,13 +161,14 @@ pub(crate) fn derive_conversation_anchor(
request: &CodexResponsesRequest,
) -> Option<(&'static str, String)> {
request
.prompt_cache_key()
.map(|value| ("prompt-cache", value.to_owned()))
.client_session_id
.as_deref()
.map(|value| ("session", value.to_owned()))
.or_else(|| {
request
.client_session_id
.client_conversation_id
.as_deref()
.map(|value| ("session", value.to_owned()))
.map(|value| ("conversation", value.to_owned()))
})
.or_else(|| {
request
@@ -177,9 +178,8 @@ pub(crate) fn derive_conversation_anchor(
})
.or_else(|| {
request
.client_conversation_id
.as_deref()
.map(|value| ("conversation", value.to_owned()))
.prompt_cache_key()
.map(|value| ("prompt-cache", value.to_owned()))
})
.or_else(|| derive_stable_conversation_key(request).map(|value| ("request", value)))
}
@@ -784,6 +784,94 @@ async fn prompt_cache_key_should_become_an_opaque_session_affinity_lookup_key()
assert!(keys[0].bytes().all(|byte| byte.is_ascii_hexdigit()));
}
#[tokio::test]
async fn explicit_session_id_should_override_turn_specific_prompt_cache_keys_for_affinity() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_session_affinity").await;
let affinity = Arc::new(MemorySessionAffinity::default());
let provider = provider_with_affinity(&store, Arc::clone(&affinity));
for (request_id, prompt_cache_key) in [
("req_session_affinity_first", "turn-cache-first"),
("req_session_affinity_second", "turn-cache-second"),
] {
let generation = GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object(
"openai",
Map::from_iter([
("model".to_owned(), json!("gpt-5.4")),
("input".to_owned(), json!("hello")),
("prompt_cache_key".to_owned(), json!(prompt_cache_key)),
]),
)
.expect("OpenAI payload")
.with_context(Map::from_iter([(
"session_id".to_owned(),
json!("stable-client-session"),
)])),
);
let stream = provider
.execute(
planned_request("openai", Operation::Generate(generation)),
context(request_id, CancellationToken::new()),
)
.await
.expect("prepare provider stream");
drop(stream);
}
let keys = affinity.lookup_keys();
assert_eq!(keys.len(), 2);
assert_eq!(keys[0], keys[1]);
}
#[tokio::test]
async fn invalid_local_conversation_id_should_fall_back_to_an_opaque_affinity_key() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_local_affinity").await;
let affinity = Arc::new(MemorySessionAffinity::default());
let generation = GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object(
"openai",
Map::from_iter([
("model".to_owned(), json!("gpt-5.4")),
("input".to_owned(), json!("hello")),
]),
)
.expect("OpenAI payload"),
)
.with_provider_session_state(
ProviderSessionState::new(
"openai",
Map::from_iter([
("account_id".to_owned(), json!("acct_local_affinity")),
(
"conversation_id".to_owned(),
json!("lc_UppercaseBase64ConversationId"),
),
("continuation_scope".to_owned(), json!("replay_required")),
("transcript".to_owned(), json!([])),
]),
)
.expect("provider session state"),
);
let stream = provider_with_affinity(&store, Arc::clone(&affinity))
.execute(
planned_request("openai", Operation::Generate(generation)),
context("req_local_affinity", CancellationToken::new()),
)
.await
.expect("prepare provider stream");
drop(stream);
let keys = affinity.lookup_keys();
assert_eq!(keys.len(), 1);
assert_eq!(keys[0].len(), 64);
assert!(keys[0].bytes().all(|byte| byte.is_ascii_hexdigit()));
}
#[tokio::test]
async fn completed_response_persists_session_affinity_before_stream_consumer_stops_polling() {
let store = Arc::new(MemoryAccountStore::default());
@@ -479,15 +479,16 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
"upstream_payment_quota_exhausted",
None,
),
GrokCredentialFailure::ModelQuotaExhausted {
upstream_model,
retry_after,
} => {
GrokCredentialFailure::ModelQuotaExhausted { retry_after, .. } => {
let retry_after =
bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN);
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
.await;
return;
// xAI 免费模型额度按账号滚动窗口恢复;持久化使页面和各调度节点收敛。
self.record_runtime_cooldown(session, retry_after).await;
(
GrokCredentialAvailability::Cooldown,
"usage_limit_exhausted",
persistent_cooldown_until(observed_at, retry_after),
)
}
GrokCredentialFailure::ModelAccessDenied {
upstream_model,
+87 -40
View File
@@ -60,8 +60,9 @@ use crate::transport::{
GrokCompactionSummaryDecoder, GrokCredentialFailure, GrokInferenceChunkStream,
GrokInferenceRequest, GrokInferenceResponse, GrokInferenceTransport,
GrokInferenceTransportError, GrokInferenceTransportErrorKind, GrokInferenceTransportMetrics,
GrokProviderConfigError, GrokRequestEncodeError, GrokResponsesRequest, GrokSessionAffinityKey,
GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError, SelectedGrokSession,
GrokProviderConfigError, GrokQuotaFailureKind, GrokRequestEncodeError, GrokResponsesRequest,
GrokSessionAffinityKey, GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError,
SelectedGrokSession, classify_grok_quota_failure,
};
use crate::{GrokCatalogCapabilityEvidence, GrokCatalogModel};
@@ -1118,9 +1119,21 @@ fn cold_compaction_http_sse_stream(
.await
.map_err(mark_transient_compaction_failure)?
{
let events = canonical.push(&chunk).map_err(|error| {
mark_transient_compaction_failure(map_continuation_failure(&context, error))
})?;
let events = match canonical.push(&chunk) {
Ok(events) => events,
Err(error) => {
let error = map_continuation_failure(&context, error);
let error = record_stream_failure(
selector.as_ref(),
&session,
error,
&upstream_model,
)
.await;
Err(mark_transient_compaction_failure(error))?;
return;
}
};
for event in events {
summary.observe(&event).map_err(map_compaction_decode_error)?;
facts.observe(&event);
@@ -1131,9 +1144,22 @@ fn cold_compaction_http_sse_stream(
}
if facts.completed.is_none() {
for event in canonical.finish_without_terminal().map_err(|error| {
mark_transient_compaction_failure(map_continuation_failure(&context, error))
})? {
let events = match canonical.finish_without_terminal() {
Ok(events) => events,
Err(error) => {
let error = map_continuation_failure(&context, error);
let error = record_stream_failure(
selector.as_ref(),
&session,
error,
&upstream_model,
)
.await;
Err(mark_transient_compaction_failure(error))?;
return;
}
};
for event in events {
summary.observe(&event).map_err(map_compaction_decode_error)?;
facts.observe(&event);
if facts.completed.is_some() {
@@ -1371,7 +1397,13 @@ fn cold_http_sse_stream(
Ok(events) => events,
Err(error) => {
let error = map_continuation_failure(&context, error);
let error = record_stream_failure(selector.as_ref(), &session, error).await;
let error = record_stream_failure(
selector.as_ref(),
&session,
error,
&upstream_model,
)
.await;
Err(error)?;
return;
}
@@ -1399,7 +1431,13 @@ fn cold_http_sse_stream(
Ok(events) => events,
Err(error) => {
let error = map_continuation_failure(&context, error);
let error = record_stream_failure(selector.as_ref(), &session, error).await;
let error = record_stream_failure(
selector.as_ref(),
&session,
error,
&upstream_model,
)
.await;
Err(error)?;
return;
}
@@ -1508,28 +1546,6 @@ fn xai_transport_metadata(
ProviderResponseMetadata::new(serde_json::to_string(&Value::Object(metadata)).ok()?)
}
async fn record_failure(
selector: &dyn GrokSessionSelector,
session: &SelectedGrokSession,
error: ProviderError,
) -> ProviderError {
if !session.allows_account_state_mutation() {
return error;
}
let failure = match error.kind() {
ProviderErrorKind::Unauthorized => Some(GrokCredentialFailure::Unauthorized),
ProviderErrorKind::RateLimited => Some(GrokCredentialFailure::RateLimited {
retry_after: error.retry_after(),
}),
ProviderErrorKind::QuotaExhausted => Some(GrokCredentialFailure::QuotaExhausted),
_ => None,
};
if let Some(failure) = failure {
selector.record_failure(session, failure).await;
}
error
}
async fn record_credential_failure(
selector: &dyn GrokSessionSelector,
session: &SelectedGrokSession,
@@ -1546,17 +1562,13 @@ async fn record_stream_failure(
selector: &dyn GrokSessionSelector,
session: &SelectedGrokSession,
error: ProviderError,
upstream_model: &UpstreamModelId,
) -> ProviderError {
if !session.allows_account_state_mutation() {
return error;
}
if matches!(
error.kind(),
ProviderErrorKind::Unauthorized
| ProviderErrorKind::RateLimited
| ProviderErrorKind::QuotaExhausted
) {
return record_failure(selector, session, error).await;
if let Some(failure) = stream_credential_failure(&error, upstream_model) {
return record_credential_failure(selector, session, error, failure).await;
}
if error.kind() != ProviderErrorKind::Cancelled {
selector
@@ -1566,6 +1578,41 @@ async fn record_stream_failure(
error
}
fn stream_credential_failure(
error: &ProviderError,
upstream_model: &UpstreamModelId,
) -> Option<GrokCredentialFailure> {
match error.kind() {
ProviderErrorKind::Unauthorized => Some(GrokCredentialFailure::Unauthorized),
ProviderErrorKind::RateLimited => Some(GrokCredentialFailure::RateLimited {
retry_after: error.retry_after(),
}),
ProviderErrorKind::QuotaExhausted => {
let visible = error.client_visible_upstream_error();
let quota_kind = classify_grok_quota_failure(
visible
.and_then(ClientVisibleUpstreamError::code)
.or_else(|| error.upstream_code().map(|code| code.as_str())),
visible.and_then(ClientVisibleUpstreamError::error_type),
visible.map(ClientVisibleUpstreamError::message),
);
Some(match quota_kind {
Some(GrokQuotaFailureKind::FreeAccount) => {
GrokCredentialFailure::FreeQuotaExhausted
}
Some(GrokQuotaFailureKind::FreeModelUsage) => {
GrokCredentialFailure::ModelQuotaExhausted {
upstream_model: upstream_model.clone(),
retry_after: error.retry_after(),
}
}
Some(GrokQuotaFailureKind::Account) | None => GrokCredentialFailure::QuotaExhausted,
})
}
_ => None,
}
}
async fn recover_or_record_failure(
selector: &dyn GrokSessionSelector,
recovery: &dyn GrokCredentialRecovery,
@@ -1644,7 +1691,7 @@ async fn map_and_record_stream_transport_failure(
match credential_failure {
Some(failure) => record_credential_failure(selector, session, error, failure).await,
None if request_scoped => error,
None => record_stream_failure(selector, session, error).await,
None => record_stream_failure(selector, session, error, upstream_model).await,
}
}
@@ -19,7 +19,7 @@ use gateway_protocol::openai::sse::{SseEvent, SseEventDecoder};
use serde_json::Value;
use super::request::{GrokResponseTransform, GrokResponsesRequest};
use super::scrub_account_fingerprints;
use super::{classify_grok_quota_failure, scrub_account_fingerprints};
const CONTENTS_PER_OUTPUT: u32 = 1_024;
const LONG_CONTEXT_THRESHOLD: u64 = 200_000;
@@ -785,14 +785,19 @@ fn incomplete_finish_reason(response: &Value) -> FinishReason {
fn upstream_event_error(value: &Value) -> ProviderError {
let code = upstream_error_field(value, "code");
let kind = match code {
Some("invalid_request" | "invalid_prompt") => ProviderErrorKind::InvalidRequest,
Some("unsupported" | "unsupported_feature") => ProviderErrorKind::Unsupported,
Some("unauthorized" | "invalid_token") => ProviderErrorKind::Unauthorized,
Some("permission_denied") => ProviderErrorKind::PermissionDenied,
Some("rate_limit_exceeded") => ProviderErrorKind::RateLimited,
Some("quota_exceeded" | "insufficient_quota") => ProviderErrorKind::QuotaExhausted,
_ => ProviderErrorKind::Unavailable,
let error_type = upstream_error_field(value, "type");
let message = upstream_error_field(value, "message");
let kind = if classify_grok_quota_failure(code, error_type, message).is_some() {
ProviderErrorKind::QuotaExhausted
} else {
match code {
Some("invalid_request" | "invalid_prompt") => ProviderErrorKind::InvalidRequest,
Some("unsupported" | "unsupported_feature") => ProviderErrorKind::Unsupported,
Some("unauthorized" | "invalid_token") => ProviderErrorKind::Unauthorized,
Some("permission_denied") => ProviderErrorKind::PermissionDenied,
Some("rate_limit_exceeded") => ProviderErrorKind::RateLimited,
_ => ProviderErrorKind::Unavailable,
}
};
let mut error = ProviderError::new(kind, UpstreamSendState::Sent)
.redact_sensitive_context("upstream event");
@@ -801,11 +806,11 @@ fn upstream_event_error(value: &Value) -> ProviderError {
}
// 结构化 message/code/type 供原客户端展示与重试分类;message 先脱去
// 账号指纹(上游限流文案内嵌 team UUID),非结构化正文不透出。
if let Some(message) = upstream_error_field(value, "message") {
if let Some(message) = message {
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
scrub_account_fingerprints(message),
code.map(str::to_owned),
upstream_error_field(value, "type").map(str::to_owned),
error_type.map(str::to_owned),
));
}
error
@@ -56,6 +56,85 @@ pub use transport::{
};
const UUID_TEXT_LEN: usize = 36;
const FREE_USAGE_SIGNALS: &[&str] = &[
"subscription:free-usage-exhausted",
"subscription_free_usage_exhausted",
"free-usage-exhausted",
"free_usage_exhausted",
"used all the included free usage",
"used all your free usage",
];
const ACCOUNT_QUOTA_SIGNALS: &[&str] = &[
"personal-team-blocked:spending-limit",
"personal_team_blocked_spending_limit",
"quota_exceeded",
"insufficient_quota",
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum GrokQuotaFailureKind {
Account,
FreeAccount,
FreeModelUsage,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum GrokQuotaSignal {
Account,
FreeUsage,
}
pub(crate) fn classify_grok_quota_failure(
code: Option<&str>,
error_type: Option<&str>,
message: Option<&str>,
) -> Option<GrokQuotaFailureKind> {
let code = code.map(str::to_ascii_lowercase);
let error_type = error_type.map(str::to_ascii_lowercase);
let message = message.map(str::to_ascii_lowercase);
// `code` and `type` are stable upstream fields. Do not let a conflicting
// human-readable message override them; use the message only as a
// fallback or to refine a confirmed free-usage signal's rolling window.
let structured_signal = code
.as_deref()
.and_then(classify_quota_signal)
.or_else(|| error_type.as_deref().and_then(classify_quota_signal));
match structured_signal {
Some(GrokQuotaSignal::Account) => Some(GrokQuotaFailureKind::Account),
Some(GrokQuotaSignal::FreeUsage) => Some(free_usage_failure_kind(message.as_deref())),
None => classify_message_quota_failure(message.as_deref()),
}
}
fn classify_quota_signal(value: &str) -> Option<GrokQuotaSignal> {
if contains_any(value, ACCOUNT_QUOTA_SIGNALS) {
Some(GrokQuotaSignal::Account)
} else if contains_any(value, FREE_USAGE_SIGNALS) {
Some(GrokQuotaSignal::FreeUsage)
} else {
None
}
}
fn classify_message_quota_failure(message: Option<&str>) -> Option<GrokQuotaFailureKind> {
match message.and_then(classify_quota_signal) {
Some(GrokQuotaSignal::Account) => Some(GrokQuotaFailureKind::Account),
Some(GrokQuotaSignal::FreeUsage) => Some(free_usage_failure_kind(message)),
None => None,
}
}
fn free_usage_failure_kind(message: Option<&str>) -> GrokQuotaFailureKind {
if message.is_some_and(|value| value.contains("for model")) {
GrokQuotaFailureKind::FreeModelUsage
} else {
GrokQuotaFailureKind::FreeAccount
}
}
fn contains_any(value: &str, needles: &[&str]) -> bool {
needles.iter().any(|needle| value.contains(needle))
}
/// 把 message 中的 UUID 替换为占位符,控制字符归一为空格,
/// 使文本满足客户端可见错误的约束且不携带可定位账号的标识。
@@ -33,7 +33,7 @@ use gateway_core::engine::UpstreamSendState;
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue};
use gateway_core::event::UpstreamHttpVersion;
use super::scrub_account_fingerprints;
use super::{GrokQuotaFailureKind, classify_grok_quota_failure, scrub_account_fingerprints};
pub(crate) const OFFICIAL_OAUTH_HOST: &str = "auth.x.ai";
const OFFICIAL_INFERENCE_HOST: &str = "cli-chat-proxy.grok.com";
@@ -1082,15 +1082,10 @@ fn first_string(object: &serde_json::Map<String, Value>, fields: &[&str]) -> Opt
}
fn classify_payment_required(metadata: &InferenceErrorMetadata) -> GrokInferenceTransportErrorKind {
if model_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::ModelQuotaExhausted
} else if paid_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::QuotaExhausted
} else if free_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::FreeQuotaExhausted
} else {
GrokInferenceTransportErrorKind::PaymentRequired
}
quota_failure(metadata).map_or(
GrokInferenceTransportErrorKind::PaymentRequired,
quota_transport_error_kind,
)
}
fn classify_forbidden(
@@ -1103,12 +1098,8 @@ fn classify_forbidden(
GrokInferenceTransportErrorKind::SafetyRejected
} else if credential_rejected(&text) {
GrokInferenceTransportErrorKind::Unauthorized
} else if model_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::ModelQuotaExhausted
} else if paid_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::QuotaExhausted
} else if free_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::FreeQuotaExhausted
} else if let Some(failure) = quota_failure(metadata) {
quota_transport_error_kind(failure)
} else if model_access_denied(metadata) {
GrokInferenceTransportErrorKind::ModelAccessDenied
} else {
@@ -1119,15 +1110,10 @@ fn classify_forbidden(
fn classify_too_many_requests(
metadata: &InferenceErrorMetadata,
) -> GrokInferenceTransportErrorKind {
if model_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::ModelQuotaExhausted
} else if paid_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::QuotaExhausted
} else if free_quota_exhausted(metadata) {
GrokInferenceTransportErrorKind::FreeQuotaExhausted
} else {
GrokInferenceTransportErrorKind::RateLimited
}
quota_failure(metadata).map_or(
GrokInferenceTransportErrorKind::RateLimited,
quota_transport_error_kind,
)
}
fn reasoning_decode_failed(metadata: &InferenceErrorMetadata) -> bool {
@@ -1137,32 +1123,22 @@ fn reasoning_decode_failed(metadata: &InferenceErrorMetadata) -> bool {
})
}
fn paid_quota_exhausted(metadata: &InferenceErrorMetadata) -> bool {
contains_any(
&inference_metadata_text(metadata),
&[
"personal-team-blocked:spending-limit",
"personal_team_blocked_spending_limit",
],
fn quota_failure(metadata: &InferenceErrorMetadata) -> Option<GrokQuotaFailureKind> {
classify_grok_quota_failure(
metadata.code.as_deref(),
metadata.error_type.as_deref(),
metadata.message.as_deref(),
)
}
fn free_quota_exhausted(metadata: &InferenceErrorMetadata) -> bool {
contains_any(
&inference_metadata_text(metadata),
&[
"subscription:free-usage-exhausted",
"subscription_free_usage_exhausted",
"free-usage-exhausted",
"free_usage_exhausted",
"used all the included free usage",
"used all your free usage",
],
)
}
fn model_quota_exhausted(metadata: &InferenceErrorMetadata) -> bool {
inference_metadata_text(metadata).contains("used all the included free usage for model")
fn quota_transport_error_kind(failure: GrokQuotaFailureKind) -> GrokInferenceTransportErrorKind {
match failure {
GrokQuotaFailureKind::Account => GrokInferenceTransportErrorKind::QuotaExhausted,
GrokQuotaFailureKind::FreeAccount => GrokInferenceTransportErrorKind::FreeQuotaExhausted,
GrokQuotaFailureKind::FreeModelUsage => {
GrokInferenceTransportErrorKind::ModelQuotaExhausted
}
}
}
fn model_access_denied(metadata: &InferenceErrorMetadata) -> bool {
@@ -319,7 +319,7 @@ pub enum GrokCredentialFailure {
/// 已解析并脱敏的延迟;缺失时由 selector 使用短暂默认值。
retry_after: Option<Duration>,
},
/// 选中账号只耗尽了指定模型的额度。
/// 上游声明指定模型的免费用量耗尽;selector 会收敛为账号级可恢复额度状态。
ModelQuotaExhausted {
upstream_model: UpstreamModelId,
retry_after: Option<Duration>,
@@ -282,7 +282,7 @@ pub enum GrokInferenceTransportErrorKind {
QuotaExhausted,
/// 账号订阅级免费额度耗尽。
FreeQuotaExhausted,
/// 当前账号只耗尽了指定模型的免费额度。
/// 上游声明指定模型的免费用量耗尽;selector 会收敛为账号级可恢复额度状态。
ModelQuotaExhausted,
/// 当前账号只缺少指定模型的访问权限。
ModelAccessDenied,
@@ -490,7 +490,7 @@ async fn payment_required_feedback_persists_account_quota_exhaustion() {
}
#[tokio::test]
async fn model_quota_feedback_blocks_only_the_failed_account_and_model() {
async fn free_model_quota_feedback_persists_resettable_account_exhaustion() {
let fixture = SelectorFixture::new(&["model-quota", "available"]).await;
let failed_model = UpstreamModelId::new("grok-4.5").expect("failed model");
let session = fixture
@@ -505,38 +505,47 @@ async fn model_quota_feedback_blocks_only_the_failed_account_and_model() {
&session,
GrokCredentialFailure::ModelQuotaExhausted {
upstream_model: failed_model.clone(),
retry_after: Some(Duration::from_secs(5)),
retry_after: None,
},
)
.await;
assert_eq!(
fixture
.store
.account(&selected)
.expect("selected account")
.availability(),
AccountAvailability::Ready
let account = fixture.store.account(&selected).expect("selected account");
assert_eq!(account.availability(), AccountAvailability::Cooldown);
let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60);
assert!(
account
.cooldown_until()
.is_some_and(|until| until > minimum_until)
);
let scope = ProviderCooldownScope::upstream_model(failed_model);
assert!(
fixture
.cooldowns
.scoped_cooldown(&selected, &scope)
.is_some()
.is_none()
);
let unrelated_model = fixture
assert!(
fixture
.cooldowns
.cooldown(&selected)
.is_some_and(|cooldown| cooldown.until() > minimum_until)
);
assert!(matches!(
fixture
.selector
.select(fixture.request_for_model("grok-4.6", Some(selected.clone())))
.await,
Err(GrokSessionSelectorError::AccountCoolingDown {
retry_after: Some(_)
})
));
let next = fixture
.selector
.select(fixture.request_for_model("grok-4.6", Some(selected.clone())))
.select(fixture.request_for_model("grok-4.6", None))
.await
.expect("same account remains valid for another model");
assert_eq!(unrelated_model.account_id(), &selected);
let failed_model_retry = fixture
.selector
.select(fixture.request_for_model("grok-4.5", None))
.await
.expect("another account serves the failed model");
assert_ne!(failed_model_retry.account_id(), &selected);
.expect("another account remains schedulable");
assert_ne!(next.account_id(), &selected);
}
#[tokio::test]
@@ -1518,6 +1518,73 @@ async fn model_quota_transport_failure_preserves_model_scoped_feedback() {
);
}
#[tokio::test]
async fn free_quota_sse_failure_marks_the_account_exhausted() {
let selector = StubSelector::success();
let body = concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_free_quota\",\"model\":\"grok-4.5\"}}\n\n",
"event: response.failed\n",
"data: {\"type\":\"response.failed\",\"error\":{\"code\":\"subscription:free-usage-exhausted\",\"message\":\"You have used all your free usage\"}}\n\n",
)
.as_bytes()
.to_vec();
let transport = StubInferenceTransport::sequence([InferenceMode::SuccessBody(body)]);
let provider = provider(selector.clone(), transport).await;
let mut stream = provider
.execute(
provider_request("xai"),
context(CancellationToken::new(), None),
)
.await
.expect("stream");
let error = next_provider_error(&mut stream).await;
assert_eq!(error.kind(), ProviderErrorKind::QuotaExhausted);
assert_eq!(
selector.feedback.lock().expect("feedback").as_slice(),
&[GrokCredentialFailure::FreeQuotaExhausted]
);
}
#[tokio::test]
async fn free_model_quota_sse_failure_is_classified_for_resettable_account_cooldown() {
let selector = StubSelector::success();
let body = concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_model_quota\",\"model\":\"grok-4.5\"}}\n\n",
"event: response.failed\n",
"data: {\"type\":\"response.failed\",\"error\":{\"status\":null,\"contentType\":null,\"body\":null,\"code\":\"subscription_free_usage_exhausted\",\"type\":null,\"message\":\"You've used all the included free usage for model grok-4.5-0722 for now. Usage resets over a rolling 24-hour window \u{2014} tokens (actual/limit): 500505/500000. Upgrade to a Grok subscription for higher limits: https://grok.com/supergrok\"}}\n\n",
)
.as_bytes()
.to_vec();
let transport = StubInferenceTransport::sequence([InferenceMode::SuccessBody(body)]);
let provider = provider(selector.clone(), transport).await;
let mut stream = provider
.execute(
provider_request("xai"),
context(CancellationToken::new(), None),
)
.await
.expect("stream");
let error = next_provider_error(&mut stream).await;
assert_eq!(error.kind(), ProviderErrorKind::QuotaExhausted);
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
Some("subscription_free_usage_exhausted")
);
assert_eq!(
selector.feedback.lock().expect("feedback").as_slice(),
&[GrokCredentialFailure::ModelQuotaExhausted {
upstream_model: UpstreamModelId::new("grok-4.5").expect("model"),
retry_after: None,
}]
);
}
#[tokio::test]
async fn model_access_denial_is_retryable_without_credential_recovery() {
let selector = StubSelector::success();
@@ -802,6 +802,28 @@ fn decoder_should_classify_failed_event_without_retaining_body() {
assert!(!format!("{error:?}").contains("secret"));
}
#[test]
fn decoder_should_classify_free_quota_failed_event() {
let body = concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_quota\"}}\n\n",
"event: response.failed\n",
"data: {\"type\":\"response.failed\",\"error\":{\"code\":\"subscription:free-usage-exhausted\",\"type\":\"billing_error\",\"message\":\"You have used all your free usage\"}}\n\n",
);
let error = GrokCanonicalDecoder::new("fallback")
.push(body.as_bytes())
.expect_err("free quota failure must surface");
assert_eq!(
error.kind(),
gateway_core::error::ProviderErrorKind::QuotaExhausted
);
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
Some("subscription:free-usage-exhausted")
);
}
#[test]
fn decoder_should_scrub_account_fingerprints_from_failure_messages() {
let body = concat!(
@@ -471,8 +471,12 @@ async fn inference_transport_should_classify_model_quota_for_403_and_429() {
.and(path("/v1/responses"))
.respond_with(ResponseTemplate::new(status).set_body_json(json!({
"error": {
"code": "subscription:free-usage-exhausted",
"message": "You have used all the included free usage for model grok-4.5"
"status": null,
"contentType": null,
"body": null,
"code": "subscription_free_usage_exhausted",
"type": null,
"message": "You've used all the included free usage for model grok-4.5-0722 for now. Usage resets over a rolling 24-hour window - tokens (actual/limit): 500505/500000. Upgrade to a Grok subscription for higher limits: https://grok.com/supergrok"
}
})))
.expect(1)
@@ -491,6 +495,38 @@ async fn inference_transport_should_classify_model_quota_for_403_and_429() {
}
}
#[tokio::test]
async fn inference_transport_should_prefer_structured_quota_fields_over_conflicting_message() {
for (code, error_type) in [
("quota_exceeded", Some("subscription_free_usage_exhausted")),
("rate_limit_exceeded", Some("quota_exceeded")),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/responses"))
.respond_with(ResponseTemplate::new(429).set_body_json(json!({
"error": {
"code": code,
"type": error_type,
"message": "You've used all the included free usage for model grok-4.5-0722 for now."
}
})))
.expect(1)
.mount(&server)
.await;
let origin = Url::parse(&server.uri()).expect("wiremock origin");
let error = inference_transport(&origin)
.execute(inference_request(&origin))
.await
.expect_err("quota response must fail");
assert_eq!(
error.kind(),
GrokInferenceTransportErrorKind::QuotaExhausted
);
}
}
#[tokio::test]
async fn inference_transport_should_classify_free_usage_429_as_account_free_quota_exhausted() {
let server = MockServer::start().await;
@@ -193,19 +193,6 @@ function connectionLogClass(tone: string) {
</div>
</BaseScrollbar>
</div>
<div v-if="error" class="mt-3 rounded-lg bg-(--cp-danger-bg) px-3 py-2.5">
<p class="m-0 text-[11px] font-heavy text-(--cp-danger-text)">
错误信息
</p>
<BaseScrollbar max-height="118px" view-class="pt-1.5 pr-2">
<p
class="m-0 wrap-break-word text-[12px] leading-[1.55] font-emphasis text-(--cp-danger-text)"
>
{{ error }}
</p>
</BaseScrollbar>
</div>
</section>
</div>
@@ -280,17 +280,7 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
if (event.type === 'error') {
applyAccountStatus(event.accountStatus)
connectionTestError.value = connectionTestErrorText(event)
const detail = event.upstreamStatus || event.upstreamContentType || event.upstreamBody
|| event.providerErrorCode || event.providerErrorType
? {
status: event.upstreamStatus,
contentType: event.upstreamContentType,
body: event.upstreamBody,
code: event.providerErrorCode,
type: event.providerErrorType,
}
: undefined
appendConnectionTestLog(connectionTestError.value, 'danger', detail)
appendConnectionTestLog('上游返回错误', 'danger', connectionTestError.value)
finishConnectionTest('error')
clearConnectionTestRun()
}
@@ -148,15 +148,20 @@ export function useAccountOnboarding(options: {
}
async function importAccountDocument() {
const data = parseImportJson(createForm.value.importText)
if (Array.isArray(data) || typeof data !== 'object' || data === null)
throw new Error('导入文件必须是 JSON object')
const result = await importAccounts({
provider: createForm.value.provider,
data,
})
const provider = createForm.value.provider === 'xai' ? 'xAI' : 'OpenAI'
return `${provider} 账号已导入 ${result.importedCount} 个`
const provider = requireImportProvider(createForm.value.provider)
const documents = providerImportDocuments(
parseImportJson(createForm.value.importText),
provider,
)
let importedCount = 0
for (const entry of documents) {
const result = await importAccounts({
provider,
data: entry.document,
})
importedCount += result.importedCount
}
return `${providerDisplayName(provider) ?? provider} 账号已导入 ${importedCount} 个`
}
async function importMixedAccountDocument() {
@@ -259,6 +264,27 @@ function parseImportJson(value: string) {
}
}
function requireImportProvider(value: string): ImportProvider {
if (value === 'openai' || value === 'xai')
return value
throw new Error('请选择要导入的账号平台')
}
function providerImportDocuments(value: unknown, provider: ImportProvider): MixedImportDocument[] {
if (isRecord(value) && Array.isArray(value.documents)) {
const documents = parseMixedImportDocuments(value)
.filter(entry => entry.provider === provider)
if (documents.length === 0) {
const label = providerDisplayName(provider) ?? provider
throw new Error(`批量导入文件不包含 ${label} 账号文档`)
}
return documents
}
if (!isRecord(value))
throw new Error('导入文件必须是 JSON object')
return [{ provider, document: value }]
}
function parseMixedImportDocuments(value: unknown): MixedImportDocument[] {
if (!isRecord(value) || !Array.isArray(value.documents))
throw new Error('批量导入文件必须是 CPR 多平台导出文件')
@@ -56,17 +56,17 @@ function itemValueClass(tone?: string, accent?: boolean) {
<UsageDetailPopover v-if="billing" title="计费明细" width="248px" trigger-label="查看费用明细">
<div class="grid gap-1.5 text-(--cp-text-secondary)">
<div v-for="item in amountItems" :key="item.label" class="flex justify-between gap-4">
<span>{{ item.label }}</span>
<span class="font-mono font-heavy" :class="itemValueClass(undefined, item.accent)">
<div v-for="item in amountItems" :key="item.label" class="grid grid-cols-[auto_minmax(0,1fr)] items-center gap-4">
<span class="whitespace-nowrap">{{ item.label }}</span>
<span class="justify-self-end whitespace-nowrap font-mono font-heavy" :class="itemValueClass(undefined, item.accent)">
{{ item.value }}
</span>
</div>
</div>
<div class="mt-1 grid gap-1.5 rounded-(--cp-input-radius-base) bg-(--cp-bg-subtle) p-2 text-(--cp-text-secondary)">
<div v-for="item in billingItems" :key="item.label" class="flex justify-between gap-4">
<span>{{ item.label }}</span>
<span class="font-mono font-heavy" :class="itemValueClass(item.tone)">
<div v-for="item in billingItems" :key="item.label" class="grid grid-cols-[auto_minmax(0,1fr)] items-center gap-4">
<span class="whitespace-nowrap">{{ item.label }}</span>
<span class="justify-self-end whitespace-nowrap font-mono font-heavy" :class="itemValueClass(item.tone)">
{{ item.value }}
</span>
</div>
+1 -1
View File
@@ -1 +1 @@
version: "3.0.26"
version: "3.0.28"