mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
fix(accounts): converge account state and imports
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
zyycn
Codex
@@ -374,6 +374,7 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
pub(super) struct FakeAccountStore {
|
||||
events: EventLog,
|
||||
account: AccountRecord,
|
||||
account_after_probe: Mutex<Option<AccountRecord>>,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
@@ -388,6 +389,7 @@ impl FakeAccountStore {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account,
|
||||
account_after_probe: Mutex::new(None),
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
@@ -406,6 +408,13 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
fn set_account_after_probe(&self, account: AccountRecord) {
|
||||
*self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe") = Some(account);
|
||||
}
|
||||
|
||||
fn record(&self, event: &'static str) {
|
||||
self.events.lock().expect("store events").push(event);
|
||||
}
|
||||
@@ -445,7 +454,13 @@ impl AccountStore for FakeAccountStore {
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.record("store.load_account");
|
||||
Ok((account_id == self.account.id).then(|| self.account.clone()))
|
||||
let account = self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe")
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.account.clone());
|
||||
Ok((account_id == account.id.as_str()).then_some(account))
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -802,6 +817,36 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_reloads_resettable_usage_limit_status_after_probe() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
let probe = Arc::new(FreeModelQuotaProbe {
|
||||
store: Arc::clone(&store),
|
||||
});
|
||||
let services = accounts_service_with_probe(provider, store, probe).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
@@ -1234,6 +1279,30 @@ impl AccountProbe for FailingAccountProbe {
|
||||
}
|
||||
}
|
||||
|
||||
struct FreeModelQuotaProbe {
|
||||
store: Arc<FakeAccountStore>,
|
||||
}
|
||||
|
||||
impl AccountProbe for FreeModelQuotaProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::hours(24));
|
||||
self.store.set_account_after_probe(account);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"xAI free model quota is exhausted",
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
|
||||
@@ -1653,12 +1653,7 @@ fn format_token_price(cost: &domain::CurrencyCost) -> String {
|
||||
if cost.currency != "USD" {
|
||||
return format!("{} {} / 1M Token", cost.currency, cost.amount.as_str());
|
||||
}
|
||||
let value = cost
|
||||
.amount
|
||||
.as_str()
|
||||
.parse::<f64>()
|
||||
.map_or(0.0, |value| value);
|
||||
format!("${value:.4} / 1M Token")
|
||||
format!("${} / 1M Token", cost.amount.as_str())
|
||||
}
|
||||
|
||||
fn format_service_tier(service_tier: Option<&str>) -> String {
|
||||
|
||||
@@ -340,7 +340,7 @@ mod actions {
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::QuotaExhausted,
|
||||
},
|
||||
]
|
||||
.map(|event| AccountConnectionTestEvent::from(event).data);
|
||||
@@ -374,7 +374,7 @@ mod actions {
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "active"
|
||||
"accountStatus": "quota_exhausted"
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
@@ -291,12 +291,17 @@ async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
use std::str::FromStr as _;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
use gateway_admin::model::observability::{
|
||||
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, RequestOutcome, UsageBilling,
|
||||
UsageRecord,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
@@ -304,6 +309,10 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let usd = |amount: &str| CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: DecimalAmount::from_str(amount).expect("USD amount"),
|
||||
};
|
||||
fixture
|
||||
.usage_records
|
||||
.lock()
|
||||
@@ -359,7 +368,22 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
billing: Some(UsageBilling::Calculated(Box::new(
|
||||
CalculatedBillingBreakdown {
|
||||
input_amount: usd("0.03"),
|
||||
output_amount: usd("0.07"),
|
||||
cache_read_amount: usd("0.00"),
|
||||
cache_write_amount: usd("0.00"),
|
||||
standard_amount: usd("0.10"),
|
||||
total_amount: usd("0.10"),
|
||||
input_price_per_million: usd("10.0000"),
|
||||
output_price_per_million: usd("60.0000"),
|
||||
cache_read_price_per_million: usd("1.0000"),
|
||||
cache_write_price_per_million: usd("12.5000"),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
multiplier_percent: 100,
|
||||
},
|
||||
))),
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
@@ -400,6 +424,19 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
.expect("usage response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
|
||||
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
|
||||
"$10 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
|
||||
"$60 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
|
||||
"$12.5 / 1M Token"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
|
||||
@@ -2013,7 +2013,7 @@ async fn provider_account_metrics(
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (where status in ('expired', 'disabled', 'banned'))::bigint
|
||||
count(*) filter (where status <> 'active')::bigint
|
||||
as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
|
||||
@@ -1630,14 +1630,7 @@ fn admin_account_summary(
|
||||
let unavailable = u64::try_from(
|
||||
accounts
|
||||
.iter()
|
||||
.filter(|account| {
|
||||
matches!(
|
||||
admin_account_status(account, now),
|
||||
AdminAccountStatus::Expired
|
||||
| AdminAccountStatus::Disabled
|
||||
| AdminAccountStatus::Banned
|
||||
)
|
||||
})
|
||||
.filter(|account| admin_account_status(account, now) != AdminAccountStatus::Active)
|
||||
.count(),
|
||||
)
|
||||
.unwrap_or(u64::MAX);
|
||||
|
||||
@@ -134,7 +134,7 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
|
||||
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'rate_limited', $1 + interval '2 hours', $1, $1, $1),
|
||||
('acct_usage_limit', 'openai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'usage_limit_exhausted', $1 + interval '2 hours', $1, $1, $1),
|
||||
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
|
||||
@@ -169,8 +169,9 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
),
|
||||
(6, 5, 2, 3, 1, 1, 1, 1),
|
||||
(6, 5, 2, 4, 1, 1, 1, 1),
|
||||
);
|
||||
assert_eq!(metrics.total, metrics.active + metrics.unavailable);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
|
||||
@@ -217,6 +217,11 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(usage_page.summary.total, 5);
|
||||
assert_eq!(usage_page.summary.active, 2);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 1);
|
||||
assert_eq!(usage_page.summary.unavailable, 3);
|
||||
assert_eq!(
|
||||
usage_page.summary.total,
|
||||
usage_page.summary.active + usage_page.summary.unavailable
|
||||
);
|
||||
assert_eq!(
|
||||
usage_page
|
||||
.items
|
||||
@@ -1104,6 +1109,103 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_free_quota_exhaustion_state_is_persisted() {
|
||||
let Some(database) = TestDatabase::create("provider_account_xai_free_quota").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let account_id = ProviderAccountId::new("acct_xai_free_quota").expect("account ID");
|
||||
let mut imported = account(account_id.as_str(), "user-xai-free-quota");
|
||||
imported.provider_kind = "xai".to_owned();
|
||||
repository
|
||||
.insert_provider_account(imported)
|
||||
.await
|
||||
.expect("insert xAI account");
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("upstream_free_quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI free quota exhaustion");
|
||||
|
||||
let current: (String, String, Option<String>) = sqlx::query_as(
|
||||
"select provider_kind, availability, availability_reason
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI quota state");
|
||||
assert_eq!(
|
||||
current,
|
||||
(
|
||||
"xai".to_owned(),
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("upstream_free_quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_resettable_usage_limit_state_is_persisted() {
|
||||
let Some(database) = TestDatabase::create("provider_account_xai_usage_limit").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let account_id = ProviderAccountId::new("acct_xai_usage_limit").expect("account ID");
|
||||
let mut imported = account(account_id.as_str(), "user-xai-usage-limit");
|
||||
imported.provider_kind = "xai".to_owned();
|
||||
repository
|
||||
.insert_provider_account(imported)
|
||||
.await
|
||||
.expect("insert xAI account");
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
let cooldown_until = observed_at
|
||||
.checked_add(Duration::from_secs(24 * 60 * 60))
|
||||
.expect("cooldown deadline");
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Cooldown,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(cooldown_until),
|
||||
observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI resettable usage limit");
|
||||
|
||||
let current: (String, String, Option<String>, bool) = sqlx::query_as(
|
||||
"select provider_kind, availability, availability_reason, cooldown_until is not null
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI usage limit state");
|
||||
assert_eq!(
|
||||
current,
|
||||
(
|
||||
"xai".to_owned(),
|
||||
"cooldown".to_owned(),
|
||||
Some("usage_limit_exhausted".to_owned()),
|
||||
true,
|
||||
),
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
let Some(database) = TestDatabase::create("provider_account_disabled_refresh").await else {
|
||||
|
||||
@@ -16,9 +16,16 @@ pub(crate) fn derive_codex_session_affinity_key(
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
return ProviderSessionAffinityKey::try_new(conversation_id.to_owned()).ok();
|
||||
if let Ok(key) = ProviderSessionAffinityKey::try_new(conversation_id.to_owned()) {
|
||||
return Some(key);
|
||||
}
|
||||
return opaque_affinity_key("local-conversation", conversation_id);
|
||||
}
|
||||
let (domain, value) = derive_conversation_anchor(request)?;
|
||||
opaque_affinity_key(domain, &value)
|
||||
}
|
||||
|
||||
fn opaque_affinity_key(domain: &str, value: &str) -> Option<ProviderSessionAffinityKey> {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"codex-session-affinity-v1\0");
|
||||
hasher.update(domain.as_bytes());
|
||||
|
||||
@@ -161,13 +161,14 @@ pub(crate) fn derive_conversation_anchor(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<(&'static str, String)> {
|
||||
request
|
||||
.prompt_cache_key()
|
||||
.map(|value| ("prompt-cache", value.to_owned()))
|
||||
.client_session_id
|
||||
.as_deref()
|
||||
.map(|value| ("session", value.to_owned()))
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_session_id
|
||||
.client_conversation_id
|
||||
.as_deref()
|
||||
.map(|value| ("session", value.to_owned()))
|
||||
.map(|value| ("conversation", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
@@ -177,9 +178,8 @@ pub(crate) fn derive_conversation_anchor(
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_conversation_id
|
||||
.as_deref()
|
||||
.map(|value| ("conversation", value.to_owned()))
|
||||
.prompt_cache_key()
|
||||
.map(|value| ("prompt-cache", value.to_owned()))
|
||||
})
|
||||
.or_else(|| derive_stable_conversation_key(request).map(|value| ("request", value)))
|
||||
}
|
||||
|
||||
@@ -784,6 +784,94 @@ async fn prompt_cache_key_should_become_an_opaque_session_affinity_lookup_key()
|
||||
assert!(keys[0].bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_session_id_should_override_turn_specific_prompt_cache_keys_for_affinity() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_session_affinity").await;
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let provider = provider_with_affinity(&store, Arc::clone(&affinity));
|
||||
|
||||
for (request_id, prompt_cache_key) in [
|
||||
("req_session_affinity_first", "turn-cache-first"),
|
||||
("req_session_affinity_second", "turn-cache-second"),
|
||||
] {
|
||||
let generation = GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
("prompt_cache_key".to_owned(), json!(prompt_cache_key)),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"session_id".to_owned(),
|
||||
json!("stable-client-session"),
|
||||
)])),
|
||||
);
|
||||
|
||||
let stream = provider
|
||||
.execute(
|
||||
planned_request("openai", Operation::Generate(generation)),
|
||||
context(request_id, CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
drop(stream);
|
||||
}
|
||||
|
||||
let keys = affinity.lookup_keys();
|
||||
assert_eq!(keys.len(), 2);
|
||||
assert_eq!(keys[0], keys[1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_conversation_id_should_fall_back_to_an_opaque_affinity_key() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_local_affinity").await;
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let generation = GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload"),
|
||||
)
|
||||
.with_provider_session_state(
|
||||
ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("account_id".to_owned(), json!("acct_local_affinity")),
|
||||
(
|
||||
"conversation_id".to_owned(),
|
||||
json!("lc_UppercaseBase64ConversationId"),
|
||||
),
|
||||
("continuation_scope".to_owned(), json!("replay_required")),
|
||||
("transcript".to_owned(), json!([])),
|
||||
]),
|
||||
)
|
||||
.expect("provider session state"),
|
||||
);
|
||||
|
||||
let stream = provider_with_affinity(&store, Arc::clone(&affinity))
|
||||
.execute(
|
||||
planned_request("openai", Operation::Generate(generation)),
|
||||
context("req_local_affinity", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
drop(stream);
|
||||
|
||||
let keys = affinity.lookup_keys();
|
||||
assert_eq!(keys.len(), 1);
|
||||
assert_eq!(keys[0].len(), 64);
|
||||
assert!(keys[0].bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_response_persists_session_affinity_before_stream_consumer_stops_polling() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -479,15 +479,16 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
"upstream_payment_quota_exhausted",
|
||||
None,
|
||||
),
|
||||
GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model,
|
||||
retry_after,
|
||||
} => {
|
||||
GrokCredentialFailure::ModelQuotaExhausted { retry_after, .. } => {
|
||||
let retry_after =
|
||||
bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN);
|
||||
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
|
||||
.await;
|
||||
return;
|
||||
// xAI 免费模型额度按账号滚动窗口恢复；持久化使页面和各调度节点收敛。
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
(
|
||||
GrokCredentialAvailability::Cooldown,
|
||||
"usage_limit_exhausted",
|
||||
persistent_cooldown_until(observed_at, retry_after),
|
||||
)
|
||||
}
|
||||
GrokCredentialFailure::ModelAccessDenied {
|
||||
upstream_model,
|
||||
|
||||
@@ -60,8 +60,9 @@ use crate::transport::{
|
||||
GrokCompactionSummaryDecoder, GrokCredentialFailure, GrokInferenceChunkStream,
|
||||
GrokInferenceRequest, GrokInferenceResponse, GrokInferenceTransport,
|
||||
GrokInferenceTransportError, GrokInferenceTransportErrorKind, GrokInferenceTransportMetrics,
|
||||
GrokProviderConfigError, GrokRequestEncodeError, GrokResponsesRequest, GrokSessionAffinityKey,
|
||||
GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError, SelectedGrokSession,
|
||||
GrokProviderConfigError, GrokQuotaFailureKind, GrokRequestEncodeError, GrokResponsesRequest,
|
||||
GrokSessionAffinityKey, GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError,
|
||||
SelectedGrokSession, classify_grok_quota_failure,
|
||||
};
|
||||
use crate::{GrokCatalogCapabilityEvidence, GrokCatalogModel};
|
||||
|
||||
@@ -1118,9 +1119,21 @@ fn cold_compaction_http_sse_stream(
|
||||
.await
|
||||
.map_err(mark_transient_compaction_failure)?
|
||||
{
|
||||
let events = canonical.push(&chunk).map_err(|error| {
|
||||
mark_transient_compaction_failure(map_continuation_failure(&context, error))
|
||||
})?;
|
||||
let events = match canonical.push(&chunk) {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let error = map_continuation_failure(&context, error);
|
||||
let error = record_stream_failure(
|
||||
selector.as_ref(),
|
||||
&session,
|
||||
error,
|
||||
&upstream_model,
|
||||
)
|
||||
.await;
|
||||
Err(mark_transient_compaction_failure(error))?;
|
||||
return;
|
||||
}
|
||||
};
|
||||
for event in events {
|
||||
summary.observe(&event).map_err(map_compaction_decode_error)?;
|
||||
facts.observe(&event);
|
||||
@@ -1131,9 +1144,22 @@ fn cold_compaction_http_sse_stream(
|
||||
}
|
||||
|
||||
if facts.completed.is_none() {
|
||||
for event in canonical.finish_without_terminal().map_err(|error| {
|
||||
mark_transient_compaction_failure(map_continuation_failure(&context, error))
|
||||
})? {
|
||||
let events = match canonical.finish_without_terminal() {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let error = map_continuation_failure(&context, error);
|
||||
let error = record_stream_failure(
|
||||
selector.as_ref(),
|
||||
&session,
|
||||
error,
|
||||
&upstream_model,
|
||||
)
|
||||
.await;
|
||||
Err(mark_transient_compaction_failure(error))?;
|
||||
return;
|
||||
}
|
||||
};
|
||||
for event in events {
|
||||
summary.observe(&event).map_err(map_compaction_decode_error)?;
|
||||
facts.observe(&event);
|
||||
if facts.completed.is_some() {
|
||||
@@ -1371,7 +1397,13 @@ fn cold_http_sse_stream(
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let error = map_continuation_failure(&context, error);
|
||||
let error = record_stream_failure(selector.as_ref(), &session, error).await;
|
||||
let error = record_stream_failure(
|
||||
selector.as_ref(),
|
||||
&session,
|
||||
error,
|
||||
&upstream_model,
|
||||
)
|
||||
.await;
|
||||
Err(error)?;
|
||||
return;
|
||||
}
|
||||
@@ -1399,7 +1431,13 @@ fn cold_http_sse_stream(
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let error = map_continuation_failure(&context, error);
|
||||
let error = record_stream_failure(selector.as_ref(), &session, error).await;
|
||||
let error = record_stream_failure(
|
||||
selector.as_ref(),
|
||||
&session,
|
||||
error,
|
||||
&upstream_model,
|
||||
)
|
||||
.await;
|
||||
Err(error)?;
|
||||
return;
|
||||
}
|
||||
@@ -1508,28 +1546,6 @@ fn xai_transport_metadata(
|
||||
ProviderResponseMetadata::new(serde_json::to_string(&Value::Object(metadata)).ok()?)
|
||||
}
|
||||
|
||||
async fn record_failure(
|
||||
selector: &dyn GrokSessionSelector,
|
||||
session: &SelectedGrokSession,
|
||||
error: ProviderError,
|
||||
) -> ProviderError {
|
||||
if !session.allows_account_state_mutation() {
|
||||
return error;
|
||||
}
|
||||
let failure = match error.kind() {
|
||||
ProviderErrorKind::Unauthorized => Some(GrokCredentialFailure::Unauthorized),
|
||||
ProviderErrorKind::RateLimited => Some(GrokCredentialFailure::RateLimited {
|
||||
retry_after: error.retry_after(),
|
||||
}),
|
||||
ProviderErrorKind::QuotaExhausted => Some(GrokCredentialFailure::QuotaExhausted),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(failure) = failure {
|
||||
selector.record_failure(session, failure).await;
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
async fn record_credential_failure(
|
||||
selector: &dyn GrokSessionSelector,
|
||||
session: &SelectedGrokSession,
|
||||
@@ -1546,17 +1562,13 @@ async fn record_stream_failure(
|
||||
selector: &dyn GrokSessionSelector,
|
||||
session: &SelectedGrokSession,
|
||||
error: ProviderError,
|
||||
upstream_model: &UpstreamModelId,
|
||||
) -> ProviderError {
|
||||
if !session.allows_account_state_mutation() {
|
||||
return error;
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::Unauthorized
|
||||
| ProviderErrorKind::RateLimited
|
||||
| ProviderErrorKind::QuotaExhausted
|
||||
) {
|
||||
return record_failure(selector, session, error).await;
|
||||
if let Some(failure) = stream_credential_failure(&error, upstream_model) {
|
||||
return record_credential_failure(selector, session, error, failure).await;
|
||||
}
|
||||
if error.kind() != ProviderErrorKind::Cancelled {
|
||||
selector
|
||||
@@ -1566,6 +1578,39 @@ async fn record_stream_failure(
|
||||
error
|
||||
}
|
||||
|
||||
fn stream_credential_failure(
|
||||
error: &ProviderError,
|
||||
upstream_model: &UpstreamModelId,
|
||||
) -> Option<GrokCredentialFailure> {
|
||||
match error.kind() {
|
||||
ProviderErrorKind::Unauthorized => Some(GrokCredentialFailure::Unauthorized),
|
||||
ProviderErrorKind::RateLimited => Some(GrokCredentialFailure::RateLimited {
|
||||
retry_after: error.retry_after(),
|
||||
}),
|
||||
ProviderErrorKind::QuotaExhausted => {
|
||||
let visible = error.client_visible_upstream_error();
|
||||
let quota_kind = classify_grok_quota_failure(
|
||||
visible
|
||||
.and_then(ClientVisibleUpstreamError::code)
|
||||
.or_else(|| error.upstream_code().map(|code| code.as_str())),
|
||||
visible.and_then(ClientVisibleUpstreamError::error_type),
|
||||
visible.map(ClientVisibleUpstreamError::message),
|
||||
);
|
||||
Some(match quota_kind {
|
||||
Some(GrokQuotaFailureKind::FreeAccount) => {
|
||||
GrokCredentialFailure::FreeQuotaExhausted
|
||||
}
|
||||
Some(GrokQuotaFailureKind::Model) => GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model: upstream_model.clone(),
|
||||
retry_after: error.retry_after(),
|
||||
},
|
||||
Some(GrokQuotaFailureKind::Account) | None => GrokCredentialFailure::QuotaExhausted,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_or_record_failure(
|
||||
selector: &dyn GrokSessionSelector,
|
||||
recovery: &dyn GrokCredentialRecovery,
|
||||
@@ -1644,7 +1689,7 @@ async fn map_and_record_stream_transport_failure(
|
||||
match credential_failure {
|
||||
Some(failure) => record_credential_failure(selector, session, error, failure).await,
|
||||
None if request_scoped => error,
|
||||
None => record_stream_failure(selector, session, error).await,
|
||||
None => record_stream_failure(selector, session, error, upstream_model).await,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ use gateway_protocol::openai::sse::{SseEvent, SseEventDecoder};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::request::{GrokResponseTransform, GrokResponsesRequest};
|
||||
use super::scrub_account_fingerprints;
|
||||
use super::{classify_grok_quota_failure, scrub_account_fingerprints};
|
||||
|
||||
const CONTENTS_PER_OUTPUT: u32 = 1_024;
|
||||
const LONG_CONTEXT_THRESHOLD: u64 = 200_000;
|
||||
@@ -785,14 +785,19 @@ fn incomplete_finish_reason(response: &Value) -> FinishReason {
|
||||
|
||||
fn upstream_event_error(value: &Value) -> ProviderError {
|
||||
let code = upstream_error_field(value, "code");
|
||||
let kind = match code {
|
||||
Some("invalid_request" | "invalid_prompt") => ProviderErrorKind::InvalidRequest,
|
||||
Some("unsupported" | "unsupported_feature") => ProviderErrorKind::Unsupported,
|
||||
Some("unauthorized" | "invalid_token") => ProviderErrorKind::Unauthorized,
|
||||
Some("permission_denied") => ProviderErrorKind::PermissionDenied,
|
||||
Some("rate_limit_exceeded") => ProviderErrorKind::RateLimited,
|
||||
Some("quota_exceeded" | "insufficient_quota") => ProviderErrorKind::QuotaExhausted,
|
||||
_ => ProviderErrorKind::Unavailable,
|
||||
let error_type = upstream_error_field(value, "type");
|
||||
let message = upstream_error_field(value, "message");
|
||||
let kind = if classify_grok_quota_failure(code, error_type, message).is_some() {
|
||||
ProviderErrorKind::QuotaExhausted
|
||||
} else {
|
||||
match code {
|
||||
Some("invalid_request" | "invalid_prompt") => ProviderErrorKind::InvalidRequest,
|
||||
Some("unsupported" | "unsupported_feature") => ProviderErrorKind::Unsupported,
|
||||
Some("unauthorized" | "invalid_token") => ProviderErrorKind::Unauthorized,
|
||||
Some("permission_denied") => ProviderErrorKind::PermissionDenied,
|
||||
Some("rate_limit_exceeded") => ProviderErrorKind::RateLimited,
|
||||
_ => ProviderErrorKind::Unavailable,
|
||||
}
|
||||
};
|
||||
let mut error = ProviderError::new(kind, UpstreamSendState::Sent)
|
||||
.redact_sensitive_context("upstream event");
|
||||
@@ -801,11 +806,11 @@ fn upstream_event_error(value: &Value) -> ProviderError {
|
||||
}
|
||||
// 结构化 message/code/type 供原客户端展示与重试分类；message 先脱去
|
||||
// 账号指纹（上游限流文案内嵌 team UUID），非结构化正文不透出。
|
||||
if let Some(message) = upstream_error_field(value, "message") {
|
||||
if let Some(message) = message {
|
||||
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
scrub_account_fingerprints(message),
|
||||
code.map(str::to_owned),
|
||||
upstream_error_field(value, "type").map(str::to_owned),
|
||||
error_type.map(str::to_owned),
|
||||
));
|
||||
}
|
||||
error
|
||||
|
||||
@@ -57,6 +57,57 @@ pub use transport::{
|
||||
|
||||
const UUID_TEXT_LEN: usize = 36;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum GrokQuotaFailureKind {
|
||||
Account,
|
||||
FreeAccount,
|
||||
Model,
|
||||
}
|
||||
|
||||
pub(crate) fn classify_grok_quota_failure(
|
||||
code: Option<&str>,
|
||||
error_type: Option<&str>,
|
||||
message: Option<&str>,
|
||||
) -> Option<GrokQuotaFailureKind> {
|
||||
let text = [
|
||||
code.unwrap_or_default(),
|
||||
error_type.unwrap_or_default(),
|
||||
message.unwrap_or_default(),
|
||||
]
|
||||
.join(" ")
|
||||
.to_ascii_lowercase();
|
||||
let free_usage = contains_any(
|
||||
&text,
|
||||
&[
|
||||
"subscription:free-usage-exhausted",
|
||||
"subscription_free_usage_exhausted",
|
||||
"free-usage-exhausted",
|
||||
"free_usage_exhausted",
|
||||
"used all the included free usage",
|
||||
"used all your free usage",
|
||||
],
|
||||
);
|
||||
if free_usage && text.contains("for model") {
|
||||
return Some(GrokQuotaFailureKind::Model);
|
||||
}
|
||||
if contains_any(
|
||||
&text,
|
||||
&[
|
||||
"personal-team-blocked:spending-limit",
|
||||
"personal_team_blocked_spending_limit",
|
||||
"quota_exceeded",
|
||||
"insufficient_quota",
|
||||
],
|
||||
) {
|
||||
return Some(GrokQuotaFailureKind::Account);
|
||||
}
|
||||
free_usage.then_some(GrokQuotaFailureKind::FreeAccount)
|
||||
}
|
||||
|
||||
fn contains_any(value: &str, needles: &[&str]) -> bool {
|
||||
needles.iter().any(|needle| value.contains(needle))
|
||||
}
|
||||
|
||||
/// 把 message 中的 UUID 替换为占位符，控制字符归一为空格，
|
||||
/// 使文本满足客户端可见错误的约束且不携带可定位账号的标识。
|
||||
fn scrub_account_fingerprints(message: &str) -> String {
|
||||
|
||||
@@ -33,7 +33,7 @@ use gateway_core::engine::UpstreamSendState;
|
||||
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue};
|
||||
use gateway_core::event::UpstreamHttpVersion;
|
||||
|
||||
use super::scrub_account_fingerprints;
|
||||
use super::{GrokQuotaFailureKind, classify_grok_quota_failure, scrub_account_fingerprints};
|
||||
|
||||
pub(crate) const OFFICIAL_OAUTH_HOST: &str = "auth.x.ai";
|
||||
const OFFICIAL_INFERENCE_HOST: &str = "cli-chat-proxy.grok.com";
|
||||
@@ -1138,31 +1138,26 @@ fn reasoning_decode_failed(metadata: &InferenceErrorMetadata) -> bool {
|
||||
}
|
||||
|
||||
fn paid_quota_exhausted(metadata: &InferenceErrorMetadata) -> bool {
|
||||
contains_any(
|
||||
&inference_metadata_text(metadata),
|
||||
&[
|
||||
"personal-team-blocked:spending-limit",
|
||||
"personal_team_blocked_spending_limit",
|
||||
],
|
||||
)
|
||||
matches!(quota_failure(metadata), Some(GrokQuotaFailureKind::Account))
|
||||
}
|
||||
|
||||
fn free_quota_exhausted(metadata: &InferenceErrorMetadata) -> bool {
|
||||
contains_any(
|
||||
&inference_metadata_text(metadata),
|
||||
&[
|
||||
"subscription:free-usage-exhausted",
|
||||
"subscription_free_usage_exhausted",
|
||||
"free-usage-exhausted",
|
||||
"free_usage_exhausted",
|
||||
"used all the included free usage",
|
||||
"used all your free usage",
|
||||
],
|
||||
matches!(
|
||||
quota_failure(metadata),
|
||||
Some(GrokQuotaFailureKind::FreeAccount)
|
||||
)
|
||||
}
|
||||
|
||||
fn model_quota_exhausted(metadata: &InferenceErrorMetadata) -> bool {
|
||||
inference_metadata_text(metadata).contains("used all the included free usage for model")
|
||||
matches!(quota_failure(metadata), Some(GrokQuotaFailureKind::Model))
|
||||
}
|
||||
|
||||
fn quota_failure(metadata: &InferenceErrorMetadata) -> Option<GrokQuotaFailureKind> {
|
||||
classify_grok_quota_failure(
|
||||
metadata.code.as_deref(),
|
||||
metadata.error_type.as_deref(),
|
||||
metadata.message.as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
fn model_access_denied(metadata: &InferenceErrorMetadata) -> bool {
|
||||
|
||||
@@ -319,7 +319,7 @@ pub enum GrokCredentialFailure {
|
||||
/// 已解析并脱敏的延迟；缺失时由 selector 使用短暂默认值。
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
/// 选中账号只耗尽了指定模型的额度。
|
||||
/// 上游声明指定模型的免费用量耗尽；selector 会收敛为账号级可恢复额度状态。
|
||||
ModelQuotaExhausted {
|
||||
upstream_model: UpstreamModelId,
|
||||
retry_after: Option<Duration>,
|
||||
|
||||
@@ -282,7 +282,7 @@ pub enum GrokInferenceTransportErrorKind {
|
||||
QuotaExhausted,
|
||||
/// 账号订阅级免费额度耗尽。
|
||||
FreeQuotaExhausted,
|
||||
/// 当前账号只耗尽了指定模型的免费额度。
|
||||
/// 上游声明指定模型的免费用量耗尽；selector 会收敛为账号级可恢复额度状态。
|
||||
ModelQuotaExhausted,
|
||||
/// 当前账号只缺少指定模型的访问权限。
|
||||
ModelAccessDenied,
|
||||
|
||||
@@ -490,7 +490,7 @@ async fn payment_required_feedback_persists_account_quota_exhaustion() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_quota_feedback_blocks_only_the_failed_account_and_model() {
|
||||
async fn free_model_quota_feedback_persists_resettable_account_exhaustion() {
|
||||
let fixture = SelectorFixture::new(&["model-quota", "available"]).await;
|
||||
let failed_model = UpstreamModelId::new("grok-4.5").expect("failed model");
|
||||
let session = fixture
|
||||
@@ -505,38 +505,47 @@ async fn model_quota_feedback_blocks_only_the_failed_account_and_model() {
|
||||
&session,
|
||||
GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model: failed_model.clone(),
|
||||
retry_after: Some(Duration::from_secs(5)),
|
||||
retry_after: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
fixture
|
||||
.store
|
||||
.account(&selected)
|
||||
.expect("selected account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
let account = fixture.store.account(&selected).expect("selected account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > minimum_until)
|
||||
);
|
||||
let scope = ProviderCooldownScope::upstream_model(failed_model);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.scoped_cooldown(&selected, &scope)
|
||||
.is_some()
|
||||
.is_none()
|
||||
);
|
||||
let unrelated_model = fixture
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
.cooldown(&selected)
|
||||
.is_some_and(|cooldown| cooldown.until() > minimum_until)
|
||||
);
|
||||
assert!(matches!(
|
||||
fixture
|
||||
.selector
|
||||
.select(fixture.request_for_model("grok-4.6", Some(selected.clone())))
|
||||
.await,
|
||||
Err(GrokSessionSelectorError::AccountCoolingDown {
|
||||
retry_after: Some(_)
|
||||
})
|
||||
));
|
||||
let next = fixture
|
||||
.selector
|
||||
.select(fixture.request_for_model("grok-4.6", Some(selected.clone())))
|
||||
.select(fixture.request_for_model("grok-4.6", None))
|
||||
.await
|
||||
.expect("same account remains valid for another model");
|
||||
assert_eq!(unrelated_model.account_id(), &selected);
|
||||
let failed_model_retry = fixture
|
||||
.selector
|
||||
.select(fixture.request_for_model("grok-4.5", None))
|
||||
.await
|
||||
.expect("another account serves the failed model");
|
||||
assert_ne!(failed_model_retry.account_id(), &selected);
|
||||
.expect("another account remains schedulable");
|
||||
assert_ne!(next.account_id(), &selected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1518,6 +1518,69 @@ async fn model_quota_transport_failure_preserves_model_scoped_feedback() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn free_quota_sse_failure_marks_the_account_exhausted() {
|
||||
let selector = StubSelector::success();
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_free_quota\",\"model\":\"grok-4.5\"}}\n\n",
|
||||
"event: response.failed\n",
|
||||
"data: {\"type\":\"response.failed\",\"error\":{\"code\":\"subscription:free-usage-exhausted\",\"message\":\"You have used all your free usage\"}}\n\n",
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
let transport = StubInferenceTransport::sequence([InferenceMode::SuccessBody(body)]);
|
||||
let provider = provider(selector.clone(), transport).await;
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
provider_request("xai"),
|
||||
context(CancellationToken::new(), None),
|
||||
)
|
||||
.await
|
||||
.expect("stream");
|
||||
|
||||
let error = next_provider_error(&mut stream).await;
|
||||
|
||||
assert_eq!(error.kind(), ProviderErrorKind::QuotaExhausted);
|
||||
assert_eq!(
|
||||
selector.feedback.lock().expect("feedback").as_slice(),
|
||||
&[GrokCredentialFailure::FreeQuotaExhausted]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn free_model_quota_sse_failure_is_classified_for_resettable_account_cooldown() {
|
||||
let selector = StubSelector::success();
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_model_quota\",\"model\":\"grok-4.5\"}}\n\n",
|
||||
"event: response.failed\n",
|
||||
"data: {\"type\":\"response.failed\",\"error\":{\"code\":\"subscription:free-usage-exhausted\",\"message\":\"You have used all the included free usage for model grok-4.5-0722 for now. Usage resets over a rolling 24-hour window \u{2014} tokens (actual/limit): 500505/500000. Upgrade to a Grok subscription for higher limits: https://grok.com/supergrok\"}}\n\n",
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
let transport = StubInferenceTransport::sequence([InferenceMode::SuccessBody(body)]);
|
||||
let provider = provider(selector.clone(), transport).await;
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
provider_request("xai"),
|
||||
context(CancellationToken::new(), None),
|
||||
)
|
||||
.await
|
||||
.expect("stream");
|
||||
|
||||
let error = next_provider_error(&mut stream).await;
|
||||
|
||||
assert_eq!(error.kind(), ProviderErrorKind::QuotaExhausted);
|
||||
assert_eq!(
|
||||
selector.feedback.lock().expect("feedback").as_slice(),
|
||||
&[GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model: UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
retry_after: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_access_denial_is_retryable_without_credential_recovery() {
|
||||
let selector = StubSelector::success();
|
||||
|
||||
@@ -802,6 +802,28 @@ fn decoder_should_classify_failed_event_without_retaining_body() {
|
||||
assert!(!format!("{error:?}").contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_classify_free_quota_failed_event() {
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_quota\"}}\n\n",
|
||||
"event: response.failed\n",
|
||||
"data: {\"type\":\"response.failed\",\"error\":{\"code\":\"subscription:free-usage-exhausted\",\"type\":\"billing_error\",\"message\":\"You have used all your free usage\"}}\n\n",
|
||||
);
|
||||
let error = GrokCanonicalDecoder::new("fallback")
|
||||
.push(body.as_bytes())
|
||||
.expect_err("free quota failure must surface");
|
||||
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
gateway_core::error::ProviderErrorKind::QuotaExhausted
|
||||
);
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some("subscription:free-usage-exhausted")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_scrub_account_fingerprints_from_failure_messages() {
|
||||
let body = concat!(
|
||||
|
||||
@@ -193,19 +193,6 @@ function connectionLogClass(tone: string) {
|
||||
</div>
|
||||
</BaseScrollbar>
|
||||
</div>
|
||||
|
||||
<div v-if="error" class="mt-3 rounded-lg bg-(--cp-danger-bg) px-3 py-2.5">
|
||||
<p class="m-0 text-[11px] font-heavy text-(--cp-danger-text)">
|
||||
错误信息
|
||||
</p>
|
||||
<BaseScrollbar max-height="118px" view-class="pt-1.5 pr-2">
|
||||
<p
|
||||
class="m-0 wrap-break-word text-[12px] leading-[1.55] font-emphasis text-(--cp-danger-text)"
|
||||
>
|
||||
{{ error }}
|
||||
</p>
|
||||
</BaseScrollbar>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -280,17 +280,7 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
if (event.type === 'error') {
|
||||
applyAccountStatus(event.accountStatus)
|
||||
connectionTestError.value = connectionTestErrorText(event)
|
||||
const detail = event.upstreamStatus || event.upstreamContentType || event.upstreamBody
|
||||
|| event.providerErrorCode || event.providerErrorType
|
||||
? {
|
||||
status: event.upstreamStatus,
|
||||
contentType: event.upstreamContentType,
|
||||
body: event.upstreamBody,
|
||||
code: event.providerErrorCode,
|
||||
type: event.providerErrorType,
|
||||
}
|
||||
: undefined
|
||||
appendConnectionTestLog(connectionTestError.value, 'danger', detail)
|
||||
appendConnectionTestLog('上游返回错误', 'danger', connectionTestError.value)
|
||||
finishConnectionTest('error')
|
||||
clearConnectionTestRun()
|
||||
}
|
||||
|
||||
@@ -148,15 +148,20 @@ export function useAccountOnboarding(options: {
|
||||
}
|
||||
|
||||
async function importAccountDocument() {
|
||||
const data = parseImportJson(createForm.value.importText)
|
||||
if (Array.isArray(data) || typeof data !== 'object' || data === null)
|
||||
throw new Error('导入文件必须是 JSON object')
|
||||
const result = await importAccounts({
|
||||
provider: createForm.value.provider,
|
||||
data,
|
||||
})
|
||||
const provider = createForm.value.provider === 'xai' ? 'xAI' : 'OpenAI'
|
||||
return `${provider} 账号已导入 ${result.importedCount} 个`
|
||||
const provider = requireImportProvider(createForm.value.provider)
|
||||
const documents = providerImportDocuments(
|
||||
parseImportJson(createForm.value.importText),
|
||||
provider,
|
||||
)
|
||||
let importedCount = 0
|
||||
for (const entry of documents) {
|
||||
const result = await importAccounts({
|
||||
provider,
|
||||
data: entry.document,
|
||||
})
|
||||
importedCount += result.importedCount
|
||||
}
|
||||
return `${providerDisplayName(provider) ?? provider} 账号已导入 ${importedCount} 个`
|
||||
}
|
||||
|
||||
async function importMixedAccountDocument() {
|
||||
@@ -259,6 +264,27 @@ function parseImportJson(value: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function requireImportProvider(value: string): ImportProvider {
|
||||
if (value === 'openai' || value === 'xai')
|
||||
return value
|
||||
throw new Error('请选择要导入的账号平台')
|
||||
}
|
||||
|
||||
function providerImportDocuments(value: unknown, provider: ImportProvider): MixedImportDocument[] {
|
||||
if (isRecord(value) && Array.isArray(value.documents)) {
|
||||
const documents = parseMixedImportDocuments(value)
|
||||
.filter(entry => entry.provider === provider)
|
||||
if (documents.length === 0) {
|
||||
const label = providerDisplayName(provider) ?? provider
|
||||
throw new Error(`批量导入文件不包含 ${label} 账号文档`)
|
||||
}
|
||||
return documents
|
||||
}
|
||||
if (!isRecord(value))
|
||||
throw new Error('导入文件必须是 JSON object')
|
||||
return [{ provider, document: value }]
|
||||
}
|
||||
|
||||
function parseMixedImportDocuments(value: unknown): MixedImportDocument[] {
|
||||
if (!isRecord(value) || !Array.isArray(value.documents))
|
||||
throw new Error('批量导入文件必须是 CPR 多平台导出文件')
|
||||
|
||||
@@ -56,17 +56,17 @@ function itemValueClass(tone?: string, accent?: boolean) {
|
||||
|
||||
<UsageDetailPopover v-if="billing" title="计费明细" width="248px" trigger-label="查看费用明细">
|
||||
<div class="grid gap-1.5 text-(--cp-text-secondary)">
|
||||
<div v-for="item in amountItems" :key="item.label" class="flex justify-between gap-4">
|
||||
<span>{{ item.label }}</span>
|
||||
<span class="font-mono font-heavy" :class="itemValueClass(undefined, item.accent)">
|
||||
<div v-for="item in amountItems" :key="item.label" class="grid grid-cols-[auto_minmax(0,1fr)] items-center gap-4">
|
||||
<span class="whitespace-nowrap">{{ item.label }}</span>
|
||||
<span class="justify-self-end whitespace-nowrap font-mono font-heavy" :class="itemValueClass(undefined, item.accent)">
|
||||
{{ item.value }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-1 grid gap-1.5 rounded-(--cp-input-radius-base) bg-(--cp-bg-subtle) p-2 text-(--cp-text-secondary)">
|
||||
<div v-for="item in billingItems" :key="item.label" class="flex justify-between gap-4">
|
||||
<span>{{ item.label }}</span>
|
||||
<span class="font-mono font-heavy" :class="itemValueClass(item.tone)">
|
||||
<div v-for="item in billingItems" :key="item.label" class="grid grid-cols-[auto_minmax(0,1fr)] items-center gap-4">
|
||||
<span class="whitespace-nowrap">{{ item.label }}</span>
|
||||
<span class="justify-self-end whitespace-nowrap font-mono font-heavy" :class="itemValueClass(item.tone)">
|
||||
{{ item.value }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user