diff --git a/backend/crates/gateway-admin/src/model/accounts.rs b/backend/crates/gateway-admin/src/model/accounts.rs index 10175043f..4bab12ca7 100644 --- a/backend/crates/gateway-admin/src/model/accounts.rs +++ b/backend/crates/gateway-admin/src/model/accounts.rs @@ -75,8 +75,6 @@ pub struct AccountRecord { pub next_refresh_at: Option>, pub enabled: bool, pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, pub availability_observed_at: DateTime, pub quota_observed_at: Option>, pub created_at: DateTime, diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index 414868d05..329c47b83 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -443,13 +443,15 @@ pub struct RequestMetricPoint { /// /// 首页概览的 `unavailable` 表示不可参与调度，包含配额耗尽及过期、禁用和封禁账号。 /// -/// `quota_exhausted` 仍单列，以便状态详情展示和诊断。 +/// `quota_exhausted`（402 确认耗尽）与 `rate_limited`（429 限流中）单列， +/// 以便状态详情展示和诊断。 #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct AccountPoolMetrics { pub total: u64, pub enabled: u64, pub unavailable: u64, pub active: u64, + pub rate_limited: u64, pub expired: u64, pub quota_exhausted: u64, pub refreshing: Option, diff --git a/backend/crates/gateway-admin/src/model/provider_credentials.rs b/backend/crates/gateway-admin/src/model/provider_credentials.rs index a04f3ff5d..bfbb44b82 100644 --- a/backend/crates/gateway-admin/src/model/provider_credentials.rs +++ b/backend/crates/gateway-admin/src/model/provider_credentials.rs @@ -153,8 +153,6 @@ pub struct PreparedCredentialCreate { pub next_refresh_at: Option>, pub enabled: bool, pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, pub availability_observed_at: DateTime, } @@ -573,6 +571,7 @@ pub struct ProviderQuotaWindow { pub window_seconds: Option, pub used_percent: Option, pub reset_at: Option>, + pub limit_reached: bool, pub local_usage: Option, pub provider_data: Option, } diff --git a/backend/crates/gateway-admin/src/use_case/accounts.rs b/backend/crates/gateway-admin/src/use_case/accounts.rs index 158a460ba..ee41c2ead 100644 --- a/backend/crates/gateway-admin/src/use_case/accounts.rs +++ b/backend/crates/gateway-admin/src/use_case/accounts.rs @@ -260,18 +260,37 @@ impl AccountsService for DefaultAccountsService { let mut quotas = futures::future::join_all(page.items.iter().map(|account| async { let account_id = ProviderAccountId::new(account.id.clone()) .map_err(|_| AdminError::invalid("Invalid provider account ID"))?; - let provider = self - .providers - .require(&account.provider_kind) - .map_err(|error| map_provider_error(error, "provider quota"))?; - provider + // 单个账号的 quota 投影失败（Provider 未注册或 quota 读取失败）不拖垮整页： + // 该账号降级为空额度投影，其余账号与页面状态照常返回。 + let provider = match self.providers.require(&account.provider_kind) { + Ok(provider) => provider, + Err(error) => { + tracing::warn!( + account_id = %account.id, + error = %error, + "account directory provider is not registered; showing empty quota" + ); + return Ok(empty_quota()); + } + }; + match provider .quota(ProviderQuotaRequest { account_id, refresh: false, rolling_usage: rolling_usage.get(&account.id).cloned(), }) .await - .map_err(|error| map_provider_error(error, "provider quota")) + { + Ok(quota) => Ok(quota), + Err(error) => { + tracing::warn!( + account_id = %account.id, + error = %error, + "account directory quota projection failed; showing empty quota" + ); + Ok(empty_quota()) + } + } })) .await .into_iter() @@ -486,16 +505,22 @@ impl AccountsService for DefaultAccountsService { } } +/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。 +fn empty_quota() -> ProviderQuota { + ProviderQuota { + observed_at: None, + refresh_token_expires_at: None, + windows: Vec::new(), + provider_data: None, + } +} + fn account_status(account: &AccountRecord, now: chrono::DateTime) -> AccountStatus { if !account.enabled { AccountStatus::Disabled } else if account.availability == crate::model::accounts::AccountAvailability::Banned { AccountStatus::Banned - } else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted - || (account.availability == crate::model::accounts::AccountAvailability::Cooldown - && account.availability_reason.as_deref() == Some("usage_limit_exhausted") - && account.cooldown_until.is_none_or(|until| until > now)) - { + } else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted { AccountStatus::QuotaExhausted } else if matches!( account.availability, diff --git a/backend/crates/gateway-admin/tests/main.rs b/backend/crates/gateway-admin/tests/main.rs index 67b2a01f0..511d6c6d6 100644 --- a/backend/crates/gateway-admin/tests/main.rs +++ b/backend/crates/gateway-admin/tests/main.rs @@ -146,6 +146,7 @@ fn quota_window( window_seconds, used_percent, reset_at: None, + limit_reached: false, local_usage: None, provider_data: None, } diff --git a/backend/crates/gateway-admin/tests/use_case/accounts.rs b/backend/crates/gateway-admin/tests/use_case/accounts.rs index 8bbdf9bd7..75fb7204f 100644 --- a/backend/crates/gateway-admin/tests/use_case/accounts.rs +++ b/backend/crates/gateway-admin/tests/use_case/accounts.rs @@ -373,7 +373,7 @@ impl ProviderAdmin for FakeProviderAdmin { pub(super) struct FakeAccountStore { events: EventLog, - account: AccountRecord, + accounts: Mutex>, account_after_probe: Mutex>, fail_commit: Mutex, audit_requests: Mutex>, @@ -388,7 +388,7 @@ impl FakeAccountStore { fn with_account(account: AccountRecord, events: EventLog) -> Arc { Arc::new(Self { events, - account, + accounts: Mutex::new(vec![account]), account_after_probe: Mutex::new(None), fail_commit: Mutex::new(false), audit_requests: Mutex::new(Vec::new()), @@ -408,6 +408,10 @@ impl FakeAccountStore { *self.quota_window_usage.lock().expect("quota window usage") = usage; } + pub(super) fn set_accounts(&self, accounts: Vec) { + *self.accounts.lock().expect("accounts") = accounts; + } + fn set_account_after_probe(&self, account: AccountRecord) { *self .account_after_probe @@ -439,12 +443,14 @@ impl FakeAccountStore { #[async_trait] impl AccountStore for FakeAccountStore { async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult { + self.record("store.list_accounts"); + let accounts = self.accounts.lock().expect("accounts").clone(); Ok(AccountPage { config_revision: revision(1), - items: vec![self.account.clone()], - total: 1, + items: accounts.clone(), + total: accounts.len() as u64, summary: AccountSummary { - total: 1, + total: accounts.len() as u64, active: 1, quota_exhausted: 0, unavailable: 0, @@ -454,13 +460,22 @@ impl AccountStore for FakeAccountStore { async fn load_account(&self, account_id: &str) -> AdminStoreResult> { self.record("store.load_account"); + // probe 后的账号状态覆盖只对同一 id 生效；其余按账号列表查询。 let account = self .account_after_probe .lock() .expect("account after probe") .clone() - .unwrap_or_else(|| self.account.clone()); - Ok((account_id == account.id.as_str()).then_some(account)) + .filter(|account| account.id == account_id) + .or_else(|| { + self.accounts + .lock() + .expect("accounts") + .iter() + .find(|account| account.id == account_id) + .cloned() + }); + Ok(account) } async fn load_account_usage( @@ -488,11 +503,12 @@ impl AccountStore for FakeAccountStore { _: CredentialListQuery, ) -> AdminStoreResult { self.record("store.list_credentials"); + let accounts = self.accounts.lock().expect("accounts").clone(); Ok(CredentialPage { config_revision: revision(1), - items: (provider_kind == &self.account.provider_kind) - .then(|| self.account.clone()) + items: accounts .into_iter() + .filter(|account| &account.provider_kind == provider_kind) .collect(), next_cursor: None, }) @@ -504,14 +520,19 @@ impl AccountStore for FakeAccountStore { account_id: &ProviderAccountId, ) -> AdminStoreResult> { self.record("store.credential_details"); - Ok( - (provider_kind == &self.account.provider_kind - && account_id.as_str() == self.account.id) - .then(|| CredentialDetails { - config_revision: revision(1), - credential: self.account.clone(), - }), - ) + let account = self + .accounts + .lock() + .expect("accounts") + .iter() + .find(|account| { + &account.provider_kind == provider_kind && account.id == account_id.as_str() + }) + .cloned(); + Ok(account.map(|credential| CredentialDetails { + config_revision: revision(1), + credential, + })) } async fn load_credentials_for_export( @@ -520,21 +541,26 @@ impl AccountStore for FakeAccountStore { account_ids: &[ProviderAccountId], ) -> AdminStoreResult> { self.record("store.load_credentials_for_export"); - if provider_kind != &self.account.provider_kind - || account_ids - .iter() - .any(|account_id| account_id.as_str() != self.account.id) - { + let accounts = self.accounts.lock().expect("accounts").clone(); + if accounts.iter().any(|account| { + &account.provider_kind != provider_kind + || !account_ids + .iter() + .any(|account_id| account_id.as_str() == account.id) + }) { return Err(AdminStoreError::new( AdminStoreErrorKind::NotFound, "test account", "credential not found", )); } - Ok(vec![ProviderExportCredentialInput { - account: self.account.clone(), - provider_material: document(), - }]) + Ok(accounts + .into_iter() + .map(|account| ProviderExportCredentialInput { + account, + provider_material: document(), + }) + .collect()) } async fn commit_credential_import( @@ -818,7 +844,7 @@ async fn connection_test_should_probe_unavailable_account() { } #[tokio::test] -async fn connection_test_reloads_resettable_usage_limit_status_after_probe() { +async fn connection_test_rate_limited_probe_preserves_active_account_status() { let events = events(); let provider = FakeProviderAdmin::new("xai", events.clone()); let store = FakeAccountStore::new("xai", events); @@ -841,7 +867,7 @@ async fn connection_test_reloads_resettable_usage_limit_status_after_probe() { assert!(matches!( events.last(), Some(AccountConnectionTestEvent::Failed { - account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted, + account_status: gateway_admin::model::accounts::AccountStatus::Active, .. }) )); @@ -880,7 +906,7 @@ async fn connection_test_should_preserve_disabled_account_status() { async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() { let provider = FakeProviderAdmin::new("openai", events()); let store = FakeAccountStore::new("openai", events()); - let expected = store.account.clone(); + let expected = store.accounts.lock().expect("accounts")[0].clone(); accounts_service(provider.clone(), store) .await .accounts() @@ -968,12 +994,75 @@ async fn accounts_list_should_return_complete_directory_semantics() { } #[tokio::test] -async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() { +async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_dropping_page() { + let events = events(); + let openai = FakeProviderAdmin::new("openai", events.clone()); + openai.set_quota(ProviderQuota { + observed_at: Some(Utc::now()), + refresh_token_expires_at: None, + windows: vec![ProviderQuotaWindow { + key: "primary".to_owned(), + group: "shortTerm".to_owned(), + label: "5小时限额".to_owned(), + source: None, + window_seconds: Some(5 * 60 * 60), + used_percent: Some(97.0), + reset_at: Some(Utc::now() + TimeDelta::hours(1)), + limit_reached: false, + local_usage: None, + provider_data: None, + }], + provider_data: None, + }); + let failing = FakeProviderAdmin::new("xai", events.clone()); + failing.fail_next_quota(ProviderAdminErrorKind::Invalid); + let store = FakeAccountStore::new("openai", events.clone()); + let mut xai_account = account_record("xai"); + xai_account.id = "acct_bad_xai".to_owned(); + store.set_accounts(vec![account_record("openai"), xai_account]); + let services = super::AdminHarness::new() + .accounts(store.clone()) + .settings(Arc::new(StaticSettingsStore)) + .provider(openai) + .provider(failing) + .probe(Arc::new(SuccessfulAccountProbe)) + .build() + .await; + + let page = services + .accounts() + .list(AccountListQuery { + page: 1, + page_size: gateway_admin::model::PageSize::new(20).expect("page size"), + provider_kind: None, + search: None, + status: None, + sort: None, + }) + .await + .expect("one failing quota must not fail the directory page"); + + assert_eq!(page.items.len(), 2); + let by_id = page + .items + .iter() + .map(|item| (item.account.id.as_str(), item)) + .collect::>(); + let healthy = by_id.get("acct_test").expect("healthy account item"); + assert_eq!(healthy.quota.windows.len(), 1); + assert_eq!(healthy.quota.windows[0].key, "primary"); + let degraded = by_id.get("acct_bad_xai").expect("degraded account item"); + assert!(degraded.quota.windows.is_empty()); + assert!(degraded.quota.observed_at.is_none()); + assert!(degraded.quota.provider_data.is_none()); + assert_eq!(page.summary.total, 2); +} + +#[tokio::test] +async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() { let provider = FakeProviderAdmin::new("openai", events()); let mut account = account_record("openai"); - account.availability = AccountAvailability::Cooldown; - account.availability_reason = Some("usage_limit_exhausted".to_owned()); - account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5)); + account.availability = AccountAvailability::QuotaExhausted; account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5)); let store = FakeAccountStore::with_account(account, events()); @@ -1012,6 +1101,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() { window_seconds: Some(5 * 60 * 60), used_percent: Some(97.0), reset_at: Some(reset_at), + limit_reached: false, local_usage: None, provider_data: None, }], @@ -1166,8 +1256,6 @@ pub(super) fn account_record(kind: &str) -> AccountRecord { next_refresh_at: Some(now + TimeDelta::minutes(30)), enabled: true, availability: AccountAvailability::Ready, - availability_reason: None, - cooldown_until: None, availability_observed_at: now, quota_observed_at: None, created_at: now, @@ -1204,8 +1292,6 @@ fn prepared_create_with_id( next_refresh_at: Some(now + TimeDelta::minutes(30)), enabled: true, availability: AccountAvailability::Ready, - availability_reason: None, - cooldown_until: None, availability_observed_at: now, } } @@ -1290,9 +1376,7 @@ impl AccountProbe for FreeModelQuotaProbe { ) -> BoxFuture<'_, Result> { Box::pin(async move { let mut account = account_record("xai"); - account.availability = AccountAvailability::Cooldown; - account.availability_reason = Some("usage_limit_exhausted".to_owned()); - account.cooldown_until = Some(Utc::now() + TimeDelta::hours(24)); + account.availability = AccountAvailability::Ready; self.store.set_account_after_probe(account); Err(GatewayError::new( GatewayErrorKind::RateLimited, diff --git a/backend/crates/gateway-api/src/admin/accounts.rs b/backend/crates/gateway-api/src/admin/accounts.rs index 43795dd68..7990e6b39 100644 --- a/backend/crates/gateway-api/src/admin/accounts.rs +++ b/backend/crates/gateway-api/src/admin/accounts.rs @@ -194,7 +194,6 @@ pub struct AccountView { pub has_refresh_token: bool, pub status: String, pub display_status: String, - pub token_refreshing: bool, pub availability: String, pub enabled: bool, pub access_token_expires_at: Option, @@ -227,6 +226,7 @@ pub struct AccountQuotaWindowView { pub label_display: String, pub used_percent: Option, pub used_percent_display: String, + pub limit_reached: bool, #[serde(skip_serializing_if = "Option::is_none")] pub local_usage: Option, pub reset_at_display: String, @@ -1387,7 +1387,6 @@ fn account_view(item: AccountDirectoryItem, now: DateTime) -> AccountView { has_refresh_token: account.has_refresh_token, status: status.clone(), display_status: status, - token_refreshing: false, availability: account.availability.as_str().to_owned(), enabled: account.enabled, access_token_expires_at: expires_at, @@ -1435,6 +1434,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView { window_seconds, used_percent, reset_at, + limit_reached, local_usage, provider_data: _, } = window; @@ -1446,6 +1446,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView { used_percent, used_percent_display: used_percent .map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")), + limit_reached, local_usage: local_usage.as_ref().map(quota_local_usage), reset_at_display: reset_at.map_or_else(|| "—".to_owned(), |value| china_datetime(&value)), } diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs index 503b87bac..38dea83ea 100644 --- a/backend/crates/gateway-api/src/admin/observability/presenter.rs +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -784,6 +784,7 @@ pub(crate) fn dashboard_view( pool_summary: DashboardPoolSummaryView { total: provider_accounts.total, active: provider_accounts.active, + rate_limited: provider_accounts.rate_limited, expired: provider_accounts.expired, quota_exhausted: provider_accounts.quota_exhausted, refreshing: provider_accounts.refreshing, diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs index c475c8af0..ee2592b21 100644 --- a/backend/crates/gateway-api/src/admin/observability/wire.rs +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -348,6 +348,7 @@ pub struct DashboardAccountRequestBucketView { pub struct DashboardPoolSummaryView { pub total: u64, pub active: u64, + pub rate_limited: u64, pub expired: u64, pub quota_exhausted: u64, pub refreshing: Option, diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index 8f162bc1c..9bc3c14ef 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -224,6 +224,7 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli enabled: 804, unavailable: 792, active: 15, + rate_limited: 0, expired: 2, quota_exhausted: 785, refreshing: None, diff --git a/backend/crates/gateway-core/src/engine/credential.rs b/backend/crates/gateway-core/src/engine/credential.rs index 41448fdb0..28404c2c2 100644 --- a/backend/crates/gateway-core/src/engine/credential.rs +++ b/backend/crates/gateway-core/src/engine/credential.rs @@ -188,11 +188,13 @@ impl fmt::Debug for OpaqueProviderData { } /// 数据库中固定的账号运行状态。 +/// +/// 只承载终态或需要外部干预的状态；临时限流（429）不进入此状态，由 +/// Provider quota 数据驱动（`AccountQuotaSignals::limit_reached`）。 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AccountAvailability { Unknown, Ready, - Cooldown, QuotaExhausted, Expired, Banned, @@ -206,7 +208,6 @@ impl AccountAvailability { match self { Self::Unknown => "unknown", Self::Ready => "ready", - Self::Cooldown => "cooldown", Self::QuotaExhausted => "quota_exhausted", Self::Expired => "expired", Self::Banned => "banned", @@ -220,7 +221,6 @@ impl AccountAvailability { match value { "unknown" => Some(Self::Unknown), "ready" => Some(Self::Ready), - "cooldown" => Some(Self::Cooldown), "quota_exhausted" => Some(Self::QuotaExhausted), "expired" => Some(Self::Expired), "banned" => Some(Self::Banned), @@ -244,7 +244,6 @@ pub struct ProviderAccount { revision: CredentialRevision, enabled: bool, availability: AccountAvailability, - cooldown_until: Option, access_token_expires_at: Option, next_refresh_at: Option, has_refresh_token: bool, @@ -274,7 +273,6 @@ impl ProviderAccount { revision, enabled: true, availability: AccountAvailability::Unknown, - cooldown_until: None, access_token_expires_at, next_refresh_at: None, has_refresh_token: false, @@ -299,11 +297,9 @@ impl ProviderAccount { mut self, enabled: bool, availability: AccountAvailability, - cooldown_until: Option, ) -> Self { self.enabled = enabled; self.availability = availability; - self.cooldown_until = cooldown_until; self } @@ -373,11 +369,6 @@ impl ProviderAccount { self.enabled } - #[must_use] - pub const fn cooldown_until(&self) -> Option { - self.cooldown_until - } - #[must_use] pub const fn access_token_expires_at(&self) -> Option { self.access_token_expires_at @@ -398,7 +389,6 @@ impl ProviderAccount { pub fn is_schedulable(&self, now: SystemTime) -> bool { let available = match self.availability { AccountAvailability::Ready => true, - AccountAvailability::Cooldown => self.cooldown_until.is_some_and(|until| until <= now), AccountAvailability::Unknown | AccountAvailability::QuotaExhausted | AccountAvailability::Expired @@ -614,8 +604,6 @@ pub struct AccountStateChange { pub account_id: ProviderAccountId, pub expected_revision: CredentialRevision, pub availability: AccountAvailability, - pub reason: Option, - pub cooldown_until: Option, pub observed_at: SystemTime, } @@ -756,6 +744,7 @@ pub struct AccountRuntimeSignals { pub last_started_at: Option, pub quota_reset_at: Option, pub quota_remaining_rank: Option, + pub quota_limit_reached: bool, pub failure_rate_basis_points: Option, pub first_output_latency_ms: Option, } @@ -903,6 +892,7 @@ impl AccountRuntimeSignals { if let Some(quota) = quota { self.quota_reset_at = quota.reset_at; self.quota_remaining_rank = quota.remaining_rank; + self.quota_limit_reached = quota.limit_reached; } self } @@ -924,14 +914,20 @@ impl AccountRuntimeSignals { pub struct AccountQuotaSignals { reset_at: Option, remaining_rank: Option, + limit_reached: bool, } impl AccountQuotaSignals { #[must_use] - pub const fn new(reset_at: Option, remaining_rank: Option) -> Self { + pub const fn new( + reset_at: Option, + remaining_rank: Option, + limit_reached: bool, + ) -> Self { Self { reset_at, remaining_rank, + limit_reached, } } @@ -944,6 +940,12 @@ impl AccountQuotaSignals { pub const fn remaining_rank(self) -> Option { self.remaining_rank } + + /// 任一计量窗口已触顶（`limit_reached`/`used_percent >= 100`）；调度据此排除候选。 + #[must_use] + pub const fn limit_reached(self) -> bool { + self.limit_reached + } } /// 账号持久事实与可重建运行信号的请求级组合。 diff --git a/backend/crates/gateway-core/tests/engine/credential.rs b/backend/crates/gateway-core/tests/engine/credential.rs index 0f0f9626a..bd284aa53 100644 --- a/backend/crates/gateway-core/tests/engine/credential.rs +++ b/backend/crates/gateway-core/tests/engine/credential.rs @@ -24,7 +24,7 @@ fn account(id: &str) -> ProviderAccount { CredentialRevision::new(1).expect("valid revision"), Some(SystemTime::now() + Duration::from_secs(3600)), ) - .with_runtime_state(true, AccountAvailability::Ready, None) + .with_runtime_state(true, AccountAvailability::Ready) } fn candidate(id: &str, in_flight: u32, remaining: Option) -> AccountCandidate { @@ -35,6 +35,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option) -> AccountCandida last_started_at: None, quota_reset_at: None, quota_remaining_rank: remaining, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -110,31 +111,27 @@ fn availability_should_round_trip_all_database_values() { #[test] fn diagnostic_selection_bypasses_all_local_account_availability() { let exhausted = AccountCandidate { - account: account("acct_exhausted").with_runtime_state( - true, - AccountAvailability::QuotaExhausted, - None, - ), + account: account("acct_exhausted") + .with_runtime_state(true, AccountAvailability::QuotaExhausted), signals: AccountRuntimeSignals { in_flight: 0, last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, }; let disabled = AccountCandidate { - account: account("acct_disabled").with_runtime_state( - false, - AccountAvailability::QuotaExhausted, - None, - ), + account: account("acct_disabled") + .with_runtime_state(false, AccountAvailability::QuotaExhausted), signals: AccountRuntimeSignals { in_flight: 0, last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -158,33 +155,11 @@ fn diagnostic_selection_bypasses_all_local_account_availability() { #[test] fn disabled_account_should_not_be_schedulable() { - let account = - account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready, None); + let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready); assert!(!account.is_schedulable(SystemTime::now())); } -#[test] -fn cooldown_account_should_become_schedulable_at_its_deadline() { - let now = SystemTime::now(); - let account = - account("acct_cooldown").with_runtime_state(true, AccountAvailability::Cooldown, Some(now)); - - assert!(account.is_schedulable(now)); -} - -#[test] -fn cooldown_account_should_remain_isolated_before_its_deadline() { - let now = SystemTime::now(); - let account = account("acct_cooldown").with_runtime_state( - true, - AccountAvailability::Cooldown, - Some(now + Duration::from_secs(1)), - ); - - assert!(!account.is_schedulable(now)); -} - #[test] fn smart_selector_should_prefer_lower_inflight_count() { let candidates = vec![ @@ -311,10 +286,15 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() { last_started_at: Some(last_started_at), quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, } - .with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75)))); + .with_provider_quota(Some(AccountQuotaSignals::new( + Some(reset_at), + Some(75), + false, + ))); assert_eq!( ( diff --git a/backend/crates/gateway-core/tests/provider_ports/mod.rs b/backend/crates/gateway-core/tests/provider_ports/mod.rs index 85256918a..6109e343b 100644 --- a/backend/crates/gateway-core/tests/provider_ports/mod.rs +++ b/backend/crates/gateway-core/tests/provider_ports/mod.rs @@ -101,6 +101,7 @@ fn scheduling_state_preserves_provider_neutral_signals() { last_started_at: None, quota_reset_at: None, quota_remaining_rank: Some(7), + quota_limit_reached: false, failure_rate_basis_points: Some(125), first_output_latency_ms: Some(250), }, diff --git a/backend/crates/gateway-store/src/bundle.rs b/backend/crates/gateway-store/src/bundle.rs index 8ba6c8e87..2ebd070c6 100644 --- a/backend/crates/gateway-store/src/bundle.rs +++ b/backend/crates/gateway-store/src/bundle.rs @@ -1,8 +1,10 @@ //! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。 +use gateway_core::engine::credential::ProviderAccountStore; + use super::*; -/// 已完成连接、迁移与 cooldown hydration 的 Store 能力集合。 +/// 已完成连接、迁移与 hydration 的 Store 能力集合。 pub struct StoreBundle { admin_ports: AdminStorePorts, core_ports: CoreStorePorts, @@ -61,12 +63,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult { redis_connection.clone(), REDIS_NAMESPACE, )?); - let account_store = Arc::new(redis::CooldownCachingProviderAccountStore::new( - provider_accounts, - cooldowns.clone(), - )); - let hydrated = account_store.hydrate(std::time::SystemTime::now()).await; - tracing::info!(hydrated, "账号 cooldown Redis 热缓存重建完成"); + let account_store: Arc = provider_accounts; let credential_leases = redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?; diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index cf690069e..464a715a2 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -263,6 +263,7 @@ pub(crate) const fn admin_account_pool_metrics( enabled: metrics.enabled, unavailable: metrics.unavailable, active: metrics.active, + rate_limited: metrics.rate_limited, expired: metrics.expired, quota_exhausted: metrics.quota_exhausted, refreshing: None, diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs index 92a7e6219..83f9adb1c 100644 --- a/backend/crates/gateway-store/src/postgres/observability/model.rs +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -366,6 +366,7 @@ pub struct ProviderAccountMetrics { pub enabled: u64, pub unavailable: u64, pub active: u64, + pub rate_limited: u64, pub expired: u64, pub quota_exhausted: u64, pub disabled: u64, diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs index 95da964cd..c9df4e928 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs @@ -348,17 +348,11 @@ pub(crate) async fn provider_account_metrics( ) -> StoreResult { let row = sqlx::query( "with normalized as ( - select enabled, + select enabled, availability, provider_quota_json, case when not enabled then 'disabled' when availability = 'banned' then 'banned' - when availability = 'quota_exhausted' - or ( - availability = 'cooldown' - and availability_reason = 'usage_limit_exhausted' - and (cooldown_until is null or cooldown_until > $1) - ) - then 'quota_exhausted' + when availability = 'quota_exhausted' then 'quota_exhausted' when availability in ('expired', 'invalid') or (access_token_expires_at is not null and access_token_expires_at <= $1) then 'expired' @@ -370,6 +364,17 @@ pub(crate) async fn provider_account_metrics( count(*) filter (where enabled)::bigint as enabled, count(*) filter (where status <> 'active')::bigint as unavailable, count(*) filter (where status = 'active')::bigint as active, + count(*) filter ( + where status = 'active' + and ( + provider_quota_json is not null + and ( + provider_quota_json->'rate_limit'->>'limit_reached' = 'true' + or (provider_quota_json->'rate_limit'->'primary_window'->>'used_percent')::numeric >= 100 + or (provider_quota_json->'rate_limit'->'secondary_window'->>'used_percent')::numeric >= 100 + ) + ) + )::bigint as rate_limited, count(*) filter (where status = 'expired')::bigint as expired, count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted, count(*) filter (where status = 'disabled')::bigint as disabled, @@ -385,6 +390,7 @@ pub(crate) async fn provider_account_metrics( enabled: unsigned(&row, "enabled")?, unavailable: unsigned(&row, "unavailable")?, active: unsigned(&row, "active")?, + rate_limited: unsigned(&row, "rate_limited")?, expired: unsigned(&row, "expired")?, quota_exhausted: unsigned(&row, "quota_exhausted")?, disabled: unsigned(&row, "disabled")?, @@ -402,13 +408,7 @@ pub(crate) async fn active_provider_account_ids( case when not enabled then 'disabled' when availability = 'banned' then 'banned' - when availability = 'quota_exhausted' - or ( - availability = 'cooldown' - and availability_reason = 'usage_limit_exhausted' - and (cooldown_until is null or cooldown_until > $1) - ) - then 'quota_exhausted' + when availability = 'quota_exhausted' then 'quota_exhausted' when availability in ('expired', 'invalid') or (access_token_expires_at is not null and access_token_expires_at <= $1) then 'expired' diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs index 5787d767c..796cc6bb1 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs @@ -32,8 +32,6 @@ impl ProviderAccountStore for PgProviderAccountRepository { next_refresh_at: account.account.next_refresh_at().map(DateTime::::from), enabled: account.account.enabled(), availability: account.account.availability(), - availability_reason: None, - cooldown_until: account.account.cooldown_until().map(DateTime::::from), availability_observed_at: Utc::now(), }) .await @@ -242,8 +240,6 @@ impl ProviderAccountStore for PgProviderAccountRepository { expected_revision: Revision::new(change.expected_revision.get()) .map_err(core_store_error)?, availability: change.availability, - availability_reason: change.reason, - cooldown_until: change.cooldown_until.map(DateTime::::from), availability_observed_at: DateTime::::from(change.observed_at), }) .await diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs index 3d4a0363a..22a0a4df8 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs @@ -63,11 +63,7 @@ pub(crate) fn admin_account_status( AdminAccountStatus::Disabled } else if account.availability == AccountAvailability::Banned { AdminAccountStatus::Banned - } else if account.availability == AccountAvailability::QuotaExhausted - || (account.availability == AccountAvailability::Cooldown - && account.availability_reason.as_deref() == Some("usage_limit_exhausted") - && account.cooldown_until.is_none_or(|until| until > now)) - { + } else if account.availability == AccountAvailability::QuotaExhausted { AdminAccountStatus::QuotaExhausted } else if account.availability == AccountAvailability::Expired || account.availability == AccountAvailability::Invalid @@ -184,8 +180,6 @@ pub(crate) fn admin_account_record( next_refresh_at: summary.next_refresh_at, enabled: summary.enabled, availability: summary.availability, - availability_reason: summary.availability_reason, - cooldown_until: summary.cooldown_until, availability_observed_at: summary.availability_observed_at, quota_observed_at: summary.quota_observed_at, created_at: summary.created_at, @@ -211,8 +205,6 @@ pub(crate) fn prepared_account( next_refresh_at: credential.next_refresh_at, enabled: credential.enabled, availability: credential.availability, - availability_reason: credential.availability_reason, - cooldown_until: credential.cooldown_until, availability_observed_at: credential.availability_observed_at, }) } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs index 48b0bf4a7..fdfd8db73 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs @@ -93,7 +93,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { "select id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - availability_reason, cooldown_until, availability_observed_at, + availability_observed_at, quota_observed_at, created_at, updated_at from provider_accounts where ($1::text is null or provider_kind = $1) and ($2 or enabled) @@ -114,11 +114,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository { id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, - availability, availability_reason, cooldown_until, provider_quota_json, + availability, provider_quota_json, availability_observed_at, quota_observed_at, created_at, updated_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, - $14, null, $15, null, $16, null, now(), greatest(now(), $16) + $14, null, $15, null, now(), greatest(now(), $15) )", ) .bind(account.id) @@ -135,7 +135,6 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(account.next_refresh_at) .bind(account.enabled) .bind(account.availability.as_str()) - .bind(account.cooldown_until) .bind(account.availability_observed_at) .execute(&self.pool) .await @@ -210,21 +209,17 @@ impl ProviderAccountRepository for PgProviderAccountRepository { let result = sqlx::query( "update provider_accounts set availability = case when enabled then $3 else availability end, - availability_reason = case when enabled then $4 else availability_reason end, - cooldown_until = case when enabled then $5 else cooldown_until end, availability_observed_at = case - when enabled then $6 + when enabled then $4 else availability_observed_at end, - updated_at = case when enabled then greatest(now(), $6) else updated_at end + updated_at = case when enabled then greatest(now(), $4) else updated_at end where id = $1 and credential_revision = $2 - and (availability_observed_at is null or availability_observed_at <= $6)", + and (availability_observed_at is null or availability_observed_at <= $4)", ) .bind(update.account_id) .bind(to_i64(update.expected_revision.get())?) .bind(update.availability.as_str()) - .bind(update.availability_reason) - .bind(update.cooldown_until) .bind(update.availability_observed_at) .execute(&self.pool) .await @@ -466,11 +461,11 @@ pub(crate) async fn upsert_provider_account_in_transaction( id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, - availability, availability_reason, cooldown_until, provider_quota_json, + availability, provider_quota_json, availability_observed_at, quota_observed_at, created_at, updated_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, - $14, $15, $16, null, $17, null, now(), greatest(now(), $17) + $14, null, $15, null, now(), greatest(now(), $15) ) on conflict ( provider_kind, @@ -488,8 +483,6 @@ pub(crate) async fn upsert_provider_account_in_transaction( next_refresh_at = excluded.next_refresh_at, enabled = excluded.enabled, availability = excluded.availability, - availability_reason = excluded.availability_reason, - cooldown_until = excluded.cooldown_until, provider_quota_json = null, availability_observed_at = excluded.availability_observed_at, quota_observed_at = null, @@ -510,8 +503,6 @@ pub(crate) async fn upsert_provider_account_in_transaction( .bind(account.next_refresh_at) .bind(account.enabled) .bind(account.availability.as_str()) - .bind(&account.availability_reason) - .bind(account.cooldown_until) .bind(account.availability_observed_at) .fetch_optional(&mut **transaction) .await @@ -565,16 +556,6 @@ pub(crate) async fn rotate_provider_account_in_transaction( when availability <> 'quota_exhausted' then 'ready' else availability end, - availability_reason = case - when not enabled then availability_reason - when availability <> 'quota_exhausted' then null - else availability_reason - end, - cooldown_until = case - when not enabled then cooldown_until - when availability <> 'quota_exhausted' then null - else cooldown_until - end, availability_observed_at = case when not enabled then availability_observed_at when availability <> 'quota_exhausted' then now() diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs index 47ec13f9c..010c24ac7 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -70,26 +70,6 @@ pub(crate) fn parse_availability(value: &str) -> StoreResult StoreResult<()> { - if availability == AccountAvailability::Cooldown && !has_cooldown_until { - return Err(invalid("cooldown availability requires cooldown_until")); - } - if has_cooldown_until - && !matches!( - availability, - AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted - ) - { - return Err(invalid( - "cooldown_until is only allowed for cooldown or quota_exhausted availability", - )); - } - Ok(()) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProviderAccountSummary { pub id: String, @@ -106,8 +86,6 @@ pub struct ProviderAccountSummary { pub next_refresh_at: Option>, pub enabled: bool, pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, pub availability_observed_at: DateTime, pub quota_observed_at: Option>, pub created_at: DateTime, @@ -151,8 +129,6 @@ pub struct NewProviderAccount { pub next_refresh_at: Option>, pub enabled: bool, pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, pub availability_observed_at: DateTime, } @@ -173,7 +149,6 @@ impl fmt::Debug for NewProviderAccount { .field("name", &self.name) .field("enabled", &self.enabled) .field("availability", &self.availability) - .field("cooldown_until", &self.cooldown_until) .field("provider_credentials_json", &"[REDACTED]") .finish_non_exhaustive() } @@ -189,7 +164,6 @@ impl NewProviderAccount { if !self.has_refresh_token && self.next_refresh_at.is_some() { return Err(invalid("next_refresh_at requires a refresh token")); } - validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; validate_object_size( "provider_credentials_json", &self.provider_credentials_json, @@ -321,8 +295,6 @@ impl fmt::Debug for ProviderCredentialUpdate { pub struct ProviderAccountObservation { pub account_id: String, pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, pub provider_quota_json: Option, pub availability_observed_at: DateTime, pub quota_observed_at: Option>, @@ -333,15 +305,12 @@ pub struct ProviderAccountStateUpdate { pub account_id: String, pub expected_revision: Revision, pub availability: AccountAvailability, - pub availability_reason: Option, - pub cooldown_until: Option>, pub availability_observed_at: DateTime, } impl ProviderAccountStateUpdate { pub fn validate(&self) -> StoreResult<()> { require_nonempty(ENTITY, "account_id", &self.account_id)?; - validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; Ok(()) } } @@ -349,7 +318,6 @@ impl ProviderAccountStateUpdate { impl ProviderAccountObservation { pub fn validate(&self) -> StoreResult<()> { require_nonempty(ENTITY, "account_id", &self.account_id)?; - validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?; if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() { return Err(invalid("quota JSON and quota_observed_at must agree")); } @@ -363,14 +331,14 @@ impl ProviderAccountObservation { pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - availability_reason, cooldown_until, provider_quota_json, + provider_quota_json, availability_observed_at, quota_observed_at, created_at, updated_at from provider_accounts where id = $1"; pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - availability_reason, cooldown_until, provider_quota_json, + provider_quota_json, availability_observed_at, quota_observed_at, created_at, updated_at from provider_accounts where id = any($1::text[]) and provider_kind = $2 @@ -420,11 +388,7 @@ pub(crate) fn core_account_from_summary( summary.upstream_account_id, summary.plan_type, ) - .with_runtime_state( - summary.enabled, - summary.availability, - summary.cooldown_until.map(Into::into), - ) + .with_runtime_state(summary.enabled, summary.availability) .with_refresh_schedule( summary.has_refresh_token, summary.next_refresh_at.map(Into::into), @@ -474,8 +438,6 @@ pub(crate) fn account_summary_from_row( next_refresh_at: get(&row, "next_refresh_at")?, enabled: get(&row, "enabled")?, availability: parse_availability(&availability)?, - availability_reason: get(&row, "availability_reason")?, - cooldown_until: get(&row, "cooldown_until")?, availability_observed_at: get(&row, "availability_observed_at")?, quota_observed_at: get(&row, "quota_observed_at")?, created_at: get(&row, "created_at")?, diff --git a/backend/crates/gateway-store/src/redis/credential_cooldown.rs b/backend/crates/gateway-store/src/redis/credential_cooldown.rs index a1ccc0af1..ecd713fe1 100644 --- a/backend/crates/gateway-store/src/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/src/redis/credential_cooldown.rs @@ -1,22 +1,13 @@ -//! 请求调度用的可丢失账号 cooldown Redis 存储。 - -use std::{sync::Arc, time::SystemTime}; +//! 请求调度用的可丢失 Provider 级 cooldown Redis 存储。 use async_trait::async_trait; use chrono::{DateTime, Utc}; use gateway_core::{ - engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision, LoadedCredential, NewProviderAccount, ProviderAccount, - ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation, - QuotaWriteOutcome, - }, - error::StoreError as CoreStoreError, + engine::credential::{CredentialRevision, ProviderAccountId}, provider_ports::{ ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown, ProviderStoreError, ProviderStoreErrorKind, }, - routing::ProviderKind, }; use redis::{Script, aio::ConnectionManager}; @@ -352,216 +343,6 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { } } -/// 账号状态写入到可丢失 cooldown cache 的桥接 adapter。 -pub struct CooldownCachingProviderAccountStore { - authoritative: Arc, - cooldowns: Arc, -} - -impl CooldownCachingProviderAccountStore { - #[must_use] - pub const fn new( - authoritative: Arc, - cooldowns: Arc, - ) -> Self { - Self { - authoritative, - cooldowns, - } - } - - /// 启动时从 PostgreSQL 尽力重建仍有效的 cooldown cache。 - pub async fn hydrate(&self, now: SystemTime) -> u64 { - let Ok(accounts) = self.authoritative.list_accounts().await else { - return 0; - }; - let mut hydrated = 0_u64; - for account in accounts { - let Some(cooldown) = projection( - account.id(), - account.revision(), - account.availability(), - account.cooldown_until(), - now, - ) else { - continue; - }; - if matches!( - self.cooldowns.cache_credential_cooldown(&cooldown).await, - Ok(true) - ) { - hydrated = hydrated.saturating_add(1); - } - } - hydrated - } - - async fn mirror( - &self, - account_id: &ProviderAccountId, - credential_revision: CredentialRevision, - availability: AccountAvailability, - cooldown_until: Option, - ) { - let Some(cooldown) = projection( - account_id, - credential_revision, - availability, - cooldown_until, - SystemTime::now(), - ) else { - let Ok(revision) = Revision::new(credential_revision.get()) else { - return; - }; - let _ = self - .cooldowns - .invalidate_credential_cooldown(account_id.as_str(), revision) - .await; - return; - }; - let _ = self.cooldowns.cache_credential_cooldown(&cooldown).await; - } -} - -#[async_trait] -impl ProviderAccountStore for CooldownCachingProviderAccountStore { - async fn create_account(&self, account: NewProviderAccount) -> Result<(), CoreStoreError> { - self.authoritative.create_account(account).await - } - - async fn get_account( - &self, - account: &ProviderAccountId, - ) -> Result, CoreStoreError> { - self.authoritative.get_account(account).await - } - - async fn list_accounts(&self) -> Result, CoreStoreError> { - self.authoritative.list_accounts().await - } - - async fn list_for_provider( - &self, - provider: &ProviderKind, - ) -> Result, CoreStoreError> { - self.authoritative.list_for_provider(provider).await - } - - async fn load_credential( - &self, - account: &ProviderAccountId, - expected_revision: CredentialRevision, - ) -> Result { - self.authoritative - .load_credential(account, expected_revision) - .await - } - - async fn load_current_credential( - &self, - account: &ProviderAccountId, - ) -> Result { - self.authoritative.load_current_credential(account).await - } - - async fn compare_and_swap_credential( - &self, - update: CredentialCasUpdate, - ) -> Result { - let account_id = update.account_id().clone(); - let outcome = self - .authoritative - .compare_and_swap_credential(update) - .await?; - if let CredentialCasOutcome::Updated(credential_revision) = outcome - && let Ok(revision) = Revision::new(credential_revision.get()) - { - let _ = self - .cooldowns - .invalidate_credential_cooldown(account_id.as_str(), revision) - .await; - } - Ok(outcome) - } - - async fn get_quotas( - &self, - accounts: &[ProviderAccountId], - ) -> Result, CoreStoreError> { - self.authoritative.get_quotas(accounts).await - } - - async fn compare_and_swap_quota( - &self, - observation: QuotaObservation, - ) -> Result { - self.authoritative.compare_and_swap_quota(observation).await - } - - async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), CoreStoreError> { - let account_id = change.account_id.clone(); - let credential_revision = change.expected_revision; - let availability = change.availability; - let cooldown_until = change.cooldown_until; - self.authoritative.apply_state_change(change).await?; - let Some(account) = self.authoritative.get_account(&account_id).await? else { - return Ok(()); - }; - if !account.enabled() { - return Ok(()); - } - self.mirror( - &account_id, - credential_revision, - availability, - cooldown_until, - ) - .await; - Ok(()) - } - - async fn update_account(&self, update: ProviderAccountUpdate) -> Result<(), CoreStoreError> { - self.authoritative.update_account(update).await - } - - async fn set_enabled( - &self, - account: &ProviderAccountId, - enabled: bool, - ) -> Result<(), CoreStoreError> { - self.authoritative.set_enabled(account, enabled).await - } - - async fn delete_account(&self, account: &ProviderAccountId) -> Result<(), CoreStoreError> { - self.authoritative.delete_account(account).await?; - if let Ok(through_revision) = Revision::new(u64::MAX) { - let _ = self - .cooldowns - .invalidate_credential_cooldown(account.as_str(), through_revision) - .await; - } - Ok(()) - } -} - -fn projection( - account_id: &ProviderAccountId, - credential_revision: CredentialRevision, - availability: AccountAvailability, - cooldown_until: Option, - now: SystemTime, -) -> Option { - let cooldown_until = cooldown_until - .filter(|_| availability == AccountAvailability::Cooldown) - .filter(|until| *until > now) - .map(DateTime::::from)?; - Some(CredentialCooldown { - provider_account_id: account_id.as_str().to_owned(), - credential_revision: Revision::new(credential_revision.get()).ok()?, - cooldown_until, - }) -} - fn invalid(message: &str) -> StoreError { StoreError::InvalidData { entity: "credential cooldown", diff --git a/backend/crates/gateway-store/src/redis/credential_leases.rs b/backend/crates/gateway-store/src/redis/credential_leases.rs index d303ac204..f6b3c24b9 100644 --- a/backend/crates/gateway-store/src/redis/credential_leases.rs +++ b/backend/crates/gateway-store/src/redis/credential_leases.rs @@ -512,6 +512,7 @@ impl RedisProviderLeaseCoordinator { last_started_at: signal.last_started_at.map(Into::into), quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts.rs index 9e9589543..eab199c3e 100644 --- a/backend/crates/gateway-store/tests/postgres/provider_accounts.rs +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts.rs @@ -28,9 +28,9 @@ use gateway_store::{ postgres::{ AdminAuditActorKind, AdminAuditEvent, DeleteProviderAccounts, ImportProviderAccounts, NewProviderAccount, PgAdminAccountStore, PgProviderAccountRepository, - ProviderAccountAdminRepository, ProviderAccountAdminScope, ProviderAccountObservation, - ProviderAccountRepository, ProviderCredentialUpdate, RotateProviderAccount, - SetProviderAccountEnabled, UpdateProviderAccount, + ProviderAccountAdminRepository, ProviderAccountAdminScope, ProviderAccountRepository, + ProviderCredentialUpdate, RotateProviderAccount, SetProviderAccountEnabled, + UpdateProviderAccount, }, }; use serde_json::json; @@ -161,8 +161,6 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() { account_id: account_id.clone(), expected_revision: revision, availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: Some(newer), observed_at: newer, }) .await @@ -173,8 +171,6 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() { account_id: account_id.clone(), expected_revision: revision, availability: AccountAvailability::Ready, - reason: None, - cooldown_until: None, observed_at: older, }) .await @@ -747,7 +743,6 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it() updated.name = "updated import".to_owned(); updated.provider_credentials_json = credential_json("updated-import-secret"); updated.availability = AccountAvailability::Banned; - updated.availability_reason = Some("upstream_account_deactivated".to_owned()); let imported = repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), @@ -819,8 +814,6 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() { next_refresh_at: None, enabled: true, availability: AccountAvailability::Ready, - availability_reason: None, - cooldown_until: None, availability_observed_at: Utc::now(), }) .await @@ -942,8 +935,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() { }; let mut cooldown_account = account("acct_admin_b", "user-admin-b"); - cooldown_account.availability = AccountAvailability::Cooldown; - cooldown_account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(10)); + cooldown_account.availability = AccountAvailability::Ready; let imported = repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), @@ -1158,7 +1150,6 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() { }; let mut exhausted = account("acct_rotation_quota", "user-rotation-quota"); exhausted.availability = AccountAvailability::QuotaExhausted; - exhausted.availability_reason = Some("quota_exhausted".to_owned()); repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), @@ -1224,29 +1215,20 @@ async fn xai_free_quota_exhaustion_state_is_persisted() { account_id: account_id.clone(), expected_revision: CredentialRevision::new(1).expect("credential revision"), availability: AccountAvailability::QuotaExhausted, - reason: Some("upstream_free_quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await .expect("persist xAI free quota exhaustion"); - let current: (String, String, Option) = sqlx::query_as( - "select provider_kind, availability, availability_reason + let current: (String, String) = sqlx::query_as( + "select provider_kind, availability from provider_accounts where id = $1", ) .bind(account_id.as_str()) .fetch_one(&database.pool) .await .expect("load persisted xAI quota state"); - assert_eq!( - current, - ( - "xai".to_owned(), - "quota_exhausted".to_owned(), - Some("upstream_free_quota_exhausted".to_owned()), - ), - ); + assert_eq!(current, ("xai".to_owned(), "quota_exhausted".to_owned()),); database.close().await; } @@ -1266,38 +1248,25 @@ async fn xai_resettable_usage_limit_state_is_persisted() { .expect("insert xAI account"); let observed_at = SystemTime::now(); - let cooldown_until = observed_at - .checked_add(Duration::from_secs(24 * 60 * 60)) - .expect("cooldown deadline"); repository .apply_state_change(AccountStateChange { account_id: account_id.clone(), expected_revision: CredentialRevision::new(1).expect("credential revision"), - availability: AccountAvailability::Cooldown, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: Some(cooldown_until), + availability: AccountAvailability::Ready, observed_at, }) .await .expect("persist xAI resettable usage limit"); - let current: (String, String, Option, bool) = sqlx::query_as( - "select provider_kind, availability, availability_reason, cooldown_until is not null + let current: (String, String) = sqlx::query_as( + "select provider_kind, availability from provider_accounts where id = $1", ) .bind(account_id.as_str()) .fetch_one(&database.pool) .await .expect("load persisted xAI usage limit state"); - assert_eq!( - current, - ( - "xai".to_owned(), - "cooldown".to_owned(), - Some("usage_limit_exhausted".to_owned()), - true, - ), - ); + assert_eq!(current, ("xai".to_owned(), "ready".to_owned())); database.close().await; } @@ -1315,7 +1284,6 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() { let mut disabled = account(account_id.as_str(), "user-disabled-refresh"); disabled.enabled = false; disabled.availability = AccountAvailability::Expired; - disabled.availability_reason = Some("credential_expired".to_owned()); repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), @@ -1334,8 +1302,6 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() { account_id: account_id.clone(), expected_revision: CredentialRevision::new(1).expect("credential revision"), availability: AccountAvailability::Ready, - reason: None, - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -1389,8 +1355,6 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount { next_refresh_at: None, enabled: true, availability: AccountAvailability::Ready, - availability_reason: None, - cooldown_until: None, availability_observed_at: Utc::now(), } } @@ -1517,31 +1481,15 @@ fn provider_credentials_are_redacted_from_debug() { assert!(!format!("{object:?}").contains(secret)); } -#[test] -fn cooldown_observation_requires_expiry() { - let observation = ProviderAccountObservation { - account_id: "account-1".to_owned(), - availability: AccountAvailability::Cooldown, - availability_reason: None, - cooldown_until: None, - provider_quota_json: None, - availability_observed_at: Utc::now(), - quota_observed_at: None, - }; - assert!(observation.validate().is_err()); -} - #[test] fn quota_exhausted_account_accepts_cooldown_runtime_facts() { let mut imported = account("acct_quota_cooldown", "user-quota-cooldown"); imported.availability = AccountAvailability::QuotaExhausted; - imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5)); assert!(imported.validate().is_ok()); } #[test] -fn imported_account_rejects_cooldown_until_for_ready_runtime_facts() { - let mut imported = account("acct_invalid_cooldown", "user-invalid-cooldown"); - imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5)); - assert!(imported.validate().is_err()); +fn imported_ready_account_passes_validation() { + let imported = account("acct_ready_import", "user-ready-import"); + assert!(imported.validate().is_ok()); } diff --git a/backend/crates/gateway-store/tests/redis/credential_cooldown.rs b/backend/crates/gateway-store/tests/redis/credential_cooldown.rs index 3e2c83f17..c923c825d 100644 --- a/backend/crates/gateway-store/tests/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/tests/redis/credential_cooldown.rs @@ -1,27 +1,13 @@ -use std::{ - sync::{Arc, Mutex}, - time::{Duration as StdDuration, SystemTime}, -}; +use std::time::{Duration as StdDuration, SystemTime}; -use async_trait::async_trait; use chrono::{DateTime, Duration, Utc}; use gateway_core::{ - engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision, LoadedCredential, NewProviderAccount, PlaintextCredential, - ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, - QuotaObservation, QuotaWriteOutcome, - }, - error::{StoreError as CoreStoreError, StoreErrorKind}, + engine::credential::{CredentialRevision, ProviderAccountId}, provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown}, - routing::ProviderKind, }; use gateway_store::{ - Revision, StoreBackend, StoreError, StoreResult, - redis::{ - CooldownCachingProviderAccountStore, CredentialCooldown, CredentialCooldownRepository, - RedisCredentialCooldownRepository, - }, + Revision, + redis::{CredentialCooldown, CredentialCooldownRepository, RedisCredentialCooldownRepository}, }; use redis::aio::ConnectionManager; use uuid::Uuid; @@ -36,409 +22,6 @@ fn credential_cooldown_is_revision_fenced() { assert_eq!(cooldown.credential_revision.get(), 2); } -struct CooldownTestAccountStore { - accounts: Vec, - calls: Arc>>, - fail_state_write: bool, -} - -#[async_trait] -impl ProviderAccountStore for CooldownTestAccountStore { - async fn create_account(&self, _account: NewProviderAccount) -> Result<(), CoreStoreError> { - unreachable!("cooldown tests do not create accounts") - } - - async fn get_account( - &self, - account: &ProviderAccountId, - ) -> Result, CoreStoreError> { - Ok(self - .accounts - .iter() - .find(|candidate| candidate.id() == account) - .cloned()) - } - - async fn list_accounts(&self) -> Result, CoreStoreError> { - Ok(self.accounts.clone()) - } - - async fn list_for_provider( - &self, - _provider: &ProviderKind, - ) -> Result, CoreStoreError> { - unreachable!("cooldown tests do not list one provider") - } - - async fn load_credential( - &self, - _account: &ProviderAccountId, - _expected_revision: CredentialRevision, - ) -> Result { - unreachable!("cooldown tests do not load credentials") - } - - async fn load_current_credential( - &self, - _account: &ProviderAccountId, - ) -> Result { - unreachable!("cooldown tests do not load credentials") - } - - async fn compare_and_swap_credential( - &self, - _update: CredentialCasUpdate, - ) -> Result { - self.calls.lock().expect("calls lock").push("postgres"); - Ok(CredentialCasOutcome::Updated( - CredentialRevision::new(8).expect("positive revision"), - )) - } - - async fn get_quotas( - &self, - _accounts: &[ProviderAccountId], - ) -> Result, CoreStoreError> { - unreachable!("cooldown tests do not read quota") - } - - async fn compare_and_swap_quota( - &self, - _observation: QuotaObservation, - ) -> Result { - unreachable!("cooldown tests do not write quota") - } - - async fn apply_state_change(&self, _change: AccountStateChange) -> Result<(), CoreStoreError> { - self.calls.lock().expect("calls lock").push("postgres"); - if self.fail_state_write { - Err(CoreStoreError::new(StoreErrorKind::Unavailable)) - } else { - Ok(()) - } - } - - async fn update_account(&self, _update: ProviderAccountUpdate) -> Result<(), CoreStoreError> { - unreachable!("cooldown tests do not update account metadata") - } - - async fn set_enabled( - &self, - _account: &ProviderAccountId, - _enabled: bool, - ) -> Result<(), CoreStoreError> { - unreachable!("cooldown tests do not change account switches") - } - - async fn delete_account(&self, _account: &ProviderAccountId) -> Result<(), CoreStoreError> { - self.calls.lock().expect("calls lock").push("postgres"); - Ok(()) - } -} - -struct CooldownTestCache { - calls: Arc>>, - cached: Mutex>, - invalidated: Mutex>, - fail: bool, -} - -#[async_trait] -impl CredentialCooldownRepository for CooldownTestCache { - async fn cache_credential_cooldown(&self, cooldown: &CredentialCooldown) -> StoreResult { - self.calls.lock().expect("calls lock").push("redis"); - if self.fail { - return Err(StoreError::Unavailable { - backend: StoreBackend::Redis, - message: "test cache unavailable".to_owned(), - }); - } - self.cached - .lock() - .expect("cached lock") - .push(cooldown.clone()); - Ok(true) - } - - async fn read_credential_cooldown( - &self, - _provider_account_id: &str, - ) -> StoreResult> { - unreachable!("write-through adapter does not consume cooldown cache") - } - - async fn invalidate_credential_cooldown( - &self, - provider_account_id: &str, - through_revision: Revision, - ) -> StoreResult { - self.calls.lock().expect("calls lock").push("redis"); - if self.fail { - return Err(StoreError::Unavailable { - backend: StoreBackend::Redis, - message: "test cache unavailable".to_owned(), - }); - } - self.invalidated - .lock() - .expect("invalidated lock") - .push((provider_account_id.to_owned(), through_revision)); - Ok(true) - } -} - -fn composition_fixture( - fail_state_write: bool, - fail_cache: bool, -) -> ( - CooldownCachingProviderAccountStore, - Arc, - Arc>>, -) { - composition_fixture_with_accounts( - vec![ready_test_account("acct_cooldown_composition")], - fail_state_write, - fail_cache, - ) -} - -fn composition_fixture_with_accounts( - accounts: Vec, - fail_state_write: bool, - fail_cache: bool, -) -> ( - CooldownCachingProviderAccountStore, - Arc, - Arc>>, -) { - let calls = Arc::new(Mutex::new(Vec::new())); - let authoritative = Arc::new(CooldownTestAccountStore { - accounts, - calls: Arc::clone(&calls), - fail_state_write, - }); - let cache = Arc::new(CooldownTestCache { - calls: Arc::clone(&calls), - cached: Mutex::new(Vec::new()), - invalidated: Mutex::new(Vec::new()), - fail: fail_cache, - }); - let store = CooldownCachingProviderAccountStore::new( - authoritative, - Arc::clone(&cache) as Arc, - ); - (store, cache, calls) -} - -#[tokio::test] -async fn cooldown_composition_writes_cache_only_after_postgres_commit() { - let (store, cache, calls) = composition_fixture(false, false); - - store - .apply_state_change(cooldown_state_change(60)) - .await - .expect("persist cooldown"); - - assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]); - assert_eq!(cache.cached.lock().expect("cached lock").len(), 1); -} - -#[tokio::test] -async fn cooldown_composition_does_not_cache_failed_postgres_state() { - let (store, cache, calls) = composition_fixture(true, false); - - assert!( - store - .apply_state_change(cooldown_state_change(60)) - .await - .is_err() - ); - assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]); - assert!(cache.cached.lock().expect("cached lock").is_empty()); -} - -#[tokio::test] -async fn cooldown_composition_keeps_postgres_terminal_when_redis_fails() { - let (store, _cache, calls) = composition_fixture(false, true); - - store - .apply_state_change(cooldown_state_change(60)) - .await - .expect("Redis is only a cache"); - - assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]); -} - -#[tokio::test] -async fn disabled_account_state_write_does_not_change_runtime_cooldown() { - let disabled = test_provider_account("acct_cooldown_composition").with_runtime_state( - false, - AccountAvailability::Ready, - None, - ); - let (store, cache, calls) = composition_fixture_with_accounts(vec![disabled], false, false); - - store - .apply_state_change(cooldown_state_change(60)) - .await - .expect("disabled state write is a no-op"); - - assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]); - assert!(cache.cached.lock().expect("cached lock").is_empty()); - assert!( - cache - .invalidated - .lock() - .expect("invalidated lock") - .is_empty() - ); -} - -#[tokio::test] -async fn ready_state_invalidates_same_revision_cooldown_after_postgres_commit() { - let (store, cache, calls) = composition_fixture(false, false); - let mut change = cooldown_state_change(60); - change.availability = AccountAvailability::Ready; - change.reason = None; - change.cooldown_until = None; - - store - .apply_state_change(change) - .await - .expect("persist ready state"); - - assert_eq!( - cache.invalidated.lock().expect("invalidated lock")[0].1, - Revision::new(7).expect("positive revision") - ); - assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]); -} - -#[tokio::test] -async fn credential_rotation_invalidates_cooldown_through_new_revision() { - let (store, cache, calls) = composition_fixture(false, false); - - let outcome = store - .compare_and_swap_credential(credential_update()) - .await - .expect("rotate credential"); - - assert_eq!( - outcome, - CredentialCasOutcome::Updated(CredentialRevision::new(8).expect("positive revision")) - ); - assert_eq!( - cache.invalidated.lock().expect("invalidated lock")[0].1, - Revision::new(8).expect("positive revision") - ); - assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]); -} - -#[tokio::test] -async fn account_delete_invalidates_all_stale_cooldown_revisions() { - let (store, cache, calls) = composition_fixture(false, false); - let account_id = ProviderAccountId::new("acct_cooldown_delete").expect("valid account ID"); - - store - .delete_account(&account_id) - .await - .expect("delete account"); - - assert_eq!( - cache.invalidated.lock().expect("invalidated lock")[0], - ( - account_id.to_string(), - Revision::new(u64::MAX).expect("positive revision") - ) - ); - assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]); -} - -#[tokio::test] -async fn cooldown_startup_hydrates_only_active_postgres_cooldowns() { - let now = SystemTime::now(); - let calls = Arc::new(Mutex::new(Vec::new())); - let authoritative = Arc::new(CooldownTestAccountStore { - accounts: vec![ - cooldown_test_account("acct_hydrate_active", now + StdDuration::from_secs(60)), - cooldown_test_account("acct_hydrate_expired", now - StdDuration::from_secs(1)), - ready_test_account("acct_hydrate_ready"), - ], - calls: Arc::clone(&calls), - fail_state_write: false, - }); - let cache = Arc::new(CooldownTestCache { - calls, - cached: Mutex::new(Vec::new()), - invalidated: Mutex::new(Vec::new()), - fail: false, - }); - let store = CooldownCachingProviderAccountStore::new( - authoritative, - Arc::clone(&cache) as Arc, - ); - - assert_eq!(store.hydrate(now).await, 1); - assert_eq!( - cache.cached.lock().expect("cached lock")[0].provider_account_id, - "acct_hydrate_active" - ); -} - -fn cooldown_state_change(seconds: u64) -> AccountStateChange { - AccountStateChange { - account_id: ProviderAccountId::new("acct_cooldown_composition").expect("valid account ID"), - expected_revision: CredentialRevision::new(7).expect("positive revision"), - availability: AccountAvailability::Cooldown, - reason: Some("rate_limited".to_owned()), - cooldown_until: Some(SystemTime::now() + StdDuration::from_secs(seconds)), - observed_at: SystemTime::now(), - } -} - -fn credential_update() -> CredentialCasUpdate { - let account_id = ProviderAccountId::new("acct_cooldown_rotation").expect("valid account ID"); - CredentialCasUpdate::new( - account_id.clone(), - CredentialRevision::new(7).expect("positive revision"), - ProviderAccountUpdate { - account_id, - name: "Rotated cooldown account".to_owned(), - email: None, - plan_type: None, - }, - PlaintextCredential::new(serde_json::Map::new()), - false, - Some(SystemTime::now() + StdDuration::from_secs(3_600)), - None, - ) - .expect("valid credential update") -} - -fn cooldown_test_account(account_id: &str, cooldown_until: SystemTime) -> ProviderAccount { - test_provider_account(account_id).with_runtime_state( - true, - AccountAvailability::Cooldown, - Some(cooldown_until), - ) -} - -fn ready_test_account(account_id: &str) -> ProviderAccount { - test_provider_account(account_id).with_runtime_state(true, AccountAvailability::Ready, None) -} - -fn test_provider_account(account_id: &str) -> ProviderAccount { - ProviderAccount::new( - ProviderAccountId::new(account_id).expect("valid account ID"), - ProviderKind::new("openai").expect("valid provider kind"), - "Cooldown test".to_owned(), - "upstream-user".to_owned(), - "oauth".to_owned(), - CredentialRevision::new(7).expect("positive revision"), - Some(SystemTime::now() + StdDuration::from_secs(3_600)), - ) -} - #[tokio::test] async fn credential_cooldown_round_trips_without_raw_account_id_in_key() { let Some((repository, mut connection, namespace)) = repository().await else { diff --git a/backend/crates/providers/openai/src/admin.rs b/backend/crates/providers/openai/src/admin.rs index 40dc154f9..366bbe08f 100644 --- a/backend/crates/providers/openai/src/admin.rs +++ b/backend/crates/providers/openai/src/admin.rs @@ -579,8 +579,6 @@ fn prepared_create( next_refresh_at: account.next_refresh_at().map(DateTime::::from), enabled: account.enabled(), availability, - availability_reason: None, - cooldown_until: account.cooldown_until().map(DateTime::::from), availability_observed_at: observed_at, }) } @@ -682,11 +680,7 @@ fn account_from_record(account: &AccountRecord) -> Result, account: &ProviderAccount, ) -> ProviderQuota { - let mut quota = snapshot.map_or_else(empty_quota, project_quota_snapshot); + // 展示前先滚动已过期窗口，避免 reset 后的旧窗口值悬挂。 + let mut quota = snapshot + .map(|snapshot| project_quota_snapshot(snapshot.roll_expired_windows(SystemTime::now()))) + .unwrap_or_else(empty_quota); if account.availability() == AccountAvailability::QuotaExhausted { - force_confirmed_exhaustion_projection(&mut quota, account.cooldown_until()); + force_confirmed_exhaustion_projection(&mut quota); } quota } @@ -790,6 +787,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota key: window.key().to_owned(), group: quota_group(window.kind()).to_owned(), label: codex_quota_window_label( + window.limit_name(), window.kind(), window.source(), window.window_seconds(), @@ -798,6 +796,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota window_seconds: window.window_seconds(), used_percent: window.used_percent(), reset_at: window.reset_at(), + limit_reached: window.limit_reached(), local_usage: None, provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(data))), } @@ -813,12 +812,9 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota } } -fn force_confirmed_exhaustion_projection( - quota: &mut ProviderQuota, - cooldown_until: Option, -) { - // 真实 429 对“当前已耗尽”的证明强于可能延迟结算的 usage 文档。这里只覆盖 - // Admin 展示投影，原始 Provider JSON 仍作为后续判断窗口是否真正恢复的基线。 +fn force_confirmed_exhaustion_projection(quota: &mut ProviderQuota) { + // 402 已确认耗尽：只覆盖 Admin 展示投影，原始 Provider JSON 仍作为 + // 后续判断窗口是否真正恢复的基线。 quota.provider_data = Some(ProviderDocument::new(OpaqueProviderData::new( Map::from_iter([ ( @@ -828,7 +824,6 @@ fn force_confirmed_exhaustion_projection( ("exhausted".to_owned(), Value::Bool(true)), ]), ))); - let reset_at = cooldown_until.map(DateTime::::from); if quota.windows.is_empty() { quota.windows.push(ProviderQuotaWindow { key: "confirmed-quota-exhaustion".to_owned(), @@ -837,42 +832,28 @@ fn force_confirmed_exhaustion_projection( source: None, window_seconds: None, used_percent: Some(100.0), - reset_at, + reset_at: None, + limit_reached: true, local_usage: None, provider_data: None, }); return; } - let matched_reset = reset_at.and_then(|reset_at| { - quota - .windows - .iter() - .enumerate() - .filter_map(|(index, window)| { - let distance = window.reset_at.map(|window_reset| { - (window_reset.timestamp() - reset_at.timestamp()).unsigned_abs() - })?; - Some((index, distance)) - }) - .min_by_key(|(_, distance)| *distance) - .map(|(index, _)| index) - }); - let index = matched_reset - .or_else(|| { - quota - .windows - .iter() - .enumerate() - .max_by(|(_, left), (_, right)| { - left.used_percent - .unwrap_or(-1.0) - .total_cmp(&right.used_percent.unwrap_or(-1.0)) - }) - .map(|(index, _)| index) + // 选已用比例最高的窗口强制 100%，保留其自身 reset_at。 + let index = quota + .windows + .iter() + .enumerate() + .max_by(|(_, left), (_, right)| { + left.used_percent + .unwrap_or(-1.0) + .total_cmp(&right.used_percent.unwrap_or(-1.0)) }) + .map(|(index, _)| index) .unwrap_or(0); quota.windows[index].used_percent = Some(100.0); + quota.windows[index].limit_reached = true; } const fn quota_group(kind: CodexQuotaWindowKind) -> &'static str { @@ -891,11 +872,17 @@ const fn quota_role(role: CodexQuotaWindowRole) -> &'static str { } } +/// 窗口展示名，优先使用上游 `limit_name`（原文，`_` 转空格）； +/// 无 limit_name 时按窗口时长猜（±5% 容差，汉化输出），猜不中兜底「额度」。 fn codex_quota_window_label( + limit_name: Option<&str>, kind: CodexQuotaWindowKind, source: &str, window_seconds: Option, ) -> String { + if let Some(limit_name) = limit_name { + return limit_name.replace('_', " "); + } let base = match kind { CodexQuotaWindowKind::Monthly => "月限额".to_owned(), CodexQuotaWindowKind::Weekly => "周限额".to_owned(), @@ -908,12 +895,9 @@ fn codex_quota_window_label( } CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds), }; - if matches!(source, "core" | "codex" | "spend_control" | "monthly_limit") { + if matches!(source, "core" | "codex" | "code_review" | "spend_control") { return base; } - if is_codex_review_limit(source) { - return format!("代码审查 · {base}"); - } format!("{source} · {base}") } @@ -930,15 +914,6 @@ fn custom_quota_window_label(window_seconds: Option) -> String { } } -fn is_codex_review_limit(value: &str) -> bool { - let normalized = value.trim().to_ascii_lowercase().replace(['-', ' '], "_"); - matches!( - normalized.as_str(), - "review" | "code_review" | "codex_review" | "codex_code_review" - ) || normalized.contains("code_review") - || normalized.contains("codex_review") -} - /// 将 Provider-owned PKCE/OIDC 状态保存到 Store 提供的 Redis 原子端口。 pub(crate) struct OpenAiOAuthPendingStore { port: Arc, diff --git a/backend/crates/providers/openai/src/credential/admin.rs b/backend/crates/providers/openai/src/credential/admin.rs index d5fff00eb..8775f8a62 100644 --- a/backend/crates/providers/openai/src/credential/admin.rs +++ b/backend/crates/providers/openai/src/credential/admin.rs @@ -409,7 +409,7 @@ impl CodexCredentialAdmin { Some(input.verified_account.chatgpt_account_id), input.verified_account.plan_type, ) - .with_runtime_state(input.enabled, AccountAvailability::Ready, None) + .with_runtime_state(input.enabled, AccountAvailability::Ready) .with_refresh_schedule( input.secret.refresh_token.is_some(), optional_time(input.next_refresh_at), @@ -806,10 +806,7 @@ impl CodexCredentialAdminService { )) { return Err(CodexCredentialAdminError::InvalidInput); } - account.account = - account - .account - .with_runtime_state(enabled, availability, None); + account.account = account.account.with_runtime_state(enabled, availability); account } }; @@ -933,7 +930,7 @@ impl CodexCredentialAdminService { Some(upstream_account_id), candidate.plan_type.clone(), ) - .with_runtime_state(enabled, AccountAvailability::Ready, None) + .with_runtime_state(enabled, AccountAvailability::Ready) .with_refresh_schedule(false, None); Ok(NewProviderAccount { account, @@ -1000,9 +997,7 @@ fn cpr_status(account: &ProviderAccount) -> &'static str { AccountAvailability::QuotaExhausted => "quota_exhausted", AccountAvailability::Expired | AccountAvailability::Invalid => "expired", AccountAvailability::Banned => "banned", - AccountAvailability::Unknown - | AccountAvailability::Ready - | AccountAvailability::Cooldown => "active", + AccountAvailability::Unknown | AccountAvailability::Ready => "active", } } diff --git a/backend/crates/providers/openai/src/credential/catalog.rs b/backend/crates/providers/openai/src/credential/catalog.rs index 1507e3c22..5dd4825fc 100644 --- a/backend/crates/providers/openai/src/credential/catalog.rs +++ b/backend/crates/providers/openai/src/credential/catalog.rs @@ -783,7 +783,6 @@ fn eligible_catalog_account(account: &ProviderAccount, now: SystemTime) -> bool && match account.availability() { AccountAvailability::Unknown | AccountAvailability::Ready - | AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted => true, AccountAvailability::Expired | AccountAvailability::Banned diff --git a/backend/crates/providers/openai/src/credential/quota.rs b/backend/crates/providers/openai/src/credential/quota.rs index 9c5807b3b..9d6a10766 100644 --- a/backend/crates/providers/openai/src/credential/quota.rs +++ b/backend/crates/providers/openai/src/credential/quota.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, RwLock}; -use std::time::{Duration, Instant, SystemTime}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ @@ -14,7 +14,7 @@ use gateway_protocol::openai::events::{ }; use reqwest::{Client, StatusCode}; use secrecy::ExposeSecret; -use serde_json::{Map, Value}; +use serde_json::{Map, Number, Value}; use thiserror::Error; use tokio::sync::Mutex; use uuid::Uuid; @@ -26,11 +26,10 @@ use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeC use super::repository::{CodexCredentialRepository, CredentialRepositoryError}; const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60); -const MAX_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60 * 60); const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60); const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5); const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60); -// 5xx 上游拒绝的短退避重试预算；指数退避 1s/2s，吞掉瞬时抖动（对齐 TS withRetry）。 +// 5xx 上游拒绝的短退避重试预算；指数退避 1s/2s，吞掉瞬时抖动。 const QUOTA_FETCH_5XX_MAX_RETRIES: u32 = 2; const QUOTA_FETCH_5XX_BASE_DELAY: Duration = Duration::from_secs(1); @@ -82,11 +81,14 @@ pub enum CodexQuotaWindowRole { pub struct CodexQuotaWindow { key: String, source: String, + /// 上游提供的限流名称（`metered_feature`/`limit_name`/`limitId`）；显示优先用它。 + limit_name: Option, kind: CodexQuotaWindowKind, role: CodexQuotaWindowRole, window_seconds: Option, used_percent: Option, reset_at: Option>, + limit_reached: bool, } impl CodexQuotaWindow { @@ -100,6 +102,11 @@ impl CodexQuotaWindow { &self.source } + #[must_use] + pub fn limit_name(&self) -> Option<&str> { + self.limit_name.as_deref() + } + #[must_use] pub const fn kind(&self) -> CodexQuotaWindowKind { self.kind @@ -124,6 +131,12 @@ impl CodexQuotaWindow { pub const fn reset_at(&self) -> Option> { self.reset_at } + + /// 该窗口是否已触顶（`limit_reached` 或 `used_percent >= 100`）。 + #[must_use] + pub const fn limit_reached(&self) -> bool { + self.limit_reached + } } #[derive(Debug, Clone, PartialEq)] @@ -160,6 +173,37 @@ impl CodexAccountQuotaSnapshot { pub fn windows(&self) -> &[CodexQuotaWindow] { &self.windows } + + /// 投影滚动已过期窗口：`reset_at` 已过的窗口 `used_percent=0`、 + /// `limit_reached=false`、`reset_at` 滚到下一个周期（reset 到期自动归零）。 + /// 只作用于展示/调度投影，不写回持久化 raw JSON。 + #[must_use] + pub fn roll_expired_windows(mut self, now: SystemTime) -> Self { + for window in &mut self.windows { + let Some(reset_at) = window.reset_at else { + continue; + }; + let reset = SystemTime::from(reset_at); + if reset > now { + continue; + } + window.used_percent = Some(0.0); + window.limit_reached = false; + if let Some(window_seconds) = window.window_seconds.filter(|seconds| *seconds > 0) { + let mut next = reset; + while next <= now { + let Some(after) = next.checked_add(Duration::from_secs(window_seconds)) else { + break; + }; + next = after; + } + window.reset_at = Some(DateTime::::from(next)); + } else { + window.reset_at = None; + } + } + self + } } impl CodexQuotaFact { @@ -179,20 +223,11 @@ impl CodexQuotaFact { } } -fn quota_fact_is_active_exhaustion(fact: CodexQuotaFact, now: SystemTime) -> bool { - fact.exhausted() - && fact - .resets_at() - .map(SystemTime::from) - .is_none_or(|reset_at| reset_at > now) -} - #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct CodexQuotaSyncSummary { pub updated: u64, pub exhausted: u64, pub banned: u64, - pub cooldown: u64, pub transient: u64, pub stale: u64, } @@ -292,38 +327,6 @@ enum CodexQuotaFetchError { }, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CodexQuotaStateTransition { - Banned { reason: &'static str }, - Exhausted, - ResettableLimit { until: Option }, -} - -impl CodexQuotaStateTransition { - const fn availability(self) -> AccountAvailability { - match self { - Self::Banned { .. } => AccountAvailability::Banned, - Self::Exhausted => AccountAvailability::QuotaExhausted, - Self::ResettableLimit { .. } => AccountAvailability::QuotaExhausted, - } - } - - const fn reason(self) -> &'static str { - match self { - Self::Banned { reason } => reason, - Self::Exhausted => "quota_exhausted", - Self::ResettableLimit { .. } => "quota_rate_limited", - } - } - - const fn cooldown_until(self) -> Option { - match self { - Self::ResettableLimit { until } => until, - Self::Banned { .. } | Self::Exhausted => None, - } - } -} - impl CodexQuotaSchedulingProjection { fn hydration_targets(&self, accounts: &[ProviderAccount]) -> Vec { let state = self @@ -347,21 +350,15 @@ impl CodexQuotaSchedulingProjection { .collect() } - fn observe( - &self, - account_id: ProviderAccountId, - revision: CredentialRevision, - observed_at: SystemTime, - fact: CodexQuotaFact, - ) -> bool { - let Some(remaining_ttl) = quota_projection_ttl(observed_at) else { + fn observe(&self, snapshot: &CodexAccountQuotaSnapshot) -> bool { + let Some(remaining_ttl) = quota_projection_ttl(snapshot.observed_at()) else { return false; }; self.replace( - account_id, - revision, + snapshot.account_id().clone(), + snapshot.credential_revision(), remaining_ttl, - quota_scheduling_signals(fact), + scheduling_signals_from_snapshot(snapshot), ); true } @@ -413,7 +410,7 @@ impl CodexQuotaSchedulingProjection { snapshot.account_id().clone(), snapshot.credential_revision(), remaining_ttl, - quota_scheduling_signals(snapshot.fact()), + scheduling_signals_from_snapshot(snapshot), ); true } @@ -453,7 +450,12 @@ impl CodexQuotaSchedulingProjection { ) -> Vec { let candidates = accounts .into_iter() - .filter_map(|account| quota_refresh_candidate(account, now)) + .filter_map(|account| { + let limit_reached = self + .signals(&account) + .is_some_and(|signals| signals.limit_reached()); + quota_refresh_candidate(account, now, limit_reached) + }) .collect::>(); let candidate_ids = candidates .iter() @@ -484,36 +486,16 @@ impl CodexQuotaSchedulingProjection { } } -fn quota_refresh_candidate(account: ProviderAccount, now: SystemTime) -> Option { - (eligible_periodic_quota_refresh(&account, now) - && account.availability() == AccountAvailability::QuotaExhausted) - .then_some(account) -} - -fn quota_snapshot_for_account( - account: &ProviderAccount, - observation: Option<&QuotaObservation>, -) -> Option { - observation - .filter(|observation| observation.expected_revision == account.revision()) - .and_then(quota_snapshot_from_observation) -} - -fn quota_reset_at_from_snapshot( - snapshot: Option<&CodexAccountQuotaSnapshot>, -) -> Option { - snapshot - .and_then(|snapshot| snapshot.fact().resets_at()) - .map(SystemTime::from) -} - -fn quota_observation_confirms_active_exhaustion( - account: &ProviderAccount, - observation: Option<&QuotaObservation>, +fn quota_refresh_candidate( + account: ProviderAccount, now: SystemTime, -) -> bool { - quota_snapshot_for_account(account, observation) - .is_some_and(|snapshot| quota_fact_is_active_exhaustion(snapshot.fact(), now)) + limit_reached: bool, +) -> Option { + // 定时器只兜底「已耗尽（402）」或「任一窗口触顶（limit_reached 死锁）」账号， + // 正常账号由真实请求的响应头被动同步。 + (eligible_periodic_quota_refresh(&account, now) + && (account.availability() == AccountAvailability::QuotaExhausted || limit_reached)) + .then_some(account) } fn periodic_quota_refresh_due( @@ -612,28 +594,9 @@ impl CodexCredentialQuotaService { } pub async fn synchronize(&self) -> Result { - let mut accounts = self.repository.list_for_provider().await?; + let accounts = self.repository.list_for_provider().await?; let mut summary = CodexQuotaSyncSummary::default(); let now = SystemTime::now(); - let account_ids = accounts - .iter() - .map(|account| account.id().clone()) - .collect::>(); - let observations = match self.store.get_quotas(&account_ids).await { - Ok(observations) => Some( - observations - .into_iter() - .map(|observation| (observation.account_id.clone(), observation)) - .collect::>(), - ), - Err(error) => { - // 持久观察不可用时跳过旧状态修正；仍复核当前已标记耗尽的账号。 - tracing::warn!(error = %error, "OpenAI quota refresh skipped persisted exhaustion reconciliation"); - None - } - }; - self.reconcile_persisted_exhaustion(&mut accounts, observations.as_ref(), now) - .await; let accounts = self.scheduling.reserve_periodic_refreshes(accounts, now); if accounts.is_empty() { return Ok(summary); @@ -671,102 +634,42 @@ impl CodexCredentialQuotaService { ); } Err(CodexQuotaFetchError::Upstream { account, error }) => { - let Some(transition) = quota_state_transition(&error, observed_at) else { - summary.transient += 1; - tracing::warn!( - account_id = %account.id(), - error = %error, - "OpenAI quota upstream rejection was not classified" - ); - continue; - }; - match transition { - CodexQuotaStateTransition::Banned { .. } => summary.banned += 1, - CodexQuotaStateTransition::Exhausted - | CodexQuotaStateTransition::ResettableLimit { .. } => { + match quota_state_transition(&error) { + // 402：真额度耗尽，写入 QuotaExhausted（worker 按 reset_at 自动恢复）。 + Some(AccountAvailability::QuotaExhausted) => { summary.exhausted += 1; + let _ = self + .repository + .apply_state( + &account, + AccountAvailability::QuotaExhausted, + observed_at, + ) + .await; + } + Some(AccountAvailability::Banned) => { + summary.banned += 1; + let _ = self + .repository + .apply_state(&account, AccountAvailability::Banned, observed_at) + .await; + } + // 429/503 等临时拒绝：不写 availability（限流不改变账号可用性），仅记失败待重试。 + None | Some(_) => { + summary.transient += 1; + tracing::warn!( + account_id = %account.id(), + error = %error, + "OpenAI quota upstream rejection; refresh cycle will retry later" + ); } } - let _ = self - .repository - .apply_state( - &account, - transition.availability(), - Some(transition.reason().to_owned()), - transition.cooldown_until(), - observed_at, - ) - .await; } } } Ok(summary) } - async fn reconcile_persisted_exhaustion( - &self, - accounts: &mut [ProviderAccount], - observations: Option<&BTreeMap>, - observed_at: SystemTime, - ) { - let Some(observations) = observations else { - return; - }; - for account in accounts { - if account.availability() != AccountAvailability::Cooldown - || !quota_observation_confirms_active_exhaustion( - account, - observations.get(account.id()), - observed_at, - ) - { - continue; - } - let Ok(Some(current)) = self.store.get_account(account.id()).await else { - continue; - }; - if current.revision() != account.revision() - || current.availability() != AccountAvailability::Cooldown - { - continue; - } - let snapshot = match self.read_snapshot_for(¤t).await { - Ok(Some(snapshot)) - if quota_fact_is_active_exhaustion(snapshot.fact(), observed_at) => - { - snapshot - } - _ => continue, - }; - let cooldown_until = - quota_reset_at_from_snapshot(Some(&snapshot)).or_else(|| current.cooldown_until()); - if let Err(error) = self - .repository - .apply_state( - ¤t, - AccountAvailability::QuotaExhausted, - Some("quota_exhausted".to_owned()), - cooldown_until, - observed_at, - ) - .await - { - tracing::warn!( - account_id = %account.id(), - error = %error, - "OpenAI persisted quota exhaustion reconciliation failed" - ); - continue; - } - let enabled = current.enabled(); - *account = current.with_runtime_state( - enabled, - AccountAvailability::QuotaExhausted, - cooldown_until, - ); - } - } - /// 解析并 revision-fenced 落库单账号的 Provider quota JSON。 async fn apply_fetched_quota( &self, @@ -807,29 +710,21 @@ impl CodexCredentialQuotaService { summary.stale += 1; return Ok(()); } - self.scheduling - .observe(current.id().clone(), current.revision(), observed_at, fact); + self.scheduling.observe(&snapshot); if fact.exhausted() { summary.exhausted += 1; } else { summary.updated += 1; } - if let Some((availability, cooldown_until)) = quota_success_state( + if let Some(availability) = quota_success_state( current.availability(), - current.cooldown_until(), fact, SystemTime::now(), QuotaRecoveryEvidence::RefreshedSnapshot, ) { let _ = self .repository - .apply_state( - ¤t, - availability, - fact.exhausted().then_some("quota_exhausted".to_owned()), - cooldown_until, - observed_at, - ) + .apply_state(¤t, availability, observed_at) .await; } Ok(()) @@ -857,8 +752,14 @@ impl CodexCredentialQuotaService { .map(OpaqueProviderData::into_inner) .unwrap_or_default(); let quota = merge_passive_quota(existing, &rate_limits); - let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?; let observed_at = SystemTime::now(); + let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?; + let snapshot = parse_account_quota_snapshot( + account.id().clone(), + account.revision(), + observed_at, + &Value::Object(quota.clone()), + ); let outcome = self .store .compare_and_swap_quota(QuotaObservation { @@ -879,28 +780,107 @@ impl CodexCredentialQuotaService { if current.revision() != account.revision() { return Ok(false); } - self.scheduling - .observe(current.id().clone(), current.revision(), observed_at, fact); - if let Some((availability, cooldown_until)) = quota_success_state( + if let Ok(snapshot) = snapshot { + self.scheduling.observe(&snapshot); + } + if let Some(availability) = quota_success_state( current.availability(), - current.cooldown_until(), fact, observed_at, QuotaRecoveryEvidence::SuccessfulResponse, ) { self.repository - .apply_state( - ¤t, - availability, - fact.exhausted().then_some("quota_exhausted".to_owned()), - cooldown_until, - observed_at, - ) + .apply_state(¤t, availability, observed_at) .await?; } Ok(true) } + /// 真实 429 的单一事实入口：合成/更新 quota 主窗口（`limit_reached=true`、 + /// `used_percent=100`、`reset_at=retry-after`）。availability 不改变， + /// 窗口 reset 到期后由投影自动滚动恢复（429 只记录限流窗口）。 + /// 已有更晚的 `reset_at` 不会被缩短（never-shrink）。 + pub async fn apply_rate_limit_429( + &self, + account: &ProviderAccount, + retry_after: Option, + observed_at: SystemTime, + ) -> Result<(), CodexCredentialQuotaError> { + let mut object = self + .store + .get_quotas(std::slice::from_ref(account.id())) + .await? + .into_iter() + .find(|observation| { + observation.account_id == *account.id() + && observation.expected_revision == account.revision() + }) + .and_then(|observation| observation.quota) + .map(OpaqueProviderData::into_inner) + .unwrap_or_default(); + let new_reset_at = observed_at + .checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)) + .and_then(|reset| reset.duration_since(UNIX_EPOCH).ok()) + .map(|duration| duration.as_secs() as i64); + let existing_reset_at = object + .get("rate_limit") + .and_then(Value::as_object) + .and_then(|rate_limit| rate_limit.get("primary_window")) + .and_then(Value::as_object) + .and_then(|window| window.get("reset_at")) + .and_then(Value::as_i64); + let reset_at = match (existing_reset_at, new_reset_at) { + (Some(existing), Some(next)) if existing > next => Some(existing), + (_, next) => next, + }; + let mut rate_limit = object + .get("rate_limit") + .cloned() + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + rate_limit.insert("allowed".to_owned(), Value::Bool(false)); + rate_limit.insert("limit_reached".to_owned(), Value::Bool(true)); + let mut primary = rate_limit + .get("primary_window") + .cloned() + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + let used_percent = primary + .get("used_percent") + .and_then(Value::as_f64) + .unwrap_or(0.0) + .max(100.0); + primary.insert( + "used_percent".to_owned(), + Number::from_f64(used_percent) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(100))), + ); + if let Some(reset_at) = reset_at { + primary.insert("reset_at".to_owned(), Value::from(reset_at)); + } + rate_limit.insert("primary_window".to_owned(), Value::Object(primary)); + object.insert("rate_limit".to_owned(), Value::Object(rate_limit)); + if self + .store + .compare_and_swap_quota(QuotaObservation { + account_id: account.id().clone(), + expected_revision: account.revision(), + quota: Some(OpaqueProviderData::new(object)), + observed_at: Some(observed_at), + }) + .await? + == QuotaWriteOutcome::Conflict + { + return Ok(()); + } + // 429 合成的限流窗口要立即反映到调度信号（reset 到期由投影滚动解除）。 + if let Ok(Some(snapshot)) = self.read_snapshot_for(account).await { + self.scheduling.observe(&snapshot); + } + Ok(()) + } + /// 读取单账号最后一次落库的 Provider quota，并由 Codex 域解析展示窗口。 pub async fn read_account( &self, @@ -949,12 +929,7 @@ impl CodexCredentialQuotaService { observed_at, &Value::Object(data.expose_to_provider().clone()), )?; - self.scheduling.observe( - snapshot.account_id().clone(), - snapshot.credential_revision(), - snapshot.observed_at(), - snapshot.fact(), - ); + self.scheduling.observe(&snapshot); Ok(Some(snapshot)) } @@ -1005,16 +980,11 @@ impl CodexCredentialQuotaService { }); } Err(CodexQuotaFetchError::Upstream { account, error }) => { - if let Some(transition) = quota_state_transition(&error, observed_at) { + // 402/deactivated → 写终态；429/503 等临时拒绝不写 availability（限流不改变账号可用性）。 + if let Some(availability) = quota_state_transition(&error) { let _ = self .repository - .apply_state( - &account, - transition.availability(), - Some(transition.reason().to_owned()), - transition.cooldown_until(), - observed_at, - ) + .apply_state(&account, availability, observed_at) .await; } return Err(CodexCredentialQuotaError::Upstream { @@ -1053,30 +1023,15 @@ impl CodexCredentialQuotaService { if current.revision() != account.revision() { return Err(CodexCredentialQuotaError::RevisionConflict); } - self.scheduling.observe( - snapshot.account_id().clone(), - snapshot.credential_revision(), - snapshot.observed_at(), - snapshot.fact(), - ); - if let Some((availability, cooldown_until)) = quota_success_state( + self.scheduling.observe(&snapshot); + if let Some(availability) = quota_success_state( current.availability(), - current.cooldown_until(), snapshot.fact(), SystemTime::now(), QuotaRecoveryEvidence::RefreshedSnapshot, ) { self.repository - .apply_state( - ¤t, - availability, - snapshot - .fact() - .exhausted() - .then_some("quota_exhausted".to_owned()), - cooldown_until, - observed_at, - ) + .apply_state(¤t, availability, observed_at) .await?; } Ok(snapshot) @@ -1315,91 +1270,56 @@ fn passive_rate_limit_window(window: RateLimitWindow) -> Map { snapshot } +/// 402 恢复判定：新观测未耗尽时返回 `Ready`；仍耗尽或未到恢复时机返回 `None`。 +/// +/// 只处理 `QuotaExhausted` 的恢复——429/触顶不再进入该状态，由 quota 数据 +/// 驱动调度排除（限流不改变账号可用性）。`Invalid/Expired/Banned` 等终态由 selector 的 +/// 成功响应恢复路径处理。 fn quota_success_state( current: AccountAvailability, - cooldown_until: Option, fact: CodexQuotaFact, now: SystemTime, recovery_evidence: QuotaRecoveryEvidence, -) -> Option<(AccountAvailability, Option)> { - if fact.exhausted() { - let observed_cooldown = match (fact.resets_at().map(SystemTime::from), cooldown_until) { - (Some(observed), Some(confirmed)) => Some(observed.max(confirmed)), - (observed, confirmed) => observed.or(confirmed), - }; - return (current != AccountAvailability::QuotaExhausted - || cooldown_until != observed_cooldown) - .then_some((AccountAvailability::QuotaExhausted, observed_cooldown)); +) -> Option { + if current != AccountAvailability::QuotaExhausted || fact.exhausted() { + return None; } - - let availability = match current { - AccountAvailability::Invalid - | AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Unknown => Some(AccountAvailability::Ready), - AccountAvailability::QuotaExhausted - if !quota_recovery_confirmed(recovery_evidence, cooldown_until, now) => - { - None - } - AccountAvailability::QuotaExhausted => Some(AccountAvailability::Ready), - AccountAvailability::Ready => None, - AccountAvailability::Cooldown - if cooldown_until.is_some_and(|cooldown_until| cooldown_until <= now) => - { - Some(AccountAvailability::Ready) - } - AccountAvailability::Cooldown => None, - }; - availability.map(|availability| (availability, None)) + quota_recovery_confirmed( + recovery_evidence, + fact.resets_at().map(SystemTime::from), + now, + ) + .then_some(AccountAvailability::Ready) } fn quota_recovery_confirmed( evidence: QuotaRecoveryEvidence, - cooldown_until: Option, + reset_at: Option, now: SystemTime, ) -> bool { - if cooldown_until.is_some_and(|cooldown_until| cooldown_until <= now) { + if reset_at.is_some_and(|reset_at| reset_at <= now) { return true; } match evidence { + // 一次真实成功响应是恢复的最强证据。 QuotaRecoveryEvidence::SuccessfulResponse => true, - // used_percent 是可滞后、会取整的观测值；确认 429 后的 100% 也是 - // 管理端投影。它们的回落不能覆盖仍未到期的上游 reset 时间。 + // used_percent 是可滞后、会取整的观测值；其回落不能覆盖仍未到期的上游 reset 时间。 QuotaRecoveryEvidence::RefreshedSnapshot => false, } } -fn quota_state_transition( - error: &CodexClientError, - observed_at: SystemTime, -) -> Option { - let CodexClientError::Upstream { - status, - retry_after_seconds, - body, - .. - } = error - else { +fn quota_state_transition(error: &CodexClientError) -> Option { + let CodexClientError::Upstream { status, body, .. } = error else { return None; }; if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) { - return Some(CodexQuotaStateTransition::Banned { - reason: "quota_deactivated_workspace", - }); + return Some(AccountAvailability::Banned); } match *status { - StatusCode::PAYMENT_REQUIRED => Some(CodexQuotaStateTransition::Exhausted), - // 429 与 503 都是"稍后重试"语义：优先尊重 Retry-After，缺失时用默认冷却。 - StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE => { - let duration = Duration::from_secs( - retry_after_seconds.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN.as_secs()), - ) - .min(MAX_RATE_LIMIT_COOLDOWN); - Some(CodexQuotaStateTransition::ResettableLimit { - until: observed_at.checked_add(duration), - }) - } + // 402：真额度耗尽 → QuotaExhausted（worker 按 reset_at 自动恢复）。 + StatusCode::PAYMENT_REQUIRED => Some(AccountAvailability::QuotaExhausted), + // 429/503：临时限流，不写 availability（限流不改变账号可用性），由 quota 数据驱动。 + StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE => None, // quota/usage 接口只能产生额度域状态；鉴权拒绝不具备判定 RT // 永久失效的证据，终态只能由 OAuth refresh/身份校验状态机写入。 _ => None, @@ -1420,22 +1340,13 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result Option { .filter(|remaining| !remaining.is_zero()) } -fn quota_scheduling_signals(fact: CodexQuotaFact) -> Option { - let remaining_rank = fact.remaining_percent().map(|value| u64::from(value) * 100); +/// 从快照的滚动窗口派生中立调度事实（`limit_reached` = 任一窗口触顶）。 +fn scheduling_signals_from_snapshot( + snapshot: &CodexAccountQuotaSnapshot, +) -> Option { + let rolled = snapshot.clone().roll_expired_windows(SystemTime::now()); + quota_scheduling_signals(&rolled) +} + +fn quota_scheduling_signals(snapshot: &CodexAccountQuotaSnapshot) -> Option { let now = SystemTime::now(); - let reset_at = fact - .resets_at() - .map(SystemTime::from) - .filter(|reset_at| *reset_at > now); + let mut limit_reached = false; + let mut remaining_rank: Option = None; + let mut reset_at: Option = None; + for window in &snapshot.windows { + limit_reached |= window.limit_reached(); + if let Some(used) = window + .used_percent() + .filter(|used| used.is_finite() && (0.0..=100.0).contains(used)) + { + let rank = ((100.0 - used) * 100.0).round() as u64; + remaining_rank = Some(remaining_rank.map_or(rank, |current| current.min(rank))); + } + if let Some(reset) = window + .reset_at() + .map(SystemTime::from) + .filter(|reset| *reset > now) + { + reset_at = Some(reset_at.map_or(reset, |current| current.min(reset))); + } + } (remaining_rank.is_some() || reset_at.is_some()) - .then(|| AccountQuotaSignals::new(reset_at, remaining_rank)) + .then(|| AccountQuotaSignals::new(reset_at, remaining_rank, limit_reached)) } fn parse_account_quota_snapshot( @@ -1495,14 +1429,13 @@ fn parse_account_quota_snapshot( .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; let mut windows = Vec::new(); - for (source, rate_limit) in canonical_rate_limits(object)? { - parse_rate_limit_windows(&source, rate_limit, &mut windows)?; - } - if let Some(spend_control) = object.get("spend_control").filter(|value| !value.is_null()) { - parse_spend_control_window(spend_control, &mut windows)?; - } - if let Some(monthly_limit) = object.get("monthly_limit").filter(|value| !value.is_null()) { - parse_monthly_limit_window(monthly_limit, &mut windows)?; + for limit in canonical_rate_limits(object)? { + parse_rate_limit_windows( + &limit.source, + limit.limit_name.as_deref(), + limit.rate_limit, + &mut windows, + )?; } windows.sort_by(|left, right| left.key.cmp(&right.key)); Ok(CodexAccountQuotaSnapshot { @@ -1514,12 +1447,33 @@ fn parse_account_quota_snapshot( }) } +/// 一个规范化后的限流桶：source 是调度/展示分组键，limit_name 是上游显示名。 +struct CanonicalRateLimit<'a> { + source: String, + limit_name: Option, + rate_limit: &'a Value, +} + fn canonical_rate_limits( object: &Map, -) -> Result, CodexCredentialQuotaError> { - let mut limits = BTreeMap::new(); +) -> Result>, CodexCredentialQuotaError> { + let mut limits = Vec::new(); if let Some(rate_limit) = object.get("rate_limit") { - limits.insert("core".to_owned(), rate_limit); + limits.push(CanonicalRateLimit { + source: "core".to_owned(), + limit_name: None, + rate_limit, + }); + } + if let Some(rate_limit) = object + .get("code_review_rate_limit") + .filter(|value| !value.is_null()) + { + limits.push(CanonicalRateLimit { + source: "code_review".to_owned(), + limit_name: Some("code_review".to_owned()), + rate_limit, + }); } let Some(additional) = object .get("additional_rate_limits") @@ -1536,15 +1490,28 @@ fn canonical_rate_limits( let item = value .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let source = canonical_rate_limit_source(item, index); + let (source, limit_name) = canonical_rate_limit_source(item, index); if let Some(rate_limit) = item.get("rate_limit") { - limits.insert(source, rate_limit); + // additional 里的 codex 桶覆盖顶层 rate_limit（同源替换）。 + if let Some(existing) = limits.iter_mut().find(|existing| existing.source == source) { + existing.rate_limit = rate_limit; + existing.limit_name = limit_name; + } else { + limits.push(CanonicalRateLimit { + source, + limit_name, + rate_limit, + }); + } } } Ok(limits) } -fn canonical_rate_limit_source(item: &Map, index: usize) -> String { +fn canonical_rate_limit_source( + item: &Map, + index: usize, +) -> (String, Option) { let source = item .get("metered_feature") .or_else(|| item.get("limit_name")) @@ -1554,15 +1521,24 @@ fn canonical_rate_limit_source(item: &Map, index: usize) -> Strin }) .map(str::to_owned) .unwrap_or_else(|| format!("additional-{index}")); + let limit_name = item + .get("limit_name") + .or_else(|| item.get("metered_feature")) + .and_then(Value::as_str) + .filter(|value| { + !value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control) + }) + .map(str::to_owned); if source.trim().to_ascii_lowercase().replace(['-', ' '], "_") == "codex" { - "core".to_owned() + ("core".to_owned(), limit_name) } else { - source + (source, limit_name) } } fn parse_rate_limit_windows( source: &str, + limit_name: Option<&str>, value: &Value, output: &mut Vec, ) -> Result<(), CodexCredentialQuotaError> { @@ -1606,134 +1582,27 @@ fn parse_rate_limit_windows( .ok_or(CodexCredentialQuotaError::InvalidCredentialData) }) .transpose()?; + let limit_reached = window + .get("limit_reached") + .and_then(Value::as_bool) + .unwrap_or(false) + || used_percent.is_some_and(|used| used >= 100.0); let kind = quota_window_kind(window_seconds); output.push(CodexQuotaWindow { key: format!("{}-{}", quota_key(source), quota_role_name(role, kind)), source: source.to_owned(), + limit_name: limit_name.map(str::to_owned), kind, role, window_seconds, used_percent, reset_at, + limit_reached, }); } Ok(()) } -fn parse_spend_control_window( - value: &Value, - output: &mut Vec, -) -> Result<(), CodexCredentialQuotaError> { - let object = value - .as_object() - .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let Some(individual) = object.get("individual_limit") else { - return Ok(()); - }; - if individual.is_null() { - return Ok(()); - } - let individual = individual - .as_object() - .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let reached = object - .get("reached") - .map(|value| { - value - .as_bool() - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) - }) - .transpose()? - .unwrap_or(false); - let used_percent = quota_percent(individual, "used_percent")?; - let remaining_percent = quota_percent(individual, "remaining_percent")?; - let used_percent = - used_percent.or_else(|| remaining_percent.map(|value| (100.0 - value).clamp(0.0, 100.0))); - let reset_at = quota_reset_at(individual)?; - let window_seconds = quota_window_seconds(individual)?.or(Some(2_592_000)); - output.push(CodexQuotaWindow { - key: "spend-control-monthly".to_owned(), - source: "spend_control".to_owned(), - kind: CodexQuotaWindowKind::Monthly, - role: CodexQuotaWindowRole::Monthly, - window_seconds, - used_percent: used_percent.map(|value| if reached { 100.0_f64.max(value) } else { value }), - reset_at, - }); - Ok(()) -} - -fn parse_monthly_limit_window( - value: &Value, - output: &mut Vec, -) -> Result<(), CodexCredentialQuotaError> { - let object = value - .as_object() - .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let used_percent = quota_percent(object, "used_percent")?; - let remaining_percent = quota_percent(object, "remaining_percent")?; - let used_percent = - used_percent.or_else(|| remaining_percent.map(|value| (100.0 - value).clamp(0.0, 100.0))); - output.push(CodexQuotaWindow { - key: "monthly-limit".to_owned(), - source: "monthly_limit".to_owned(), - kind: CodexQuotaWindowKind::Monthly, - role: CodexQuotaWindowRole::Monthly, - window_seconds: quota_window_seconds(object)?.or(Some(2_592_000)), - used_percent, - reset_at: quota_reset_at(object)?, - }); - Ok(()) -} - -fn quota_percent( - object: &Map, - key: &str, -) -> Result, CodexCredentialQuotaError> { - object - .get(key) - .map(|value| { - value - .as_f64() - .filter(|value| value.is_finite() && (0.0..=100.0).contains(value)) - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) - }) - .transpose() -} - -fn quota_reset_at( - object: &Map, -) -> Result>, CodexCredentialQuotaError> { - object - .get("reset_at") - .map(|value| { - value - .as_i64() - .filter(|value| *value > 0) - .and_then(|value| DateTime::::from_timestamp(value, 0)) - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) - }) - .transpose() -} - -fn quota_window_seconds( - object: &Map, -) -> Result, CodexCredentialQuotaError> { - if let Some(seconds) = optional_positive_u64(object, "window_seconds")? { - return Ok(Some(seconds)); - } - if let Some(seconds) = optional_positive_u64(object, "limit_window_seconds")? { - return Ok(Some(seconds)); - } - optional_positive_u64(object, "window_minutes")? - .map(|minutes| { - minutes - .checked_mul(60) - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) - }) - .transpose() -} - fn optional_positive_u64( object: &Map, key: &str, diff --git a/backend/crates/providers/openai/src/credential/refresh.rs b/backend/crates/providers/openai/src/credential/refresh.rs index 2c58eaff0..33b82079c 100644 --- a/backend/crates/providers/openai/src/credential/refresh.rs +++ b/backend/crates/providers/openai/src/credential/refresh.rs @@ -192,7 +192,6 @@ impl CodexCredentialRefreshService { self.persist_terminal( &due.account, AccountAvailability::Expired, - "invalid_grant", CodexCredentialRefreshOutcome::Invalidated { account_id }, ) .await @@ -201,7 +200,6 @@ impl CodexCredentialRefreshService { self.persist_terminal( &due.account, AccountAvailability::Banned, - "account_banned", CodexCredentialRefreshOutcome::Banned { account_id }, ) .await @@ -330,7 +328,6 @@ impl CodexCredentialRefreshService { .persist_terminal( &due.account, AccountAvailability::Invalid, - "refreshed_identity_rejected", CodexCredentialRefreshOutcome::Invalidated { account_id: due.account.id().to_string(), }, @@ -433,7 +430,6 @@ impl CodexCredentialRefreshService { &self, account: &ProviderAccount, availability: AccountAvailability, - reason: &'static str, outcome: CodexCredentialRefreshOutcome, ) -> Result { match self.repository.load_runtime_credential(account).await { @@ -447,13 +443,7 @@ impl CodexCredentialRefreshService { } match self .repository - .apply_state( - account, - availability, - Some(reason.to_owned()), - None, - SystemTime::now(), - ) + .apply_state(account, availability, SystemTime::now()) .await { Ok(()) => Ok(outcome), diff --git a/backend/crates/providers/openai/src/credential/repository.rs b/backend/crates/providers/openai/src/credential/repository.rs index f5e527820..9108f7251 100644 --- a/backend/crates/providers/openai/src/credential/repository.rs +++ b/backend/crates/providers/openai/src/credential/repository.rs @@ -233,8 +233,6 @@ impl CodexCredentialRepository { &self, account: &ProviderAccount, availability: AccountAvailability, - reason: Option, - cooldown_until: Option, observed_at: SystemTime, ) -> Result<(), CredentialRepositoryError> { if !account.enabled() { @@ -245,8 +243,6 @@ impl CodexCredentialRepository { account_id: account.id().clone(), expected_revision: account.revision(), availability, - reason, - cooldown_until, observed_at, }) .await?; diff --git a/backend/crates/providers/openai/src/credential/selector.rs b/backend/crates/providers/openai/src/credential/selector.rs index a81bfc824..e4d4a1446 100644 --- a/backend/crates/providers/openai/src/credential/selector.rs +++ b/backend/crates/providers/openai/src/credential/selector.rs @@ -32,8 +32,6 @@ use super::types::{ RuntimeCodexCookie, }; -const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60); -const CLOUDFLARE_PATH_BLOCK_COOLDOWN: Duration = Duration::from_secs(30); const CLOUDFLARE_RECOVERY_STALE_AFTER: Duration = Duration::from_secs(60 * 60); const CLOUDFLARE_CHALLENGE_BACKOFF: [Duration; 4] = [ Duration::from_secs(10), @@ -84,7 +82,6 @@ struct RiskRecoveryState { #[derive(Debug, Clone, Copy)] enum CookieRecovery { - None, ExpireAt(SystemTime), Clear, } @@ -321,6 +318,7 @@ impl CodexCredentialSelector { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }) @@ -329,9 +327,16 @@ impl CodexCredentialSelector { AccountCandidate { account, signals } }) .collect::>(); - // 旧版开关只影响“候选资格”。保留原始账号供 revision-fenced 凭据加载， - // 用影子候选承载放宽后的资格，避免伪造已持久化账号的运行状态。 - let quota_exhausted_candidates = (!self.skip_exhausted).then(|| { + // skip_exhausted 语义：true 时排除任一窗口触顶（limit_reached）的账号， + // 仅由上游 429 兜底；false 时保留全部候选（含 QuotaExhausted 的放宽投影）。 + // 保留原始账号供 revision-fenced 凭据加载，用影子候选承载资格放宽。 + let scheduling_candidates = if self.skip_exhausted { + candidates + .iter() + .filter(|candidate| !candidate.signals.quota_limit_reached) + .cloned() + .collect::>() + } else { candidates .iter() .cloned() @@ -340,8 +345,8 @@ impl CodexCredentialSelector { candidate }) .collect::>() - }); - let scheduling_candidates = quota_exhausted_candidates.as_deref().unwrap_or(&candidates); + }; + let scheduling_candidates = scheduling_candidates.as_slice(); let mut affinity = if diagnostic { AffinitySelection::default() } else { @@ -682,118 +687,64 @@ impl CodexCredentialSelector { failure: CodexAccountFailure, ) -> Result<(), CredentialSelectionError> { let now = SystemTime::now(); - let (availability, reason, cooldown_until, cookie_recovery) = match failure { - CodexAccountFailure::CredentialExpired => ( - AccountAvailability::Expired, - Some("credential_expired".to_owned()), - None, - CookieRecovery::None, - ), - CodexAccountFailure::IdentityVerificationRequired => ( - AccountAvailability::Invalid, - Some("identity_verification_required".to_owned()), - None, - CookieRecovery::None, - ), - CodexAccountFailure::Banned => ( - AccountAvailability::Banned, - Some("account_banned".to_owned()), - None, - CookieRecovery::None, - ), - CodexAccountFailure::QuotaExhausted => ( - AccountAvailability::QuotaExhausted, - Some("quota_exhausted".to_owned()), - None, - CookieRecovery::None, - ), - CodexAccountFailure::UsageLimitExhausted { retry_after } => { - return self - .record_resettable_limit_failure( - account, - retry_after, - "usage_limit_exhausted", - now, - ) - .await; + match failure { + CodexAccountFailure::CredentialExpired => { + self.apply_state(account, AccountAvailability::Expired, now) + .await } + CodexAccountFailure::IdentityVerificationRequired => { + self.apply_state(account, AccountAvailability::Invalid, now) + .await + } + CodexAccountFailure::Banned => { + self.apply_state(account, AccountAvailability::Banned, now) + .await + } + // 402：真额度耗尽，由 quota worker 按窗口 reset 自动恢复。 + CodexAccountFailure::QuotaExhausted + | CodexAccountFailure::UsageLimitExhausted { .. } => { + self.apply_state(account, AccountAvailability::QuotaExhausted, now) + .await + } + // 429：临时限流只写 quota，availability 不改变。 CodexAccountFailure::RateLimited { retry_after } => { - return self - .record_resettable_limit_failure(account, retry_after, "rate_limited", now) - .await; + self.quota + .apply_rate_limit_429(account, retry_after, now) + .await + .map_err(|_| CredentialSelectionError::Store)?; + Ok(()) } + // Cloudflare 挑战：内存退避表（记录风险计数），不写 availability。 CodexAccountFailure::CloudflareChallenge { retry_after } => { let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after); - let cooldown_until = now.checked_add(delay); - ( - AccountAvailability::Cooldown, - Some("cloudflare_challenge".to_owned()), - cooldown_until, - cooldown_until.map_or(CookieRecovery::Clear, CookieRecovery::ExpireAt), - ) + let recovery = now + .checked_add(delay) + .map_or(CookieRecovery::Clear, CookieRecovery::ExpireAt); + self.apply_cookie_recovery(account, recovery).await?; + Ok(()) } + // Cloudflare 路径被封：连续超阈值才置 Invalid；否则内存退避。 CodexAccountFailure::CloudflarePathBlocked => { let blocked = self.record_cloudflare_path_block(account.id(), now); if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD { - ( - AccountAvailability::Invalid, - Some("cloudflare_path_blocked".to_owned()), - None, - CookieRecovery::Clear, - ) - } else { - ( - AccountAvailability::Cooldown, - Some("cloudflare_path_blocked".to_owned()), - now.checked_add(CLOUDFLARE_PATH_BLOCK_COOLDOWN), - CookieRecovery::Clear, - ) + self.apply_state(account, AccountAvailability::Invalid, now) + .await?; } + self.apply_cookie_recovery(account, CookieRecovery::Clear) + .await?; + Ok(()) } - }; - self.repository - .apply_state(account, availability, reason, cooldown_until, now) - .await?; - self.apply_cookie_recovery(account, cookie_recovery).await?; - Ok(()) + } } - async fn record_resettable_limit_failure( + async fn apply_state( &self, account: &ProviderAccount, - retry_after: Option, - reason: &'static str, + availability: AccountAvailability, observed_at: SystemTime, ) -> Result<(), CredentialSelectionError> { - let current = self.current_account(account.id()).await?; - if current.revision() != account.revision() { - return Err(CredentialRepositoryError::RevisionConflict.into()); - } - let cooldown_until = - observed_at.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)); - let cooldown_until = match (current.availability(), current.cooldown_until()) { - (AccountAvailability::QuotaExhausted, Some(existing)) => { - cooldown_until.map_or(Some(existing), |until| Some(existing.max(until))) - } - _ => cooldown_until, - }; - let reason = if current.availability() == AccountAvailability::QuotaExhausted - && current.cooldown_until().is_none() - { - Some("quota_exhausted".to_owned()) - } else { - Some(reason.to_owned()) - }; - // 一次真实 429 证明上游已经接受当前凭据，可纠正旧的认证/封禁终态。 - // resettable 用量限制沿用 v2 的 quota lock 语义，并保留/延长复核时间。 self.repository - .apply_state( - ¤t, - AccountAvailability::QuotaExhausted, - reason, - cooldown_until, - observed_at, - ) + .apply_state(account, availability, observed_at) .await?; Ok(()) } @@ -857,7 +808,6 @@ impl CodexCredentialSelector { || !matches!( current.availability(), AccountAvailability::Unknown - | AccountAvailability::Cooldown | AccountAvailability::Expired | AccountAvailability::Invalid | AccountAvailability::Banned @@ -867,13 +817,7 @@ impl CodexCredentialSelector { } if let Err(error) = self .repository - .apply_state( - ¤t, - AccountAvailability::Ready, - None, - None, - SystemTime::now(), - ) + .apply_state(¤t, AccountAvailability::Ready, SystemTime::now()) .await { tracing::warn!( @@ -985,15 +929,11 @@ impl CodexCredentialSelector { account: &ProviderAccount, recovery: CookieRecovery, ) -> Result<(), CredentialSelectionError> { - if matches!(recovery, CookieRecovery::None) { - return Ok(()); - } let mut data = self.repository.load_complete_data(account).await?; if data.cookies().is_empty() { return Ok(()); } match recovery { - CookieRecovery::None => return Ok(()), CookieRecovery::ExpireAt(expires_at) => { let expires_at = chrono::DateTime::::from(expires_at); for cookie in data.cookies_mut() { @@ -1013,11 +953,9 @@ impl CodexCredentialSelector { fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderAccount { if account.availability() == AccountAvailability::QuotaExhausted { - return account.clone().with_runtime_state( - account.enabled(), - AccountAvailability::Ready, - account.cooldown_until(), - ); + return account + .clone() + .with_runtime_state(account.enabled(), AccountAvailability::Ready); } account.clone() } @@ -1032,7 +970,6 @@ fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> Af } match account.availability() { AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted, - AccountAvailability::Cooldown => AffinityEscapeReason::Cooldown, AccountAvailability::Unknown | AccountAvailability::Ready | AccountAvailability::Expired diff --git a/backend/crates/providers/openai/src/provider.rs b/backend/crates/providers/openai/src/provider.rs index 8d56b80fb..342739406 100644 --- a/backend/crates/providers/openai/src/provider.rs +++ b/backend/crates/providers/openai/src/provider.rs @@ -2533,7 +2533,6 @@ impl ScheduledTask for OpenAiQuotaTask { updated = summary.updated, exhausted = summary.exhausted, banned = summary.banned, - cooldown = summary.cooldown, transient = summary.transient, stale = summary.stale, "OpenAI quota cycle contained operational failures" diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index 7106c8960..ee4af744b 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -498,7 +498,8 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu .collect::>(); assert_eq!(monthly.len(), 1); - assert_eq!(monthly[0].label, "月限额"); + // limit_name 优先显示，不再猜成「月限额」。 + assert_eq!(monthly[0].label, "custom codex label"); assert_eq!(monthly[0].source.as_deref(), Some("core")); assert_eq!(monthly[0].used_percent, Some(2.0)); } @@ -544,8 +545,6 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati account_id: account.id().clone(), expected_revision: account.revision(), availability: CoreAccountAvailability::QuotaExhausted, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(reset_at)), observed_at: SystemTime::now(), }) .await @@ -586,8 +585,6 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati account_id: account.id().clone(), expected_revision: account.revision(), availability: CoreAccountAvailability::Ready, - reason: None, - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -645,7 +642,6 @@ async fn openai_admin_provider_rejects_unprepared_mutations_before_store_commit( stale_record.next_refresh_at = None; stale_record.enabled = false; stale_record.availability = AdminAccountAvailability::Banned; - stale_record.cooldown_until = Some(Utc::now() + chrono::Duration::minutes(5)); let rotation_error = admin .prepare_rotation(PrepareCredentialRotation { account: stale_record, @@ -712,8 +708,6 @@ fn account_record(account: &ProviderAccount) -> AccountRecord { next_refresh_at: account.next_refresh_at().map(DateTime::::from), enabled: account.enabled(), availability: AdminAccountAvailability::Ready, - availability_reason: None, - cooldown_until: account.cooldown_until().map(DateTime::::from), availability_observed_at: now, quota_observed_at: None, created_at: now, diff --git a/backend/crates/providers/openai/tests/credential/admin.rs b/backend/crates/providers/openai/tests/credential/admin.rs index 7e03c35a5..fc741ced2 100644 --- a/backend/crates/providers/openai/tests/credential/admin.rs +++ b/backend/crates/providers/openai/tests/credential/admin.rs @@ -194,7 +194,7 @@ fn agent_export_item(id: &str) -> (ExportManagedCodexCredential, String) { Some("agent-account-export".to_owned()), Some("pro".to_owned()), ) - .with_runtime_state(true, AccountAvailability::Ready, None) + .with_runtime_state(true, AccountAvailability::Ready) .with_refresh_schedule(false, None); ( ExportManagedCodexCredential { diff --git a/backend/crates/providers/openai/tests/credential/agent_identity.rs b/backend/crates/providers/openai/tests/credential/agent_identity.rs index 29ee3985a..27736a617 100644 --- a/backend/crates/providers/openai/tests/credential/agent_identity.rs +++ b/backend/crates/providers/openai/tests/credential/agent_identity.rs @@ -66,7 +66,7 @@ fn agent_account( Some("agent-account".to_owned()), Some("pro".to_owned()), ) - .with_runtime_state(true, AccountAvailability::Ready, None) + .with_runtime_state(true, AccountAvailability::Ready) .with_refresh_schedule(false, None); NewProviderAccount { account, diff --git a/backend/crates/providers/openai/tests/credential/contract.rs b/backend/crates/providers/openai/tests/credential/contract.rs index b58f371db..e60bef372 100644 --- a/backend/crates/providers/openai/tests/credential/contract.rs +++ b/backend/crates/providers/openai/tests/credential/contract.rs @@ -510,8 +510,6 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() { account_id: first.id().clone(), expected_revision: first.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -550,57 +548,6 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() { ); } -#[tokio::test] -async fn selector_should_escape_an_affinity_account_during_cooldown() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_first", "at-first"); - create_account(&store, "acct_second", "at-second"); - let first = store.account("acct_first").expect("first account"); - store - .apply_state_change(AccountStateChange { - account_id: first.id().clone(), - expected_revision: first.revision(), - availability: AccountAvailability::Cooldown, - reason: Some("rate_limited".to_owned()), - cooldown_until: Some(SystemTime::now() + Duration::from_secs(60)), - observed_at: SystemTime::now(), - }) - .await - .expect("mark first account cooling down"); - let affinity = Arc::new(MemorySessionAffinity::default()); - let provider = ProviderKind::new("openai").expect("provider"); - let key = ProviderSessionAffinityKey::try_new("cooldown-session").expect("affinity key"); - affinity - .bind(&provider, &key, first.id(), Duration::from_secs(60)) - .await - .expect("seed affinity"); - let selector = - selector_with_affinity(&store, Arc::new(TestLeaseCoordinator::default()), affinity); - let request_url = - Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL"); - let request_attempt = attempt(BTreeSet::new()); - - let selected = selector - .select(&SelectCodexCredential { - upstream_model: "gpt-5.4", - request_url: &request_url, - attempt: &request_attempt, - session_affinity_key: Some(&key), - }) - .await - .expect("select fallback account"); - - assert_eq!( - ( - selected.account_id().as_str(), - selected.affinity_hit(), - selected.escape_reason(), - selected.account_switch(), - ), - ("acct_second", false, Some("cooldown"), true) - ); -} - #[test] fn selector_honors_attempt_local_account_exclusion() { let store = Arc::new(MemoryAccountStore::default()); @@ -728,7 +675,7 @@ fn credential_expired_failure_marks_unified_account_expired() { } #[test] -fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() { +fn rate_limited_failure_records_quota_window_without_changing_availability() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); @@ -753,12 +700,33 @@ fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() { )) .expect("record rate-limit failure"); + // 429 不改变 availability（限流不改变账号可用性），只把限流窗口合成进 quota JSON。 let account = store.account("acct_primary").expect("account"); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); + assert_eq!(account.availability(), AccountAvailability::Ready); + let quota = store.quota_json("acct_primary").expect("quota"); + assert_eq!( + quota + .pointer("/rate_limit/limit_reached") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + quota + .pointer("/rate_limit/primary_window/used_percent") + .and_then(serde_json::Value::as_f64), + Some(100.0) + ); assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) + quota + .pointer("/rate_limit/primary_window/reset_at") + .and_then(serde_json::Value::as_i64) + .is_some_and(|reset_at| { + reset_at + > SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_secs() as i64 + }) ); } @@ -781,11 +749,6 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() { let account = store.account("acct_primary").expect("persisted account"); assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); assert_eq!(store.quota_reads(), 0); } @@ -798,8 +761,6 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), })) .expect("mark exhausted"); @@ -813,6 +774,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() { )) .expect("record rate-limit failure"); + // 429 不写 availability，402 的 QuotaExhausted 保持不变。 assert_eq!( store .account("acct_primary") @@ -820,47 +782,6 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() { .availability(), AccountAvailability::QuotaExhausted ); - assert!( - store - .account("acct_primary") - .expect("persisted account") - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); -} - -#[test] -fn rate_limited_failure_keeps_longer_existing_quota_deadline() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_primary", "at-primary"); - let account = store.account("acct_primary").expect("account"); - let existing_deadline = SystemTime::now() + Duration::from_secs(300); - block_on(store.apply_state_change(AccountStateChange { - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: Some(existing_deadline), - observed_at: SystemTime::now(), - })) - .expect("mark exhausted"); - let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); - - block_on(selector.record_failure( - &account, - CodexAccountFailure::RateLimited { - retry_after: Some(Duration::from_secs(30)), - }, - )) - .expect("record shorter rate-limit failure"); - - assert_eq!( - store - .account("acct_primary") - .expect("persisted account") - .cooldown_until(), - Some(existing_deadline) - ); } #[test] @@ -883,13 +804,14 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() { .account("acct_primary") .expect("persisted account") .availability(), - AccountAvailability::QuotaExhausted + AccountAvailability::Ready ); - assert_eq!(store.quota_reads(), 0); + // 429 合成限流窗口需要读写 quota JSON。 + assert!(store.quota_reads() >= 1); } #[test] -fn authenticated_rate_limit_recovers_stale_authentication_state() { +fn rate_limited_failure_does_not_overwrite_stale_authentication_state() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let account = store.account("acct_primary").expect("account"); @@ -897,8 +819,6 @@ fn authenticated_rate_limit_recovers_stale_authentication_state() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::Invalid, - reason: Some("identity_verification_required".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), })) .expect("mark account invalid"); @@ -913,12 +833,13 @@ fn authenticated_rate_limit_recovers_stale_authentication_state() { )) .expect("record rate-limit failure"); + // 429 不写 availability（限流不改变账号可用性），认证终态保持不变。 assert_eq!( store .account("acct_primary") .expect("persisted account") .availability(), - AccountAvailability::QuotaExhausted + AccountAvailability::Invalid ); } @@ -936,8 +857,6 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() { account_id: account.id().clone(), expected_revision: account.revision(), availability: stale, - reason: Some("stale_observation".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), })) .expect("seed stale state"); @@ -996,7 +915,7 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() { .expect("persisted second account") .availability(), ], - [AccountAvailability::QuotaExhausted; 2] + [AccountAvailability::Ready; 2] ); } @@ -1082,7 +1001,7 @@ fn identity_verification_failure_isolates_only_selected_account() { } #[test] -fn cloudflare_challenge_backoff_escalates_and_success_resets_it() { +fn cloudflare_challenge_does_not_change_availability() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); @@ -1099,48 +1018,21 @@ fn cloudflare_challenge_backoff_escalates_and_success_resets_it() { ) .expect("select account"); + // Cloudflare 挑战只走内存退避表，不写 availability（限流不改变账号可用性）。 block_on(selector.record_failure( lease.account(), CodexAccountFailure::CloudflareChallenge { retry_after: None }, )) - .expect("record first challenge"); - let first = store - .account("acct_primary") - .expect("account") - .cooldown_until() - .expect("first cooldown") - .duration_since(SystemTime::now()) - .expect("future cooldown"); - assert!(first >= Duration::from_secs(9) && first <= Duration::from_secs(10)); - - block_on(selector.record_failure( - lease.account(), - CodexAccountFailure::CloudflareChallenge { retry_after: None }, - )) - .expect("record second challenge"); - let second = store - .account("acct_primary") - .expect("account") - .cooldown_until() - .expect("second cooldown") - .duration_since(SystemTime::now()) - .expect("future cooldown"); - assert!(second >= Duration::from_secs(29) && second <= Duration::from_secs(30)); + .expect("record challenge"); + assert_eq!( + store + .account("acct_primary") + .expect("account") + .availability(), + AccountAvailability::Ready + ); block_on(selector.record_success(lease.account(), None)); - block_on(selector.record_failure( - lease.account(), - CodexAccountFailure::CloudflareChallenge { retry_after: None }, - )) - .expect("record reset challenge"); - let reset = store - .account("acct_primary") - .expect("account") - .cooldown_until() - .expect("reset cooldown") - .duration_since(SystemTime::now()) - .expect("future cooldown"); - assert!(reset >= Duration::from_secs(9) && reset <= Duration::from_secs(10)); } #[test] @@ -1226,14 +1118,12 @@ fn cloudflare_challenge_expires_provider_owned_cookies_at_cooldown_boundary() { .expect("record challenge"); let account = store.account("acct_primary").expect("account"); - let cooldown_until = account.cooldown_until().expect("cooldown"); let data = block_on(store.repository().load_complete_data(&account)).expect("credential data"); assert_eq!(data.cookies().len(), 1); - assert!( - data.cookies()[0] - .expires_at - .is_some_and(|expires_at| SystemTime::from(expires_at) <= cooldown_until) - ); + assert!(data.cookies()[0].expires_at.is_some_and(|expires_at| { + let expires_at = SystemTime::from(expires_at); + expires_at > SystemTime::now() && expires_at <= SystemTime::now() + Duration::from_secs(120) + })); } #[test] diff --git a/backend/crates/providers/openai/tests/credential/quota.rs b/backend/crates/providers/openai/tests/credential/quota.rs index cb895cc1d..830c1c608 100644 --- a/backend/crates/providers/openai/tests/credential/quota.rs +++ b/backend/crates/providers/openai/tests/credential/quota.rs @@ -291,14 +291,13 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io() .expect("read quota") .expect("quota snapshot"); - assert_eq!(snapshot.windows().len(), 3); + assert_eq!(snapshot.windows().len(), 2); assert_eq!( snapshot.windows()[0].kind(), CodexQuotaWindowKind::ShortTerm ); assert_eq!(snapshot.windows()[0].window_seconds(), Some(18_000)); assert_eq!(snapshot.windows()[1].kind(), CodexQuotaWindowKind::Weekly); - assert_eq!(snapshot.windows()[2].kind(), CodexQuotaWindowKind::Monthly); } #[tokio::test] @@ -358,7 +357,78 @@ async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() { } #[tokio::test] -async fn passive_rate_limit_headers_update_quota_and_account_state_with_revision_fence() { +async fn code_review_limit_projects_as_independent_window_with_limit_name() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_code_review").await; + let account = store.account("acct_code_review").expect("account"); + let raw = json!({ + "rate_limit": { + "primary_window": { + "used_percent": 37, + "reset_at": 1_900_000_000, + "limit_window_seconds": 18_000 + } + }, + "code_review_rate_limit": { + "primary_window": { + "used_percent": 80, + "reset_at": 1_900_604_800, + "limit_window_seconds": 604_800 + } + }, + "additional_rate_limits": [{ + "limit_name": "code review", + "metered_feature": "code_review", + "rate_limit": { + "primary_window": { + "used_percent": 55, + "reset_at": 1_900_604_800, + "limit_window_seconds": 604_800 + } + } + }], + "spend_control": { + "reached": false, + "individual_limit": { + "used_percent": 12, + "reset_at": 1_902_592_000 + } + } + }); + store + .compare_and_swap_quota(QuotaObservation { + account_id: account.id().clone(), + expected_revision: account.revision(), + quota: Some(OpaqueProviderData::new( + raw.as_object().expect("quota object").clone(), + )), + observed_at: Some(SystemTime::now()), + }) + .await + .expect("persist quota"); + + let snapshot = quota_service(&store) + .read_account(account.id()) + .await + .expect("read quota") + .expect("quota snapshot"); + + // spend_control 不再生成窗口（只作 exhaustion 信号）， + // 顶层 code_review_rate_limit 与 additional 的 code_review 桶同源替换后保留一个。 + let review = snapshot + .windows() + .iter() + .filter(|window| window.source() == "code_review") + .collect::>(); + assert_eq!(review.len(), 1); + assert_eq!(review[0].limit_name(), Some("code review")); + assert_eq!(review[0].used_percent(), Some(55.0)); + assert_eq!(snapshot.windows().len(), 2); + assert!(!snapshot.fact().exhausted()); +} + +#[tokio::test] +async fn passive_rate_limit_headers_update_quota_with_revision_fence() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_passive_quota").await; let account = store.account("acct_passive_quota").expect("account"); @@ -384,7 +454,8 @@ async fn passive_rate_limit_headers_update_quota_and_account_state_with_revision let current = store .account("acct_passive_quota") .expect("current account"); - assert_eq!(current.availability(), AccountAvailability::QuotaExhausted); + // 被动头限流只写 quota，不改变 availability（限流不改变账号可用性）。 + assert_eq!(current.availability(), AccountAvailability::Ready); let snapshot = service .read_account(account.id()) .await @@ -434,11 +505,10 @@ async fn passive_full_percent_with_explicit_allowance_keeps_account_ready() { assert_eq!( ( current.availability(), - current.cooldown_until(), snapshot.fact().remaining_percent(), snapshot.fact().exhausted(), ), - (AccountAvailability::Ready, None, Some(0), false) + (AccountAvailability::Ready, Some(0), false) ); } @@ -453,8 +523,6 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::Expired, - reason: Some("stale_expired_state".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -495,7 +563,8 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese .account(account_id) .expect("exhausted account") .availability(), - AccountAvailability::QuotaExhausted + AccountAvailability::Expired, + "manual refresh never changes non-QuotaExhausted availability" ); server.reset().await; @@ -526,8 +595,8 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese .account(account_id) .expect("account after contradictory quota") .availability(), - AccountAvailability::QuotaExhausted, - "an unchanged usage window cannot override a confirmed quota failure" + AccountAvailability::Expired, + "an unchanged usage window cannot override an expired credential state" ); server.reset().await; @@ -558,8 +627,8 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese .account(account_id) .expect("account after lower usage") .availability(), - AccountAvailability::QuotaExhausted, - "a lower used_percent before the recorded reset is not quota recovery" + AccountAvailability::Expired, + "a lower used_percent before the recorded reset is not quota recovery for an expired credential" ); } @@ -574,8 +643,6 @@ async fn manual_quota_refresh_preserves_disabled_account_state() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::Expired, - reason: Some("disabled_expired_state".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -629,8 +696,6 @@ async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential( account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -782,7 +847,6 @@ async fn synchronize_without_accounts_is_a_noop_before_network_io() { assert_eq!(summary.updated, 0); assert_eq!(summary.exhausted, 0); assert_eq!(summary.banned, 0); - assert_eq!(summary.cooldown, 0); assert_eq!(summary.transient, 0); assert_eq!(summary.stale, 0); } @@ -824,7 +888,7 @@ async fn periodic_quota_synchronization_does_not_scan_stale_ready_accounts() { } #[tokio::test] -async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_quota() { +async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_periodic_stuck_cooldown").await; let account = store @@ -835,9 +899,7 @@ async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_q .apply_state_change(AccountStateChange { account_id: account.id().clone(), expected_revision: account.revision(), - availability: AccountAvailability::Cooldown, - reason: Some("rate_limited".to_owned()), - cooldown_until: SystemTime::now().checked_add(Duration::from_secs(24 * 60 * 60)), + availability: AccountAvailability::Ready, observed_at: SystemTime::now(), }) .await @@ -884,19 +946,11 @@ async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_q service.synchronize().await.expect("quota synchronization"); + // 429/触顶不再写入 availability（限流不改变账号可用性），worker 只更新 quota 数据。 let reconciled = store .account("acct_periodic_stuck_cooldown") .expect("reconciled account"); - assert_eq!( - (reconciled.availability(), reconciled.cooldown_until()), - ( - AccountAvailability::QuotaExhausted, - Some( - SystemTime::UNIX_EPOCH - + Duration::from_secs(u64::try_from(reset_at).expect("positive reset")), - ), - ) - ); + assert_eq!(reconciled.availability(), AccountAvailability::Ready); } #[tokio::test] @@ -911,8 +965,6 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -938,11 +990,6 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li .account("acct_periodic_exhausted") .expect("persisted account"); assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); } #[tokio::test] @@ -957,8 +1004,6 @@ async fn periodic_quota_synchronization_respects_retry_after_on_service_unavaila account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -985,15 +1030,10 @@ async fn periodic_quota_synchronization_respects_retry_after_on_service_unavaila .account("acct_periodic_503_retry_after") .expect("persisted account"); assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); } #[tokio::test] -async fn periodic_quota_synchronization_skips_ordinary_cooldown_without_exhausted_quota() { +async fn periodic_quota_synchronization_skips_ready_accounts_without_quota_reached() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_periodic_temporary_cooldown").await; let account = store @@ -1003,9 +1043,7 @@ async fn periodic_quota_synchronization_skips_ordinary_cooldown_without_exhauste .apply_state_change(AccountStateChange { account_id: account.id().clone(), expected_revision: account.revision(), - availability: AccountAvailability::Cooldown, - reason: Some("rate_limited".to_owned()), - cooldown_until: SystemTime::now().checked_add(Duration::from_secs(60)), + availability: AccountAvailability::Ready, observed_at: SystemTime::now(), }) .await @@ -1022,7 +1060,7 @@ async fn periodic_quota_synchronization_skips_ordinary_cooldown_without_exhauste .account("acct_periodic_temporary_cooldown") .expect("cooldown account") .availability(), - AccountAvailability::Cooldown + AccountAvailability::Ready ); } @@ -1038,8 +1076,6 @@ async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -1065,8 +1101,6 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: SystemTime::now().checked_add(Duration::from_secs(24 * 60 * 60)), observed_at: SystemTime::now(), }) .await @@ -1131,8 +1165,6 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: SystemTime::now().checked_add(Duration::from_secs(24 * 60 * 60)), observed_at: SystemTime::now(), }) .await @@ -1223,21 +1255,19 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: SystemTime::now().checked_sub(Duration::from_secs(60)), observed_at: SystemTime::now(), }) .await .expect("mark account exhausted with an elapsed reset"); let server = MockServer::start().await; - let next_reset = Utc::now().timestamp() + 24 * 60 * 60; + let elapsed_reset = Utc::now().timestamp() - 1; Mock::given(method("GET")) .and(path("/api/codex/usage")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "rate_limit": { "allowed": true, "limit_reached": false, - "primary_window": {"used_percent": 1, "reset_at": next_reset} + "primary_window": {"used_percent": 1, "reset_at": elapsed_reset} } }))) .expect(1) @@ -1260,7 +1290,6 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() { .account(account_id) .expect("account after elapsed reset"); assert_eq!(current.availability(), AccountAvailability::Ready); - assert_eq!(current.cooldown_until(), None); } #[tokio::test] @@ -1275,8 +1304,6 @@ async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -1295,15 +1322,11 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset let account_id = "acct_periodic_new_reset"; create_account(&store, account_id).await; let account = store.account(account_id).expect("created account"); - let initial_reset = SystemTime::UNIX_EPOCH + Duration::from_secs(1); - let confirmed_reset = SystemTime::UNIX_EPOCH + Duration::from_secs(2); store .apply_state_change(AccountStateChange { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("usage_limit_exhausted".to_owned()), - cooldown_until: Some(initial_reset), observed_at: SystemTime::now(), }) .await @@ -1330,10 +1353,6 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset ); let first = service.synchronize().await.expect("first reset recheck"); - let persisted_reset = store - .account(account_id) - .expect("account after first recheck") - .cooldown_until(); let second = service .synchronize() .await @@ -1346,20 +1365,8 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset .len(); assert_eq!( - ( - first.exhausted, - persisted_reset, - second.exhausted, - repeated, - request_count, - ), - ( - 1, - Some(confirmed_reset), - 0, - CodexQuotaSyncSummary::default(), - 1, - ) + (first.exhausted, second.exhausted, repeated, request_count,), + (1, 0, CodexQuotaSyncSummary::default(), 1,) ); } @@ -1374,8 +1381,6 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() { account_id: account.id().clone(), expected_revision: account.revision(), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await diff --git a/backend/crates/providers/openai/tests/credential/refresh.rs b/backend/crates/providers/openai/tests/credential/refresh.rs index 12c6c5412..6bc98a392 100644 --- a/backend/crates/providers/openai/tests/credential/refresh.rs +++ b/backend/crates/providers/openai/tests/credential/refresh.rs @@ -621,8 +621,6 @@ async fn expired_access_token_refreshes_even_when_next_refresh_is_deferred() { .apply_state( &deferred, AccountAvailability::QuotaExhausted, - Some("quota_exhausted".to_owned()), - None, SystemTime::now(), ) .await @@ -685,18 +683,11 @@ async fn expired_access_token_refreshes_during_active_cooldown() { .await .expect("seed expired access token with deferred refresh"); let deferred = store.account("acct_refresh").expect("deferred account"); - let cooldown_until = SystemTime::now() + Duration::from_secs(12 * 60 * 60); store .repository() - .apply_state( - &deferred, - AccountAvailability::Cooldown, - Some("rate_limited".to_owned()), - Some(cooldown_until), - SystemTime::now(), - ) + .apply_state(&deferred, AccountAvailability::Ready, SystemTime::now()) .await - .expect("mark account cooling down"); + .expect("mark account ready"); let refresher = Arc::new(Refresher::new(Ok(success_tokens()))); let service = CodexCredentialRefreshService::new( store.repository(), @@ -724,8 +715,7 @@ async fn expired_access_token_refreshes_during_active_cooldown() { ["rt-old-access"] ); let refreshed = store.account("acct_refresh").expect("refreshed account"); - assert_eq!(refreshed.availability(), AccountAvailability::Cooldown); - assert_eq!(refreshed.cooldown_until(), Some(cooldown_until)); + assert_eq!(refreshed.availability(), AccountAvailability::Ready); assert!( refreshed .access_token_expires_at() diff --git a/backend/crates/providers/openai/tests/provider/contract.rs b/backend/crates/providers/openai/tests/provider/contract.rs index 01cbc8e6d..1d3e3fd1e 100644 --- a/backend/crates/providers/openai/tests/provider/contract.rs +++ b/backend/crates/providers/openai/tests/provider/contract.rs @@ -1263,12 +1263,8 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc vec!["response.created", "response.failed"] ); let account = store.account(account_id).expect("rate-limited account"); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); + // 429 只记录限流窗口，不改变 availability（限流不改变账号可用性）。 + assert_eq!(account.availability(), AccountAvailability::Ready); let _ = release.send(()); server.abort(); let _ = server.await; @@ -1358,11 +1354,6 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted); let account = store.account(account_id).expect("usage-limit account"); assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); tokio::time::sleep(Duration::from_millis(200)).await; let requests = server.received_requests().await.expect("received requests"); assert_eq!( @@ -1799,8 +1790,6 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st .apply_state( &account, AccountAvailability::QuotaExhausted, - Some("quota_exhausted".to_owned()), - None, SystemTime::now(), ) .await @@ -1907,7 +1896,7 @@ async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers( .account(account_id) .expect("account after successful response") .availability(), - AccountAvailability::QuotaExhausted + AccountAvailability::Ready ); assert!(store.has_quota(account_id)); } @@ -1969,16 +1958,8 @@ async fn successful_http_sse_rate_limit_event_persists_structured_exhaustion() { .account(account_id) .expect("account after HTTP SSE response"); assert_eq!( - ( - account.availability(), - account.cooldown_until(), - store.has_quota(account_id), - ), - ( - AccountAvailability::QuotaExhausted, - Some(SystemTime::UNIX_EPOCH + Duration::from_secs(reset_at)), - true, - ) + (account.availability(), store.has_quota(account_id),), + (AccountAvailability::Ready, true,) ); } diff --git a/backend/crates/providers/openai/tests/support.rs b/backend/crates/providers/openai/tests/support.rs index bd0f71e2d..1cc259f77 100644 --- a/backend/crates/providers/openai/tests/support.rs +++ b/backend/crates/providers/openai/tests/support.rs @@ -117,6 +117,17 @@ impl MemoryAccountStore { .is_some_and(|stored| stored.quota.is_some()) } + pub(crate) fn quota_json(&self, id: &str) -> Option { + let id = ProviderAccountId::new(id).ok()?; + self.accounts + .lock() + .expect("account store lock") + .get(&id) + .and_then(|stored| stored.quota.as_ref()) + .and_then(|observation| observation.quota.as_ref()) + .map(|data| serde_json::Value::Object(data.expose_to_provider().clone())) + } + pub(crate) fn fail_provider_listing(&self) { self.fail_provider_listing.store(true, Ordering::SeqCst); } @@ -235,7 +246,6 @@ impl ProviderAccountStore for MemoryAccountStore { revision: next, enabled: stored.account.enabled(), availability: stored.account.availability(), - cooldown_until: stored.account.cooldown_until(), access_token_expires_at, has_refresh_token, next_refresh_at, @@ -303,7 +313,6 @@ impl ProviderAccountStore for MemoryAccountStore { revision: stored.account.revision(), enabled: stored.account.enabled(), availability: change.availability, - cooldown_until: change.cooldown_until, access_token_expires_at: stored.account.access_token_expires_at(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -325,7 +334,6 @@ impl ProviderAccountStore for MemoryAccountStore { revision: stored.account.revision(), enabled: stored.account.enabled(), availability: stored.account.availability(), - cooldown_until: stored.account.cooldown_until(), access_token_expires_at: stored.account.access_token_expires_at(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -350,7 +358,6 @@ impl ProviderAccountStore for MemoryAccountStore { revision: stored.account.revision(), enabled, availability: stored.account.availability(), - cooldown_until: stored.account.cooldown_until(), access_token_expires_at: stored.account.access_token_expires_at(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -378,7 +385,6 @@ struct AccountRebuild { revision: CredentialRevision, enabled: bool, availability: AccountAvailability, - cooldown_until: Option, access_token_expires_at: Option, has_refresh_token: bool, next_refresh_at: Option, @@ -407,11 +413,7 @@ fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> Provid current.upstream_account_id().map(str::to_owned), plan_type, ) - .with_runtime_state( - rebuild.enabled, - rebuild.availability, - rebuild.cooldown_until, - ) + .with_runtime_state(rebuild.enabled, rebuild.availability) .with_refresh_schedule(rebuild.has_refresh_token, rebuild.next_refresh_at) } @@ -464,6 +466,7 @@ impl ProviderLeasePort for TestLeaseCoordinator { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -765,6 +768,6 @@ pub(crate) fn codex_account(id: &str) -> ProviderAccount { Some(format!("chatgpt-{id}")), Some("pro".to_owned()), ) - .with_runtime_state(true, AccountAvailability::Ready, None) + .with_runtime_state(true, AccountAvailability::Ready) .with_refresh_schedule(true, Some(SystemTime::now() + Duration::from_secs(2_700))) } diff --git a/backend/crates/providers/xai/src/admin.rs b/backend/crates/providers/xai/src/admin.rs index ab39cc00d..a3afb52fc 100644 --- a/backend/crates/providers/xai/src/admin.rs +++ b/backend/crates/providers/xai/src/admin.rs @@ -913,8 +913,6 @@ fn prepared_create( next_refresh_at: account.next_refresh_at().map(Into::into), enabled: account.enabled(), availability: account.availability(), - availability_reason: None, - cooldown_until: account.cooldown_until().map(Into::into), availability_observed_at: observed_at, }) } @@ -1018,11 +1016,7 @@ fn account_from_record(account: &AccountRecord) -> Result= 100.0), local_usage: None, provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(data))), } @@ -1099,6 +1096,7 @@ fn project_quota( window_seconds: Some(crate::GROK_FREE_ROLLING_WINDOW_SECONDS), used_percent: None, reset_at: None, + limit_reached: false, local_usage: rolling_usage, provider_data: None, } diff --git a/backend/crates/providers/xai/src/credential/catalog.rs b/backend/crates/providers/xai/src/credential/catalog.rs index ade2731a1..8607689a0 100644 --- a/backend/crates/providers/xai/src/credential/catalog.rs +++ b/backend/crates/providers/xai/src/credential/catalog.rs @@ -482,7 +482,6 @@ impl GrokCredentialQuotaService { GrokCredentialAvailability::QuotaExhausted ) .then_some("quota_exhausted".to_owned()), - cooldown_until: None, observed_at, }) .await @@ -1151,7 +1150,6 @@ fn eligible_catalog_candidate(candidate: &LoadedGrokCredential) -> bool { && match account.availability() { AccountAvailability::Unknown | AccountAvailability::Ready - | AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted => true, AccountAvailability::Expired | AccountAvailability::Banned @@ -1294,8 +1292,15 @@ fn quota_scheduling_signals(billing: &GrokBillingPresentation) -> Option now); - (remaining_rank.is_some() || reset_at.is_some()) - .then(|| AccountQuotaSignals::new(reset_at, remaining_rank)) + (remaining_rank.is_some() || reset_at.is_some()).then(|| { + AccountQuotaSignals::new( + reset_at, + remaining_rank, + billing + .used_percent() + .is_some_and(|used| used.is_finite() && used >= 100.0), + ) + }) } fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool { @@ -1343,7 +1348,6 @@ fn quota_refresh_may_update_state(before: &ProviderAccount, current: &ProviderAc .is_some_and(|expires_at| expires_at > SystemTime::now()) && current.revision() == before.revision() && current.availability() == before.availability() - && current.cooldown_until() == before.cooldown_until() } fn quota_refresh_availability( @@ -1357,15 +1361,13 @@ fn quota_refresh_availability( AccountAvailability::QuotaExhausted => { matches!(exhausted, Some(false)).then_some(GrokCredentialAvailability::Ready) } - AccountAvailability::Cooldown | AccountAvailability::Unknown => { - exhausted.map(|exhausted| { - if exhausted { - GrokCredentialAvailability::QuotaExhausted - } else { - GrokCredentialAvailability::Ready - } - }) - } + AccountAvailability::Unknown => exhausted.map(|exhausted| { + if exhausted { + GrokCredentialAvailability::QuotaExhausted + } else { + GrokCredentialAvailability::Ready + } + }), AccountAvailability::Ready => { matches!(exhausted, Some(true)).then_some(GrokCredentialAvailability::QuotaExhausted) } diff --git a/backend/crates/providers/xai/src/credential/refresh.rs b/backend/crates/providers/xai/src/credential/refresh.rs index 300043160..c392b3185 100644 --- a/backend/crates/providers/xai/src/credential/refresh.rs +++ b/backend/crates/providers/xai/src/credential/refresh.rs @@ -705,7 +705,6 @@ impl GrokCredentialRefreshService { expected_revision: credential.credential_revision, availability, availability_reason: Some(reason.to_owned()), - cooldown_until: None, observed_at: Utc::now(), }) .await @@ -852,8 +851,6 @@ fn account_due( | AccountAvailability::Banned | AccountAvailability::Invalid ) - && (availability != AccountAvailability::Cooldown - || account.cooldown_until().is_some_and(|until| until <= now)) && account.next_refresh_at().is_some_and(|next| next <= now) } diff --git a/backend/crates/providers/xai/src/credential/repository.rs b/backend/crates/providers/xai/src/credential/repository.rs index 9152ee502..b29ef253a 100644 --- a/backend/crates/providers/xai/src/credential/repository.rs +++ b/backend/crates/providers/xai/src/credential/repository.rs @@ -168,11 +168,7 @@ impl GrokCredentialAdmin { input.account.upstream_account_id.clone(), input.account.plan_type.clone(), ) - .with_runtime_state( - input.enabled, - input.initial_availability.into(), - input.initial_cooldown_until.map(to_system_time), - ) + .with_runtime_state(input.enabled, input.initial_availability.into()) .with_refresh_schedule(true, Some(to_system_time(input.next_refresh_at))); Ok(NewProviderAccount { account, @@ -222,7 +218,6 @@ impl GrokCredentialAdmin { enabled: input.enabled, initial_availability: GrokCredentialAvailability::Ready, initial_availability_reason: None, - initial_cooldown_until: None, }) } @@ -434,15 +429,12 @@ impl GrokCredentialRepository { input: &UpdateGrokCredentialState, ) -> Result<(), GrokCredentialRepositoryError> { validate_reason(input.availability_reason.as_deref())?; - validate_cooldown(input.availability, input.cooldown_until)?; self.ensure_xai_account(&input.account_id).await?; self.store .apply_state_change(AccountStateChange { account_id: input.account_id.clone(), expected_revision: input.expected_revision, availability: input.availability.into(), - reason: input.availability_reason.clone(), - cooldown_until: input.cooldown_until.map(to_system_time), observed_at: to_system_time(input.observed_at), }) .await @@ -708,8 +700,7 @@ fn validate_create(input: &CreateGrokCredential) -> Result<(), GrokCredentialRep validate_profile(&input.account)?; validate_refresh_schedule(input.next_refresh_at, input.account.access_token_expires_at)?; validate_secret(&input.secret)?; - validate_reason(input.initial_availability_reason.as_deref())?; - validate_cooldown(input.initial_availability, input.initial_cooldown_until) + validate_reason(input.initial_availability_reason.as_deref()) } fn validate_profile(profile: &GrokAccountProfile) -> Result<(), GrokCredentialRepositoryError> { @@ -816,18 +807,6 @@ fn validate_reason(reason: Option<&str>) -> Result<(), GrokCredentialRepositoryE Ok(()) } -fn validate_cooldown( - availability: GrokCredentialAvailability, - cooldown_until: Option>, -) -> Result<(), GrokCredentialRepositoryError> { - if matches!(availability, GrokCredentialAvailability::Cooldown) != cooldown_until.is_some() { - return Err(GrokCredentialRepositoryError::InvalidInput( - "cooldown_until", - )); - } - Ok(()) -} - fn ensure_xai(account: &ProviderAccount) -> Result<(), GrokCredentialRepositoryError> { if account.provider().as_str() == XAI_PROVIDER_KIND && account.authentication_kind() == XAI_AUTHENTICATION_KIND_OAUTH diff --git a/backend/crates/providers/xai/src/credential/selector.rs b/backend/crates/providers/xai/src/credential/selector.rs index 63f9304d0..8b00ddde3 100644 --- a/backend/crates/providers/xai/src/credential/selector.rs +++ b/backend/crates/providers/xai/src/credential/selector.rs @@ -4,7 +4,7 @@ use std::fmt; use std::sync::Arc; use std::time::{Duration, SystemTime}; -use chrono::{DateTime, Utc}; +use chrono::Utc; use gateway_core::engine::credential::{ AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats, AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId, @@ -300,36 +300,26 @@ impl GrokAccountSessionSelector { for ((account, account_read), model_read) in accounts.iter().zip(account_reads).zip(model_reads) { - if let Ok(Some(cooldown)) = account_read { - if cooldown.credential_revision() != account.revision() { - if cooldown.credential_revision() < account.revision() { - let _ = self.cooldowns.clear(account.id(), account.revision()).await; - } - } else if cooldown.until() > now { - insert_runtime_cooldown( - &mut cooled, - account.id(), - cooldown.until(), - RuntimeCooldownScope::Account, - ); - } + if let Ok(Some(cooldown)) = account_read + && cooldown.until() > now + { + // 账号级限流冷却跨凭据轮换保留：上游限流不因本地凭据更换解除。 + insert_runtime_cooldown( + &mut cooled, + account.id(), + cooldown.until(), + RuntimeCooldownScope::Account, + ); } - if let Ok(Some(cooldown)) = model_read { - if cooldown.credential_revision() != account.revision() { - if cooldown.credential_revision() < account.revision() { - let _ = self - .cooldowns - .clear_scoped(account.id(), &model_scope, account.revision()) - .await; - } - } else if cooldown.until() > now { - insert_runtime_cooldown( - &mut cooled, - account.id(), - cooldown.until(), - RuntimeCooldownScope::Model, - ); - } + if let Ok(Some(cooldown)) = model_read + && cooldown.until() > now + { + insert_runtime_cooldown( + &mut cooled, + account.id(), + cooldown.until(), + RuntimeCooldownScope::Model, + ); } } cooled @@ -374,7 +364,6 @@ impl GrokAccountSessionSelector { session: &SelectedGrokSession, availability: GrokCredentialAvailability, reason: &'static str, - cooldown_until: Option>, observed_at: chrono::DateTime, ) { let update = |expected_revision| UpdateGrokCredentialState { @@ -382,7 +371,6 @@ impl GrokAccountSessionSelector { expected_revision, availability, availability_reason: Some(reason.to_owned()), - cooldown_until, observed_at, }; match self @@ -467,28 +455,22 @@ impl GrokSessionSelector for GrokAccountSessionSelector { GrokCredentialFailure::QuotaExhausted => ( GrokCredentialAvailability::QuotaExhausted, "upstream_quota_exhausted", - None, ), GrokCredentialFailure::FreeQuotaExhausted => ( GrokCredentialAvailability::QuotaExhausted, "upstream_free_quota_exhausted", - None, ), GrokCredentialFailure::PaymentRequired { .. } => ( GrokCredentialAvailability::QuotaExhausted, "upstream_payment_quota_exhausted", - None, ), + // xAI 免费模型额度按账号滚动窗口恢复：限流只写运行时冷却 + // （Redis 跨重启保留），不进入持久化账号状态。 GrokCredentialFailure::ModelQuotaExhausted { retry_after, .. } => { let retry_after = bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN); - // xAI 免费模型额度按账号滚动窗口恢复；持久化使页面和各调度节点收敛。 self.record_runtime_cooldown(session, retry_after).await; - ( - GrokCredentialAvailability::Cooldown, - "usage_limit_exhausted", - persistent_cooldown_until(observed_at, retry_after), - ) + return; } GrokCredentialFailure::ModelAccessDenied { upstream_model, @@ -508,31 +490,17 @@ impl GrokSessionSelector for GrokAccountSessionSelector { .unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN) .min(MAX_RATE_LIMIT_COOLDOWN); self.record_runtime_cooldown(session, retry_after).await; - ( - GrokCredentialAvailability::Cooldown, - "upstream_rate_limited", - persistent_cooldown_until(observed_at, retry_after), - ) + return; } GrokCredentialFailure::StreamInterrupted => { self.record_runtime_cooldown(session, STREAM_INTERRUPTION_COOLDOWN) .await; - ( - GrokCredentialAvailability::Cooldown, - "upstream_stream_interrupted", - persistent_cooldown_until(observed_at, STREAM_INTERRUPTION_COOLDOWN), - ) + return; } }; - let (availability, reason, cooldown_until) = persistent; - self.persist_account_failure( - session, - availability, - reason, - cooldown_until, - observed_at, - ) - .await; + let (availability, reason) = persistent; + self.persist_account_failure(session, availability, reason, observed_at) + .await; }) } @@ -552,9 +520,7 @@ impl GrokSessionSelector for GrokAccountSessionSelector { || !account.enabled() || !matches!( account.availability(), - AccountAvailability::Unknown - | AccountAvailability::Cooldown - | AccountAvailability::Expired + AccountAvailability::Unknown | AccountAvailability::Expired ) { return; @@ -566,7 +532,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector { expected_revision: account.revision(), availability: GrokCredentialAvailability::Ready, availability_reason: None, - cooldown_until: None, observed_at: Utc::now(), }) .await @@ -600,15 +565,6 @@ fn bounded_cooldown( .min(maximum) } -fn persistent_cooldown_until( - observed_at: DateTime, - duration: Duration, -) -> Option> { - chrono::Duration::from_std(duration) - .ok() - .map(|duration| observed_at + duration) -} - fn insert_runtime_cooldown( cooldowns: &mut std::collections::BTreeMap, account_id: &ProviderAccountId, diff --git a/backend/crates/providers/xai/src/credential/types.rs b/backend/crates/providers/xai/src/credential/types.rs index 6998d997c..55c47d685 100644 --- a/backend/crates/providers/xai/src/credential/types.rs +++ b/backend/crates/providers/xai/src/credential/types.rs @@ -64,11 +64,13 @@ impl fmt::Debug for GrokAccountProfile { } /// 与 `provider_accounts.availability` 一一对应的 xAI 状态。 +/// +/// 限流（429/滚动窗口）不进入持久化状态：由 `ProviderCooldownPort`（Redis 跨重启 +/// 保留）驱动调度排除，账号状态保持 `Ready`。 #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum GrokCredentialAvailability { Unknown, Ready, - Cooldown, QuotaExhausted, Expired, Banned, @@ -81,7 +83,6 @@ impl From for AccountAvailability { match value { GrokCredentialAvailability::Unknown => Self::Unknown, GrokCredentialAvailability::Ready => Self::Ready, - GrokCredentialAvailability::Cooldown => Self::Cooldown, GrokCredentialAvailability::QuotaExhausted => Self::QuotaExhausted, GrokCredentialAvailability::Expired => Self::Expired, GrokCredentialAvailability::Banned => Self::Banned, @@ -100,7 +101,6 @@ pub struct CreateGrokCredential { pub enabled: bool, pub initial_availability: GrokCredentialAvailability, pub initial_availability_reason: Option, - pub initial_cooldown_until: Option>, } impl fmt::Debug for CreateGrokCredential { @@ -243,7 +243,6 @@ pub struct UpdateGrokCredentialState { pub expected_revision: CredentialRevision, pub availability: GrokCredentialAvailability, pub availability_reason: Option, - pub cooldown_until: Option>, pub observed_at: DateTime, } diff --git a/backend/crates/providers/xai/tests/admin.rs b/backend/crates/providers/xai/tests/admin.rs index a8981f1a4..a0f2cc385 100644 --- a/backend/crates/providers/xai/tests/admin.rs +++ b/backend/crates/providers/xai/tests/admin.rs @@ -472,8 +472,6 @@ fn account_record(account: &ProviderAccount) -> AccountRecord { next_refresh_at: account.next_refresh_at().map(DateTime::::from), enabled: account.enabled(), availability: account.availability(), - availability_reason: None, - cooldown_until: account.cooldown_until().map(DateTime::::from), availability_observed_at: now, quota_observed_at: None, created_at: now, @@ -504,6 +502,7 @@ impl ProviderLeasePort for TestLeases { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/providers/xai/tests/credential/catalog.rs b/backend/crates/providers/xai/tests/credential/catalog.rs index 43ea468e3..debdbdaa8 100644 --- a/backend/crates/providers/xai/tests/credential/catalog.rs +++ b/backend/crates/providers/xai/tests/credential/catalog.rs @@ -201,15 +201,12 @@ async fn set_account_state( store: &MemoryProviderAccountStore, id: &ProviderAccountId, availability: AccountAvailability, - cooldown_until: Option, ) { store .apply_state_change(AccountStateChange { account_id: id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), availability, - reason: Some("test isolation".to_owned()), - cooldown_until, observed_at: SystemTime::now(), }) .await @@ -491,8 +488,6 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() { account_id: id, expected_revision: CredentialRevision::new(1).expect("revision"), availability: AccountAvailability::QuotaExhausted, - reason: Some("quota exhausted".to_owned()), - cooldown_until: None, observed_at: SystemTime::now(), }) .await @@ -686,18 +681,14 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie } #[tokio::test] -async fn recovered_quota_refresh_releases_existing_quota_and_cooldown_isolation() { - for (suffix, availability, cooldown_until) in [ - ("recover-quota", AccountAvailability::QuotaExhausted, None), - ( - "recover-cooldown", - AccountAvailability::Cooldown, - SystemTime::now().checked_add(Duration::from_secs(300)), - ), +async fn recovered_quota_refresh_releases_existing_quota_isolation() { + for (suffix, availability) in [ + ("recover-quota", AccountAvailability::QuotaExhausted), + ("recover-ready", AccountAvailability::Ready), ] { let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await; let id = account_id(suffix); - set_account_state(&store, &id, availability, cooldown_until).await; + set_account_state(&store, &id, availability).await; crate::support::grok_quota_service( repository, QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":25}}"#), @@ -708,7 +699,6 @@ async fn recovered_quota_refresh_releases_existing_quota_and_cooldown_isolation( let account = store.account(&id).expect("account"); assert_eq!(account.availability(), AccountAvailability::Ready); - assert_eq!(account.cooldown_until(), None); } } @@ -717,7 +707,7 @@ async fn authoritative_exhaustion_preserves_quota_exhausted_state() { let (store, repository) = repository_with_accounts(&[("still-exhausted", "still-exhausted")]).await; let id = account_id("still-exhausted"); - set_account_state(&store, &id, AccountAvailability::QuotaExhausted, None).await; + set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await; crate::support::grok_quota_service( repository, @@ -742,7 +732,7 @@ async fn recovered_quota_does_not_clear_terminal_account_states() { ] { let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await; let id = account_id(suffix); - set_account_state(&store, &id, availability, None).await; + set_account_state(&store, &id, availability).await; crate::support::grok_quota_service( repository, QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":10}}"#), @@ -759,20 +749,15 @@ async fn recovered_quota_does_not_clear_terminal_account_states() { } #[tokio::test] -async fn quota_refresh_does_not_overwrite_a_newer_cooldown() { +async fn quota_refresh_preserves_ready_availability_across_concurrent_state_write() { let (store, repository) = repository_with_accounts(&[("new-cooldown", "new-cooldown")]).await; let id = account_id("new-cooldown"); - let new_cooldown = SystemTime::now() - .checked_add(Duration::from_secs(600)) - .expect("cooldown time"); let transport = Arc::new(MutatingBillingTransport { store: Arc::clone(&store), mutation: Mutex::new(Some(BillingMutation::State(AccountStateChange { account_id: id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability: AccountAvailability::Cooldown, - reason: Some("new rate limit".to_owned()), - cooldown_until: Some(new_cooldown), + availability: AccountAvailability::Ready, observed_at: SystemTime::now(), }))), body: br#"{"config":{"creditUsagePercent":10}}"#.to_vec(), @@ -781,11 +766,10 @@ async fn quota_refresh_does_not_overwrite_a_newer_cooldown() { crate::support::grok_quota_service(repository, transport) .refresh_account(&id) .await - .expect("refresh around concurrent cooldown"); + .expect("refresh around concurrent state write"); let account = store.account(&id).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Cooldown); - assert_eq!(account.cooldown_until(), Some(new_cooldown)); + assert_eq!(account.availability(), AccountAvailability::Ready); } #[tokio::test] @@ -878,7 +862,7 @@ async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustio let (store, repository) = repository_with_accounts(&[("free-quota", "subject-free-quota")]).await; let id = account_id("free-quota"); - set_account_state(&store, &id, AccountAvailability::QuotaExhausted, None).await; + set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await; let transport = QueueBillingTransport::success( br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"onDemandCap":{"val":0},"onDemandUsed":{"val":0},"prepaidBalance":{"val":0}}}"#, ); diff --git a/backend/crates/providers/xai/tests/credential/contract.rs b/backend/crates/providers/xai/tests/credential/contract.rs index f92acb250..0c3f6252a 100644 --- a/backend/crates/providers/xai/tests/credential/contract.rs +++ b/backend/crates/providers/xai/tests/credential/contract.rs @@ -219,44 +219,18 @@ async fn state_update_uses_credential_revision_fence() { let (store, repository) = repository(); let input = create_input("state", "subject-state"); seed_input(&store, &input).await.expect("create"); - let cooldown_until = Utc::now() + chrono::Duration::minutes(1); repository .update_state(&UpdateGrokCredentialState { account_id: input.account_id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability: GrokCredentialAvailability::Cooldown, - availability_reason: Some("rate_limited".to_owned()), - cooldown_until: Some(cooldown_until), + availability: GrokCredentialAvailability::Ready, + availability_reason: Some("upstream_rate_limited".to_owned()), observed_at: Utc::now(), }) .await .expect("state update"); let account = store.account(&input.account_id).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Cooldown); - assert!(account.cooldown_until().is_some()); -} - -#[tokio::test] -async fn cooldown_requires_matching_deadline() { - let (store, repository) = repository(); - let input = create_input("bad-cooldown", "subject-cooldown"); - seed_input(&store, &input).await.expect("create"); - let result = repository - .update_state(&UpdateGrokCredentialState { - account_id: input.account_id, - expected_revision: CredentialRevision::new(1).expect("revision"), - availability: GrokCredentialAvailability::Cooldown, - availability_reason: None, - cooldown_until: None, - observed_at: Utc::now(), - }) - .await; - assert_eq!( - result, - Err(GrokCredentialRepositoryError::InvalidInput( - "cooldown_until" - )) - ); + assert_eq!(account.availability(), AccountAvailability::Ready); } #[tokio::test] diff --git a/backend/crates/providers/xai/tests/credential/selector.rs b/backend/crates/providers/xai/tests/credential/selector.rs index 7cf52e602..12f792e32 100644 --- a/backend/crates/providers/xai/tests/credential/selector.rs +++ b/backend/crates/providers/xai/tests/credential/selector.rs @@ -110,7 +110,6 @@ impl SelectorFixture { expected_revision: CredentialRevision::new(1).expect("revision"), availability: GrokCredentialAvailability::Ready, availability_reason: None, - cooldown_until: None, observed_at: Utc::now(), }) .await @@ -132,6 +131,7 @@ impl SelectorFixture { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, + quota_limit_reached: false, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -369,21 +369,17 @@ async fn account_scoped_cooldown_survives_credential_rotation() { CredentialCasOutcome::Updated(revision) if revision.get() == 2 )); + // 账号级限流冷却跨凭据轮换保留：轮换后仍被冷却排除。 assert!(matches!( fixture .selector .select(fixture.request(BTreeSet::new())) .await, - Err(GrokSessionSelectorError::NoEligibleSession) + Err(GrokSessionSelectorError::AccountCoolingDown { .. }) )); let account = fixture.store.account(&id).expect("rotated account"); assert_eq!(account.revision().get(), 2); - assert_eq!(account.availability(), AccountAvailability::Cooldown); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); + assert_eq!(account.availability(), AccountAvailability::Ready); } #[tokio::test] @@ -405,12 +401,7 @@ async fn rate_limit_feedback_persists_the_grok2api_cooldown_state() { ) .await; let account = fixture.store.account(&selected).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Cooldown); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); + assert_eq!(account.availability(), AccountAvailability::Ready); assert!( fixture .cooldowns @@ -450,7 +441,6 @@ async fn successful_request_clears_the_persisted_cooldown_state() { .account(session.account_id()) .expect("account"); assert_eq!(account.availability(), AccountAvailability::Ready); - assert!(account.cooldown_until().is_none()); } #[tokio::test] @@ -511,13 +501,8 @@ async fn free_model_quota_feedback_persists_resettable_account_exhaustion() { .await; let account = fixture.store.account(&selected).expect("selected account"); - assert_eq!(account.availability(), AccountAvailability::Cooldown); + assert_eq!(account.availability(), AccountAvailability::Ready); let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > minimum_until) - ); let scope = ProviderCooldownScope::upstream_model(failed_model); assert!( fixture @@ -604,12 +589,7 @@ async fn interrupted_stream_feedback_persists_the_grok2api_cooldown_state() { .store .account(session.account_id()) .expect("account"); - assert_eq!(account.availability(), AccountAvailability::Cooldown); - assert!( - account - .cooldown_until() - .is_some_and(|until| until > SystemTime::now()) - ); + assert_eq!(account.availability(), AccountAvailability::Ready); assert!( fixture .cooldowns diff --git a/backend/crates/providers/xai/tests/support.rs b/backend/crates/providers/xai/tests/support.rs index 78755712b..1c7f9f67e 100644 --- a/backend/crates/providers/xai/tests/support.rs +++ b/backend/crates/providers/xai/tests/support.rs @@ -328,7 +328,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore { revision: next, access_token_expires_at, availability: stored.account.availability(), - cooldown_until: stored.account.cooldown_until(), enabled: stored.account.enabled(), has_refresh_token, next_refresh_at, @@ -381,7 +380,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore { revision: stored.account.revision(), access_token_expires_at: stored.account.access_token_expires_at(), availability: change.availability, - cooldown_until: change.cooldown_until, enabled: stored.account.enabled(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -402,7 +400,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore { revision: stored.account.revision(), access_token_expires_at: stored.account.access_token_expires_at(), availability: stored.account.availability(), - cooldown_until: stored.account.cooldown_until(), enabled: stored.account.enabled(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -427,7 +424,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore { revision: stored.account.revision(), access_token_expires_at: stored.account.access_token_expires_at(), availability: stored.account.availability(), - cooldown_until: stored.account.cooldown_until(), enabled, has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -449,7 +445,6 @@ struct AccountReplacement { revision: CredentialRevision, access_token_expires_at: Option, availability: AccountAvailability, - cooldown_until: Option, enabled: bool, has_refresh_token: bool, next_refresh_at: Option, @@ -473,11 +468,7 @@ fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement) previous.upstream_account_id().map(str::to_owned), replacement.plan_type, ) - .with_runtime_state( - replacement.enabled, - replacement.availability, - replacement.cooldown_until, - ) + .with_runtime_state(replacement.enabled, replacement.availability) .with_refresh_schedule(replacement.has_refresh_token, replacement.next_refresh_at) } @@ -549,7 +540,6 @@ pub fn create_input(suffix: &str, subject: &str) -> CreateGrokCredential { enabled: true, initial_availability: GrokCredentialAvailability::Unknown, initial_availability_reason: None, - initial_cooldown_until: None, } } diff --git a/backend/migrations/0002_remove_account_cooldown.sql b/backend/migrations/0002_remove_account_cooldown.sql new file mode 100644 index 000000000..29f0b1ac8 --- /dev/null +++ b/backend/migrations/0002_remove_account_cooldown.sql @@ -0,0 +1,43 @@ +-- 移除账号级 cooldown 状态模型（照 v2 语义：429 限流不进 availability，由 quota 数据驱动）。 +-- +-- 存量迁移： +-- - availability='cooldown' 的账号：429/Cloudflare 等临时冷却 → 'ready' +-- （限流事实留在 provider_quota_json，由投影滚动兜底）； +-- 402 语义的 usage_limit_exhausted 且未到期 → 'quota_exhausted'。 + +-- 1. 迁移存量 cooldown 账号。 +update provider_accounts +set availability = case + when availability_reason = 'usage_limit_exhausted' + and cooldown_until is not null + and cooldown_until > now() + then 'quota_exhausted' + else 'ready' + end +where availability = 'cooldown'; + +-- 2. 删除依赖 cooldown_until 的约束与索引。 +alter table provider_accounts drop constraint provider_accounts_cooldown_ck; +drop index provider_accounts_availability_idx; + +-- 3. availability CHECK 去掉 'cooldown'。 +alter table provider_accounts drop constraint provider_accounts_availability_ck; +alter table provider_accounts + add constraint provider_accounts_availability_ck check ( + availability in ( + 'unknown', + 'ready', + 'quota_exhausted', + 'expired', + 'banned', + 'invalid' + ) + ); + +-- 4. 删除列。 +alter table provider_accounts drop column availability_reason; +alter table provider_accounts drop column cooldown_until; + +-- 5. 重建可用性索引（不含 cooldown_until）。 +create index provider_accounts_availability_idx + on provider_accounts (availability, id); diff --git a/frontend/src/api/modules/accounts.ts b/frontend/src/api/modules/accounts.ts index 60bad13b4..12e65e0e4 100644 --- a/frontend/src/api/modules/accounts.ts +++ b/frontend/src/api/modules/accounts.ts @@ -7,6 +7,7 @@ export interface AccountQuotaWindow { labelDisplay: string usedPercent: number | null usedPercentDisplay: string + limitReached: boolean localUsage?: unknown resetAtDisplay: string } @@ -102,7 +103,6 @@ export interface Account { hasRefreshToken: boolean status: string displayStatus: string - tokenRefreshing: boolean availability: string enabled: boolean accessTokenExpiresAt: string | null diff --git a/frontend/src/api/modules/dashboard.ts b/frontend/src/api/modules/dashboard.ts index 46d5088ea..e03fe0355 100644 --- a/frontend/src/api/modules/dashboard.ts +++ b/frontend/src/api/modules/dashboard.ts @@ -144,6 +144,7 @@ export interface DashboardAccountUsage { export interface DashboardPoolSummary { total: number active: number + rateLimited: number expired: number quotaExhausted: number refreshing: number | null diff --git a/frontend/src/views/accounts/composables/useAccountConnectionTest.ts b/frontend/src/views/accounts/composables/useAccountConnectionTest.ts index b898e7109..30f48f25d 100644 --- a/frontend/src/views/accounts/composables/useAccountConnectionTest.ts +++ b/frontend/src/views/accounts/composables/useAccountConnectionTest.ts @@ -230,7 +230,7 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions) testingAccount.value = { ...account, status, - displayStatus: account.tokenRefreshing ? account.displayStatus : status, + displayStatus: status, } options.onAccountStatus(account.id, status) } diff --git a/frontend/src/views/accounts/composables/useAccountMutations.ts b/frontend/src/views/accounts/composables/useAccountMutations.ts index 114f603b6..574e73adc 100644 --- a/frontend/src/views/accounts/composables/useAccountMutations.ts +++ b/frontend/src/views/accounts/composables/useAccountMutations.ts @@ -221,7 +221,7 @@ export function useAccountMutations(options: { ? { ...account, status, - displayStatus: account.tokenRefreshing ? 'refreshing' : status, + displayStatus: status, } : account, ) diff --git a/frontend/src/views/accounts/constants.ts b/frontend/src/views/accounts/constants.ts index 9abf22502..8015da660 100644 --- a/frontend/src/views/accounts/constants.ts +++ b/frontend/src/views/accounts/constants.ts @@ -128,7 +128,7 @@ export const statusLabels: Record = { disabled: '已禁用', banned: '已封禁', quota_exhausted: '配额耗尽', - refreshing: '刷新中', + rate_limited: '限流中', } export const statusTones: Record = { @@ -137,18 +137,33 @@ export const statusTones: Record window.limitReached) +} + export function visibleSummaryQuotaWindows(windows: AccountQuotaWindow[]) { const known = [...windows] .filter(window => summaryGroupOrder.has(window.group)) diff --git a/frontend/src/views/accounts/index.vue b/frontend/src/views/accounts/index.vue index feb87995c..86455bd3e 100644 --- a/frontend/src/views/accounts/index.vue +++ b/frontend/src/views/accounts/index.vue @@ -23,7 +23,7 @@ import { useAccountConnectionTest } from './composables/useAccountConnectionTest import { useAccountMutations } from './composables/useAccountMutations' import { useAccountsQuery } from './composables/useAccountsQuery' import { useAccountsTable } from './composables/useAccountsTable' -import { accountColumns } from './constants' +import { accountColumns, derivedAccountStatus } from './constants' const selectedIds = ref>(new Set()) const { @@ -204,7 +204,7 @@ const {