diff --git a/backend/crates/gateway-admin/src/lib.rs b/backend/crates/gateway-admin/src/lib.rs index 2ff747863..9c3e8981a 100644 --- a/backend/crates/gateway-admin/src/lib.rs +++ b/backend/crates/gateway-admin/src/lib.rs @@ -255,7 +255,6 @@ pub async fn initialize( let accounts = Arc::new(DefaultAccountsService::new( store.accounts(), - store.settings(), registry.clone(), snapshot.clone(), probe.clone(), diff --git a/backend/crates/gateway-admin/src/model/accounts.rs b/backend/crates/gateway-admin/src/model/accounts.rs index 1d4452ffe..84a44696e 100644 --- a/backend/crates/gateway-admin/src/model/accounts.rs +++ b/backend/crates/gateway-admin/src/model/accounts.rs @@ -1,6 +1,6 @@ //! 多 Provider 账号目录与连接测试的公共事实。 -use std::{pin::Pin, time::SystemTime}; +use std::pin::Pin; use chrono::{DateTime, Utc}; use futures::Stream; @@ -9,151 +9,10 @@ use gateway_core::routing::ProviderKind; use super::{PageSize, Revision, observability::TimeRange}; -/// Provider 账号当前可用性;权威定义与字符串编解码在 gateway-core,此处仅复用。 -pub use gateway_core::engine::credential::AccountAvailability; - -/// 管理页使用的归一化账号状态。 -/// -/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举, -/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AccountStatus { - /// 正常,可调度。 - Normal, - /// 配额耗尽(持久化终态),等待额度恢复,无需干预。 - QuotaExhausted, - /// 限流中(quota 快照级信号),自动恢复,无需干预。 - RateLimited, - /// 管理员手动停用(`enabled = false`)。 - Disabled, - /// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。 - Error, -} - -impl AccountStatus { - /// 返回稳定 wire 值。 - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Normal => "normal", - Self::QuotaExhausted => "quota_exhausted", - Self::RateLimited => "rate_limited", - Self::Disabled => "disabled", - Self::Error => "error", - } - } - - /// 解析稳定 wire 值。 - #[must_use] - pub fn parse(value: &str) -> Option { - match value { - "normal" => Some(Self::Normal), - "quota_exhausted" => Some(Self::QuotaExhausted), - "rate_limited" => Some(Self::RateLimited), - "disabled" => Some(Self::Disabled), - "error" => Some(Self::Error), - _ => None, - } - } -} - -/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AccountStatusSignals { - pub enabled: bool, - pub availability: AccountAvailability, - pub access_token_expired: bool, - pub quota_limit_reached: bool, - /// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。 - pub rate_limited_until: Option, -} - -impl AccountStatusSignals { - /// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。 - #[must_use] - pub const fn derive(self) -> AccountStatus { - if !self.enabled { - AccountStatus::Disabled - } else if matches!(self.availability, AccountAvailability::QuotaExhausted) { - AccountStatus::QuotaExhausted - } else if self.access_token_expired - || matches!( - self.availability, - AccountAvailability::Expired - | AccountAvailability::Invalid - | AccountAvailability::Banned - | AccountAvailability::Unknown - ) - { - AccountStatus::Error - } else if self.rate_limited_until.is_some() { - AccountStatus::RateLimited - } else if self.quota_limit_reached { - AccountStatus::QuotaExhausted - } else { - AccountStatus::Normal - } - } -} - -/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。 -/// -/// 受控枚举用于展示与聚合,扩展不改变状态分类。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AccountErrorReason { - Expired, - TokenExpired, - Invalid, - Banned, - Unknown, -} - -impl AccountErrorReason { - /// 返回稳定 wire 值。 - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Expired => "expired", - Self::TokenExpired => "token_expired", - Self::Invalid => "invalid", - Self::Banned => "banned", - Self::Unknown => "unknown", - } - } - - /// 解析稳定 wire 值。 - #[must_use] - pub fn parse(value: &str) -> Option { - match value { - "expired" => Some(Self::Expired), - "token_expired" => Some(Self::TokenExpired), - "invalid" => Some(Self::Invalid), - "banned" => Some(Self::Banned), - "unknown" => Some(Self::Unknown), - _ => None, - } - } -} - -/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。 -/// -/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。 -#[must_use] -pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option { - if !signals.enabled { - return None; - } - match signals.availability { - AccountAvailability::QuotaExhausted => None, - AccountAvailability::Ready => signals - .access_token_expired - .then_some(AccountErrorReason::TokenExpired), - AccountAvailability::Expired => Some(AccountErrorReason::Expired), - AccountAvailability::Invalid => Some(AccountErrorReason::Invalid), - AccountAvailability::Banned => Some(AccountErrorReason::Banned), - AccountAvailability::Unknown => Some(AccountErrorReason::Unknown), - } -} +pub use gateway_core::engine::credential::{ + AccountErrorReason, AccountStatus, AccountStatusFacts, AccountStatusProjection, + CredentialState, QuotaAccessState, QuotaEvidence, QuotaState, resolve_account_status, +}; /// 账号列表排序字段。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -207,11 +66,11 @@ pub struct AccountRecord { pub access_token_expires_at: Option>, pub next_refresh_at: Option>, pub enabled: bool, - pub availability: AccountAvailability, - pub availability_observed_at: DateTime, - /// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。 + pub credential_state: CredentialState, + pub credential_observed_at: DateTime, + pub quota: QuotaState, + pub last_error_reason: Option, pub last_error_message: Option, - pub quota_observed_at: Option>, pub created_at: DateTime, pub updated_at: DateTime, } @@ -296,11 +155,18 @@ pub struct AccountUsageWindowResult { #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountPage { pub config_revision: Revision, - pub items: Vec, + pub items: Vec, pub total: u64, pub summary: AccountSummary, } +/// 同一状态快照下的账号事实与唯一状态投影。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountPageItem { + pub account: AccountRecord, + pub projection: AccountStatusProjection, +} + /// 统一账号目录的全局状态计数,不受当前筛选和分页影响。 /// /// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。 @@ -342,9 +208,7 @@ pub enum AccountConnectionTestEvent { Content { text: String, }, - Completed { - account_status: AccountStatus, - }, + Completed, Failed { message: String, provider_error_code: Option, @@ -352,7 +216,6 @@ pub enum AccountConnectionTestEvent { upstream_status: Option, upstream_content_type: Option, upstream_body: Option, - account_status: AccountStatus, }, } diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index 018474966..00740dc09 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -439,25 +439,15 @@ pub struct RequestMetricPoint { pub costs: Vec, } -/// 账号池容量统计。 -/// -/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。 -/// -/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列, -/// 以便状态详情展示和诊断。 +/// 账号池的统一五态统计。 #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct AccountPoolMetrics { pub total: u64, - pub enabled: u64, - pub unavailable: u64, - pub active: u64, - pub rate_limited: u64, - pub expired: u64, - pub invalid: u64, + pub normal: u64, pub quota_exhausted: u64, - pub refreshing: Option, + pub rate_limited: u64, pub disabled: u64, - pub banned: u64, + pub error: u64, } /// Dashboard 中一个账号的模型级用量事实。 @@ -488,7 +478,7 @@ pub struct DashboardAccountRequestBucket { pub request_count: u64, } -/// Dashboard 中一个账号的完整用量与公共状态事实。 +/// Dashboard 中一个账号的完整用量事实。 #[derive(Debug, Clone, PartialEq)] pub struct DashboardAccountUsage { pub account_id: String, @@ -497,8 +487,6 @@ pub struct DashboardAccountUsage { pub name: String, pub email: Option, pub plan_type: Option, - pub enabled: bool, - pub availability: String, pub request_count: u64, pub success_count: u64, pub input_tokens: Option, diff --git a/backend/crates/gateway-admin/src/model/provider_credentials.rs b/backend/crates/gateway-admin/src/model/provider_credentials.rs index 3489bb012..970068f21 100644 --- a/backend/crates/gateway-admin/src/model/provider_credentials.rs +++ b/backend/crates/gateway-admin/src/model/provider_credentials.rs @@ -11,10 +11,7 @@ use gateway_core::{ use super::{ AdminError, MutationActor, MutationContext, PageSize, Revision, - accounts::{ - AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary, - AccountUsage, - }, + accounts::{AccountRecord, AccountSummary, AccountUsage, CredentialState}, }; /// Provider-owned JSON;公共层只搬运且 Debug 不输出值。 @@ -55,24 +52,24 @@ pub struct CredentialCursor { /// Provider credential 列表查询。 #[derive(Debug, Clone, PartialEq, Eq)] pub struct CredentialListQuery { - pub availability: Option, + pub credential_state: Option, pub enabled: Option, pub window: CredentialListWindow, } -/// Credential 可用性筛选;支持一个 wire 值对应多个持久状态。 +/// Credential 状态筛选。 #[derive(Debug, Clone, PartialEq, Eq)] -pub enum CredentialAvailabilityFilter { - Exact(AccountAvailability), - AnyOf(Vec), +pub enum CredentialStateFilter { + Exact(CredentialState), + AnyOf(Vec), } -impl CredentialAvailabilityFilter { +impl CredentialStateFilter { #[must_use] - pub fn matches(&self, availability: AccountAvailability) -> bool { + pub fn matches(&self, credential_state: CredentialState) -> bool { match self { - Self::Exact(expected) => *expected == availability, - Self::AnyOf(expected) => expected.contains(&availability), + Self::Exact(expected) => *expected == credential_state, + Self::AnyOf(expected) => expected.contains(&credential_state), } } } @@ -155,8 +152,8 @@ pub struct PreparedCredentialCreate { pub access_token_expires_at: Option>, pub next_refresh_at: Option>, pub enabled: bool, - pub availability: AccountAvailability, - pub availability_observed_at: DateTime, + pub credential_state: CredentialState, + pub credential_observed_at: DateTime, } /// Provider 对一份导入文档的完整验证结果。 @@ -603,15 +600,31 @@ pub struct ProviderQuota { pub observed_at: Option>, pub refresh_token_expires_at: Option>, pub windows: Vec, - /// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。 + /// 展示用快照级触顶事实(顶层或任一窗口触顶);不参与账号五态派生。 pub limit_reached: bool, - /// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。 - /// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。 - pub rate_limited_until: Option>, pub provider_data: Option, } impl ProviderQuota { + /// 返回账号展开面板使用的当前额度窗口用量。 + /// + /// 只选择具备完整时间边界且可按账号归属的窗口;优先级与 Dashboard + /// 代表性额度窗口一致,同一优先级保持 Provider 投影顺序。 + #[must_use] + pub fn representative_window_usage(&self) -> Option<&AccountUsage> { + self.windows + .iter() + .enumerate() + .filter(|(_, window)| { + window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide + && window.window_seconds.is_some() + && window.reset_at.is_some() + && window.local_usage.is_some() + }) + .min_by_key(|(index, window)| (quota_usage_priority(window), *index)) + .and_then(|(_, window)| window.local_usage.as_ref()) + } + /// 返回 Dashboard 使用的代表性额度比例。 /// /// 优先使用短周期窗口,再依次使用周、月和其它窗口;同一优先级取较高的已用比例。 @@ -703,9 +716,7 @@ impl fmt::Debug for ProviderExport { #[derive(Debug, Clone, PartialEq)] pub struct AccountDirectoryItem { pub account: AccountRecord, - pub status: AccountStatus, - /// `status == Error` 时的具体原因;其余状态为 `None`。 - pub error_reason: Option, + pub projection: gateway_core::engine::credential::AccountStatusProjection, pub usage: Option, pub quota: ProviderQuota, } diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index 7e8584fac..411f0072d 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -11,7 +11,7 @@ use super::backup::BackupStorePorts; use crate::model::{ MutationContext, Revision, accounts::{ - AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery, + AccountListQuery, AccountPage, AccountPageItem, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled, }, auth::{AdminAuditEvent, AdminSession}, @@ -83,7 +83,7 @@ pub type AdminStoreResult = Result; pub trait AccountStore: Send + Sync { async fn list_accounts(&self, query: AccountListQuery) -> AdminStoreResult; - async fn load_account(&self, account_id: &str) -> AdminStoreResult>; + async fn load_account(&self, account_id: &str) -> AdminStoreResult>; async fn load_account_usage( &self, diff --git a/backend/crates/gateway-admin/src/use_case/accounts.rs b/backend/crates/gateway-admin/src/use_case/accounts.rs index 24eae9c09..d57df6043 100644 --- a/backend/crates/gateway-admin/src/use_case/accounts.rs +++ b/backend/crates/gateway-admin/src/use_case/accounts.rs @@ -1,6 +1,6 @@ //! 统一账号目录与跨 Provider 动态分派。 -use std::{collections::BTreeMap, sync::Arc, time::SystemTime}; +use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; use chrono::{Duration, Utc}; @@ -18,8 +18,7 @@ use crate::{ AdminError, MutationContext, accounts::{ AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery, - AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery, - derive_error_reason, + AccountPageItem, AccountUsageWindowQuery, }, observability::TimeRange, provider_credentials::{ @@ -28,10 +27,7 @@ use crate::{ ProviderQuotaWindow, QuotaLocalUsageAttribution, }, }, - ports::{ - provider::ProviderAdminRegistry, - store::{AccountStore, SettingsStore}, - }, + ports::{provider::ProviderAdminRegistry, store::AccountStore}, }; use super::{ @@ -79,7 +75,6 @@ pub trait AccountsService: Send + Sync { pub(crate) struct DefaultAccountsService { accounts: Arc, - settings: Arc, providers: ProviderAdminRegistry, snapshot: Arc, probe: Arc, @@ -89,14 +84,12 @@ impl DefaultAccountsService { #[must_use] pub(crate) const fn new( accounts: Arc, - settings: Arc, providers: ProviderAdminRegistry, snapshot: Arc, probe: Arc, ) -> Self { Self { accounts, - settings, providers, snapshot, probe, @@ -106,7 +99,7 @@ impl DefaultAccountsService { async fn load_account( &self, account_id: &ProviderAccountId, - ) -> Result { + ) -> Result { self.accounts .load_account(account_id.as_str()) .await @@ -119,32 +112,32 @@ impl DefaultAccountsService { account_id: &ProviderAccountId, ) -> Result< ( - AccountRecord, + AccountPageItem, Arc, ), AdminError, > { - let account = self.load_account(account_id).await?; + let item = self.load_account(account_id).await?; let provider = self .providers - .require(&account.provider_kind) + .require(&item.account.provider_kind) .map_err(|error| map_provider_error(error, "provider account"))?; - Ok((account, provider)) + Ok((item, provider)) } async fn attach_quota_local_usage( &self, - accounts: &[AccountRecord], + accounts: &[AccountPageItem], quotas: &mut [ProviderQuota], ) -> Result<(), AdminError> { let windows = accounts .iter() .zip(quotas.iter()) - .flat_map(|(account, quota)| { + .flat_map(|(item, quota)| { quota .windows .iter() - .filter_map(|window| quota_usage_window(&account.id, window)) + .filter_map(|window| quota_usage_window(&item.account.id, window)) }) .collect::>(); if windows.is_empty() { @@ -158,12 +151,12 @@ impl DefaultAccountsService { .into_iter() .map(|result| ((result.account_id, result.key), result.usage)) .collect::>(); - for (account, quota) in accounts.iter().zip(quotas) { + for (item, quota) in accounts.iter().zip(quotas) { for window in &mut quota.windows { if window.local_usage.is_none() && window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide { - let key = (account.id.clone(), window.key.clone()); + let key = (item.account.id.clone(), window.key.clone()); if let Some(usage) = usage_by_window.get(&key) { window.local_usage = Some(usage.clone()); } @@ -178,33 +171,19 @@ impl DefaultAccountsService { account_id: &ProviderAccountId, refresh_quota: bool, ) -> Result { - let (account, provider) = self.provider_for_account(account_id).await?; - let settings = self - .settings - .load_runtime_settings() - .await - .map_err(|error| map_store_error(error, "runtime settings"))?; + let (stored, provider) = self.provider_for_account(account_id).await?; + let account = &stored.account; let now = Utc::now(); - let retained_range = retained_usage_range(now, settings.usage_retention_days); let rolling_range = TimeRange { start: now - Duration::hours(24), end: now, }; let ids = vec![account.id.clone()]; - let (usage, rolling_usage) = futures::try_join!( - async { - self.accounts - .load_account_usage(retained_range, &ids) - .await - .map_err(|error| map_store_error(error, "account usage")) - }, - async { - self.accounts - .load_account_usage(rolling_range, &ids) - .await - .map_err(|error| map_store_error(error, "rolling account usage")) - }, - )?; + let rolling_usage = self + .accounts + .load_account_usage(rolling_range, &ids) + .await + .map_err(|error| map_store_error(error, "rolling account usage"))?; let rolling_usage = rolling_usage.into_iter().next(); let mut quota = provider .quota(ProviderQuotaRequest { @@ -215,17 +194,15 @@ impl DefaultAccountsService { .await .map_err(|error| map_provider_error(error, "provider quota"))?; self.attach_quota_local_usage( - std::slice::from_ref(&account), + std::slice::from_ref(&stored), std::slice::from_mut(&mut quota), ) .await?; - let signals = account_signals(&account, "a, now); - let error_reason = derive_error_reason(&signals); + let usage = quota.representative_window_usage().cloned(); Ok(AccountDirectoryItem { - status: signals.derive(), - error_reason, - usage: usage.into_iter().next(), - account, + projection: stored.projection, + usage, + account: stored.account, quota, }) } @@ -234,13 +211,12 @@ impl DefaultAccountsService { #[async_trait] impl AccountsService for DefaultAccountsService { async fn list(&self, query: AccountListQuery) -> Result { - let (page, settings) = futures::try_join!( - self.accounts.list_accounts(query), - self.settings.load_runtime_settings(), - ) - .map_err(|error| map_store_error(error, "account directory"))?; + let page = self + .accounts + .list_accounts(query) + .await + .map_err(|error| map_store_error(error, "account directory"))?; let now = Utc::now(); - let retained_range = retained_usage_range(now, settings.usage_retention_days); let rolling_range = TimeRange { start: now - Duration::hours(24), end: now, @@ -248,22 +224,19 @@ impl AccountsService for DefaultAccountsService { let ids = page .items .iter() - .map(|account| account.id.clone()) + .map(|item| item.account.id.clone()) .collect::>(); - let (usage, rolling_usage) = futures::try_join!( - self.accounts.load_account_usage(retained_range, &ids), - self.accounts.load_account_usage(rolling_range, &ids), - ) - .map_err(|error| map_store_error(error, "account usage"))?; - let usage = usage - .into_iter() - .map(|usage| (usage.account_id.clone(), usage)) - .collect::>(); + let rolling_usage = self + .accounts + .load_account_usage(rolling_range, &ids) + .await + .map_err(|error| map_store_error(error, "rolling account usage"))?; let rolling_usage = rolling_usage .into_iter() .map(|usage| (usage.account_id.clone(), usage)) .collect::>(); - let mut quotas = futures::future::join_all(page.items.iter().map(|account| async { + let mut quotas = futures::future::join_all(page.items.iter().map(|item| async { + let account = &item.account; let account_id = ProviderAccountId::new(account.id.clone()) .map_err(|_| AdminError::invalid("Invalid provider account ID"))?; // 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页: @@ -307,14 +280,12 @@ impl AccountsService for DefaultAccountsService { .items .into_iter() .zip(quotas) - .map(|(account, quota)| { - let signals = account_signals(&account, "a, now); - let error_reason = derive_error_reason(&signals); + .map(|(item, quota)| { + let usage = quota.representative_window_usage().cloned(); AccountDirectoryItem { - status: signals.derive(), - error_reason, - usage: usage.get(&account.id).cloned(), - account, + usage, + account: item.account, + projection: item.projection, quota, } }) @@ -342,7 +313,7 @@ impl AccountsService for DefaultAccountsService { for account_id in account_ids { let account = self.load_account(&account_id).await?; grouped - .entry(account.provider_kind) + .entry(account.account.provider_kind) .or_default() .push(account_id); } @@ -385,7 +356,8 @@ impl AccountsService for DefaultAccountsService { context: &MutationContext, account_id: ProviderAccountId, ) -> Result { - let (account, provider) = self.provider_for_account(&account_id).await?; + let (stored, provider) = self.provider_for_account(&account_id).await?; + let account = stored.account; let prepared = provider .prepare_refresh(PrepareCredentialRefresh { account: account.clone(), @@ -436,8 +408,8 @@ impl AccountsService for DefaultAccountsService { account_id: ProviderAccountId, upstream_model: UpstreamModelId, ) -> Result { - let (account, provider) = self.provider_for_account(&account_id).await?; - let initial_status = account_status(&account, false, Utc::now()); + let (stored, provider) = self.provider_for_account(&account_id).await?; + let account = stored.account; let model = upstream_model.as_str().to_owned(); let operation = provider .connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT) @@ -454,8 +426,6 @@ impl AccountsService for DefaultAccountsService { }, ]; let probe = Arc::clone(&self.probe); - let accounts = Arc::clone(&self.accounts); - let stored_account_id = account.id.clone(); let terminal = futures::stream::once(async move { let result = probe .probe(AccountProbeRequest { @@ -465,22 +435,12 @@ impl AccountsService for DefaultAccountsService { operation, }) .await; - let status = accounts - .load_account(&stored_account_id) - .await - .ok() - .flatten() - .map_or(initial_status, |account| { - account_status(&account, false, Utc::now()) - }); match result { Ok(result) => result .text .into_iter() .map(|text| AccountConnectionTestEvent::Content { text }) - .chain(std::iter::once(AccountConnectionTestEvent::Completed { - account_status: status, - })) + .chain(std::iter::once(AccountConnectionTestEvent::Completed)) .collect(), Err(error) => { let upstream_status = error @@ -506,7 +466,6 @@ impl AccountsService for DefaultAccountsService { upstream_status, upstream_content_type, upstream_body, - account_status: status, }] } } @@ -523,49 +482,10 @@ fn empty_quota() -> ProviderQuota { refresh_token_expires_at: None, windows: Vec::new(), limit_reached: false, - rate_limited_until: None, provider_data: None, } } -/// 构造状态派生所需的信号快照。 -fn account_signals( - account: &AccountRecord, - quota: &ProviderQuota, - now: chrono::DateTime, -) -> AccountStatusSignals { - AccountStatusSignals { - enabled: account.enabled, - availability: account.availability, - access_token_expired: account - .access_token_expires_at - .is_some_and(|expires_at| expires_at <= now), - quota_limit_reached: quota.limit_reached, - rate_limited_until: quota.rate_limited_until.map(SystemTime::from), - } -} - -/// 账号运营分类(互斥展示状态)。 -fn account_status( - account: &AccountRecord, - quota_limit_reached: bool, - now: chrono::DateTime, -) -> AccountStatus { - let quota = ProviderQuota { - limit_reached: quota_limit_reached, - ..empty_quota() - }; - account_signals(account, "a, now).derive() -} - -fn retained_usage_range(now: chrono::DateTime, retention_days: u32) -> TimeRange { - // 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。 - TimeRange { - start: now - Duration::days(i64::from(retention_days)), - end: now, - } -} - fn quota_usage_window( account_id: &str, window: &ProviderQuotaWindow, diff --git a/backend/crates/gateway-admin/src/use_case/observability.rs b/backend/crates/gateway-admin/src/use_case/observability.rs index eaa809890..a116f81b6 100644 --- a/backend/crates/gateway-admin/src/use_case/observability.rs +++ b/backend/crates/gateway-admin/src/use_case/observability.rs @@ -127,7 +127,7 @@ impl ObservabilityService for DefaultObservabilityService { let max_concurrent_per_account = u64::from(settings.max_concurrent_per_account); let active_accounts = runtime_slots .as_ref() - .map_or(observation.provider_accounts.active, |slots| { + .map_or(observation.provider_accounts.normal, |slots| { slots.active_accounts }); let total_slots = active_accounts.saturating_mul(max_concurrent_per_account); @@ -484,8 +484,7 @@ impl DefaultObservabilityService { }); let quota_used = futures::future::join_all(quota_reads).await; for (account, used_percent) in accounts.iter_mut().zip(quota_used) { - account.quota_used_percent = used_percent - .or_else(|| (account.availability == "quota_exhausted").then_some(100.0)); + account.quota_used_percent = used_percent; } } diff --git a/backend/crates/gateway-admin/tests/main.rs b/backend/crates/gateway-admin/tests/main.rs index ee8b8d334..735ff1fe1 100644 --- a/backend/crates/gateway-admin/tests/main.rs +++ b/backend/crates/gateway-admin/tests/main.rs @@ -129,7 +129,6 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() { quota_window("shortTerm", Some(14_400), Some(45.0)), ], limit_reached: false, - rate_limited_until: None, provider_data: None, }; diff --git a/backend/crates/gateway-admin/tests/use_case/accounts.rs b/backend/crates/gateway-admin/tests/use_case/accounts.rs index 4f1e97377..43b4d1fec 100644 --- a/backend/crates/gateway-admin/tests/use_case/accounts.rs +++ b/backend/crates/gateway-admin/tests/use_case/accounts.rs @@ -5,7 +5,10 @@ use chrono::{TimeDelta, Utc}; use futures::{StreamExt as _, future::BoxFuture}; use gateway_core::{ engine::{ - credential::{OpaqueProviderData, ProviderAccountId}, + credential::{ + AccountStatusFacts, CredentialState, OpaqueProviderData, ProviderAccountId, + QuotaEvidence, QuotaState, resolve_account_status, + }, probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult}, }, error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind}, @@ -18,7 +21,7 @@ use gateway_admin::{ model::{ AdminError, MutationContext, Revision, accounts::{ - AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage, + AccountConnectionTestEvent, AccountListQuery, AccountPage, AccountPageItem, AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled, }, @@ -378,6 +381,7 @@ pub(super) struct FakeAccountStore { fail_commit: Mutex, audit_requests: Mutex>, quota_window_usage: Mutex>, + quota_window_queries: Mutex>, } impl FakeAccountStore { @@ -393,6 +397,7 @@ impl FakeAccountStore { fail_commit: Mutex::new(false), audit_requests: Mutex::new(Vec::new()), quota_window_usage: Mutex::new(Vec::new()), + quota_window_queries: Mutex::new(Vec::new()), }) } @@ -408,6 +413,13 @@ impl FakeAccountStore { *self.quota_window_usage.lock().expect("quota window usage") = usage; } + fn quota_window_queries(&self) -> Vec { + self.quota_window_queries + .lock() + .expect("quota window queries") + .clone() + } + pub(super) fn set_accounts(&self, accounts: Vec) { *self.accounts.lock().expect("accounts") = accounts; } @@ -438,6 +450,22 @@ impl FakeAccountStore { .expect("audit requests") .push(context.request_id.clone()); } + + fn page_item(account: AccountRecord) -> AccountPageItem { + let facts = AccountStatusFacts { + enabled: account.enabled, + credential_state: account.credential_state, + access_token_expires_at: account.access_token_expires_at.map(Into::into), + quota: account.quota, + rate_limited_until: None, + last_error_reason: account.last_error_reason, + last_error_message: account.last_error_message.clone(), + }; + AccountPageItem { + account, + projection: resolve_account_status(&facts, std::time::SystemTime::now()), + } + } } #[async_trait] @@ -445,12 +473,13 @@ impl AccountStore for FakeAccountStore { async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult { self.record("store.list_accounts"); let accounts = self.accounts.lock().expect("accounts").clone(); + let total = accounts.len() as u64; Ok(AccountPage { config_revision: revision(1), - items: accounts.clone(), - total: accounts.len() as u64, + items: accounts.into_iter().map(Self::page_item).collect(), + total, summary: AccountSummary { - total: accounts.len() as u64, + total, normal: 1, quota_exhausted: 0, rate_limited: 0, @@ -460,7 +489,7 @@ impl AccountStore for FakeAccountStore { }) } - async fn load_account(&self, account_id: &str) -> AdminStoreResult> { + async fn load_account(&self, account_id: &str) -> AdminStoreResult> { self.record("store.load_account"); // probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。 let account = self @@ -477,7 +506,7 @@ impl AccountStore for FakeAccountStore { .find(|account| account.id == account_id) .cloned() }); - Ok(account) + Ok(account.map(Self::page_item)) } async fn load_account_usage( @@ -490,8 +519,12 @@ impl AccountStore for FakeAccountStore { async fn load_account_usage_by_windows( &self, - _: &[AccountUsageWindowQuery], + windows: &[AccountUsageWindowQuery], ) -> AdminStoreResult> { + *self + .quota_window_queries + .lock() + .expect("quota window queries") = windows.to_vec(); Ok(self .quota_window_usage .lock() @@ -815,7 +848,11 @@ fn provider_registry_should_reject_duplicate_kind() { async fn connection_test_should_probe_unavailable_account() { let provider = FakeProviderAdmin::new("xai", events()); let mut account = account_record("xai"); - account.availability = AccountAvailability::QuotaExhausted; + account.quota = QuotaState::exhausted( + QuotaEvidence::UsageLimitReached, + std::time::SystemTime::now(), + None, + ); let store = FakeAccountStore::with_account(account, events()); let services = accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await; @@ -837,7 +874,6 @@ async fn connection_test_should_probe_unavailable_account() { message, provider_error_code: Some(code), provider_error_type: Some(error_type), - account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted, .. }) if message == "included usage exhausted" && code == "usage_exhausted" @@ -846,7 +882,7 @@ async fn connection_test_should_probe_unavailable_account() { } #[tokio::test] -async fn connection_test_rate_limited_probe_preserves_normal_account_status() { +async fn connection_test_rate_limited_probe_returns_provider_failure() { let events = events(); let provider = FakeProviderAdmin::new("xai", events.clone()); let store = FakeAccountStore::new("xai", events); @@ -869,9 +905,10 @@ async fn connection_test_rate_limited_probe_preserves_normal_account_status() { assert!(matches!( events.last(), Some(AccountConnectionTestEvent::Failed { - account_status: gateway_admin::model::accounts::AccountStatus::Normal, + message, .. }) + if message == "xAI free model quota is exhausted" )); } @@ -897,10 +934,7 @@ async fn connection_test_should_preserve_disabled_account_status() { assert!(matches!( events.last(), - Some(AccountConnectionTestEvent::Failed { - account_status: gateway_admin::model::accounts::AccountStatus::Disabled, - .. - }) + Some(AccountConnectionTestEvent::Failed { .. }) )); } @@ -949,7 +983,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() { assert_eq!(result.config_revision, revision(2)); assert_eq!(result.account.account.provider_kind.as_str(), "openai"); assert_eq!( - result.account.status, + result.account.projection.status, gateway_admin::model::accounts::AccountStatus::Normal ); @@ -990,7 +1024,7 @@ async fn accounts_list_should_return_complete_directory_semantics() { let account = page.items.first().expect("account item"); assert_eq!(account.account.provider_kind.as_str(), "openai"); assert_eq!( - account.status, + account.projection.status, gateway_admin::model::accounts::AccountStatus::Normal ); } @@ -1016,7 +1050,6 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop provider_data: None, }], limit_reached: false, - rate_limited_until: None, provider_data: None, }); let failing = FakeProviderAdmin::new("xai", events.clone()); @@ -1064,10 +1097,14 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop } #[tokio::test] -async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() { +async fn accounts_list_should_prefer_credential_error_over_quota_exhaustion() { let provider = FakeProviderAdmin::new("openai", events()); let mut account = account_record("openai"); - account.availability = AccountAvailability::QuotaExhausted; + account.quota = QuotaState::exhausted( + QuotaEvidence::ProviderDenied, + std::time::SystemTime::now(), + None, + ); account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5)); let store = FakeAccountStore::with_account(account, events()); @@ -1086,17 +1123,17 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() .expect("usage-limited account directory"); assert_eq!( - page.items.first().expect("account item").status, - gateway_admin::model::accounts::AccountStatus::QuotaExhausted, + page.items.first().expect("account item").projection.status, + gateway_admin::model::accounts::AccountStatus::Error, ); } #[tokio::test] -async fn accounts_list_should_map_unknown_availability_to_error_not_normal() { +async fn accounts_list_should_map_unknown_credential_to_error_not_normal() { // Unknown 不可调度,Admin 不得显示为 normal。 let provider = FakeProviderAdmin::new("openai", events()); let mut account = account_record("openai"); - account.availability = AccountAvailability::Unknown; + account.credential_state = CredentialState::Unknown; let store = FakeAccountStore::with_account(account, events()); let page = accounts_service(provider, store) @@ -1114,15 +1151,13 @@ async fn accounts_list_should_map_unknown_availability_to_error_not_normal() { .expect("unknown account directory"); assert_eq!( - page.items.first().expect("account item").status, + page.items.first().expect("account item").projection.status, gateway_admin::model::accounts::AccountStatus::Error, ); } #[tokio::test] -async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() { - // ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽); - // rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。 +async fn accounts_list_should_not_derive_rate_limited_from_provider_quota_view() { let provider = FakeProviderAdmin::new("openai", events()); provider.set_quota(ProviderQuota { observed_at: Some(Utc::now()), @@ -1141,7 +1176,6 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() { provider_data: None, }], limit_reached: true, - rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)), provider_data: None, }); let store = FakeAccountStore::new("openai", events()); @@ -1158,17 +1192,16 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() { sort: None, }) .await - .expect("rate limited account directory"); + .expect("account directory with display quota cooldown"); assert_eq!( - page.items.first().expect("account item").status, - gateway_admin::model::accounts::AccountStatus::RateLimited, + page.items.first().expect("account item").projection.status, + gateway_admin::model::accounts::AccountStatus::Normal, ); } #[tokio::test] -async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() { - // ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。 +async fn accounts_list_should_not_derive_exhaustion_from_provider_quota_view() { let provider = FakeProviderAdmin::new("openai", events()); provider.set_quota(ProviderQuota { observed_at: Some(Utc::now()), @@ -1187,7 +1220,6 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() { provider_data: None, }], limit_reached: true, - rate_limited_until: None, provider_data: None, }); let store = FakeAccountStore::new("openai", events()); @@ -1204,11 +1236,11 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() { sort: None, }) .await - .expect("quota exhausted account directory"); + .expect("account directory with display quota limit"); assert_eq!( - page.items.first().expect("account item").status, - gateway_admin::model::accounts::AccountStatus::QuotaExhausted, + page.items.first().expect("account item").projection.status, + gateway_admin::model::accounts::AccountStatus::Normal, ); } @@ -1233,7 +1265,6 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() { provider_data: None, }], limit_reached: false, - rate_limited_until: None, provider_data: None, }); let store = FakeAccountStore::new("openai", events()); @@ -1243,7 +1274,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() { usage: quota_local_usage("acct_test", 4_330_000), }]); - let page = accounts_service(provider, store) + let page = accounts_service(provider, store.clone()) .await .accounts() .list(AccountListQuery { @@ -1257,11 +1288,23 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() { .await .expect("quota window usage"); - let usage = page.items[0].quota.windows[0] + let item = &page.items[0]; + let usage = item.quota.windows[0] .local_usage .as_ref() .expect("quota window local usage"); assert_eq!(usage.total_tokens, Some(4_330_000)); + assert_eq!( + item.usage.as_ref().and_then(|usage| usage.total_tokens), + Some(4_330_000), + ); + + let queries = store.quota_window_queries(); + assert_eq!(queries.len(), 1); + assert_eq!(queries[0].account_id, "acct_test"); + assert_eq!(queries[0].key, "primary"); + assert_eq!(queries[0].range.start, reset_at - TimeDelta::hours(5)); + assert_eq!(queries[0].range.end, reset_at); } #[tokio::test] @@ -1300,7 +1343,6 @@ async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_w }, ], limit_reached: false, - rate_limited_until: None, provider_data: None, }); let store = FakeAccountStore::new("openai", events()); @@ -1352,6 +1394,13 @@ async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_w ("codex-bengalfox-primary", None), ], ); + assert_eq!( + page.items[0] + .usage + .as_ref() + .and_then(|usage| usage.total_tokens), + Some(12_818_806), + ); } #[tokio::test] @@ -1432,7 +1481,6 @@ fn empty_quota() -> ProviderQuota { refresh_token_expires_at: None, windows: Vec::new(), limit_reached: false, - rate_limited_until: None, provider_data: None, } } @@ -1476,10 +1524,11 @@ pub(super) fn account_record(kind: &str) -> AccountRecord { access_token_expires_at: Some(now + TimeDelta::hours(1)), next_refresh_at: Some(now + TimeDelta::minutes(30)), enabled: true, - availability: AccountAvailability::Ready, - availability_observed_at: now, + credential_state: CredentialState::Ready, + credential_observed_at: now, + quota: QuotaState::allowed(now.into()), + last_error_reason: None, last_error_message: None, - quota_observed_at: None, created_at: now, updated_at: now, } @@ -1513,8 +1562,8 @@ fn prepared_create_with_id( access_token_expires_at: Some(now + TimeDelta::hours(1)), next_refresh_at: Some(now + TimeDelta::minutes(30)), enabled: true, - availability: AccountAvailability::Ready, - availability_observed_at: now, + credential_state: CredentialState::Ready, + credential_observed_at: now, } } @@ -1597,9 +1646,7 @@ impl AccountProbe for FreeModelQuotaProbe { _: AccountProbeRequest, ) -> BoxFuture<'_, Result> { Box::pin(async move { - let mut account = account_record("xai"); - account.availability = AccountAvailability::Ready; - self.store.set_account_after_probe(account); + self.store.set_account_after_probe(account_record("xai")); Err(GatewayError::new( GatewayErrorKind::RateLimited, "xAI free model quota is exhausted", diff --git a/backend/crates/gateway-admin/tests/use_case/mod.rs b/backend/crates/gateway-admin/tests/use_case/mod.rs index 576ff5ae5..c7b1b196e 100644 --- a/backend/crates/gateway-admin/tests/use_case/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/mod.rs @@ -20,7 +20,7 @@ use gateway_admin::{ model::{ MutationContext, Revision, accounts::{ - AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery, + AccountListQuery, AccountPage, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled, }, auth::{AdminAuditEvent, AdminSession}, @@ -240,7 +240,10 @@ impl AccountStore for UnavailableStore { Err(unavailable("accounts")) } - async fn load_account(&self, _: &str) -> AdminStoreResult> { + async fn load_account( + &self, + _: &str, + ) -> AdminStoreResult> { Err(unavailable("account")) } diff --git a/backend/crates/gateway-api/src/admin/accounts.rs b/backend/crates/gateway-api/src/admin/accounts.rs index 7d0fa5ac0..254f3b62f 100644 --- a/backend/crates/gateway-api/src/admin/accounts.rs +++ b/backend/crates/gateway-api/src/admin/accounts.rs @@ -37,7 +37,7 @@ use gateway_core::{ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use super::presenter::{format_compact_number, format_number}; +use super::presenter::{format_compact_number, format_decimal_currency, format_number}; use super::{ AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta, WireValidationError, @@ -185,12 +185,10 @@ pub struct AccountView { pub authentication_kind: String, pub has_refresh_token: bool, pub status: String, - pub display_status: String, /// `status == "error"` 时的具体原因;其余状态为 `null`。 pub error_reason: Option, /// 最近一次失败的上游错误描述;仅错误状态存在。 pub error_message: Option, - pub availability: String, pub enabled: bool, pub access_token_expires_at: Option, pub access_token_expires_at_display: Option, @@ -522,10 +520,9 @@ impl From for AccountConnectionTestEvent { DomainConnectionTestEvent::Content { text } => { serde_json::json!({ "type": "content", "text": text }) } - DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({ + DomainConnectionTestEvent::Completed => serde_json::json!({ "type": "test_complete", - "success": true, - "accountStatus": account_status.as_str() + "success": true }), DomainConnectionTestEvent::Failed { message, @@ -534,7 +531,6 @@ impl From for AccountConnectionTestEvent { upstream_status, upstream_content_type, upstream_body, - account_status, } => serde_json::json!({ "type": "error", "error": message, @@ -542,8 +538,7 @@ impl From for AccountConnectionTestEvent { "providerErrorType": provider_error_type, "upstreamStatus": upstream_status, "upstreamContentType": upstream_content_type, - "upstreamBody": upstream_body, - "accountStatus": account_status.as_str() + "upstreamBody": upstream_body }), }; Self { data } @@ -1365,16 +1360,16 @@ fn account_models_data(result: ProviderModels) -> AccountModelsData { fn account_view(item: AccountDirectoryItem, now: DateTime) -> AccountView { let AccountDirectoryItem { account, - status, - error_reason, + projection, usage, quota, } = item; - let status = status.as_str().to_owned(); + let status = projection.status.as_str().to_owned(); + let rate_limited_until = projection.rate_limited_until.map(DateTime::::from); let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339); let added_at = china_rfc3339(&account.created_at); let updated_at = china_rfc3339(&account.updated_at); - let (quota, refresh_token_expires_at) = account_quota_view(quota, now); + let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now); AccountView { id: account.id.clone(), name: account.name, @@ -1387,11 +1382,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime) -> AccountView { plan_type: account.plan_type, authentication_kind: account.authentication_kind, has_refresh_token: account.has_refresh_token, - status: status.clone(), - display_status: status, - error_reason: error_reason.map(|reason| reason.as_str().to_owned()), - error_message: account.last_error_message, - availability: account.availability.as_str().to_owned(), + status, + error_reason: projection + .error_reason + .map(|reason| reason.as_str().to_owned()), + error_message: projection.error_message, enabled: account.enabled, access_token_expires_at: expires_at, access_token_expires_at_display: account @@ -1411,6 +1406,7 @@ fn account_view(item: AccountDirectoryItem, now: DateTime) -> AccountView { fn account_quota_view( quota: ProviderQuota, + rate_limited_until: Option>, now: DateTime, ) -> (AccountQuotaView, Option) { let refresh_token_expires_at = quota @@ -1420,7 +1416,7 @@ fn account_quota_view( .observed_at .map_or_else(|| "—".to_owned(), |value| relative_time(value, now)); let windows = quota.windows.into_iter().map(quota_window_view).collect(); - let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until)); + let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until)); ( AccountQuotaView { refreshed_at_display, @@ -1590,7 +1586,7 @@ fn account_model_usage_view(usage: AccountModelUsage, now: DateTime) -> Mod billing_amount_usd: usd.map(|cost| cost.amount.as_str().to_owned()), billing_amount_usd_display: usd.map_or_else( || "—".to_owned(), - |cost| format!("${}", cost.amount.as_str()), + |cost| format_decimal_currency(cost.amount.as_str(), "USD"), ), cost_estimate_status: cost_estimate_status.to_owned(), known_cost_count: known_count, diff --git a/backend/crates/gateway-api/src/admin/observability/mod.rs b/backend/crates/gateway-api/src/admin/observability/mod.rs index c1cfceec5..5c917eab5 100644 --- a/backend/crates/gateway-api/src/admin/observability/mod.rs +++ b/backend/crates/gateway-api/src/admin/observability/mod.rs @@ -15,7 +15,7 @@ use gateway_admin::model::{PageSize as DomainPageSize, observability as domain}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use super::presenter::{format_compact_number, format_number}; +use super::presenter::{format_compact_number, format_decimal_currency, format_number}; use super::{ AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta, WireValidationError, wire::map_admin_service_error, diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs index 820e542e2..17dbaebb0 100644 --- a/backend/crates/gateway-api/src/admin/observability/presenter.rs +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -127,20 +127,6 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView { } } -pub(crate) fn format_decimal_currency(amount: &str, currency: &str) -> String { - if currency == "USD" { - let value = amount.parse::().map_or(0.0, |value| value); - let precision = if value != 0.0 && value.abs() < 0.01 { - 4 - } else { - 2 - }; - format!("${value:.precision$}") - } else { - format!("{currency} {amount}") - } -} - pub(crate) fn format_money(cost: &domain::CurrencyCost) -> String { format_decimal_currency(cost.amount.as_str(), &cost.currency) } @@ -747,10 +733,16 @@ pub(crate) fn dashboard_view( credentials: DashboardCredentialsCardView { total: format_compact_number(provider_accounts.total), total_value: provider_accounts.total, - available: format_compact_number(provider_accounts.active), - available_value: provider_accounts.active, - unavailable: format_compact_number(provider_accounts.unavailable), - unavailable_value: provider_accounts.unavailable, + available: format_compact_number(provider_accounts.normal), + available_value: provider_accounts.normal, + unavailable: format_compact_number( + provider_accounts + .total + .saturating_sub(provider_accounts.normal), + ), + unavailable_value: provider_accounts + .total + .saturating_sub(provider_accounts.normal), }, traffic: DashboardTrafficCardView { today_requests: format_compact_number(today.request_count), @@ -783,14 +775,11 @@ pub(crate) fn dashboard_view( usage_records: recent_requests.into_iter().map(usage_record_view).collect(), pool_summary: DashboardPoolSummaryView { total: provider_accounts.total, - active: provider_accounts.active, - rate_limited: provider_accounts.rate_limited, - expired: provider_accounts.expired, - invalid: provider_accounts.invalid, + normal: provider_accounts.normal, quota_exhausted: provider_accounts.quota_exhausted, - refreshing: provider_accounts.refreshing, + rate_limited: provider_accounts.rate_limited, disabled: provider_accounts.disabled, - banned: provider_accounts.banned, + error: provider_accounts.error, }, capacity_info: DashboardCapacityInfoView { max_concurrent_per_account: capacity.max_concurrent_per_account, diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs index 04a944ae1..f91fbc86f 100644 --- a/backend/crates/gateway-api/src/admin/observability/wire.rs +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -347,14 +347,11 @@ pub struct DashboardAccountRequestBucketView { #[serde(rename_all = "camelCase")] pub struct DashboardPoolSummaryView { pub total: u64, - pub active: u64, - pub rate_limited: u64, - pub expired: u64, - pub invalid: u64, + pub normal: u64, pub quota_exhausted: u64, - pub refreshing: Option, + pub rate_limited: u64, pub disabled: u64, - pub banned: u64, + pub error: u64, } /// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。 diff --git a/backend/crates/gateway-api/src/admin/presenter.rs b/backend/crates/gateway-api/src/admin/presenter.rs index e8838a3c3..b80cf114f 100644 --- a/backend/crates/gateway-api/src/admin/presenter.rs +++ b/backend/crates/gateway-api/src/admin/presenter.rs @@ -3,6 +3,8 @@ //! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终 //! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。 +use gateway_core::accounting::Decimal; + /// 千分位格式化。 #[must_use] pub fn format_number(value: u64) -> String { @@ -37,3 +39,33 @@ pub fn format_compact_number(value: u64) -> String { } format_number(value) } + +/// 货币展示格式化。USD 常规金额保留两位,小于 1 美元时最多保留四位。 +#[must_use] +pub fn format_decimal_currency(amount: &str, currency: &str) -> String { + if currency != "USD" { + return format!("{currency} {amount}"); + } + + const DECIMAL_SCALE: u128 = 10_000_000_000; + let scaled = amount.parse::().unwrap_or_default().scaled(); + let precision = if scaled != 0 && scaled < DECIMAL_SCALE { + 4_u32 + } else { + 2_u32 + }; + let rounding_unit = 10_u128.pow(10 - precision); + let rounded = (scaled + rounding_unit / 2) / rounding_unit; + let display_scale = 10_u128.pow(precision); + let whole = rounded / display_scale; + let fraction = rounded % display_scale; + let mut display = format!("{whole}.{fraction:0width$}", width = precision as usize); + while display.ends_with('0') + && display + .split_once('.') + .is_some_and(|(_, fraction)| fraction.len() > 2) + { + display.pop(); + } + format!("${display}") +} diff --git a/backend/crates/gateway-api/tests/admin/accounts.rs b/backend/crates/gateway-api/tests/admin/accounts.rs index 7a331db5c..79fa600c3 100644 --- a/backend/crates/gateway-api/tests/admin/accounts.rs +++ b/backend/crates/gateway-api/tests/admin/accounts.rs @@ -72,6 +72,7 @@ mod query { mod response { use gateway_api::admin::accounts::AccountUsageView; + use gateway_api::admin::presenter::format_decimal_currency; #[test] fn account_usage_view_should_keep_unobserved_numbers_null() { @@ -112,12 +113,21 @@ mod response { assert!(value["totalTokens"].is_null()); assert_eq!(value["createdTokensDisplay"], "-"); } + + #[test] + fn account_usage_currency_should_limit_usd_to_four_fraction_digits() { + assert_eq!(format_decimal_currency("0.1204956", "USD"), "$0.1205"); + assert_eq!(format_decimal_currency("0.99996", "USD"), "$1.00"); + assert_eq!(format_decimal_currency("0.0300", "USD"), "$0.03"); + assert_eq!(format_decimal_currency("12.3456", "USD"), "$12.35"); + assert_eq!(format_decimal_currency("0.1204956", "CNY"), "CNY 0.1204956"); + } } mod actions { use gateway_admin::model::{ Revision, - accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus}, + accounts::AccountConnectionTestEvent as DomainConnectionTestEvent, provider_credentials::{ CredentialDeletionResult, CredentialImportResult, CredentialMutationResult, }, @@ -343,9 +353,7 @@ mod actions { DomainConnectionTestEvent::Content { text: "OK".to_owned(), }, - DomainConnectionTestEvent::Completed { - account_status: AccountStatus::Normal, - }, + DomainConnectionTestEvent::Completed {}, DomainConnectionTestEvent::Failed { message: "upstream unavailable".to_owned(), provider_error_code: Some("usage_exhausted".to_owned()), @@ -353,7 +361,6 @@ mod actions { upstream_status: Some(429), upstream_content_type: Some("application/json".to_owned()), upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()), - account_status: AccountStatus::QuotaExhausted, }, ] .map(|event| AccountConnectionTestEvent::from(event).data); @@ -378,7 +385,7 @@ mod actions { } }), json!({ "type": "content", "text": "OK" }), - json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }), + json!({ "type": "test_complete", "success": true }), json!({ "type": "error", "error": "upstream unavailable", @@ -386,8 +393,7 @@ mod actions { "providerErrorType": "invalid_request_error", "upstreamStatus": 429, "upstreamContentType": "application/json", - "upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#, - "accountStatus": "quota_exhausted" + "upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"# }), ] ); diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index eecd47365..08c148842 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -14,7 +14,7 @@ use gateway_admin::{ model::{ MutationContext, Revision, accounts::{ - AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery, + AccountListQuery, AccountPage, AccountPageItem, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled, }, auth::{AdminAuditEvent, AdminSession}, @@ -426,7 +426,7 @@ impl AccountStore for UnusedStore { Err(unavailable("account list")) } - async fn load_account(&self, _: &str) -> AdminStoreResult> { + async fn load_account(&self, _: &str) -> AdminStoreResult> { Err(unavailable("account")) } diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index 7c1817847..2b2b26c4a 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -222,16 +222,11 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli totals: Default::default(), provider_accounts: AccountPoolMetrics { total: 807, - enabled: 804, - unavailable: 792, - active: 15, - rate_limited: 0, - expired: 2, - invalid: 0, + normal: 15, quota_exhausted: 785, - refreshing: None, + rate_limited: 0, disabled: 3, - banned: 2, + error: 4, }, trend: Vec::new(), account_usage: Vec::new(), diff --git a/backend/crates/gateway-core/src/engine/credential.rs b/backend/crates/gateway-core/src/engine/credential.rs index 24631e1e8..1c44eac15 100644 --- a/backend/crates/gateway-core/src/engine/credential.rs +++ b/backend/crates/gateway-core/src/engine/credential.rs @@ -187,45 +187,41 @@ impl fmt::Debug for OpaqueProviderData { } } -/// 数据库中固定的账号运行状态。 +/// 已持久化的凭据/账号身份状态。 /// -/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由 -/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。 +/// 这里只保存凭据与账号身份事实。额度耗尽属于 [`QuotaState`],临时 429 属于 +/// 运行时冷却;二者都不得写入此枚举。 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum AccountAvailability { +pub enum CredentialState { Unknown, Ready, - QuotaExhausted, Expired, Banned, Invalid, } -impl AccountAvailability { +impl CredentialState { /// 返回数据库稳定值。 #[must_use] pub const fn as_str(self) -> &'static str { match self { Self::Unknown => "unknown", Self::Ready => "ready", - Self::QuotaExhausted => "quota_exhausted", Self::Expired => "expired", Self::Banned => "banned", Self::Invalid => "invalid", } } - /// 返回账号进入不可恢复错误态、但 Provider 未给出更具体上游原因时的稳定兜底。 - /// - /// Provider 应优先持久化已脱敏的上游 message 或原因码;该值只避免错误态沿用 - /// 上一次无关的错误信息。 + /// 返回该凭据状态对应的稳定错误原因。 #[must_use] - pub const fn fallback_error_reason(self) -> Option<&'static str> { + pub const fn error_reason(self) -> Option { match self { - Self::Expired => Some("credential_expired"), - Self::Banned => Some("account_banned"), - Self::Invalid => Some("credential_invalid"), - Self::Unknown | Self::Ready | Self::QuotaExhausted => None, + Self::Unknown => Some(AccountErrorReason::AccountUnverified), + Self::Expired => Some(AccountErrorReason::CredentialExpired), + Self::Banned => Some(AccountErrorReason::AccountBanned), + Self::Invalid => Some(AccountErrorReason::CredentialInvalid), + Self::Ready => None, } } @@ -235,7 +231,6 @@ impl AccountAvailability { match value { "unknown" => Some(Self::Unknown), "ready" => Some(Self::Ready), - "quota_exhausted" => Some(Self::QuotaExhausted), "expired" => Some(Self::Expired), "banned" => Some(Self::Banned), "invalid" => Some(Self::Invalid), @@ -244,6 +239,373 @@ impl AccountAvailability { } } +/// 额度访问结论;百分比和余额等展示值不进入此枚举。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum QuotaAccessState { + Unknown, + Allowed, + Exhausted, +} + +impl QuotaAccessState { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::Allowed => "allowed", + Self::Exhausted => "exhausted", + } + } + + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "unknown" => Some(Self::Unknown), + "allowed" => Some(Self::Allowed), + "exhausted" => Some(Self::Exhausted), + _ => None, + } + } +} + +/// 权威额度耗尽证据。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum QuotaEvidence { + ProviderDenied, + AccountLimitReached, + UsageLimitReached, + PaymentRequired, +} + +impl QuotaEvidence { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ProviderDenied => "provider_denied", + Self::AccountLimitReached => "account_limit_reached", + Self::UsageLimitReached => "usage_limit_reached", + Self::PaymentRequired => "payment_required", + } + } + + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "provider_denied" => Some(Self::ProviderDenied), + "account_limit_reached" => Some(Self::AccountLimitReached), + "usage_limit_reached" => Some(Self::UsageLimitReached), + "payment_required" => Some(Self::PaymentRequired), + _ => None, + } + } +} + +/// 可审计的额度访问事实。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct QuotaState { + access: QuotaAccessState, + evidence: Option, + observed_at: Option, + reset_at: Option, +} + +impl Default for QuotaState { + fn default() -> Self { + Self::unknown() + } +} + +impl QuotaState { + #[must_use] + pub const fn unknown() -> Self { + Self { + access: QuotaAccessState::Unknown, + evidence: None, + observed_at: None, + reset_at: None, + } + } + + #[must_use] + pub const fn observed_unknown(observed_at: SystemTime) -> Self { + Self { + access: QuotaAccessState::Unknown, + evidence: None, + observed_at: Some(observed_at), + reset_at: None, + } + } + + #[must_use] + pub const fn allowed(observed_at: SystemTime) -> Self { + Self { + access: QuotaAccessState::Allowed, + evidence: None, + observed_at: Some(observed_at), + reset_at: None, + } + } + + #[must_use] + pub const fn exhausted( + evidence: QuotaEvidence, + observed_at: SystemTime, + reset_at: Option, + ) -> Self { + Self { + access: QuotaAccessState::Exhausted, + evidence: Some(evidence), + observed_at: Some(observed_at), + reset_at, + } + } + + /// 从持久化列恢复额度状态;非法组合返回 `None`。 + #[must_use] + pub const fn from_persisted( + access: QuotaAccessState, + evidence: Option, + observed_at: Option, + reset_at: Option, + ) -> Option { + match access { + QuotaAccessState::Unknown if evidence.is_none() && reset_at.is_none() => Some(Self { + access, + evidence, + observed_at, + reset_at, + }), + QuotaAccessState::Allowed + if evidence.is_none() && observed_at.is_some() && reset_at.is_none() => + { + Some(Self { + access, + evidence, + observed_at, + reset_at, + }) + } + QuotaAccessState::Exhausted if evidence.is_some() && observed_at.is_some() => { + Some(Self { + access, + evidence, + observed_at, + reset_at, + }) + } + _ => None, + } + } + + #[must_use] + pub const fn access(self) -> QuotaAccessState { + self.access + } + + #[must_use] + pub const fn evidence(self) -> Option { + self.evidence + } + + #[must_use] + pub const fn observed_at(self) -> Option { + self.observed_at + } + + #[must_use] + pub const fn reset_at(self) -> Option { + self.reset_at + } + + /// 返回当前已确认的额度耗尽结论。 + /// + /// `reset_at` 只表示何时应重新向 Provider 求证;时间到期本身不是额度恢复证据。 + #[must_use] + pub fn is_exhausted(self) -> bool { + self.access == QuotaAccessState::Exhausted + } + + /// 判断已耗尽额度是否到达 Provider 的下一次复核时间。 + /// + /// 有明确 `reset_at` 时严格等待该时刻;没有时由 Provider 传入自己的保守复核周期。 + #[must_use] + pub fn exhaustion_refresh_due(self, now: SystemTime, fallback_interval: Duration) -> bool { + if !self.is_exhausted() { + return false; + } + let due_at = self.reset_at.or_else(|| { + self.observed_at + .and_then(|observed_at| observed_at.checked_add(fallback_interval)) + }); + due_at.is_none_or(|due_at| due_at <= now) + } + + /// 合并一次额度访问观察;`Unknown` 不能擦除已经确认的访问结论。 + #[must_use] + pub fn merge_observation(self, observation: Self) -> Self { + if observation.access == QuotaAccessState::Unknown + && self.access != QuotaAccessState::Unknown + { + self + } else { + observation + } + } +} + +/// 对外唯一的五态账号状态。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AccountStatus { + Normal, + QuotaExhausted, + RateLimited, + Disabled, + Error, +} + +impl AccountStatus { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Normal => "normal", + Self::QuotaExhausted => "quota_exhausted", + Self::RateLimited => "rate_limited", + Self::Disabled => "disabled", + Self::Error => "error", + } + } + + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "normal" => Some(Self::Normal), + "quota_exhausted" => Some(Self::QuotaExhausted), + "rate_limited" => Some(Self::RateLimited), + "disabled" => Some(Self::Disabled), + "error" => Some(Self::Error), + _ => None, + } + } + + /// 管理页稳定业务排序:正常、限流、耗尽、错误、停用。 + #[must_use] + pub const fn sort_rank(self) -> u8 { + match self { + Self::Normal => 0, + Self::RateLimited => 1, + Self::QuotaExhausted => 2, + Self::Error => 3, + Self::Disabled => 4, + } + } +} + +/// `error` 状态下的稳定原因码。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AccountErrorReason { + AccountUnverified, + AccessTokenExpired, + CredentialExpired, + CredentialInvalid, + AccountBanned, +} + +impl AccountErrorReason { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::AccountUnverified => "account_unverified", + Self::AccessTokenExpired => "access_token_expired", + Self::CredentialExpired => "credential_expired", + Self::CredentialInvalid => "credential_invalid", + Self::AccountBanned => "account_banned", + } + } + + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "account_unverified" => Some(Self::AccountUnverified), + "access_token_expired" => Some(Self::AccessTokenExpired), + "credential_expired" => Some(Self::CredentialExpired), + "credential_invalid" => Some(Self::CredentialInvalid), + "account_banned" => Some(Self::AccountBanned), + _ => None, + } + } +} + +/// 唯一状态解析器的完整输入事实。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountStatusFacts { + pub enabled: bool, + pub credential_state: CredentialState, + pub access_token_expires_at: Option, + pub quota: QuotaState, + pub rate_limited_until: Option, + pub last_error_reason: Option, + pub last_error_message: Option, +} + +/// 唯一状态解析器的输出。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountStatusProjection { + pub status: AccountStatus, + pub error_reason: Option, + pub error_message: Option, + /// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。 + pub rate_limited_until: Option, +} + +/// 从独立事实派生唯一、互斥的对外状态。 +#[must_use] +pub fn resolve_account_status( + facts: &AccountStatusFacts, + now: SystemTime, +) -> AccountStatusProjection { + if !facts.enabled { + return status_projection(AccountStatus::Disabled); + } + let credential_error = facts.credential_state.error_reason().or_else(|| { + facts + .access_token_expires_at + .is_some_and(|expires_at| expires_at <= now) + .then_some(AccountErrorReason::AccessTokenExpired) + }); + if let Some(default_reason) = credential_error { + return AccountStatusProjection { + status: AccountStatus::Error, + error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)), + error_message: facts.last_error_message.clone(), + rate_limited_until: None, + }; + } + if facts.quota.is_exhausted() { + return status_projection(AccountStatus::QuotaExhausted); + } + if facts + .rate_limited_until + .is_some_and(|rate_limited_until| rate_limited_until > now) + { + return AccountStatusProjection { + status: AccountStatus::RateLimited, + error_reason: None, + error_message: None, + rate_limited_until: facts.rate_limited_until, + }; + } + status_projection(AccountStatus::Normal) +} + +const fn status_projection(status: AccountStatus) -> AccountStatusProjection { + AccountStatusProjection { + status, + error_reason: None, + error_message: None, + rate_limited_until: None, + } +} + /// 不含 secret 的账号持久事实。 #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProviderAccount { @@ -257,7 +619,10 @@ pub struct ProviderAccount { authentication_kind: String, revision: CredentialRevision, enabled: bool, - availability: AccountAvailability, + credential_state: CredentialState, + quota: QuotaState, + last_error_reason: Option, + last_error_message: Option, access_token_expires_at: Option, next_refresh_at: Option, has_refresh_token: bool, @@ -286,7 +651,10 @@ impl ProviderAccount { authentication_kind, revision, enabled: true, - availability: AccountAvailability::Unknown, + credential_state: CredentialState::Unknown, + quota: QuotaState::unknown(), + last_error_reason: None, + last_error_message: None, access_token_expires_at, next_refresh_at: None, has_refresh_token: false, @@ -307,17 +675,23 @@ impl ProviderAccount { } #[must_use] - pub const fn with_runtime_state( + pub fn with_account_facts( mut self, enabled: bool, - availability: AccountAvailability, + credential_state: CredentialState, + quota: QuotaState, + last_error_reason: Option, + last_error_message: Option, ) -> Self { self.enabled = enabled; - self.availability = if self.upstream_user_id.is_some() { - availability + self.credential_state = if self.upstream_user_id.is_some() { + credential_state } else { - AccountAvailability::Unknown + CredentialState::Unknown }; + self.quota = quota; + self.last_error_reason = last_error_reason; + self.last_error_message = last_error_message; self } @@ -382,8 +756,23 @@ impl ProviderAccount { } #[must_use] - pub const fn availability(&self) -> AccountAvailability { - self.availability + pub const fn credential_state(&self) -> CredentialState { + self.credential_state + } + + #[must_use] + pub const fn quota(&self) -> QuotaState { + self.quota + } + + #[must_use] + pub const fn last_error_reason(&self) -> Option { + self.last_error_reason + } + + #[must_use] + pub fn last_error_message(&self) -> Option<&str> { + self.last_error_message.as_deref() } #[must_use] @@ -407,23 +796,25 @@ impl ProviderAccount { self.has_refresh_token } - /// 判断账号当前能否进入同 target 的候选池。 + /// 组合持久事实与请求级冷却,交给唯一解析器派生状态。 #[must_use] - pub fn is_schedulable(&self, now: SystemTime) -> bool { - let available = match self.availability { - AccountAvailability::Ready => true, - AccountAvailability::Unknown - | AccountAvailability::QuotaExhausted - | AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid => false, - }; - self.upstream_user_id.is_some() - && self.enabled - && available - && self - .access_token_expires_at - .is_none_or(|expires_at| expires_at > now) + pub fn status_projection( + &self, + now: SystemTime, + rate_limited_until: Option, + ) -> AccountStatusProjection { + resolve_account_status( + &AccountStatusFacts { + enabled: self.enabled, + credential_state: self.credential_state, + access_token_expires_at: self.access_token_expires_at, + quota: self.quota, + rate_limited_until, + last_error_reason: self.last_error_reason, + last_error_message: self.last_error_message.clone(), + }, + now, + ) } } @@ -600,10 +991,11 @@ pub enum CredentialCasOutcome { pub struct QuotaObservation { pub account_id: ProviderAccountId, pub expected_revision: CredentialRevision, - pub quota: Option, - pub observed_at: Option, - /// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。 - pub limit_reached: Option, + pub quota: OpaqueProviderData, + /// Provider 原始 quota document 的观察时间;不代表访问结论发生变化。 + pub observed_at: SystemTime, + /// Provider 已从私有 JSON 归一化出的额度访问事实。 + pub state: QuotaState, } impl fmt::Debug for QuotaObservation { @@ -614,6 +1006,7 @@ impl fmt::Debug for QuotaObservation { .field("expected_revision", &self.expected_revision) .field("quota", &self.quota) .field("observed_at", &self.observed_at) + .field("state", &self.state) .finish() } } @@ -624,14 +1017,24 @@ pub enum QuotaWriteOutcome { Conflict, } +/// 不改 Provider 原始 quota JSON 的额度访问事实写入。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QuotaAccessChange { + pub account_id: ProviderAccountId, + pub expected_revision: CredentialRevision, + pub state: QuotaState, +} + /// 账号状态的 revision-fenced 写入。 #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStateChange { pub account_id: ProviderAccountId, pub expected_revision: CredentialRevision, - pub availability: AccountAvailability, + pub credential_state: CredentialState, pub observed_at: SystemTime, - /// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。 + /// 受控错误原因;恢复 Ready 时必须清空。 + pub error_reason: Option, + /// 已脱敏的上游诊断信息;恢复 Ready 时必须清空。 pub message: Option, } @@ -682,6 +1085,11 @@ pub trait ProviderAccountStore: Send + Sync { observation: QuotaObservation, ) -> Result; + async fn apply_quota_access( + &self, + change: QuotaAccessChange, + ) -> Result; + async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), StoreError>; async fn update_account(&self, update: ProviderAccountUpdate) -> Result<(), StoreError>; @@ -772,7 +1180,7 @@ pub struct AccountRuntimeSignals { pub last_started_at: Option, pub quota_reset_at: Option, pub quota_remaining_rank: Option, - pub quota_limit_reached: bool, + pub rate_limited_until: Option, pub failure_rate_basis_points: Option, pub first_output_latency_ms: Option, } @@ -920,11 +1328,16 @@ impl AccountRuntimeSignals { if let Some(quota) = quota { self.quota_reset_at = quota.reset_at; self.quota_remaining_rank = quota.remaining_rank; - self.quota_limit_reached = quota.limit_reached; } self } + #[must_use] + pub const fn with_rate_limit(mut self, rate_limited_until: Option) -> Self { + self.rate_limited_until = rate_limited_until; + self + } + #[must_use] pub const fn with_runtime_health( mut self, @@ -942,20 +1355,14 @@ impl AccountRuntimeSignals { pub struct AccountQuotaSignals { reset_at: Option, remaining_rank: Option, - limit_reached: bool, } impl AccountQuotaSignals { #[must_use] - pub const fn new( - reset_at: Option, - remaining_rank: Option, - limit_reached: bool, - ) -> Self { + pub const fn new(reset_at: Option, remaining_rank: Option) -> Self { Self { reset_at, remaining_rank, - limit_reached, } } @@ -968,12 +1375,6 @@ impl AccountQuotaSignals { pub const fn remaining_rank(self) -> Option { self.remaining_rank } - - /// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。 - #[must_use] - pub const fn limit_reached(self) -> bool { - self.limit_reached - } } /// 账号持久事实与可重建运行信号的请求级组合。 @@ -991,24 +1392,24 @@ pub struct AccountSelectionContext { pub excluded_accounts: BTreeSet, pub preferred_account: Option, pub round_robin_cursor: u64, - pub availability: AccountAvailabilityPolicy, + pub eligibility: AccountEligibilityPolicy, } -/// 选择账号时是否信任本地可用性投影。 +/// 选择账号时是否执行本地调度资格投影。 /// /// 管理端对指定账号执行诊断时,会直接向上游确认实际状态;该模式跳过 /// `enabled`、可用性、冷却和 token 到期等本地投影,仍保留租约、并发和 /// 请求间隔约束,且只能与固定账号约束组合使用。 #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum AccountAvailabilityPolicy { +pub enum AccountEligibilityPolicy { #[default] Enforce, BypassForDiagnostic, } -impl AccountAvailabilityPolicy { +impl AccountEligibilityPolicy { #[must_use] - pub const fn bypasses_local_availability(self) -> bool { + pub const fn bypasses_local_eligibility(self) -> bool { matches!(self, Self::BypassForDiagnostic) } } @@ -1132,10 +1533,14 @@ impl AccountSelector { candidate: &AccountCandidate, context: &AccountSelectionContext, ) -> Option { - if !context.availability.bypasses_local_availability() - && !candidate.account.is_schedulable(context.now) - { - return Some(AccountSchedulingBlocker::LocalAvailability); + if !context.eligibility.bypasses_local_eligibility() { + let status = candidate + .account + .status_projection(context.now, candidate.signals.rate_limited_until) + .status; + if status != AccountStatus::Normal { + return Some(AccountSchedulingBlocker::LocalAvailability); + } } if context.excluded_accounts.contains(candidate.account.id()) { return Some(AccountSchedulingBlocker::Excluded); diff --git a/backend/crates/gateway-core/src/provider_ports/mod.rs b/backend/crates/gateway-core/src/provider_ports/mod.rs index 689db3f6a..2bd85a32f 100644 --- a/backend/crates/gateway-core/src/provider_ports/mod.rs +++ b/backend/crates/gateway-core/src/provider_ports/mod.rs @@ -9,7 +9,7 @@ use std::time::{Duration, SystemTime}; use futures::future::BoxFuture; use crate::engine::credential::{ - AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision, + AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccountId, ProviderAccountStore, }; use crate::error::{IdentifierError, validate_text}; @@ -411,7 +411,7 @@ pub struct ProviderCredentialState { account_id: ProviderAccountId, credential_revision: CredentialRevision, enabled: bool, - availability: AccountAvailability, + credential_state: CredentialState, observed_at: SystemTime, } @@ -421,14 +421,14 @@ impl ProviderCredentialState { account_id: ProviderAccountId, credential_revision: CredentialRevision, enabled: bool, - availability: AccountAvailability, + credential_state: CredentialState, observed_at: SystemTime, ) -> Self { Self { account_id, credential_revision, enabled, - availability, + credential_state, observed_at, } } @@ -449,8 +449,8 @@ impl ProviderCredentialState { } #[must_use] - pub const fn availability(&self) -> AccountAvailability { - self.availability + pub const fn credential_state(&self) -> CredentialState { + self.credential_state } #[must_use] diff --git a/backend/crates/gateway-core/tests/engine/credential.rs b/backend/crates/gateway-core/tests/engine/credential.rs index 2590657cf..3afbedac7 100644 --- a/backend/crates/gateway-core/tests/engine/credential.rs +++ b/backend/crates/gateway-core/tests/engine/credential.rs @@ -5,12 +5,12 @@ use std::time::{Duration, SystemTime}; use serde_json::{Map, Value}; use gateway_core::engine::credential::{ - AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, - AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker, - AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate, - CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection, - ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate, - RotationStrategy, + AccountAttemptFeedback, AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats, + AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, + AccountSelectionPolicy, AccountSelector, AccountStatus, CredentialCasUpdate, + CredentialRevision, CredentialState, OpaqueProviderData, PlaintextCredential, + PreferredAccountSelection, ProviderAccount, ProviderAccountId, ProviderAccountIdentity, + ProviderAccountUpdate, QuotaEvidence, QuotaState, RotationStrategy, }; use gateway_core::routing::ProviderKind; @@ -24,7 +24,13 @@ fn account(id: &str) -> ProviderAccount { CredentialRevision::new(1).expect("valid revision"), Some(SystemTime::now() + Duration::from_secs(3600)), ) - .with_runtime_state(true, AccountAvailability::Ready) + .with_account_facts( + true, + CredentialState::Ready, + QuotaState::unknown(), + None, + None, + ) } fn candidate(id: &str, in_flight: u32, remaining: Option) -> AccountCandidate { @@ -35,7 +41,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option) -> AccountCandida last_started_at: None, quota_reset_at: None, quota_remaining_rank: remaining, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -53,7 +59,7 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext { excluded_accounts: BTreeSet::new(), preferred_account: None, round_robin_cursor: 0, - availability: AccountAvailabilityPolicy::Enforce, + eligibility: AccountEligibilityPolicy::Enforce, } } @@ -101,43 +107,99 @@ fn provider_account_identity_debug_should_redact_upstream_ids() { } #[test] -fn availability_should_round_trip_all_database_values() { +fn account_fact_values_should_round_trip() { assert_eq!( - AccountAvailability::parse(AccountAvailability::QuotaExhausted.as_str()), - Some(AccountAvailability::QuotaExhausted) + CredentialState::parse(CredentialState::Banned.as_str()), + Some(CredentialState::Banned) + ); + assert_eq!( + QuotaEvidence::parse("provider_denied"), + Some(QuotaEvidence::ProviderDenied) ); } #[test] -fn diagnostic_selection_bypasses_all_local_account_availability() { +fn exhausted_quota_refresh_is_due_at_reset_or_provider_fallback() { + let now = SystemTime::now(); + let observed_at = now - Duration::from_secs(60); + let before_reset = QuotaState::exhausted( + QuotaEvidence::ProviderDenied, + observed_at, + Some(now + Duration::from_secs(1)), + ); + let after_reset = QuotaState::exhausted( + QuotaEvidence::ProviderDenied, + observed_at, + Some(now - Duration::from_secs(1)), + ); + let without_reset = QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None); + + assert!(!before_reset.exhaustion_refresh_due(now, Duration::from_secs(30))); + assert!(after_reset.exhaustion_refresh_due(now, Duration::from_secs(30))); + assert!(without_reset.exhaustion_refresh_due(now, Duration::from_secs(30))); + assert!(!without_reset.exhaustion_refresh_due(now, Duration::from_secs(120))); +} + +#[test] +fn inconclusive_quota_observation_cannot_erase_confirmed_access() { + let now = SystemTime::now(); + let exhausted = QuotaState::exhausted(QuotaEvidence::ProviderDenied, now, None); + + assert_eq!( + exhausted + .merge_observation(QuotaState::observed_unknown(now + Duration::from_secs(1))) + .access(), + gateway_core::engine::credential::QuotaAccessState::Exhausted + ); + assert_eq!( + exhausted + .merge_observation(QuotaState::allowed(now + Duration::from_secs(1))) + .access(), + gateway_core::engine::credential::QuotaAccessState::Allowed + ); +} + +#[test] +fn diagnostic_selection_bypasses_all_local_account_eligibility() { + let observed_at = SystemTime::now(); let exhausted = AccountCandidate { - account: account("acct_exhausted") - .with_runtime_state(true, AccountAvailability::QuotaExhausted), + account: account("acct_exhausted").with_account_facts( + true, + CredentialState::Ready, + QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), + None, + None, + ), signals: AccountRuntimeSignals { in_flight: 0, last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, }; let disabled = AccountCandidate { - account: account("acct_disabled") - .with_runtime_state(false, AccountAvailability::QuotaExhausted), + account: account("acct_disabled").with_account_facts( + false, + CredentialState::Ready, + QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), + None, + None, + ), signals: AccountRuntimeSignals { in_flight: 0, last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, }; let mut context = context(RotationStrategy::Sticky); - context.availability = AccountAvailabilityPolicy::BypassForDiagnostic; + context.eligibility = AccountEligibilityPolicy::BypassForDiagnostic; assert_eq!( AccountSelector @@ -155,9 +217,33 @@ fn diagnostic_selection_bypasses_all_local_account_availability() { #[test] fn disabled_account_should_not_be_schedulable() { - let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready); + let account = account("acct_disabled").with_account_facts( + false, + CredentialState::Ready, + QuotaState::unknown(), + None, + None, + ); - assert!(!account.is_schedulable(SystemTime::now())); + assert_eq!( + account.status_projection(SystemTime::now(), None).status, + AccountStatus::Disabled + ); +} + +#[test] +fn rate_limited_projection_carries_only_the_active_cooldown_deadline() { + let now = SystemTime::now(); + let active_until = now + Duration::from_secs(60); + let account = account("acct_rate_limited"); + + let active = account.status_projection(now, Some(active_until)); + assert_eq!(active.status, AccountStatus::RateLimited); + assert_eq!(active.rate_limited_until, Some(active_until)); + + let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1))); + assert_eq!(elapsed.status, AccountStatus::Normal); + assert_eq!(elapsed.rate_limited_until, None); } #[test] @@ -171,14 +257,20 @@ fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() { CredentialRevision::new(1).expect("valid revision"), Some(SystemTime::now() + Duration::from_secs(3600)), ) - .with_runtime_state(true, AccountAvailability::Ready); + .with_account_facts( + true, + CredentialState::Ready, + QuotaState::unknown(), + None, + None, + ); assert_eq!( ( - account.availability(), - account.is_schedulable(SystemTime::now()) + account.credential_state(), + account.status_projection(SystemTime::now(), None).status ), - (AccountAvailability::Unknown, false) + (CredentialState::Unknown, AccountStatus::Error) ); } @@ -308,15 +400,11 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() { last_started_at: Some(last_started_at), quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, } - .with_provider_quota(Some(AccountQuotaSignals::new( - Some(reset_at), - Some(75), - false, - ))); + .with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75)))); assert_eq!( ( @@ -633,21 +721,31 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() { } #[test] -fn enforced_availability_excludes_terminal_states_despite_quota_signals() { - // availability 终态(Banned/Expired/Invalid)始终排除, - // 不受 quota 信号或 skip_exhausted 放宽影响。 +fn enforced_eligibility_excludes_credential_errors_despite_quota_signals() { let mut banned = candidate("acct_banned", 0, Some(100)); - banned.account = banned - .account - .with_runtime_state(true, AccountAvailability::Banned); + banned.account = banned.account.with_account_facts( + true, + CredentialState::Banned, + QuotaState::unknown(), + None, + None, + ); let mut expired = candidate("acct_expired", 0, Some(100)); - expired.account = expired - .account - .with_runtime_state(true, AccountAvailability::Expired); + expired.account = expired.account.with_account_facts( + true, + CredentialState::Expired, + QuotaState::unknown(), + None, + None, + ); let mut invalid = candidate("acct_invalid", 0, Some(100)); - invalid.account = invalid - .account - .with_runtime_state(true, AccountAvailability::Invalid); + invalid.account = invalid.account.with_account_facts( + true, + CredentialState::Invalid, + QuotaState::unknown(), + None, + None, + ); let healthy = candidate("acct_healthy", 0, Some(100)); let candidates = vec![banned, expired, invalid, healthy]; @@ -659,14 +757,15 @@ fn enforced_availability_excludes_terminal_states_despite_quota_signals() { } #[test] -fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() { - // QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的 - // 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身 - // 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。 +fn enforced_eligibility_excludes_authoritative_quota_exhaustion() { let mut exhausted = candidate("acct_exhausted", 0, Some(100)); - exhausted.account = exhausted - .account - .with_runtime_state(true, AccountAvailability::QuotaExhausted); + exhausted.account = exhausted.account.with_account_facts( + true, + CredentialState::Ready, + QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None), + None, + None, + ); let healthy = candidate("acct_healthy", 0, Some(100)); let candidates = vec![exhausted, healthy]; @@ -676,3 +775,31 @@ fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy"); } + +#[test] +fn elapsed_quota_reset_does_not_fabricate_recovery() { + let now = SystemTime::now(); + let mut exhausted = candidate("acct_exhausted", 0, Some(100)); + exhausted.account = exhausted.account.with_account_facts( + true, + CredentialState::Ready, + QuotaState::exhausted( + QuotaEvidence::ProviderDenied, + now - Duration::from_secs(2), + Some(now - Duration::from_secs(1)), + ), + None, + None, + ); + let healthy = candidate("acct_healthy", 0, Some(100)); + + assert_eq!( + exhausted.account.status_projection(now, None).status, + AccountStatus::QuotaExhausted + ); + let candidates = [exhausted, healthy]; + let selected = AccountSelector + .select(&candidates, &context(RotationStrategy::Smart)) + .expect("healthy candidate available"); + assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy"); +} diff --git a/backend/crates/gateway-core/tests/provider_ports/mod.rs b/backend/crates/gateway-core/tests/provider_ports/mod.rs index 2f538d8d7..18de8934d 100644 --- a/backend/crates/gateway-core/tests/provider_ports/mod.rs +++ b/backend/crates/gateway-core/tests/provider_ports/mod.rs @@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() { last_started_at: None, quota_reset_at: None, quota_remaining_rank: Some(7), - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: Some(125), first_output_latency_ms: Some(250), }, diff --git a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs index ceaa9d706..98b438f66 100644 --- a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs @@ -1,12 +1,15 @@ //! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。 -use std::sync::Arc; -use std::time::SystemTime; - use gateway_core::provider_ports::ProviderCooldownPort; +use std::sync::Arc; use super::*; +use crate::postgres::{ + PgProviderAccountRepository, ProviderAccountRepository, account_status_projection, + load_rate_limited_until, +}; + use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository}; #[derive(Clone)] @@ -21,73 +24,48 @@ impl PgObservabilityRepository { Self { pool, cooldowns } } - /// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。 - /// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。 - async fn provider_account_metrics_with_cooldowns( + /// 从账号事实和当前冷却一次性派生 Dashboard 五态;不在 SQL 中复制状态机。 + async fn account_status_snapshot( &self, observed_at: DateTime, - ) -> StoreResult { - let base = provider_account_metrics(&self.pool, observed_at).await?; - let Some(cooldowns) = &self.cooldowns else { - return Ok(base); + ) -> StoreResult<(ProviderAccountMetrics, Vec)> { + let accounts = PgProviderAccountRepository::new(self.pool.clone()) + .list_provider_accounts(None, true) + .await?; + let now = observed_at.into(); + let rate_limited_until = + load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await; + let mut metrics = ProviderAccountMetrics { + total: u64::try_from(accounts.len()).unwrap_or(u64::MAX), + ..ProviderAccountMetrics::default() }; - let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?; - if candidates.is_empty() { - return Ok(base); - } - // Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶; - // 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。 - let mut active = base.active; - let mut rate_limited = base.rate_limited; - let mut quota_exhausted = base.quota_exhausted; - let now = SystemTime::now(); - for (account_id, revision, quota_reached) in candidates { - let Ok(account_id) = - gateway_core::engine::credential::ProviderAccountId::new(account_id) - else { - continue; - }; - let cooling = cooldowns - .read(&account_id) - .await - .ok() - .flatten() - .filter(|cooldown| cooldown.credential_revision().get() as i64 == revision) - .filter(|cooldown| cooldown.until() > now) - .is_some(); - if !cooling { - continue; - } - if quota_reached { - if quota_exhausted == 0 { - continue; + let mut normal_account_ids = Vec::new(); + for account in &accounts { + let projection = account_status_projection( + account, + now, + rate_limited_until.get(&account.id).copied(), + ); + match projection.status { + gateway_core::engine::credential::AccountStatus::Normal => { + metrics.normal = metrics.normal.saturating_add(1); + normal_account_ids.push(account.id.clone()); } - quota_exhausted -= 1; - } else { - if active == 0 { - continue; + gateway_core::engine::credential::AccountStatus::QuotaExhausted => { + metrics.quota_exhausted = metrics.quota_exhausted.saturating_add(1); + } + gateway_core::engine::credential::AccountStatus::RateLimited => { + metrics.rate_limited = metrics.rate_limited.saturating_add(1); + } + gateway_core::engine::credential::AccountStatus::Disabled => { + metrics.disabled = metrics.disabled.saturating_add(1); + } + gateway_core::engine::credential::AccountStatus::Error => { + metrics.error = metrics.error.saturating_add(1); } - active -= 1; } - rate_limited += 1; } - Ok(ProviderAccountMetrics { - total: base.total, - enabled: base.enabled, - unavailable: rate_limited - + quota_exhausted - + base.expired - + base.invalid - + base.disabled - + base.banned, - active, - rate_limited, - expired: base.expired, - invalid: base.invalid, - quota_exhausted, - disabled: base.disabled, - banned: base.banned, - }) + Ok((metrics, normal_account_ids)) } } @@ -145,9 +123,7 @@ impl ObservabilityRepository for PgObservabilityRepository { attempt_metrics(&self.pool, range, &filter), dashboard_totals(&self.pool), )?; - let provider_accounts = self - .provider_account_metrics_with_cooldowns(range.end) - .await?; + let (provider_accounts, _) = self.account_status_snapshot(range.end).await?; let (trend, account_usage, recent_requests) = futures::try_join!( request_metric_series(&self.pool, range, &filter), provider_account_usage(&self.pool, account_usage_query), @@ -252,12 +228,14 @@ impl AdminObservabilityStore for PgAdminObservabilityStore { &self, observed_at: DateTime, ) -> AdminStoreResult> { - let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at) + let (_, normal_account_ids) = self + .repository + .account_status_snapshot(observed_at) .await .map_err(observability_error)?; - let active_accounts = u64::try_from(active_account_ids.len()) + let active_accounts = u64::try_from(normal_account_ids.len()) .map_err(|_| observability_error(invalid("active account count overflows u64")))?; - if active_account_ids.is_empty() { + if normal_account_ids.is_empty() { return Ok(Some(admin_observability::DashboardRuntimeSlots { active_accounts, used_slots: Some(0), @@ -267,7 +245,7 @@ impl AdminObservabilityStore for PgAdminObservabilityStore { return Ok(None); }; let signals = match runtime_signals - .credential_runtime_signals(&active_account_ids) + .credential_runtime_signals(&normal_account_ids) .await { Ok(signals) => signals, diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index eb8925dca..4e692d371 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -274,16 +274,11 @@ pub(crate) const fn admin_account_pool_metrics( ) -> admin_observability::AccountPoolMetrics { admin_observability::AccountPoolMetrics { total: metrics.total, - enabled: metrics.enabled, - unavailable: metrics.unavailable, - active: metrics.active, - rate_limited: metrics.rate_limited, - expired: metrics.expired, - invalid: metrics.invalid, + normal: metrics.normal, quota_exhausted: metrics.quota_exhausted, - refreshing: None, + rate_limited: metrics.rate_limited, disabled: metrics.disabled, - banned: metrics.banned, + error: metrics.error, } } @@ -297,8 +292,6 @@ pub(crate) fn admin_dashboard_account_usage( name: usage.name, email: usage.email, plan_type: usage.plan_type, - enabled: usage.enabled, - availability: usage.availability, request_count: usage.request_count, success_count: usage.success_count, input_tokens: usage.input_tokens, diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs index 260f391b8..33d8083d3 100644 --- a/backend/crates/gateway-store/src/postgres/observability/model.rs +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -363,15 +363,11 @@ pub struct CalculatedUsageBillingFact { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ProviderAccountMetrics { pub total: u64, - pub enabled: u64, - pub unavailable: u64, - pub active: u64, - pub rate_limited: u64, - pub expired: u64, - pub invalid: u64, + pub normal: u64, pub quota_exhausted: u64, + pub rate_limited: u64, pub disabled: u64, - pub banned: u64, + pub error: u64, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -388,8 +384,6 @@ pub struct ProviderAccountUsageObservation { pub name: String, pub email: Option, pub plan_type: Option, - pub enabled: bool, - pub availability: String, pub request_count: u64, pub success_count: u64, pub input_tokens: Option, diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs index 43036f8e8..b354a7d39 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs @@ -15,7 +15,7 @@ pub(crate) async fn provider_account_usage( let mut statement = QueryBuilder::::new( "select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, - pa.plan_type, pa.enabled, pa.availability, + pa.plan_type, count(mr.id)::bigint as request_count, count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count, sum(mr.input_tokens)::bigint as input_tokens, @@ -52,7 +52,7 @@ pub(crate) async fn provider_account_usage( } statement.push( " group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email, - pa.plan_type, pa.enabled, pa.availability + pa.plan_type order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ", ); statement.push_bind(i64::from(query.limit)); @@ -313,8 +313,6 @@ pub(crate) fn provider_account_from_row( name: get(row, "name")?, email: get(row, "email")?, plan_type: get(row, "plan_type")?, - enabled: get(row, "enabled")?, - availability: get(row, "availability")?, request_count: unsigned(row, "request_count")?, success_count: unsigned(row, "success_count")?, input_tokens: optional_unsigned(row, "input_tokens")?, diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs index 8edf883ca..a5de6fd28 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs @@ -369,118 +369,6 @@ pub(crate) async fn request_costs( .collect() } -pub(crate) async fn provider_account_metrics( - pool: &PgPool, - observed_at: DateTime, -) -> StoreResult { - let row = sqlx::query( - "with normalized as ( - select enabled, availability, quota_limit_reached, - case - when not enabled then 'disabled' - when availability = 'banned' then 'banned' - when availability = 'quota_exhausted' then 'quota_exhausted' - when availability = 'invalid' then 'invalid' - when availability = 'expired' - or (access_token_expires_at is not null and access_token_expires_at <= $1) - then 'expired' - when availability = 'unknown' then 'invalid' - when quota_limit_reached then 'quota_exhausted' - else 'active' - end as status - from provider_accounts - ) - select count(*)::bigint as total, - count(*) filter (where enabled)::bigint as enabled, - count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable, - count(*) filter (where status = 'active')::bigint as active, - count(*) filter (where status = 'rate_limited')::bigint as rate_limited, - count(*) filter (where status = 'expired')::bigint as expired, - count(*) filter (where status = 'invalid')::bigint as invalid, - count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted, - count(*) filter (where status = 'disabled')::bigint as disabled, - count(*) filter (where status = 'banned')::bigint as banned - from normalized", - ) - .bind(observed_at) - .fetch_one(pool) - .await - .map_err(|_| postgres_unavailable("load provider account metrics"))?; - Ok(ProviderAccountMetrics { - total: unsigned(&row, "total")?, - enabled: unsigned(&row, "enabled")?, - unavailable: unsigned(&row, "unavailable")?, - active: unsigned(&row, "active")?, - rate_limited: unsigned(&row, "rate_limited")?, - expired: unsigned(&row, "expired")?, - invalid: unsigned(&row, "invalid")?, - quota_exhausted: unsigned(&row, "quota_exhausted")?, - disabled: unsigned(&row, "disabled")?, - banned: unsigned(&row, "banned")?, - }) -} - -pub(crate) async fn active_provider_account_ids( - pool: &PgPool, - observed_at: DateTime, -) -> StoreResult> { - sqlx::query( - "with normalized as ( - select id, - case - when not enabled then 'disabled' - when availability = 'banned' then 'banned' - when availability = 'quota_exhausted' then 'quota_exhausted' - when availability in ('expired', 'invalid') - or (access_token_expires_at is not null and access_token_expires_at <= $1) - then 'expired' - else 'active' - end as status - from provider_accounts - ) - select id - from normalized - where status = 'active' - order by id", - ) - .bind(observed_at) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load active provider account IDs"))? - .iter() - .map(|row| get(row, "id")) - .collect() -} - -/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。 -/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。 -pub(crate) async fn schedulable_metric_candidates( - pool: &PgPool, - observed_at: DateTime, -) -> StoreResult> { - sqlx::query( - "select id, credential_revision, quota_limit_reached - from provider_accounts - where enabled - and availability = 'ready' - and (access_token_expires_at is null or access_token_expires_at > $1) - order by id", - ) - .bind(observed_at) - .fetch_all(pool) - .await - .map_err(|_| postgres_unavailable("load schedulable metric candidates"))? - .into_iter() - .map(|row| { - Ok(( - get(&row, "id")?, - get(&row, "credential_revision")?, - get(&row, "quota_limit_reached")?, - )) - }) - .collect::>>() -} - pub(crate) async fn provider_observations( pool: &PgPool, range: ObservabilityRange, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs index e92752fc2..cdf042650 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -2,7 +2,6 @@ use std::collections::BTreeMap; use std::sync::Arc; -use std::time::SystemTime; use gateway_core::provider_ports::ProviderCooldownPort; @@ -32,34 +31,6 @@ impl PgAdminAccountStore { } } - /// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。 - /// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。 - async fn load_rate_limited_until( - &self, - accounts: &[ProviderAccountSummary], - ) -> AdminStoreResult> { - let Some(cooldowns) = &self.cooldowns else { - return Ok(BTreeMap::new()); - }; - let mut map = BTreeMap::new(); - for account in accounts { - let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else { - continue; - }; - let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else { - continue; - }; - if cooldown.credential_revision().get() != account.credential_revision.get() { - continue; - } - let until = cooldown.until(); - if until > SystemTime::now() { - map.insert(account.id.clone(), until); - } - } - Ok(map) - } - async fn usage_observations( &self, range: ObservabilityRange, @@ -278,21 +249,18 @@ impl AccountStore for PgAdminAccountStore { ) .map_err(|error| admin_store_error(ENTITY, error))?; let now = Utc::now(); - // 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。 - let rate_limited_until = self - .load_rate_limited_until(&accounts) - .await - .unwrap_or_default(); + let rate_limited_until = + load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now.into()).await; let summary = admin_account_summary(&accounts, now, &rate_limited_until); let mut items = accounts .into_iter() .filter_map(|account| { let until = rate_limited_until.get(&account.id).copied(); - let status = admin_account_status(&account, now, until); - account_matches_admin_query(&account, status, &query).then_some( + let projection = account_status_projection(&account, now.into(), until); + account_matches_admin_query(&account, projection.status, &query).then_some( AdminAccountListItem { account, - status, + projection, usage: None, }, ) @@ -330,7 +298,12 @@ impl AccountStore for PgAdminAccountStore { .into_iter() .skip(offset) .take(page_size) - .map(|item| admin_account_record(item.account)) + .map(|item| { + Ok(AccountPageItem { + account: admin_account_record(item.account)?, + projection: item.projection, + }) + }) .collect::>>()?; Ok(AccountPage { config_revision: admin_revision(control_plane.settings.config_revision)?, @@ -340,13 +313,29 @@ impl AccountStore for PgAdminAccountStore { }) } - async fn load_account(&self, account_id: &str) -> AdminStoreResult> { - self.accounts + async fn load_account(&self, account_id: &str) -> AdminStoreResult> { + let record = self + .accounts .load_provider_account(account_id) .await - .map_err(|error| admin_store_error(ENTITY, error))? - .map(|record| admin_account_record(record.summary)) - .transpose() + .map_err(|error| admin_store_error(ENTITY, error))?; + let Some(record) = record else { + return Ok(None); + }; + let now = Utc::now(); + let rate_limited_until = load_rate_limited_until( + self.cooldowns.as_deref(), + std::slice::from_ref(&record.summary), + now.into(), + ) + .await + .get(account_id) + .copied(); + let projection = account_status_projection(&record.summary, now.into(), rate_limited_until); + Ok(Some(AccountPageItem { + account: admin_account_record(record.summary)?, + projection, + })) } async fn load_account_usage( @@ -386,9 +375,9 @@ impl AccountStore for PgAdminAccountStore { .filter(|account| account.provider_kind == provider_kind.as_str()) .filter(|account| { query - .availability + .credential_state .as_ref() - .is_none_or(|expected| expected.matches(account.availability)) + .is_none_or(|expected| expected.matches(account.credential_state)) }) .filter(|account| { query diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs index 62f853bb9..a4a09f1a2 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs @@ -31,8 +31,8 @@ impl ProviderAccountStore for PgProviderAccountRepository { .map(DateTime::::from), next_refresh_at: account.account.next_refresh_at().map(DateTime::::from), enabled: account.account.enabled(), - availability: account.account.availability(), - availability_observed_at: Utc::now(), + credential_state: account.account.credential_state(), + credential_observed_at: Utc::now(), }) .await .map_err(core_store_error) @@ -164,9 +164,9 @@ impl ProviderAccountStore for PgProviderAccountRepository { .collect::>(); let rows = sqlx::query( "select id, credential_revision, provider_quota_json, quota_observed_at, \ - quota_limit_reached \ + quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at \ from provider_accounts \ - where id = any($1) and provider_quota_json is not null", + where id = any($1) and quota_observed_at is not null", ) .bind(account_ids) .fetch_all(&self.pool) @@ -184,24 +184,49 @@ impl ProviderAccountStore for PgProviderAccountRepository { .and_then(|value| CoreCredentialRevision::new(value).ok()) .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; let quota = row - .try_get::("provider_quota_json") - .ok() + .try_get::, _>("provider_quota_json") + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))? .and_then(|value| value.as_object().cloned()) .map(OpaqueProviderData::new) - .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .unwrap_or_else(|| OpaqueProviderData::new(serde_json::Map::new())); let observed_at = row .try_get::, _>("quota_observed_at") .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let limit_reached = row - .try_get::("quota_limit_reached") + let access = row + .try_get::("quota_access_state") + .ok() + .and_then(|value| QuotaAccessState::parse(&value)) + .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let evidence = row + .try_get::, _>("quota_evidence") .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let evidence = match evidence { + Some(value) => Some( + QuotaEvidence::parse(&value) + .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + ), + None => None, + }; + let reset_at = row + .try_get::>, _>("quota_reset_at") + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let access_observed_at = row + .try_get::>, _>("quota_access_observed_at") + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let state = QuotaState::from_persisted( + access, + evidence, + access_observed_at.map(Into::into), + reset_at.map(Into::into), + ) + .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; Ok(QuotaObservation { account_id: CoreProviderAccountId::new(account_id) .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, expected_revision: revision, - quota: Some(quota), - observed_at: Some(observed_at.into()), - limit_reached: Some(limit_reached), + quota, + observed_at: observed_at.into(), + state, }) }) .collect() @@ -211,24 +236,38 @@ impl ProviderAccountStore for PgProviderAccountRepository { &self, observation: QuotaObservation, ) -> Result { - let quota = observation - .quota - .map(|value| { - JsonObject::try_from_value( - "provider_quota_json", - serde_json::Value::Object(value.into_inner()), - QUOTA_MAX_BYTES, - ) - }) - .transpose() - .map_err(core_store_error)?; + let quota = JsonObject::try_from_value( + "provider_quota_json", + serde_json::Value::Object(observation.quota.into_inner()), + QUOTA_MAX_BYTES, + ) + .map_err(core_store_error)?; let updated = self .compare_and_swap_provider_quota( observation.account_id.as_str(), Revision::new(observation.expected_revision.get()).map_err(core_store_error)?, quota, - observation.observed_at.map(DateTime::::from), - observation.limit_reached, + DateTime::::from(observation.observed_at), + observation.state, + ) + .await + .map_err(core_store_error)?; + Ok(if updated { + QuotaWriteOutcome::Updated + } else { + QuotaWriteOutcome::Conflict + }) + } + + async fn apply_quota_access( + &self, + change: QuotaAccessChange, + ) -> Result { + let updated = self + .apply_provider_quota_access( + change.account_id.as_str(), + Revision::new(change.expected_revision.get()).map_err(core_store_error)?, + change.state, ) .await .map_err(core_store_error)?; @@ -245,8 +284,9 @@ impl ProviderAccountStore for PgProviderAccountRepository { account_id: change.account_id.as_str().to_owned(), expected_revision: Revision::new(change.expected_revision.get()) .map_err(core_store_error)?, - availability: change.availability, - availability_observed_at: DateTime::::from(change.observed_at), + credential_state: change.credential_state, + credential_observed_at: DateTime::::from(change.observed_at), + error_reason: change.error_reason, message: change.message, }) .await diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs index 49d9bcdf9..882c2adef 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mapping.rs @@ -1,12 +1,12 @@ //! Admin 账号列表、用量与成本的领域映射。 +use std::time::SystemTime; + use super::*; -use gateway_admin::model::accounts::AccountStatusSignals; - pub(crate) struct AdminAccountListItem { pub(crate) account: ProviderAccountSummary, - pub(crate) status: AdminAccountStatus, + pub(crate) projection: gateway_core::engine::credential::AccountStatusProjection, pub(crate) usage: Option, } @@ -57,24 +57,6 @@ pub(crate) fn account_matches_admin_query( provider_matches && search_matches && status_matches } -pub(crate) fn admin_account_status( - account: &ProviderAccountSummary, - now: DateTime, - rate_limited_until: Option, -) -> AdminAccountStatus { - // 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。 - AccountStatusSignals { - enabled: account.enabled, - availability: account.availability, - access_token_expired: account - .access_token_expires_at - .is_some_and(|expires_at| expires_at <= now), - quota_limit_reached: account.quota_limit_reached, - rate_limited_until, - } - .derive() -} - pub(crate) fn admin_account_summary( accounts: &[ProviderAccountSummary], now: DateTime, @@ -90,7 +72,7 @@ pub(crate) fn admin_account_summary( }; for account in accounts { let until = rate_limited_until.get(&account.id).copied(); - match admin_account_status(account, now, until) { + match account_status_projection(account, now.into(), until).status { AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1), AdminAccountStatus::QuotaExhausted => { summary.quota_exhausted = summary.quota_exhausted.saturating_add(1); @@ -118,7 +100,11 @@ pub(crate) fn sort_admin_account_items( items.sort_by(|left, right| { let ordering = match sort.field { AdminAccountSortField::Email => left.account.email.cmp(&right.account.email), - AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()), + AdminAccountSortField::Status => left + .projection + .status + .sort_rank() + .cmp(&right.projection.status.sort_rank()), AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type), AdminAccountSortField::Usage => left .usage @@ -171,10 +157,11 @@ pub(crate) fn admin_account_record( access_token_expires_at: summary.access_token_expires_at, next_refresh_at: summary.next_refresh_at, enabled: summary.enabled, - availability: summary.availability, - availability_observed_at: summary.availability_observed_at, + credential_state: summary.credential_state, + credential_observed_at: summary.credential_observed_at, + quota: summary.quota, + last_error_reason: summary.last_error_reason, last_error_message: summary.last_error_message, - quota_observed_at: summary.quota_observed_at, created_at: summary.created_at, updated_at: summary.updated_at, }) @@ -197,8 +184,8 @@ pub(crate) fn prepared_account( access_token_expires_at: credential.access_token_expires_at, next_refresh_at: credential.next_refresh_at, enabled: credential.enabled, - availability: credential.availability, - availability_observed_at: credential.availability_observed_at, + credential_state: credential.credential_state, + credential_observed_at: credential.credential_observed_at, }) } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs index b4b32b579..a2f89ebd3 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/mod.rs @@ -4,7 +4,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt, str::FromStr, - time::SystemTime, }; use async_trait::async_trait; @@ -14,7 +13,7 @@ use gateway_admin::{ MutationContext, Revision as AdminRevision, accounts::{ AccountCost, AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage, - AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort, + AccountPageItem, AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort, AccountSortField as AdminAccountSortField, AccountStatus as AdminAccountStatus, AccountSummary, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled, SortDirection as AdminSortDirection, @@ -35,12 +34,13 @@ use gateway_admin::{ use sqlx::{PgPool, Postgres, Row, Transaction}; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision as CoreCredentialRevision, LoadedCredential, + AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, + CredentialRevision as CoreCredentialRevision, CredentialState, LoadedCredential, NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential, ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId, ProviderAccountIdentity, ProviderAccountStore, - ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome, + ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaAccessChange, QuotaAccessState, + QuotaEvidence, QuotaObservation, QuotaState, QuotaWriteOutcome, }; use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind}; use gateway_core::routing::ProviderKind; @@ -63,8 +63,10 @@ mod core_adapter; mod mapping; mod repository; mod rows; +mod runtime; pub use admin_adapter::*; pub(crate) use mapping::*; pub use repository::*; pub use rows::*; +pub(crate) use runtime::*; diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs index 0a9eafbb9..f08137e1f 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs @@ -25,9 +25,15 @@ pub trait ProviderAccountRepository: Send + Sync { &self, account_id: &str, expected_revision: Revision, - quota: Option, - observed_at: Option>, - limit_reached: Option, + quota: JsonObject, + observed_at: DateTime, + state: QuotaState, + ) -> StoreResult; + async fn apply_provider_quota_access( + &self, + account_id: &str, + expected_revision: Revision, + state: QuotaState, ) -> StoreResult; async fn delete_provider_account(&self, id: &str) -> StoreResult; } @@ -93,9 +99,10 @@ impl ProviderAccountRepository for PgProviderAccountRepository { let rows = sqlx::query( "select id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token, - access_token_expires_at, next_refresh_at, enabled, availability, - availability_observed_at, - quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at + access_token_expires_at, next_refresh_at, enabled, credential_state, + credential_observed_at, quota_access_state, quota_evidence, + quota_access_observed_at, quota_reset_at, + quota_observed_at, last_error_reason, last_error_message, created_at, updated_at from provider_accounts where ($1::text is null or provider_kind = $1) and ($2 or enabled) order by provider_kind, name, id", @@ -110,21 +117,21 @@ impl ProviderAccountRepository for PgProviderAccountRepository { async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> { account.validate()?; - let availability = if account.upstream_user_id.is_some() { - account.availability + let credential_state = if account.upstream_user_id.is_some() { + account.credential_state } else { - AccountAvailability::Unknown + CredentialState::Unknown }; sqlx::query( "insert into provider_accounts ( id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, - availability, provider_quota_json, - availability_observed_at, quota_observed_at, created_at, updated_at + credential_state, provider_quota_json, + credential_observed_at, quota_access_observed_at, quota_observed_at, created_at, updated_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, - $14, null, $15, null, now(), greatest(now(), $15) + $14, null, $15, null, null, now(), greatest(now(), $15) )", ) .bind(account.id) @@ -140,8 +147,8 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(account.access_token_expires_at) .bind(account.next_refresh_at) .bind(account.enabled) - .bind(availability.as_str()) - .bind(account.availability_observed_at) + .bind(credential_state.as_str()) + .bind(account.credential_observed_at) .execute(&self.pool) .await .map_err(|_| postgres_unavailable("insert provider account"))?; @@ -214,27 +221,33 @@ impl ProviderAccountRepository for PgProviderAccountRepository { update.validate()?; let result = sqlx::query( "update provider_accounts - set availability = case + set credential_state = case when enabled and upstream_user_id is not null then $3 - else availability + else credential_state end, - availability_observed_at = case + credential_observed_at = case when enabled and upstream_user_id is not null then $4 - else availability_observed_at + else credential_observed_at + end, + last_error_reason = case + when enabled and upstream_user_id is not null then + case when $3 = 'ready' then null else $5 end + else last_error_reason end, last_error_message = case when enabled and upstream_user_id is not null then - case when $3 = 'ready' then null else coalesce($5, last_error_message) end + case when $3 = 'ready' then null else $6 end else last_error_message end, updated_at = case when enabled then greatest(now(), $4) else updated_at end where id = $1 and credential_revision = $2 - and (availability_observed_at is null or availability_observed_at <= $4)", + and (credential_observed_at is null or credential_observed_at <= $4)", ) .bind(update.account_id) .bind(to_i64(update.expected_revision.get())?) - .bind(update.availability.as_str()) - .bind(update.availability_observed_at) + .bind(update.credential_state.as_str()) + .bind(update.credential_observed_at) + .bind(update.error_reason.map(AccountErrorReason::as_str)) .bind(update.message.as_deref()) .execute(&self.pool) .await @@ -259,36 +272,81 @@ impl ProviderAccountRepository for PgProviderAccountRepository { &self, account_id: &str, expected_revision: Revision, - quota: Option, - observed_at: Option>, - limit_reached: Option, + quota: JsonObject, + observed_at: DateTime, + state: QuotaState, ) -> StoreResult { require_nonempty(ENTITY, "account_id", account_id)?; - if quota.is_some() != observed_at.is_some() { - return Err(invalid("quota JSON and observed_at must agree")); - } - if let Some(quota) = quota.as_ref() { - validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?; - } + validate_object_size("provider_quota_json", "a, QUOTA_MAX_BYTES)?; + let access_observed_at = state.observed_at().map(DateTime::::from); let result = sqlx::query( "update provider_accounts set provider_quota_json = $3, quota_observed_at = $4, - quota_limit_reached = coalesce($5, quota_limit_reached), - updated_at = greatest(now(), coalesce($4, now())) + quota_access_state = case + when $7::timestamptz is not null + and (quota_access_observed_at is null or quota_access_observed_at <= $7) + then $5 else quota_access_state end, + quota_evidence = case + when $7::timestamptz is not null + and (quota_access_observed_at is null or quota_access_observed_at <= $7) + then $6 else quota_evidence end, + quota_access_observed_at = case + when $7::timestamptz is not null + and (quota_access_observed_at is null or quota_access_observed_at <= $7) + then $7 else quota_access_observed_at end, + quota_reset_at = case + when $7::timestamptz is not null + and (quota_access_observed_at is null or quota_access_observed_at <= $7) + then $8 else quota_reset_at end, + updated_at = greatest(now(), $4) where id = $1 and credential_revision = $2 - and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)", + and (quota_observed_at is null or quota_observed_at <= $4)", ) .bind(account_id) .bind(to_i64(expected_revision.get())?) - .bind(quota.map(|value| value.as_value())) + .bind(quota.as_value()) .bind(observed_at) - .bind(limit_reached) + .bind(state.access().as_str()) + .bind(state.evidence().map(QuotaEvidence::as_str)) + .bind(access_observed_at) + .bind(state.reset_at().map(DateTime::::from)) .execute(&self.pool) .await .map_err(|_| postgres_unavailable("compare and swap provider quota"))?; Ok(result.rows_affected() == 1) } + async fn apply_provider_quota_access( + &self, + account_id: &str, + expected_revision: Revision, + state: QuotaState, + ) -> StoreResult { + require_nonempty(ENTITY, "account_id", account_id)?; + let observed_at = state + .observed_at() + .map(DateTime::::from) + .ok_or_else(|| invalid("quota access change requires observed_at"))?; + let result = sqlx::query( + "update provider_accounts + set quota_access_observed_at = $3, quota_access_state = $4, + quota_evidence = $5, quota_reset_at = $6, + updated_at = greatest(now(), $3) + where id = $1 and credential_revision = $2 + and (quota_access_observed_at is null or quota_access_observed_at <= $3)", + ) + .bind(account_id) + .bind(to_i64(expected_revision.get())?) + .bind(observed_at) + .bind(state.access().as_str()) + .bind(state.evidence().map(QuotaEvidence::as_str)) + .bind(state.reset_at().map(DateTime::::from)) + .execute(&self.pool) + .await + .map_err(|_| postgres_unavailable("apply provider quota access"))?; + Ok(result.rows_affected() == 1) + } + async fn delete_provider_account(&self, id: &str) -> StoreResult { require_nonempty(ENTITY, "id", id)?; let result = sqlx::query("delete from provider_accounts where id = $1 and not enabled") @@ -474,21 +532,21 @@ pub(crate) async fn upsert_provider_account_in_transaction( account: &NewProviderAccount, ) -> StoreResult { account.validate()?; - let availability = if account.upstream_user_id.is_some() { - account.availability + let credential_state = if account.upstream_user_id.is_some() { + account.credential_state } else { - AccountAvailability::Unknown + CredentialState::Unknown }; let imported_id = sqlx::query_scalar::<_, String>( "insert into provider_accounts ( id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, access_token_expires_at, next_refresh_at, enabled, - availability, provider_quota_json, - availability_observed_at, quota_observed_at, created_at, updated_at + credential_state, provider_quota_json, + credential_observed_at, quota_access_observed_at, quota_observed_at, created_at, updated_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13, - $14, null, $15, null, now(), greatest(now(), $15) + $14, null, $15, null, null, now(), greatest(now(), $15) ) on conflict ( provider_kind, @@ -505,11 +563,17 @@ pub(crate) async fn upsert_provider_account_in_transaction( access_token_expires_at = excluded.access_token_expires_at, next_refresh_at = excluded.next_refresh_at, enabled = excluded.enabled, - availability = excluded.availability, + credential_state = excluded.credential_state, provider_quota_json = null, - availability_observed_at = excluded.availability_observed_at, + quota_access_state = 'unknown', + quota_evidence = null, + quota_access_observed_at = null, + quota_reset_at = null, + credential_observed_at = excluded.credential_observed_at, quota_observed_at = null, - updated_at = greatest(now(), excluded.availability_observed_at) + last_error_reason = null, + last_error_message = null, + updated_at = greatest(now(), excluded.credential_observed_at) returning id", ) .bind(&account.id) @@ -525,8 +589,8 @@ pub(crate) async fn upsert_provider_account_in_transaction( .bind(account.access_token_expires_at) .bind(account.next_refresh_at) .bind(account.enabled) - .bind(availability.as_str()) - .bind(account.availability_observed_at) + .bind(credential_state.as_str()) + .bind(account.credential_observed_at) .fetch_optional(&mut **transaction) .await .map_err(|error| { @@ -574,17 +638,17 @@ pub(crate) async fn rotate_provider_account_in_transaction( next_refresh_at = $10, upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end, upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end, - availability = case - when not enabled then availability - when availability = 'quota_exhausted' then availability + credential_state = case + when not enabled then credential_state when coalesce($12::text, upstream_user_id) is not null then 'ready' else 'unknown' end, - availability_observed_at = case - when not enabled then availability_observed_at - when availability <> 'quota_exhausted' then now() - else availability_observed_at + credential_observed_at = case + when not enabled then credential_observed_at + else now() end, + last_error_reason = case when enabled then null else last_error_reason end, + last_error_message = case when enabled then null else last_error_message end, updated_at = now() where id = $1 and provider_kind = $2 and credential_revision = $3 diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs index af35c670a..ce213471a 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -65,9 +65,29 @@ impl ProviderAccountAdminScope { } } -/// 解析数据库 availability 列;字符串映射本体唯一归属 gateway-core。 -pub(crate) fn parse_availability(value: &str) -> StoreResult { - AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value")) +pub(crate) fn parse_credential_state(value: &str) -> StoreResult { + CredentialState::parse(value).ok_or_else(|| invalid("unknown credential_state value")) +} + +pub(crate) fn parse_quota_access_state(value: &str) -> StoreResult { + QuotaAccessState::parse(value).ok_or_else(|| invalid("unknown quota_access_state value")) +} + +pub(crate) fn parse_quota_evidence(value: Option) -> StoreResult> { + value + .map(|value| { + QuotaEvidence::parse(&value).ok_or_else(|| invalid("unknown quota_evidence value")) + }) + .transpose() +} + +pub(crate) fn parse_error_reason(value: Option) -> StoreResult> { + value + .map(|value| { + AccountErrorReason::parse(&value) + .ok_or_else(|| invalid("unknown last_error_reason value")) + }) + .transpose() } #[derive(Debug, Clone, PartialEq, Eq)] @@ -85,12 +105,10 @@ pub struct ProviderAccountSummary { pub access_token_expires_at: Option>, pub next_refresh_at: Option>, pub enabled: bool, - pub availability: AccountAvailability, - pub availability_observed_at: DateTime, - pub quota_observed_at: Option>, - /// 快照级限流事实(Provider 物化)。 - pub quota_limit_reached: bool, - /// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。 + pub credential_state: CredentialState, + pub credential_observed_at: DateTime, + pub quota: QuotaState, + pub last_error_reason: Option, pub last_error_message: Option, pub created_at: DateTime, pub updated_at: DateTime, @@ -132,8 +150,8 @@ pub struct NewProviderAccount { pub access_token_expires_at: Option>, pub next_refresh_at: Option>, pub enabled: bool, - pub availability: AccountAvailability, - pub availability_observed_at: DateTime, + pub credential_state: CredentialState, + pub credential_observed_at: DateTime, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -152,7 +170,7 @@ impl fmt::Debug for NewProviderAccount { .field("provider_kind", &self.provider_kind) .field("name", &self.name) .field("enabled", &self.enabled) - .field("availability", &self.availability) + .field("credential_state", &self.credential_state) .field("provider_credentials_json", &"[REDACTED]") .finish_non_exhaustive() } @@ -297,24 +315,13 @@ impl fmt::Debug for ProviderCredentialUpdate { } } -#[derive(Debug, Clone, PartialEq)] -pub struct ProviderAccountObservation { - pub account_id: String, - pub availability: AccountAvailability, - pub provider_quota_json: Option, - pub availability_observed_at: DateTime, - pub quota_observed_at: Option>, - /// 快照级限流事实(Provider 物化)。 - pub quota_limit_reached: bool, -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProviderAccountStateUpdate { pub account_id: String, pub expected_revision: Revision, - pub availability: AccountAvailability, - pub availability_observed_at: DateTime, - /// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。 + pub credential_state: CredentialState, + pub credential_observed_at: DateTime, + pub error_reason: Option, pub message: Option, } @@ -325,31 +332,20 @@ impl ProviderAccountStateUpdate { } } -impl ProviderAccountObservation { - pub fn validate(&self) -> StoreResult<()> { - require_nonempty(ENTITY, "account_id", &self.account_id)?; - if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() { - return Err(invalid("quota JSON and quota_observed_at must agree")); - } - if let Some(quota) = &self.provider_quota_json { - validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?; - } - Ok(()) - } -} - pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - provider_quota_json, quota_limit_reached, last_error_message, - availability_observed_at, quota_observed_at, created_at, updated_at + has_refresh_token, access_token_expires_at, next_refresh_at, enabled, credential_state, + provider_quota_json, quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at, + last_error_reason, last_error_message, + credential_observed_at, quota_observed_at, created_at, updated_at from provider_accounts where id = $1"; pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability, - provider_quota_json, quota_limit_reached, last_error_message, - availability_observed_at, quota_observed_at, created_at, updated_at + has_refresh_token, access_token_expires_at, next_refresh_at, enabled, credential_state, + provider_quota_json, quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at, + last_error_reason, last_error_message, + credential_observed_at, quota_observed_at, created_at, updated_at from provider_accounts where id = any($1::text[]) and provider_kind = $2 order by id"; @@ -398,7 +394,13 @@ pub(crate) fn core_account_from_summary( summary.upstream_account_id, summary.plan_type, ) - .with_runtime_state(summary.enabled, summary.availability) + .with_account_facts( + summary.enabled, + summary.credential_state, + summary.quota, + summary.last_error_reason, + summary.last_error_message, + ) .with_refresh_schedule( summary.has_refresh_token, summary.next_refresh_at.map(Into::into), @@ -430,9 +432,20 @@ pub(crate) fn account_summary_from_row( let revision = row .try_get::("credential_revision") .map_err(|_| invalid("invalid credential revision"))?; - let availability = row - .try_get::("availability") - .map_err(|_| invalid("invalid availability"))?; + let credential_state = row + .try_get::("credential_state") + .map_err(|_| invalid("invalid credential_state"))?; + let quota_access_state = parse_quota_access_state(&get::(&row, "quota_access_state")?)?; + let quota_evidence = parse_quota_evidence(get(&row, "quota_evidence")?)?; + let quota_access_observed_at = get::>>(&row, "quota_access_observed_at")?; + let quota_reset_at = get::>>(&row, "quota_reset_at")?; + let quota = QuotaState::from_persisted( + quota_access_state, + quota_evidence, + quota_access_observed_at.map(Into::into), + quota_reset_at.map(Into::into), + ) + .ok_or_else(|| invalid("invalid persisted quota fact"))?; Ok(ProviderAccountSummary { id: get(&row, "id")?, provider_kind: get(&row, "provider_kind")?, @@ -447,10 +460,10 @@ pub(crate) fn account_summary_from_row( access_token_expires_at: get(&row, "access_token_expires_at")?, next_refresh_at: get(&row, "next_refresh_at")?, enabled: get(&row, "enabled")?, - availability: parse_availability(&availability)?, - availability_observed_at: get(&row, "availability_observed_at")?, - quota_observed_at: get(&row, "quota_observed_at")?, - quota_limit_reached: get(&row, "quota_limit_reached")?, + credential_state: parse_credential_state(&credential_state)?, + credential_observed_at: get(&row, "credential_observed_at")?, + quota, + last_error_reason: parse_error_reason(get(&row, "last_error_reason")?)?, last_error_message: get(&row, "last_error_message")?, created_at: get(&row, "created_at")?, updated_at: get(&row, "updated_at")?, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs new file mode 100644 index 000000000..68aff5855 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs @@ -0,0 +1,53 @@ +//! 持久账号事实与可丢失冷却事实的统一状态投影。 + +use std::{collections::BTreeMap, time::SystemTime}; + +use futures::future::join_all; +use gateway_core::{ + engine::credential::{ + AccountStatusFacts, AccountStatusProjection, ProviderAccountId, resolve_account_status, + }, + provider_ports::ProviderCooldownPort, +}; + +use super::ProviderAccountSummary; + +#[must_use] +pub(crate) fn account_status_projection( + account: &ProviderAccountSummary, + now: SystemTime, + rate_limited_until: Option, +) -> AccountStatusProjection { + resolve_account_status( + &AccountStatusFacts { + enabled: account.enabled, + credential_state: account.credential_state, + access_token_expires_at: account.access_token_expires_at.map(Into::into), + quota: account.quota, + rate_limited_until, + last_error_reason: account.last_error_reason, + last_error_message: account.last_error_message.clone(), + }, + now, + ) +} + +/// 尽力读取账号当前有效的 429 冷却。 +/// +/// 冷却属于可丢失的账号级运行时事实:端口不可用、单条读取失败或脏账号 ID +/// 都视为没有冷却,不污染持久账号状态。凭据轮换不会解除上游对同一账号的限流。 +pub(crate) async fn load_rate_limited_until( + cooldowns: Option<&dyn ProviderCooldownPort>, + accounts: &[ProviderAccountSummary], + now: SystemTime, +) -> BTreeMap { + let Some(cooldowns) = cooldowns else { + return BTreeMap::new(); + }; + let reads = accounts.iter().map(|account| async move { + let account_id = ProviderAccountId::new(account.id.clone()).ok()?; + let cooldown = cooldowns.read(&account_id).await.ok().flatten()?; + (cooldown.until() > now).then(|| (account.id.clone(), cooldown.until())) + }); + join_all(reads).await.into_iter().flatten().collect() +} diff --git a/backend/crates/gateway-store/src/postgres/snapshot.rs b/backend/crates/gateway-store/src/postgres/snapshot.rs index 9d220742d..158ff150d 100644 --- a/backend/crates/gateway-store/src/postgres/snapshot.rs +++ b/backend/crates/gateway-store/src/postgres/snapshot.rs @@ -31,7 +31,7 @@ pub struct SnapshotRuntimeSettings { pub struct CredentialRevisionVectorEntry { pub provider_account_id: String, pub credential_revision: Revision, - pub availability_observed_at: DateTime, + pub credential_observed_at: DateTime, pub quota_observed_at: Option>, } @@ -108,7 +108,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { async fn credential_revision_vector(&self) -> StoreResult> { let rows = sqlx::query_as::<_, (String, i64, DateTime, Option>)>( - "select id, credential_revision, availability_observed_at, quota_observed_at + "select id, credential_revision, credential_observed_at, quota_observed_at from provider_accounts order by id", ) .fetch_all(&self.pool) @@ -119,7 +119,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { Ok(CredentialRevisionVectorEntry { provider_account_id: row.0, credential_revision: revision_from_i64(row.1)?, - availability_observed_at: row.2, + credential_observed_at: row.2, quota_observed_at: row.3, }) }) diff --git a/backend/crates/gateway-store/src/redis/credential_leases.rs b/backend/crates/gateway-store/src/redis/credential_leases.rs index f6b3c24b9..e4090ef66 100644 --- a/backend/crates/gateway-store/src/redis/credential_leases.rs +++ b/backend/crates/gateway-store/src/redis/credential_leases.rs @@ -512,7 +512,7 @@ impl RedisProviderLeaseCoordinator { last_started_at: signal.last_started_at.map(Into::into), quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/gateway-store/src/redis/credential_state.rs b/backend/crates/gateway-store/src/redis/credential_state.rs index d2bc4321b..55f740dcf 100644 --- a/backend/crates/gateway-store/src/redis/credential_state.rs +++ b/backend/crates/gateway-store/src/redis/credential_state.rs @@ -1,4 +1,4 @@ -//! Provider account availability 的可重建 Redis cache fence。 +//! Provider credential state 的可重建 Redis cache fence。 use std::time::Duration; @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use redis::{Script, aio::ConnectionManager}; use gateway_core::engine::credential::{ - AccountAvailability, CredentialRevision, OpaqueProviderData, ProviderAccountId, + CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccountId, }; use gateway_core::provider_ports::{ ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCredentialState, @@ -24,7 +24,7 @@ if current >= tonumber(ARGV[1]) then return 0 end redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'enabled', ARGV[2], - 'availability', ARGV[3], + 'credential_state', ARGV[3], 'observed_at_ms', ARGV[4]) redis.call('PEXPIRE', KEYS[1], ARGV[5]) return 1 @@ -44,7 +44,7 @@ pub struct CredentialStateCache { pub provider_account_id: String, pub revision: Revision, pub enabled: bool, - pub availability: String, + pub credential_state: String, pub observed_at: DateTime, } @@ -55,8 +55,8 @@ impl CredentialStateCache { "provider_account_id", &self.provider_account_id, )?; - if AccountAvailability::parse(&self.availability).is_none() { - return Err(invalid("availability is not registered")); + if CredentialState::parse(&self.credential_state).is_none() { + return Err(invalid("credential_state is not registered")); } Ok(()) } @@ -172,7 +172,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository { .key(self.key(&state.provider_account_id)?) .arg(state.revision.get()) .arg(u8::from(state.enabled)) - .arg(&state.availability) + .arg(&state.credential_state) .arg(state.observed_at.timestamp_millis()) .arg(ttl_ms) .invoke_async::(&mut connection) @@ -195,12 +195,12 @@ impl CredentialStateRepository for RedisCredentialStateRepository { .arg(self.key(provider_account_id)?) .arg("revision") .arg("enabled") - .arg("availability") + .arg("credential_state") .arg("observed_at_ms") .query_async(&mut connection) .await .map_err(|_| redis_unavailable("read credential state"))?; - let (Some(revision), Some(enabled), Some(availability), Some(observed_at_ms)) = values + let (Some(revision), Some(enabled), Some(credential_state), Some(observed_at_ms)) = values else { return Ok(None); }; @@ -208,7 +208,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository { provider_account_id: provider_account_id.to_owned(), revision: Revision::new(revision)?, enabled: enabled == 1, - availability, + credential_state, observed_at: DateTime::from_timestamp_millis(observed_at_ms) .ok_or_else(|| invalid("observed_at is invalid"))?, }; @@ -299,7 +299,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { revision: Revision::new(state.credential_revision().get()) .map_err(|_| provider_invalid("encode credential state"))?, enabled: state.enabled(), - availability: state.availability().as_str().to_owned(), + credential_state: state.credential_state().as_str().to_owned(), observed_at, }; CredentialStateRepository::cache_credential_state( @@ -329,13 +329,13 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { .map_err(|_| provider_invalid("decode credential state"))?; let revision = CredentialRevision::new(state.revision.get()) .map_err(|_| provider_invalid("decode credential state"))?; - let availability = AccountAvailability::parse(&state.availability) + let credential_state = CredentialState::parse(&state.credential_state) .ok_or_else(|| provider_invalid("decode credential state"))?; Ok(ProviderCredentialState::new( account_id, revision, state.enabled, - availability, + credential_state, state.observed_at.into(), )) }) diff --git a/backend/crates/gateway-store/tests/postgres/execution.rs b/backend/crates/gateway-store/tests/postgres/execution.rs index b7ae28c8e..1a6e8d3fd 100644 --- a/backend/crates/gateway-store/tests/postgres/execution.rs +++ b/backend/crates/gateway-store/tests/postgres/execution.rs @@ -55,8 +55,8 @@ async fn merged_first_attempt_insert_should_match_sequential_semantics() { id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, enabled, availability, - availability_observed_at, created_at, updated_at + has_refresh_token, access_token_expires_at, enabled, credential_state, + credential_observed_at, created_at, updated_at ) values ( 'acct_merged', 'openai', 'merged', null, 'user-merged', null, null, 'oauth', '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', $1, $1, $1 diff --git a/backend/crates/gateway-store/tests/postgres/observability.rs b/backend/crates/gateway-store/tests/postgres/observability.rs index 11d441953..c506487bc 100644 --- a/backend/crates/gateway-store/tests/postgres/observability.rs +++ b/backend/crates/gateway-store/tests/postgres/observability.rs @@ -134,26 +134,27 @@ async fn dashboard_account_metrics_should_partition_account_statuses() { "insert into provider_accounts ( id, provider_kind, name, upstream_user_id, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, - access_token_expires_at, enabled, availability, - availability_observed_at, created_at, updated_at + access_token_expires_at, enabled, credential_state, + quota_access_state, quota_evidence, quota_access_observed_at, + credential_observed_at, created_at, updated_at ) values ('acct_available', 'openai', 'available', 'user-available', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1, $1, $1, $1), ('acct_expired', 'openai', 'expired', 'user-expired', 'oauth', - '{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready', + '{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready', 'allowed', null, $1, $1, $1, $1), ('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1, $1, $1, $1), ('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'usage_limit_reached', $1, $1, $1, $1), ('acct_banned', 'xai', 'banned', 'user-banned', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned', + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned', 'allowed', null, $1, $1, $1, $1), ('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', + '{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', 'allowed', null, $1, $1, $1, $1)", ) .bind(now) @@ -173,22 +174,22 @@ async fn dashboard_account_metrics_should_partition_account_statuses() { assert_eq!( ( metrics.total, - metrics.enabled, - metrics.active, - metrics.unavailable, - metrics.expired, + metrics.normal, metrics.quota_exhausted, + metrics.rate_limited, metrics.disabled, - metrics.banned, + metrics.error, ), - (6, 5, 2, 4, 1, 1, 1, 1), + (6, 2, 1, 0, 1, 2), ); - // 首页按调度可用性聚合:配额耗尽也属于不可用。 assert_eq!( - metrics.unavailable, - metrics.quota_exhausted + metrics.expired + metrics.disabled + metrics.banned + metrics.total, + metrics.normal + + metrics.quota_exhausted + + metrics.rate_limited + + metrics.disabled + + metrics.error ); - assert_eq!(metrics.total, metrics.active + metrics.unavailable); database.close().await; } @@ -202,35 +203,36 @@ async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligibl "insert into provider_accounts ( id, provider_kind, name, upstream_user_id, authentication_kind, provider_credentials_json, credential_revision, has_refresh_token, - access_token_expires_at, enabled, availability, quota_limit_reached, - availability_observed_at, created_at, updated_at + access_token_expires_at, enabled, credential_state, + quota_access_state, quota_evidence, quota_access_observed_at, + credential_observed_at, created_at, updated_at ) values ('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false, + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1, $1, $1, $1), ('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth', - '{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false, + '{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1, $1, $1, $1), ('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true, + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1, $1, $1, $1), ('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth', - '{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true, + '{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1, $1, $1, $1), ('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth', - '{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true, + '{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1, $1, $1, $1), ('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true, + '{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1, $1, $1, $1), ('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth', - '{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true, + '{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', 'exhausted', 'provider_denied', $1, $1, $1, $1), ('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth', - '{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false, + '{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', 'unknown', null, null, $1, $1, $1), ('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth', - '{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false, + '{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', 'allowed', null, $1, $1, $1, $1)", ) .bind(now) @@ -258,36 +260,21 @@ async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligibl assert_eq!( ( metrics.total, - metrics.enabled, - metrics.active, + metrics.normal, metrics.rate_limited, metrics.quota_exhausted, - metrics.expired, - metrics.invalid, metrics.disabled, - metrics.banned, - metrics.unavailable, + metrics.error, ), - (9, 8, 1, 2, 3, 1, 1, 1, 0, 8), + (9, 1, 2, 3, 1, 2), ); assert_eq!( metrics.total, - metrics.active + metrics.normal + metrics.rate_limited + metrics.quota_exhausted - + metrics.expired - + metrics.invalid + metrics.disabled - + metrics.banned - ); - assert_eq!( - metrics.unavailable, - metrics.rate_limited - + metrics.quota_exhausted - + metrics.expired - + metrics.invalid - + metrics.disabled - + metrics.banned + + metrics.error ); database.close().await; } @@ -1013,8 +1000,8 @@ async fn seed_observability_facts( id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, enabled, availability, - availability_observed_at, created_at, updated_at + has_refresh_token, access_token_expires_at, enabled, credential_state, + credential_observed_at, created_at, updated_at ) values ( 'acct_observe', 'openai', 'primary', 'account@example.invalid', 'user-observe', null, 'pro', 'oauth', '{}'::jsonb, 1, false, $1 + interval '1 day', diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts.rs index 7b1253172..9b4d7855b 100644 --- a/backend/crates/gateway-store/tests/postgres/provider_accounts.rs +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts.rs @@ -10,16 +10,15 @@ use gateway_admin::{ SortDirection, }, observability::TimeRange, - provider_credentials::{ - CredentialAvailabilityFilter, CredentialListQuery, CredentialListWindow, - }, + provider_credentials::{CredentialListQuery, CredentialListWindow, CredentialStateFilter}, }, ports::store::AccountStore, }; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision, OpaqueProviderData, PlaintextCredential, ProviderAccountId, - ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation, + AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, + CredentialRevision, CredentialState, OpaqueProviderData, PlaintextCredential, + ProviderAccountId, ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate, + QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaState, QuotaWriteOutcome, }; use gateway_core::routing::ProviderKind; @@ -59,7 +58,7 @@ async fn provider_account_should_allow_missing_upstream_user_id() { let mut pending = account("acct_pending_identity", "unused"); pending.upstream_user_id = None; pending.email = None; - pending.availability = AccountAvailability::Ready; + pending.credential_state = CredentialState::Ready; repository .insert_provider_account(pending) .await @@ -71,8 +70,11 @@ async fn provider_account_should_allow_missing_upstream_user_id() { .expect("load account") .expect("stored account"); assert_eq!( - (stored.summary.upstream_user_id, stored.summary.availability,), - (None, AccountAvailability::Unknown) + ( + stored.summary.upstream_user_id, + stored.summary.credential_state, + ), + (None, CredentialState::Unknown) ); database.close().await; @@ -92,18 +94,19 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() { } let revision = CredentialRevision::new(1).expect("revision"); for (id, remaining) in [("acct_quota_a", 20), ("acct_quota_b", 80)] { + let observed_at = SystemTime::now(); let outcome = repository .compare_and_swap_quota(QuotaObservation { account_id: ProviderAccountId::new(id).expect("account id"), expected_revision: revision, - quota: Some(OpaqueProviderData::new( + quota: OpaqueProviderData::new( json!({"remaining": remaining}) .as_object() .expect("quota object") .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + ), + observed_at, + state: QuotaState::allowed(observed_at), }) .await .expect("persist quota"); @@ -123,15 +126,12 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() { assert_eq!(observations.len(), 2); assert_eq!(observations[0].account_id.as_str(), "acct_quota_a"); assert_eq!(observations[0].expected_revision, revision); - assert_eq!( - observations[0] - .quota - .as_ref() - .expect("quota") - .expose_to_provider()["remaining"], - 20 + assert_eq!(observations[0].quota.expose_to_provider()["remaining"], 20); + assert!( + observations + .iter() + .all(|value| value.state.access() == QuotaAccessState::Allowed) ); - assert!(observations.iter().all(|value| value.observed_at.is_some())); assert_eq!( current_revision(&database.pool).await, 1, @@ -163,14 +163,14 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() { let quota = |marker: &str, observed_at| QuotaObservation { account_id: account_id.clone(), expected_revision: revision, - quota: Some(OpaqueProviderData::new( + quota: OpaqueProviderData::new( json!({"marker": marker}) .as_object() .expect("quota object") .clone(), - )), - observed_at: Some(observed_at), - limit_reached: None, + ), + observed_at, + state: QuotaState::allowed(observed_at), }; assert_eq!( repository @@ -192,8 +192,9 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() { message: None, account_id: account_id.clone(), expected_revision: revision, - availability: AccountAvailability::QuotaExhausted, + credential_state: CredentialState::Invalid, observed_at: newer, + error_reason: Some(AccountErrorReason::CredentialInvalid), }) .await .expect("persist newer account state"); @@ -203,8 +204,9 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() { message: None, account_id: account_id.clone(), expected_revision: revision, - availability: AccountAvailability::Ready, + credential_state: CredentialState::Ready, observed_at: older, + error_reason: None, }) .await .is_err(), @@ -216,23 +218,20 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() { .await .expect("load fenced account") .expect("fenced account"); - assert_eq!(current.availability(), AccountAvailability::QuotaExhausted); + assert_eq!(current.credential_state(), CredentialState::Invalid); let observed = repository .get_quotas(std::slice::from_ref(&account_id)) .await .expect("load fenced quota") .pop() .expect("fenced quota"); - assert_eq!( - observed.quota.expect("quota document").expose_to_provider()["marker"], - "newer" - ); + assert_eq!(observed.quota.expose_to_provider()["marker"], "newer"); database.close().await; } #[tokio::test] -async fn account_state_message_is_persisted_and_cleared_on_recovery() { +async fn credential_error_message_is_persisted_and_cleared_on_recovery() { let Some(database) = TestDatabase::create("provider_account_state_message").await else { return; }; @@ -249,8 +248,9 @@ async fn account_state_message_is_persisted_and_cleared_on_recovery() { message: Some("upstream returned 402 payment required".to_owned()), account_id: account_id.clone(), expected_revision: revision, - availability: AccountAvailability::QuotaExhausted, + credential_state: CredentialState::Invalid, observed_at: SystemTime::now(), + error_reason: Some(AccountErrorReason::CredentialInvalid), }) .await .expect("persist error message"); @@ -270,8 +270,9 @@ async fn account_state_message_is_persisted_and_cleared_on_recovery() { message: None, account_id: account_id.clone(), expected_revision: revision, - availability: AccountAvailability::Ready, + credential_state: CredentialState::Ready, observed_at: SystemTime::now(), + error_reason: None, }) .await .expect("recover account"); @@ -299,39 +300,34 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u let mut beta = account("acct_beta", "user-beta"); beta.provider_kind = "xai".to_owned(); beta.email = Some("beta@example.invalid".to_owned()); - beta.availability = AccountAvailability::Banned; + beta.credential_state = CredentialState::Banned; let mut charlie = account("acct_charlie", "user-charlie"); charlie.email = Some("charlie@example.invalid".to_owned()); let mut invalid = account("acct_invalid", "user-invalid"); invalid.email = Some("invalid@example.invalid".to_owned()); - invalid.availability = AccountAvailability::Invalid; + invalid.credential_state = CredentialState::Invalid; let mut disabled = account("acct_disabled", "user-disabled"); disabled.email = Some("disabled@example.invalid".to_owned()); disabled.enabled = false; let mut quota_exhausted = account("acct_quota_exhausted", "user-quota-exhausted"); quota_exhausted.email = Some("quota-exhausted@example.invalid".to_owned()); - quota_exhausted.availability = AccountAvailability::QuotaExhausted; for account in [alpha, beta, charlie, invalid, disabled, quota_exhausted] { repository .insert_provider_account(account) .await .expect("insert account list fixture"); } - repository - .compare_and_swap_quota(QuotaObservation { - account_id: ProviderAccountId::new("acct_charlie").expect("account id"), - expected_revision: CredentialRevision::new(1).expect("credential revision"), - quota: Some(OpaqueProviderData::new( - json!({"rate_limit": {"limit_reached": true}}) - .as_object() - .expect("quota object") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: Some(true), - }) - .await - .expect("seed rate-limited account"); + for account_id in ["acct_charlie", "acct_quota_exhausted"] { + let observed_at = SystemTime::now(); + repository + .apply_quota_access(QuotaAccessChange { + account_id: ProviderAccountId::new(account_id).expect("account id"), + expected_revision: CredentialRevision::new(1).expect("credential revision"), + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), + }) + .await + .expect("seed exhausted account"); + } seed_model_request( &database.pool, @@ -425,7 +421,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u usage_page .items .iter() - .map(|account| account.id.as_str()) + .map(|item| item.account.id.as_str()) .collect::>(), ["acct_charlie", "acct_beta"] ); @@ -448,7 +444,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u last_used_page .items .iter() - .map(|account| account.id.as_str()) + .map(|item| item.account.id.as_str()) .collect::>(), ["acct_beta", "acct_charlie"] ); @@ -466,8 +462,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u .expect("filter account directory"); assert_eq!(filtered.total, 1); assert_eq!(filtered.summary, usage_page.summary); - assert_eq!(filtered.items[0].id, "acct_alpha"); - assert_eq!(filtered.items[0].provider_kind.as_str(), "openai"); + assert_eq!(filtered.items[0].account.id, "acct_alpha"); + assert_eq!(filtered.items[0].account.provider_kind.as_str(), "openai"); let error_accounts = store .list_accounts(AccountListQuery { @@ -481,8 +477,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u .await .expect("filter error accounts"); assert_eq!(error_accounts.items.len(), 2); - assert_eq!(error_accounts.items[0].id, "acct_beta"); - assert_eq!(error_accounts.items[1].id, "acct_invalid"); + assert_eq!(error_accounts.items[0].account.id, "acct_beta"); + assert_eq!(error_accounts.items[1].account.id, "acct_invalid"); database.close().await; } @@ -498,11 +494,11 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti &format!("user-xai-{index:03}"), ); credential.provider_kind = "xai".to_owned(); - credential.availability = match index { - 0 => AccountAvailability::Expired, - 1 => AccountAvailability::Banned, - 2 => AccountAvailability::Invalid, - _ => AccountAvailability::Ready, + credential.credential_state = match index { + 0 => CredentialState::Expired, + 1 => CredentialState::Banned, + 2 => CredentialState::Invalid, + _ => CredentialState::Ready, }; repository .insert_provider_account(credential) @@ -516,7 +512,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti .list_credentials( &provider, CredentialListQuery { - availability: None, + credential_state: None, enabled: None, window: CredentialListWindow::All, }, @@ -530,7 +526,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti .list_credentials( &provider, CredentialListQuery { - availability: None, + credential_state: None, enabled: None, window: CredentialListWindow::Page { cursor: None, @@ -547,10 +543,10 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti .list_credentials( &provider, CredentialListQuery { - availability: Some(CredentialAvailabilityFilter::AnyOf(vec![ - gateway_admin::model::accounts::AccountAvailability::Expired, - gateway_admin::model::accounts::AccountAvailability::Banned, - gateway_admin::model::accounts::AccountAvailability::Invalid, + credential_state: Some(CredentialStateFilter::AnyOf(vec![ + CredentialState::Expired, + CredentialState::Banned, + CredentialState::Invalid, ])), enabled: None, window: CredentialListWindow::All, @@ -851,7 +847,7 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it() let mut updated = account("acct_admin_reimport", "user-admin-upsert"); updated.name = "updated import".to_owned(); updated.provider_credentials_json = credential_json("updated-import-secret"); - updated.availability = AccountAvailability::Banned; + updated.credential_state = CredentialState::Banned; let imported = repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), @@ -871,7 +867,7 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it() String, ) = sqlx::query_as( "select name, provider_credentials_json, credential_revision, provider_quota_json, - availability + credential_state from provider_accounts where id = 'acct_admin_upsert'", ) .fetch_one(&database.pool) @@ -920,8 +916,8 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() { access_token_expires_at: Some(Utc::now() + TimeDelta::minutes(5)), next_refresh_at: None, enabled: true, - availability: AccountAvailability::Ready, - availability_observed_at: Utc::now(), + credential_state: CredentialState::Ready, + credential_observed_at: Utc::now(), }) .await .expect("seed provider account"); @@ -1041,32 +1037,32 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() { provider_kind: "openai".to_owned(), }; - let mut cooldown_account = account("acct_admin_b", "user-admin-b"); - cooldown_account.availability = AccountAvailability::Ready; let imported = repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), - accounts: vec![account("acct_admin_a", "user-admin-a"), cooldown_account], + accounts: vec![ + account("acct_admin_a", "user-admin-a"), + account("acct_admin_b", "user-admin-b"), + ], audit: audit("audit_account_batch", "import_batch", "provider_accounts"), }) .await .expect("import provider accounts"); assert_eq!(imported.config_revision.get(), 2); let ready: (bool, String) = - sqlx::query_as("select enabled, availability from provider_accounts where id = $1") + sqlx::query_as("select enabled, credential_state from provider_accounts where id = $1") .bind("acct_admin_a") .fetch_one(&database.pool) .await .expect("load ready imported account"); assert_eq!(ready, (true, "ready".to_owned())); - let cooldown: (bool, String) = - sqlx::query_as("select enabled, availability from provider_accounts where id = $1") + let second: (bool, String) = + sqlx::query_as("select enabled, credential_state from provider_accounts where id = $1") .bind("acct_admin_b") .fetch_one(&database.pool) .await .expect("load cooldown imported account"); - assert!(cooldown.0); - assert_eq!(cooldown.1, "ready"); + assert_eq!(second, (true, "ready".to_owned())); repository .import_provider_accounts(ImportProviderAccounts { @@ -1113,7 +1109,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() { sqlx::query( "update provider_accounts - set availability = 'expired' + set credential_state = 'expired' where id = $1", ) .bind("acct_admin_a") @@ -1137,7 +1133,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() { assert_eq!(rotation.config_revision.get(), 3); assert_eq!(rotation.credential_revision.get(), 2); let restored: (String, String, Option) = sqlx::query_as( - "select availability, upstream_user_id, upstream_account_id + "select credential_state, upstream_user_id, upstream_account_id from provider_accounts where id = $1", ) .bind("acct_admin_a") @@ -1243,12 +1239,10 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() { let scope = ProviderAccountAdminScope { provider_kind: "openai".to_owned(), }; - let mut exhausted = account("acct_rotation_quota", "user-rotation-quota"); - exhausted.availability = AccountAvailability::QuotaExhausted; repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), - accounts: vec![exhausted], + accounts: vec![account("acct_rotation_quota", "user-rotation-quota")], audit: audit( "audit_rotation_quota_import", "import", @@ -1257,6 +1251,15 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() { }) .await .expect("import exhausted account"); + let observed_at = SystemTime::now(); + repository + .apply_quota_access(QuotaAccessChange { + account_id: ProviderAccountId::new("acct_rotation_quota").expect("account ID"), + expected_revision: CredentialRevision::new(1).expect("credential revision"), + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), + }) + .await + .expect("persist quota exhaustion"); repository .rotate_provider_account(RotateProviderAccount { @@ -1273,13 +1276,13 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() { .await .expect("rotate exhausted account credential"); - let availability: String = - sqlx::query_scalar("select availability from provider_accounts where id = $1") + let quota_access_state: String = + sqlx::query_scalar("select quota_access_state from provider_accounts where id = $1") .bind("acct_rotation_quota") .fetch_one(&database.pool) .await .expect("load exhausted account after credential rotation"); - assert_eq!(availability, "quota_exhausted".to_owned()); + assert_eq!(quota_access_state, "exhausted".to_owned()); database.close().await; } @@ -1298,26 +1301,25 @@ async fn xai_free_quota_exhaustion_state_is_persisted() { .await .expect("insert xAI account"); + let observed_at = SystemTime::now(); repository - .apply_state_change(AccountStateChange { - message: None, + .apply_quota_access(QuotaAccessChange { account_id: account_id.clone(), expected_revision: CredentialRevision::new(1).expect("credential revision"), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), + state: QuotaState::exhausted(QuotaEvidence::AccountLimitReached, observed_at, None), }) .await .expect("persist xAI free quota exhaustion"); let current: (String, String) = sqlx::query_as( - "select provider_kind, availability + "select provider_kind, quota_access_state from provider_accounts where id = $1", ) .bind(account_id.as_str()) .fetch_one(&database.pool) .await .expect("load persisted xAI quota state"); - assert_eq!(current, ("xai".to_owned(), "quota_exhausted".to_owned()),); + assert_eq!(current, ("xai".to_owned(), "exhausted".to_owned()),); database.close().await; } @@ -1338,25 +1340,29 @@ async fn xai_resettable_usage_limit_state_is_persisted() { let observed_at = SystemTime::now(); repository - .apply_state_change(AccountStateChange { - message: None, + .apply_quota_access(QuotaAccessChange { account_id: account_id.clone(), expected_revision: CredentialRevision::new(1).expect("credential revision"), - availability: AccountAvailability::Ready, - observed_at, + state: QuotaState::exhausted( + QuotaEvidence::UsageLimitReached, + observed_at, + Some(observed_at + Duration::from_secs(60)), + ), }) .await .expect("persist xAI resettable usage limit"); - let current: (String, String) = sqlx::query_as( - "select provider_kind, availability + let current: (String, String, Option>) = sqlx::query_as( + "select provider_kind, quota_access_state, quota_reset_at from provider_accounts where id = $1", ) .bind(account_id.as_str()) .fetch_one(&database.pool) .await .expect("load persisted xAI usage limit state"); - assert_eq!(current, ("xai".to_owned(), "ready".to_owned())); + assert_eq!(current.0, "xai"); + assert_eq!(current.1, "exhausted"); + assert!(current.2.is_some()); database.close().await; } @@ -1373,7 +1379,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() { let account_id = ProviderAccountId::new("acct_disabled_refresh").expect("account ID"); let mut disabled = account(account_id.as_str(), "user-disabled-refresh"); disabled.enabled = false; - disabled.availability = AccountAvailability::Expired; + disabled.credential_state = CredentialState::Expired; repository .import_provider_accounts(ImportProviderAccounts { scope: scope.clone(), @@ -1392,8 +1398,9 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() { message: None, account_id: account_id.clone(), expected_revision: CredentialRevision::new(1).expect("credential revision"), - availability: AccountAvailability::Ready, + credential_state: CredentialState::Ready, observed_at: SystemTime::now(), + error_reason: None, }) .await .expect("disabled state write is a no-op"); @@ -1413,7 +1420,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() { .expect("refresh disabled account credential"); let current: (bool, String, serde_json::Value, i64) = sqlx::query_as( - "select enabled, availability, provider_credentials_json, credential_revision + "select enabled, credential_state, provider_credentials_json, credential_revision from provider_accounts where id = $1", ) .bind(account_id.as_str()) @@ -1443,8 +1450,8 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount { access_token_expires_at: Some(Utc::now() + TimeDelta::hours(1)), next_refresh_at: None, enabled: true, - availability: AccountAvailability::Ready, - availability_observed_at: Utc::now(), + credential_state: CredentialState::Ready, + credential_observed_at: Utc::now(), } } @@ -1570,13 +1577,6 @@ fn provider_credentials_are_redacted_from_debug() { assert!(!format!("{object:?}").contains(secret)); } -#[test] -fn quota_exhausted_account_accepts_cooldown_runtime_facts() { - let mut imported = account("acct_quota_cooldown", "user-quota-cooldown"); - imported.availability = AccountAvailability::QuotaExhausted; - assert!(imported.validate().is_ok()); -} - #[test] fn imported_ready_account_passes_validation() { let imported = account("acct_ready_import", "user-ready-import"); diff --git a/backend/crates/gateway-store/tests/postgres/runtime_settings.rs b/backend/crates/gateway-store/tests/postgres/runtime_settings.rs index 11b02ba99..55a9143e5 100644 --- a/backend/crates/gateway-store/tests/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/tests/postgres/runtime_settings.rs @@ -108,7 +108,7 @@ async fn insert_refreshable_account( "insert into provider_accounts ( id, provider_kind, name, upstream_user_id, authentication_kind, provider_credentials_json, has_refresh_token, access_token_expires_at, - next_refresh_at, availability, availability_observed_at, created_at, updated_at + next_refresh_at, credential_state, credential_observed_at, created_at, updated_at ) values ($1, 'openai', $1, $1, 'oauth', '{}'::jsonb, true, $2, $3, 'ready', now(), now(), now())", ) diff --git a/backend/crates/gateway-store/tests/postgres/snapshots.rs b/backend/crates/gateway-store/tests/postgres/snapshots.rs index e8467d937..7b43a7a83 100644 --- a/backend/crates/gateway-store/tests/postgres/snapshots.rs +++ b/backend/crates/gateway-store/tests/postgres/snapshots.rs @@ -722,8 +722,8 @@ async fn seed_account( id, provider_kind, name, email, upstream_user_id, upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision, - has_refresh_token, access_token_expires_at, enabled, availability, - availability_observed_at, created_at, updated_at + has_refresh_token, access_token_expires_at, enabled, credential_state, + credential_observed_at, created_at, updated_at ) values ( $1, 'openai', $2, $3, 'user-' || $1, null, null, $4, '{}'::jsonb, 1, false, $5 + interval '1 day', true, 'ready', diff --git a/backend/crates/gateway-store/tests/redis/credential_state.rs b/backend/crates/gateway-store/tests/redis/credential_state.rs index 39d97d4dc..71c27c0ca 100644 --- a/backend/crates/gateway-store/tests/redis/credential_state.rs +++ b/backend/crates/gateway-store/tests/redis/credential_state.rs @@ -19,7 +19,7 @@ fn credential_state_rejects_provider_specific_status() { provider_account_id: "account-1".to_owned(), revision: Revision::new(1).expect("positive revision"), enabled: true, - availability: "codex_special".to_owned(), + credential_state: "codex_special".to_owned(), observed_at: Utc::now(), }; assert!(state.validate().is_err()); diff --git a/backend/crates/providers/openai/src/admin.rs b/backend/crates/providers/openai/src/admin.rs index a63906458..1bc659587 100644 --- a/backend/crates/providers/openai/src/admin.rs +++ b/backend/crates/providers/openai/src/admin.rs @@ -25,9 +25,8 @@ use gateway_admin::model::{MutationActor, MutationContext, Revision}; use gateway_admin::ports::provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind}; use gateway_core::accounting::Money; use gateway_core::engine::credential::{ - AccountAvailability, CredentialRevision, LoadedCredential, NewProviderAccount, - OpaqueProviderData, PlaintextCredential, ProviderAccount, ProviderAccountId, - ProviderAccountStore, + CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData, + PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore, }; use gateway_core::error::StoreErrorKind; use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload}; @@ -424,14 +423,7 @@ impl ProviderAdmin for OpenAiAdminProvider { .await .map_err(map_quota_error)? }; - let mut quota = project_quota(snapshot, &account); - quota.rate_limited_until = self - .quota - .rate_limited_until(account.id(), account.revision()) - .await - .map_err(map_quota_error)? - .map(DateTime::::from); - Ok(quota) + Ok(project_quota(snapshot, &account)) } async fn models( @@ -519,7 +511,6 @@ fn prepared_create( account, credential, } = prepared; - let availability = account.availability(); Ok(PreparedCredentialCreate { account_id: account.id().clone(), provider_kind: account.provider().clone(), @@ -534,8 +525,8 @@ fn prepared_create( access_token_expires_at: account.access_token_expires_at().map(DateTime::::from), next_refresh_at: account.next_refresh_at().map(DateTime::::from), enabled: account.enabled(), - availability, - availability_observed_at: observed_at, + credential_state: account.credential_state(), + credential_observed_at: observed_at, }) } @@ -627,7 +618,13 @@ fn account_from_record(account: &AccountRecord) -> Result ProviderQuota { refresh_token_expires_at: None, windows: Vec::new(), limit_reached: false, - rate_limited_until: None, provider_data: None, } } @@ -696,13 +692,10 @@ fn project_quota( snapshot: Option, account: &ProviderAccount, ) -> ProviderQuota { - // 展示前先滚动已过期窗口,避免 reset 后的旧窗口值悬挂。 let mut quota = snapshot - .map(|snapshot| project_quota_snapshot(snapshot.roll_expired_windows(SystemTime::now()))) + .map(project_quota_snapshot) .unwrap_or_else(empty_quota); - if account.availability() == AccountAvailability::QuotaExhausted { - force_confirmed_exhaustion_projection(&mut quota); - } + quota.limit_reached = account.quota().is_exhausted(); quota } @@ -717,7 +710,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota ); provider_data.insert( "exhausted".to_owned(), - Value::Bool(snapshot.fact().exhausted()), + Value::Bool(snapshot.quota().is_exhausted()), ); let windows: Vec = snapshot .windows() @@ -766,7 +759,6 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota refresh_token_expires_at: None, windows, limit_reached, - rate_limited_until: None, provider_data: Some(ProviderDocument::new(OpaqueProviderData::new( provider_data, ))), @@ -788,52 +780,6 @@ fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool { windows.iter().any(|window| window.limit_reached) } -fn force_confirmed_exhaustion_projection(quota: &mut ProviderQuota) { - // 402 已确认耗尽:只覆盖 Admin 展示投影,原始 Provider JSON 仍作为 - // 后续判断窗口是否真正恢复的基线。 - quota.limit_reached = true; - quota.provider_data = Some(ProviderDocument::new(OpaqueProviderData::new( - Map::from_iter([ - ( - "remaining_percent".to_owned(), - Value::Number(Number::from(0)), - ), - ("exhausted".to_owned(), Value::Bool(true)), - ]), - ))); - if quota.windows.is_empty() { - quota.windows.push(ProviderQuotaWindow { - key: "confirmed-quota-exhaustion".to_owned(), - group: "other".to_owned(), - label: "额度".to_owned(), - source: None, - local_usage_attribution: QuotaLocalUsageAttribution::Unavailable, - window_seconds: None, - used_percent: Some(100.0), - reset_at: None, - limit_reached: true, - local_usage: None, - provider_data: None, - }); - return; - } - - // 选已用比例最高的窗口强制 100%,保留其自身 reset_at。 - let index = quota - .windows - .iter() - .enumerate() - .max_by(|(_, left), (_, right)| { - left.used_percent - .unwrap_or(-1.0) - .total_cmp(&right.used_percent.unwrap_or(-1.0)) - }) - .map(|(index, _)| index) - .unwrap_or(0); - quota.windows[index].used_percent = Some(100.0); - quota.windows[index].limit_reached = true; -} - const fn quota_group(kind: CodexQuotaWindowKind) -> &'static str { match kind { CodexQuotaWindowKind::Monthly => "monthly", diff --git a/backend/crates/providers/openai/src/config.rs b/backend/crates/providers/openai/src/config.rs index 5e9b83d40..3bf56df7b 100644 --- a/backend/crates/providers/openai/src/config.rs +++ b/backend/crates/providers/openai/src/config.rs @@ -68,11 +68,6 @@ impl OpenAiConfig { self.quota.refresh_policy() } - #[must_use] - pub const fn quota_skip_exhausted(&self) -> bool { - self.quota.skip_exhausted - } - #[must_use] pub fn token_client_config(&self) -> TokenClientConfig { TokenClientConfig { @@ -209,14 +204,12 @@ impl CodexWebSocketPoolSettings { #[serde(deny_unknown_fields)] pub struct CodexQuotaSettings { pub refresh_interval_minutes: u64, - pub skip_exhausted: bool, } impl Default for CodexQuotaSettings { fn default() -> Self { Self { refresh_interval_minutes: 15, - skip_exhausted: true, } } } diff --git a/backend/crates/providers/openai/src/credential/admin.rs b/backend/crates/providers/openai/src/credential/admin.rs index 70f5cf36e..13a1b5b91 100644 --- a/backend/crates/providers/openai/src/credential/admin.rs +++ b/backend/crates/providers/openai/src/credential/admin.rs @@ -10,9 +10,9 @@ use std::time::SystemTime; use chrono::{DateTime, FixedOffset, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential, + AccountErrorReason, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountIdentity, - ProviderAccountUpdate, + ProviderAccountUpdate, QuotaEvidence, QuotaState, }; use gateway_core::provider_ports::{ ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest, @@ -433,7 +433,13 @@ impl CodexCredentialAdmin { Some(input.verified_account.chatgpt_account_id), input.verified_account.plan_type, ) - .with_runtime_state(input.enabled, AccountAvailability::Ready) + .with_account_facts( + input.enabled, + CredentialState::Ready, + QuotaState::unknown(), + None, + None, + ) .with_refresh_schedule( input.secret.refresh_token.is_some(), optional_time(input.next_refresh_at), @@ -479,13 +485,16 @@ impl CodexCredentialAdmin { optional_time(input.access_token_expires_at), ) .with_profile(email, upstream_account_id, plan_type) - .with_runtime_state( + .with_account_facts( input.enabled, if has_upstream_user_id { - AccountAvailability::Ready + CredentialState::Ready } else { - AccountAvailability::Unknown + CredentialState::Unknown }, + QuotaState::unknown(), + (!has_upstream_user_id).then_some(AccountErrorReason::AccountUnverified), + None, ) .with_refresh_schedule( input.secret.refresh_token.is_some(), @@ -898,10 +907,10 @@ impl CodexCredentialAdminService { if !account_ids.insert(account_id.clone()) { return Err(CodexCredentialAdminError::InvalidInput); } - let (enabled, availability) = - import_runtime_state(candidate.status.as_deref())?; + let facts = + parse_cpr_account_facts(candidate.status.as_deref(), SystemTime::now())?; let mut account = - self.prepare_agent_identity_import(&candidate, account_id, enabled)?; + self.prepare_agent_identity_import(&candidate, account_id, facts.enabled)?; let upstream_user_id = account .account .upstream_user_id() @@ -915,7 +924,7 @@ impl CodexCredentialAdminService { if !upstream_identities.insert((upstream_user_id, upstream_account_id)) { return Err(CodexCredentialAdminError::InvalidInput); } - account.account = account.account.with_runtime_state(enabled, availability); + account.account = facts.apply(account.account); account } }; @@ -1048,7 +1057,13 @@ impl CodexCredentialAdminService { Some(upstream_account_id), candidate.plan_type.clone(), ) - .with_runtime_state(enabled, AccountAvailability::Ready) + .with_account_facts( + enabled, + CredentialState::Ready, + QuotaState::unknown(), + None, + None, + ) .with_refresh_schedule(false, None); Ok(NewProviderAccount { account, @@ -1065,11 +1080,13 @@ fn cpr_status(account: &ProviderAccount) -> &'static str { if !account.enabled() { return "disabled"; } - match account.availability() { - AccountAvailability::QuotaExhausted => "quota_exhausted", - AccountAvailability::Expired | AccountAvailability::Invalid => "expired", - AccountAvailability::Banned => "banned", - AccountAvailability::Unknown | AccountAvailability::Ready => "active", + if account.quota().is_exhausted() { + return "quota_exhausted"; + } + match account.credential_state() { + CredentialState::Expired | CredentialState::Invalid => "expired", + CredentialState::Banned => "banned", + CredentialState::Unknown | CredentialState::Ready => "active", } } @@ -1292,20 +1309,65 @@ fn normalize_import_text(value: &str, max_bytes: usize) -> Option { .then(|| value.to_owned()) } -fn import_runtime_state( +struct ImportedCprAccountFacts { + enabled: bool, + credential_state: CredentialState, + quota: QuotaState, + error_reason: Option, +} + +impl ImportedCprAccountFacts { + fn apply(self, account: ProviderAccount) -> ProviderAccount { + account.with_account_facts( + self.enabled, + self.credential_state, + self.quota, + self.error_reason, + None, + ) + } +} + +fn parse_cpr_account_facts( status: Option<&str>, -) -> Result<(bool, AccountAvailability), CodexCredentialAdminError> { + observed_at: SystemTime, +) -> Result { match status .map(str::trim) .unwrap_or("active") .to_ascii_lowercase() .as_str() { - "active" => Ok((true, AccountAvailability::Ready)), - "disabled" => Ok((false, AccountAvailability::Ready)), - "expired" => Ok((true, AccountAvailability::Expired)), - "quota_exhausted" => Ok((true, AccountAvailability::QuotaExhausted)), - "banned" => Ok((true, AccountAvailability::Banned)), + "active" => Ok(ImportedCprAccountFacts { + enabled: true, + credential_state: CredentialState::Ready, + quota: QuotaState::unknown(), + error_reason: None, + }), + "disabled" => Ok(ImportedCprAccountFacts { + enabled: false, + credential_state: CredentialState::Ready, + quota: QuotaState::unknown(), + error_reason: None, + }), + "expired" => Ok(ImportedCprAccountFacts { + enabled: true, + credential_state: CredentialState::Expired, + quota: QuotaState::unknown(), + error_reason: Some(AccountErrorReason::CredentialExpired), + }), + "quota_exhausted" => Ok(ImportedCprAccountFacts { + enabled: true, + credential_state: CredentialState::Ready, + quota: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), + error_reason: None, + }), + "banned" => Ok(ImportedCprAccountFacts { + enabled: true, + credential_state: CredentialState::Banned, + quota: QuotaState::unknown(), + error_reason: Some(AccountErrorReason::AccountBanned), + }), _ => Err(CodexCredentialAdminError::InvalidInput), } } diff --git a/backend/crates/providers/openai/src/credential/catalog.rs b/backend/crates/providers/openai/src/credential/catalog.rs index 5dd4825fc..6418cb879 100644 --- a/backend/crates/providers/openai/src/credential/catalog.rs +++ b/backend/crates/providers/openai/src/credential/catalog.rs @@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, OpaqueProviderData, ProviderAccount, ProviderAccountId, + CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId, }; use gateway_core::engine::provider::ProviderCatalogGeneration; use gateway_core::provider_ports::{ @@ -780,12 +780,8 @@ fn eligible_catalog_account(account: &ProviderAccount, now: SystemTime) -> bool && account .access_token_expires_at() .is_none_or(|expires_at| expires_at > now) - && match account.availability() { - AccountAvailability::Unknown - | AccountAvailability::Ready - | AccountAvailability::QuotaExhausted => true, - AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid => false, - } + && matches!( + account.credential_state(), + CredentialState::Unknown | CredentialState::Ready + ) } diff --git a/backend/crates/providers/openai/src/credential/quota/evidence.rs b/backend/crates/providers/openai/src/credential/quota/evidence.rs new file mode 100644 index 000000000..3f374359f --- /dev/null +++ b/backend/crates/providers/openai/src/credential/quota/evidence.rs @@ -0,0 +1,46 @@ +//! 额度接口失败到规范化账号事实的唯一分类边界。 + +use gateway_core::engine::credential::{AccountErrorReason, CredentialState, QuotaEvidence}; +use reqwest::StatusCode; +use serde_json::Value; + +use crate::transport::CodexClientError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum QuotaEndpointFailure { + Credential { + state: CredentialState, + reason: AccountErrorReason, + }, + Exhausted(QuotaEvidence), +} + +/// 额度 endpoint 只有明确 402 能改变账号事实。401/403、429、5xx 与传输失败 +/// 均不足以判断凭据或账号状态,留给 OAuth 与真实推理请求确认。 +pub(crate) fn classify_quota_endpoint_failure( + error: &CodexClientError, +) -> Option { + let CodexClientError::Upstream { status, body, .. } = error else { + return None; + }; + if *status != StatusCode::PAYMENT_REQUIRED { + return None; + } + if is_deactivated_workspace(body) { + return Some(QuotaEndpointFailure::Credential { + state: CredentialState::Banned, + reason: AccountErrorReason::AccountBanned, + }); + } + Some(QuotaEndpointFailure::Exhausted( + QuotaEvidence::PaymentRequired, + )) +} + +fn is_deactivated_workspace(body: &str) -> bool { + serde_json::from_str::(body) + .ok() + .is_some_and(|value| { + value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace") + }) +} diff --git a/backend/crates/providers/openai/src/credential/quota/mod.rs b/backend/crates/providers/openai/src/credential/quota/mod.rs index bac8a6c0d..4c9cfd964 100644 --- a/backend/crates/providers/openai/src/credential/quota/mod.rs +++ b/backend/crates/providers/openai/src/credential/quota/mod.rs @@ -1,9 +1,9 @@ -//! Codex quota 服务编排:请求级额度投影、被动/主动同步与 429 合成。 +//! Codex quota 服务编排:权威额度事实、展示投影、主动/被动同步与 429 冷却。 //! //! - [`snapshot`]:快照/窗口解析、聚合、O(1) 滚动与调度信号。 -//! - [`recovery`]:402 恢复证据与 availability 转换。 +//! - [`evidence`]:额度接口错误到凭据/额度事实的分类。 -pub(crate) mod recovery; +pub(crate) mod evidence; pub(crate) mod snapshot; pub use snapshot::{ @@ -13,12 +13,14 @@ pub use snapshot::{ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, RwLock}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime}; use chrono::Utc; use gateway_core::engine::credential::{ - AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData, - ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, QuotaWriteOutcome, + AccountErrorReason, AccountQuotaSignals, CredentialRevision, CredentialState, + OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore, + QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaState, + QuotaWriteOutcome, }; use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort}; use gateway_protocol::openai::events::{ @@ -36,10 +38,7 @@ use crate::transport::{CodexBackendClient, CodexClientError, CodexRequestContext use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeCredential}; use super::repository::{CodexCredentialRepository, CredentialRepositoryError}; -use recovery::{ - authoritative_quota_snapshot_state, quota_state_reason, quota_state_transition, - quota_success_state, -}; +use evidence::{QuotaEndpointFailure, classify_quota_endpoint_failure}; use snapshot::{ parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation, scheduling_signals_from_snapshot, @@ -48,7 +47,8 @@ use snapshot::{ const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60); pub(crate) const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60); const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5); -const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60); +const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(30 * 60); +const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60); /// 首次 OAuth 异步观察失败时,由既有 quota worker 兜底重试的单轮上限。 const INITIAL_QUOTA_SYNC_BATCH: usize = 100; // 5xx 上游拒绝的短退避重试预算;指数退避 1s/2s,吞掉瞬时抖动。 @@ -130,6 +130,12 @@ pub struct CodexCredentialQuotaService { scheduling: CodexQuotaSchedulingProjection, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum QuotaRefreshAuthority { + ObserveAccess, + PreserveAccess, +} + #[derive(Clone, Default)] struct CodexQuotaSchedulingProjection { state: Arc>, @@ -299,12 +305,7 @@ impl CodexQuotaSchedulingProjection { ) -> Vec { let candidates = accounts .into_iter() - .filter_map(|account| { - let limit_reached = self - .signals(&account) - .is_some_and(|signals| signals.limit_reached()); - quota_refresh_candidate(account, now, limit_reached) - }) + .filter_map(|account| quota_refresh_candidate(account, now)) .collect::>(); let candidate_ids = candidates .iter() @@ -319,8 +320,8 @@ impl CodexQuotaSchedulingProjection { .last_periodic_refresh_at .retain(|account_id, _| candidate_ids.contains(account_id)); - // 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只兜底 - // 已耗尽账号,且不依据 reset 时间跳过复核;每个账号仍至少间隔 30 分钟。 + // 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只复核 + // 已耗尽且已到 reset/fallback 时刻的账号;重复复核至少间隔 30 分钟。 let mut reserved = Vec::new(); for account in candidates { if !periodic_quota_refresh_due(&state, account.id(), refreshed_at) { @@ -335,16 +336,13 @@ impl CodexQuotaSchedulingProjection { } } -fn quota_refresh_candidate( - account: ProviderAccount, - now: SystemTime, - limit_reached: bool, -) -> Option { - // 定时器只兜底已耗尽或任一窗口触顶的账号; - // 正常账号由真实请求的响应头被动同步。 +fn quota_refresh_candidate(account: ProviderAccount, now: SystemTime) -> Option { + // reset 到期是“应刷新”的信号,不是“已恢复”的证据。 (eligible_periodic_quota_refresh(&account, now) - && (account.availability() == AccountAvailability::QuotaExhausted || limit_reached)) - .then_some(account) + && account + .quota() + .exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL)) + .then_some(account) } fn periodic_quota_refresh_due( @@ -356,7 +354,7 @@ fn periodic_quota_refresh_due( .last_periodic_refresh_at .get(account_id) .is_none_or(|last| { - now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL + now.saturating_duration_since(*last) >= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL }) } @@ -400,70 +398,55 @@ impl CodexCredentialQuotaService { } } - /// 写入新快照前读取 core primary 已确认耗尽窗口的 reset,用于判断该窗口是否真的推进。 - async fn previous_confirmed_reset(&self, account: &ProviderAccount) -> Option { - let observations = self - .store - .get_quotas(std::slice::from_ref(account.id())) - .await - .ok()?; - let observation = observations.into_iter().find(|observation| { - observation.account_id == *account.id() - && observation.expected_revision == account.revision() - })?; - let snapshot = quota_snapshot_from_observation(&observation)?; - snapshot - .core_primary_window() - .and_then(|window| window.reset_at()) - .map(SystemTime::from) - } - - /// 将已确认的额度拒绝投影为 100%,使展示、调度和恢复共用同一窗口事实。 - /// - /// `reset_at` 来自官方 `usage_limit_reached` 错误体;缺失时保留当前快照的 reset。 + /// 真实推理错误只更新额度访问事实,不伪造 Provider JSON 或展示百分比。 pub(crate) async fn record_confirmed_exhaustion( &self, account: &ProviderAccount, - reset_at: Option, + evidence: QuotaEvidence, + reset_at: Option, + observed_at: SystemTime, ) -> Result<(), CodexCredentialQuotaError> { - let existing = self - .store - .get_quotas(std::slice::from_ref(account.id())) - .await? - .into_iter() - .find(|observation| { - observation.account_id == *account.id() - && observation.expected_revision == account.revision() - }) - .and_then(|observation| observation.quota) - .map(OpaqueProviderData::into_inner) - .unwrap_or_default(); - let quota = confirmed_exhaustion_projection( - normalize_quota_window_placeholders(existing), - reset_at, - ); - let observed_at = SystemTime::now(); let outcome = self .store - .compare_and_swap_quota(QuotaObservation { + .apply_quota_access(QuotaAccessChange { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new(quota.clone())), - observed_at: Some(observed_at), - limit_reached: Some(true), + state: QuotaState::exhausted(evidence, observed_at, reset_at), }) .await?; if outcome == QuotaWriteOutcome::Conflict { return Err(CodexCredentialQuotaError::RevisionConflict); } - if let Ok(snapshot) = parse_account_quota_snapshot( - account.id().clone(), - account.revision(), - observed_at, - &Value::Object(quota), - ) { - self.scheduling.observe(&snapshot); + Ok(()) + } + + /// 成功推理是额度可访问的权威证据,同时解除账号级 429 冷却。 + pub(crate) async fn record_successful_inference( + &self, + account: &ProviderAccount, + observed_at: SystemTime, + ) -> Result<(), CodexCredentialQuotaError> { + // 已经发出的并发请求可能晚于耗尽事实成功返回;真实成功仍是独立的 + // Allowed 权威证据,但调度不会为了试探恢复而放行耗尽账号。 + if account.quota().access() == QuotaAccessState::Exhausted { + let outcome = self + .store + .apply_quota_access(QuotaAccessChange { + account_id: account.id().clone(), + expected_revision: account.revision(), + state: QuotaState::allowed(observed_at), + }) + .await?; + if outcome == QuotaWriteOutcome::Conflict { + return Err(CodexCredentialQuotaError::RevisionConflict); + } } + self.cooldowns + .clear(account.id(), account.revision()) + .await + .map_err(|error| CodexCredentialQuotaError::Store { + detail: error.to_string(), + })?; Ok(()) } @@ -562,30 +545,27 @@ impl CodexCredentialQuotaService { ); } Err(CodexQuotaFetchError::Upstream { account, error }) => { - match quota_state_transition(&error) { - // 402:真额度耗尽,写入 QuotaExhausted(worker 按 reset_at 自动恢复)。 - Some(AccountAvailability::QuotaExhausted) => { + match classify_quota_endpoint_failure(&error) { + Some(QuotaEndpointFailure::Exhausted(evidence)) => { summary.exhausted += 1; - self.persist_upstream_state( - &account, - AccountAvailability::QuotaExhausted, - &error, - observed_at, - ) - .await; + if let Err(write_error) = self + .record_confirmed_exhaustion(&account, evidence, None, observed_at) + .await + { + summary.transient += 1; + tracing::warn!( + account_id = %account.id(), + error = %write_error, + "OpenAI quota exhaustion fact write failed" + ); + } } - Some(AccountAvailability::Banned) => { + Some(QuotaEndpointFailure::Credential { state, reason }) => { summary.banned += 1; - self.persist_upstream_state( - &account, - AccountAvailability::Banned, - &error, - observed_at, - ) - .await; + self.persist_credential_failure(&account, state, reason, observed_at) + .await; } - // usage 401/403 与 429/503 都不写 availability;前者等待 OAuth 或真实请求确认。 - None | Some(_) => { + None => { summary.transient += 1; tracing::warn!( account_id = %account.id(), @@ -613,7 +593,7 @@ impl CodexCredentialQuotaService { let observed = self.store.get_quotas(&account_ids).await?; let observed_ids = observed .into_iter() - .filter_map(|observation| observation.observed_at.map(|_| observation.account_id)) + .map(|observation| observation.account_id) .collect::>(); Ok(accounts .iter() @@ -633,79 +613,40 @@ impl CodexCredentialQuotaService { observed_at: SystemTime, summary: &mut CodexQuotaSyncSummary, ) -> Result<(), CodexCredentialQuotaError> { - let refreshed_snapshot = parse_account_quota_snapshot( - account.id().clone(), - account.revision(), - observed_at, - value, - )?; - let previous_reset_at = self.previous_confirmed_reset(account).await; let object = normalize_quota_window_placeholders( value .as_object() .cloned() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?, ); - let object = if account.availability() == AccountAvailability::QuotaExhausted - && quota_success_state( - account.availability(), - &refreshed_snapshot, - previous_reset_at, - ) - .is_none() - { - confirmed_exhaustion_projection( - object, - confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot), - ) - } else { - object - }; let snapshot = parse_account_quota_snapshot( account.id().clone(), account.revision(), observed_at, &Value::Object(object.clone()), )?; - let fact = snapshot.fact(); + let state = account.quota().merge_observation(snapshot.quota()); + let snapshot = snapshot.with_quota_state(state); let outcome = self .store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new(object)), - observed_at: Some(observed_at), - limit_reached: Some(fact.exhausted()), + quota: OpaqueProviderData::new(object), + observed_at, + state, }) .await?; if outcome == QuotaWriteOutcome::Conflict { summary.stale += 1; return Ok(()); } - let Some(current) = self.store.get_account(account.id()).await? else { - summary.stale += 1; - return Ok(()); - }; - if current.revision() != account.revision() { - summary.stale += 1; - return Ok(()); - } self.scheduling.observe(&snapshot); - if fact.exhausted() { + if snapshot.quota().is_exhausted() { summary.exhausted += 1; } else { summary.updated += 1; } - if let Some(availability) = authoritative_quota_snapshot_state( - current.availability(), - &refreshed_snapshot, - previous_reset_at, - ) { - let _ = self - .repository - .apply_state(¤t, availability, observed_at) - .await; - } Ok(()) } @@ -722,11 +663,6 @@ impl CodexCredentialQuotaService { .limits .values() .any(|details| passive_rate_limit_snapshot(details).is_some()); - let previous_reset_at = if account.availability() == AccountAvailability::QuotaExhausted { - self.previous_confirmed_reset(account).await - } else { - None - }; let existing = self .store .get_quotas(std::slice::from_ref(account.id())) @@ -736,17 +672,14 @@ impl CodexCredentialQuotaService { observation.account_id == *account.id() && observation.expected_revision == account.revision() }); - let existing_observed_at = existing - .as_ref() - .and_then(|observation| observation.observed_at); + let existing_state = existing.as_ref().map(|observation| observation.state); let existing = existing - .and_then(|observation| observation.quota) - .map(OpaqueProviderData::into_inner) + .map(|observation| observation.quota.into_inner()) .unwrap_or_default(); // 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻, // 也不能借旧快照重新推导 quota state。 if !has_quota_facts { - let Some(observed_at) = existing_observed_at else { + let Some(state) = existing_state else { return Ok(false); }; let outcome = self @@ -754,62 +687,42 @@ impl CodexCredentialQuotaService { .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new(merge_passive_quota( - existing, - &rate_limits, - ))), - observed_at: Some(observed_at), - limit_reached: None, + quota: OpaqueProviderData::new(merge_passive_quota(existing, &rate_limits)), + observed_at: SystemTime::now(), + state, }) .await?; return Ok(outcome != QuotaWriteOutcome::Conflict); } let quota = merge_passive_quota(existing, &rate_limits); let observed_at = SystemTime::now(); - let refreshed_snapshot = parse_account_quota_snapshot( - account.id().clone(), - account.revision(), - observed_at, - &Value::Object(quota.clone()), - )?; - let quota = if account.availability() == AccountAvailability::QuotaExhausted { - // 被动成功响应不能解除确认额度耗尽;继续展示 100%,直到权威 usage 快照 - // 证明旧 reset 已跨过且新窗口已经推进。 - confirmed_exhaustion_projection( - quota, - confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot), - ) - } else { - quota - }; - let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?; let snapshot = parse_account_quota_snapshot( account.id().clone(), account.revision(), observed_at, &Value::Object(quota.clone()), - ); + )?; + // 这些 headers 来自一次成功推理,访问结论优先于可能滞后的百分比。 + let state = QuotaState::allowed(observed_at); let outcome = self .store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new(quota)), - observed_at: Some(observed_at), - limit_reached: Some(fact.exhausted()), + quota: OpaqueProviderData::new(quota), + observed_at, + state, }) .await?; if outcome == QuotaWriteOutcome::Conflict { return Ok(false); } - if let Ok(snapshot) = snapshot { - self.scheduling.observe(&snapshot); - } + self.scheduling.observe(&snapshot); Ok(true) } /// 真实 429 的单一事实入口:写入 Redis 临时限流冷却(`until = now + retry_after`)。 - /// availability 与 quota 主窗口(额度重置时间)都不改变——临时限流是独立维度, + /// 凭据与额度主窗口(额度重置时间)都不改变——临时限流是独立维度, /// 到期由 Redis key 过期自动解除,不污染配额耗尽状态。 /// 已有更晚的冷却不会被缩短(put_if_later)。 pub async fn apply_rate_limit_429( @@ -838,7 +751,6 @@ impl CodexCredentialQuotaService { pub async fn rate_limited_until( &self, account_id: &ProviderAccountId, - credential_revision: CredentialRevision, ) -> Result, CodexCredentialQuotaError> { let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| { CodexCredentialQuotaError::Store { @@ -848,9 +760,6 @@ impl CodexCredentialQuotaService { else { return Ok(None); }; - if cooldown.credential_revision() != credential_revision { - return Ok(None); - } let until = cooldown.until(); if until <= SystemTime::now() { return Ok(None); @@ -894,18 +803,17 @@ impl CodexCredentialQuotaService { { return Err(CodexCredentialQuotaError::RevisionConflict); } - let Some(data) = observation.quota else { + if observation.quota.expose_to_provider().is_empty() { return Ok(None); - }; - let observed_at = observation - .observed_at - .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; + } + let observed_at = observation.observed_at; let snapshot = parse_account_quota_snapshot( account_id.clone(), account.revision(), observed_at, - &Value::Object(data.expose_to_provider().clone()), - )?; + &Value::Object(observation.quota.expose_to_provider().clone()), + )? + .with_quota_state(observation.state); self.scheduling.observe(&snapshot); Ok(Some(snapshot)) } @@ -915,7 +823,8 @@ impl CodexCredentialQuotaService { &self, account_id: &ProviderAccountId, ) -> Result { - self.refresh_account_snapshot(account_id).await + self.refresh_account_snapshot(account_id, QuotaRefreshAuthority::ObserveAccess) + .await } /// 真实限额失败后的异步刷新只补齐展示快照,不允许 usage 快照撤销已确认的失败状态。 @@ -923,12 +832,14 @@ impl CodexCredentialQuotaService { &self, account_id: &ProviderAccountId, ) -> Result { - self.refresh_account_snapshot(account_id).await + self.refresh_account_snapshot(account_id, QuotaRefreshAuthority::PreserveAccess) + .await } async fn refresh_account_snapshot( &self, account_id: &ProviderAccountId, + authority: QuotaRefreshAuthority, ) -> Result { let account = self .store @@ -957,104 +868,77 @@ impl CodexCredentialQuotaService { }); } Err(CodexQuotaFetchError::Upstream { account, error }) => { - // 402 与明确的停用错误写对应状态;usage 401/403 不写 availability。 - if let Some(availability) = quota_state_transition(&error) { - self.persist_upstream_state(&account, availability, &error, observed_at) - .await; + match classify_quota_endpoint_failure(&error) { + Some(QuotaEndpointFailure::Exhausted(evidence)) => { + self.record_confirmed_exhaustion(&account, evidence, None, observed_at) + .await?; + } + Some(QuotaEndpointFailure::Credential { state, reason }) => { + self.persist_credential_failure(&account, state, reason, observed_at) + .await; + } + None => {} } return Err(CodexCredentialQuotaError::Upstream { detail: error.to_string(), }); } }; - let refreshed_snapshot = parse_account_quota_snapshot( - account.id().clone(), - account.revision(), - observed_at, - &value, - )?; - let previous_reset_at = self.previous_confirmed_reset(&account).await; let object = normalize_quota_window_placeholders( value .as_object() .cloned() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?, ); - let object = if account.availability() == AccountAvailability::QuotaExhausted - && quota_success_state( - account.availability(), - &refreshed_snapshot, - previous_reset_at, - ) - .is_none() - { - confirmed_exhaustion_projection( - object, - confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot), - ) - } else { - object - }; let snapshot = parse_account_quota_snapshot( account.id().clone(), account.revision(), observed_at, &Value::Object(object.clone()), )?; + let state = match authority { + QuotaRefreshAuthority::ObserveAccess => { + account.quota().merge_observation(snapshot.quota()) + } + QuotaRefreshAuthority::PreserveAccess => account.quota(), + }; + let snapshot = snapshot.with_quota_state(state); if self .store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new(object)), - observed_at: Some(observed_at), - limit_reached: Some(snapshot.fact().exhausted()), + quota: OpaqueProviderData::new(object), + observed_at, + state, }) .await? == QuotaWriteOutcome::Conflict { return Err(CodexCredentialQuotaError::RevisionConflict); } - let current = self - .store - .get_account(account.id()) - .await? - .ok_or(CodexCredentialQuotaError::NotFound)?; - if current.revision() != account.revision() { - return Err(CodexCredentialQuotaError::RevisionConflict); - } self.scheduling.observe(&snapshot); - if let Some(availability) = authoritative_quota_snapshot_state( - current.availability(), - &refreshed_snapshot, - previous_reset_at, - ) { - self.repository - .apply_state(¤t, availability, observed_at) - .await?; - } Ok(snapshot) } - async fn persist_upstream_state( + async fn persist_credential_failure( &self, account: &ProviderAccount, - availability: AccountAvailability, - upstream_error: &CodexClientError, + credential_state: CredentialState, + reason: AccountErrorReason, observed_at: SystemTime, ) { - let reason = quota_state_reason(upstream_error).map(str::to_owned); if let Err(error) = self .repository - .apply_state_with_message(account, availability, observed_at, reason.clone()) + .apply_state_with_reason(account, credential_state, observed_at, Some(reason), None) .await { tracing::warn!( account_id = %account.id(), - ?availability, - reason = reason.as_deref().unwrap_or_default(), + ?credential_state, + reason = reason.as_str(), error = %error, - "OpenAI quota upstream state write failed" + "OpenAI quota credential fact write failed" ); } } @@ -1163,10 +1047,8 @@ fn eligible_periodic_quota_refresh(account: &ProviderAccount, now: SystemTime) - fn eligible_initial_quota_sync(account: &ProviderAccount, now: SystemTime) -> bool { // 首次观察只兜底刚入库、尚无 quota 快照的账号。已耗尽等运行时状态必须由 // periodic 路径处理,才能保留同一账号的最小复核间隔。 - matches!( - account.availability(), - AccountAvailability::Ready | AccountAvailability::Unknown - ) && eligible_periodic_quota_refresh(account, now) + account.credential_state() == CredentialState::Ready + && eligible_periodic_quota_refresh(account, now) } fn access_token_is_current(account: &ProviderAccount, now: SystemTime) -> bool { @@ -1297,71 +1179,6 @@ fn secondary_window_is_placeholder(window: &Map) -> bool { }) } -/// 用已确认的上游额度拒绝覆盖 core 限额投影。 -/// -/// 上游 usage 快照可能在结算期间短暂滞后为 98/99% 或 `allowed=true`;保留其他 -/// 原始 metadata,但把 core 窗口固定为 100%,直到权威快照证明进入下一窗口。 -fn confirmed_exhaustion_projection( - mut quota: Map, - reset_at: Option, -) -> Map { - let mut rate_limit = quota - .remove("rate_limit") - .and_then(|value| value.as_object().cloned()) - .unwrap_or_default(); - rate_limit.insert("allowed".to_owned(), Value::Bool(false)); - rate_limit.insert("limit_reached".to_owned(), Value::Bool(true)); - for field in ["primary_window", "secondary_window"] { - let existed = rate_limit.contains_key(field); - let mut window = rate_limit - .remove(field) - .and_then(|value| value.as_object().cloned()) - .unwrap_or_default(); - if field == "primary_window" || existed { - window.insert("used_percent".to_owned(), Value::from(100_u64)); - if field == "primary_window" - && let Some(reset_at) = reset_at - { - window.insert("reset_at".to_owned(), Value::from(reset_at)); - } - rate_limit.insert(field.to_owned(), Value::Object(window)); - } - } - quota.insert("rate_limit".to_owned(), Value::Object(rate_limit)); - quota -} - -/// 选择应继续锁定的 reset:新鲜快照仍确认耗尽时可把锁推进到更晚窗口; -/// 否则保留已确认失败的旧窗口,避免 98/99% 快照反向解锁。 -fn confirmed_exhaustion_reset_at( - previous_reset_at: Option, - refreshed_snapshot: &CodexAccountQuotaSnapshot, -) -> Option { - let refreshed_reset_at = refreshed_snapshot - .core_primary_window() - .and_then(|window| window.reset_at()) - .map(SystemTime::from); - let reset_at = match (previous_reset_at, refreshed_reset_at) { - (Some(previous), Some(refreshed)) - if refreshed_snapshot.fact().exhausted() && refreshed > previous => - { - refreshed - } - (Some(previous), _) => previous, - (None, refreshed) => refreshed?, - }; - system_time_to_unix_seconds(reset_at) -} - -fn system_time_to_unix_seconds(value: SystemTime) -> Option { - value - .duration_since(UNIX_EPOCH) - .ok()? - .as_secs() - .try_into() - .ok() -} - fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option> { // 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。 let mut snapshot = Map::new(); diff --git a/backend/crates/providers/openai/src/credential/quota/recovery.rs b/backend/crates/providers/openai/src/credential/quota/recovery.rs deleted file mode 100644 index e3865d0d2..000000000 --- a/backend/crates/providers/openai/src/credential/quota/recovery.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! 权威额度快照的 availability 转换。 -//! -//! 已触顶的 core primary 会立即把可调度账号标为 `QuotaExhausted`; -//! `QuotaExhausted -> Ready` 则只能由同一 credential revision 的权威 usage 快照证明: -//! 新快照未耗尽、core primary reset 已推进到下一窗口,且该窗口用量低于 10%。 - -use std::time::SystemTime; - -use gateway_core::engine::credential::AccountAvailability; -use reqwest::StatusCode; -use serde_json::Value; - -use super::snapshot::CodexAccountQuotaSnapshot; -use crate::transport::CodexClientError; - -const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0; - -/// 基于成功的权威 usage 快照收敛可调度账号的额度状态。 -/// -/// 只接受上游明确拒绝使用且 core primary 触顶的证据;secondary、附加桶、 -/// credits 与 spend control 都不能单独把整个账号标成额度耗尽。 -pub(crate) fn authoritative_quota_snapshot_state( - current: AccountAvailability, - snapshot: &CodexAccountQuotaSnapshot, - previous_reset_at: Option, -) -> Option { - match current { - AccountAvailability::Ready | AccountAvailability::Unknown - if snapshot.authoritative_core_primary_exhausted() => - { - Some(AccountAvailability::QuotaExhausted) - } - AccountAvailability::QuotaExhausted => { - quota_success_state(current, snapshot, previous_reset_at) - } - AccountAvailability::Ready | AccountAvailability::Unknown => None, - AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid => None, - } -} - -/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独覆盖 -/// 一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。 -/// -/// `previous_reset_at` 是写入新快照前读取的 core primary 已确认耗尽窗口。上游可能只 -/// 调整 reset 时间而尚未重置额度,因此还要求同一新鲜 primary 窗口的用量低于 10%。 -pub(crate) fn quota_success_state( - current: AccountAvailability, - snapshot: &CodexAccountQuotaSnapshot, - previous_reset_at: Option, -) -> Option { - if current != AccountAvailability::QuotaExhausted || snapshot.fact().exhausted() { - return None; - } - let previous_reset_at = previous_reset_at?; - let primary_window = snapshot.core_primary_window()?; - let refreshed_reset_at = primary_window.reset_at().map(SystemTime::from)?; - let used_percent = primary_window.used_percent()?; - (refreshed_reset_at > previous_reset_at && used_percent < RESET_RECOVERY_MAX_USED_PERCENT) - .then_some(AccountAvailability::Ready) -} - -pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option { - let CodexClientError::Upstream { status, body, .. } = error else { - return None; - }; - if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) { - return Some(AccountAvailability::Banned); - } - match *status { - // 402:真额度耗尽 → QuotaExhausted(worker 按 reset_at 自动恢复)。 - StatusCode::PAYMENT_REQUIRED => Some(AccountAvailability::QuotaExhausted), - // usage 401/403 不能单独判定账号不可用:保持现有状态,交给 OAuth 或真实请求确认。 - // 429/503 同样不写 availability,仅记失败待重试。 - StatusCode::UNAUTHORIZED - | StatusCode::FORBIDDEN - | StatusCode::TOO_MANY_REQUESTS - | StatusCode::SERVICE_UNAVAILABLE => None, - _ => None, - } -} - -/// 配额接口将账号变为非 ready 状态时可安全持久化的上游原因码。 -pub(crate) fn quota_state_reason(error: &CodexClientError) -> Option<&'static str> { - match quota_state_transition(error)? { - AccountAvailability::Banned => Some("deactivated_workspace"), - AccountAvailability::QuotaExhausted => Some("payment_required"), - AccountAvailability::Unknown - | AccountAvailability::Ready - | AccountAvailability::Expired - | AccountAvailability::Invalid => None, - } -} - -fn is_deactivated_workspace(body: &str) -> bool { - serde_json::from_str::(body) - .ok() - .is_some_and(|value| { - value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace") - }) -} diff --git a/backend/crates/providers/openai/src/credential/quota/snapshot.rs b/backend/crates/providers/openai/src/credential/quota/snapshot.rs index 995640a70..5382e4313 100644 --- a/backend/crates/providers/openai/src/credential/quota/snapshot.rs +++ b/backend/crates/providers/openai/src/credential/quota/snapshot.rs @@ -7,19 +7,17 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ - AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaObservation, + AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaEvidence, QuotaObservation, + QuotaState, }; use serde_json::{Map, Value}; use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL}; -const CORE_PRIMARY_WINDOW_KEY: &str = "core-primary"; - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct CodexQuotaFact { remaining_percent: Option, resets_at: Option>, - exhausted: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -112,7 +110,7 @@ pub struct CodexAccountQuotaSnapshot { credential_revision: CredentialRevision, observed_at: SystemTime, fact: CodexQuotaFact, - authoritative_core_primary_exhausted: bool, + quota: QuotaState, windows: Vec, } @@ -143,54 +141,14 @@ impl CodexAccountQuotaSnapshot { } #[must_use] - pub(crate) fn core_primary_window(&self) -> Option<&CodexQuotaWindow> { - self.windows - .iter() - .find(|window| window.key() == CORE_PRIMARY_WINDOW_KEY) + pub const fn quota(&self) -> QuotaState { + self.quota } - /// 上游成功 usage 明确拒绝请求,且 canonical core primary 已确认触顶。 + /// 将已持久化的规范化事实覆盖到 raw JSON 展示快照上。 #[must_use] - pub(crate) const fn authoritative_core_primary_exhausted(&self) -> bool { - self.authoritative_core_primary_exhausted - } - - /// 投影滚动已过期窗口:`reset_at` 已过的窗口 `used_percent=0`、 - /// `limit_reached=false`、`reset_at` 滚到下一个周期(reset 到期自动归零)。 - /// 只作用于展示/调度投影,不写回持久化 raw JSON。 - /// - /// O(1) 计算:`next_reset = reset + (floor((now - reset) / window) + 1) * window`, - /// 不做逐周期线性循环。 - #[must_use] - pub fn roll_expired_windows(mut self, now: SystemTime) -> Self { - for window in &mut self.windows { - let Some(reset_at) = window.reset_at else { - continue; - }; - let reset = SystemTime::from(reset_at); - if reset > now { - continue; - } - window.used_percent = Some(0.0); - window.limit_reached = false; - let Some(window_seconds) = window.window_seconds.filter(|seconds| *seconds > 0) else { - // 没有合法窗口时长:清除已过期的 reset,等待下一次权威观测。 - window.reset_at = None; - continue; - }; - let elapsed = now - .duration_since(reset) - .unwrap_or(Duration::ZERO) - .as_secs(); - let periods = elapsed / window_seconds + 1; - let Some(after) = - reset.checked_add(Duration::from_secs(periods.saturating_mul(window_seconds))) - else { - window.reset_at = None; - continue; - }; - window.reset_at = Some(DateTime::::from(after)); - } + pub(crate) const fn with_quota_state(mut self, quota: QuotaState) -> Self { + self.quota = quota; self } } @@ -205,11 +163,6 @@ impl CodexQuotaFact { pub const fn resets_at(&self) -> Option> { self.resets_at } - - #[must_use] - pub const fn exhausted(&self) -> bool { - self.exhausted - } } pub fn parse_codex_quota_usage(usage: &Value) -> Result { @@ -220,13 +173,12 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result Result Option { - let observed_at = observation.observed_at?; - let quota = observation.quota.as_ref()?; - parse_account_quota_snapshot( + let observed_at = observation.observed_at; + let quota = &observation.quota; + let mut snapshot = parse_account_quota_snapshot( observation.account_id.clone(), observation.expected_revision, observed_at, &Value::Object(quota.expose_to_provider().clone()), ) - .ok() + .ok()?; + // 归一化列是持久化权威事实;raw JSON 只负责 Provider 展示结构。 + snapshot.quota = observation.state; + Some(snapshot) } /// 调度投影缓存 TTL:`min(常规 TTL, 最近 active reset - now)`。 @@ -275,46 +229,36 @@ pub(crate) fn quota_projection_ttl(snapshot: &CodexAccountQuotaSnapshot) -> Opti (!ttl.is_zero()).then_some(ttl) } -/// 从快照的滚动窗口派生中立调度事实(`limit_reached` = 任一窗口触顶)。 +/// 从快照窗口派生容量排序事实;额度访问资格由 `QuotaState` 统一判断。 pub(crate) fn scheduling_signals_from_snapshot( snapshot: &CodexAccountQuotaSnapshot, ) -> Option { - let rolled = snapshot.clone().roll_expired_windows(SystemTime::now()); - quota_scheduling_signals(&rolled) + quota_scheduling_signals(snapshot) } pub(crate) fn quota_scheduling_signals( snapshot: &CodexAccountQuotaSnapshot, ) -> Option { let now = SystemTime::now(); - let mut limit_reached = false; let mut remaining_rank: Option = None; let mut reset_at: Option = None; for window in &snapshot.windows { - limit_reached |= window.limit_reached(); - if let Some(used) = window - .used_percent() - .filter(|used| used.is_finite() && (0.0..=100.0).contains(used)) + let window_reset_at = window.reset_at().map(SystemTime::from); + let observation_expired = window_reset_at.is_some_and(|reset_at| reset_at <= now); + if !observation_expired + && let Some(used) = window + .used_percent() + .filter(|used| used.is_finite() && (0.0..=100.0).contains(used)) { let rank = ((100.0 - used) * 100.0).round() as u64; remaining_rank = Some(remaining_rank.map_or(rank, |current| current.min(rank))); } - if let Some(reset) = window - .reset_at() - .map(SystemTime::from) - .filter(|reset| *reset > now) - { + if let Some(reset) = window_reset_at.filter(|reset| *reset > now) { reset_at = Some(reset_at.map_or(reset, |current| current.min(reset))); } } - // 快照级 limit_reached 是调度排除的充分条件:即使没有任何窗口带 - // percent/reset(如顶层限流标记),也必须产出信号;不能返回 None 丢失排除。 - if limit_reached || remaining_rank.is_some() || reset_at.is_some() { - Some(AccountQuotaSignals::new( - reset_at, - remaining_rank, - limit_reached, - )) + if remaining_rank.is_some() || reset_at.is_some() { + Some(AccountQuotaSignals::new(reset_at, remaining_rank)) } else { None } @@ -330,7 +274,7 @@ pub(crate) fn parse_account_quota_snapshot( let object = usage .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let authoritative_core_primary_exhausted = authoritative_core_primary_exhausted(object)?; + let quota = authoritative_quota_state(object, observed_at)?; let mut windows = Vec::new(); for limit in canonical_rate_limits(object)? { parse_rate_limit_windows( @@ -352,7 +296,7 @@ pub(crate) fn parse_account_quota_snapshot( credential_revision, observed_at, fact, - authoritative_core_primary_exhausted, + quota, windows, }) } @@ -365,48 +309,73 @@ fn quota_source_order(source: &str) -> u8 { } } -/// 账号状态只能由顶层 canonical Codex 额度桶的明确拒绝来改变。 -/// -/// `fact.exhausted()` 和窗口 `limit_reached` 都会聚合或继承 secondary、附加桶等事实, -/// 因此不能单独用来将整个账号标为 `QuotaExhausted`。 -fn authoritative_core_primary_exhausted( +/// 顶层 account-wide Codex 桶是唯一可改变账号额度访问结论的 quota 响应字段。 +fn authoritative_quota_state( usage: &Map, -) -> Result { + observed_at: SystemTime, +) -> Result { let Some(rate_limit) = usage.get("rate_limit") else { - return Ok(false); + return Ok(QuotaState::observed_unknown(observed_at)); }; if rate_limit.is_null() { - return Ok(false); + return Ok(QuotaState::observed_unknown(observed_at)); } let rate_limit = rate_limit .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - if optional_bool(rate_limit, "allowed")? != Some(false) { - return Ok(false); + let allowed = optional_bool(rate_limit, "allowed")?; + let top_level_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false); + let primary_window = rate_limit + .get("primary_window") + .filter(|value| !value.is_null()) + .map(|value| { + value + .as_object() + .ok_or(CodexCredentialQuotaError::InvalidCredentialData) + }) + .transpose()?; + let primary_reached = primary_window + .map(|window| optional_bool(window, "limit_reached")) + .transpose()? + .flatten() + .unwrap_or(false); + let reset_at = primary_window + .and_then(|window| window.get("reset_at")) + .map(|value| { + value + .as_i64() + .filter(|value| *value > 0) + .and_then(|value| DateTime::::from_timestamp(value, 0)) + .map(SystemTime::from) + .ok_or(CodexCredentialQuotaError::InvalidCredentialData) + }) + .transpose()?; + + if allowed == Some(true) { + if top_level_reached || primary_reached { + tracing::warn!( + top_level_reached, + primary_reached, + "OpenAI quota returned contradictory access facts; explicit allowed=true wins" + ); + } + return Ok(QuotaState::allowed(observed_at)); } - let Some(primary_window) = rate_limit.get("primary_window") else { - return Ok(false); - }; - if primary_window.is_null() { - return Ok(false); + if allowed == Some(false) { + return Ok(QuotaState::exhausted( + QuotaEvidence::ProviderDenied, + observed_at, + reset_at, + )); } - let primary_window = primary_window - .as_object() - .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let primary_window_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false) - || optional_bool(primary_window, "limit_reached")?.unwrap_or(false) - || primary_window - .get("used_percent") - .map(|value| { - value - .as_f64() - .filter(|value| value.is_finite() && *value >= 0.0) - .map(|value| value >= 100.0) - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) - }) - .transpose()? - .unwrap_or(false); - Ok(primary_window_reached) + if top_level_reached || primary_reached { + return Ok(QuotaState::exhausted( + QuotaEvidence::AccountLimitReached, + observed_at, + reset_at, + )); + } + Ok(QuotaState::observed_unknown(observed_at)) } /// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。 @@ -523,10 +492,9 @@ fn parse_rate_limit_windows( let object = value .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - // 顶层限流标记是快照级事实:任一窗口都要继承所属 - // rate_limit 的 `limit_reached=true` / `allowed=false`。 - let top_level_reached = optional_bool(object, "limit_reached")?.unwrap_or(false) - || optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed); + let allowed = optional_bool(object, "allowed")?; + let top_level_reached = allowed != Some(true) + && (optional_bool(object, "limit_reached")?.unwrap_or(false) || allowed == Some(false)); for (name, role) in [ ("primary_window", CodexQuotaWindowRole::Primary), ("secondary_window", CodexQuotaWindowRole::Secondary), @@ -564,12 +532,12 @@ fn parse_rate_limit_windows( .ok_or(CodexCredentialQuotaError::InvalidCredentialData) }) .transpose()?; - let limit_reached = top_level_reached - || window - .get("limit_reached") - .and_then(Value::as_bool) - .unwrap_or(false) - || used_percent.is_some_and(|used| used >= 100.0); + let limit_reached = allowed != Some(true) + && (top_level_reached + || window + .get("limit_reached") + .and_then(Value::as_bool) + .unwrap_or(false)); let kind = quota_window_kind(window_seconds); output.push(CodexQuotaWindow { key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)), @@ -649,7 +617,6 @@ const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind) #[derive(Default)] struct QuotaAggregate { recognized: bool, - exhausted: bool, remaining_percent: Option, resets_at: Option>, } @@ -660,8 +627,8 @@ impl QuotaAggregate { .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; self.recognized = true; - self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false); - self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed); + let _ = optional_bool(object, "limit_reached")?; + let _ = optional_bool(object, "allowed")?; for key in ["primary_window", "secondary_window"] { if let Some(window) = object.get(key) { self.observe_window(window)?; @@ -682,8 +649,6 @@ impl QuotaAggregate { .as_f64() .filter(|value| value.is_finite() && *value >= 0.0) .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - // 子窗口触顶(used>=100)是快照级 exhaustion 事实。 - self.exhausted |= used >= 100.0; let remaining = (100.0 - used).clamp(0.0, 100.0).round() as u8; self.remaining_percent = Some( self.remaining_percent @@ -702,7 +667,7 @@ impl QuotaAggregate { Ok(()) } - fn observe_exhaustion_object( + fn observe_metadata_object( &mut self, value: &Value, key: &str, @@ -714,7 +679,7 @@ impl QuotaAggregate { .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; self.recognized = true; - self.exhausted |= optional_bool(object, key)?.unwrap_or(false); + let _ = optional_bool(object, key)?; Ok(()) } } diff --git a/backend/crates/providers/openai/src/credential/refresh.rs b/backend/crates/providers/openai/src/credential/refresh.rs index c9f0bd1cf..4602b41f2 100644 --- a/backend/crates/providers/openai/src/credential/refresh.rs +++ b/backend/crates/providers/openai/src/credential/refresh.rs @@ -6,7 +6,9 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use futures::{StreamExt as _, stream}; -use gateway_core::engine::credential::{AccountAvailability, ProviderAccount, ProviderAccountId}; +use gateway_core::engine::credential::{ + AccountErrorReason, CredentialState, ProviderAccount, ProviderAccountId, +}; use gateway_core::provider_ports::{ ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest, ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy, @@ -253,7 +255,8 @@ impl CodexCredentialRefreshService { return self .persist_terminal( &due.account, - AccountAvailability::Expired, + CredentialState::Expired, + AccountErrorReason::CredentialExpired, "refresh_recovery_window_exhausted", CodexCredentialRefreshOutcome::Invalidated { account_id }, ) @@ -293,7 +296,8 @@ impl CodexCredentialRefreshService { Err(RefreshFailure::InvalidGrant) => { self.persist_terminal( &due.account, - AccountAvailability::Expired, + CredentialState::Expired, + AccountErrorReason::CredentialExpired, "refresh_invalid_grant", CodexCredentialRefreshOutcome::Invalidated { account_id }, ) @@ -302,7 +306,8 @@ impl CodexCredentialRefreshService { Err(RefreshFailure::Banned) => { self.persist_terminal( &due.account, - AccountAvailability::Banned, + CredentialState::Banned, + AccountErrorReason::AccountBanned, "account_banned", CodexCredentialRefreshOutcome::Banned { account_id }, ) @@ -353,11 +358,9 @@ impl CodexCredentialRefreshService { && account.enabled() && account.has_refresh_token() && !excluded.contains(account.id()) - && !matches!( - account.availability(), - AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid + && matches!( + account.credential_state(), + CredentialState::Unknown | CredentialState::Ready ) && refresh_due_at(account, policy, now).is_some() }); @@ -463,7 +466,8 @@ impl CodexCredentialRefreshService { async fn persist_terminal( &self, account: &ProviderAccount, - availability: AccountAvailability, + credential_state: CredentialState, + error_reason: AccountErrorReason, reason: &'static str, outcome: CodexCredentialRefreshOutcome, ) -> Result { @@ -478,10 +482,11 @@ impl CodexCredentialRefreshService { } match self .repository - .apply_state_with_message( + .apply_state_with_reason( account, - availability, + credential_state, SystemTime::now(), + Some(error_reason), Some(reason.to_owned()), ) .await @@ -489,7 +494,7 @@ impl CodexCredentialRefreshService { Ok(()) => { tracing::warn!( account_id = %account.id(), - ?availability, + ?credential_state, reason, access_token_expires_at = ?account.access_token_expires_at() .map(DateTime::::from), diff --git a/backend/crates/providers/openai/src/credential/repository.rs b/backend/crates/providers/openai/src/credential/repository.rs index 7d6131298..c2aba671d 100644 --- a/backend/crates/providers/openai/src/credential/repository.rs +++ b/backend/crates/providers/openai/src/credential/repository.rs @@ -5,8 +5,8 @@ use std::time::SystemTime; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision, ProviderAccount, ProviderAccountId, ProviderAccountStore, + AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, + CredentialRevision, CredentialState, ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, }; use gateway_core::routing::ProviderKind; @@ -220,34 +220,46 @@ impl CodexCredentialRepository { pub async fn apply_state( &self, account: &ProviderAccount, - availability: AccountAvailability, + credential_state: CredentialState, observed_at: SystemTime, ) -> Result<(), CredentialRepositoryError> { - self.apply_state_with_message(account, availability, observed_at, None) - .await + self.apply_state_with_reason( + account, + credential_state, + observed_at, + credential_state.error_reason(), + None, + ) + .await } - /// 带上游错误描述的状态写入;上游 message 优先,终态缺失 message 时写稳定兜底, - /// 恢复 `Ready` 时清空。 - pub async fn apply_state_with_message( + /// 写入凭据事实及其稳定错误原因;额度事实不经过此入口。 + pub async fn apply_state_with_reason( &self, account: &ProviderAccount, - availability: AccountAvailability, + credential_state: CredentialState, observed_at: SystemTime, + error_reason: Option, message: Option, ) -> Result<(), CredentialRepositoryError> { if !account.enabled() { return Ok(()); } - let message = message - .filter(|value| !value.trim().is_empty()) - .or_else(|| availability.fallback_error_reason().map(str::to_owned)); + let (error_reason, message) = if credential_state == CredentialState::Ready { + (None, None) + } else { + ( + error_reason.or_else(|| credential_state.error_reason()), + message.filter(|value| !value.trim().is_empty()), + ) + }; self.store .apply_state_change(AccountStateChange { account_id: account.id().clone(), expected_revision: account.revision(), - availability, + credential_state, observed_at, + error_reason, message, }) .await?; diff --git a/backend/crates/providers/openai/src/credential/selector.rs b/backend/crates/providers/openai/src/credential/selector.rs index 7c714cb4e..b6c2c18ff 100644 --- a/backend/crates/providers/openai/src/credential/selector.rs +++ b/backend/crates/providers/openai/src/credential/selector.rs @@ -6,9 +6,10 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use gateway_core::engine::credential::{ - AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats, + AccountCandidate, AccountEligibilityPolicy, AccountErrorReason, AccountFeedbackStats, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, AccountSelector, - PreferredAccountSelection, ProviderAccount, ProviderAccountId, + AccountStatus, CredentialState, PreferredAccountSelection, ProviderAccount, ProviderAccountId, + QuotaEvidence, }; use gateway_core::engine::{AttemptContext, ContinuationAttempt}; use gateway_core::provider_ports::{ @@ -57,8 +58,8 @@ pub enum CodexAccountFailure { QuotaExhausted, /// 当前用量窗口已耗尽;到重置时间后可自动恢复。 UsageLimitExhausted { - /// 上游返回的窗口重置时长。 - retry_after: Option, + /// 上游返回的窗口绝对重置时刻。 + reset_at: Option, }, /// 账号触发临时用量限制。 RateLimited { @@ -74,21 +75,6 @@ pub enum CodexAccountFailure { CloudflarePathBlocked, } -impl CodexAccountFailure { - /// 上游未提供可展示错误文本时,保留已分类的稳定原因。 - const fn fallback_error_reason(self) -> Option<&'static str> { - match self { - Self::CredentialExpired => Some("upstream_credential_expired"), - Self::IdentityVerificationRequired => Some("upstream_identity_verification_required"), - Self::Banned => Some("upstream_account_banned"), - Self::QuotaExhausted => Some("upstream_quota_exhausted"), - Self::UsageLimitExhausted { .. } => Some("upstream_usage_limit_exhausted"), - Self::CloudflarePathBlocked => Some("upstream_cloudflare_path_blocked"), - Self::RateLimited { .. } | Self::CloudflareChallenge { .. } => None, - } - } -} - #[derive(Debug, Clone, Copy)] struct RiskRecoveryState { challenge_count: u32, @@ -127,7 +113,6 @@ pub struct CodexCredentialSelector { cookie_policy: CodexCookiePolicy, risk_recovery: Mutex>, account_feedback: Arc, - skip_exhausted: bool, } enum SessionAffinityLookup { @@ -263,7 +248,6 @@ impl CodexCredentialSelector { agent_identity: Arc, account_feedback: Arc, cookie_policy: CodexCookiePolicy, - skip_exhausted: bool, ) -> Self { Self { provider_kind, @@ -277,7 +261,6 @@ impl CodexCredentialSelector { cookie_policy, risk_recovery: Mutex::new(HashMap::new()), account_feedback, - skip_exhausted, } } @@ -310,24 +293,17 @@ impl CodexCredentialSelector { if !diagnostic { self.quota.prepare_scheduling(&accounts).await; } - // 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度, - // 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。 - let accounts = if diagnostic { - accounts - } else { - let mut retained = Vec::with_capacity(accounts.len()); - for account in accounts { - let cooling = self + let mut rate_limits = HashMap::with_capacity(accounts.len()); + if !diagnostic { + for account in &accounts { + let until = self .quota - .rate_limited_until(account.id(), account.revision()) + .rate_limited_until(account.id()) .await .unwrap_or(None); - if cooling.is_none() { - retained.push(account); - } + rate_limits.insert(account.id().clone(), until); } - retained - }; + } let account_ids = accounts .iter() .map(|account| account.id().clone()) @@ -352,11 +328,12 @@ impl CodexCredentialSelector { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }) .with_provider_quota(self.quota.scheduling_signals(&account)) + .with_rate_limit(rate_limits.get(account.id()).copied().flatten()) .with_runtime_health(health.0, health.1); AccountCandidate { account, signals } }) @@ -383,41 +360,6 @@ impl CodexCredentialSelector { return Err(CredentialSelectionError::NoEligibleCredential); } let pinned_account = required_account.or_else(|| continuation_account.clone()); - let preserve_native_continuation_state = matches!( - request.attempt.continuation_attempt(), - ContinuationAttempt::Native - ) && continuation_account.is_some(); - // skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号, - // 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。 - // 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。 - let scheduling_candidates = if diagnostic { - // 连接测试必须实际探测指定账号;额度快照只影响常规调度。 - candidates.clone() - } else if self.skip_exhausted { - candidates - .iter() - .filter(|candidate| !candidate.signals.quota_limit_reached) - .cloned() - .collect::>() - } else { - candidates - .iter() - .cloned() - .map(|mut candidate| { - // 已命中 previous_response_id 的 native continuation 先检查原账号。 - // 即使常规调度允许探测额度耗尽账号,也必须保留原账号真实状态, - // 由 Coordinator 据此切换到可携带重放,而非伪装为 Ready。 - if !preserve_native_continuation_state - || pinned_account.as_ref() != Some(candidate.account.id()) - { - candidate.account = - quota_exhausted_scheduling_projection(&candidate.account); - } - candidate - }) - .collect::>() - }; - let scheduling_candidates = scheduling_candidates.as_slice(); let mut affinity = if diagnostic { AffinitySelection::default() } else { @@ -432,18 +374,6 @@ impl CodexCredentialSelector { .prepare_cyber_policy_scope(cyber_policy_session_key) .await; let mut excluded = request.attempt.excluded_accounts().clone(); - if !diagnostic - && preserve_native_continuation_state - && let Some(pinned) = pinned_account.as_ref() - && candidates.iter().any(|candidate| { - candidate.account.id() == pinned && candidate.signals.quota_limit_reached - }) - { - // `skip_exhausted = false` 只允许普通新请求探测可疑账号;已绑定的 - // native response handle 先将真实触顶状态交给 Coordinator,随后才可 - // 通过 ReplayAny 按当前调度策略换号。 - excluded.insert(pinned.clone()); - } if let Some(state) = cyber_policy_scope .as_ref() .and_then(|scope| scope.state.as_ref()) @@ -476,13 +406,13 @@ impl CodexCredentialSelector { excluded_accounts: excluded.clone(), preferred_account: preferred.clone(), round_robin_cursor, - availability: if diagnostic { - AccountAvailabilityPolicy::BypassForDiagnostic + eligibility: if diagnostic { + AccountEligibilityPolicy::BypassForDiagnostic } else { - AccountAvailabilityPolicy::Enforce + AccountEligibilityPolicy::Enforce }, }; - let Some(selection) = AccountSelector.select(scheduling_candidates, &context) else { + let Some(selection) = AccountSelector.select(&candidates, &context) else { return match shortest_retry { Some(retry_after) => Err(CredentialSelectionError::CapacityUnavailable { retry_after: Some(retry_after), @@ -597,10 +527,15 @@ impl CodexCredentialSelector { self.clear_session_affinity(key).await; return AffinitySelection::escaped(account_id, AffinityEscapeReason::HardUnavailable); }; - if candidate.account.is_schedulable(now) { + if candidate + .account + .status_projection(now, candidate.signals.rate_limited_until) + .status + == AccountStatus::Normal + { return AffinitySelection::preferred(account_id); } - let reason = affinity_unavailable_reason(&candidate.account, now); + let reason = affinity_unavailable_reason(candidate, now); self.clear_session_affinity(key).await; AffinitySelection::escaped(account_id, reason) } @@ -749,9 +684,7 @@ impl CodexCredentialSelector { message: Option, ) -> Result<(), CredentialSelectionError> { let now = SystemTime::now(); - let message = message - .filter(|value| !value.trim().is_empty()) - .or_else(|| failure.fallback_error_reason().map(str::to_owned)); + let message = message.filter(|value| !value.trim().is_empty()); match failure { CodexAccountFailure::CredentialExpired => { if account.authentication_kind() == CODEX_AUTHENTICATION_KIND_OAUTH @@ -772,29 +705,51 @@ impl CodexCredentialSelector { ); return Ok(()); } - self.apply_state_with_message(account, AccountAvailability::Expired, now, message) - .await - } - CodexAccountFailure::IdentityVerificationRequired => { - self.apply_state_with_message(account, AccountAvailability::Invalid, now, message) - .await - } - CodexAccountFailure::Banned => { - self.apply_state_with_message(account, AccountAvailability::Banned, now, message) - .await - } - // 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。 - CodexAccountFailure::QuotaExhausted - | CodexAccountFailure::UsageLimitExhausted { .. } => { - self.apply_state_with_message( + self.apply_credential_state( account, - AccountAvailability::QuotaExhausted, + CredentialState::Expired, + AccountErrorReason::AccessTokenExpired, now, message, ) .await } - // 429:临时限流只写 quota,availability 不改变。 + CodexAccountFailure::IdentityVerificationRequired => { + self.apply_credential_state( + account, + CredentialState::Invalid, + AccountErrorReason::AccountUnverified, + now, + message, + ) + .await + } + CodexAccountFailure::Banned => { + self.apply_credential_state( + account, + CredentialState::Banned, + AccountErrorReason::AccountBanned, + now, + message, + ) + .await + } + CodexAccountFailure::QuotaExhausted => self + .quota + .record_confirmed_exhaustion(account, QuotaEvidence::PaymentRequired, None, now) + .await + .map_err(|_| CredentialSelectionError::Store), + CodexAccountFailure::UsageLimitExhausted { reset_at } => self + .quota + .record_confirmed_exhaustion( + account, + QuotaEvidence::UsageLimitReached, + reset_at, + now, + ) + .await + .map_err(|_| CredentialSelectionError::Store), + // 429:临时限流只写运行时冷却,不改变凭据或额度事实。 CodexAccountFailure::RateLimited { retry_after } => { self.quota .apply_rate_limit_429(account, retry_after, now) @@ -802,7 +757,7 @@ impl CodexCredentialSelector { .map_err(|_| CredentialSelectionError::Store)?; Ok(()) } - // Cloudflare 挑战:内存退避表(记录风险计数),不写 availability。 + // Cloudflare 挑战:内存退避表(记录风险计数),不写账号事实。 CodexAccountFailure::CloudflareChallenge { retry_after } => { let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after); let recovery = now @@ -815,9 +770,10 @@ impl CodexCredentialSelector { CodexAccountFailure::CloudflarePathBlocked => { let blocked = self.record_cloudflare_path_block(account.id(), now); if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD { - self.apply_state_with_message( + self.apply_credential_state( account, - AccountAvailability::Invalid, + CredentialState::Invalid, + AccountErrorReason::CredentialInvalid, now, message, ) @@ -830,15 +786,22 @@ impl CodexCredentialSelector { } } - async fn apply_state_with_message( + async fn apply_credential_state( &self, account: &ProviderAccount, - availability: AccountAvailability, + credential_state: CredentialState, + error_reason: AccountErrorReason, observed_at: SystemTime, message: Option, ) -> Result<(), CredentialSelectionError> { self.repository - .apply_state_with_message(account, availability, observed_at, message) + .apply_state_with_reason( + account, + credential_state, + observed_at, + Some(error_reason), + message, + ) .await?; Ok(()) } @@ -895,29 +858,34 @@ impl CodexCredentialSelector { let Ok(current) = self.current_account(account.id()).await else { return; }; - // QuotaExhausted 只能由结构化额度事实解除;一次成功响应本身不能证明额度已经恢复。 if current.provider() != &self.provider_kind || current.revision() != account.revision() || !current.enabled() - || !matches!( - current.availability(), - AccountAvailability::Unknown - | AccountAvailability::Expired - | AccountAvailability::Invalid - | AccountAvailability::Banned - ) { return; } + let observed_at = SystemTime::now(); + if current.credential_state() != CredentialState::Ready + && let Err(error) = self + .repository + .apply_state(¤t, CredentialState::Ready, observed_at) + .await + { + tracing::warn!( + account_id = %account.id(), + error = %error, + "OpenAI credential recovery after successful upstream response failed" + ); + } if let Err(error) = self - .repository - .apply_state(¤t, AccountAvailability::Ready, SystemTime::now()) + .quota + .record_successful_inference(¤t, observed_at) .await { tracing::warn!( account_id = %account.id(), error = %error, - "OpenAI account state recovery after successful upstream response failed" + "OpenAI quota recovery after successful upstream response failed" ); } } @@ -1045,30 +1013,19 @@ impl CodexCredentialSelector { } } -fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderAccount { - if account.availability() == AccountAvailability::QuotaExhausted { - return account - .clone() - .with_runtime_state(account.enabled(), AccountAvailability::Ready); - } - account.clone() -} - -fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> AffinityEscapeReason { - if !account.enabled() - || account - .access_token_expires_at() - .is_some_and(|expires_at| expires_at <= now) +fn affinity_unavailable_reason( + candidate: &AccountCandidate, + now: SystemTime, +) -> AffinityEscapeReason { + match candidate + .account + .status_projection(now, candidate.signals.rate_limited_until) + .status { - return AffinityEscapeReason::HardUnavailable; - } - match account.availability() { - AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted, - AccountAvailability::Unknown - | AccountAvailability::Ready - | AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid => AffinityEscapeReason::HardUnavailable, + AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted, + AccountStatus::RateLimited => AffinityEscapeReason::Cooldown, + AccountStatus::Normal => AffinityEscapeReason::SelectionInvariant, + AccountStatus::Disabled | AccountStatus::Error => AffinityEscapeReason::HardUnavailable, } } diff --git a/backend/crates/providers/openai/src/lib.rs b/backend/crates/providers/openai/src/lib.rs index af0189beb..d1ae276bf 100644 --- a/backend/crates/providers/openai/src/lib.rs +++ b/backend/crates/providers/openai/src/lib.rs @@ -110,7 +110,6 @@ pub async fn initialize( Arc::clone(&agent_identity), ports.cooldowns(), )); - let quota_skip_exhausted = config.quota_skip_exhausted(); let selector = Arc::new(CodexCredentialSelector::new( provider_kind.clone(), repository.clone(), @@ -122,7 +121,6 @@ pub async fn initialize( Arc::clone(&agent_identity), Arc::clone(&account_feedback), CodexCookiePolicy::official().map_err(|_| OpenAiInitializeError::CookiePolicy)?, - quota_skip_exhausted, )); let core_provider: Arc = Arc::new( CodexProvider::new( diff --git a/backend/crates/providers/openai/src/provider.rs b/backend/crates/providers/openai/src/provider.rs index 8228b2c71..974cf6332 100644 --- a/backend/crates/providers/openai/src/provider.rs +++ b/backend/crates/providers/openai/src/provider.rs @@ -1704,13 +1704,11 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError { struct MappedProviderFailure { error: ProviderError, account_failure: Option, - /// 原始上游错误描述,随 availability 状态一并持久化。 + /// 原始上游错误描述,仅在凭据错误状态下持久化。 error_message: Option, cyber_policy_failure: bool, set_cookie_headers: Vec, rate_limit_headers: Vec<(String, String)>, - /// 结构化 usage-limit 错误给出的窗口 reset(Unix 秒)。 - confirmed_exhaustion_reset_at: Option, observation: Option, capture_response_cookies: bool, } @@ -1724,7 +1722,6 @@ impl MappedProviderFailure { cyber_policy_failure: false, set_cookie_headers: Vec::new(), rate_limit_headers: Vec::new(), - confirmed_exhaustion_reset_at: None, observation: None, capture_response_cookies: false, } @@ -1882,18 +1879,6 @@ async fn apply_failure( failure.account_failure, Some(CodexAccountFailure::QuotaExhausted | CodexAccountFailure::UsageLimitExhausted { .. }) ); - if needs_authoritative_quota_refresh - && let Err(error) = context - .quota - .record_confirmed_exhaustion(account, failure.confirmed_exhaustion_reset_at) - .await - { - tracing::warn!( - account_id = %account.id(), - error = %error, - "Failed to project confirmed OpenAI quota exhaustion" - ); - } if failure.cyber_policy_failure { context .selector @@ -2186,12 +2171,15 @@ fn map_upstream_failure( } MappedProviderFailure { error, - account_failure: account_failure(category, failure.retry_after_seconds), + account_failure: account_failure( + category, + failure.retry_after_seconds, + failure.usage_limit_resets_at, + ), error_message: failure.client_message, cyber_policy_failure, set_cookie_headers: failure.set_cookie_headers, rate_limit_headers: failure.rate_limit_headers, - confirmed_exhaustion_reset_at: failure.usage_limit_resets_at, observation, capture_response_cookies: !matches!( category, @@ -2245,6 +2233,7 @@ const fn upstream_send_state(phase: CodexUpstreamSendPhase) -> UpstreamSendState fn account_failure( category: CodexFailureCategory, retry_after_seconds: Option, + usage_limit_resets_at: Option, ) -> Option { match category { CodexFailureCategory::CredentialExpired => Some(CodexAccountFailure::CredentialExpired), @@ -2254,7 +2243,11 @@ fn account_failure( CodexFailureCategory::Banned => Some(CodexAccountFailure::Banned), CodexFailureCategory::UsageLimitExhausted => { Some(CodexAccountFailure::UsageLimitExhausted { - retry_after: retry_after_seconds.map(Duration::from_secs), + reset_at: usage_limit_resets_at + .and_then(|seconds| u64::try_from(seconds).ok()) + .and_then(|seconds| { + SystemTime::UNIX_EPOCH.checked_add(Duration::from_secs(seconds)) + }), }) } CodexFailureCategory::RateLimited => Some(CodexAccountFailure::RateLimited { diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index a137dd7eb..e136a9246 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -5,9 +5,7 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, TimeZone as _, Utc}; use futures::future::BoxFuture; -use gateway_admin::model::accounts::{ - AccountAvailability as AdminAccountAvailability, AccountRecord, -}; +use gateway_admin::model::accounts::AccountRecord; use gateway_admin::model::observability::{ CurrencyCost, DesktopReleaseStatus, ProviderBillingInput, }; @@ -20,8 +18,8 @@ use gateway_admin::model::provider_credentials::{ use gateway_admin::model::{MutationActor, MutationContext, Revision}; use gateway_admin::ports::provider::ProviderAdminErrorKind; use gateway_core::engine::credential::{ - AccountAvailability as CoreAccountAvailability, AccountStateChange, CredentialRevision, - OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, + CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId, + ProviderAccountStore, QuotaAccessChange, QuotaEvidence, QuotaObservation, QuotaState, }; use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload}; use gateway_core::provider_ports::{ @@ -515,15 +513,14 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc } }] }); + let observed_at = SystemTime::now(); store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at, + state: QuotaState::observed_unknown(observed_at), }) .await .expect("persist quota"); @@ -596,7 +593,7 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc } #[tokio::test] -async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutating_raw_usage() { +async fn openai_admin_keeps_confirmed_exhaustion_separate_from_raw_usage_display() { let store = Arc::new(MemoryAccountStore::default()); let account_id = "acct_admin_confirmed_exhaustion"; store @@ -611,11 +608,12 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati .await; let account = store.account(account_id).expect("stored account"); let reset_at = 1_900_000_000_u64; + let observed_at = SystemTime::now(); store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( + quota: OpaqueProviderData::new( json!({ "rate_limit": { "allowed": true, @@ -626,19 +624,17 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati .as_object() .expect("quota object") .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + ), + observed_at, + state: QuotaState::allowed(observed_at), }) .await .expect("persist raw quota"); store - .apply_state_change(AccountStateChange { - message: None, + .apply_quota_access(QuotaAccessChange { account_id: account.id().clone(), expected_revision: account.revision(), - availability: CoreAccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), + state: QuotaState::exhausted(QuotaEvidence::UsageLimitReached, SystemTime::now(), None), }) .await .expect("mark account exhausted"); @@ -661,7 +657,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati .expect("project exhausted quota"); assert_eq!(exhausted.windows.len(), 1); - assert_eq!(exhausted.windows[0].used_percent, Some(100.0)); + assert_eq!(exhausted.windows[0].used_percent, Some(86.0)); let raw = store .get_quotas(std::slice::from_ref(account.id())) .await @@ -669,17 +665,15 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati .pop() .expect("raw quota"); assert_eq!( - raw.quota.expect("raw document").expose_to_provider()["rate_limit"]["primary_window"]["used_percent"], + raw.quota.expose_to_provider()["rate_limit"]["primary_window"]["used_percent"], 86 ); store - .apply_state_change(AccountStateChange { - message: None, + .apply_quota_access(QuotaAccessChange { account_id: account.id().clone(), expected_revision: account.revision(), - availability: CoreAccountAvailability::Ready, - observed_at: SystemTime::now(), + state: QuotaState::allowed(SystemTime::now()), }) .await .expect("recover account"); @@ -735,7 +729,7 @@ async fn openai_admin_provider_rejects_unprepared_mutations_before_store_commit( stale_record.access_token_expires_at = None; stale_record.next_refresh_at = None; stale_record.enabled = false; - stale_record.availability = AdminAccountAvailability::Banned; + stale_record.credential_state = CredentialState::Banned; let rotation_error = admin .prepare_rotation(PrepareCredentialRotation { account: stale_record, @@ -801,10 +795,11 @@ fn account_record(account: &ProviderAccount) -> AccountRecord { access_token_expires_at: account.access_token_expires_at().map(DateTime::::from), next_refresh_at: account.next_refresh_at().map(DateTime::::from), enabled: account.enabled(), - availability: AdminAccountAvailability::Ready, - availability_observed_at: now, + credential_state: account.credential_state(), + credential_observed_at: now, + quota: account.quota(), + last_error_reason: account.last_error_reason(), last_error_message: None, - quota_observed_at: None, created_at: now, updated_at: now, } diff --git a/backend/crates/providers/openai/tests/config.rs b/backend/crates/providers/openai/tests/config.rs index ff7acf608..2f6b55605 100644 --- a/backend/crates/providers/openai/tests/config.rs +++ b/backend/crates/providers/openai/tests/config.rs @@ -82,14 +82,6 @@ fn openai_config_restricts_upstream_base_url_to_https_or_loopback_http() { } } -#[test] -fn openai_config_keeps_the_quota_exhaustion_scheduling_switch() { - let mut config = valid_config(); - config.quota.skip_exhausted = false; - - assert!(!config.quota_skip_exhausted()); -} - #[test] fn openai_config_defaults_to_the_provider_owned_operating_values() { let config = OpenAiConfig::default(); @@ -104,7 +96,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() { config.ws_pool.max_connecting, config.ws_pool.initial_event_timeout_ms, config.quota.refresh_interval_minutes, - config.quota.skip_exhausted, config.auth.refresh_enabled, config.auth.oauth_client_id.as_str(), config.auth.oauth_token_endpoint.as_str(), @@ -119,7 +110,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() { 20_000, 15, true, - true, "app_EMoamEEZ73f0CkXaXp7hrann", "https://auth.openai.com/oauth/token", ) diff --git a/backend/crates/providers/openai/tests/credential/agent_identity.rs b/backend/crates/providers/openai/tests/credential/agent_identity.rs index 8146a314a..41bc81b5d 100644 --- a/backend/crates/providers/openai/tests/credential/agent_identity.rs +++ b/backend/crates/providers/openai/tests/credential/agent_identity.rs @@ -9,8 +9,8 @@ use chrono::{TimeZone, Utc}; use ed25519_dalek::pkcs8::EncodePrivateKey as _; use ed25519_dalek::{Signature, SigningKey, Verifier as _}; use gateway_core::engine::credential::{ - AccountAvailability, CredentialRevision, NewProviderAccount, ProviderAccount, - ProviderAccountId, ProviderAccountStore, + CredentialRevision, CredentialState, NewProviderAccount, ProviderAccount, ProviderAccountId, + ProviderAccountStore, QuotaState, }; use gateway_core::routing::ProviderKind; use provider_openai::credential::{ @@ -65,7 +65,13 @@ fn agent_account( Some("agent-account".to_owned()), Some("pro".to_owned()), ) - .with_runtime_state(true, AccountAvailability::Ready) + .with_account_facts( + true, + CredentialState::Ready, + QuotaState::unknown(), + None, + None, + ) .with_refresh_schedule(false, None); NewProviderAccount { account, diff --git a/backend/crates/providers/openai/tests/credential/contract.rs b/backend/crates/providers/openai/tests/credential/contract.rs index 0aec695b8..823ddde5b 100644 --- a/backend/crates/providers/openai/tests/credential/contract.rs +++ b/backend/crates/providers/openai/tests/credential/contract.rs @@ -8,9 +8,10 @@ use gateway_core::engine::continuation::{ ContinuationBinding, NativeContinuationPin, PreviousResponseId, }; use gateway_core::engine::credential::{ - AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy, - AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _, - QuotaObservation, QuotaWriteOutcome, RotationStrategy, + AccountAttemptFeedback, AccountErrorReason, AccountFeedbackStats, AccountSelectionPolicy, + AccountStateChange, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId, + ProviderAccountStore as _, QuotaAccessChange, QuotaAccessState, QuotaEvidence, + QuotaObservation, QuotaState, QuotaWriteOutcome, RotationStrategy, }; use gateway_core::engine::{ AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext, @@ -105,7 +106,6 @@ fn selector_with_affinity( leases, session_affinity, Arc::new(AccountFeedbackStats::default()), - true, Arc::new(MemoryCooldownPort::new()), ) } @@ -115,7 +115,6 @@ fn selector_with_runtime( leases: Arc, session_affinity: Arc, account_feedback: Arc, - skip_exhausted: bool, cooldowns: Arc, ) -> CodexCredentialSelector { let profile = CodexWireProfileState::new(CodexWireProfile { @@ -158,10 +157,42 @@ fn selector_with_runtime( agent_identity, account_feedback, CodexCookiePolicy::official().expect("official cookie policy"), - skip_exhausted, ) } +fn persist_credential_state( + store: &MemoryAccountStore, + account: &ProviderAccount, + credential_state: CredentialState, +) { + block_on(store.apply_state_change(AccountStateChange { + account_id: account.id().clone(), + expected_revision: account.revision(), + credential_state, + observed_at: SystemTime::now(), + error_reason: credential_state.error_reason(), + message: None, + })) + .expect("persist credential state"); +} + +fn persist_quota_exhaustion( + store: &MemoryAccountStore, + account: &ProviderAccount, + reset_at: Option, +) { + block_on(store.apply_quota_access(QuotaAccessChange { + account_id: account.id().clone(), + expected_revision: account.revision(), + state: QuotaState::exhausted( + QuotaEvidence::UsageLimitReached, + SystemTime::now(), + reset_at, + ), + })) + .expect("persist quota exhaustion"); +} + #[test] fn codec_persists_tokens_as_plaintext_provider_json() { let encoded = CodexCredentialCodec::encode_new( @@ -485,7 +516,6 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health Arc::new(TestLeaseCoordinator::default()), affinity, Arc::clone(&account_feedback), - true, Arc::new(MemoryCooldownPort::new()), ); for _ in 0..4 { @@ -528,16 +558,7 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() { create_account(&store, "acct_first", "at-first"); create_account(&store, "acct_second", "at-second"); let first = store.account("acct_first").expect("first account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: first.id().clone(), - expected_revision: first.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark first account exhausted"); + persist_quota_exhaustion(&store, &first, None); let affinity = Arc::new(MemorySessionAffinity::default()); let provider = ProviderKind::new("openai").expect("provider"); let key = ProviderSessionAffinityKey::try_new("quota-session").expect("affinity key"); @@ -697,13 +718,15 @@ fn credential_expired_failure_marks_unified_account_expired() { store .account("acct_primary") .expect("account") - .availability(), - AccountAvailability::Expired + .credential_state(), + CredentialState::Expired ); + let account = store.account("acct_primary").expect("expired account"); assert_eq!( - store.last_error_message("acct_primary").as_deref(), - Some("upstream_credential_expired") + account.last_error_reason(), + Some(AccountErrorReason::AccessTokenExpired) ); + assert_eq!(account.last_error_message(), None); } #[test] @@ -734,13 +757,13 @@ fn credential_expired_failure_keeps_expired_oauth_for_bounded_refresh_recovery() store .account("acct_primary") .expect("account retained for refresh") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); } #[test] -fn rate_limited_failure_records_quota_window_without_changing_availability() { +fn rate_limited_failure_records_runtime_cooldown_without_changing_persisted_facts() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let cooldowns = Arc::new(MemoryCooldownPort::new()); @@ -749,7 +772,6 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() { Arc::new(TestLeaseCoordinator::default()), Arc::new(MemorySessionAffinity::default()), Arc::new(AccountFeedbackStats::default()), - true, Arc::clone(&cooldowns) as Arc, ); let attempt = attempt(BTreeSet::new()); @@ -774,10 +796,9 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() { )) .expect("record rate-limit failure"); - // 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON—— - // 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。 let account = store.account("acct_primary").expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); + assert_eq!(account.quota().access(), QuotaAccessState::Unknown); assert!( store.quota_json("acct_primary").is_none(), "429 must not synthesize quota window" @@ -800,7 +821,7 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() { .record_failure( &account, CodexAccountFailure::UsageLimitExhausted { - retry_after: Some(Duration::from_secs(30)), + reset_at: Some(SystemTime::now() + Duration::from_secs(30)), }, None, ) @@ -808,7 +829,11 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() { .expect("record usage-limit exhaustion"); let account = store.account("acct_primary").expect("persisted account"); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); + assert_eq!(account.quota().access(), QuotaAccessState::Exhausted); + assert_eq!( + account.quota().evidence(), + Some(QuotaEvidence::UsageLimitReached) + ); assert_eq!(store.quota_reads(), 0); } @@ -817,14 +842,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let account = store.account("acct_primary").expect("account"); - block_on(store.apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - })) - .expect("mark exhausted"); + persist_quota_exhaustion(&store, &account, None); let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); block_on(selector.record_failure( @@ -836,13 +854,13 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() { )) .expect("record rate-limit failure"); - // 429 不写 availability,402 的 QuotaExhausted 保持不变。 assert_eq!( store .account("acct_primary") .expect("persisted account") - .availability(), - AccountAvailability::QuotaExhausted + .quota() + .access(), + QuotaAccessState::Exhausted ); } @@ -857,7 +875,6 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() { Arc::new(TestLeaseCoordinator::default()), Arc::new(MemorySessionAffinity::default()), Arc::new(AccountFeedbackStats::default()), - true, Arc::clone(&cooldowns) as Arc, ); @@ -874,8 +891,8 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() { store .account("acct_primary") .expect("persisted account") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); // 429 临时限流写入 Redis 冷却,不读写 quota JSON。 assert_eq!(store.quota_reads(), 0); @@ -891,14 +908,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let account = store.account("acct_primary").expect("account"); - block_on(store.apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::Invalid, - observed_at: SystemTime::now(), - })) - .expect("mark account invalid"); + persist_credential_state(&store, &account, CredentialState::Invalid); let current = store.account("acct_primary").expect("invalid account"); let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); @@ -911,34 +921,26 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() { )) .expect("record rate-limit failure"); - // 429 不写 availability(限流不改变账号可用性),认证终态保持不变。 assert_eq!( store .account("acct_primary") .expect("persisted account") - .availability(), - AccountAvailability::Invalid + .credential_state(), + CredentialState::Invalid ); } #[test] fn successful_upstream_response_recovers_non_quota_terminal_states() { for stale in [ - AccountAvailability::Expired, - AccountAvailability::Invalid, - AccountAvailability::Banned, + CredentialState::Expired, + CredentialState::Invalid, + CredentialState::Banned, ] { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let account = store.account("acct_primary").expect("account"); - block_on(store.apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: stale, - observed_at: SystemTime::now(), - })) - .expect("seed stale state"); + persist_credential_state(&store, &account, stale); let current = store.account("acct_primary").expect("stale account"); let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); @@ -948,13 +950,34 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() { store .account("acct_primary") .expect("recovered account") - .availability(), - AccountAvailability::Ready, + .credential_state(), + CredentialState::Ready, "stale state {stale:?}", ); } } +#[test] +fn elapsed_quota_reset_remains_blocked_until_authoritative_recovery() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_primary", "at-primary"); + let account = store.account("acct_primary").expect("account"); + persist_quota_exhaustion( + &store, + &account, + Some(SystemTime::now() - Duration::from_secs(1)), + ); + let blocked = store.account("acct_primary").expect("blocked account"); + assert!(blocked.quota().is_exhausted()); + assert_eq!( + blocked + .status_projection(SystemTime::now(), None) + .status + .as_str(), + "quota_exhausted" + ); +} + #[test] fn rate_limited_failures_for_distinct_accounts_do_not_conflict() { let store = Arc::new(MemoryAccountStore::default()); @@ -990,52 +1013,13 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() { store .account("acct_first") .expect("persisted first account") - .availability(), + .credential_state(), store .account("acct_second") .expect("persisted second account") - .availability(), + .credential_state(), ], - [AccountAvailability::Ready; 2] - ); -} - -#[test] -fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip_it() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_primary", "at-primary"); - let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); - let account = store.account("acct_primary").expect("account"); - block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None)) - .expect("mark account exhausted"); - - let selector = selector_with_runtime( - &store, - Arc::new(TestLeaseCoordinator::default()), - Arc::new(MemorySessionAffinity::default()), - Arc::new(AccountFeedbackStats::default()), - false, - Arc::new(MemoryCooldownPort::new()), - ); - let request_url = - Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL"); - let request_attempt = attempt(BTreeSet::new()); - - let lease = block_on(selector.select(&SelectCodexCredential { - upstream_model: "gpt-5.4", - request_url: &request_url, - attempt: &request_attempt, - session_affinity_key: None, - })) - .expect("select quota exhausted account when skipping is disabled"); - - assert_eq!(lease.account_id().as_str(), "acct_primary"); - assert_eq!( - store - .account("acct_primary") - .expect("persisted account") - .availability(), - AccountAvailability::QuotaExhausted + [CredentialState::Ready; 2] ); } @@ -1054,7 +1038,6 @@ fn native_continuation_surfaces_the_original_accounts_quota_status_to_the_coordi Arc::new(TestLeaseCoordinator::default()), Arc::new(MemorySessionAffinity::default()), Arc::new(AccountFeedbackStats::default()), - false, Arc::new(MemoryCooldownPort::new()), ); let continuation = NativeContinuationPin::new( @@ -1108,14 +1091,13 @@ fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordi "primary_window": {"used_percent": 98} } }); + let observed_at = SystemTime::now(); let outcome = block_on(store.compare_and_swap_quota(QuotaObservation { account_id: original.id().clone(), expected_revision: original.revision(), - quota: Some(OpaqueProviderData::new( - quota.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(quota.as_object().expect("quota object").clone()), + observed_at, + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), })) .expect("persist quota signal"); assert!(matches!(outcome, QuotaWriteOutcome::Updated)); @@ -1125,7 +1107,6 @@ fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordi Arc::new(TestLeaseCoordinator::default()), Arc::new(MemorySessionAffinity::default()), Arc::new(AccountFeedbackStats::default()), - false, Arc::new(MemoryCooldownPort::new()), ); let continuation = NativeContinuationPin::new( @@ -1194,8 +1175,8 @@ fn identity_verification_failure_isolates_only_selected_account() { store .account(lease.account_id().as_str()) .expect("selected account") - .availability(), - AccountAvailability::Invalid + .credential_state(), + CredentialState::Invalid ); let other = if lease.account_id().as_str() == "acct_primary" { "acct_other" @@ -1203,13 +1184,16 @@ fn identity_verification_failure_isolates_only_selected_account() { "acct_primary" }; assert_eq!( - store.account(other).expect("other account").availability(), - AccountAvailability::Ready + store + .account(other) + .expect("other account") + .credential_state(), + CredentialState::Ready ); } #[test] -fn cloudflare_challenge_does_not_change_availability() { +fn cloudflare_challenge_does_not_change_persisted_account_facts() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_primary", "at-primary"); let selector = selector(&store, Arc::new(TestLeaseCoordinator::default())); @@ -1226,7 +1210,6 @@ fn cloudflare_challenge_does_not_change_availability() { ) .expect("select account"); - // Cloudflare 挑战只走内存退避表,不写 availability(限流不改变账号可用性)。 block_on(selector.record_failure( lease.account(), CodexAccountFailure::CloudflareChallenge { retry_after: None }, @@ -1238,8 +1221,8 @@ fn cloudflare_challenge_does_not_change_availability() { store .account("acct_primary") .expect("account") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); block_on(selector.record_success(lease.account(), None)); } @@ -1279,15 +1262,15 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() { store .account("acct_primary") .expect("affected account") - .availability(), - AccountAvailability::Invalid + .credential_state(), + CredentialState::Invalid ); assert_eq!( store .account("acct_other") .expect("other account") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); } @@ -1416,7 +1399,8 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes() store .account("acct_primary") .expect("updated account") - .availability(), - AccountAvailability::QuotaExhausted + .quota() + .access(), + QuotaAccessState::Exhausted ); } diff --git a/backend/crates/providers/openai/tests/credential/oauth.rs b/backend/crates/providers/openai/tests/credential/oauth.rs index 128ec2c0f..6d8ac7de2 100644 --- a/backend/crates/providers/openai/tests/credential/oauth.rs +++ b/backend/crates/providers/openai/tests/credential/oauth.rs @@ -7,6 +7,7 @@ use gateway_admin::model::provider_credentials::{ AuthorizationMutationTarget, AuthorizationOwnerBinding, PendingAuthorizationMutation, }; use gateway_admin::model::{MutationActor, MutationContext}; +use gateway_core::engine::credential::AccountStatus; use gateway_core::routing::ProviderKind; use provider_openai::credential::token_client::{ AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant, @@ -182,7 +183,13 @@ async fn first_exchange_uses_official_id_token_claim_mapping_without_signature_v assert_eq!(account.account.upstream_user_id(), Some("chatgpt-user")); assert_eq!(account.account.upstream_account_id(), Some("workspace-id")); assert_eq!(account.account.plan_type(), Some("pro")); - assert!(account.account.is_schedulable(SystemTime::now())); + assert_eq!( + account + .account + .status_projection(SystemTime::now(), None) + .status, + AccountStatus::Normal + ); assert!( account.account.next_refresh_at().is_none(), "normal OAuth creation must not persist a margin-derived refresh time" @@ -201,7 +208,13 @@ async fn first_exchange_permits_missing_identity_claims_but_requires_a_parseable }; assert_eq!(account.account.email(), Some("profile@example.com")); assert_eq!(account.account.upstream_user_id(), None); - assert!(!account.account.is_schedulable(SystemTime::now())); + assert_eq!( + account + .account + .status_projection(SystemTime::now(), None) + .status, + AccountStatus::Error + ); let error = complete("header.not-base64.signature".to_owned()) .await diff --git a/backend/crates/providers/openai/tests/credential/quota/mod.rs b/backend/crates/providers/openai/tests/credential/quota/mod.rs index a835223a0..453c4a88a 100644 --- a/backend/crates/providers/openai/tests/credential/quota/mod.rs +++ b/backend/crates/providers/openai/tests/credential/quota/mod.rs @@ -1,7 +1,4 @@ -//! quota 测试入口与共享 fixture。 -//! -//! - [`snapshot`]:快照聚合、窗口滚动与调度信号回归。 -//! - [`recovery`]:402 恢复证据回归。 +//! OpenAI 额度事实边界与展示快照回归。 mod recovery; mod snapshot; @@ -11,16 +8,15 @@ use std::time::{Duration, SystemTime}; use chrono::{TimeZone as _, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, OpaqueProviderData, ProviderAccountStore as _, - QuotaObservation, + AccountStateChange, AccountStatus, CredentialState, ProviderAccount, ProviderAccountStore as _, + QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaState, }; use provider_openai::OFFICIAL_CODEX_BASE_URL; use provider_openai::credential::{ - CodexCredentialQuotaError, CodexCredentialQuotaService, CodexQuotaSyncSummary, - CodexQuotaWindowKind, ImportCodexOAuthCredential, + CodexCredentialQuotaError, CodexCredentialQuotaService, ImportCodexOAuthCredential, }; use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState}; -use serde_json::{Value, json}; +use serde_json::json; use wiremock::matchers::{header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -44,22 +40,16 @@ fn wire_profile() -> CodexWireProfileState { } fn quota_service(store: &Arc) -> CodexCredentialQuotaService { - quota_service_with_http( + quota_service_with_base_url( store, reqwest::Client::builder() .no_proxy() .build() .expect("client"), + OFFICIAL_CODEX_BASE_URL.to_owned(), ) } -fn quota_service_with_http( - store: &Arc, - http: reqwest::Client, -) -> CodexCredentialQuotaService { - quota_service_with_base_url(store, http, OFFICIAL_CODEX_BASE_URL.to_owned()) -} - fn quota_service_with_base_url( store: &Arc, http: reqwest::Client, @@ -75,17 +65,6 @@ fn quota_service_with_base_url( ) } -fn blocked_network_quota_service(store: &Arc) -> CodexCredentialQuotaService { - let proxy = reqwest::Proxy::all("http://127.0.0.1:9").expect("loopback proxy"); - let http = reqwest::Client::builder() - .proxy(proxy) - .connect_timeout(Duration::from_secs(1)) - .timeout(Duration::from_secs(1)) - .build() - .expect("blocked client"); - quota_service_with_http(store, http) -} - async fn create_account(store: &Arc, account_id: &str) { create_account_with_enabled(store, account_id, true).await; } @@ -107,21 +86,58 @@ async fn create_account_with_enabled( .await; } +async fn persist_credential_state( + store: &MemoryAccountStore, + account: &ProviderAccount, + credential_state: CredentialState, +) { + store + .apply_state_change(AccountStateChange { + account_id: account.id().clone(), + expected_revision: account.revision(), + credential_state, + observed_at: SystemTime::now(), + error_reason: credential_state.error_reason(), + message: None, + }) + .await + .expect("persist credential state"); +} + +async fn persist_quota_state( + store: &MemoryAccountStore, + account: &ProviderAccount, + state: QuotaState, +) { + store + .apply_quota_access(QuotaAccessChange { + account_id: account.id().clone(), + expected_revision: account.revision(), + state, + }) + .await + .expect("persist quota state"); +} + +fn exhausted_quota(reset_at: Option) -> QuotaState { + QuotaState::exhausted( + QuotaEvidence::UsageLimitReached, + SystemTime::now(), + reset_at, + ) +} + #[tokio::test] -async fn passive_rate_limit_headers_update_quota_with_revision_fence() { +async fn successful_inference_headers_are_authoritative_even_at_full_display_usage() { let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_passive_quota").await; - let account = store.account("acct_passive_quota").expect("account"); + let account_id = "acct_passive_allowed"; + create_account(&store, account_id).await; + let account = store.account(account_id).expect("account"); + persist_quota_state(&store, &account, exhausted_quota(None)).await; let service = quota_service(&store); - let reset_at = 1_900_000_000_i64; let headers = vec![ ("x-codex-active-limit".to_owned(), "codex".to_owned()), ("x-codex-primary-used-percent".to_owned(), "100".to_owned()), - ( - "x-codex-primary-window-minutes".to_owned(), - "300".to_owned(), - ), - ("x-codex-primary-reset-at".to_owned(), reset_at.to_string()), ("x-codex-limit-reached".to_owned(), "true".to_owned()), ]; @@ -131,1598 +147,39 @@ async fn passive_rate_limit_headers_update_quota_with_revision_fence() { .await .expect("passive quota") ); - let current = store - .account("acct_passive_quota") - .expect("current account"); - // 被动头限流只写 quota,不改变 availability(限流不改变账号可用性)。 - assert_eq!(current.availability(), AccountAvailability::Ready); - let snapshot = service - .read_account(account.id()) - .await - .expect("read quota") - .expect("quota snapshot"); - assert_eq!(snapshot.windows()[0].window_seconds(), Some(18_000)); - assert_eq!( - snapshot.windows()[0] - .reset_at() - .map(|value| value.timestamp()), - Some(reset_at) - ); -} - -#[tokio::test] -async fn passive_full_percent_with_explicit_allowance_keeps_account_ready() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_passive_full_allowed").await; - let account = store.account("acct_passive_full_allowed").expect("account"); - let service = quota_service(&store); - let headers = vec![ - ("x-codex-active-limit".to_owned(), "codex".to_owned()), - ("x-codex-primary-used-percent".to_owned(), "100".to_owned()), - ( - "x-codex-primary-reset-at".to_owned(), - "1900000000".to_owned(), - ), - ("x-codex-allowed".to_owned(), "true".to_owned()), - ("x-codex-limit-reached".to_owned(), "false".to_owned()), - ]; - - assert!( - service - .synchronize_passive_headers(&account, &headers) - .await - .expect("passive quota") - ); - let current = store - .account("acct_passive_full_allowed") - .expect("current account"); + let current = store.account(account_id).expect("updated account"); let snapshot = service .read_account(account.id()) .await .expect("read quota") .expect("quota snapshot"); - assert_eq!( - ( - current.availability(), - snapshot.fact().remaining_percent(), - snapshot.fact().exhausted(), - ), - // used=100 是快照级 exhaustion,但 availability 保持 Ready - // (限流不改变账号可用性)。 - (AccountAvailability::Ready, Some(0), true) - ); -} - -#[tokio::test] -async fn passive_headers_should_materialize_false_when_quota_is_not_exhausted() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_passive_quota_available").await; - let account = store - .account("acct_passive_quota_available") - .expect("account"); - let service = quota_service(&store); - let headers = vec![ - ("x-codex-active-limit".to_owned(), "codex".to_owned()), - ("x-codex-primary-used-percent".to_owned(), "31".to_owned()), - ("x-codex-allowed".to_owned(), "true".to_owned()), - ("x-codex-limit-reached".to_owned(), "false".to_owned()), - ]; - - assert!( - service - .synchronize_passive_headers(&account, &headers) - .await - .expect("passive quota") - ); - - let observations = store - .get_quotas(std::slice::from_ref(account.id())) - .await - .expect("read quota observation"); - assert_eq!( - observations - .first() - .and_then(|observation| observation.limit_reached), - Some(false) - ); -} - -#[tokio::test] -async fn manual_quota_refresh_preserves_disabled_account_state() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_disabled_quota_state"; - create_account_with_enabled(&store, account_id, false).await; - let account = store.account(account_id).expect("created disabled account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::Expired, - observed_at: SystemTime::now(), - }) - .await - .expect("seed disabled account state"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 20, "reset_at": 1_900_000_000} - } - }))) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - service - .refresh_account(account.id()) - .await - .expect("refresh disabled account quota"); - - let current = store - .account(account_id) - .expect("disabled account after quota refresh"); - assert!(!current.enabled()); - assert_eq!(current.availability(), AccountAvailability::Expired); - assert!(store.has_quota(account_id)); -} - -#[tokio::test] -async fn manual_quota_unauthorized_rejection_preserves_current_availability() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_manual_quota_auth_rejected"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(401)) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - assert!(matches!( - service.refresh_account(account.id()).await, - Err(CodexCredentialQuotaError::Upstream { .. }) - )); - assert_eq!( - store - .account(account_id) - .expect("account after rejection") - .availability(), - AccountAvailability::QuotaExhausted - ); -} - -#[tokio::test] -async fn manual_quota_forbidden_rejection_preserves_current_availability() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_manual_quota_forbidden"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(403)) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - assert!(matches!( - service.refresh_account(account.id()).await, - Err(CodexCredentialQuotaError::Upstream { .. }) - )); - assert_eq!( - store - .account(account_id) - .expect("account after rejection") - .availability(), - AccountAvailability::QuotaExhausted - ); -} - -#[tokio::test] -async fn manual_quota_refresh_waits_for_expired_oauth_token_to_be_refreshed() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_manual_quota_expired_token"; - let mut expired_profile = profile(&format!("chatgpt-{account_id}")); - expired_profile.access_token_expires_at = Some(Utc::now() - chrono::Duration::minutes(1)); - store - .seed_oauth_credential(ImportCodexOAuthCredential { - account_id: account_id.to_owned(), - name: account_id.to_owned(), - secret: secret(&format!("token-{account_id}")), - verified_account: expired_profile, - next_refresh_at: Some(Utc::now() - chrono::Duration::minutes(5)), - enabled: true, - }) - .await; - let account = store.account(account_id).expect("created account"); - let server = MockServer::start().await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - assert!(matches!( - service.refresh_account(account.id()).await, - Err(CodexCredentialQuotaError::CredentialRefreshRequired) - )); - assert!( - server - .received_requests() - .await - .expect("received requests") - .is_empty() - ); -} - -#[tokio::test] -async fn passive_rate_limit_headers_replace_stale_secondary_window() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_passive_quota_snapshot").await; - let account = store - .account("acct_passive_quota_snapshot") - .expect("account"); - let baseline = json!({ - "rate_limit": { - "primary_window": { - "used_percent": 91, - "reset_at": 1_900_000_000, - "limit_window_seconds": 2_592_000 - }, - "secondary_window": { - "used_percent": 43, - "reset_at": 1_900_000_000, - "limit_window_seconds": 2_592_000 - } - } - }); - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - baseline.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("persist baseline quota"); - let service = quota_service(&store); - let headers = vec![ - ("x-codex-active-limit".to_owned(), "codex".to_owned()), - ("x-codex-primary-used-percent".to_owned(), "92".to_owned()), - ( - "x-codex-primary-window-minutes".to_owned(), - "43200".to_owned(), - ), - ( - "x-codex-primary-reset-at".to_owned(), - "1900000000".to_owned(), - ), - ]; - - assert!( - service - .synchronize_passive_headers(&account, &headers) - .await - .expect("passive quota") - ); - let snapshot = service - .read_account(account.id()) - .await - .expect("read quota") - .expect("quota snapshot"); - let raw = store - .quota_json("acct_passive_quota_snapshot") - .expect("stored passive quota"); - let monthly = snapshot - .windows() - .iter() - .filter(|window| window.kind() == CodexQuotaWindowKind::Monthly) - .collect::>(); - - assert_eq!(snapshot.windows().len(), 1); - assert_eq!(monthly.len(), 1); - assert_eq!(monthly[0].used_percent(), Some(92.0)); - assert!( - raw.pointer("/rate_limit/secondary_window").is_none(), - "a normal response without a secondary window must delete stale raw state" - ); -} - -#[tokio::test] -async fn manual_quota_refresh_marks_ready_account_exhausted_and_drops_empty_secondary_window() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_manual_secondary_placeholder"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": { - "used_percent": 100, - "reset_at": 1_900_000_000, - "limit_window_seconds": 2_592_000 - }, - "secondary_window": { - "used_percent": 0, - "limit_reached": false - } - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let snapshot = service - .refresh_account(account.id()) - .await - .expect("manual quota refresh"); - - assert_eq!(snapshot.windows().len(), 1); - assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0)); - assert_eq!( - store - .account(account_id) - .expect("account after refresh") - .availability(), - AccountAvailability::QuotaExhausted - ); -} - -#[tokio::test] -async fn manual_quota_refresh_drops_null_secondary_window_during_confirmed_exhaustion() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_manual_null_secondary_placeholder"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - let reset_at = 1_900_000_000; - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": { - "used_percent": 100, - "reset_at": reset_at, - "limit_window_seconds": 2_592_000 - } - } - }) - .as_object() - .expect("quota object") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("seed confirmed exhaustion"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": { - "used_percent": 86, - "reset_at": reset_at, - "reset_after_seconds": 2_133_000, - "limit_window_seconds": 2_592_000 - }, - "secondary_window": null - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let snapshot = service - .refresh_account(account.id()) - .await - .expect("manual quota refresh"); - let persisted = store - .get_quotas(&[account.id().clone()]) - .await - .expect("read persisted quota") - .into_iter() - .next() - .and_then(|observation| observation.quota) - .map(OpaqueProviderData::into_inner) - .map(Value::Object) - .expect("persisted quota"); - - assert_eq!(snapshot.windows().len(), 1); - assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0)); - assert_eq!( - snapshot.windows()[0] - .reset_at() - .map(|value| value.timestamp()), - Some(reset_at) - ); - assert!(snapshot.fact().exhausted()); - assert_eq!( - store - .account(account_id) - .expect("account after refresh") - .availability(), - AccountAvailability::QuotaExhausted - ); - assert!( - persisted.pointer("/rate_limit/secondary_window").is_none(), - "a null secondary window must not be projected as 100%" - ); -} - -#[tokio::test] -async fn passive_active_limit_keeps_the_top_level_codex_bucket_canonical() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_passive_active_limit").await; - let account = store.account("acct_passive_active_limit").expect("account"); - let baseline = json!({ - "additional_rate_limits": [{ - "metered_feature": "codex", - "rate_limit": { - "allowed": true, - "primary_window": { - "used_percent": 20, - "reset_at": 1_900_000_000, - "limit_window_seconds": 604_800 - } - } - }] - }); - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - baseline.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("persist baseline quota"); - let service = quota_service(&store); - let headers = vec![ - ("x-codex-active-limit".to_owned(), "premium".to_owned()), - ("x-premium-primary-used-percent".to_owned(), "55".to_owned()), - ( - "x-premium-primary-window-minutes".to_owned(), - "10080".to_owned(), - ), - ( - "x-premium-primary-reset-at".to_owned(), - "1900000000".to_owned(), - ), - ]; - - assert!( - service - .synchronize_passive_headers(&account, &headers) - .await - .expect("passive quota") - ); - let snapshot = service - .read_account(account.id()) - .await - .expect("read quota") - .expect("quota snapshot"); - - assert_eq!(snapshot.windows().len(), 1); - assert!( - snapshot - .windows() - .iter() - .any(|window| { window.source() == "codex" && window.used_percent() == Some(55.0) }) - ); -} - -#[tokio::test] -async fn passive_metadata_headers_preserve_quota_observation_time() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_passive_metadata").await; - let account = store.account("acct_passive_metadata").expect("account"); - let baseline = json!({ - "rate_limit": { - "allowed": true, - "primary_window": { - "used_percent": 20, - "reset_at": 1_900_000_000, - "limit_window_seconds": 604_800 - } - } - }); - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - baseline.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("persist baseline quota"); - let before = store - .get_quotas(std::slice::from_ref(account.id())) - .await - .expect("read baseline quota"); - let service = quota_service(&store); - let headers = vec![ - ("x-codex-plan-type".to_owned(), "free".to_owned()), - ("x-codex-credits-has-credits".to_owned(), "true".to_owned()), - ("x-codex-credits-unlimited".to_owned(), "false".to_owned()), - ]; - - assert!( - service - .synchronize_passive_headers(&account, &headers) - .await - .expect("metadata-only passive response") - ); - let after = store - .get_quotas(std::slice::from_ref(account.id())) - .await - .expect("read updated quota metadata"); - - assert_eq!(after[0].observed_at, before[0].observed_at); - assert_eq!(after[0].limit_reached, before[0].limit_reached); - assert_eq!( - store - .quota_json("acct_passive_metadata") - .and_then(|quota| quota.get("plan_type").cloned()), - Some(json!("free")) - ); -} - -#[tokio::test] -async fn synchronize_without_accounts_is_a_noop_before_network_io() { - let store = Arc::new(MemoryAccountStore::default()); - - let summary = quota_service(&store) - .synchronize() - .await - .expect("empty synchronization"); - - assert_eq!(summary.updated, 0); - assert_eq!(summary.exhausted, 0); - assert_eq!(summary.banned, 0); - assert_eq!(summary.transient, 0); - assert_eq!(summary.stale, 0); -} - -#[tokio::test] -async fn periodic_quota_synchronization_does_not_scan_stale_ready_accounts() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_ready").await; - let account = store - .account("acct_periodic_ready") - .expect("created account"); - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": true, - "primary_window": {"used_percent": 10, "reset_at": 1_900_000_000} - } - }) - .as_object() - .expect("quota object") - .clone(), - )), - // 正常账号即使缓存已旧,也只等待真实请求被动同步,不能被定时任务主动拉取。 - observed_at: Some(SystemTime::UNIX_EPOCH), - limit_reached: None, - }) - .await - .expect("persist stale quota"); - - let summary = blocked_network_quota_service(&store) - .synchronize() - .await - .expect("ready account must not query upstream"); - - assert_eq!(summary, CodexQuotaSyncSummary::default()); -} - -#[tokio::test] -async fn periodic_quota_synchronization_marks_ready_account_exhausted_when_primary_is_exhausted() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_stuck_cooldown").await; - let account = store - .account("acct_periodic_stuck_cooldown") - .expect("created account"); - let reset_at = Utc::now().timestamp() + 24 * 60 * 60; - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::Ready, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account cooldown"); - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": false, - "primary_window": {"used_percent": 100, "reset_at": reset_at} - } - }) - .as_object() - .expect("quota object") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("persist exhausted quota"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": {"used_percent": 100, "reset_at": reset_at} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - service - .prepare_scheduling(std::slice::from_ref(&account)) - .await; - - let summary = service.synchronize().await.expect("quota synchronization"); - - assert_eq!(summary.exhausted, 1); - let reconciled = store - .account("acct_periodic_stuck_cooldown") - .expect("reconciled account"); - assert_eq!( - reconciled.availability(), - AccountAvailability::QuotaExhausted - ); -} - -#[tokio::test] -async fn periodic_quota_synchronization_keeps_ready_when_only_secondary_window_is_exhausted() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_secondary_only").await; - let reset_at = Utc::now().timestamp() + 24 * 60 * 60; - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 5, "reset_at": reset_at}, - "secondary_window": {"used_percent": 100, "reset_at": reset_at} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let summary = service.synchronize().await.expect("quota synchronization"); - - assert_eq!(summary.exhausted, 1); - assert_eq!( - store - .account("acct_periodic_secondary_only") - .expect("reconciled account") - .availability(), - AccountAvailability::Ready - ); -} - -#[tokio::test] -async fn periodic_quota_synchronization_keeps_ready_when_primary_is_full_but_requests_are_allowed() -{ - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_primary_full_but_allowed").await; - let reset_at = Utc::now().timestamp() + 24 * 60 * 60; - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 100, "reset_at": reset_at} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let summary = service.synchronize().await.expect("quota synchronization"); - - assert_eq!(summary.exhausted, 1); - assert_eq!( - store - .account("acct_periodic_primary_full_but_allowed") - .expect("reconciled account") - .availability(), - AccountAvailability::Ready - ); -} - -#[tokio::test] -async fn periodic_quota_synchronization_keeps_ready_without_primary_exhaustion_evidence() { - let store = Arc::new(MemoryAccountStore::default()); - create_account( - &store, - "acct_periodic_disallowed_without_primary_exhaustion", - ) - .await; - let reset_at = Utc::now().timestamp() + 24 * 60 * 60; - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": false, - "limit_reached": false, - "primary_window": {"used_percent": 5, "reset_at": reset_at} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let summary = service.synchronize().await.expect("quota synchronization"); - - assert_eq!(summary.exhausted, 1); - assert_eq!( - store - .account("acct_periodic_disallowed_without_primary_exhaustion") - .expect("reconciled account") - .availability(), - AccountAvailability::Ready - ); -} - -#[tokio::test] -async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_limit() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_exhausted").await; - let account = store - .account("acct_periodic_exhausted") - .expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(429)) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - service.synchronize().await.expect("quota synchronization"); - - let account = store - .account("acct_periodic_exhausted") - .expect("persisted account"); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); -} - -#[tokio::test] -async fn periodic_quota_refresh_drops_empty_secondary_window() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_periodic_secondary_placeholder"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": { - "used_percent": 99, - "reset_at": 1_900_000_000, - "limit_window_seconds": 2_592_000 - }, - "secondary_window": {} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let summary = service.synchronize().await.expect("periodic quota refresh"); - let snapshot = service - .read_account(account.id()) - .await - .expect("read quota") - .expect("quota snapshot"); - - assert_eq!(summary.exhausted, 1); - assert_eq!(snapshot.windows().len(), 1); - assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0)); -} - -#[tokio::test] -async fn periodic_quota_refresh_drops_null_secondary_window() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_periodic_null_secondary_placeholder"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": { - "used_percent": 86, - "reset_at": 1_900_000_000, - "reset_after_seconds": 2_133_000, - "limit_window_seconds": 2_592_000 - }, - "secondary_window": null - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let summary = service.synchronize().await.expect("periodic quota refresh"); - let snapshot = service - .read_account(account.id()) - .await - .expect("read quota") - .expect("quota snapshot"); - - assert_eq!(summary.exhausted, 1); - assert_eq!(snapshot.windows().len(), 1); - assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0)); -} - -#[tokio::test] -async fn periodic_quota_refresh_keeps_nonempty_secondary_window_during_confirmed_exhaustion() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_periodic_real_secondary"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let secondary_reset_at = 1_900_604_800; - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": { - "used_percent": 99, - "reset_at": 1_900_000_000, - "limit_window_seconds": 2_592_000 - }, - "secondary_window": { - "used_percent": 0, - "reset_at": secondary_reset_at, - "limit_window_seconds": 604_800 - } - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let summary = service.synchronize().await.expect("periodic quota refresh"); - let snapshot = service - .read_account(account.id()) - .await - .expect("read quota") - .expect("quota snapshot"); - - assert_eq!(summary.exhausted, 1); - assert_eq!(snapshot.windows().len(), 2); - assert!(snapshot.windows().iter().any(|window| { - window.window_seconds() == Some(604_800) - && window.used_percent() == Some(100.0) - && window - .reset_at() - .is_some_and(|reset_at| reset_at.timestamp() == secondary_reset_at) - })); -} - -#[tokio::test] -async fn periodic_quota_synchronization_respects_retry_after_on_service_unavailable() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_503_retry_after").await; - let account = store - .account("acct_periodic_503_retry_after") - .expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - // 5xx 短退避重试后仍失败,Retry-After 用于写冷却,不再落入"未分类"。 - .respond_with(ResponseTemplate::new(503).insert_header("Retry-After", "120")) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - service.synchronize().await.expect("quota synchronization"); - - let account = store - .account("acct_periodic_503_retry_after") - .expect("persisted account"); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); -} - -#[tokio::test] -async fn initial_quota_synchronization_retries_ready_accounts_without_observation() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_temporary_cooldown").await; - let account = store - .account("acct_periodic_temporary_cooldown") - .expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::Ready, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account cooldown"); - - let summary = blocked_network_quota_service(&store) - .synchronize() - .await - .expect("initial quota sync must not fail the worker"); - - assert_eq!(summary.transient, 1); - assert_eq!( - store - .account("acct_periodic_temporary_cooldown") - .expect("cooldown account") - .availability(), - AccountAvailability::Ready - ); -} - -#[tokio::test] -async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_exhausted").await; - let account = store - .account("acct_periodic_exhausted") - .expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - - let summary = blocked_network_quota_service(&store) - .synchronize() - .await - .expect("quota synchronization"); - - assert_eq!(summary.transient, 1); -} - -#[tokio::test] -async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_waiting_for_reset() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_exhausted_cooldown").await; - let account = store - .account("acct_periodic_exhausted_cooldown") - .expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted with a future reset"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(500)) - // 5xx 短退避重试 1s/2s 后仍失败,最终计入 transient。 - .expect(3) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - // reset 仅用于展示/调度状态,不能推迟额度耗尽账号的定时复核。 - let due = service.synchronize().await.expect("quota synchronization"); - let repeated = service - .synchronize() - .await - .expect("quota synchronization repeated"); - - assert_eq!(due.transient, 1); - assert_eq!(repeated, CodexQuotaSyncSummary::default()); -} - -#[tokio::test] -async fn periodic_quota_synchronization_keeps_exhaustion_at_ten_percent_after_advanced_reset() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_periodic_recovery_evidence"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - let reset_at = 1_900_000_000; - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 100, "reset_at": reset_at} - } - }) - .as_object() - .expect("quota object") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("seed post-failure quota snapshot"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 100, "reset_at": reset_at} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let unchanged = service - .synchronize() - .await - .expect("unchanged quota recheck"); - - // used=100 是快照级 exhaustion,计 exhausted 而非 updated。 - assert_eq!(unchanged.exhausted, 1); - assert_eq!( - store - .account(account_id) - .expect("account after unchanged snapshot") - .availability(), - AccountAvailability::QuotaExhausted - ); - - server.reset().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - // reset 已后移,但 10% 仍不能证明这是刚重置的新窗口。 - "primary_window": {"used_percent": 10, "reset_at": reset_at + 3600} - } - }))) - .expect(1) - .mount(&server) - .await; - // 新 worker 实例模拟下一次 30 分钟复核,避免测试等待真实节流周期。 - let next_cycle = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let ten_percent = next_cycle - .synchronize() - .await - .expect("ten-percent quota recheck"); - - assert_eq!(ten_percent.exhausted, 1); - assert_eq!( - store - .account(account_id) - .expect("account after ten-percent snapshot") - .availability(), - AccountAvailability::QuotaExhausted, - "an advanced reset at exactly 10% is not sufficient to release confirmed exhaustion" - ); - let snapshot = next_cycle - .read_account(account.id()) - .await - .expect("read held exhaustion projection") - .expect("held exhaustion projection"); assert_eq!(snapshot.fact().remaining_percent(), Some(0)); -} - -#[tokio::test] -async fn periodic_quota_synchronization_recovers_with_advanced_reset_and_low_primary_usage() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_periodic_reset_elapsed"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - let old_reset = Utc::now().timestamp() + 3_600; - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": {"used_percent": 100, "reset_at": old_reset} - } - }) - .as_object() - .expect("recorded exhausted quota") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: Some(true), - }) - .await - .expect("record future exhausted reset"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted with a future reset"); - let server = MockServer::start().await; - let next_reset = old_reset + 3_600; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 1, "reset_at": next_reset} - } - }))) - .expect(1) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - // 定时复核可在旧 reset 前执行;新主窗口已推进且用量低于 10% 才释放额度耗尽状态。 - let summary = service.synchronize().await.expect("quota synchronization"); - - assert_eq!(summary.updated, 1); - let current = store - .account(account_id) - .expect("account after elapsed reset"); - assert_eq!(current.availability(), AccountAvailability::Ready); -} - -#[tokio::test] -async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() { - let store = Arc::new(MemoryAccountStore::default()); - create_account(&store, "acct_periodic_throttled").await; - let account = store - .account("acct_periodic_throttled") - .expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - let service = blocked_network_quota_service(&store); - - service.synchronize().await.expect("first quota cycle"); - let summary = service.synchronize().await.expect("throttled quota cycle"); - - assert_eq!(summary, CodexQuotaSyncSummary::default()); -} - -#[tokio::test] -async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset() { - let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_periodic_new_reset"; - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted with due reset"); - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": {"used_percent": 100, "reset_at": 2} - } - }))) - .mount(&server) - .await; - let service = quota_service_with_base_url( - &store, - reqwest::Client::builder() - .no_proxy() - .build() - .expect("client"), - server.uri(), - ); - - let first = service.synchronize().await.expect("first reset recheck"); - let second = service - .synchronize() - .await - .expect("throttled reset recheck"); - let repeated = service.synchronize().await.expect("same reset throttled"); - let request_count = server - .received_requests() - .await - .expect("received requests") - .len(); - + assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed); + assert_eq!(current.quota().access(), QuotaAccessState::Allowed); assert_eq!( - (first.exhausted, second.exhausted, repeated, request_count,), - (1, 0, CodexQuotaSyncSummary::default(), 1,) + current.status_projection(SystemTime::now(), None).status, + AccountStatus::Normal ); } #[tokio::test] -async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() { +async fn quota_refresh_persists_explicit_provider_denial() { let store = Arc::new(MemoryAccountStore::default()); - for account_id in ["acct_quota_batch_bad", "acct_quota_batch_good"] { - create_account(&store, account_id).await; - let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); - } - let good = store - .account("acct_quota_batch_good") - .expect("good account"); - let elapsed_reset = Utc::now().timestamp() - 60; - store - .compare_and_swap_quota(QuotaObservation { - account_id: good.id().clone(), - expected_revision: good.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": {"used_percent": 100, "reset_at": elapsed_reset} - } - }) - .as_object() - .expect("recorded exhausted quota") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: Some(true), - }) - .await - .expect("seed good account old reset"); + let account_id = "acct_explicit_quota_denial"; + create_account(&store, account_id).await; + let account = store.account(account_id).expect("account"); let server = MockServer::start().await; - let next_reset = Utc::now().timestamp() + 3600; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .and(header("authorization", "Bearer token-acct_quota_batch_bad")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({"rate_limit": {"allowed": "not-a-boolean"}})), - ) - .mount(&server) - .await; Mock::given(method("GET")) .and(path("/api/codex/usage")) .and(header( "authorization", - "Bearer token-acct_quota_batch_good", + format!("Bearer token-{account_id}"), )) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "rate_limit": { - "allowed": true, - "primary_window": {"used_percent": 1.0, "reset_at": next_reset} + "allowed": false, + "primary_window": {"used_percent": 8, "reset_at": 1_900_000_000} } }))) .mount(&server) @@ -1736,20 +193,125 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() { server.uri(), ); - let summary = service - .synchronize() - .await - .expect("batch must survive one invalid account"); - - assert_eq!(summary.updated, 1); - assert_eq!(summary.transient, 1); - let good = store - .account("acct_quota_batch_good") - .expect("good account"); let snapshot = service - .read_account(good.id()) + .refresh_account(account.id()) .await - .expect("read good quota") - .expect("good quota persisted"); - assert_eq!(snapshot.fact().remaining_percent(), Some(99)); + .expect("refresh quota"); + let current = store.account(account_id).expect("updated account"); + + assert_eq!(snapshot.fact().remaining_percent(), Some(92)); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted); + assert_eq!( + snapshot.quota().evidence(), + Some(QuotaEvidence::ProviderDenied) + ); + assert_eq!( + current.status_projection(SystemTime::now(), None).status, + AccountStatus::QuotaExhausted + ); +} + +#[tokio::test] +async fn quota_endpoint_auth_rejections_do_not_reclassify_credentials_or_quota() { + for status in [401, 403, 429, 503] { + let store = Arc::new(MemoryAccountStore::default()); + let account_id = format!("acct_quota_rejection_{status}"); + create_account(&store, &account_id).await; + let account = store.account(&account_id).expect("account"); + persist_quota_state(&store, &account, exhausted_quota(None)).await; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/codex/usage")) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + let service = quota_service_with_base_url( + &store, + reqwest::Client::builder() + .no_proxy() + .build() + .expect("client"), + server.uri(), + ); + + assert!(matches!( + service.refresh_account(account.id()).await, + Err(CodexCredentialQuotaError::Upstream { .. }) + )); + let current = store.account(&account_id).expect("preserved account"); + assert_eq!(current.credential_state(), CredentialState::Ready); + assert_eq!(current.quota().access(), QuotaAccessState::Exhausted); + } +} + +#[tokio::test] +async fn payment_required_is_authoritative_quota_exhaustion_without_fabricated_usage() { + let store = Arc::new(MemoryAccountStore::default()); + let account_id = "acct_payment_required"; + create_account(&store, account_id).await; + let account = store.account(account_id).expect("account"); + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/codex/usage")) + .respond_with(ResponseTemplate::new(402).set_body_json(json!({ + "detail": {"code": "billing_required"} + }))) + .mount(&server) + .await; + let service = quota_service_with_base_url( + &store, + reqwest::Client::builder() + .no_proxy() + .build() + .expect("client"), + server.uri(), + ); + + assert!(service.refresh_account(account.id()).await.is_err()); + let current = store.account(account_id).expect("updated account"); + assert_eq!(current.quota().access(), QuotaAccessState::Exhausted); + assert_eq!( + current.quota().evidence(), + Some(QuotaEvidence::PaymentRequired) + ); + assert!(!store.has_quota(account_id)); +} + +#[tokio::test] +async fn quota_refresh_preserves_disabled_and_credential_error_facts() { + let store = Arc::new(MemoryAccountStore::default()); + let account_id = "acct_disabled_credential_error"; + create_account_with_enabled(&store, account_id, false).await; + let account = store.account(account_id).expect("account"); + persist_credential_state(&store, &account, CredentialState::Expired).await; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/codex/usage")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "rate_limit": {"allowed": true, "primary_window": {"used_percent": 1}} + }))) + .mount(&server) + .await; + let service = quota_service_with_base_url( + &store, + reqwest::Client::builder() + .no_proxy() + .build() + .expect("client"), + server.uri(), + ); + + service + .refresh_account(account.id()) + .await + .expect("refresh quota"); + let current = store.account(account_id).expect("updated account"); + + assert!(!current.enabled()); + assert_eq!(current.credential_state(), CredentialState::Expired); + assert_eq!(current.quota().access(), QuotaAccessState::Allowed); + assert_eq!( + current.status_projection(SystemTime::now(), None).status, + AccountStatus::Disabled + ); } diff --git a/backend/crates/providers/openai/tests/credential/quota/recovery.rs b/backend/crates/providers/openai/tests/credential/quota/recovery.rs index 5e72bbe96..17a61f40c 100644 --- a/backend/crates/providers/openai/tests/credential/quota/recovery.rs +++ b/backend/crates/providers/openai/tests/credential/quota/recovery.rs @@ -1,25 +1,15 @@ -//! 402 恢复回归。 -//! -//! 覆盖 `QuotaExhausted -> Ready` 的证据规则:新 core primary reset 后移 + 新快照用量低于 10%。 +//! 额度刷新与凭证事实相互独立的回归测试。 use super::*; #[tokio::test] -async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_reset() { +async fn quota_refresh_updates_access_fact_without_recovering_credential_error() { let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_manual_quota_state"; + let account_id = "acct_independent_quota_and_credential"; create_account(&store, account_id).await; let account = store.account(account_id).expect("created account"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::Expired, - observed_at: SystemTime::now(), - }) - .await - .expect("seed stale expired state"); + persist_credential_state(&store, &account, CredentialState::Expired).await; + let server = MockServer::start().await; Mock::given(method("GET")) .and(path("/api/codex/usage")) @@ -45,20 +35,15 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese server.uri(), ); - let exhausted = service + let snapshot = service .refresh_account(account.id()) .await .expect("refresh exhausted quota"); + let current = store.account(account_id).expect("updated account"); - assert!(exhausted.fact().exhausted()); - assert_eq!( - store - .account(account_id) - .expect("exhausted account") - .availability(), - AccountAvailability::Expired, - "manual refresh never changes non-QuotaExhausted availability" - ); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted); + assert_eq!(current.quota().access(), QuotaAccessState::Exhausted); + assert_eq!(current.credential_state(), CredentialState::Expired); server.reset().await; Mock::given(method("GET")) @@ -77,111 +62,39 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese .mount(&server) .await; - let stale = service + let snapshot = service .refresh_account(account.id()) .await - .expect("refresh contradictory full quota"); + .expect("refresh allowed quota"); + let current = store.account(account_id).expect("recovered quota account"); - // used=100 是快照级 exhaustion。 - assert!(stale.fact().exhausted()); - assert_eq!( - store - .account(account_id) - .expect("account after contradictory quota") - .availability(), - AccountAvailability::Expired, - "an unchanged usage window cannot override an expired credential state" - ); - - server.reset().await; - Mock::given(method("GET")) - .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 99, "reset_at": 1_900_000_000} - } - }))) - .mount(&server) - .await; - - let unchanged_reset = service - .refresh_account(account.id()) - .await - .expect("refresh lower usage quota"); - - assert!(!unchanged_reset.fact().exhausted()); - assert_eq!( - store - .account(account_id) - .expect("account after lower usage") - .availability(), - AccountAvailability::Expired, - "a lower used_percent before the recorded reset is not quota recovery for an expired credential" - ); + assert_eq!(snapshot.fact().remaining_percent(), Some(0)); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed); + assert_eq!(current.quota().access(), QuotaAccessState::Allowed); + assert_eq!(current.credential_state(), CredentialState::Expired); } #[tokio::test] -async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() { - // 旧 reset 仍在未来;同一主窗口 reset 后移且用量低于 10% 时才恢复 Ready。 +async fn explicit_allowance_replaces_confirmed_exhaustion_without_usage_heuristics() { let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_cross_window_recovery"; + let account_id = "acct_authoritative_quota_recovery"; create_account(&store, account_id).await; let account = store.account(account_id).expect("created account"); - let old_reset = Utc::now().timestamp() + 3_600; - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": { - "used_percent": 100, - "reset_at": old_reset, - "limit_window_seconds": 3600 - } - } - }) - .as_object() - .expect("quota object") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("seed old exhausted snapshot"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); + persist_quota_state( + &store, + &account, + exhausted_quota(Some(SystemTime::now() + Duration::from_secs(3_600))), + ) + .await; + let server = MockServer::start().await; - let future_reset = old_reset + 3_600; Mock::given(method("GET")) .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "rate_limit": { "allowed": true, "limit_reached": false, - "primary_window": {"used_percent": 9.9, "reset_at": future_reset} + "primary_window": {"used_percent": 100, "reset_at": 1_900_000_000} } }))) .mount(&server) @@ -195,77 +108,36 @@ async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() { server.uri(), ); - let recovered = service + let snapshot = service .refresh_account(account.id()) .await - .expect("refresh new window quota"); + .expect("refresh authoritative allowance"); - assert!(!recovered.fact().exhausted()); + assert_eq!(snapshot.fact().remaining_percent(), Some(0)); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed); assert_eq!( store .account(account_id) - .expect("account after cross-window recovery") - .availability(), - AccountAvailability::Ready, - "an advanced reset with a nearly unused primary window must recover QuotaExhausted" + .expect("updated account") + .quota() + .access(), + QuotaAccessState::Allowed ); } #[tokio::test] -async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaustion() { +async fn inconclusive_quota_refresh_preserves_confirmed_exhaustion() { let store = Arc::new(MemoryAccountStore::default()); - let account_id = "acct_cross_window_ten_percent"; + let account_id = "acct_inconclusive_quota_refresh"; create_account(&store, account_id).await; let account = store.account(account_id).expect("created account"); - let old_reset = Utc::now().timestamp() + 3_600; - store - .compare_and_swap_quota(QuotaObservation { - account_id: account.id().clone(), - expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - json!({ - "rate_limit": { - "allowed": false, - "limit_reached": true, - "primary_window": { - "used_percent": 100, - "reset_at": old_reset, - "limit_window_seconds": 3600 - } - } - }) - .as_object() - .expect("quota object") - .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, - }) - .await - .expect("seed old exhausted snapshot"); - store - .apply_state_change(AccountStateChange { - message: None, - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), - }) - .await - .expect("mark account exhausted"); + persist_quota_state(&store, &account, exhausted_quota(None)).await; let server = MockServer::start().await; - let future_reset = old_reset + 3_600; Mock::given(method("GET")) .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "rate_limit": { - "allowed": true, - "limit_reached": false, - "primary_window": {"used_percent": 10.0, "reset_at": future_reset} + "primary_window": {"used_percent": 12, "reset_at": 1_900_000_000} } }))) .mount(&server) @@ -279,24 +151,24 @@ async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaus server.uri(), ); - let held = service + let snapshot = service .refresh_account(account.id()) .await - .expect("refresh non-empty new window quota"); + .expect("refresh inconclusive quota"); - assert!(held.fact().exhausted()); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted); assert_eq!( store .account(account_id) - .expect("account after ten-percent snapshot") - .availability(), - AccountAvailability::QuotaExhausted, - "a reset advance at exactly 10% must not release confirmed exhaustion" + .expect("preserved account") + .quota() + .access(), + QuotaAccessState::Exhausted ); } #[tokio::test] -async fn deactivated_workspace_quota_response_persists_the_upstream_reason() { +async fn deactivated_workspace_quota_response_persists_credential_error_reason() { let store = Arc::new(MemoryAccountStore::default()); let account_id = "acct_deactivated_workspace"; create_account(&store, account_id).await; @@ -304,10 +176,6 @@ async fn deactivated_workspace_quota_response_persists_the_upstream_reason() { let server = MockServer::start().await; Mock::given(method("GET")) .and(path("/api/codex/usage")) - .and(header( - "authorization", - format!("Bearer token-{account_id}"), - )) .respond_with(ResponseTemplate::new(402).set_body_json(json!({ "detail": { "code": "deactivated_workspace", @@ -326,15 +194,12 @@ async fn deactivated_workspace_quota_response_persists_the_upstream_reason() { ); assert!(service.refresh_account(account.id()).await.is_err()); + let current = store + .account(account_id) + .expect("account after deactivation"); + assert_eq!(current.credential_state(), CredentialState::Banned); assert_eq!( - store - .account(account_id) - .expect("account after deactivation") - .availability(), - AccountAvailability::Banned - ); - assert_eq!( - store.last_error_message(account_id).as_deref(), - Some("deactivated_workspace") + current.last_error_reason(), + Some(gateway_core::engine::credential::AccountErrorReason::AccountBanned) ); } diff --git a/backend/crates/providers/openai/tests/credential/quota/snapshot.rs b/backend/crates/providers/openai/tests/credential/quota/snapshot.rs index 2b3851e13..e20f85326 100644 --- a/backend/crates/providers/openai/tests/credential/quota/snapshot.rs +++ b/backend/crates/providers/openai/tests/credential/quota/snapshot.rs @@ -7,7 +7,8 @@ use std::time::{Duration, SystemTime}; use futures::future::join_all; use gateway_core::engine::credential::{ - OpaqueProviderData, ProviderAccountStore as _, QuotaObservation, QuotaWriteOutcome, + OpaqueProviderData, ProviderAccountStore as _, QuotaAccessState, QuotaObservation, QuotaState, + QuotaWriteOutcome, }; use provider_openai::credential::{CodexQuotaWindowKind, parse_codex_quota_usage}; use serde_json::json; @@ -34,11 +35,11 @@ fn parser_extracts_dynamic_windows_without_a_fixed_database_shape() { fact.resets_at().map(|value| value.timestamp()), Some(1_800_000_100) ); - assert!(!fact.exhausted()); + assert_eq!(fact.remaining_percent(), Some(20)); } #[test] -fn parser_treats_any_confirmed_provider_limit_as_exhausted() { +fn parser_keeps_additional_limit_as_display_data() { let fact = parse_codex_quota_usage(&json!({ "rate_limit": {"primary_window": {"used_percent": 10}}, "additional_rate_limits": [{ @@ -47,7 +48,7 @@ fn parser_treats_any_confirmed_provider_limit_as_exhausted() { })) .expect("valid quota"); - assert!(fact.exhausted()); + assert_eq!(fact.remaining_percent(), Some(0)); } #[test] @@ -61,14 +62,12 @@ fn parser_keeps_full_percent_as_display_without_provider_exhaustion_signal() { })) .expect("valid full-percent quota"); - // 任一子窗口 used>=100 是快照级 exhaustion 事实, - // 即使顶层 allowed=true。 + // 百分比只负责展示;额度访问结论由规范化 QuotaState 承载。 assert_eq!(fact.remaining_percent(), Some(0)); - assert!(fact.exhausted()); } #[test] -fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() { +fn parser_projects_percent_when_access_is_denied() { let fact = parse_codex_quota_usage(&json!({ "rate_limit": { "allowed": false, @@ -79,7 +78,6 @@ fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() .expect("valid provider-confirmed quota"); assert_eq!(fact.remaining_percent(), Some(2)); - assert!(fact.exhausted()); } #[test] @@ -94,7 +92,7 @@ fn parser_does_not_infer_exhaustion_from_unknown_credit_fields() { })) .expect("recognized credits object"); - assert!(!fact.exhausted()); + assert_eq!(fact.remaining_percent(), None); } #[test] @@ -129,7 +127,6 @@ fn parser_accepts_official_null_additional_rate_limits() { .expect("official null additional quota"); assert_eq!(fact.remaining_percent(), Some(70)); - assert!(!fact.exhausted()); } #[test] @@ -192,11 +189,9 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io() .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::allowed(SystemTime::now()), }) .await .expect("persist quota"); @@ -218,8 +213,7 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io() } #[tokio::test] -async fn roll_expired_windows_is_o1_for_extremely_old_reset() { - // 过去数年的 1 秒窗口:O(1) 完成,不能线性循环。 +async fn expired_window_remains_the_last_observation_until_refresh() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_o1_roll").await; let account = store.account("acct_o1_roll").expect("account"); @@ -240,11 +234,9 @@ async fn roll_expired_windows_is_o1_for_extremely_old_reset() { .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::observed_unknown(SystemTime::now()), }) .await .expect("persist quota"); @@ -254,20 +246,18 @@ async fn roll_expired_windows_is_o1_for_extremely_old_reset() { .await .expect("read quota") .expect("quota snapshot"); - let rolled = snapshot.roll_expired_windows(SystemTime::now()); - let window = &rolled.windows()[0]; - assert_eq!(window.used_percent(), Some(0.0)); - assert!(!window.limit_reached()); - // 下一 reset 必须落在 now 之后(O(1) 公式而非线性累加)。 + let window = &snapshot.windows()[0]; + assert_eq!(window.used_percent(), Some(100.0)); assert!( window .reset_at() - .is_some_and(|reset| SystemTime::from(reset) > SystemTime::now()) + .is_some_and(|reset| { SystemTime::from(reset) < SystemTime::now() }) ); + assert_eq!(service.scheduling_signals(&account), None); } #[tokio::test] -async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() { +async fn persisted_access_fact_survives_limit_without_percent_or_reset() { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_signal_only_limit").await; let account = store.account("acct_signal_only_limit").expect("account"); @@ -282,11 +272,13 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() { .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::exhausted( + gateway_core::engine::credential::QuotaEvidence::ProviderDenied, + SystemTime::now(), + None, + ), }) .await .expect("persist quota"); @@ -296,12 +288,7 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() { .await .expect("read quota") .expect("quota snapshot"); - assert!(snapshot.fact().exhausted()); - let signals = service.scheduling_signals(&account); - assert!( - signals.is_some_and(|signals| signals.limit_reached()), - "limit_reached without percent/reset must still produce scheduling exclusion" - ); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted); } #[tokio::test] @@ -334,11 +321,9 @@ async fn persisted_codex_alias_keeps_the_top_level_rate_limit_canonical() { .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::observed_unknown(SystemTime::now()), }) .await .expect("persist quota"); @@ -393,11 +378,9 @@ async fn persisted_quota_orders_core_window_before_additional_limit() { .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::observed_unknown(SystemTime::now()), }) .await .expect("persist quota"); @@ -459,11 +442,9 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() { .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( - raw.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::observed_unknown(SystemTime::now()), }) .await .expect("persist quota"); @@ -490,5 +471,5 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() { })); assert_ne!(review[0].key(), review[1].key()); assert_eq!(snapshot.windows().len(), 3); - assert!(!snapshot.fact().exhausted()); + assert_eq!(snapshot.quota().access(), QuotaAccessState::Unknown); } diff --git a/backend/crates/providers/openai/tests/provider/contract.rs b/backend/crates/providers/openai/tests/provider/contract.rs index 608bac059..c5591e06f 100644 --- a/backend/crates/providers/openai/tests/provider/contract.rs +++ b/backend/crates/providers/openai/tests/provider/contract.rs @@ -7,8 +7,9 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use chrono::Utc; use futures::{SinkExt, StreamExt}; use gateway_core::engine::credential::{ - AccountAvailability, AccountFeedbackStats, OpaqueProviderData, ProviderAccountId, - ProviderAccountStore as _, QuotaObservation, + AccountFeedbackStats, CredentialState, OpaqueProviderData, ProviderAccountId, + ProviderAccountStore as _, QuotaAccessChange, QuotaAccessState, QuotaEvidence, + QuotaObservation, QuotaState, }; use gateway_core::engine::provider::{Provider as _, ProviderRequest}; use gateway_core::engine::{ @@ -166,7 +167,6 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases( Arc::clone(&agent_identity), Arc::clone(&account_feedback), CodexCookiePolicy::official().expect("cookie policy"), - true, )); let provider = CodexProvider::new( @@ -1414,25 +1414,26 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc vec!["response.created", "response.failed"] ); let account = store.account(account_id).expect("rate-limited account"); - // 429 只记录限流窗口,不改变 availability(限流不改变账号可用性)。 - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); + assert_eq!(account.quota().access(), QuotaAccessState::Unknown); let _ = release.send(()); server.abort(); let _ = server.await; } #[tokio::test] -async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset_advances() { +async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() { let store = Arc::new(MemoryAccountStore::default()); let account_id = "acct_usage_limit_request_path"; let reset_at = 1_900_000_000; create_account(&store, account_id).await; let account = store.account(account_id).expect("created account"); + let observed_at = SystemTime::now(); store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( + quota: OpaqueProviderData::new( json!({ "rate_limit": { "allowed": true, @@ -1444,9 +1445,9 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset .as_object() .expect("stale quota object") .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + ), + observed_at, + state: QuotaState::allowed(observed_at), }) .await .expect("seed stale passive quota"); @@ -1505,7 +1506,11 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted); let account = store.account(account_id).expect("usage-limit account"); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); + assert_eq!(account.quota().access(), QuotaAccessState::Exhausted); + assert_eq!( + account.quota().evidence(), + Some(QuotaEvidence::UsageLimitReached) + ); let projected = store .get_quotas(&[account.id().clone()]) .await @@ -1513,22 +1518,20 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset .into_iter() .next() .expect("confirmed quota observation"); - let projected_observed_at = projected.observed_at.expect("projection observed at"); - let projected = Value::Object( - projected - .quota - .map(OpaqueProviderData::into_inner) - .expect("confirmed quota projection"), - ); + let projected_observed_at = projected.observed_at; + let projected = Value::Object(projected.quota.into_inner()); assert_eq!( projected .pointer("/rate_limit/primary_window/used_percent") .and_then(Value::as_u64), - Some(100) + Some(99) ); - assert!( - projected.pointer("/rate_limit/secondary_window").is_none(), - "request failure must not manufacture a secondary quota" + assert_eq!( + projected + .pointer("/rate_limit/secondary_window/used_percent") + .and_then(Value::as_u64), + Some(0), + "request failure must not rewrite the raw display document" ); tokio::time::sleep(Duration::from_millis(200)).await; let requests = server.received_requests().await.expect("received requests"); @@ -1549,7 +1552,7 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset .expect("read refreshed quota observation") .into_iter() .next() - .and_then(|observation| observation.observed_at); + .map(|observation| observation.observed_at); if observed_at.is_some_and(|observed_at| observed_at > projected_observed_at) { break; } @@ -1564,15 +1567,14 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset .expect("read refreshed quota") .into_iter() .next() - .and_then(|observation| observation.quota) - .map(OpaqueProviderData::into_inner) + .map(|observation| observation.quota.into_inner()) .map(Value::Object) .expect("authoritative quota projection"); assert_eq!( quota .pointer("/rate_limit/primary_window/used_percent") .and_then(Value::as_u64), - Some(100) + Some(99) ); assert_eq!( quota @@ -1582,7 +1584,7 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset ); assert!( quota.pointer("/rate_limit/secondary_window").is_none(), - "background quota refresh must retain the same window projection" + "background refresh may normalize empty display windows" ); let requests = server.received_requests().await.expect("received requests"); assert_eq!( @@ -1596,8 +1598,9 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset store .account(account_id) .expect("account after stale full usage refresh") - .availability(), - AccountAvailability::QuotaExhausted, + .quota() + .access(), + QuotaAccessState::Exhausted, "a contradictory full usage snapshot must not unlock a confirmed exhausted account" ); } @@ -1976,12 +1979,11 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st create_account(&store, account_id).await; let account = store.account(account_id).expect("test account"); store - .repository() - .apply_state( - &account, - AccountAvailability::QuotaExhausted, - SystemTime::now(), - ) + .apply_quota_access(QuotaAccessChange { + account_id: account.id().clone(), + expected_revision: account.revision(), + state: QuotaState::exhausted(QuotaEvidence::UsageLimitReached, SystemTime::now(), None), + }) .await .expect("seed quota-exhausted state"); store @@ -2033,7 +2035,7 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st .account(account_id) .expect("disabled test account after test"); assert!(!account.enabled()); - assert_eq!(account.availability(), AccountAvailability::QuotaExhausted); + assert_eq!(account.quota().access(), QuotaAccessState::Exhausted); assert!(!store.has_quota(account_id)); assert_eq!(affinity.binding_count(), 0); } @@ -2051,18 +2053,19 @@ async fn quota_limited_account_diagnostic_uses_upstream() { "primary_window": {"used_percent": 100, "reset_at": 1_900_000_000} } }); + let observed_at = SystemTime::now(); store .compare_and_swap_quota(QuotaObservation { account_id: account.id().clone(), expected_revision: account.revision(), - quota: Some(OpaqueProviderData::new( + quota: OpaqueProviderData::new( raw_quota .as_object() .expect("quota snapshot object") .clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: Some(true), + ), + observed_at, + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None), }) .await .expect("seed quota-limited snapshot"); @@ -2118,7 +2121,7 @@ async fn quota_limited_account_diagnostic_uses_upstream() { } #[tokio::test] -async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() { +async fn successful_response_treats_inference_success_as_authoritative_allowance() { let store = Arc::new(MemoryAccountStore::default()); let account_id = "acct_success_exhausted"; create_account(&store, account_id).await; @@ -2164,8 +2167,9 @@ async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers( store .account(account_id) .expect("account after successful response") - .availability(), - AccountAvailability::Ready + .quota() + .access(), + QuotaAccessState::Allowed ); assert!(store.has_quota(account_id)); } @@ -2226,10 +2230,9 @@ async fn successful_http_sse_rate_limit_event_persists_structured_exhaustion() { let account = store .account(account_id) .expect("account after HTTP SSE response"); - assert_eq!( - (account.availability(), store.has_quota(account_id),), - (AccountAvailability::Ready, true,) - ); + assert_eq!(account.credential_state(), CredentialState::Ready); + assert_eq!(account.quota().access(), QuotaAccessState::Allowed); + assert!(store.has_quota(account_id)); } #[test] diff --git a/backend/crates/providers/openai/tests/support.rs b/backend/crates/providers/openai/tests/support.rs index bfd763be0..879120f46 100644 --- a/backend/crates/providers/openai/tests/support.rs +++ b/backend/crates/providers/openai/tests/support.rs @@ -9,10 +9,10 @@ use std::time::{Duration, SystemTime}; use async_trait::async_trait; use futures::future::BoxFuture; use gateway_core::engine::credential::{ - AccountAvailability, AccountRuntimeSignals, AccountStateChange, CredentialCasOutcome, - CredentialCasUpdate, CredentialRevision, LoadedCredential, NewProviderAccount, + AccountErrorReason, AccountRuntimeSignals, AccountStateChange, CredentialCasOutcome, + CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount, OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore, - ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome, + ProviderAccountUpdate, QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome, }; use gateway_core::error::{StoreError, StoreErrorKind}; use gateway_core::provider_ports::{ @@ -38,7 +38,6 @@ struct StoredAccount { credential: gateway_core::engine::credential::PlaintextCredential, quota: Option, state_observed_at: Option, - last_error_message: Option, } #[derive(Default)] @@ -71,15 +70,6 @@ impl MemoryAccountStore { .map(|stored| stored.account.clone()) } - pub(crate) fn last_error_message(&self, id: &str) -> Option { - let id = ProviderAccountId::new(id).ok()?; - self.accounts - .lock() - .expect("account store lock") - .get(&id) - .and_then(|stored| stored.last_error_message.clone()) - } - pub(crate) fn quota_reads(&self) -> usize { self.quota_reads.load(Ordering::SeqCst) } @@ -102,8 +92,9 @@ impl MemoryAccountStore { .expect("account store lock") .get(&id) .and_then(|stored| stored.quota.as_ref()) - .and_then(|observation| observation.quota.as_ref()) - .map(|data| serde_json::Value::Object(data.expose_to_provider().clone())) + .map(|observation| { + serde_json::Value::Object(observation.quota.expose_to_provider().clone()) + }) } pub(crate) fn fail_provider_listing(&self) { @@ -125,7 +116,6 @@ impl ProviderAccountStore for MemoryAccountStore { credential: input.credential, quota: None, state_observed_at: None, - last_error_message: None, }, ); Ok(()) @@ -224,7 +214,10 @@ impl ProviderAccountStore for MemoryAccountStore { AccountRebuild { revision: next, enabled: stored.account.enabled(), - availability: stored.account.availability(), + credential_state: stored.account.credential_state(), + quota: stored.account.quota(), + last_error_reason: stored.account.last_error_reason(), + last_error_message: stored.account.last_error_message().map(str::to_owned), access_token_expires_at, has_refresh_token, next_refresh_at, @@ -262,13 +255,37 @@ impl ProviderAccountStore for MemoryAccountStore { if stored .quota .as_ref() - .and_then(|quota| quota.observed_at) - .zip(observation.observed_at) - .is_some_and(|(current, incoming)| current > incoming) + .is_some_and(|quota| quota.observed_at > observation.observed_at) { return Ok(QuotaWriteOutcome::Conflict); } + let quota = observation.state; stored.quota = Some(observation); + stored.account = rebuild_account( + &stored.account, + AccountRebuild::preserving(&stored.account).with_quota(quota), + ); + Ok(QuotaWriteOutcome::Updated) + } + + async fn apply_quota_access( + &self, + change: QuotaAccessChange, + ) -> Result { + let mut accounts = self.accounts.lock().expect("account store lock"); + let stored = accounts + .get_mut(&change.account_id) + .ok_or_else(|| store_error(StoreErrorKind::InvalidData))?; + if stored.account.revision() != change.expected_revision { + return Ok(QuotaWriteOutcome::Conflict); + } + stored.account = rebuild_account( + &stored.account, + AccountRebuild::preserving(&stored.account).with_quota(change.state), + ); + if let Some(observation) = &mut stored.quota { + observation.state = change.state; + } Ok(QuotaWriteOutcome::Updated) } @@ -291,7 +308,10 @@ impl ProviderAccountStore for MemoryAccountStore { AccountRebuild { revision: stored.account.revision(), enabled: stored.account.enabled(), - availability: change.availability, + credential_state: change.credential_state, + quota: stored.account.quota(), + last_error_reason: change.error_reason, + last_error_message: change.message, access_token_expires_at: stored.account.access_token_expires_at(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -299,11 +319,6 @@ impl ProviderAccountStore for MemoryAccountStore { }, ); stored.state_observed_at = Some(change.observed_at); - if change.availability == AccountAvailability::Ready { - stored.last_error_message = None; - } else if let Some(message) = change.message { - stored.last_error_message = Some(message); - } Ok(()) } @@ -317,7 +332,10 @@ impl ProviderAccountStore for MemoryAccountStore { AccountRebuild { revision: stored.account.revision(), enabled: stored.account.enabled(), - availability: stored.account.availability(), + credential_state: stored.account.credential_state(), + quota: stored.account.quota(), + last_error_reason: stored.account.last_error_reason(), + last_error_message: stored.account.last_error_message().map(str::to_owned), access_token_expires_at: stored.account.access_token_expires_at(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -341,7 +359,10 @@ impl ProviderAccountStore for MemoryAccountStore { AccountRebuild { revision: stored.account.revision(), enabled, - availability: stored.account.availability(), + credential_state: stored.account.credential_state(), + quota: stored.account.quota(), + last_error_reason: stored.account.last_error_reason(), + last_error_message: stored.account.last_error_message().map(str::to_owned), access_token_expires_at: stored.account.access_token_expires_at(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -368,13 +389,38 @@ const fn store_error(kind: StoreErrorKind) -> StoreError { struct AccountRebuild { revision: CredentialRevision, enabled: bool, - availability: AccountAvailability, + credential_state: CredentialState, + quota: QuotaState, + last_error_reason: Option, + last_error_message: Option, access_token_expires_at: Option, has_refresh_token: bool, next_refresh_at: Option, profile: Option<(String, Option, Option)>, } +impl AccountRebuild { + fn preserving(account: &ProviderAccount) -> Self { + Self { + revision: account.revision(), + enabled: account.enabled(), + credential_state: account.credential_state(), + quota: account.quota(), + last_error_reason: account.last_error_reason(), + last_error_message: account.last_error_message().map(str::to_owned), + access_token_expires_at: account.access_token_expires_at(), + has_refresh_token: account.has_refresh_token(), + next_refresh_at: account.next_refresh_at(), + profile: None, + } + } + + fn with_quota(mut self, quota: QuotaState) -> Self { + self.quota = quota; + self + } +} + fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> ProviderAccount { let (name, email, plan_type) = rebuild.profile.unwrap_or_else(|| { ( @@ -397,7 +443,13 @@ fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> Provid current.upstream_account_id().map(str::to_owned), plan_type, ) - .with_runtime_state(rebuild.enabled, rebuild.availability) + .with_account_facts( + rebuild.enabled, + rebuild.credential_state, + rebuild.quota, + rebuild.last_error_reason, + rebuild.last_error_message, + ) .with_refresh_schedule(rebuild.has_refresh_token, rebuild.next_refresh_at) } @@ -450,7 +502,7 @@ impl ProviderLeasePort for TestLeaseCoordinator { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/providers/xai/src/admin.rs b/backend/crates/providers/xai/src/admin.rs index f1b1db017..f70f0d59e 100644 --- a/backend/crates/providers/xai/src/admin.rs +++ b/backend/crates/providers/xai/src/admin.rs @@ -660,19 +660,21 @@ impl ProviderAdmin for XaiAdminProvider { refresh, rolling_usage, } = request; - self.account(&account_id).await?; + let mut account = self.account(&account_id).await?; let lifecycle = self .repository .read_lifecycle(&account_id) .await .map_err(map_repository_error)?; let snapshot = if refresh { - Some( + let snapshot = Some( self.quota .refresh_account(&account_id) .await .map_err(map_quota_error)?, - ) + ); + account = self.account(&account_id).await?; + snapshot } else { self.quota .read_account(&account_id) @@ -681,6 +683,7 @@ impl ProviderAdmin for XaiAdminProvider { }; Ok(project_quota( snapshot, + account.quota().is_exhausted(), lifecycle.refresh_token_expires_at().copied(), rolling_usage, )) @@ -891,8 +894,8 @@ fn prepared_create( access_token_expires_at: account.access_token_expires_at().map(DateTime::::from), next_refresh_at: account.next_refresh_at().map(Into::into), enabled: account.enabled(), - availability: account.availability(), - availability_observed_at: observed_at, + credential_state: account.credential_state(), + credential_observed_at: observed_at, }) } @@ -995,7 +998,13 @@ fn account_from_record(account: &AccountRecord) -> Result fn project_quota( snapshot: Option, + quota_exhausted: bool, refresh_token_expires_at: Option>, rolling_usage: Option, ) -> ProviderQuota { @@ -1020,8 +1030,7 @@ fn project_quota( observed_at: None, refresh_token_expires_at, windows: Vec::new(), - limit_reached: false, - rate_limited_until: None, + limit_reached: quota_exhausted, provider_data: None, }; }; @@ -1088,10 +1097,7 @@ fn project_quota( observed_at: Some(snapshot.observed_at()), refresh_token_expires_at, windows: vec![window], - limit_reached: billing - .used_percent() - .is_some_and(|used| used.is_finite() && used >= 100.0), - rate_limited_until: None, + limit_reached: quota_exhausted, provider_data: None, } } diff --git a/backend/crates/providers/xai/src/credential/catalog.rs b/backend/crates/providers/xai/src/credential/catalog.rs index 803f270d0..bc1b93452 100644 --- a/backend/crates/providers/xai/src/credential/catalog.rs +++ b/backend/crates/providers/xai/src/credential/catalog.rs @@ -8,8 +8,8 @@ use std::time::{Duration, Instant, SystemTime}; use async_trait::async_trait; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData, - ProviderAccount, ProviderAccountId, QuotaObservation, + AccountQuotaSignals, CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount, + ProviderAccountId, QuotaObservation, QuotaState, }; use gateway_core::engine::provider::ProviderCatalogGeneration; use gateway_core::provider_ports::{ @@ -19,7 +19,6 @@ use gateway_core::routing::ProviderKind; use tokio::sync::Mutex; use super::repository::{GrokCredentialRepository, LoadedGrokCredential}; -use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState}; use crate::XaiWireProfileState; use crate::transport::catalog::{MAX_CATALOG_MODELS, valid_model_slug, validate_etag}; use crate::{ @@ -445,15 +444,17 @@ impl GrokCredentialQuotaService { let observed_at = Utc::now(); let document = billing.into_document(); let presentation = billing_presentation(&document)?; + let quota = loaded + .account + .quota() + .merge_observation(quota_access_observation(&presentation, observed_at.into())); self.repository .replace_quota( loaded.account.id().clone(), loaded.account.revision(), document, observed_at.into(), - presentation - .used_percent() - .is_some_and(|used| used.is_finite() && used >= 100.0), + quota, ) .await .map_err(map_quota_repository_error)?; @@ -464,32 +465,6 @@ impl GrokCredentialQuotaService { billing: presentation, }; self.scheduling.observe(&snapshot); - let current = self - .repository - .load_current(account_id) - .await - .map_err(map_quota_repository_error)?; - if quota_refresh_may_update_state(&loaded.account, ¤t.account) - && let Some(availability) = quota_refresh_availability( - current.account.availability(), - authoritative_quota_exhaustion(snapshot.billing()), - ) - { - self.repository - .update_state(&UpdateGrokCredentialState { - account_id: current.account.id().clone(), - expected_revision: current.account.revision(), - availability, - availability_reason: matches!( - availability, - GrokCredentialAvailability::QuotaExhausted - ) - .then_some("quota_exhausted".to_owned()), - observed_at, - }) - .await - .map_err(map_quota_repository_error)?; - } Ok(snapshot) } @@ -506,10 +481,8 @@ impl GrokCredentialQuotaService { else { return Ok(None); }; - let (Some(quota), Some(observed_at)) = (observation.quota, observation.observed_at) else { - return Err(GrokQuotaError::InvalidData); - }; - let document = quota.into_inner(); + let observed_at = observation.observed_at; + let document = observation.quota.into_inner(); let snapshot = GrokQuotaSnapshot { account_id: observation.account_id, credential_revision: observation.expected_revision, @@ -1150,14 +1123,10 @@ fn eligible_catalog_candidate(candidate: &LoadedGrokCredential) -> bool { && candidate .refresh_token_expires_at .is_none_or(|expires_at| expires_at > Utc::now()) - && match account.availability() { - AccountAvailability::Unknown - | AccountAvailability::Ready - | AccountAvailability::QuotaExhausted => true, - AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid => false, - } + && matches!( + account.credential_state(), + CredentialState::Unknown | CredentialState::Ready + ) } async fn fetch_candidate_catalog( @@ -1273,8 +1242,8 @@ fn billing_presentation( } fn quota_snapshot_from_observation(observation: &QuotaObservation) -> Option { - let observed_at = DateTime::::from(observation.observed_at?); - let document = observation.quota.as_ref()?.expose_to_provider(); + let observed_at = DateTime::::from(observation.observed_at); + let document = observation.quota.expose_to_provider(); Some(GrokQuotaSnapshot { account_id: observation.account_id.clone(), credential_revision: observation.expected_revision, @@ -1293,29 +1262,54 @@ fn quota_projection_ttl(observed_at: DateTime) -> Option { } fn quota_scheduling_signals(billing: &GrokBillingPresentation) -> Option { - let remaining_rank = billing - .used_percent() - .filter(|used| used.is_finite() && (0.0..=100.0).contains(used)) - .map(|used| ((100.0 - used) * 100.0).round() as u64); let now = SystemTime::now(); - let reset_at = billing + let period_end = billing + .period_end() + .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) + .map(|value| SystemTime::from(value.to_utc())); + let observation_expired = period_end.is_some_and(|period_end| period_end <= now); + let remaining_rank = (!observation_expired) + .then(|| { + billing + .used_percent() + .filter(|used| used.is_finite() && (0.0..=100.0).contains(used)) + .map(|used| ((100.0 - used) * 100.0).round() as u64) + }) + .flatten(); + let reset_at = period_end.filter(|reset_at| *reset_at > now); + (remaining_rank.is_some() || reset_at.is_some()) + .then(|| AccountQuotaSignals::new(reset_at, remaining_rank)) +} + +/// 把 xAI billing 文档归一化为访问结论。缺少正反证据时返回 Unknown, +/// 由 `QuotaState::merge_observation` 保留既有已确认结论。 +fn quota_access_observation( + billing: &GrokBillingPresentation, + observed_at: SystemTime, +) -> QuotaState { + if !billing.has_authoritative_quota() { + return QuotaState::observed_unknown(observed_at); + } + if billing_proves_access_allowed(billing, observed_at) { + QuotaState::allowed(observed_at) + } else { + QuotaState::observed_unknown(observed_at) + } +} + +fn billing_proves_access_allowed( + billing: &GrokBillingPresentation, + observed_at: SystemTime, +) -> bool { + if billing .period_end() .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) .map(|value| SystemTime::from(value.to_utc())) - .filter(|reset_at| *reset_at > now); - (remaining_rank.is_some() || reset_at.is_some()).then(|| { - AccountQuotaSignals::new( - reset_at, - remaining_rank, - billing - .used_percent() - .is_some_and(|used| used.is_finite() && used >= 100.0), - ) - }) -} - -fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool { - if billing + .is_some_and(|period_end| period_end <= observed_at) + { + return false; + } + billing .prepaid_balance_cents() .is_some_and(|balance| balance > 0) || billing @@ -1329,60 +1323,6 @@ fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool { (billing.included_used_cents(), billing.monthly_limit_cents()), (Some(used), Some(limit)) if limit > 0 && used < limit ) - { - return false; - } - billing - .used_percent() - .is_some_and(|used| used.is_finite() && used >= 100.0) - || matches!( - (billing.included_used_cents(), billing.monthly_limit_cents()), - (Some(used), Some(limit)) if limit > 0 && used >= limit - ) - || matches!( - (billing.on_demand_used_cents(), billing.on_demand_cap_cents()), - (Some(used), Some(cap)) if cap > 0 && used >= cap - ) -} - -/// `None` means the billing document cannot prove either quota state. -fn authoritative_quota_exhaustion(billing: &GrokBillingPresentation) -> Option { - billing - .has_authoritative_quota() - .then(|| quota_is_exhausted(billing)) -} - -fn quota_refresh_may_update_state(before: &ProviderAccount, current: &ProviderAccount) -> bool { - current.enabled() - && current - .access_token_expires_at() - .is_some_and(|expires_at| expires_at > SystemTime::now()) - && current.revision() == before.revision() - && current.availability() == before.availability() -} - -fn quota_refresh_availability( - current: AccountAvailability, - exhausted: Option, -) -> Option { - match current { - AccountAvailability::Invalid - | AccountAvailability::Expired - | AccountAvailability::Banned => None, - AccountAvailability::QuotaExhausted => { - matches!(exhausted, Some(false)).then_some(GrokCredentialAvailability::Ready) - } - AccountAvailability::Unknown => exhausted.map(|exhausted| { - if exhausted { - GrokCredentialAvailability::QuotaExhausted - } else { - GrokCredentialAvailability::Ready - } - }), - AccountAvailability::Ready => { - matches!(exhausted, Some(true)).then_some(GrokCredentialAvailability::QuotaExhausted) - } - } } fn dynamic_string( diff --git a/backend/crates/providers/xai/src/credential/mod.rs b/backend/crates/providers/xai/src/credential/mod.rs index f2344712e..5a6e7d0ff 100644 --- a/backend/crates/providers/xai/src/credential/mod.rs +++ b/backend/crates/providers/xai/src/credential/mod.rs @@ -63,10 +63,9 @@ pub use token::{ parse_refresh_success, }; pub use types::{ - CreateGrokCredential, GrokAccountProfile, GrokCredentialAvailability, GrokCredentialRecord, - GrokOAuthSecret, PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, - RotateGrokCredential, RotateManagedGrokCredential, UpdateGrokCredentialState, - XAI_AUTHENTICATION_KIND_OAUTH, + CreateGrokCredential, GrokAccountProfile, GrokCredentialRecord, GrokOAuthSecret, + PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, RotateGrokCredential, + RotateManagedGrokCredential, UpdateGrokCredentialState, XAI_AUTHENTICATION_KIND_OAUTH, }; pub use verification::{ FailClosedTokenVerifier, TokenCandidate, TokenVerificationContext, TokenVerifier, diff --git a/backend/crates/providers/xai/src/credential/refresh.rs b/backend/crates/providers/xai/src/credential/refresh.rs index 239d91799..1704ff098 100644 --- a/backend/crates/providers/xai/src/credential/refresh.rs +++ b/backend/crates/providers/xai/src/credential/refresh.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::{StreamExt as _, stream}; use gateway_core::engine::credential::{ - AccountAvailability, CredentialRevision, LoadedCredential, ProviderAccountId, + AccountErrorReason, CredentialRevision, CredentialState, LoadedCredential, ProviderAccountId, }; use gateway_core::provider_ports::{ ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest, @@ -23,9 +23,8 @@ use super::repository::{ GrokCredentialAdmin, GrokCredentialRepository, GrokCredentialRepositoryError, }; use super::types::{ - GrokAccountProfile, GrokCredentialAvailability, GrokOAuthSecret, - PreparedGrokCredentialRotation, RotateGrokCredential, RotateManagedGrokCredential, - UpdateGrokCredentialState, + GrokAccountProfile, GrokOAuthSecret, PreparedGrokCredentialRotation, RotateGrokCredential, + RotateManagedGrokCredential, UpdateGrokCredentialState, }; use crate::{ DiscoveryDocument, FailureClass, GrokOAuthClient, OAuthError, RefreshTokenGrant, @@ -540,7 +539,8 @@ impl GrokCredentialRefreshService { return self .persist_terminal_failure( credential, - GrokCredentialAvailability::Expired, + CredentialState::Expired, + AccountErrorReason::CredentialExpired, "refresh_token_expired", ) .await; @@ -549,7 +549,8 @@ impl GrokCredentialRefreshService { return self .persist_terminal_failure( credential, - GrokCredentialAvailability::Expired, + CredentialState::Expired, + AccountErrorReason::CredentialExpired, "refresh_recovery_window_exhausted", ) .await; @@ -587,7 +588,8 @@ impl GrokCredentialRefreshService { Err(GrokRefreshFailure::InvalidGrant) => { self.persist_terminal_failure( credential, - GrokCredentialAvailability::Expired, + CredentialState::Expired, + AccountErrorReason::CredentialExpired, "refresh_invalid_grant", ) .await @@ -595,7 +597,8 @@ impl GrokCredentialRefreshService { Err(GrokRefreshFailure::Banned) => { self.persist_terminal_failure( credential, - GrokCredentialAvailability::Banned, + CredentialState::Banned, + AccountErrorReason::AccountBanned, "account_banned", ) .await @@ -717,7 +720,8 @@ impl GrokCredentialRefreshService { async fn persist_terminal_failure( &self, credential: DueGrokCredential, - availability: GrokCredentialAvailability, + credential_state: CredentialState, + error_reason: AccountErrorReason, reason: &str, ) -> Result { let account_id = credential.account_id.clone(); @@ -737,8 +741,9 @@ impl GrokCredentialRefreshService { .update_state(&UpdateGrokCredentialState { account_id: account_id.clone(), expected_revision: credential.credential_revision, - availability, - availability_reason: Some(reason.to_owned()), + credential_state, + error_reason: Some(error_reason), + error_message: Some(reason.to_owned()), observed_at: Utc::now(), }) .await @@ -746,7 +751,7 @@ impl GrokCredentialRefreshService { Ok(()) => { tracing::warn!( account_id = %account_id, - ?availability, + ?credential_state, reason, access_token_expires_at = ?credential.access_token_expires_at .map(DateTime::::from), @@ -915,14 +920,11 @@ fn account_due( policy: ProviderRefreshPolicy, now: SystemTime, ) -> bool { - let availability = account.availability(); account.enabled() && account.has_refresh_token() && !matches!( - availability, - AccountAvailability::Expired - | AccountAvailability::Banned - | AccountAvailability::Invalid + account.credential_state(), + CredentialState::Expired | CredentialState::Banned | CredentialState::Invalid ) && (refresh_recovery_window_exhausted(account.access_token_expires_at(), now) || (access_token_refresh_due(account.access_token_expires_at(), policy, now) diff --git a/backend/crates/providers/xai/src/credential/repository.rs b/backend/crates/providers/xai/src/credential/repository.rs index afc9e8c74..a66e90a30 100644 --- a/backend/crates/providers/xai/src/credential/repository.rs +++ b/backend/crates/providers/xai/src/credential/repository.rs @@ -6,10 +6,10 @@ use std::time::SystemTime; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData, - PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore, - ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome, + AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision, + CredentialState, LoadedCredential, NewProviderAccount, OpaqueProviderData, PlaintextCredential, + ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, + QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome, }; use gateway_core::error::StoreErrorKind; use gateway_core::routing::ProviderKind; @@ -17,8 +17,8 @@ use serde_json::Value; use thiserror::Error; use super::types::{ - CreateGrokCredential, GrokAccountProfile, GrokCredentialAvailability, GrokCredentialRecord, - GrokOAuthSecret, GrokOAuthSecretWire, OwnedGrokOAuthSecretWire, PreparedGrokCredentialRotation, + CreateGrokCredential, GrokAccountProfile, GrokCredentialRecord, GrokOAuthSecret, + GrokOAuthSecretWire, OwnedGrokOAuthSecretWire, PreparedGrokCredentialRotation, RotateGrokCredential, RotateManagedGrokCredential, UpdateGrokCredentialState, XAI_AUTHENTICATION_KIND_OAUTH, }; @@ -167,7 +167,13 @@ impl GrokCredentialAdmin { input.account.upstream_account_id.clone(), input.account.plan_type.clone(), ) - .with_runtime_state(input.enabled, input.initial_availability.into()) + .with_account_facts( + input.enabled, + input.initial_credential_state, + QuotaState::unknown(), + input.initial_error_reason, + input.initial_error_message.clone(), + ) .with_refresh_schedule(true, None); Ok(NewProviderAccount { account, @@ -210,8 +216,9 @@ impl GrokCredentialAdmin { refresh_token_expires_at: None, }, enabled: input.enabled, - initial_availability: GrokCredentialAvailability::Ready, - initial_availability_reason: None, + initial_credential_state: CredentialState::Ready, + initial_error_reason: None, + initial_error_message: None, }) } @@ -420,21 +427,30 @@ impl GrokCredentialRepository { &self, input: &UpdateGrokCredentialState, ) -> Result<(), GrokCredentialRepositoryError> { - let availability = AccountAvailability::from(input.availability); - let availability_reason = input - .availability_reason + let error_message = input + .error_message .clone() - .filter(|value| !value.trim().is_empty()) - .or_else(|| availability.fallback_error_reason().map(str::to_owned)); - validate_reason(availability_reason.as_deref())?; + .filter(|value| !value.trim().is_empty()); + validate_reason(error_message.as_deref())?; + let (error_reason, error_message) = if input.credential_state == CredentialState::Ready { + (None, None) + } else { + ( + input + .error_reason + .or_else(|| input.credential_state.error_reason()), + error_message, + ) + }; self.ensure_xai_account(&input.account_id).await?; self.store .apply_state_change(AccountStateChange { account_id: input.account_id.clone(), expected_revision: input.expected_revision, - availability, + credential_state: input.credential_state, observed_at: to_system_time(input.observed_at), - message: availability_reason, + error_reason, + message: error_message, }) .await .map_err(map_store_error) @@ -518,16 +534,39 @@ impl GrokCredentialRepository { expected_revision: CredentialRevision, document: serde_json::Map, observed_at: SystemTime, - limit_reached: bool, + state: QuotaState, ) -> Result<(), GrokCredentialRepositoryError> { let outcome = self .store .compare_and_swap_quota(QuotaObservation { account_id, expected_revision, - quota: Some(OpaqueProviderData::new(document)), - observed_at: Some(observed_at), - limit_reached: Some(limit_reached), + quota: OpaqueProviderData::new(document), + observed_at, + state, + }) + .await + .map_err(map_store_error)?; + match outcome { + QuotaWriteOutcome::Updated => Ok(()), + QuotaWriteOutcome::Conflict => { + Err(GrokCredentialRepositoryError::StaleCredentialRevision) + } + } + } + + pub(crate) async fn update_quota_access( + &self, + account_id: ProviderAccountId, + expected_revision: CredentialRevision, + state: QuotaState, + ) -> Result<(), GrokCredentialRepositoryError> { + let outcome = self + .store + .apply_quota_access(QuotaAccessChange { + account_id, + expected_revision, + state, }) .await .map_err(map_store_error)?; @@ -698,7 +737,7 @@ fn validate_create(input: &CreateGrokCredential) -> Result<(), GrokCredentialRep validate_name(&input.name)?; validate_profile(&input.account)?; validate_secret(&input.secret)?; - validate_reason(input.initial_availability_reason.as_deref()) + validate_reason(input.initial_error_message.as_deref()) } fn validate_profile(profile: &GrokAccountProfile) -> Result<(), GrokCredentialRepositoryError> { @@ -786,9 +825,7 @@ fn validate_reason(reason: Option<&str>) -> Result<(), GrokCredentialRepositoryE if reason .is_some_and(|value| value.len() > MAX_REASON_BYTES || value.chars().any(char::is_control)) { - return Err(GrokCredentialRepositoryError::InvalidInput( - "availability_reason", - )); + return Err(GrokCredentialRepositoryError::InvalidInput("error_message")); } Ok(()) } diff --git a/backend/crates/providers/xai/src/credential/selector.rs b/backend/crates/providers/xai/src/credential/selector.rs index e6838a2aa..e9a4a0a88 100644 --- a/backend/crates/providers/xai/src/credential/selector.rs +++ b/backend/crates/providers/xai/src/credential/selector.rs @@ -6,8 +6,9 @@ use std::time::{Duration, SystemTime}; use chrono::Utc; use gateway_core::engine::credential::{ - AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats, - AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId, + AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats, AccountSelectionContext, + AccountSelector, CredentialState, ProviderAccount, ProviderAccountId, QuotaAccessState, + QuotaEvidence, QuotaState, }; use gateway_core::provider_ports::{ ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition, @@ -18,7 +19,7 @@ use gateway_core::routing::{ProviderKind, UpstreamModelId}; use super::catalog::{GrokCatalogScope, GrokCredentialCatalogCache, GrokCredentialQuotaService}; use super::repository::{GrokCredentialRepository, GrokCredentialRepositoryError}; -use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState}; +use super::types::UpdateGrokCredentialState; use crate::{ GrokCredentialFailure, GrokCredentialFeedbackFuture, GrokSessionBinding, GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError, GrokSessionSelectorFuture, SecretValue, @@ -82,7 +83,7 @@ impl GrokAccountSessionSelector { &self, request: GrokSessionSelection, ) -> Result { - let diagnostic = request.availability() == AccountAvailabilityPolicy::BypassForDiagnostic; + let diagnostic = request.eligibility() == AccountEligibilityPolicy::BypassForDiagnostic; let accounts = self .repository .list_accounts_for_provider() @@ -177,7 +178,7 @@ impl GrokAccountSessionSelector { excluded_accounts: request.excluded_accounts().clone(), preferred_account: request.required_account().cloned().or(affinity_account), round_robin_cursor: scheduling.round_robin_cursor(), - availability: request.availability(), + eligibility: request.eligibility(), }; let mut capacity_denied = false; let mut retry_after = None; @@ -363,23 +364,18 @@ impl GrokAccountSessionSelector { .await; } - async fn persist_account_failure( + async fn persist_quota_failure( &self, session: &SelectedGrokSession, - availability: GrokCredentialAvailability, - reason: &'static str, observed_at: chrono::DateTime, ) { - let update = |expected_revision| UpdateGrokCredentialState { - account_id: session.account_id().clone(), - expected_revision, - availability, - availability_reason: Some(reason.to_owned()), - observed_at, - }; match self .repository - .update_state(&update(session.credential_revision())) + .update_quota_access( + session.account_id().clone(), + session.credential_revision(), + QuotaState::exhausted(QuotaEvidence::UsageLimitReached, observed_at.into(), None), + ) .await { Ok(()) => return, @@ -410,7 +406,11 @@ impl GrokAccountSessionSelector { }; if let Err(error) = self .repository - .update_state(&update(current.account.revision())) + .update_quota_access( + session.account_id().clone(), + current.account.revision(), + QuotaState::exhausted(QuotaEvidence::UsageLimitReached, observed_at.into(), None), + ) .await { tracing::warn!( @@ -450,20 +450,15 @@ impl GrokSessionSelector for GrokAccountSessionSelector { return; } let observed_at = Utc::now(); - let persistent = match failure { + match failure { GrokCredentialFailure::Unauthorized => { self.record_runtime_cooldown(session, UNAUTHORIZED_COOLDOWN) .await; - return; } - GrokCredentialFailure::QuotaExhausted => ( - GrokCredentialAvailability::QuotaExhausted, - "upstream_quota_exhausted", - ), - GrokCredentialFailure::FreeQuotaExhausted => ( - GrokCredentialAvailability::QuotaExhausted, - "upstream_free_quota_exhausted", - ), + GrokCredentialFailure::QuotaExhausted + | GrokCredentialFailure::FreeQuotaExhausted => { + self.persist_quota_failure(session, observed_at).await; + } // bare 402 没有结构化 quota code,不能证明账号额度耗尽: // 只写短期账号级 runtime cooldown(TransientBackoff), // 不持久化 QuotaExhausted(结构化 QuotaExhausted/FreeQuotaExhausted 走上面)。 @@ -472,7 +467,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector { .unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN) .min(MAX_RATE_LIMIT_COOLDOWN); self.record_runtime_cooldown(session, retry_after).await; - return; } // xAI 免费模型额度按该模型滚动窗口恢复:限流写 model-scoped // runtime cooldown(Redis 跨重启保留),不进入持久化账号状态, @@ -485,7 +479,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector { bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN); self.record_model_runtime_cooldown(session, upstream_model, retry_after) .await; - return; } GrokCredentialFailure::ModelAccessDenied { upstream_model, @@ -498,24 +491,18 @@ impl GrokSessionSelector for GrokAccountSessionSelector { ); self.record_model_runtime_cooldown(session, upstream_model, retry_after) .await; - return; } GrokCredentialFailure::RateLimited { retry_after } => { let retry_after = retry_after .unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN) .min(MAX_RATE_LIMIT_COOLDOWN); self.record_runtime_cooldown(session, retry_after).await; - return; } GrokCredentialFailure::StreamInterrupted => { self.record_runtime_cooldown(session, STREAM_INTERRUPTION_COOLDOWN) .await; - return; } - }; - let (availability, reason) = persistent; - self.persist_account_failure(session, availability, reason, observed_at) - .await; + } }) } @@ -531,25 +518,22 @@ impl GrokSessionSelector for GrokAccountSessionSelector { return; }; let account = ¤t.account; - if account.revision() != session.credential_revision() - || !account.enabled() - || !matches!( - account.availability(), - AccountAvailability::Unknown | AccountAvailability::Expired - ) - { + if account.revision() != session.credential_revision() || !account.enabled() { return; } - if let Err(error) = self - .repository - .update_state(&UpdateGrokCredentialState { - account_id: account.id().clone(), - expected_revision: account.revision(), - availability: GrokCredentialAvailability::Ready, - availability_reason: None, - observed_at: Utc::now(), - }) - .await + let observed_at = Utc::now(); + if account.credential_state() != CredentialState::Ready + && let Err(error) = self + .repository + .update_state(&UpdateGrokCredentialState { + account_id: account.id().clone(), + expected_revision: account.revision(), + credential_state: CredentialState::Ready, + error_reason: None, + error_message: None, + observed_at, + }) + .await { tracing::warn!( account_id = %account.id(), @@ -557,6 +541,24 @@ impl GrokSessionSelector for GrokAccountSessionSelector { "xAI account state recovery after successful upstream response failed" ); } + // 已经发出的并发请求可能晚于耗尽事实成功返回;真实成功可以收敛 + // 该事实,但 selector 不会为恢复探测而放行耗尽账号。 + if account.quota().access() == QuotaAccessState::Exhausted + && let Err(error) = self + .repository + .update_quota_access( + account.id().clone(), + account.revision(), + QuotaState::allowed(observed_at.into()), + ) + .await + { + tracing::warn!( + account_id = %account.id(), + error = %error, + "xAI quota recovery after successful upstream response failed" + ); + } }) } } diff --git a/backend/crates/providers/xai/src/credential/types.rs b/backend/crates/providers/xai/src/credential/types.rs index e1eb9448e..af1cbf535 100644 --- a/backend/crates/providers/xai/src/credential/types.rs +++ b/backend/crates/providers/xai/src/credential/types.rs @@ -4,7 +4,7 @@ use std::fmt; use chrono::{DateTime, Utc}; use gateway_core::engine::credential::{ - AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential, + AccountErrorReason, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential, ProviderAccountId, ProviderAccountUpdate, }; use gateway_core::provider_ports::ProviderLeaseGuard; @@ -63,34 +63,6 @@ impl fmt::Debug for GrokAccountProfile { } } -/// 与 `provider_accounts.availability` 一一对应的 xAI 状态。 -/// -/// 限流(429/滚动窗口)不进入持久化状态:由 `ProviderCooldownPort`(Redis 跨重启 -/// 保留)驱动调度排除,账号状态保持 `Ready`。 -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GrokCredentialAvailability { - Unknown, - Ready, - QuotaExhausted, - Expired, - Banned, - Invalid, -} - -/// Grok 可用性语义翻入内核标准态的唯一汇入点。 -impl From for AccountAvailability { - fn from(value: GrokCredentialAvailability) -> Self { - match value { - GrokCredentialAvailability::Unknown => Self::Unknown, - GrokCredentialAvailability::Ready => Self::Ready, - GrokCredentialAvailability::QuotaExhausted => Self::QuotaExhausted, - GrokCredentialAvailability::Expired => Self::Expired, - GrokCredentialAvailability::Banned => Self::Banned, - GrokCredentialAvailability::Invalid => Self::Invalid, - } - } -} - /// 创建一个明文 OAuth Provider account。 pub struct CreateGrokCredential { pub account_id: ProviderAccountId, @@ -98,8 +70,9 @@ pub struct CreateGrokCredential { pub secret: GrokOAuthSecret, pub account: GrokAccountProfile, pub enabled: bool, - pub initial_availability: GrokCredentialAvailability, - pub initial_availability_reason: Option, + pub initial_credential_state: CredentialState, + pub initial_error_reason: Option, + pub initial_error_message: Option, } impl fmt::Debug for CreateGrokCredential { @@ -111,7 +84,7 @@ impl fmt::Debug for CreateGrokCredential { .field("secret", &"[REDACTED]") .field("account", &self.account) .field("enabled", &self.enabled) - .field("initial_availability", &self.initial_availability) + .field("initial_credential_state", &self.initial_credential_state) .finish_non_exhaustive() } } @@ -235,8 +208,9 @@ impl fmt::Debug for RotateGrokCredential { pub struct UpdateGrokCredentialState { pub account_id: ProviderAccountId, pub expected_revision: CredentialRevision, - pub availability: GrokCredentialAvailability, - pub availability_reason: Option, + pub credential_state: CredentialState, + pub error_reason: Option, + pub error_message: Option, pub observed_at: DateTime, } diff --git a/backend/crates/providers/xai/src/lib.rs b/backend/crates/providers/xai/src/lib.rs index 8247c3c56..fa5084907 100644 --- a/backend/crates/providers/xai/src/lib.rs +++ b/backend/crates/providers/xai/src/lib.rs @@ -30,25 +30,25 @@ pub use credential::{ FailClosedTokenVerifier, FailureClass, FormField, FormValue, GROK_FREE_ROLLING_WINDOW_SECONDS, GrokAccountExport, GrokAccountProfile, GrokAccountSessionSelector, GrokBillingPresentation, GrokCatalogCache, GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, - GrokCredentialAvailability, GrokCredentialCatalogCache, GrokCredentialCatalogError, - GrokCredentialCatalogSeed, GrokCredentialCatalogService, GrokCredentialLifecycle, - GrokCredentialQuotaService, GrokCredentialRecord, GrokCredentialRecovery, - GrokCredentialRecoveryOutcome, GrokCredentialRefreshError, GrokCredentialRefreshOutcome, - GrokCredentialRefreshService, GrokCredentialRefresher, GrokCredentialRepository, - GrokCredentialRepositoryError, GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, - GrokOAuthImportDocument, GrokOAuthImportEntry, GrokOAuthImportError, GrokOAuthImportMetadata, - GrokOAuthImportTokens, GrokOAuthRefreshClient, GrokOAuthSecret, GrokPlanCatalog, - GrokQuotaError, GrokQuotaPeriodKind, GrokQuotaSnapshot, GrokRefreshFailure, GrokRefreshTokens, - HttpHeader, HttpMethod, OAuthError, OAuthErrorCode, OAuthHttpRequest, OAuthHttpResponse, - OAuthHttpTransport, OAuthOperation, OAuthPrincipal, OFFICIAL_CLIENT_ID, OFFICIAL_ISSUER, - OFFICIAL_REDIRECT_URI, OFFICIAL_SCOPES, PendingAuthorization, Pkce, - PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, ProtocolViolation, - RedirectUriAllowlist, RefreshTokenGrant, RefreshedTokenSet, ReqwestOidcTokenVerifier, - RotateGrokCredential, RotateManagedGrokCredential, SecretValue, TokenCandidate, - TokenVerificationContext, TokenVerifier, TransportFailure, TransportFailureKind, - TransportFuture, UpdateGrokCredentialState, VerificationEvidence, VerificationFailure, - VerificationFlow, VerificationFuture, VerificationMethod, VerifiedGrokAccount, - VerifiedTokenSet, parse_oauth_error, parse_refresh_success, + GrokCredentialCatalogCache, GrokCredentialCatalogError, GrokCredentialCatalogSeed, + GrokCredentialCatalogService, GrokCredentialLifecycle, GrokCredentialQuotaService, + GrokCredentialRecord, GrokCredentialRecovery, GrokCredentialRecoveryOutcome, + GrokCredentialRefreshError, GrokCredentialRefreshOutcome, GrokCredentialRefreshService, + GrokCredentialRefresher, GrokCredentialRepository, GrokCredentialRepositoryError, + GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, GrokOAuthImportDocument, + GrokOAuthImportEntry, GrokOAuthImportError, GrokOAuthImportMetadata, GrokOAuthImportTokens, + GrokOAuthRefreshClient, GrokOAuthSecret, GrokPlanCatalog, GrokQuotaError, GrokQuotaPeriodKind, + GrokQuotaSnapshot, GrokRefreshFailure, GrokRefreshTokens, HttpHeader, HttpMethod, OAuthError, + OAuthErrorCode, OAuthHttpRequest, OAuthHttpResponse, OAuthHttpTransport, OAuthOperation, + OAuthPrincipal, OFFICIAL_CLIENT_ID, OFFICIAL_ISSUER, OFFICIAL_REDIRECT_URI, OFFICIAL_SCOPES, + PendingAuthorization, Pkce, PreparedGrokCredentialRotation, + PreparedGrokCredentialRotationGuard, ProtocolViolation, RedirectUriAllowlist, + RefreshTokenGrant, RefreshedTokenSet, ReqwestOidcTokenVerifier, RotateGrokCredential, + RotateManagedGrokCredential, SecretValue, TokenCandidate, TokenVerificationContext, + TokenVerifier, TransportFailure, TransportFailureKind, TransportFuture, + UpdateGrokCredentialState, VerificationEvidence, VerificationFailure, VerificationFlow, + VerificationFuture, VerificationMethod, VerifiedGrokAccount, VerifiedTokenSet, + parse_oauth_error, parse_refresh_success, }; pub use provider::GrokBuildProvider; pub use transport::profile::{ diff --git a/backend/crates/providers/xai/src/provider.rs b/backend/crates/providers/xai/src/provider.rs index fe9201eb9..edc11babf 100644 --- a/backend/crates/providers/xai/src/provider.rs +++ b/backend/crates/providers/xai/src/provider.rs @@ -8,8 +8,8 @@ use async_trait::async_trait; use futures::{StreamExt, future::BoxFuture}; use gateway_core::engine::continuation::ContinuationBinding; use gateway_core::engine::credential::{ - AccountAvailability, AccountAvailabilityPolicy, AccountFeedbackStats, ProviderAccount, - ProviderAccountId, ProviderAccountStore, + AccountEligibilityPolicy, AccountFeedbackStats, ProviderAccount, ProviderAccountId, + ProviderAccountStore, }; use gateway_core::engine::provider::{ EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration, @@ -392,10 +392,10 @@ async fn select_grok_session( context.account_selection_policy(), context.deadline(), ) - .with_availability_policy(if context.is_diagnostic_required_account() { - AccountAvailabilityPolicy::BypassForDiagnostic + .with_eligibility_policy(if context.is_diagnostic_required_account() { + AccountEligibilityPolicy::BypassForDiagnostic } else { - AccountAvailabilityPolicy::Enforce + AccountEligibilityPolicy::Enforce }) .with_affinity(affinity); let selection_deadline = remaining(context.deadline()) @@ -1981,7 +1981,8 @@ const OAUTH_REFRESH_INTERVAL: Duration = Duration::from_secs(30); const QUOTA_CATALOG_INTERVAL: Duration = Duration::from_secs(5 * 60); // grok2api 对 Free/未知额度耗尽采用 24 小时恢复探测;当前公共账号状态没有 // 独立的 paid period-end 字段,因此 Build 账号统一使用同一保守下限。 -const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); +const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); +const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = QUOTA_CATALOG_INTERVAL; const CLI_RELEASE_WORKER_OWNER: &str = "xai-cli-release"; pub(crate) fn worker_contributions( @@ -2175,7 +2176,8 @@ impl XaiQuotaCatalogTask { let due = last_periodic_refresh_at .get(account.id()) .is_none_or(|last| { - now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL + now.saturating_duration_since(*last) + >= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL }); if due { last_periodic_refresh_at.insert(account.id().clone(), now); @@ -2191,5 +2193,7 @@ fn eligible_quota_worker_account(account: &ProviderAccount, now: SystemTime) -> && account .access_token_expires_at() .is_some_and(|expires_at| expires_at > now) - && account.availability() == AccountAvailability::QuotaExhausted + && account + .quota() + .exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL) } diff --git a/backend/crates/providers/xai/src/transport/session.rs b/backend/crates/providers/xai/src/transport/session.rs index 97e0bebdf..e2b6b644a 100644 --- a/backend/crates/providers/xai/src/transport/session.rs +++ b/backend/crates/providers/xai/src/transport/session.rs @@ -5,7 +5,7 @@ use std::pin::Pin; use std::time::{Duration, SystemTime}; use gateway_core::engine::credential::{ - AccountAvailabilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId, + AccountEligibilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId, }; use gateway_core::engine::provider::ProviderResource; use gateway_core::routing::UpstreamModelId; @@ -210,7 +210,7 @@ pub struct GrokSessionSelection { excluded_accounts: BTreeSet, required_account: Option, account_selection_policy: AccountSelectionPolicy, - availability: AccountAvailabilityPolicy, + eligibility: AccountEligibilityPolicy, affinity: Option, deadline: SystemTime, } @@ -230,7 +230,7 @@ impl GrokSessionSelection { excluded_accounts, required_account, account_selection_policy, - availability: AccountAvailabilityPolicy::Enforce, + eligibility: AccountEligibilityPolicy::Enforce, affinity: None, deadline, } @@ -245,11 +245,11 @@ impl GrokSessionSelection { /// 为固定账号的管理端诊断指定本地可用性判定策略。 #[must_use] - pub(crate) const fn with_availability_policy( + pub(crate) const fn with_eligibility_policy( mut self, - availability: AccountAvailabilityPolicy, + eligibility: AccountEligibilityPolicy, ) -> Self { - self.availability = availability; + self.eligibility = eligibility; self } @@ -278,8 +278,8 @@ impl GrokSessionSelection { } #[must_use] - pub const fn availability(&self) -> AccountAvailabilityPolicy { - self.availability + pub const fn eligibility(&self) -> AccountEligibilityPolicy { + self.eligibility } /// 返回不含原始客户端会话值的账号亲和键。 diff --git a/backend/crates/providers/xai/tests/admin.rs b/backend/crates/providers/xai/tests/admin.rs index 1c72dd73b..c7eed7f06 100644 --- a/backend/crates/providers/xai/tests/admin.rs +++ b/backend/crates/providers/xai/tests/admin.rs @@ -471,10 +471,11 @@ fn account_record(account: &ProviderAccount) -> AccountRecord { access_token_expires_at: account.access_token_expires_at().map(DateTime::::from), next_refresh_at: account.next_refresh_at().map(DateTime::::from), enabled: account.enabled(), - availability: account.availability(), - availability_observed_at: now, - last_error_message: None, - quota_observed_at: None, + credential_state: account.credential_state(), + credential_observed_at: now, + quota: account.quota(), + last_error_reason: account.last_error_reason(), + last_error_message: account.last_error_message().map(str::to_owned), created_at: now, updated_at: now, } @@ -503,7 +504,7 @@ impl ProviderLeasePort for TestLeases { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/providers/xai/tests/credential/catalog.rs b/backend/crates/providers/xai/tests/credential/catalog.rs index 6d0ba6290..ea77c04a3 100644 --- a/backend/crates/providers/xai/tests/credential/catalog.rs +++ b/backend/crates/providers/xai/tests/credential/catalog.rs @@ -5,9 +5,9 @@ use std::time::{Duration, SystemTime}; use futures::future::{BoxFuture, join_all}; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasUpdate, CredentialRevision, + AccountStateChange, CredentialCasUpdate, CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, - QuotaObservation, + QuotaAccessChange, QuotaEvidence, QuotaObservation, QuotaState, }; use gateway_core::provider_ports::{ ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderStoreError, @@ -200,14 +200,15 @@ async fn repository_with_accounts( async fn set_account_state( store: &MemoryProviderAccountStore, id: &ProviderAccountId, - availability: AccountAvailability, + credential_state: CredentialState, ) { store .apply_state_change(AccountStateChange { message: None, account_id: id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability, + credential_state, + error_reason: credential_state.error_reason(), observed_at: SystemTime::now(), }) .await @@ -485,12 +486,10 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() { repository_with_accounts(&[("quota-exhausted", "subject-quota-exhausted")]).await; let id = account_id("quota-exhausted"); store - .apply_state_change(AccountStateChange { - message: None, + .apply_quota_access(QuotaAccessChange { account_id: id, expected_revision: CredentialRevision::new(1).expect("revision"), - availability: AccountAvailability::QuotaExhausted, - observed_at: SystemTime::now(), + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None), }) .await .expect("mark quota exhausted"); @@ -662,7 +661,7 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie .expect("read persisted quota") .pop() .expect("quota exists"); - let document = persisted.quota.expect("provider quota"); + let document = persisted.quota; assert_eq!(transport.calls.load(Ordering::SeqCst), 1); assert_eq!(snapshot.billing().used_percent(), Some(37.5)); @@ -683,33 +682,18 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie } #[tokio::test] -async fn recovered_quota_refresh_releases_existing_quota_isolation() { - for (suffix, availability) in [ - ("recover-quota", AccountAvailability::QuotaExhausted), - ("recover-ready", AccountAvailability::Ready), - ] { - let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await; - let id = account_id(suffix); - set_account_state(&store, &id, availability).await; - crate::support::grok_quota_service( - repository, - QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":25}}"#), - ) - .refresh_account(&id) - .await - .expect("refresh recovered quota"); - - let account = store.account(&id).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); - } -} - -#[tokio::test] -async fn authoritative_exhaustion_preserves_quota_exhausted_state() { +async fn billing_percent_does_not_rewrite_quota_access_fact() { let (store, repository) = repository_with_accounts(&[("still-exhausted", "still-exhausted")]).await; let id = account_id("still-exhausted"); - set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await; + store + .apply_quota_access(QuotaAccessChange { + account_id: id.clone(), + expected_revision: CredentialRevision::new(1).expect("revision"), + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None), + }) + .await + .expect("mark quota exhausted"); crate::support::grok_quota_service( repository, @@ -720,21 +704,75 @@ async fn authoritative_exhaustion_preserves_quota_exhausted_state() { .expect("refresh exhausted quota"); assert_eq!( - store.account(&id).expect("account").availability(), - AccountAvailability::QuotaExhausted + store.account(&id).expect("account").quota().access(), + gateway_core::engine::credential::QuotaAccessState::Exhausted ); } +#[tokio::test] +async fn authoritative_billing_refresh_clears_existing_quota_exhaustion() { + let (store, repository) = + repository_with_accounts(&[("recovered-quota", "recovered-quota")]).await; + let id = account_id("recovered-quota"); + store + .apply_quota_access(QuotaAccessChange { + account_id: id.clone(), + expected_revision: CredentialRevision::new(1).expect("revision"), + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None), + }) + .await + .expect("mark quota exhausted"); + + crate::support::grok_quota_service( + repository, + QueueBillingTransport::success( + br#"{"config":{"creditUsagePercent":12.5,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2099-07-22T00:00:00Z"}}}"#, + ), + ) + .refresh_account(&id) + .await + .expect("refresh recovered quota"); + + assert_eq!( + store.account(&id).expect("account").quota().access(), + gateway_core::engine::credential::QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn full_usage_display_does_not_invent_quota_exhaustion() { + let (store, repository) = + repository_with_accounts(&[("exhausted-quota", "exhausted-quota")]).await; + let id = account_id("exhausted-quota"); + + crate::support::grok_quota_service( + repository, + QueueBillingTransport::success( + br#"{"config":{"creditUsagePercent":100,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2099-07-22T00:00:00Z"}}}"#, + ), + ) + .refresh_account(&id) + .await + .expect("refresh exhausted quota"); + + let quota = store.account(&id).expect("account").quota(); + assert_eq!( + quota.access(), + gateway_core::engine::credential::QuotaAccessState::Unknown + ); + assert!(quota.reset_at().is_none()); +} + #[tokio::test] async fn recovered_quota_does_not_clear_terminal_account_states() { - for (suffix, availability) in [ - ("keep-banned", AccountAvailability::Banned), - ("keep-expired", AccountAvailability::Expired), - ("keep-invalid", AccountAvailability::Invalid), + for (suffix, credential_state) in [ + ("keep-banned", CredentialState::Banned), + ("keep-expired", CredentialState::Expired), + ("keep-invalid", CredentialState::Invalid), ] { let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await; let id = account_id(suffix); - set_account_state(&store, &id, availability).await; + set_account_state(&store, &id, credential_state).await; crate::support::grok_quota_service( repository, QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":10}}"#), @@ -744,14 +782,14 @@ async fn recovered_quota_does_not_clear_terminal_account_states() { .expect("refresh terminal account quota"); assert_eq!( - store.account(&id).expect("account").availability(), - availability + store.account(&id).expect("account").credential_state(), + credential_state ); } } #[tokio::test] -async fn quota_refresh_preserves_ready_availability_across_concurrent_state_write() { +async fn quota_refresh_preserves_ready_credential_state_across_concurrent_write() { let (store, repository) = repository_with_accounts(&[("new-cooldown", "new-cooldown")]).await; let id = account_id("new-cooldown"); let transport = Arc::new(MutatingBillingTransport { @@ -760,7 +798,8 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ message: None, account_id: id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability: AccountAvailability::Ready, + credential_state: CredentialState::Ready, + error_reason: None, observed_at: SystemTime::now(), }))), body: br#"{"config":{"creditUsagePercent":10}}"#.to_vec(), @@ -772,7 +811,7 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ .expect("refresh around concurrent state write"); let account = store.account(&id).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); } #[tokio::test] @@ -860,12 +899,48 @@ async fn quota_projection_preserves_unknown_period_for_dynamic_duration_fallback ); } +#[tokio::test] +async fn expired_billing_window_does_not_participate_in_scheduling_rank() { + let (store, repository) = + repository_with_accounts(&[("expired-billing", "expired-billing")]).await; + let id = account_id("expired-billing"); + let account = store.account(&id).expect("account"); + let service = crate::support::grok_quota_service( + repository, + QueueBillingTransport::success( + br#"{"config":{"creditUsagePercent":25,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2000-01-01T00:00:00Z","end":"2000-01-08T00:00:00Z"}}}"#, + ), + ); + + service + .refresh_account(&id) + .await + .expect("refresh expired billing window"); + + assert_eq!(service.scheduling_signals(&account), None); + assert_eq!( + store + .account(&id) + .expect("refreshed account") + .quota() + .access(), + gateway_core::engine::credential::QuotaAccessState::Unknown + ); +} + #[tokio::test] async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustion() { let (store, repository) = repository_with_accounts(&[("free-quota", "subject-free-quota")]).await; let id = account_id("free-quota"); - set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await; + store + .apply_quota_access(QuotaAccessChange { + account_id: id.clone(), + expected_revision: CredentialRevision::new(1).expect("revision"), + state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None), + }) + .await + .expect("mark quota exhausted"); let transport = QueueBillingTransport::success( br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"onDemandCap":{"val":0},"onDemandUsed":{"val":0},"prepaidBalance":{"val":0}}}"#, ); @@ -876,8 +951,8 @@ async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustio assert!(!snapshot.billing().has_authoritative_quota()); assert_eq!( - store.account(&id).expect("account").availability(), - AccountAvailability::QuotaExhausted + store.account(&id).expect("account").quota().access(), + gateway_core::engine::credential::QuotaAccessState::Exhausted ); } @@ -920,9 +995,9 @@ async fn quota_read_rejects_corrupt_provider_document() { .compare_and_swap_quota(QuotaObservation { account_id: account_id("corrupt"), expected_revision: CredentialRevision::new(1).expect("revision"), - quota: Some(OpaqueProviderData::new(document)), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(document), + observed_at: SystemTime::now(), + state: QuotaState::unknown(), }) .await .expect("seed corrupt quota"); @@ -975,8 +1050,7 @@ async fn failed_quota_refresh_does_not_replace_last_good_observation() { .expect("read quota") .pop() .expect("quota remains") - .quota - .expect("provider document"); + .quota; assert_eq!( persisted.expose_to_provider()["config"]["creditUsagePercent"].as_f64(), Some(10.0), diff --git a/backend/crates/providers/xai/tests/credential/client.rs b/backend/crates/providers/xai/tests/credential/client.rs index 0c46d94a2..a531bb858 100644 --- a/backend/crates/providers/xai/tests/credential/client.rs +++ b/backend/crates/providers/xai/tests/credential/client.rs @@ -4,7 +4,7 @@ use std::collections::VecDeque; use std::future::ready; use std::sync::{Arc, Mutex}; -use gateway_core::engine::credential::AccountAvailability; +use gateway_core::engine::credential::CredentialState; use provider_xai::{ AuthorizationCallback, FailClosedTokenVerifier, FailureClass, FormValue, GrokCredentialAdmin, GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, GrokOAuthImportMetadata, @@ -128,7 +128,7 @@ async fn credential_import_should_require_exact_metadata_and_official_userinfo() enabled: true, }) .expect("Provider projects verified token lifetime"); - assert_eq!(prepared.account.availability(), AccountAvailability::Ready); + assert_eq!(prepared.account.credential_state(), CredentialState::Ready); assert!(prepared.account.next_refresh_at().is_none()); assert!( prepared diff --git a/backend/crates/providers/xai/tests/credential/contract.rs b/backend/crates/providers/xai/tests/credential/contract.rs index 43a3d9d83..515f571f9 100644 --- a/backend/crates/providers/xai/tests/credential/contract.rs +++ b/backend/crates/providers/xai/tests/credential/contract.rs @@ -2,15 +2,14 @@ use std::sync::Arc; use chrono::Utc; use gateway_core::engine::credential::{ - AccountAvailability, CredentialCasOutcome, CredentialRevision, NewProviderAccount, - PlaintextCredential, ProviderAccount, ProviderAccountStore, + AccountErrorReason, CredentialCasOutcome, CredentialRevision, CredentialState, + NewProviderAccount, PlaintextCredential, ProviderAccount, ProviderAccountStore, }; use gateway_core::error::StoreErrorKind; use gateway_core::routing::ProviderKind; use provider_xai::{ - GrokCredentialAdmin, GrokCredentialAvailability, GrokCredentialRepository, - GrokCredentialRepositoryError, GrokOAuthSecret, RotateManagedGrokCredential, SecretValue, - UpdateGrokCredentialState, + GrokCredentialAdmin, GrokCredentialRepository, GrokCredentialRepositoryError, GrokOAuthSecret, + RotateManagedGrokCredential, SecretValue, UpdateGrokCredentialState, }; use crate::support::{ @@ -219,14 +218,15 @@ async fn state_update_uses_credential_revision_fence() { .update_state(&UpdateGrokCredentialState { account_id: input.account_id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability: GrokCredentialAvailability::Ready, - availability_reason: Some("upstream_rate_limited".to_owned()), + credential_state: CredentialState::Ready, + error_reason: None, + error_message: Some("upstream_rate_limited".to_owned()), observed_at: Utc::now(), }) .await .expect("state update"); let account = store.account(&input.account_id).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); } #[tokio::test] @@ -239,16 +239,20 @@ async fn terminal_state_without_upstream_message_uses_stable_error_reason() { .update_state(&UpdateGrokCredentialState { account_id: input.account_id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability: GrokCredentialAvailability::Banned, - availability_reason: None, + credential_state: CredentialState::Banned, + error_reason: Some(AccountErrorReason::AccountBanned), + error_message: None, observed_at: Utc::now(), }) .await .expect("state update"); assert_eq!( - store.last_error_message(&input.account_id).as_deref(), - Some("account_banned") + store + .account(&input.account_id) + .expect("account") + .last_error_reason(), + Some(AccountErrorReason::AccountBanned) ); } diff --git a/backend/crates/providers/xai/tests/credential/refresh.rs b/backend/crates/providers/xai/tests/credential/refresh.rs index 23867cd53..69446604b 100644 --- a/backend/crates/providers/xai/tests/credential/refresh.rs +++ b/backend/crates/providers/xai/tests/credential/refresh.rs @@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime}; use async_trait::async_trait; use chrono::Utc; use gateway_core::engine::credential::{ - AccountAvailability, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision, + CredentialCasOutcome, CredentialCasUpdate, CredentialRevision, CredentialState, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, }; use gateway_core::provider_ports::{ @@ -514,8 +514,8 @@ async fn unauthorized_recovery_marks_a_permanently_rejected_refresh_expired() { assert_eq!(outcome, GrokCredentialRecoveryOutcome::Rejected); assert_eq!( - store.account(&id).expect("account").availability(), - AccountAvailability::Expired + store.account(&id).expect("account").credential_state(), + CredentialState::Expired ); assert_eq!( store.last_error_message(&id).as_deref(), @@ -664,8 +664,8 @@ async fn invalid_grant_marks_account_expired() { fixture(input, [Err(GrokRefreshFailure::InvalidGrant)], true).await; service.refresh_due().await.expect("refresh"); assert_eq!( - store.account(&id).expect("account").availability(), - AccountAvailability::Expired + store.account(&id).expect("account").credential_state(), + CredentialState::Expired ); } @@ -676,8 +676,8 @@ async fn banned_failure_marks_account_banned() { let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Banned)], true).await; service.refresh_due().await.expect("refresh"); assert_eq!( - store.account(&id).expect("account").availability(), - AccountAvailability::Banned + store.account(&id).expect("account").credential_state(), + CredentialState::Banned ); assert_eq!( store.last_error_message(&id).as_deref(), @@ -704,8 +704,8 @@ async fn ambiguous_refresh_applies_bounded_oauth_backoff() { [GrokCredentialRefreshOutcome::Ambiguous { account_id }] if account_id == &id )); assert_eq!( - store.account(&id).expect("account").availability(), - AccountAvailability::Unknown + store.account(&id).expect("account").credential_state(), + CredentialState::Unknown ); let account = store.account(&id).expect("account"); assert_eq!(account.revision().get(), 2); @@ -736,7 +736,7 @@ async fn pre_send_transient_failure_applies_bounded_oauth_backoff() { let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Transient)], true).await; service.refresh_due().await.expect("refresh"); let account = store.account(&id).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Unknown); + assert_eq!(account.credential_state(), CredentialState::Unknown); assert_eq!(account.revision().get(), 2); assert!( account @@ -908,8 +908,11 @@ async fn recovery_window_exhaustion_marks_account_expired_without_token_exchange [GrokCredentialRefreshOutcome::Invalidated { account_id }] if account_id == &id )); assert_eq!( - store.account(&id).expect("expired account").availability(), - AccountAvailability::Expired + store + .account(&id) + .expect("expired account") + .credential_state(), + CredentialState::Expired ); assert_eq!(refresher.prepare_calls.load(Ordering::SeqCst), 1); assert_eq!(refresher.responses.lock().expect("queue").len(), 1); diff --git a/backend/crates/providers/xai/tests/credential/selector.rs b/backend/crates/providers/xai/tests/credential/selector.rs index 8b013a384..3a6b749d2 100644 --- a/backend/crates/providers/xai/tests/credential/selector.rs +++ b/backend/crates/providers/xai/tests/credential/selector.rs @@ -4,10 +4,10 @@ use std::time::{Duration, SystemTime}; use chrono::Utc; use gateway_core::engine::credential::{ - AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals, - AccountSelectionPolicy, CredentialCasOutcome, CredentialRevision, OpaqueProviderData, - ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation, - QuotaWriteOutcome, RotationStrategy, + AccountAttemptFeedback, AccountFeedbackStats, AccountRuntimeSignals, AccountSelectionPolicy, + CredentialCasOutcome, CredentialRevision, CredentialState, OpaqueProviderData, + ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaAccessState, + QuotaObservation, QuotaState, QuotaWriteOutcome, RotationStrategy, }; use gateway_core::provider_ports::{ ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, @@ -17,10 +17,10 @@ use gateway_core::routing::UpstreamModelId; use provider_xai::{ GrokAccountSessionSelector, GrokBillingRequest, GrokBillingTransport, GrokBillingTransportError, GrokBillingTransportErrorKind, GrokBillingTransportFuture, - GrokCatalogScope, GrokCredentialAdmin, GrokCredentialAvailability, GrokCredentialCatalogCache, - GrokCredentialCatalogSeed, GrokCredentialFailure, GrokCredentialRepository, GrokPlanCatalog, - GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError, - RotateManagedGrokCredential, UpdateGrokCredentialState, + GrokCatalogScope, GrokCredentialAdmin, GrokCredentialCatalogCache, GrokCredentialCatalogSeed, + GrokCredentialFailure, GrokCredentialRepository, GrokPlanCatalog, GrokSessionSelection, + GrokSessionSelector, GrokSessionSelectorError, RotateManagedGrokCredential, + UpdateGrokCredentialState, }; use crate::support::{ @@ -108,8 +108,9 @@ impl SelectorFixture { .update_state(&UpdateGrokCredentialState { account_id: input.account_id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - availability: GrokCredentialAvailability::Ready, - availability_reason: None, + credential_state: CredentialState::Ready, + error_reason: None, + error_message: None, observed_at: Utc::now(), }) .await @@ -131,7 +132,7 @@ impl SelectorFixture { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - quota_limit_reached: false, + rate_limited_until: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -241,11 +242,9 @@ impl SelectorFixture { .compare_and_swap_quota(QuotaObservation { account_id: id.clone(), expected_revision: CredentialRevision::new(1).expect("revision"), - quota: Some(OpaqueProviderData::new( - document.as_object().expect("quota object").clone(), - )), - observed_at: Some(SystemTime::now()), - limit_reached: None, + quota: OpaqueProviderData::new(document.as_object().expect("quota object").clone()), + observed_at: SystemTime::now(), + state: QuotaState::unknown(), }) .await .expect("persist quota"); @@ -305,16 +304,20 @@ async fn unauthorized_feedback_records_runtime_cooldown_without_persisting_accou .store .account(&selected) .expect("selected") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); let other = [account_id("feedback-a"), account_id("feedback-b")] .into_iter() .find(|id| id != &selected) .expect("other account"); assert_eq!( - fixture.store.account(&other).expect("other").availability(), - AccountAvailability::Ready + fixture + .store + .account(&other) + .expect("other") + .credential_state(), + CredentialState::Ready ); assert!( fixture @@ -379,7 +382,7 @@ async fn account_scoped_cooldown_survives_credential_rotation() { )); let account = fixture.store.account(&id).expect("rotated account"); assert_eq!(account.revision().get(), 2); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); } #[tokio::test] @@ -401,7 +404,7 @@ async fn rate_limit_feedback_persists_the_grok2api_cooldown_state() { ) .await; let account = fixture.store.account(&selected).expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); assert!( fixture .cooldowns @@ -440,7 +443,7 @@ async fn successful_request_clears_the_persisted_cooldown_state() { .store .account(session.account_id()) .expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); } #[tokio::test] @@ -469,8 +472,8 @@ async fn payment_required_feedback_writes_short_account_cooldown_without_persist .store .account(&selected) .expect("selected account") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); assert!( fixture @@ -508,7 +511,7 @@ async fn model_quota_feedback_writes_model_scoped_cooldown_without_blocking_othe .await; let account = fixture.store.account(&selected).expect("selected account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60); let scope = ProviderCooldownScope::upstream_model(failed_model); // model-scoped cooldown:目标模型被排除,账号级无 cooldown。 @@ -563,8 +566,8 @@ async fn model_access_feedback_reports_model_cooldown_without_blocking_the_accou .store .account(session.account_id()) .expect("account") - .availability(), - AccountAvailability::Ready + .credential_state(), + CredentialState::Ready ); assert!(matches!( fixture @@ -594,7 +597,7 @@ async fn interrupted_stream_feedback_persists_the_grok2api_cooldown_state() { .store .account(session.account_id()) .expect("account"); - assert_eq!(account.availability(), AccountAvailability::Ready); + assert_eq!(account.credential_state(), CredentialState::Ready); assert!( fixture .cooldowns @@ -630,8 +633,9 @@ async fn quota_feedback_uses_common_quota_exhausted_state() { .store .account(session.account_id()) .expect("account") - .availability(), - AccountAvailability::QuotaExhausted + .quota() + .access(), + QuotaAccessState::Exhausted ); } @@ -673,13 +677,18 @@ async fn quota_feedback_should_follow_the_account_across_a_credential_rotation() .await; assert_eq!( - fixture.store.account(&id).expect("account").availability(), - AccountAvailability::QuotaExhausted + fixture + .store + .account(&id) + .expect("account") + .quota() + .access(), + QuotaAccessState::Exhausted ); } #[tokio::test] -async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() { +async fn ordinary_success_clears_confirmed_exhaustion() { let fixture = SelectorFixture::new(&["quota-success"]).await; let session = fixture .selector @@ -690,7 +699,6 @@ async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() { .selector .record_failure(&session, GrokCredentialFailure::FreeQuotaExhausted) .await; - fixture.selector.record_success(&session).await; assert_eq!( @@ -698,8 +706,9 @@ async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() { .store .account(session.account_id()) .expect("account") - .availability(), - AccountAvailability::QuotaExhausted + .quota() + .access(), + QuotaAccessState::Allowed ); } @@ -1061,8 +1070,8 @@ async fn bare_402_cooldown_expires_and_account_recovers_without_persisted_exhaus .store .account(&selected) .expect("account") - .availability(), - AccountAvailability::Ready, + .credential_state(), + CredentialState::Ready, "bare 402 must not persist QuotaExhausted" ); let next = fixture diff --git a/backend/crates/providers/xai/tests/support.rs b/backend/crates/providers/xai/tests/support.rs index f9dbe2f2e..9b2062cad 100644 --- a/backend/crates/providers/xai/tests/support.rs +++ b/backend/crates/providers/xai/tests/support.rs @@ -8,10 +8,10 @@ use std::time::{Duration, SystemTime}; use async_trait::async_trait; use chrono::Utc; use gateway_core::engine::credential::{ - AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, - CredentialRevision, LoadedCredential, NewProviderAccount, PlaintextCredential, ProviderAccount, - ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation, - QuotaWriteOutcome, + AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, + CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount, PlaintextCredential, + ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, + QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome, }; use gateway_core::error::{StoreError, StoreErrorKind}; use gateway_core::provider_ports::{ @@ -23,11 +23,10 @@ use gateway_core::provider_ports::{ use gateway_core::routing::ProviderKind; use provider_xai::{ CreateGrokCredential, GROK_BILLING_URL, GrokAccountProfile, GrokBillingTransport, - GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, GrokCredentialAvailability, - GrokCredentialCatalogCache, GrokCredentialCatalogService, GrokCredentialQuotaService, - GrokCredentialRepository, GrokCredentialRepositoryError, GrokEndpointPolicy, - GrokModelCatalogTransport, GrokOAuthSecret, GrokPlanCatalog, GrokReqwestTransportBuildError, - SecretValue, XaiConfig, XaiWireProfileState, + GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, GrokCredentialCatalogCache, + GrokCredentialCatalogService, GrokCredentialQuotaService, GrokCredentialRepository, + GrokCredentialRepositoryError, GrokEndpointPolicy, GrokModelCatalogTransport, GrokOAuthSecret, + GrokPlanCatalog, GrokReqwestTransportBuildError, SecretValue, XaiConfig, XaiWireProfileState, }; use reqwest::Client; use reqwest::redirect::Policy; @@ -194,7 +193,6 @@ struct StoredAccount { account: ProviderAccount, credential: PlaintextCredential, quota: Option, - last_error_message: Option, } #[derive(Default)] @@ -218,7 +216,7 @@ impl MemoryProviderAccountStore { pub fn last_error_message(&self, id: &ProviderAccountId) -> Option { lock(&self.accounts) .get(id) - .and_then(|stored| stored.last_error_message.clone()) + .and_then(|stored| stored.account.last_error_message().map(str::to_owned)) } pub fn credential(&self, id: &ProviderAccountId) -> Option { @@ -253,7 +251,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore { account: account.account, credential: account.credential, quota: None, - last_error_message: None, }, ); Ok(()) @@ -335,7 +332,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore { AccountReplacement { revision: next, access_token_expires_at, - availability: stored.account.availability(), + credential_state: stored.account.credential_state(), + quota: stored.account.quota(), + last_error_reason: stored.account.last_error_reason(), + last_error_message: stored.account.last_error_message().map(str::to_owned), enabled: stored.account.enabled(), has_refresh_token, next_refresh_at, @@ -372,7 +372,31 @@ impl ProviderAccountStore for MemoryProviderAccountStore { if stored.account.revision() != observation.expected_revision { return Ok(QuotaWriteOutcome::Conflict); } + let quota = observation.state; stored.quota = Some(observation); + stored.account = rebuild_account( + &stored.account, + AccountReplacement::preserving(&stored.account).with_quota(quota), + ); + Ok(QuotaWriteOutcome::Updated) + } + + async fn apply_quota_access( + &self, + change: QuotaAccessChange, + ) -> Result { + let mut accounts = lock(&self.accounts); + let stored = accounts.get_mut(&change.account_id).ok_or_else(invalid)?; + if stored.account.revision() != change.expected_revision { + return Ok(QuotaWriteOutcome::Conflict); + } + stored.account = rebuild_account( + &stored.account, + AccountReplacement::preserving(&stored.account).with_quota(change.state), + ); + if let Some(observation) = &mut stored.quota { + observation.state = change.state; + } Ok(QuotaWriteOutcome::Updated) } @@ -387,7 +411,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore { AccountReplacement { revision: stored.account.revision(), access_token_expires_at: stored.account.access_token_expires_at(), - availability: change.availability, + credential_state: change.credential_state, + quota: stored.account.quota(), + last_error_reason: change.error_reason, + last_error_message: change.message.clone(), enabled: stored.account.enabled(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -396,11 +423,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore { plan_type: stored.account.plan_type().map(str::to_owned), }, ); - if change.availability == AccountAvailability::Ready { - stored.last_error_message = None; - } else if let Some(message) = change.message { - stored.last_error_message = Some(message); - } Ok(()) } @@ -412,7 +434,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore { AccountReplacement { revision: stored.account.revision(), access_token_expires_at: stored.account.access_token_expires_at(), - availability: stored.account.availability(), + credential_state: stored.account.credential_state(), + quota: stored.account.quota(), + last_error_reason: stored.account.last_error_reason(), + last_error_message: stored.account.last_error_message().map(str::to_owned), enabled: stored.account.enabled(), has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -436,7 +461,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore { AccountReplacement { revision: stored.account.revision(), access_token_expires_at: stored.account.access_token_expires_at(), - availability: stored.account.availability(), + credential_state: stored.account.credential_state(), + quota: stored.account.quota(), + last_error_reason: stored.account.last_error_reason(), + last_error_message: stored.account.last_error_message().map(str::to_owned), enabled, has_refresh_token: stored.account.has_refresh_token(), next_refresh_at: stored.account.next_refresh_at(), @@ -457,7 +485,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore { struct AccountReplacement { revision: CredentialRevision, access_token_expires_at: Option, - availability: AccountAvailability, + credential_state: CredentialState, + quota: QuotaState, + last_error_reason: Option, + last_error_message: Option, enabled: bool, has_refresh_token: bool, next_refresh_at: Option, @@ -466,6 +497,30 @@ struct AccountReplacement { plan_type: Option, } +impl AccountReplacement { + fn preserving(account: &ProviderAccount) -> Self { + Self { + revision: account.revision(), + access_token_expires_at: account.access_token_expires_at(), + credential_state: account.credential_state(), + quota: account.quota(), + last_error_reason: account.last_error_reason(), + last_error_message: account.last_error_message().map(str::to_owned), + enabled: account.enabled(), + has_refresh_token: account.has_refresh_token(), + next_refresh_at: account.next_refresh_at(), + name: account.name().to_owned(), + email: account.email().map(str::to_owned), + plan_type: account.plan_type().map(str::to_owned), + } + } + + fn with_quota(mut self, quota: QuotaState) -> Self { + self.quota = quota; + self + } +} + fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement) -> ProviderAccount { ProviderAccount::new( previous.id().clone(), @@ -481,7 +536,13 @@ fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement) previous.upstream_account_id().map(str::to_owned), replacement.plan_type, ) - .with_runtime_state(replacement.enabled, replacement.availability) + .with_account_facts( + replacement.enabled, + replacement.credential_state, + replacement.quota, + replacement.last_error_reason, + replacement.last_error_message, + ) .with_refresh_schedule(replacement.has_refresh_token, replacement.next_refresh_at) } @@ -549,8 +610,9 @@ pub fn create_input(suffix: &str, subject: &str) -> CreateGrokCredential { }, account: profile(subject), enabled: true, - initial_availability: GrokCredentialAvailability::Unknown, - initial_availability_reason: None, + initial_credential_state: CredentialState::Unknown, + initial_error_reason: None, + initial_error_message: None, } } diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index ce10cb924..bdcefc8ac 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -1,3 +1,4 @@ 5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql f4168f7e23d42cf9d37096339e931b0e0c823f7a8dbfc6580a04f096c4c3d8e3 0002_provider_accounts_nullable_upstream_user_id.sql 4163f61183d9e90f1368c08f614040d6a1a0048de3886a77dd708ce21430b2d0 0003_backfill_openai_access_token_expiry.sql +b41bb3be5a84b95852158d82267456f8415811a1137025581eefd1c96c0882d1 0004_normalize_account_state_facts.sql diff --git a/backend/migrations/0004_normalize_account_state_facts.sql b/backend/migrations/0004_normalize_account_state_facts.sql new file mode 100644 index 000000000..8b511e6ab --- /dev/null +++ b/backend/migrations/0004_normalize_account_state_facts.sql @@ -0,0 +1,165 @@ +-- Separate credential identity facts from quota access facts. The public account +-- status remains derived at read time because Redis cooldowns and reset times are +-- time-dependent. + +alter table provider_accounts + rename column availability to credential_state; + +alter table provider_accounts + rename column availability_observed_at to credential_observed_at; + +alter table provider_accounts + drop constraint provider_accounts_availability_ck; + +alter table provider_accounts + add constraint provider_accounts_credential_state_ck check ( + credential_state in ('unknown', 'ready', 'expired', 'banned', 'invalid') + ) not valid; + +alter table provider_accounts + add column quota_access_state text not null default 'unknown', + add column quota_evidence text, + add column quota_access_observed_at timestamptz, + add column quota_reset_at timestamptz, + add column last_error_reason text; + +alter table provider_accounts + drop constraint provider_accounts_quota_observation_ck; + +-- Old quota_exhausted credential states are quota facts. Re-derive OpenAI quota +-- access from the raw authoritative response and never trust the old percentage- +-- derived quota_limit_reached boolean. +update provider_accounts +set + quota_access_state = case + when provider_kind = 'openai' + and provider_quota_json #>> '{rate_limit,allowed}' = 'true' + then 'allowed' + when provider_kind = 'openai' + and ( + provider_quota_json #>> '{rate_limit,allowed}' = 'false' + or ( + coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true' + and ( + provider_quota_json #>> '{rate_limit,limit_reached}' = 'true' + or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true' + ) + ) + ) + then 'exhausted' + when provider_kind = 'xai' and credential_state = 'quota_exhausted' + then 'exhausted' + when provider_quota_json is not null then 'unknown' + else 'unknown' + end, + quota_evidence = case + when provider_kind = 'openai' + and provider_quota_json #>> '{rate_limit,allowed}' = 'false' + then 'provider_denied' + when provider_kind = 'openai' + and coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true' + and ( + provider_quota_json #>> '{rate_limit,limit_reached}' = 'true' + or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true' + ) + then 'account_limit_reached' + when provider_kind = 'xai' and credential_state = 'quota_exhausted' + then 'usage_limit_reached' + else null + end, + quota_access_observed_at = case + when provider_kind = 'openai' and provider_quota_json is not null + then quota_observed_at + when provider_kind = 'xai' and credential_state = 'quota_exhausted' + then credential_observed_at + else null + end, + quota_reset_at = case + when provider_kind = 'openai' + and jsonb_typeof(provider_quota_json #> '{rate_limit,primary_window,reset_at}') = 'number' + then to_timestamp((provider_quota_json #>> '{rate_limit,primary_window,reset_at}')::double precision) + else null + end, + credential_state = case + when credential_state = 'quota_exhausted' and upstream_user_id is not null then 'ready' + when credential_state = 'quota_exhausted' then 'unknown' + else credential_state + end, + last_error_reason = case credential_state + when 'unknown' then 'account_unverified' + when 'expired' then 'credential_expired' + when 'invalid' then 'credential_invalid' + when 'banned' then 'account_banned' + else null + end, + last_error_message = case + when credential_state in ('unknown', 'expired', 'invalid', 'banned') then last_error_message + else null + end; + +alter table provider_accounts + validate constraint provider_accounts_credential_state_ck; + +alter table provider_accounts + add constraint provider_accounts_quota_access_state_ck check ( + quota_access_state in ('unknown', 'allowed', 'exhausted') + ), + add constraint provider_accounts_quota_evidence_ck check ( + quota_evidence is null + or quota_evidence in ( + 'provider_denied', + 'account_limit_reached', + 'usage_limit_reached', + 'payment_required' + ) + ), + add constraint provider_accounts_quota_fact_ck check ( + (quota_access_state = 'unknown' and quota_evidence is null and quota_reset_at is null) + or ( + quota_access_state = 'allowed' + and quota_evidence is null + and quota_access_observed_at is not null + and quota_reset_at is null + ) + or ( + quota_access_state = 'exhausted' + and quota_evidence is not null + and quota_access_observed_at is not null + ) + ), + add constraint provider_accounts_quota_observation_ck check ( + (provider_quota_json is null) = (quota_observed_at is null) + ), + add constraint provider_accounts_error_reason_ck check ( + last_error_reason is null + or last_error_reason in ( + 'account_unverified', + 'access_token_expired', + 'credential_expired', + 'credential_invalid', + 'account_banned' + ) + ); + +drop index provider_accounts_availability_idx; +create index provider_accounts_credential_state_idx + on provider_accounts (credential_state, id); +create index provider_accounts_quota_access_idx + on provider_accounts (quota_access_state, quota_reset_at, id); + +alter table provider_accounts + drop column quota_limit_reached; + +alter table provider_accounts + rename constraint provider_accounts_time_ck to provider_accounts_time_ck_old; + +alter table provider_accounts + drop constraint provider_accounts_time_ck_old; + +alter table provider_accounts + add constraint provider_accounts_time_ck check ( + created_at <= updated_at + and credential_observed_at <= updated_at + and (quota_access_observed_at is null or quota_access_observed_at <= updated_at) + and (quota_observed_at is null or quota_observed_at <= updated_at) + ); diff --git a/docs/api.md b/docs/api.md index 3e7870a7b..013f042b2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -112,7 +112,7 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段 账号列表支持以下稳定值: - `provider`: `all`、`openai`、`xai`; -- `status`: `active`、`expired`、`quota_exhausted`、`disabled`、`banned`; +- `status`: `normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`; - `sortBy`: `email`、`status`、`planType`、`usage`、`lastUsedAt`、`expiresAt`; - `sortDirection`: `asc`、`desc`。 @@ -184,13 +184,19 @@ OAuth start 使用: 但上游 authorization code 本身通常只能交换一次;已完成过 token exchange 时应重新创建 OAuth flow。 - `GET /accounts/quota` 只读取最后一次落库快照;`POST /accounts/quota/refresh` 才访问上游。access token 已过期时,额度刷新要求先走 credential 刷新或重新授权,不会拿过期 token 探测额度。 -- 成功额度观测会 revision-fenced 写入 quota,并在 `active` 与 `quota_exhausted` 等额度所属状态间同步; - 不会清除 `expired`、`banned` 等 credential/封禁终态。额度接口的 401/403 也不足以判定 refresh token - 永久失效,credential 终态只由 OAuth refresh 的明确永久错误写入。 +- 成功额度观测会 revision-fenced 写入 quota;明确 `Allowed` 投影为 `normal`,明确耗尽投影为 + `quota_exhausted`。额度观测不会清除凭据过期、无效或封禁事实;这些事实统一投影为 `error`,并由 + `errorReason` 区分。额度接口的 401/403 也不足以判定 refresh token 永久失效,credential 终态只由 + OAuth refresh 的明确永久错误写入。 - 正常 Responses 请求会解析上游响应的 rate-limit headers,合并进同一 quota 快照并同步状态。Free、 K12 等套餐共用该状态机;套餐只参与账号展示和按套餐隔离的模型目录 cache,不存在 K12 专属额度路径。 -- 账号页没有定时静默轮询。手工额度刷新完成后页面会重新读取该账号;请求驱动或后台任务产生的状态 - 变化,需要下一次显式查询账号列表后才会显示。 +- 账号展开区的 Token 结构和模型排行使用代表性账号级额度窗口聚合,查询边界严格为 + `[resetAt - windowSeconds, resetAt)`;额度刷新若返回了更早的重置时间,会按新边界重新聚合。无法取得 + 完整窗口边界或只有模型专属额度时显示无数据,不回退成历史累计。金额原值保持完整精度,USD 展示值 + 小于 1 美元时最多保留四位小数,其余保留两位。 +- 账号页没有定时静默轮询。手工额度刷新只替换响应中的账号行并同步状态汇总,不触发整页 loading;若 + 新状态不符合当前筛选,该行从当前页移除。请求驱动或后台任务产生的状态变化,需要下一次显式查询账号 + 列表后才会显示。 ## 6. Client Key diff --git a/docs/architecture.md b/docs/architecture.md index db4b9301e..8ef75f0eb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -116,7 +116,17 @@ Provider 独占以下职责: - OpenAI 是逐字节透明代理:请求 body 的未知字段和字段顺序保持不变(受控模型映射除外),上游 wire 是客户端可见的事实来源,HTTP SSE 帧以原始字节原样下发,WebSocket 上游 JSON 文本原样下发;response ID 以 opaque bytes 存储,不假设 UUID 或固定长度。canonical facts 只从同一帧旁路解析,用于观测、亲和与计费。仅网关内部帧(rate limits、metadata)被就地消费,不向客户端转发。 - xAI 是翻译层:把 Grok wire 解码后重新合成 Responses wire 事件,不承诺字节透明;上游错误按结构化 message/code/type 透出,message 中的 UUID 账号指纹替换为占位符。 -credential 更新使用 `credential_revision` CAS。认证永久失败、封禁、额度耗尽和带截止时间的 cooldown 是账号运行时事实。cooldown 到期后,Core 的有效调度谓词会自然允许账号重新参与选择;Redis cooldown 只是热缓存。成功调用或成功额度观测才可把账号状态重新观测为 `ready`。推理请求的账号/model cooldown 只约束数据面调度,不参与 OAuth refresh;OpenAI 与 xAI 的 OAuth refresh 请求状态不明确时只按 OAuth 自身的有界退避推进该 credential 的 `next_refresh_at`:前五次约为 5/15/45/135/300 秒,之后约每 10 分钟恢复;仍有效 AT 时不晚于其到期时刻,AT 到期后仍按该节奏恢复,至 `AT 到期 + 24h` 才终态化。自动 refresh 的瞬态失败(限流、5xx、超时、畸形响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;上游明确的永久错误(`invalid_grant`、封禁)仍立即写入终态失效。 +credential 更新使用 `credential_revision` CAS。PostgreSQL 分别保存凭据事实与额度访问事实;带截止时间的 +429 cooldown 只保存在 Redis,是唯一的临时冷却运行时事实,不进入 Provider quota 展示模型。Store 把三类 +事实交给 Core 的唯一解析器,投影为 `normal`、`quota_exhausted`、`rate_limited`、`disabled` 或 `error`; +投影为 `rate_limited` 时同时携带有效的 `rate_limited_until`,到期重算后该时间和状态一起消失。若没有其它 +阻塞事实,账号才恢复为 `normal`。成功推理或明确的成功额度观测可以写入 `Allowed`;本地时钟与不确定响应 +不能伪造恢复。推理请求的账号/model cooldown 只约束数据面调度,不参与 OAuth refresh;OpenAI 与 xAI 的 +OAuth refresh 请求状态不明确时只按 OAuth 自身的有界退避推进该 credential 的 `next_refresh_at`:前五次 +约为 5/15/45/135/300 秒,之后约每 10 分钟恢复;仍有效 Access Token 时不晚于其到期时刻,Access Token +到期后仍按该节奏恢复,至“到期 + 24 小时”才终态化。自动 refresh 的瞬态失败(限流、5xx、超时、畸形 +响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;上游明确的永久错误 +(`invalid_grant`、封禁)仍立即写入终态失效。 `refresh_token_expires_at` 不是公共 SQL 列或 Core 权威状态。xAI 可在 `provider_credentials_json` 内保存它作为 Provider 私有提示;真正失效以 refresh endpoint 返回的永久错误为准。 @@ -127,12 +137,15 @@ exchange 成功后执行同一补全。两条路径不执行 JWKS 验签,不 credential revision;complete 阶段重新读取目标账号及其当前 revision,并以当前 revision 做最终 CAS。 重新授权和手工或后台 RT refresh 只轮换目标账号的 token,保留既有账号资料与 OAuth principal。 -credential 与 quota 是两套状态机。主动 quota refresh 会拒绝过期 access token;quota 探测的 -401/403 不具备判定 refresh token 永久失效的证据。成功主动观测和正常推理响应中的 rate-limit header -观测都 revision-fenced 写入同一 quota,并转换 readiness、`QuotaExhausted` 或 `Cooldown` 等额度事实; -只有明确的 deactivated workspace 才能由额度链路写入 `Banned`,且额度观测不能清除 `Invalid`、 -`Expired`、`Banned`。Free、K12 等套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider -事实并用于隔离不同套餐的 catalog cache。 +credential 与 quota 是两组独立事实,不是两套对外状态。主动 quota refresh 会拒绝过期 Access Token; +quota 探测的 401/403 不具备判定 Refresh Token 永久失效的证据。成功主动观测和正常推理响应中的 +rate-limit header 观测都 revision-fenced 写入同一 quota;只有明确的访问证据才写 `Allowed` 或 +`Exhausted`。只有明确的 deactivated workspace 才能由额度链路写入 `Banned` 凭据事实,且额度观测不能 +清除 `Invalid`、`Expired`、`Banned`。这些凭据事实对外统一投影为 `error` 并携带稳定原因。Free、K12 等 +套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider 事实并用于隔离不同套餐的 catalog cache。 +账号展开用量不读取 retention 范围累计值,而是选择带完整边界且可归属到账号的代表性额度窗口,并按 +`[reset_at - window_seconds, reset_at)` 聚合请求、Token、模型和成本;额度重置边界变化后,同一查询随刷新 +结果重算。模型专属窗口不会错误复用账号总用量,边界不完整时也不伪造“当前窗口”统计。 ## 7. 控制面与内部 revision @@ -156,7 +169,7 @@ Admin API 不要求客户端提交配置 revision,也不向客户端暴露配 | `admin_audit_events` | 管理 mutation 审计 | | `client_api_keys` | 客户端鉴权、限额与授权范围 | | `runtime_settings` | 全局配置与 `config_revision` | -| `provider_accounts` | 账号资料、Provider-owned 明文 credential JSON、revision、quota、cooldown | +| `provider_accounts` | 账号资料、Provider-owned 明文 credential JSON、revision、凭据事实与规范化 quota 事实 | | `model_requests` | 请求、attempt、计费、交付与恢复事实 | | `ops_events` | 脱敏运行事件 | | `backup_settings` | S3/R2 存储、Cron 计划与保留策略单例配置 | @@ -184,13 +197,14 @@ heartbeat;计划时间点冲突只记录日志/指标并推进游标,不产 业务 schema 由 `backend/migrations/0001_initial.sql` 直接定义当前大版本的完整新库基线,其中 response ID 使用无格式假设的 UTF-8 bytes,`model_requests.service_tier` 保存响应观测到的服务档位, -且额度耗尽账号可以携带冷却截止时间。同一大版本内已合入的迁移永久冻结—— +账号对外五态由 PostgreSQL 凭据/额度事实与 Redis 429 冷却通过 Core 唯一解析器动态派生, +不保存重复的 `status` 列。同一大版本内已合入的迁移永久冻结—— `.frozen-sha256` 是冻结清单,CI 校验清单相对 PR base 只增不改,启动时 sqlx 重校验已应用 迁移的 checksum,不一致直接拒绝启动。规则见 `backend/migrations/README.md`。 ## 9. Redis 终态 -Redis 保存的协调状态恰好是:客户端 admission 热状态、credential lease/fencing 与调度信号、账号 cooldown 与 availability 热缓存、catalog 缓存、Provider circuit、会话亲和与会话级账号排除、continuation pin、OAuth pending flow、worker leader lease 以及 runtime change 通知。Redis 丢失后必须从 PostgreSQL 或 Provider 事实恢复;恢复完成前需要保护的 acquire 路径 fail closed。 +Redis 保存的协调状态恰好是:客户端 admission 热状态、credential lease/fencing 与调度信号、账号 429 cooldown 与 credential state 可重建缓存、catalog 缓存、Provider circuit、会话亲和与会话级账号排除、continuation pin、OAuth pending flow、worker leader lease以及 runtime change 通知。Redis 丢失后必须从 PostgreSQL 或 Provider 事实恢复;恢复完成前需要保护的 acquire 路径 fail closed。 多副本部署时,跨副本协调面只有上述 Redis 状态与 PostgreSQL 事实;WebSocket 连接池、请求画像与 RuntimeSnapshot 副本都是进程内状态,其中 RuntimeSnapshot 依靠周期对账收敛,请求画像依靠各自的官方版本检查收敛。 diff --git a/frontend/src/api/modules/accounts.ts b/frontend/src/api/modules/accounts.ts index a9e7fe678..ed4004a20 100644 --- a/frontend/src/api/modules/accounts.ts +++ b/frontend/src/api/modules/accounts.ts @@ -1,5 +1,15 @@ import request from '../request' +export type AccountStatus + = 'normal' | 'quota_exhausted' | 'rate_limited' | 'disabled' | 'error' + +export type AccountErrorReason + = 'account_unverified' + | 'access_token_expired' + | 'credential_expired' + | 'credential_invalid' + | 'account_banned' + export interface AccountQuotaWindow { key: string group: string @@ -104,11 +114,9 @@ export interface Account { planType: string | null authenticationKind: string hasRefreshToken: boolean - status: string - displayStatus: string - errorReason: string | null + status: AccountStatus + errorReason: AccountErrorReason | null errorMessage: string | null - availability: string enabled: boolean accessTokenExpiresAt: string | null accessTokenExpiresAtDisplay: string | null diff --git a/frontend/src/api/modules/dashboard.ts b/frontend/src/api/modules/dashboard.ts index a900193bc..dc60e077b 100644 --- a/frontend/src/api/modules/dashboard.ts +++ b/frontend/src/api/modules/dashboard.ts @@ -143,14 +143,11 @@ export interface DashboardAccountUsage { export interface DashboardPoolSummary { total: number - active: number - rateLimited: number - expired: number - invalid: number + normal: number quotaExhausted: number - refreshing: number | null + rateLimited: number disabled: number - banned: number + error: number } export interface DashboardCapacityInfo { diff --git a/frontend/src/views/accounts/components/AccountStatusBadge.vue b/frontend/src/views/accounts/components/AccountStatusBadge.vue index 30d39342a..820fbe053 100644 --- a/frontend/src/views/accounts/components/AccountStatusBadge.vue +++ b/frontend/src/views/accounts/components/AccountStatusBadge.vue @@ -1,4 +1,5 @@ @@ -305,8 +312,8 @@ const statusBars = computed(() => {
{{ availabilityRate }} - 可用率 + >{{ normalRate }} + 正常率
@@ -326,11 +333,11 @@ const statusBars = computed(() => { -
+