refactor(accounts): unify five-state account status

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-08-12 23:50:31 +08:00
co-authored by Codex
parent 4127886584
commit 2b8b3de0e4
97 changed files with 3607 additions and 4848 deletions
-1
View File
@@ -255,7 +255,6 @@ pub async fn initialize(
let accounts = Arc::new(DefaultAccountsService::new(
store.accounts(),
store.settings(),
registry.clone(),
snapshot.clone(),
probe.clone(),
@@ -1,6 +1,6 @@
//! 多 Provider 账号目录与连接测试的公共事实。
use std::{pin::Pin, time::SystemTime};
use std::pin::Pin;
use chrono::{DateTime, Utc};
use futures::Stream;
@@ -9,151 +9,10 @@ use gateway_core::routing::ProviderKind;
use super::{PageSize, Revision, observability::TimeRange};
/// Provider 账号当前可用性;权威定义与字符串编解码在 gateway-core,此处仅复用。
pub use gateway_core::engine::credential::AccountAvailability;
/// 管理页使用的归一化账号状态。
///
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountStatus {
/// 正常,可调度。
Normal,
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
QuotaExhausted,
/// 限流中(quota 快照级信号),自动恢复,无需干预。
RateLimited,
/// 管理员手动停用(`enabled = false`)。
Disabled,
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
Error,
}
impl AccountStatus {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Normal => "normal",
Self::QuotaExhausted => "quota_exhausted",
Self::RateLimited => "rate_limited",
Self::Disabled => "disabled",
Self::Error => "error",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"normal" => Some(Self::Normal),
"quota_exhausted" => Some(Self::QuotaExhausted),
"rate_limited" => Some(Self::RateLimited),
"disabled" => Some(Self::Disabled),
"error" => Some(Self::Error),
_ => None,
}
}
}
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusSignals {
pub enabled: bool,
pub availability: AccountAvailability,
pub access_token_expired: bool,
pub quota_limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
pub rate_limited_until: Option<SystemTime>,
}
impl AccountStatusSignals {
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
#[must_use]
pub const fn derive(self) -> AccountStatus {
if !self.enabled {
AccountStatus::Disabled
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
AccountStatus::QuotaExhausted
} else if self.access_token_expired
|| matches!(
self.availability,
AccountAvailability::Expired
| AccountAvailability::Invalid
| AccountAvailability::Banned
| AccountAvailability::Unknown
)
{
AccountStatus::Error
} else if self.rate_limited_until.is_some() {
AccountStatus::RateLimited
} else if self.quota_limit_reached {
AccountStatus::QuotaExhausted
} else {
AccountStatus::Normal
}
}
}
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
///
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountErrorReason {
Expired,
TokenExpired,
Invalid,
Banned,
Unknown,
}
impl AccountErrorReason {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Expired => "expired",
Self::TokenExpired => "token_expired",
Self::Invalid => "invalid",
Self::Banned => "banned",
Self::Unknown => "unknown",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"expired" => Some(Self::Expired),
"token_expired" => Some(Self::TokenExpired),
"invalid" => Some(Self::Invalid),
"banned" => Some(Self::Banned),
"unknown" => Some(Self::Unknown),
_ => None,
}
}
}
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
///
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
#[must_use]
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
if !signals.enabled {
return None;
}
match signals.availability {
AccountAvailability::QuotaExhausted => None,
AccountAvailability::Ready => signals
.access_token_expired
.then_some(AccountErrorReason::TokenExpired),
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
}
}
pub use gateway_core::engine::credential::{
AccountErrorReason, AccountStatus, AccountStatusFacts, AccountStatusProjection,
CredentialState, QuotaAccessState, QuotaEvidence, QuotaState, resolve_account_status,
};
/// 账号列表排序字段。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -207,11 +66,11 @@ pub struct AccountRecord {
pub access_token_expires_at: Option<DateTime<Utc>>,
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
pub quota: QuotaState,
pub last_error_reason: Option<AccountErrorReason>,
pub last_error_message: Option<String>,
pub quota_observed_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -296,11 +155,18 @@ pub struct AccountUsageWindowResult {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountPage {
pub config_revision: Revision,
pub items: Vec<AccountRecord>,
pub items: Vec<AccountPageItem>,
pub total: u64,
pub summary: AccountSummary,
}
/// 同一状态快照下的账号事实与唯一状态投影。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountPageItem {
pub account: AccountRecord,
pub projection: AccountStatusProjection,
}
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
///
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
@@ -342,9 +208,7 @@ pub enum AccountConnectionTestEvent {
Content {
text: String,
},
Completed {
account_status: AccountStatus,
},
Completed,
Failed {
message: String,
provider_error_code: Option<String>,
@@ -352,7 +216,6 @@ pub enum AccountConnectionTestEvent {
upstream_status: Option<u16>,
upstream_content_type: Option<String>,
upstream_body: Option<String>,
account_status: AccountStatus,
},
}
@@ -439,25 +439,15 @@ pub struct RequestMetricPoint {
pub costs: Vec<CurrencyCost>,
}
/// 账号池容量统计。
///
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
///
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
/// 以便状态详情展示和诊断。
/// 账号池的统一五态统计。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AccountPoolMetrics {
pub total: u64,
pub enabled: u64,
pub unavailable: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub refreshing: Option<u64>,
pub rate_limited: u64,
pub disabled: u64,
pub banned: u64,
pub error: u64,
}
/// Dashboard 中一个账号的模型级用量事实。
@@ -488,7 +478,7 @@ pub struct DashboardAccountRequestBucket {
pub request_count: u64,
}
/// Dashboard 中一个账号的完整用量与公共状态事实。
/// Dashboard 中一个账号的完整用量事实。
#[derive(Debug, Clone, PartialEq)]
pub struct DashboardAccountUsage {
pub account_id: String,
@@ -497,8 +487,6 @@ pub struct DashboardAccountUsage {
pub name: String,
pub email: Option<String>,
pub plan_type: Option<String>,
pub enabled: bool,
pub availability: String,
pub request_count: u64,
pub success_count: u64,
pub input_tokens: Option<u64>,
@@ -11,10 +11,7 @@ use gateway_core::{
use super::{
AdminError, MutationActor, MutationContext, PageSize, Revision,
accounts::{
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
AccountUsage,
},
accounts::{AccountRecord, AccountSummary, AccountUsage, CredentialState},
};
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
@@ -55,24 +52,24 @@ pub struct CredentialCursor {
/// Provider credential 列表查询。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CredentialListQuery {
pub availability: Option<CredentialAvailabilityFilter>,
pub credential_state: Option<CredentialStateFilter>,
pub enabled: Option<bool>,
pub window: CredentialListWindow,
}
/// Credential 可用性筛选;支持一个 wire 值对应多个持久状态。
/// Credential 状态筛选。
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CredentialAvailabilityFilter {
Exact(AccountAvailability),
AnyOf(Vec<AccountAvailability>),
pub enum CredentialStateFilter {
Exact(CredentialState),
AnyOf(Vec<CredentialState>),
}
impl CredentialAvailabilityFilter {
impl CredentialStateFilter {
#[must_use]
pub fn matches(&self, availability: AccountAvailability) -> bool {
pub fn matches(&self, credential_state: CredentialState) -> bool {
match self {
Self::Exact(expected) => *expected == availability,
Self::AnyOf(expected) => expected.contains(&availability),
Self::Exact(expected) => *expected == credential_state,
Self::AnyOf(expected) => expected.contains(&credential_state),
}
}
}
@@ -155,8 +152,8 @@ pub struct PreparedCredentialCreate {
pub access_token_expires_at: Option<DateTime<Utc>>,
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
}
/// Provider 对一份导入文档的完整验证结果。
@@ -603,15 +600,31 @@ pub struct ProviderQuota {
pub observed_at: Option<DateTime<Utc>>,
pub refresh_token_expires_at: Option<DateTime<Utc>>,
pub windows: Vec<ProviderQuotaWindow>,
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
/// 展示用快照级触顶事实(顶层或任一窗口触顶);不参与账号五态派生。
pub limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
pub rate_limited_until: Option<DateTime<Utc>>,
pub provider_data: Option<ProviderDocument>,
}
impl ProviderQuota {
/// 返回账号展开面板使用的当前额度窗口用量。
///
/// 只选择具备完整时间边界且可按账号归属的窗口;优先级与 Dashboard
/// 代表性额度窗口一致,同一优先级保持 Provider 投影顺序。
#[must_use]
pub fn representative_window_usage(&self) -> Option<&AccountUsage> {
self.windows
.iter()
.enumerate()
.filter(|(_, window)| {
window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
&& window.window_seconds.is_some()
&& window.reset_at.is_some()
&& window.local_usage.is_some()
})
.min_by_key(|(index, window)| (quota_usage_priority(window), *index))
.and_then(|(_, window)| window.local_usage.as_ref())
}
/// 返回 Dashboard 使用的代表性额度比例。
///
/// 优先使用短周期窗口,再依次使用周、月和其它窗口;同一优先级取较高的已用比例。
@@ -703,9 +716,7 @@ impl fmt::Debug for ProviderExport {
#[derive(Debug, Clone, PartialEq)]
pub struct AccountDirectoryItem {
pub account: AccountRecord,
pub status: AccountStatus,
/// `status == Error` 时的具体原因;其余状态为 `None`。
pub error_reason: Option<AccountErrorReason>,
pub projection: gateway_core::engine::credential::AccountStatusProjection,
pub usage: Option<super::accounts::AccountUsage>,
pub quota: ProviderQuota,
}
@@ -11,7 +11,7 @@ use super::backup::BackupStorePorts;
use crate::model::{
MutationContext, Revision,
accounts::{
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
AccountListQuery, AccountPage, AccountPageItem, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
},
auth::{AdminAuditEvent, AdminSession},
@@ -83,7 +83,7 @@ pub type AdminStoreResult<T> = Result<T, AdminStoreError>;
pub trait AccountStore: Send + Sync {
async fn list_accounts(&self, query: AccountListQuery) -> AdminStoreResult<AccountPage>;
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>>;
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>>;
async fn load_account_usage(
&self,
@@ -1,6 +1,6 @@
//! 统一账号目录与跨 Provider 动态分派。
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
use std::{collections::BTreeMap, sync::Arc};
use async_trait::async_trait;
use chrono::{Duration, Utc};
@@ -18,8 +18,7 @@ use crate::{
AdminError, MutationContext,
accounts::{
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
derive_error_reason,
AccountPageItem, AccountUsageWindowQuery,
},
observability::TimeRange,
provider_credentials::{
@@ -28,10 +27,7 @@ use crate::{
ProviderQuotaWindow, QuotaLocalUsageAttribution,
},
},
ports::{
provider::ProviderAdminRegistry,
store::{AccountStore, SettingsStore},
},
ports::{provider::ProviderAdminRegistry, store::AccountStore},
};
use super::{
@@ -79,7 +75,6 @@ pub trait AccountsService: Send + Sync {
pub(crate) struct DefaultAccountsService {
accounts: Arc<dyn AccountStore>,
settings: Arc<dyn SettingsStore>,
providers: ProviderAdminRegistry,
snapshot: Arc<dyn SnapshotControl>,
probe: Arc<dyn AccountProbe>,
@@ -89,14 +84,12 @@ impl DefaultAccountsService {
#[must_use]
pub(crate) const fn new(
accounts: Arc<dyn AccountStore>,
settings: Arc<dyn SettingsStore>,
providers: ProviderAdminRegistry,
snapshot: Arc<dyn SnapshotControl>,
probe: Arc<dyn AccountProbe>,
) -> Self {
Self {
accounts,
settings,
providers,
snapshot,
probe,
@@ -106,7 +99,7 @@ impl DefaultAccountsService {
async fn load_account(
&self,
account_id: &ProviderAccountId,
) -> Result<AccountRecord, AdminError> {
) -> Result<AccountPageItem, AdminError> {
self.accounts
.load_account(account_id.as_str())
.await
@@ -119,32 +112,32 @@ impl DefaultAccountsService {
account_id: &ProviderAccountId,
) -> Result<
(
AccountRecord,
AccountPageItem,
Arc<dyn crate::ports::provider::ProviderAdmin>,
),
AdminError,
> {
let account = self.load_account(account_id).await?;
let item = self.load_account(account_id).await?;
let provider = self
.providers
.require(&account.provider_kind)
.require(&item.account.provider_kind)
.map_err(|error| map_provider_error(error, "provider account"))?;
Ok((account, provider))
Ok((item, provider))
}
async fn attach_quota_local_usage(
&self,
accounts: &[AccountRecord],
accounts: &[AccountPageItem],
quotas: &mut [ProviderQuota],
) -> Result<(), AdminError> {
let windows = accounts
.iter()
.zip(quotas.iter())
.flat_map(|(account, quota)| {
.flat_map(|(item, quota)| {
quota
.windows
.iter()
.filter_map(|window| quota_usage_window(&account.id, window))
.filter_map(|window| quota_usage_window(&item.account.id, window))
})
.collect::<Vec<_>>();
if windows.is_empty() {
@@ -158,12 +151,12 @@ impl DefaultAccountsService {
.into_iter()
.map(|result| ((result.account_id, result.key), result.usage))
.collect::<BTreeMap<_, _>>();
for (account, quota) in accounts.iter().zip(quotas) {
for (item, quota) in accounts.iter().zip(quotas) {
for window in &mut quota.windows {
if window.local_usage.is_none()
&& window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
{
let key = (account.id.clone(), window.key.clone());
let key = (item.account.id.clone(), window.key.clone());
if let Some(usage) = usage_by_window.get(&key) {
window.local_usage = Some(usage.clone());
}
@@ -178,33 +171,19 @@ impl DefaultAccountsService {
account_id: &ProviderAccountId,
refresh_quota: bool,
) -> Result<AccountDirectoryItem, AdminError> {
let (account, provider) = self.provider_for_account(account_id).await?;
let settings = self
.settings
.load_runtime_settings()
.await
.map_err(|error| map_store_error(error, "runtime settings"))?;
let (stored, provider) = self.provider_for_account(account_id).await?;
let account = &stored.account;
let now = Utc::now();
let retained_range = retained_usage_range(now, settings.usage_retention_days);
let rolling_range = TimeRange {
start: now - Duration::hours(24),
end: now,
};
let ids = vec![account.id.clone()];
let (usage, rolling_usage) = futures::try_join!(
async {
self.accounts
.load_account_usage(retained_range, &ids)
.await
.map_err(|error| map_store_error(error, "account usage"))
},
async {
self.accounts
.load_account_usage(rolling_range, &ids)
.await
.map_err(|error| map_store_error(error, "rolling account usage"))
},
)?;
let rolling_usage = self
.accounts
.load_account_usage(rolling_range, &ids)
.await
.map_err(|error| map_store_error(error, "rolling account usage"))?;
let rolling_usage = rolling_usage.into_iter().next();
let mut quota = provider
.quota(ProviderQuotaRequest {
@@ -215,17 +194,15 @@ impl DefaultAccountsService {
.await
.map_err(|error| map_provider_error(error, "provider quota"))?;
self.attach_quota_local_usage(
std::slice::from_ref(&account),
std::slice::from_ref(&stored),
std::slice::from_mut(&mut quota),
)
.await?;
let signals = account_signals(&account, &quota, now);
let error_reason = derive_error_reason(&signals);
let usage = quota.representative_window_usage().cloned();
Ok(AccountDirectoryItem {
status: signals.derive(),
error_reason,
usage: usage.into_iter().next(),
account,
projection: stored.projection,
usage,
account: stored.account,
quota,
})
}
@@ -234,13 +211,12 @@ impl DefaultAccountsService {
#[async_trait]
impl AccountsService for DefaultAccountsService {
async fn list(&self, query: AccountListQuery) -> Result<AccountDirectoryPage, AdminError> {
let (page, settings) = futures::try_join!(
self.accounts.list_accounts(query),
self.settings.load_runtime_settings(),
)
.map_err(|error| map_store_error(error, "account directory"))?;
let page = self
.accounts
.list_accounts(query)
.await
.map_err(|error| map_store_error(error, "account directory"))?;
let now = Utc::now();
let retained_range = retained_usage_range(now, settings.usage_retention_days);
let rolling_range = TimeRange {
start: now - Duration::hours(24),
end: now,
@@ -248,22 +224,19 @@ impl AccountsService for DefaultAccountsService {
let ids = page
.items
.iter()
.map(|account| account.id.clone())
.map(|item| item.account.id.clone())
.collect::<Vec<_>>();
let (usage, rolling_usage) = futures::try_join!(
self.accounts.load_account_usage(retained_range, &ids),
self.accounts.load_account_usage(rolling_range, &ids),
)
.map_err(|error| map_store_error(error, "account usage"))?;
let usage = usage
.into_iter()
.map(|usage| (usage.account_id.clone(), usage))
.collect::<BTreeMap<_, _>>();
let rolling_usage = self
.accounts
.load_account_usage(rolling_range, &ids)
.await
.map_err(|error| map_store_error(error, "rolling account usage"))?;
let rolling_usage = rolling_usage
.into_iter()
.map(|usage| (usage.account_id.clone(), usage))
.collect::<BTreeMap<_, _>>();
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
let mut quotas = futures::future::join_all(page.items.iter().map(|item| async {
let account = &item.account;
let account_id = ProviderAccountId::new(account.id.clone())
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
@@ -307,14 +280,12 @@ impl AccountsService for DefaultAccountsService {
.items
.into_iter()
.zip(quotas)
.map(|(account, quota)| {
let signals = account_signals(&account, &quota, now);
let error_reason = derive_error_reason(&signals);
.map(|(item, quota)| {
let usage = quota.representative_window_usage().cloned();
AccountDirectoryItem {
status: signals.derive(),
error_reason,
usage: usage.get(&account.id).cloned(),
account,
usage,
account: item.account,
projection: item.projection,
quota,
}
})
@@ -342,7 +313,7 @@ impl AccountsService for DefaultAccountsService {
for account_id in account_ids {
let account = self.load_account(&account_id).await?;
grouped
.entry(account.provider_kind)
.entry(account.account.provider_kind)
.or_default()
.push(account_id);
}
@@ -385,7 +356,8 @@ impl AccountsService for DefaultAccountsService {
context: &MutationContext,
account_id: ProviderAccountId,
) -> Result<AccountRefreshResult, AdminError> {
let (account, provider) = self.provider_for_account(&account_id).await?;
let (stored, provider) = self.provider_for_account(&account_id).await?;
let account = stored.account;
let prepared = provider
.prepare_refresh(PrepareCredentialRefresh {
account: account.clone(),
@@ -436,8 +408,8 @@ impl AccountsService for DefaultAccountsService {
account_id: ProviderAccountId,
upstream_model: UpstreamModelId,
) -> Result<AccountConnectionTestEventStream, AdminError> {
let (account, provider) = self.provider_for_account(&account_id).await?;
let initial_status = account_status(&account, false, Utc::now());
let (stored, provider) = self.provider_for_account(&account_id).await?;
let account = stored.account;
let model = upstream_model.as_str().to_owned();
let operation = provider
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
@@ -454,8 +426,6 @@ impl AccountsService for DefaultAccountsService {
},
];
let probe = Arc::clone(&self.probe);
let accounts = Arc::clone(&self.accounts);
let stored_account_id = account.id.clone();
let terminal = futures::stream::once(async move {
let result = probe
.probe(AccountProbeRequest {
@@ -465,22 +435,12 @@ impl AccountsService for DefaultAccountsService {
operation,
})
.await;
let status = accounts
.load_account(&stored_account_id)
.await
.ok()
.flatten()
.map_or(initial_status, |account| {
account_status(&account, false, Utc::now())
});
match result {
Ok(result) => result
.text
.into_iter()
.map(|text| AccountConnectionTestEvent::Content { text })
.chain(std::iter::once(AccountConnectionTestEvent::Completed {
account_status: status,
}))
.chain(std::iter::once(AccountConnectionTestEvent::Completed))
.collect(),
Err(error) => {
let upstream_status = error
@@ -506,7 +466,6 @@ impl AccountsService for DefaultAccountsService {
upstream_status,
upstream_content_type,
upstream_body,
account_status: status,
}]
}
}
@@ -523,49 +482,10 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
/// 构造状态派生所需的信号快照。
fn account_signals(
account: &AccountRecord,
quota: &ProviderQuota,
now: chrono::DateTime<Utc>,
) -> AccountStatusSignals {
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached: quota.limit_reached,
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
}
}
/// 账号运营分类(互斥展示状态)。
fn account_status(
account: &AccountRecord,
quota_limit_reached: bool,
now: chrono::DateTime<Utc>,
) -> AccountStatus {
let quota = ProviderQuota {
limit_reached: quota_limit_reached,
..empty_quota()
};
account_signals(account, &quota, now).derive()
}
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
TimeRange {
start: now - Duration::days(i64::from(retention_days)),
end: now,
}
}
fn quota_usage_window(
account_id: &str,
window: &ProviderQuotaWindow,
@@ -127,7 +127,7 @@ impl ObservabilityService for DefaultObservabilityService {
let max_concurrent_per_account = u64::from(settings.max_concurrent_per_account);
let active_accounts = runtime_slots
.as_ref()
.map_or(observation.provider_accounts.active, |slots| {
.map_or(observation.provider_accounts.normal, |slots| {
slots.active_accounts
});
let total_slots = active_accounts.saturating_mul(max_concurrent_per_account);
@@ -484,8 +484,7 @@ impl DefaultObservabilityService {
});
let quota_used = futures::future::join_all(quota_reads).await;
for (account, used_percent) in accounts.iter_mut().zip(quota_used) {
account.quota_used_percent = used_percent
.or_else(|| (account.availability == "quota_exhausted").then_some(100.0));
account.quota_used_percent = used_percent;
}
}
@@ -129,7 +129,6 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
quota_window("shortTerm", Some(14_400), Some(45.0)),
],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
};
@@ -5,7 +5,10 @@ use chrono::{TimeDelta, Utc};
use futures::{StreamExt as _, future::BoxFuture};
use gateway_core::{
engine::{
credential::{OpaqueProviderData, ProviderAccountId},
credential::{
AccountStatusFacts, CredentialState, OpaqueProviderData, ProviderAccountId,
QuotaEvidence, QuotaState, resolve_account_status,
},
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
},
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
@@ -18,7 +21,7 @@ use gateway_admin::{
model::{
AdminError, MutationContext, Revision,
accounts::{
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
AccountConnectionTestEvent, AccountListQuery, AccountPage, AccountPageItem,
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
},
@@ -378,6 +381,7 @@ pub(super) struct FakeAccountStore {
fail_commit: Mutex<bool>,
audit_requests: Mutex<Vec<String>>,
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
}
impl FakeAccountStore {
@@ -393,6 +397,7 @@ impl FakeAccountStore {
fail_commit: Mutex::new(false),
audit_requests: Mutex::new(Vec::new()),
quota_window_usage: Mutex::new(Vec::new()),
quota_window_queries: Mutex::new(Vec::new()),
})
}
@@ -408,6 +413,13 @@ impl FakeAccountStore {
*self.quota_window_usage.lock().expect("quota window usage") = usage;
}
fn quota_window_queries(&self) -> Vec<AccountUsageWindowQuery> {
self.quota_window_queries
.lock()
.expect("quota window queries")
.clone()
}
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
*self.accounts.lock().expect("accounts") = accounts;
}
@@ -438,6 +450,22 @@ impl FakeAccountStore {
.expect("audit requests")
.push(context.request_id.clone());
}
fn page_item(account: AccountRecord) -> AccountPageItem {
let facts = AccountStatusFacts {
enabled: account.enabled,
credential_state: account.credential_state,
access_token_expires_at: account.access_token_expires_at.map(Into::into),
quota: account.quota,
rate_limited_until: None,
last_error_reason: account.last_error_reason,
last_error_message: account.last_error_message.clone(),
};
AccountPageItem {
account,
projection: resolve_account_status(&facts, std::time::SystemTime::now()),
}
}
}
#[async_trait]
@@ -445,12 +473,13 @@ impl AccountStore for FakeAccountStore {
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
self.record("store.list_accounts");
let accounts = self.accounts.lock().expect("accounts").clone();
let total = accounts.len() as u64;
Ok(AccountPage {
config_revision: revision(1),
items: accounts.clone(),
total: accounts.len() as u64,
items: accounts.into_iter().map(Self::page_item).collect(),
total,
summary: AccountSummary {
total: accounts.len() as u64,
total,
normal: 1,
quota_exhausted: 0,
rate_limited: 0,
@@ -460,7 +489,7 @@ impl AccountStore for FakeAccountStore {
})
}
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>> {
self.record("store.load_account");
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
let account = self
@@ -477,7 +506,7 @@ impl AccountStore for FakeAccountStore {
.find(|account| account.id == account_id)
.cloned()
});
Ok(account)
Ok(account.map(Self::page_item))
}
async fn load_account_usage(
@@ -490,8 +519,12 @@ impl AccountStore for FakeAccountStore {
async fn load_account_usage_by_windows(
&self,
_: &[AccountUsageWindowQuery],
windows: &[AccountUsageWindowQuery],
) -> AdminStoreResult<Vec<AccountUsageWindowResult>> {
*self
.quota_window_queries
.lock()
.expect("quota window queries") = windows.to_vec();
Ok(self
.quota_window_usage
.lock()
@@ -815,7 +848,11 @@ fn provider_registry_should_reject_duplicate_kind() {
async fn connection_test_should_probe_unavailable_account() {
let provider = FakeProviderAdmin::new("xai", events());
let mut account = account_record("xai");
account.availability = AccountAvailability::QuotaExhausted;
account.quota = QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
std::time::SystemTime::now(),
None,
);
let store = FakeAccountStore::with_account(account, events());
let services =
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
@@ -837,7 +874,6 @@ async fn connection_test_should_probe_unavailable_account() {
message,
provider_error_code: Some(code),
provider_error_type: Some(error_type),
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
..
}) if message == "included usage exhausted"
&& code == "usage_exhausted"
@@ -846,7 +882,7 @@ async fn connection_test_should_probe_unavailable_account() {
}
#[tokio::test]
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
async fn connection_test_rate_limited_probe_returns_provider_failure() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events);
@@ -869,9 +905,10 @@ async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
message,
..
})
if message == "xAI free model quota is exhausted"
));
}
@@ -897,10 +934,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
..
})
Some(AccountConnectionTestEvent::Failed { .. })
));
}
@@ -949,7 +983,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
assert_eq!(result.config_revision, revision(2));
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
assert_eq!(
result.account.status,
result.account.projection.status,
gateway_admin::model::accounts::AccountStatus::Normal
);
@@ -990,7 +1024,7 @@ async fn accounts_list_should_return_complete_directory_semantics() {
let account = page.items.first().expect("account item");
assert_eq!(account.account.provider_kind.as_str(), "openai");
assert_eq!(
account.status,
account.projection.status,
gateway_admin::model::accounts::AccountStatus::Normal
);
}
@@ -1016,7 +1050,6 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
provider_data: None,
}],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let failing = FakeProviderAdmin::new("xai", events.clone());
@@ -1064,10 +1097,14 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
}
#[tokio::test]
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
async fn accounts_list_should_prefer_credential_error_over_quota_exhaustion() {
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::QuotaExhausted;
account.quota = QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
std::time::SystemTime::now(),
None,
);
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
let store = FakeAccountStore::with_account(account, events());
@@ -1086,17 +1123,17 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
.expect("usage-limited account directory");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Error,
);
}
#[tokio::test]
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
async fn accounts_list_should_map_unknown_credential_to_error_not_normal() {
// Unknown 不可调度,Admin 不得显示为 normal。
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::Unknown;
account.credential_state = CredentialState::Unknown;
let store = FakeAccountStore::with_account(account, events());
let page = accounts_service(provider, store)
@@ -1114,15 +1151,13 @@ async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
.expect("unknown account directory");
assert_eq!(
page.items.first().expect("account item").status,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Error,
);
}
#[tokio::test]
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
async fn accounts_list_should_not_derive_rate_limited_from_provider_quota_view() {
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
@@ -1141,7 +1176,6 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
provider_data: None,
}],
limit_reached: true,
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1158,17 +1192,16 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
sort: None,
})
.await
.expect("rate limited account directory");
.expect("account directory with display quota cooldown");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::RateLimited,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Normal,
);
}
#[tokio::test]
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
async fn accounts_list_should_not_derive_exhaustion_from_provider_quota_view() {
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
@@ -1187,7 +1220,6 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
provider_data: None,
}],
limit_reached: true,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1204,11 +1236,11 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
sort: None,
})
.await
.expect("quota exhausted account directory");
.expect("account directory with display quota limit");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
page.items.first().expect("account item").projection.status,
gateway_admin::model::accounts::AccountStatus::Normal,
);
}
@@ -1233,7 +1265,6 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
provider_data: None,
}],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1243,7 +1274,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
usage: quota_local_usage("acct_test", 4_330_000),
}]);
let page = accounts_service(provider, store)
let page = accounts_service(provider, store.clone())
.await
.accounts()
.list(AccountListQuery {
@@ -1257,11 +1288,23 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
.await
.expect("quota window usage");
let usage = page.items[0].quota.windows[0]
let item = &page.items[0];
let usage = item.quota.windows[0]
.local_usage
.as_ref()
.expect("quota window local usage");
assert_eq!(usage.total_tokens, Some(4_330_000));
assert_eq!(
item.usage.as_ref().and_then(|usage| usage.total_tokens),
Some(4_330_000),
);
let queries = store.quota_window_queries();
assert_eq!(queries.len(), 1);
assert_eq!(queries[0].account_id, "acct_test");
assert_eq!(queries[0].key, "primary");
assert_eq!(queries[0].range.start, reset_at - TimeDelta::hours(5));
assert_eq!(queries[0].range.end, reset_at);
}
#[tokio::test]
@@ -1300,7 +1343,6 @@ async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_w
},
],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1352,6 +1394,13 @@ async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_w
("codex-bengalfox-primary", None),
],
);
assert_eq!(
page.items[0]
.usage
.as_ref()
.and_then(|usage| usage.total_tokens),
Some(12_818_806),
);
}
#[tokio::test]
@@ -1432,7 +1481,6 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
@@ -1476,10 +1524,11 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
access_token_expires_at: Some(now + TimeDelta::hours(1)),
next_refresh_at: Some(now + TimeDelta::minutes(30)),
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: now,
credential_state: CredentialState::Ready,
credential_observed_at: now,
quota: QuotaState::allowed(now.into()),
last_error_reason: None,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
}
@@ -1513,8 +1562,8 @@ fn prepared_create_with_id(
access_token_expires_at: Some(now + TimeDelta::hours(1)),
next_refresh_at: Some(now + TimeDelta::minutes(30)),
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: now,
credential_state: CredentialState::Ready,
credential_observed_at: now,
}
}
@@ -1597,9 +1646,7 @@ impl AccountProbe for FreeModelQuotaProbe {
_: AccountProbeRequest,
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
Box::pin(async move {
let mut account = account_record("xai");
account.availability = AccountAvailability::Ready;
self.store.set_account_after_probe(account);
self.store.set_account_after_probe(account_record("xai"));
Err(GatewayError::new(
GatewayErrorKind::RateLimited,
"xAI free model quota is exhausted",
@@ -20,7 +20,7 @@ use gateway_admin::{
model::{
MutationContext, Revision,
accounts::{
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
AccountListQuery, AccountPage, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
},
auth::{AdminAuditEvent, AdminSession},
@@ -240,7 +240,10 @@ impl AccountStore for UnavailableStore {
Err(unavailable("accounts"))
}
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
async fn load_account(
&self,
_: &str,
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountPageItem>> {
Err(unavailable("account"))
}
@@ -37,7 +37,7 @@ use gateway_core::{
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use super::presenter::{format_compact_number, format_number};
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError,
@@ -185,12 +185,10 @@ pub struct AccountView {
pub authentication_kind: String,
pub has_refresh_token: bool,
pub status: String,
pub display_status: String,
/// `status == "error"` 时的具体原因;其余状态为 `null`。
pub error_reason: Option<String>,
/// 最近一次失败的上游错误描述;仅错误状态存在。
pub error_message: Option<String>,
pub availability: String,
pub enabled: bool,
pub access_token_expires_at: Option<String>,
pub access_token_expires_at_display: Option<String>,
@@ -522,10 +520,9 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
DomainConnectionTestEvent::Content { text } => {
serde_json::json!({ "type": "content", "text": text })
}
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
DomainConnectionTestEvent::Completed => serde_json::json!({
"type": "test_complete",
"success": true,
"accountStatus": account_status.as_str()
"success": true
}),
DomainConnectionTestEvent::Failed {
message,
@@ -534,7 +531,6 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
upstream_status,
upstream_content_type,
upstream_body,
account_status,
} => serde_json::json!({
"type": "error",
"error": message,
@@ -542,8 +538,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
"providerErrorType": provider_error_type,
"upstreamStatus": upstream_status,
"upstreamContentType": upstream_content_type,
"upstreamBody": upstream_body,
"accountStatus": account_status.as_str()
"upstreamBody": upstream_body
}),
};
Self { data }
@@ -1365,16 +1360,16 @@ fn account_models_data(result: ProviderModels) -> AccountModelsData {
fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
let AccountDirectoryItem {
account,
status,
error_reason,
projection,
usage,
quota,
} = item;
let status = status.as_str().to_owned();
let status = projection.status.as_str().to_owned();
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
let added_at = china_rfc3339(&account.created_at);
let updated_at = china_rfc3339(&account.updated_at);
let (quota, refresh_token_expires_at) = account_quota_view(quota, now);
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
AccountView {
id: account.id.clone(),
name: account.name,
@@ -1387,11 +1382,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
plan_type: account.plan_type,
authentication_kind: account.authentication_kind,
has_refresh_token: account.has_refresh_token,
status: status.clone(),
display_status: status,
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
error_message: account.last_error_message,
availability: account.availability.as_str().to_owned(),
status,
error_reason: projection
.error_reason
.map(|reason| reason.as_str().to_owned()),
error_message: projection.error_message,
enabled: account.enabled,
access_token_expires_at: expires_at,
access_token_expires_at_display: account
@@ -1411,6 +1406,7 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
fn account_quota_view(
quota: ProviderQuota,
rate_limited_until: Option<DateTime<Utc>>,
now: DateTime<Utc>,
) -> (AccountQuotaView, Option<String>) {
let refresh_token_expires_at = quota
@@ -1420,7 +1416,7 @@ fn account_quota_view(
.observed_at
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
let windows = quota.windows.into_iter().map(quota_window_view).collect();
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
(
AccountQuotaView {
refreshed_at_display,
@@ -1590,7 +1586,7 @@ fn account_model_usage_view(usage: AccountModelUsage, now: DateTime<Utc>) -> Mod
billing_amount_usd: usd.map(|cost| cost.amount.as_str().to_owned()),
billing_amount_usd_display: usd.map_or_else(
|| "—".to_owned(),
|cost| format!("${}", cost.amount.as_str()),
|cost| format_decimal_currency(cost.amount.as_str(), "USD"),
),
cost_estimate_status: cost_estimate_status.to_owned(),
known_cost_count: known_count,
@@ -15,7 +15,7 @@ use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::presenter::{format_compact_number, format_number};
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
use super::{
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
WireValidationError, wire::map_admin_service_error,
@@ -127,20 +127,6 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
}
}
pub(crate) fn format_decimal_currency(amount: &str, currency: &str) -> String {
if currency == "USD" {
let value = amount.parse::<f64>().map_or(0.0, |value| value);
let precision = if value != 0.0 && value.abs() < 0.01 {
4
} else {
2
};
format!("${value:.precision$}")
} else {
format!("{currency} {amount}")
}
}
pub(crate) fn format_money(cost: &domain::CurrencyCost) -> String {
format_decimal_currency(cost.amount.as_str(), &cost.currency)
}
@@ -747,10 +733,16 @@ pub(crate) fn dashboard_view(
credentials: DashboardCredentialsCardView {
total: format_compact_number(provider_accounts.total),
total_value: provider_accounts.total,
available: format_compact_number(provider_accounts.active),
available_value: provider_accounts.active,
unavailable: format_compact_number(provider_accounts.unavailable),
unavailable_value: provider_accounts.unavailable,
available: format_compact_number(provider_accounts.normal),
available_value: provider_accounts.normal,
unavailable: format_compact_number(
provider_accounts
.total
.saturating_sub(provider_accounts.normal),
),
unavailable_value: provider_accounts
.total
.saturating_sub(provider_accounts.normal),
},
traffic: DashboardTrafficCardView {
today_requests: format_compact_number(today.request_count),
@@ -783,14 +775,11 @@ pub(crate) fn dashboard_view(
usage_records: recent_requests.into_iter().map(usage_record_view).collect(),
pool_summary: DashboardPoolSummaryView {
total: provider_accounts.total,
active: provider_accounts.active,
rate_limited: provider_accounts.rate_limited,
expired: provider_accounts.expired,
invalid: provider_accounts.invalid,
normal: provider_accounts.normal,
quota_exhausted: provider_accounts.quota_exhausted,
refreshing: provider_accounts.refreshing,
rate_limited: provider_accounts.rate_limited,
disabled: provider_accounts.disabled,
banned: provider_accounts.banned,
error: provider_accounts.error,
},
capacity_info: DashboardCapacityInfoView {
max_concurrent_per_account: capacity.max_concurrent_per_account,
@@ -347,14 +347,11 @@ pub struct DashboardAccountRequestBucketView {
#[serde(rename_all = "camelCase")]
pub struct DashboardPoolSummaryView {
pub total: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub refreshing: Option<u64>,
pub rate_limited: u64,
pub disabled: u64,
pub banned: u64,
pub error: u64,
}
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
@@ -3,6 +3,8 @@
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
use gateway_core::accounting::Decimal;
/// 千分位格式化。
#[must_use]
pub fn format_number(value: u64) -> String {
@@ -37,3 +39,33 @@ pub fn format_compact_number(value: u64) -> String {
}
format_number(value)
}
/// 货币展示格式化。USD 常规金额保留两位,小于 1 美元时最多保留四位。
#[must_use]
pub fn format_decimal_currency(amount: &str, currency: &str) -> String {
if currency != "USD" {
return format!("{currency} {amount}");
}
const DECIMAL_SCALE: u128 = 10_000_000_000;
let scaled = amount.parse::<Decimal>().unwrap_or_default().scaled();
let precision = if scaled != 0 && scaled < DECIMAL_SCALE {
4_u32
} else {
2_u32
};
let rounding_unit = 10_u128.pow(10 - precision);
let rounded = (scaled + rounding_unit / 2) / rounding_unit;
let display_scale = 10_u128.pow(precision);
let whole = rounded / display_scale;
let fraction = rounded % display_scale;
let mut display = format!("{whole}.{fraction:0width$}", width = precision as usize);
while display.ends_with('0')
&& display
.split_once('.')
.is_some_and(|(_, fraction)| fraction.len() > 2)
{
display.pop();
}
format!("${display}")
}
@@ -72,6 +72,7 @@ mod query {
mod response {
use gateway_api::admin::accounts::AccountUsageView;
use gateway_api::admin::presenter::format_decimal_currency;
#[test]
fn account_usage_view_should_keep_unobserved_numbers_null() {
@@ -112,12 +113,21 @@ mod response {
assert!(value["totalTokens"].is_null());
assert_eq!(value["createdTokensDisplay"], "-");
}
#[test]
fn account_usage_currency_should_limit_usd_to_four_fraction_digits() {
assert_eq!(format_decimal_currency("0.1204956", "USD"), "$0.1205");
assert_eq!(format_decimal_currency("0.99996", "USD"), "$1.00");
assert_eq!(format_decimal_currency("0.0300", "USD"), "$0.03");
assert_eq!(format_decimal_currency("12.3456", "USD"), "$12.35");
assert_eq!(format_decimal_currency("0.1204956", "CNY"), "CNY 0.1204956");
}
}
mod actions {
use gateway_admin::model::{
Revision,
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
accounts::AccountConnectionTestEvent as DomainConnectionTestEvent,
provider_credentials::{
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
},
@@ -343,9 +353,7 @@ mod actions {
DomainConnectionTestEvent::Content {
text: "OK".to_owned(),
},
DomainConnectionTestEvent::Completed {
account_status: AccountStatus::Normal,
},
DomainConnectionTestEvent::Completed {},
DomainConnectionTestEvent::Failed {
message: "upstream unavailable".to_owned(),
provider_error_code: Some("usage_exhausted".to_owned()),
@@ -353,7 +361,6 @@ mod actions {
upstream_status: Some(429),
upstream_content_type: Some("application/json".to_owned()),
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
account_status: AccountStatus::QuotaExhausted,
},
]
.map(|event| AccountConnectionTestEvent::from(event).data);
@@ -378,7 +385,7 @@ mod actions {
}
}),
json!({ "type": "content", "text": "OK" }),
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
json!({ "type": "test_complete", "success": true }),
json!({
"type": "error",
"error": "upstream unavailable",
@@ -386,8 +393,7 @@ mod actions {
"providerErrorType": "invalid_request_error",
"upstreamStatus": 429,
"upstreamContentType": "application/json",
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
"accountStatus": "quota_exhausted"
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#
}),
]
);
@@ -14,7 +14,7 @@ use gateway_admin::{
model::{
MutationContext, Revision,
accounts::{
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
AccountListQuery, AccountPage, AccountPageItem, AccountUsage, AccountUsageWindowQuery,
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
},
auth::{AdminAuditEvent, AdminSession},
@@ -426,7 +426,7 @@ impl AccountStore for UnusedStore {
Err(unavailable("account list"))
}
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountPageItem>> {
Err(unavailable("account"))
}
@@ -222,16 +222,11 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
totals: Default::default(),
provider_accounts: AccountPoolMetrics {
total: 807,
enabled: 804,
unavailable: 792,
active: 15,
rate_limited: 0,
expired: 2,
invalid: 0,
normal: 15,
quota_exhausted: 785,
refreshing: None,
rate_limited: 0,
disabled: 3,
banned: 2,
error: 4,
},
trend: Vec::new(),
account_usage: Vec::new(),
@@ -187,45 +187,41 @@ impl fmt::Debug for OpaqueProviderData {
}
}
/// 数据库中固定的账号运行状态。
/// 已持久化的凭据/账号身份状态。
///
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
/// 这里只保存凭据与账号身份事实。额度耗尽属于 [`QuotaState`],临时 429 属于
/// 运行时冷却;二者都不得写入此枚举。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AccountAvailability {
pub enum CredentialState {
Unknown,
Ready,
QuotaExhausted,
Expired,
Banned,
Invalid,
}
impl AccountAvailability {
impl CredentialState {
/// 返回数据库稳定值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::Ready => "ready",
Self::QuotaExhausted => "quota_exhausted",
Self::Expired => "expired",
Self::Banned => "banned",
Self::Invalid => "invalid",
}
}
/// 返回账号进入不可恢复错误态、但 Provider 未给出更具体上游原因时的稳定兜底。
///
/// Provider 应优先持久化已脱敏的上游 message 或原因码;该值只避免错误态沿用
/// 上一次无关的错误信息。
/// 返回该凭据状态对应的稳定错误原因。
#[must_use]
pub const fn fallback_error_reason(self) -> Option<&'static str> {
pub const fn error_reason(self) -> Option<AccountErrorReason> {
match self {
Self::Expired => Some("credential_expired"),
Self::Banned => Some("account_banned"),
Self::Invalid => Some("credential_invalid"),
Self::Unknown | Self::Ready | Self::QuotaExhausted => None,
Self::Unknown => Some(AccountErrorReason::AccountUnverified),
Self::Expired => Some(AccountErrorReason::CredentialExpired),
Self::Banned => Some(AccountErrorReason::AccountBanned),
Self::Invalid => Some(AccountErrorReason::CredentialInvalid),
Self::Ready => None,
}
}
@@ -235,7 +231,6 @@ impl AccountAvailability {
match value {
"unknown" => Some(Self::Unknown),
"ready" => Some(Self::Ready),
"quota_exhausted" => Some(Self::QuotaExhausted),
"expired" => Some(Self::Expired),
"banned" => Some(Self::Banned),
"invalid" => Some(Self::Invalid),
@@ -244,6 +239,373 @@ impl AccountAvailability {
}
}
/// 额度访问结论;百分比和余额等展示值不进入此枚举。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum QuotaAccessState {
Unknown,
Allowed,
Exhausted,
}
impl QuotaAccessState {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::Allowed => "allowed",
Self::Exhausted => "exhausted",
}
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"unknown" => Some(Self::Unknown),
"allowed" => Some(Self::Allowed),
"exhausted" => Some(Self::Exhausted),
_ => None,
}
}
}
/// 权威额度耗尽证据。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum QuotaEvidence {
ProviderDenied,
AccountLimitReached,
UsageLimitReached,
PaymentRequired,
}
impl QuotaEvidence {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::ProviderDenied => "provider_denied",
Self::AccountLimitReached => "account_limit_reached",
Self::UsageLimitReached => "usage_limit_reached",
Self::PaymentRequired => "payment_required",
}
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"provider_denied" => Some(Self::ProviderDenied),
"account_limit_reached" => Some(Self::AccountLimitReached),
"usage_limit_reached" => Some(Self::UsageLimitReached),
"payment_required" => Some(Self::PaymentRequired),
_ => None,
}
}
}
/// 可审计的额度访问事实。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct QuotaState {
access: QuotaAccessState,
evidence: Option<QuotaEvidence>,
observed_at: Option<SystemTime>,
reset_at: Option<SystemTime>,
}
impl Default for QuotaState {
fn default() -> Self {
Self::unknown()
}
}
impl QuotaState {
#[must_use]
pub const fn unknown() -> Self {
Self {
access: QuotaAccessState::Unknown,
evidence: None,
observed_at: None,
reset_at: None,
}
}
#[must_use]
pub const fn observed_unknown(observed_at: SystemTime) -> Self {
Self {
access: QuotaAccessState::Unknown,
evidence: None,
observed_at: Some(observed_at),
reset_at: None,
}
}
#[must_use]
pub const fn allowed(observed_at: SystemTime) -> Self {
Self {
access: QuotaAccessState::Allowed,
evidence: None,
observed_at: Some(observed_at),
reset_at: None,
}
}
#[must_use]
pub const fn exhausted(
evidence: QuotaEvidence,
observed_at: SystemTime,
reset_at: Option<SystemTime>,
) -> Self {
Self {
access: QuotaAccessState::Exhausted,
evidence: Some(evidence),
observed_at: Some(observed_at),
reset_at,
}
}
/// 从持久化列恢复额度状态;非法组合返回 `None`。
#[must_use]
pub const fn from_persisted(
access: QuotaAccessState,
evidence: Option<QuotaEvidence>,
observed_at: Option<SystemTime>,
reset_at: Option<SystemTime>,
) -> Option<Self> {
match access {
QuotaAccessState::Unknown if evidence.is_none() && reset_at.is_none() => Some(Self {
access,
evidence,
observed_at,
reset_at,
}),
QuotaAccessState::Allowed
if evidence.is_none() && observed_at.is_some() && reset_at.is_none() =>
{
Some(Self {
access,
evidence,
observed_at,
reset_at,
})
}
QuotaAccessState::Exhausted if evidence.is_some() && observed_at.is_some() => {
Some(Self {
access,
evidence,
observed_at,
reset_at,
})
}
_ => None,
}
}
#[must_use]
pub const fn access(self) -> QuotaAccessState {
self.access
}
#[must_use]
pub const fn evidence(self) -> Option<QuotaEvidence> {
self.evidence
}
#[must_use]
pub const fn observed_at(self) -> Option<SystemTime> {
self.observed_at
}
#[must_use]
pub const fn reset_at(self) -> Option<SystemTime> {
self.reset_at
}
/// 返回当前已确认的额度耗尽结论。
///
/// `reset_at` 只表示何时应重新向 Provider 求证;时间到期本身不是额度恢复证据。
#[must_use]
pub fn is_exhausted(self) -> bool {
self.access == QuotaAccessState::Exhausted
}
/// 判断已耗尽额度是否到达 Provider 的下一次复核时间。
///
/// 有明确 `reset_at` 时严格等待该时刻;没有时由 Provider 传入自己的保守复核周期。
#[must_use]
pub fn exhaustion_refresh_due(self, now: SystemTime, fallback_interval: Duration) -> bool {
if !self.is_exhausted() {
return false;
}
let due_at = self.reset_at.or_else(|| {
self.observed_at
.and_then(|observed_at| observed_at.checked_add(fallback_interval))
});
due_at.is_none_or(|due_at| due_at <= now)
}
/// 合并一次额度访问观察;`Unknown` 不能擦除已经确认的访问结论。
#[must_use]
pub fn merge_observation(self, observation: Self) -> Self {
if observation.access == QuotaAccessState::Unknown
&& self.access != QuotaAccessState::Unknown
{
self
} else {
observation
}
}
}
/// 对外唯一的五态账号状态。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AccountStatus {
Normal,
QuotaExhausted,
RateLimited,
Disabled,
Error,
}
impl AccountStatus {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Normal => "normal",
Self::QuotaExhausted => "quota_exhausted",
Self::RateLimited => "rate_limited",
Self::Disabled => "disabled",
Self::Error => "error",
}
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"normal" => Some(Self::Normal),
"quota_exhausted" => Some(Self::QuotaExhausted),
"rate_limited" => Some(Self::RateLimited),
"disabled" => Some(Self::Disabled),
"error" => Some(Self::Error),
_ => None,
}
}
/// 管理页稳定业务排序:正常、限流、耗尽、错误、停用。
#[must_use]
pub const fn sort_rank(self) -> u8 {
match self {
Self::Normal => 0,
Self::RateLimited => 1,
Self::QuotaExhausted => 2,
Self::Error => 3,
Self::Disabled => 4,
}
}
}
/// `error` 状态下的稳定原因码。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AccountErrorReason {
AccountUnverified,
AccessTokenExpired,
CredentialExpired,
CredentialInvalid,
AccountBanned,
}
impl AccountErrorReason {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::AccountUnverified => "account_unverified",
Self::AccessTokenExpired => "access_token_expired",
Self::CredentialExpired => "credential_expired",
Self::CredentialInvalid => "credential_invalid",
Self::AccountBanned => "account_banned",
}
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"account_unverified" => Some(Self::AccountUnverified),
"access_token_expired" => Some(Self::AccessTokenExpired),
"credential_expired" => Some(Self::CredentialExpired),
"credential_invalid" => Some(Self::CredentialInvalid),
"account_banned" => Some(Self::AccountBanned),
_ => None,
}
}
}
/// 唯一状态解析器的完整输入事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusFacts {
pub enabled: bool,
pub credential_state: CredentialState,
pub access_token_expires_at: Option<SystemTime>,
pub quota: QuotaState,
pub rate_limited_until: Option<SystemTime>,
pub last_error_reason: Option<AccountErrorReason>,
pub last_error_message: Option<String>,
}
/// 唯一状态解析器的输出。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusProjection {
pub status: AccountStatus,
pub error_reason: Option<AccountErrorReason>,
pub error_message: Option<String>,
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。
pub rate_limited_until: Option<SystemTime>,
}
/// 从独立事实派生唯一、互斥的对外状态。
#[must_use]
pub fn resolve_account_status(
facts: &AccountStatusFacts,
now: SystemTime,
) -> AccountStatusProjection {
if !facts.enabled {
return status_projection(AccountStatus::Disabled);
}
let credential_error = facts.credential_state.error_reason().or_else(|| {
facts
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now)
.then_some(AccountErrorReason::AccessTokenExpired)
});
if let Some(default_reason) = credential_error {
return AccountStatusProjection {
status: AccountStatus::Error,
error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)),
error_message: facts.last_error_message.clone(),
rate_limited_until: None,
};
}
if facts.quota.is_exhausted() {
return status_projection(AccountStatus::QuotaExhausted);
}
if facts
.rate_limited_until
.is_some_and(|rate_limited_until| rate_limited_until > now)
{
return AccountStatusProjection {
status: AccountStatus::RateLimited,
error_reason: None,
error_message: None,
rate_limited_until: facts.rate_limited_until,
};
}
status_projection(AccountStatus::Normal)
}
const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
AccountStatusProjection {
status,
error_reason: None,
error_message: None,
rate_limited_until: None,
}
}
/// 不含 secret 的账号持久事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccount {
@@ -257,7 +619,10 @@ pub struct ProviderAccount {
authentication_kind: String,
revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
quota: QuotaState,
last_error_reason: Option<AccountErrorReason>,
last_error_message: Option<String>,
access_token_expires_at: Option<SystemTime>,
next_refresh_at: Option<SystemTime>,
has_refresh_token: bool,
@@ -286,7 +651,10 @@ impl ProviderAccount {
authentication_kind,
revision,
enabled: true,
availability: AccountAvailability::Unknown,
credential_state: CredentialState::Unknown,
quota: QuotaState::unknown(),
last_error_reason: None,
last_error_message: None,
access_token_expires_at,
next_refresh_at: None,
has_refresh_token: false,
@@ -307,17 +675,23 @@ impl ProviderAccount {
}
#[must_use]
pub const fn with_runtime_state(
pub fn with_account_facts(
mut self,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
quota: QuotaState,
last_error_reason: Option<AccountErrorReason>,
last_error_message: Option<String>,
) -> Self {
self.enabled = enabled;
self.availability = if self.upstream_user_id.is_some() {
availability
self.credential_state = if self.upstream_user_id.is_some() {
credential_state
} else {
AccountAvailability::Unknown
CredentialState::Unknown
};
self.quota = quota;
self.last_error_reason = last_error_reason;
self.last_error_message = last_error_message;
self
}
@@ -382,8 +756,23 @@ impl ProviderAccount {
}
#[must_use]
pub const fn availability(&self) -> AccountAvailability {
self.availability
pub const fn credential_state(&self) -> CredentialState {
self.credential_state
}
#[must_use]
pub const fn quota(&self) -> QuotaState {
self.quota
}
#[must_use]
pub const fn last_error_reason(&self) -> Option<AccountErrorReason> {
self.last_error_reason
}
#[must_use]
pub fn last_error_message(&self) -> Option<&str> {
self.last_error_message.as_deref()
}
#[must_use]
@@ -407,23 +796,25 @@ impl ProviderAccount {
self.has_refresh_token
}
/// 判断账号当前能否进入同 target 的候选池。
/// 组合持久事实与请求级冷却,交给唯一解析器派生状态。
#[must_use]
pub fn is_schedulable(&self, now: SystemTime) -> bool {
let available = match self.availability {
AccountAvailability::Ready => true,
AccountAvailability::Unknown
| AccountAvailability::QuotaExhausted
| AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid => false,
};
self.upstream_user_id.is_some()
&& self.enabled
&& available
&& self
.access_token_expires_at
.is_none_or(|expires_at| expires_at > now)
pub fn status_projection(
&self,
now: SystemTime,
rate_limited_until: Option<SystemTime>,
) -> AccountStatusProjection {
resolve_account_status(
&AccountStatusFacts {
enabled: self.enabled,
credential_state: self.credential_state,
access_token_expires_at: self.access_token_expires_at,
quota: self.quota,
rate_limited_until,
last_error_reason: self.last_error_reason,
last_error_message: self.last_error_message.clone(),
},
now,
)
}
}
@@ -600,10 +991,11 @@ pub enum CredentialCasOutcome {
pub struct QuotaObservation {
pub account_id: ProviderAccountId,
pub expected_revision: CredentialRevision,
pub quota: Option<OpaqueProviderData>,
pub observed_at: Option<SystemTime>,
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
pub limit_reached: Option<bool>,
pub quota: OpaqueProviderData,
/// Provider 原始 quota document 的观察时间;不代表访问结论发生变化。
pub observed_at: SystemTime,
/// Provider 已从私有 JSON 归一化出的额度访问事实。
pub state: QuotaState,
}
impl fmt::Debug for QuotaObservation {
@@ -614,6 +1006,7 @@ impl fmt::Debug for QuotaObservation {
.field("expected_revision", &self.expected_revision)
.field("quota", &self.quota)
.field("observed_at", &self.observed_at)
.field("state", &self.state)
.finish()
}
}
@@ -624,14 +1017,24 @@ pub enum QuotaWriteOutcome {
Conflict,
}
/// 不改 Provider 原始 quota JSON 的额度访问事实写入。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct QuotaAccessChange {
pub account_id: ProviderAccountId,
pub expected_revision: CredentialRevision,
pub state: QuotaState,
}
/// 账号状态的 revision-fenced 写入。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStateChange {
pub account_id: ProviderAccountId,
pub expected_revision: CredentialRevision,
pub availability: AccountAvailability,
pub credential_state: CredentialState,
pub observed_at: SystemTime,
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
/// 受控错误原因;恢复 Ready 时必须清空。
pub error_reason: Option<AccountErrorReason>,
/// 已脱敏的上游诊断信息;恢复 Ready 时必须清空。
pub message: Option<String>,
}
@@ -682,6 +1085,11 @@ pub trait ProviderAccountStore: Send + Sync {
observation: QuotaObservation,
) -> Result<QuotaWriteOutcome, StoreError>;
async fn apply_quota_access(
&self,
change: QuotaAccessChange,
) -> Result<QuotaWriteOutcome, StoreError>;
async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), StoreError>;
async fn update_account(&self, update: ProviderAccountUpdate) -> Result<(), StoreError>;
@@ -772,7 +1180,7 @@ pub struct AccountRuntimeSignals {
pub last_started_at: Option<SystemTime>,
pub quota_reset_at: Option<SystemTime>,
pub quota_remaining_rank: Option<u64>,
pub quota_limit_reached: bool,
pub rate_limited_until: Option<SystemTime>,
pub failure_rate_basis_points: Option<u16>,
pub first_output_latency_ms: Option<u64>,
}
@@ -920,11 +1328,16 @@ impl AccountRuntimeSignals {
if let Some(quota) = quota {
self.quota_reset_at = quota.reset_at;
self.quota_remaining_rank = quota.remaining_rank;
self.quota_limit_reached = quota.limit_reached;
}
self
}
#[must_use]
pub const fn with_rate_limit(mut self, rate_limited_until: Option<SystemTime>) -> Self {
self.rate_limited_until = rate_limited_until;
self
}
#[must_use]
pub const fn with_runtime_health(
mut self,
@@ -942,20 +1355,14 @@ impl AccountRuntimeSignals {
pub struct AccountQuotaSignals {
reset_at: Option<SystemTime>,
remaining_rank: Option<u64>,
limit_reached: bool,
}
impl AccountQuotaSignals {
#[must_use]
pub const fn new(
reset_at: Option<SystemTime>,
remaining_rank: Option<u64>,
limit_reached: bool,
) -> Self {
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
Self {
reset_at,
remaining_rank,
limit_reached,
}
}
@@ -968,12 +1375,6 @@ impl AccountQuotaSignals {
pub const fn remaining_rank(self) -> Option<u64> {
self.remaining_rank
}
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
#[must_use]
pub const fn limit_reached(self) -> bool {
self.limit_reached
}
}
/// 账号持久事实与可重建运行信号的请求级组合。
@@ -991,24 +1392,24 @@ pub struct AccountSelectionContext {
pub excluded_accounts: BTreeSet<ProviderAccountId>,
pub preferred_account: Option<ProviderAccountId>,
pub round_robin_cursor: u64,
pub availability: AccountAvailabilityPolicy,
pub eligibility: AccountEligibilityPolicy,
}
/// 选择账号时是否信任本地可用性投影。
/// 选择账号时是否执行本地调度资格投影。
///
/// 管理端对指定账号执行诊断时,会直接向上游确认实际状态;该模式跳过
/// `enabled`、可用性、冷却和 token 到期等本地投影,仍保留租约、并发和
/// 请求间隔约束,且只能与固定账号约束组合使用。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum AccountAvailabilityPolicy {
pub enum AccountEligibilityPolicy {
#[default]
Enforce,
BypassForDiagnostic,
}
impl AccountAvailabilityPolicy {
impl AccountEligibilityPolicy {
#[must_use]
pub const fn bypasses_local_availability(self) -> bool {
pub const fn bypasses_local_eligibility(self) -> bool {
matches!(self, Self::BypassForDiagnostic)
}
}
@@ -1132,10 +1533,14 @@ impl AccountSelector {
candidate: &AccountCandidate,
context: &AccountSelectionContext,
) -> Option<AccountSchedulingBlocker> {
if !context.availability.bypasses_local_availability()
&& !candidate.account.is_schedulable(context.now)
{
return Some(AccountSchedulingBlocker::LocalAvailability);
if !context.eligibility.bypasses_local_eligibility() {
let status = candidate
.account
.status_projection(context.now, candidate.signals.rate_limited_until)
.status;
if status != AccountStatus::Normal {
return Some(AccountSchedulingBlocker::LocalAvailability);
}
}
if context.excluded_accounts.contains(candidate.account.id()) {
return Some(AccountSchedulingBlocker::Excluded);
@@ -9,7 +9,7 @@ use std::time::{Duration, SystemTime};
use futures::future::BoxFuture;
use crate::engine::credential::{
AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision,
AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision, CredentialState,
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
};
use crate::error::{IdentifierError, validate_text};
@@ -411,7 +411,7 @@ pub struct ProviderCredentialState {
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
observed_at: SystemTime,
}
@@ -421,14 +421,14 @@ impl ProviderCredentialState {
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
observed_at: SystemTime,
) -> Self {
Self {
account_id,
credential_revision,
enabled,
availability,
credential_state,
observed_at,
}
}
@@ -449,8 +449,8 @@ impl ProviderCredentialState {
}
#[must_use]
pub const fn availability(&self) -> AccountAvailability {
self.availability
pub const fn credential_state(&self) -> CredentialState {
self.credential_state
}
#[must_use]
@@ -5,12 +5,12 @@ use std::time::{Duration, SystemTime};
use serde_json::{Map, Value};
use gateway_core::engine::credential::{
AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate,
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
RotationStrategy,
AccountAttemptFeedback, AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats,
AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext,
AccountSelectionPolicy, AccountSelector, AccountStatus, CredentialCasUpdate,
CredentialRevision, CredentialState, OpaqueProviderData, PlaintextCredential,
PreferredAccountSelection, ProviderAccount, ProviderAccountId, ProviderAccountIdentity,
ProviderAccountUpdate, QuotaEvidence, QuotaState, RotationStrategy,
};
use gateway_core::routing::ProviderKind;
@@ -24,7 +24,13 @@ fn account(id: &str) -> ProviderAccount {
CredentialRevision::new(1).expect("valid revision"),
Some(SystemTime::now() + Duration::from_secs(3600)),
)
.with_runtime_state(true, AccountAvailability::Ready)
.with_account_facts(
true,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
)
}
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
@@ -35,7 +41,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: remaining,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -53,7 +59,7 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
excluded_accounts: BTreeSet::new(),
preferred_account: None,
round_robin_cursor: 0,
availability: AccountAvailabilityPolicy::Enforce,
eligibility: AccountEligibilityPolicy::Enforce,
}
}
@@ -101,43 +107,99 @@ fn provider_account_identity_debug_should_redact_upstream_ids() {
}
#[test]
fn availability_should_round_trip_all_database_values() {
fn account_fact_values_should_round_trip() {
assert_eq!(
AccountAvailability::parse(AccountAvailability::QuotaExhausted.as_str()),
Some(AccountAvailability::QuotaExhausted)
CredentialState::parse(CredentialState::Banned.as_str()),
Some(CredentialState::Banned)
);
assert_eq!(
QuotaEvidence::parse("provider_denied"),
Some(QuotaEvidence::ProviderDenied)
);
}
#[test]
fn diagnostic_selection_bypasses_all_local_account_availability() {
fn exhausted_quota_refresh_is_due_at_reset_or_provider_fallback() {
let now = SystemTime::now();
let observed_at = now - Duration::from_secs(60);
let before_reset = QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
observed_at,
Some(now + Duration::from_secs(1)),
);
let after_reset = QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
observed_at,
Some(now - Duration::from_secs(1)),
);
let without_reset = QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None);
assert!(!before_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
assert!(after_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
assert!(without_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
assert!(!without_reset.exhaustion_refresh_due(now, Duration::from_secs(120)));
}
#[test]
fn inconclusive_quota_observation_cannot_erase_confirmed_access() {
let now = SystemTime::now();
let exhausted = QuotaState::exhausted(QuotaEvidence::ProviderDenied, now, None);
assert_eq!(
exhausted
.merge_observation(QuotaState::observed_unknown(now + Duration::from_secs(1)))
.access(),
gateway_core::engine::credential::QuotaAccessState::Exhausted
);
assert_eq!(
exhausted
.merge_observation(QuotaState::allowed(now + Duration::from_secs(1)))
.access(),
gateway_core::engine::credential::QuotaAccessState::Allowed
);
}
#[test]
fn diagnostic_selection_bypasses_all_local_account_eligibility() {
let observed_at = SystemTime::now();
let exhausted = AccountCandidate {
account: account("acct_exhausted")
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
account: account("acct_exhausted").with_account_facts(
true,
CredentialState::Ready,
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
None,
None,
),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
};
let disabled = AccountCandidate {
account: account("acct_disabled")
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
account: account("acct_disabled").with_account_facts(
false,
CredentialState::Ready,
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
None,
None,
),
signals: AccountRuntimeSignals {
in_flight: 0,
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
};
let mut context = context(RotationStrategy::Sticky);
context.availability = AccountAvailabilityPolicy::BypassForDiagnostic;
context.eligibility = AccountEligibilityPolicy::BypassForDiagnostic;
assert_eq!(
AccountSelector
@@ -155,9 +217,33 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
#[test]
fn disabled_account_should_not_be_schedulable() {
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
let account = account("acct_disabled").with_account_facts(
false,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
);
assert!(!account.is_schedulable(SystemTime::now()));
assert_eq!(
account.status_projection(SystemTime::now(), None).status,
AccountStatus::Disabled
);
}
#[test]
fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
let now = SystemTime::now();
let active_until = now + Duration::from_secs(60);
let account = account("acct_rate_limited");
let active = account.status_projection(now, Some(active_until));
assert_eq!(active.status, AccountStatus::RateLimited);
assert_eq!(active.rate_limited_until, Some(active_until));
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
assert_eq!(elapsed.status, AccountStatus::Normal);
assert_eq!(elapsed.rate_limited_until, None);
}
#[test]
@@ -171,14 +257,20 @@ fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
CredentialRevision::new(1).expect("valid revision"),
Some(SystemTime::now() + Duration::from_secs(3600)),
)
.with_runtime_state(true, AccountAvailability::Ready);
.with_account_facts(
true,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
);
assert_eq!(
(
account.availability(),
account.is_schedulable(SystemTime::now())
account.credential_state(),
account.status_projection(SystemTime::now(), None).status
),
(AccountAvailability::Unknown, false)
(CredentialState::Unknown, AccountStatus::Error)
);
}
@@ -308,15 +400,11 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
last_started_at: Some(last_started_at),
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
}
.with_provider_quota(Some(AccountQuotaSignals::new(
Some(reset_at),
Some(75),
false,
)));
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
assert_eq!(
(
@@ -633,21 +721,31 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
}
#[test]
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
// availability 终态(Banned/Expired/Invalid)始终排除,
// 不受 quota 信号或 skip_exhausted 放宽影响。
fn enforced_eligibility_excludes_credential_errors_despite_quota_signals() {
let mut banned = candidate("acct_banned", 0, Some(100));
banned.account = banned
.account
.with_runtime_state(true, AccountAvailability::Banned);
banned.account = banned.account.with_account_facts(
true,
CredentialState::Banned,
QuotaState::unknown(),
None,
None,
);
let mut expired = candidate("acct_expired", 0, Some(100));
expired.account = expired
.account
.with_runtime_state(true, AccountAvailability::Expired);
expired.account = expired.account.with_account_facts(
true,
CredentialState::Expired,
QuotaState::unknown(),
None,
None,
);
let mut invalid = candidate("acct_invalid", 0, Some(100));
invalid.account = invalid
.account
.with_runtime_state(true, AccountAvailability::Invalid);
invalid.account = invalid.account.with_account_facts(
true,
CredentialState::Invalid,
QuotaState::unknown(),
None,
None,
);
let healthy = candidate("acct_healthy", 0, Some(100));
let candidates = vec![banned, expired, invalid, healthy];
@@ -659,14 +757,15 @@ fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
}
#[test]
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
fn enforced_eligibility_excludes_authoritative_quota_exhaustion() {
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
exhausted.account = exhausted
.account
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
exhausted.account = exhausted.account.with_account_facts(
true,
CredentialState::Ready,
QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
None,
None,
);
let healthy = candidate("acct_healthy", 0, Some(100));
let candidates = vec![exhausted, healthy];
@@ -676,3 +775,31 @@ fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection()
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
#[test]
fn elapsed_quota_reset_does_not_fabricate_recovery() {
let now = SystemTime::now();
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
exhausted.account = exhausted.account.with_account_facts(
true,
CredentialState::Ready,
QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
now - Duration::from_secs(2),
Some(now - Duration::from_secs(1)),
),
None,
None,
);
let healthy = candidate("acct_healthy", 0, Some(100));
assert_eq!(
exhausted.account.status_projection(now, None).status,
AccountStatus::QuotaExhausted
);
let candidates = [exhausted, healthy];
let selected = AccountSelector
.select(&candidates, &context(RotationStrategy::Smart))
.expect("healthy candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
@@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: Some(7),
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: Some(125),
first_output_latency_ms: Some(250),
},
@@ -1,12 +1,15 @@
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
use std::sync::Arc;
use std::time::SystemTime;
use gateway_core::provider_ports::ProviderCooldownPort;
use std::sync::Arc;
use super::*;
use crate::postgres::{
PgProviderAccountRepository, ProviderAccountRepository, account_status_projection,
load_rate_limited_until,
};
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
#[derive(Clone)]
@@ -21,73 +24,48 @@ impl PgObservabilityRepository {
Self { pool, cooldowns }
}
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
async fn provider_account_metrics_with_cooldowns(
/// 从账号事实和当前冷却一次性派生 Dashboard 五态;不在 SQL 中复制状态机。
async fn account_status_snapshot(
&self,
observed_at: DateTime<Utc>,
) -> StoreResult<ProviderAccountMetrics> {
let base = provider_account_metrics(&self.pool, observed_at).await?;
let Some(cooldowns) = &self.cooldowns else {
return Ok(base);
) -> StoreResult<(ProviderAccountMetrics, Vec<String>)> {
let accounts = PgProviderAccountRepository::new(self.pool.clone())
.list_provider_accounts(None, true)
.await?;
let now = observed_at.into();
let rate_limited_until =
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await;
let mut metrics = ProviderAccountMetrics {
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
..ProviderAccountMetrics::default()
};
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
if candidates.is_empty() {
return Ok(base);
}
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
let mut active = base.active;
let mut rate_limited = base.rate_limited;
let mut quota_exhausted = base.quota_exhausted;
let now = SystemTime::now();
for (account_id, revision, quota_reached) in candidates {
let Ok(account_id) =
gateway_core::engine::credential::ProviderAccountId::new(account_id)
else {
continue;
};
let cooling = cooldowns
.read(&account_id)
.await
.ok()
.flatten()
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
.filter(|cooldown| cooldown.until() > now)
.is_some();
if !cooling {
continue;
}
if quota_reached {
if quota_exhausted == 0 {
continue;
let mut normal_account_ids = Vec::new();
for account in &accounts {
let projection = account_status_projection(
account,
now,
rate_limited_until.get(&account.id).copied(),
);
match projection.status {
gateway_core::engine::credential::AccountStatus::Normal => {
metrics.normal = metrics.normal.saturating_add(1);
normal_account_ids.push(account.id.clone());
}
quota_exhausted -= 1;
} else {
if active == 0 {
continue;
gateway_core::engine::credential::AccountStatus::QuotaExhausted => {
metrics.quota_exhausted = metrics.quota_exhausted.saturating_add(1);
}
gateway_core::engine::credential::AccountStatus::RateLimited => {
metrics.rate_limited = metrics.rate_limited.saturating_add(1);
}
gateway_core::engine::credential::AccountStatus::Disabled => {
metrics.disabled = metrics.disabled.saturating_add(1);
}
gateway_core::engine::credential::AccountStatus::Error => {
metrics.error = metrics.error.saturating_add(1);
}
active -= 1;
}
rate_limited += 1;
}
Ok(ProviderAccountMetrics {
total: base.total,
enabled: base.enabled,
unavailable: rate_limited
+ quota_exhausted
+ base.expired
+ base.invalid
+ base.disabled
+ base.banned,
active,
rate_limited,
expired: base.expired,
invalid: base.invalid,
quota_exhausted,
disabled: base.disabled,
banned: base.banned,
})
Ok((metrics, normal_account_ids))
}
}
@@ -145,9 +123,7 @@ impl ObservabilityRepository for PgObservabilityRepository {
attempt_metrics(&self.pool, range, &filter),
dashboard_totals(&self.pool),
)?;
let provider_accounts = self
.provider_account_metrics_with_cooldowns(range.end)
.await?;
let (provider_accounts, _) = self.account_status_snapshot(range.end).await?;
let (trend, account_usage, recent_requests) = futures::try_join!(
request_metric_series(&self.pool, range, &filter),
provider_account_usage(&self.pool, account_usage_query),
@@ -252,12 +228,14 @@ impl AdminObservabilityStore for PgAdminObservabilityStore {
&self,
observed_at: DateTime<Utc>,
) -> AdminStoreResult<Option<admin_observability::DashboardRuntimeSlots>> {
let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at)
let (_, normal_account_ids) = self
.repository
.account_status_snapshot(observed_at)
.await
.map_err(observability_error)?;
let active_accounts = u64::try_from(active_account_ids.len())
let active_accounts = u64::try_from(normal_account_ids.len())
.map_err(|_| observability_error(invalid("active account count overflows u64")))?;
if active_account_ids.is_empty() {
if normal_account_ids.is_empty() {
return Ok(Some(admin_observability::DashboardRuntimeSlots {
active_accounts,
used_slots: Some(0),
@@ -267,7 +245,7 @@ impl AdminObservabilityStore for PgAdminObservabilityStore {
return Ok(None);
};
let signals = match runtime_signals
.credential_runtime_signals(&active_account_ids)
.credential_runtime_signals(&normal_account_ids)
.await
{
Ok(signals) => signals,
@@ -274,16 +274,11 @@ pub(crate) const fn admin_account_pool_metrics(
) -> admin_observability::AccountPoolMetrics {
admin_observability::AccountPoolMetrics {
total: metrics.total,
enabled: metrics.enabled,
unavailable: metrics.unavailable,
active: metrics.active,
rate_limited: metrics.rate_limited,
expired: metrics.expired,
invalid: metrics.invalid,
normal: metrics.normal,
quota_exhausted: metrics.quota_exhausted,
refreshing: None,
rate_limited: metrics.rate_limited,
disabled: metrics.disabled,
banned: metrics.banned,
error: metrics.error,
}
}
@@ -297,8 +292,6 @@ pub(crate) fn admin_dashboard_account_usage(
name: usage.name,
email: usage.email,
plan_type: usage.plan_type,
enabled: usage.enabled,
availability: usage.availability,
request_count: usage.request_count,
success_count: usage.success_count,
input_tokens: usage.input_tokens,
@@ -363,15 +363,11 @@ pub struct CalculatedUsageBillingFact {
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ProviderAccountMetrics {
pub total: u64,
pub enabled: u64,
pub unavailable: u64,
pub active: u64,
pub rate_limited: u64,
pub expired: u64,
pub invalid: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub rate_limited: u64,
pub disabled: u64,
pub banned: u64,
pub error: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -388,8 +384,6 @@ pub struct ProviderAccountUsageObservation {
pub name: String,
pub email: Option<String>,
pub plan_type: Option<String>,
pub enabled: bool,
pub availability: String,
pub request_count: u64,
pub success_count: u64,
pub input_tokens: Option<u64>,
@@ -15,7 +15,7 @@ pub(crate) async fn provider_account_usage(
let mut statement = QueryBuilder::<Postgres>::new(
"select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
pa.plan_type, pa.enabled, pa.availability,
pa.plan_type,
count(mr.id)::bigint as request_count,
count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count,
sum(mr.input_tokens)::bigint as input_tokens,
@@ -52,7 +52,7 @@ pub(crate) async fn provider_account_usage(
}
statement.push(
" group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
pa.plan_type, pa.enabled, pa.availability
pa.plan_type
order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ",
);
statement.push_bind(i64::from(query.limit));
@@ -313,8 +313,6 @@ pub(crate) fn provider_account_from_row(
name: get(row, "name")?,
email: get(row, "email")?,
plan_type: get(row, "plan_type")?,
enabled: get(row, "enabled")?,
availability: get(row, "availability")?,
request_count: unsigned(row, "request_count")?,
success_count: unsigned(row, "success_count")?,
input_tokens: optional_unsigned(row, "input_tokens")?,
@@ -369,118 +369,6 @@ pub(crate) async fn request_costs(
.collect()
}
pub(crate) async fn provider_account_metrics(
pool: &PgPool,
observed_at: DateTime<Utc>,
) -> StoreResult<ProviderAccountMetrics> {
let row = sqlx::query(
"with normalized as (
select enabled, availability, quota_limit_reached,
case
when not enabled then 'disabled'
when availability = 'banned' then 'banned'
when availability = 'quota_exhausted' then 'quota_exhausted'
when availability = 'invalid' then 'invalid'
when availability = 'expired'
or (access_token_expires_at is not null and access_token_expires_at <= $1)
then 'expired'
when availability = 'unknown' then 'invalid'
when quota_limit_reached then 'quota_exhausted'
else 'active'
end as status
from provider_accounts
)
select count(*)::bigint as total,
count(*) filter (where enabled)::bigint as enabled,
count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable,
count(*) filter (where status = 'active')::bigint as active,
count(*) filter (where status = 'rate_limited')::bigint as rate_limited,
count(*) filter (where status = 'expired')::bigint as expired,
count(*) filter (where status = 'invalid')::bigint as invalid,
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
count(*) filter (where status = 'disabled')::bigint as disabled,
count(*) filter (where status = 'banned')::bigint as banned
from normalized",
)
.bind(observed_at)
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load provider account metrics"))?;
Ok(ProviderAccountMetrics {
total: unsigned(&row, "total")?,
enabled: unsigned(&row, "enabled")?,
unavailable: unsigned(&row, "unavailable")?,
active: unsigned(&row, "active")?,
rate_limited: unsigned(&row, "rate_limited")?,
expired: unsigned(&row, "expired")?,
invalid: unsigned(&row, "invalid")?,
quota_exhausted: unsigned(&row, "quota_exhausted")?,
disabled: unsigned(&row, "disabled")?,
banned: unsigned(&row, "banned")?,
})
}
pub(crate) async fn active_provider_account_ids(
pool: &PgPool,
observed_at: DateTime<Utc>,
) -> StoreResult<Vec<String>> {
sqlx::query(
"with normalized as (
select id,
case
when not enabled then 'disabled'
when availability = 'banned' then 'banned'
when availability = 'quota_exhausted' then 'quota_exhausted'
when availability in ('expired', 'invalid')
or (access_token_expires_at is not null and access_token_expires_at <= $1)
then 'expired'
else 'active'
end as status
from provider_accounts
)
select id
from normalized
where status = 'active'
order by id",
)
.bind(observed_at)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load active provider account IDs"))?
.iter()
.map(|row| get(row, "id"))
.collect()
}
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
pub(crate) async fn schedulable_metric_candidates(
pool: &PgPool,
observed_at: DateTime<Utc>,
) -> StoreResult<Vec<(String, i64, bool)>> {
sqlx::query(
"select id, credential_revision, quota_limit_reached
from provider_accounts
where enabled
and availability = 'ready'
and (access_token_expires_at is null or access_token_expires_at > $1)
order by id",
)
.bind(observed_at)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
.into_iter()
.map(|row| {
Ok((
get(&row, "id")?,
get(&row, "credential_revision")?,
get(&row, "quota_limit_reached")?,
))
})
.collect::<StoreResult<Vec<_>>>()
}
pub(crate) async fn provider_observations(
pool: &PgPool,
range: ObservabilityRange,
@@ -2,7 +2,6 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::SystemTime;
use gateway_core::provider_ports::ProviderCooldownPort;
@@ -32,34 +31,6 @@ impl PgAdminAccountStore {
}
}
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
async fn load_rate_limited_until(
&self,
accounts: &[ProviderAccountSummary],
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
let Some(cooldowns) = &self.cooldowns else {
return Ok(BTreeMap::new());
};
let mut map = BTreeMap::new();
for account in accounts {
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
continue;
};
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
continue;
};
if cooldown.credential_revision().get() != account.credential_revision.get() {
continue;
}
let until = cooldown.until();
if until > SystemTime::now() {
map.insert(account.id.clone(), until);
}
}
Ok(map)
}
async fn usage_observations(
&self,
range: ObservabilityRange,
@@ -278,21 +249,18 @@ impl AccountStore for PgAdminAccountStore {
)
.map_err(|error| admin_store_error(ENTITY, error))?;
let now = Utc::now();
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
let rate_limited_until = self
.load_rate_limited_until(&accounts)
.await
.unwrap_or_default();
let rate_limited_until =
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now.into()).await;
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
let mut items = accounts
.into_iter()
.filter_map(|account| {
let until = rate_limited_until.get(&account.id).copied();
let status = admin_account_status(&account, now, until);
account_matches_admin_query(&account, status, &query).then_some(
let projection = account_status_projection(&account, now.into(), until);
account_matches_admin_query(&account, projection.status, &query).then_some(
AdminAccountListItem {
account,
status,
projection,
usage: None,
},
)
@@ -330,7 +298,12 @@ impl AccountStore for PgAdminAccountStore {
.into_iter()
.skip(offset)
.take(page_size)
.map(|item| admin_account_record(item.account))
.map(|item| {
Ok(AccountPageItem {
account: admin_account_record(item.account)?,
projection: item.projection,
})
})
.collect::<AdminStoreResult<Vec<_>>>()?;
Ok(AccountPage {
config_revision: admin_revision(control_plane.settings.config_revision)?,
@@ -340,13 +313,29 @@ impl AccountStore for PgAdminAccountStore {
})
}
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
self.accounts
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>> {
let record = self
.accounts
.load_provider_account(account_id)
.await
.map_err(|error| admin_store_error(ENTITY, error))?
.map(|record| admin_account_record(record.summary))
.transpose()
.map_err(|error| admin_store_error(ENTITY, error))?;
let Some(record) = record else {
return Ok(None);
};
let now = Utc::now();
let rate_limited_until = load_rate_limited_until(
self.cooldowns.as_deref(),
std::slice::from_ref(&record.summary),
now.into(),
)
.await
.get(account_id)
.copied();
let projection = account_status_projection(&record.summary, now.into(), rate_limited_until);
Ok(Some(AccountPageItem {
account: admin_account_record(record.summary)?,
projection,
}))
}
async fn load_account_usage(
@@ -386,9 +375,9 @@ impl AccountStore for PgAdminAccountStore {
.filter(|account| account.provider_kind == provider_kind.as_str())
.filter(|account| {
query
.availability
.credential_state
.as_ref()
.is_none_or(|expected| expected.matches(account.availability))
.is_none_or(|expected| expected.matches(account.credential_state))
})
.filter(|account| {
query
@@ -31,8 +31,8 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.map(DateTime::<Utc>::from),
next_refresh_at: account.account.next_refresh_at().map(DateTime::<Utc>::from),
enabled: account.account.enabled(),
availability: account.account.availability(),
availability_observed_at: Utc::now(),
credential_state: account.account.credential_state(),
credential_observed_at: Utc::now(),
})
.await
.map_err(core_store_error)
@@ -164,9 +164,9 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.collect::<Vec<_>>();
let rows = sqlx::query(
"select id, credential_revision, provider_quota_json, quota_observed_at, \
quota_limit_reached \
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at \
from provider_accounts \
where id = any($1) and provider_quota_json is not null",
where id = any($1) and quota_observed_at is not null",
)
.bind(account_ids)
.fetch_all(&self.pool)
@@ -184,24 +184,49 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.and_then(|value| CoreCredentialRevision::new(value).ok())
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let quota = row
.try_get::<serde_json::Value, _>("provider_quota_json")
.ok()
.try_get::<Option<serde_json::Value>, _>("provider_quota_json")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?
.and_then(|value| value.as_object().cloned())
.map(OpaqueProviderData::new)
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.unwrap_or_else(|| OpaqueProviderData::new(serde_json::Map::new()));
let observed_at = row
.try_get::<DateTime<Utc>, _>("quota_observed_at")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let limit_reached = row
.try_get::<bool, _>("quota_limit_reached")
let access = row
.try_get::<String, _>("quota_access_state")
.ok()
.and_then(|value| QuotaAccessState::parse(&value))
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let evidence = row
.try_get::<Option<String>, _>("quota_evidence")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let evidence = match evidence {
Some(value) => Some(
QuotaEvidence::parse(&value)
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
),
None => None,
};
let reset_at = row
.try_get::<Option<DateTime<Utc>>, _>("quota_reset_at")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let access_observed_at = row
.try_get::<Option<DateTime<Utc>>, _>("quota_access_observed_at")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let state = QuotaState::from_persisted(
access,
evidence,
access_observed_at.map(Into::into),
reset_at.map(Into::into),
)
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
Ok(QuotaObservation {
account_id: CoreProviderAccountId::new(account_id)
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
expected_revision: revision,
quota: Some(quota),
observed_at: Some(observed_at.into()),
limit_reached: Some(limit_reached),
quota,
observed_at: observed_at.into(),
state,
})
})
.collect()
@@ -211,24 +236,38 @@ impl ProviderAccountStore for PgProviderAccountRepository {
&self,
observation: QuotaObservation,
) -> Result<QuotaWriteOutcome, CoreStoreError> {
let quota = observation
.quota
.map(|value| {
JsonObject::try_from_value(
"provider_quota_json",
serde_json::Value::Object(value.into_inner()),
QUOTA_MAX_BYTES,
)
})
.transpose()
.map_err(core_store_error)?;
let quota = JsonObject::try_from_value(
"provider_quota_json",
serde_json::Value::Object(observation.quota.into_inner()),
QUOTA_MAX_BYTES,
)
.map_err(core_store_error)?;
let updated = self
.compare_and_swap_provider_quota(
observation.account_id.as_str(),
Revision::new(observation.expected_revision.get()).map_err(core_store_error)?,
quota,
observation.observed_at.map(DateTime::<Utc>::from),
observation.limit_reached,
DateTime::<Utc>::from(observation.observed_at),
observation.state,
)
.await
.map_err(core_store_error)?;
Ok(if updated {
QuotaWriteOutcome::Updated
} else {
QuotaWriteOutcome::Conflict
})
}
async fn apply_quota_access(
&self,
change: QuotaAccessChange,
) -> Result<QuotaWriteOutcome, CoreStoreError> {
let updated = self
.apply_provider_quota_access(
change.account_id.as_str(),
Revision::new(change.expected_revision.get()).map_err(core_store_error)?,
change.state,
)
.await
.map_err(core_store_error)?;
@@ -245,8 +284,9 @@ impl ProviderAccountStore for PgProviderAccountRepository {
account_id: change.account_id.as_str().to_owned(),
expected_revision: Revision::new(change.expected_revision.get())
.map_err(core_store_error)?,
availability: change.availability,
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
credential_state: change.credential_state,
credential_observed_at: DateTime::<Utc>::from(change.observed_at),
error_reason: change.error_reason,
message: change.message,
})
.await
@@ -1,12 +1,12 @@
//! Admin 账号列表、用量与成本的领域映射。
use std::time::SystemTime;
use super::*;
use gateway_admin::model::accounts::AccountStatusSignals;
pub(crate) struct AdminAccountListItem {
pub(crate) account: ProviderAccountSummary,
pub(crate) status: AdminAccountStatus,
pub(crate) projection: gateway_core::engine::credential::AccountStatusProjection,
pub(crate) usage: Option<ProviderAccountUsageObservation>,
}
@@ -57,24 +57,6 @@ pub(crate) fn account_matches_admin_query(
provider_matches && search_matches && status_matches
}
pub(crate) fn admin_account_status(
account: &ProviderAccountSummary,
now: DateTime<Utc>,
rate_limited_until: Option<SystemTime>,
) -> AdminAccountStatus {
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached: account.quota_limit_reached,
rate_limited_until,
}
.derive()
}
pub(crate) fn admin_account_summary(
accounts: &[ProviderAccountSummary],
now: DateTime<Utc>,
@@ -90,7 +72,7 @@ pub(crate) fn admin_account_summary(
};
for account in accounts {
let until = rate_limited_until.get(&account.id).copied();
match admin_account_status(account, now, until) {
match account_status_projection(account, now.into(), until).status {
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
AdminAccountStatus::QuotaExhausted => {
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
@@ -118,7 +100,11 @@ pub(crate) fn sort_admin_account_items(
items.sort_by(|left, right| {
let ordering = match sort.field {
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
AdminAccountSortField::Status => left
.projection
.status
.sort_rank()
.cmp(&right.projection.status.sort_rank()),
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
AdminAccountSortField::Usage => left
.usage
@@ -171,10 +157,11 @@ pub(crate) fn admin_account_record(
access_token_expires_at: summary.access_token_expires_at,
next_refresh_at: summary.next_refresh_at,
enabled: summary.enabled,
availability: summary.availability,
availability_observed_at: summary.availability_observed_at,
credential_state: summary.credential_state,
credential_observed_at: summary.credential_observed_at,
quota: summary.quota,
last_error_reason: summary.last_error_reason,
last_error_message: summary.last_error_message,
quota_observed_at: summary.quota_observed_at,
created_at: summary.created_at,
updated_at: summary.updated_at,
})
@@ -197,8 +184,8 @@ pub(crate) fn prepared_account(
access_token_expires_at: credential.access_token_expires_at,
next_refresh_at: credential.next_refresh_at,
enabled: credential.enabled,
availability: credential.availability,
availability_observed_at: credential.availability_observed_at,
credential_state: credential.credential_state,
credential_observed_at: credential.credential_observed_at,
})
}
@@ -4,7 +4,6 @@ use std::{
collections::{BTreeMap, BTreeSet},
fmt,
str::FromStr,
time::SystemTime,
};
use async_trait::async_trait;
@@ -14,7 +13,7 @@ use gateway_admin::{
MutationContext, Revision as AdminRevision,
accounts::{
AccountCost, AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage,
AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort,
AccountPageItem, AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort,
AccountSortField as AdminAccountSortField, AccountStatus as AdminAccountStatus,
AccountSummary, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult,
DeleteAccounts, SetAccountEnabled, SortDirection as AdminSortDirection,
@@ -35,12 +34,13 @@ use gateway_admin::{
use sqlx::{PgPool, Postgres, Row, Transaction};
use gateway_core::engine::credential::{
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision as CoreCredentialRevision, LoadedCredential,
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision as CoreCredentialRevision, CredentialState, LoadedCredential,
NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential,
ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId,
ProviderAccountIdentity, ProviderAccountStore,
ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaAccessChange, QuotaAccessState,
QuotaEvidence, QuotaObservation, QuotaState, QuotaWriteOutcome,
};
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
use gateway_core::routing::ProviderKind;
@@ -63,8 +63,10 @@ mod core_adapter;
mod mapping;
mod repository;
mod rows;
mod runtime;
pub use admin_adapter::*;
pub(crate) use mapping::*;
pub use repository::*;
pub use rows::*;
pub(crate) use runtime::*;
@@ -25,9 +25,15 @@ pub trait ProviderAccountRepository: Send + Sync {
&self,
account_id: &str,
expected_revision: Revision,
quota: Option<JsonObject>,
observed_at: Option<DateTime<Utc>>,
limit_reached: Option<bool>,
quota: JsonObject,
observed_at: DateTime<Utc>,
state: QuotaState,
) -> StoreResult<bool>;
async fn apply_provider_quota_access(
&self,
account_id: &str,
expected_revision: Revision,
state: QuotaState,
) -> StoreResult<bool>;
async fn delete_provider_account(&self, id: &str) -> StoreResult<bool>;
}
@@ -93,9 +99,10 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
let rows = sqlx::query(
"select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, availability,
availability_observed_at,
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
access_token_expires_at, next_refresh_at, enabled, credential_state,
credential_observed_at, quota_access_state, quota_evidence,
quota_access_observed_at, quota_reset_at,
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
from provider_accounts
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
order by provider_kind, name, id",
@@ -110,21 +117,21 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> {
account.validate()?;
let availability = if account.upstream_user_id.is_some() {
account.availability
let credential_state = if account.upstream_user_id.is_some() {
account.credential_state
} else {
AccountAvailability::Unknown
CredentialState::Unknown
};
sqlx::query(
"insert into provider_accounts (
id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
availability, provider_quota_json,
availability_observed_at, quota_observed_at, created_at, updated_at
credential_state, provider_quota_json,
credential_observed_at, quota_access_observed_at, quota_observed_at, created_at, updated_at
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
$14, null, $15, null, now(), greatest(now(), $15)
$14, null, $15, null, null, now(), greatest(now(), $15)
)",
)
.bind(account.id)
@@ -140,8 +147,8 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(account.access_token_expires_at)
.bind(account.next_refresh_at)
.bind(account.enabled)
.bind(availability.as_str())
.bind(account.availability_observed_at)
.bind(credential_state.as_str())
.bind(account.credential_observed_at)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert provider account"))?;
@@ -214,27 +221,33 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
update.validate()?;
let result = sqlx::query(
"update provider_accounts
set availability = case
set credential_state = case
when enabled and upstream_user_id is not null then $3
else availability
else credential_state
end,
availability_observed_at = case
credential_observed_at = case
when enabled and upstream_user_id is not null then $4
else availability_observed_at
else credential_observed_at
end,
last_error_reason = case
when enabled and upstream_user_id is not null then
case when $3 = 'ready' then null else $5 end
else last_error_reason
end,
last_error_message = case
when enabled and upstream_user_id is not null then
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
case when $3 = 'ready' then null else $6 end
else last_error_message
end,
updated_at = case when enabled then greatest(now(), $4) else updated_at end
where id = $1 and credential_revision = $2
and (availability_observed_at is null or availability_observed_at <= $4)",
and (credential_observed_at is null or credential_observed_at <= $4)",
)
.bind(update.account_id)
.bind(to_i64(update.expected_revision.get())?)
.bind(update.availability.as_str())
.bind(update.availability_observed_at)
.bind(update.credential_state.as_str())
.bind(update.credential_observed_at)
.bind(update.error_reason.map(AccountErrorReason::as_str))
.bind(update.message.as_deref())
.execute(&self.pool)
.await
@@ -259,36 +272,81 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
&self,
account_id: &str,
expected_revision: Revision,
quota: Option<JsonObject>,
observed_at: Option<DateTime<Utc>>,
limit_reached: Option<bool>,
quota: JsonObject,
observed_at: DateTime<Utc>,
state: QuotaState,
) -> StoreResult<bool> {
require_nonempty(ENTITY, "account_id", account_id)?;
if quota.is_some() != observed_at.is_some() {
return Err(invalid("quota JSON and observed_at must agree"));
}
if let Some(quota) = quota.as_ref() {
validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?;
}
validate_object_size("provider_quota_json", &quota, QUOTA_MAX_BYTES)?;
let access_observed_at = state.observed_at().map(DateTime::<Utc>::from);
let result = sqlx::query(
"update provider_accounts
set provider_quota_json = $3, quota_observed_at = $4,
quota_limit_reached = coalesce($5, quota_limit_reached),
updated_at = greatest(now(), coalesce($4, now()))
quota_access_state = case
when $7::timestamptz is not null
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
then $5 else quota_access_state end,
quota_evidence = case
when $7::timestamptz is not null
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
then $6 else quota_evidence end,
quota_access_observed_at = case
when $7::timestamptz is not null
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
then $7 else quota_access_observed_at end,
quota_reset_at = case
when $7::timestamptz is not null
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
then $8 else quota_reset_at end,
updated_at = greatest(now(), $4)
where id = $1 and credential_revision = $2
and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)",
and (quota_observed_at is null or quota_observed_at <= $4)",
)
.bind(account_id)
.bind(to_i64(expected_revision.get())?)
.bind(quota.map(|value| value.as_value()))
.bind(quota.as_value())
.bind(observed_at)
.bind(limit_reached)
.bind(state.access().as_str())
.bind(state.evidence().map(QuotaEvidence::as_str))
.bind(access_observed_at)
.bind(state.reset_at().map(DateTime::<Utc>::from))
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("compare and swap provider quota"))?;
Ok(result.rows_affected() == 1)
}
async fn apply_provider_quota_access(
&self,
account_id: &str,
expected_revision: Revision,
state: QuotaState,
) -> StoreResult<bool> {
require_nonempty(ENTITY, "account_id", account_id)?;
let observed_at = state
.observed_at()
.map(DateTime::<Utc>::from)
.ok_or_else(|| invalid("quota access change requires observed_at"))?;
let result = sqlx::query(
"update provider_accounts
set quota_access_observed_at = $3, quota_access_state = $4,
quota_evidence = $5, quota_reset_at = $6,
updated_at = greatest(now(), $3)
where id = $1 and credential_revision = $2
and (quota_access_observed_at is null or quota_access_observed_at <= $3)",
)
.bind(account_id)
.bind(to_i64(expected_revision.get())?)
.bind(observed_at)
.bind(state.access().as_str())
.bind(state.evidence().map(QuotaEvidence::as_str))
.bind(state.reset_at().map(DateTime::<Utc>::from))
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("apply provider quota access"))?;
Ok(result.rows_affected() == 1)
}
async fn delete_provider_account(&self, id: &str) -> StoreResult<bool> {
require_nonempty(ENTITY, "id", id)?;
let result = sqlx::query("delete from provider_accounts where id = $1 and not enabled")
@@ -474,21 +532,21 @@ pub(crate) async fn upsert_provider_account_in_transaction(
account: &NewProviderAccount,
) -> StoreResult<String> {
account.validate()?;
let availability = if account.upstream_user_id.is_some() {
account.availability
let credential_state = if account.upstream_user_id.is_some() {
account.credential_state
} else {
AccountAvailability::Unknown
CredentialState::Unknown
};
let imported_id = sqlx::query_scalar::<_, String>(
"insert into provider_accounts (
id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
availability, provider_quota_json,
availability_observed_at, quota_observed_at, created_at, updated_at
credential_state, provider_quota_json,
credential_observed_at, quota_access_observed_at, quota_observed_at, created_at, updated_at
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
$14, null, $15, null, now(), greatest(now(), $15)
$14, null, $15, null, null, now(), greatest(now(), $15)
)
on conflict (
provider_kind,
@@ -505,11 +563,17 @@ pub(crate) async fn upsert_provider_account_in_transaction(
access_token_expires_at = excluded.access_token_expires_at,
next_refresh_at = excluded.next_refresh_at,
enabled = excluded.enabled,
availability = excluded.availability,
credential_state = excluded.credential_state,
provider_quota_json = null,
availability_observed_at = excluded.availability_observed_at,
quota_access_state = 'unknown',
quota_evidence = null,
quota_access_observed_at = null,
quota_reset_at = null,
credential_observed_at = excluded.credential_observed_at,
quota_observed_at = null,
updated_at = greatest(now(), excluded.availability_observed_at)
last_error_reason = null,
last_error_message = null,
updated_at = greatest(now(), excluded.credential_observed_at)
returning id",
)
.bind(&account.id)
@@ -525,8 +589,8 @@ pub(crate) async fn upsert_provider_account_in_transaction(
.bind(account.access_token_expires_at)
.bind(account.next_refresh_at)
.bind(account.enabled)
.bind(availability.as_str())
.bind(account.availability_observed_at)
.bind(credential_state.as_str())
.bind(account.credential_observed_at)
.fetch_optional(&mut **transaction)
.await
.map_err(|error| {
@@ -574,17 +638,17 @@ pub(crate) async fn rotate_provider_account_in_transaction(
next_refresh_at = $10,
upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end,
upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end,
availability = case
when not enabled then availability
when availability = 'quota_exhausted' then availability
credential_state = case
when not enabled then credential_state
when coalesce($12::text, upstream_user_id) is not null then 'ready'
else 'unknown'
end,
availability_observed_at = case
when not enabled then availability_observed_at
when availability <> 'quota_exhausted' then now()
else availability_observed_at
credential_observed_at = case
when not enabled then credential_observed_at
else now()
end,
last_error_reason = case when enabled then null else last_error_reason end,
last_error_message = case when enabled then null else last_error_message end,
updated_at = now()
where id = $1 and provider_kind = $2
and credential_revision = $3
@@ -65,9 +65,29 @@ impl ProviderAccountAdminScope {
}
}
/// 解析数据库 availability 列;字符串映射本体唯一归属 gateway-core。
pub(crate) fn parse_availability(value: &str) -> StoreResult<AccountAvailability> {
AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value"))
pub(crate) fn parse_credential_state(value: &str) -> StoreResult<CredentialState> {
CredentialState::parse(value).ok_or_else(|| invalid("unknown credential_state value"))
}
pub(crate) fn parse_quota_access_state(value: &str) -> StoreResult<QuotaAccessState> {
QuotaAccessState::parse(value).ok_or_else(|| invalid("unknown quota_access_state value"))
}
pub(crate) fn parse_quota_evidence(value: Option<String>) -> StoreResult<Option<QuotaEvidence>> {
value
.map(|value| {
QuotaEvidence::parse(&value).ok_or_else(|| invalid("unknown quota_evidence value"))
})
.transpose()
}
pub(crate) fn parse_error_reason(value: Option<String>) -> StoreResult<Option<AccountErrorReason>> {
value
.map(|value| {
AccountErrorReason::parse(&value)
.ok_or_else(|| invalid("unknown last_error_reason value"))
})
.transpose()
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -85,12 +105,10 @@ pub struct ProviderAccountSummary {
pub access_token_expires_at: Option<DateTime<Utc>>,
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
pub quota_observed_at: Option<DateTime<Utc>>,
/// 快照级限流事实(Provider 物化)。
pub quota_limit_reached: bool,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
pub quota: QuotaState,
pub last_error_reason: Option<AccountErrorReason>,
pub last_error_message: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
@@ -132,8 +150,8 @@ pub struct NewProviderAccount {
pub access_token_expires_at: Option<DateTime<Utc>>,
pub next_refresh_at: Option<DateTime<Utc>>,
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -152,7 +170,7 @@ impl fmt::Debug for NewProviderAccount {
.field("provider_kind", &self.provider_kind)
.field("name", &self.name)
.field("enabled", &self.enabled)
.field("availability", &self.availability)
.field("credential_state", &self.credential_state)
.field("provider_credentials_json", &"[REDACTED]")
.finish_non_exhaustive()
}
@@ -297,24 +315,13 @@ impl fmt::Debug for ProviderCredentialUpdate {
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ProviderAccountObservation {
pub account_id: String,
pub availability: AccountAvailability,
pub provider_quota_json: Option<JsonObject>,
pub availability_observed_at: DateTime<Utc>,
pub quota_observed_at: Option<DateTime<Utc>>,
/// 快照级限流事实(Provider 物化)。
pub quota_limit_reached: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountStateUpdate {
pub account_id: String,
pub expected_revision: Revision,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
pub credential_state: CredentialState,
pub credential_observed_at: DateTime<Utc>,
pub error_reason: Option<AccountErrorReason>,
pub message: Option<String>,
}
@@ -325,31 +332,20 @@ impl ProviderAccountStateUpdate {
}
}
impl ProviderAccountObservation {
pub fn validate(&self) -> StoreResult<()> {
require_nonempty(ENTITY, "account_id", &self.account_id)?;
if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() {
return Err(invalid("quota JSON and quota_observed_at must agree"));
}
if let Some(quota) = &self.provider_quota_json {
validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?;
}
Ok(())
}
}
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
provider_quota_json, quota_limit_reached, last_error_message,
availability_observed_at, quota_observed_at, created_at, updated_at
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, credential_state,
provider_quota_json, quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
last_error_reason, last_error_message,
credential_observed_at, quota_observed_at, created_at, updated_at
from provider_accounts where id = $1";
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
provider_quota_json, quota_limit_reached, last_error_message,
availability_observed_at, quota_observed_at, created_at, updated_at
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, credential_state,
provider_quota_json, quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
last_error_reason, last_error_message,
credential_observed_at, quota_observed_at, created_at, updated_at
from provider_accounts
where id = any($1::text[]) and provider_kind = $2
order by id";
@@ -398,7 +394,13 @@ pub(crate) fn core_account_from_summary(
summary.upstream_account_id,
summary.plan_type,
)
.with_runtime_state(summary.enabled, summary.availability)
.with_account_facts(
summary.enabled,
summary.credential_state,
summary.quota,
summary.last_error_reason,
summary.last_error_message,
)
.with_refresh_schedule(
summary.has_refresh_token,
summary.next_refresh_at.map(Into::into),
@@ -430,9 +432,20 @@ pub(crate) fn account_summary_from_row(
let revision = row
.try_get::<i64, _>("credential_revision")
.map_err(|_| invalid("invalid credential revision"))?;
let availability = row
.try_get::<String, _>("availability")
.map_err(|_| invalid("invalid availability"))?;
let credential_state = row
.try_get::<String, _>("credential_state")
.map_err(|_| invalid("invalid credential_state"))?;
let quota_access_state = parse_quota_access_state(&get::<String>(&row, "quota_access_state")?)?;
let quota_evidence = parse_quota_evidence(get(&row, "quota_evidence")?)?;
let quota_access_observed_at = get::<Option<DateTime<Utc>>>(&row, "quota_access_observed_at")?;
let quota_reset_at = get::<Option<DateTime<Utc>>>(&row, "quota_reset_at")?;
let quota = QuotaState::from_persisted(
quota_access_state,
quota_evidence,
quota_access_observed_at.map(Into::into),
quota_reset_at.map(Into::into),
)
.ok_or_else(|| invalid("invalid persisted quota fact"))?;
Ok(ProviderAccountSummary {
id: get(&row, "id")?,
provider_kind: get(&row, "provider_kind")?,
@@ -447,10 +460,10 @@ pub(crate) fn account_summary_from_row(
access_token_expires_at: get(&row, "access_token_expires_at")?,
next_refresh_at: get(&row, "next_refresh_at")?,
enabled: get(&row, "enabled")?,
availability: parse_availability(&availability)?,
availability_observed_at: get(&row, "availability_observed_at")?,
quota_observed_at: get(&row, "quota_observed_at")?,
quota_limit_reached: get(&row, "quota_limit_reached")?,
credential_state: parse_credential_state(&credential_state)?,
credential_observed_at: get(&row, "credential_observed_at")?,
quota,
last_error_reason: parse_error_reason(get(&row, "last_error_reason")?)?,
last_error_message: get(&row, "last_error_message")?,
created_at: get(&row, "created_at")?,
updated_at: get(&row, "updated_at")?,
@@ -0,0 +1,53 @@
//! 持久账号事实与可丢失冷却事实的统一状态投影。
use std::{collections::BTreeMap, time::SystemTime};
use futures::future::join_all;
use gateway_core::{
engine::credential::{
AccountStatusFacts, AccountStatusProjection, ProviderAccountId, resolve_account_status,
},
provider_ports::ProviderCooldownPort,
};
use super::ProviderAccountSummary;
#[must_use]
pub(crate) fn account_status_projection(
account: &ProviderAccountSummary,
now: SystemTime,
rate_limited_until: Option<SystemTime>,
) -> AccountStatusProjection {
resolve_account_status(
&AccountStatusFacts {
enabled: account.enabled,
credential_state: account.credential_state,
access_token_expires_at: account.access_token_expires_at.map(Into::into),
quota: account.quota,
rate_limited_until,
last_error_reason: account.last_error_reason,
last_error_message: account.last_error_message.clone(),
},
now,
)
}
/// 尽力读取账号当前有效的 429 冷却。
///
/// 冷却属于可丢失的账号级运行时事实:端口不可用、单条读取失败或脏账号 ID
/// 都视为没有冷却,不污染持久账号状态。凭据轮换不会解除上游对同一账号的限流。
pub(crate) async fn load_rate_limited_until(
cooldowns: Option<&dyn ProviderCooldownPort>,
accounts: &[ProviderAccountSummary],
now: SystemTime,
) -> BTreeMap<String, SystemTime> {
let Some(cooldowns) = cooldowns else {
return BTreeMap::new();
};
let reads = accounts.iter().map(|account| async move {
let account_id = ProviderAccountId::new(account.id.clone()).ok()?;
let cooldown = cooldowns.read(&account_id).await.ok().flatten()?;
(cooldown.until() > now).then(|| (account.id.clone(), cooldown.until()))
});
join_all(reads).await.into_iter().flatten().collect()
}
@@ -31,7 +31,7 @@ pub struct SnapshotRuntimeSettings {
pub struct CredentialRevisionVectorEntry {
pub provider_account_id: String,
pub credential_revision: Revision,
pub availability_observed_at: DateTime<Utc>,
pub credential_observed_at: DateTime<Utc>,
pub quota_observed_at: Option<DateTime<Utc>>,
}
@@ -108,7 +108,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
async fn credential_revision_vector(&self) -> StoreResult<Vec<CredentialRevisionVectorEntry>> {
let rows = sqlx::query_as::<_, (String, i64, DateTime<Utc>, Option<DateTime<Utc>>)>(
"select id, credential_revision, availability_observed_at, quota_observed_at
"select id, credential_revision, credential_observed_at, quota_observed_at
from provider_accounts order by id",
)
.fetch_all(&self.pool)
@@ -119,7 +119,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
Ok(CredentialRevisionVectorEntry {
provider_account_id: row.0,
credential_revision: revision_from_i64(row.1)?,
availability_observed_at: row.2,
credential_observed_at: row.2,
quota_observed_at: row.3,
})
})
@@ -512,7 +512,7 @@ impl RedisProviderLeaseCoordinator {
last_started_at: signal.last_started_at.map(Into::into),
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -1,4 +1,4 @@
//! Provider account availability 的可重建 Redis cache fence。
//! Provider credential state 的可重建 Redis cache fence。
use std::time::Duration;
@@ -7,7 +7,7 @@ use chrono::{DateTime, Utc};
use redis::{Script, aio::ConnectionManager};
use gateway_core::engine::credential::{
AccountAvailability, CredentialRevision, OpaqueProviderData, ProviderAccountId,
CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccountId,
};
use gateway_core::provider_ports::{
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCredentialState,
@@ -24,7 +24,7 @@ if current >= tonumber(ARGV[1]) then return 0 end
redis.call('HSET', KEYS[1],
'revision', ARGV[1],
'enabled', ARGV[2],
'availability', ARGV[3],
'credential_state', ARGV[3],
'observed_at_ms', ARGV[4])
redis.call('PEXPIRE', KEYS[1], ARGV[5])
return 1
@@ -44,7 +44,7 @@ pub struct CredentialStateCache {
pub provider_account_id: String,
pub revision: Revision,
pub enabled: bool,
pub availability: String,
pub credential_state: String,
pub observed_at: DateTime<Utc>,
}
@@ -55,8 +55,8 @@ impl CredentialStateCache {
"provider_account_id",
&self.provider_account_id,
)?;
if AccountAvailability::parse(&self.availability).is_none() {
return Err(invalid("availability is not registered"));
if CredentialState::parse(&self.credential_state).is_none() {
return Err(invalid("credential_state is not registered"));
}
Ok(())
}
@@ -172,7 +172,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
.key(self.key(&state.provider_account_id)?)
.arg(state.revision.get())
.arg(u8::from(state.enabled))
.arg(&state.availability)
.arg(&state.credential_state)
.arg(state.observed_at.timestamp_millis())
.arg(ttl_ms)
.invoke_async::<i64>(&mut connection)
@@ -195,12 +195,12 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
.arg(self.key(provider_account_id)?)
.arg("revision")
.arg("enabled")
.arg("availability")
.arg("credential_state")
.arg("observed_at_ms")
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("read credential state"))?;
let (Some(revision), Some(enabled), Some(availability), Some(observed_at_ms)) = values
let (Some(revision), Some(enabled), Some(credential_state), Some(observed_at_ms)) = values
else {
return Ok(None);
};
@@ -208,7 +208,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
provider_account_id: provider_account_id.to_owned(),
revision: Revision::new(revision)?,
enabled: enabled == 1,
availability,
credential_state,
observed_at: DateTime::from_timestamp_millis(observed_at_ms)
.ok_or_else(|| invalid("observed_at is invalid"))?,
};
@@ -299,7 +299,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
revision: Revision::new(state.credential_revision().get())
.map_err(|_| provider_invalid("encode credential state"))?,
enabled: state.enabled(),
availability: state.availability().as_str().to_owned(),
credential_state: state.credential_state().as_str().to_owned(),
observed_at,
};
CredentialStateRepository::cache_credential_state(
@@ -329,13 +329,13 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
.map_err(|_| provider_invalid("decode credential state"))?;
let revision = CredentialRevision::new(state.revision.get())
.map_err(|_| provider_invalid("decode credential state"))?;
let availability = AccountAvailability::parse(&state.availability)
let credential_state = CredentialState::parse(&state.credential_state)
.ok_or_else(|| provider_invalid("decode credential state"))?;
Ok(ProviderCredentialState::new(
account_id,
revision,
state.enabled,
availability,
credential_state,
state.observed_at.into(),
))
})
@@ -55,8 +55,8 @@ async fn merged_first_attempt_insert_should_match_sequential_semantics() {
id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind,
provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, enabled, availability,
availability_observed_at, created_at, updated_at
has_refresh_token, access_token_expires_at, enabled, credential_state,
credential_observed_at, created_at, updated_at
) values (
'acct_merged', 'openai', 'merged', null, 'user-merged', null, null, 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', $1, $1, $1
@@ -134,26 +134,27 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
"insert into provider_accounts (
id, provider_kind, name, upstream_user_id, authentication_kind,
provider_credentials_json, credential_revision, has_refresh_token,
access_token_expires_at, enabled, availability,
availability_observed_at, created_at, updated_at
access_token_expires_at, enabled, credential_state,
quota_access_state, quota_evidence, quota_access_observed_at,
credential_observed_at, created_at, updated_at
) values
('acct_available', 'openai', 'available', 'user-available', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
$1, $1, $1),
('acct_expired', 'openai', 'expired', 'user-expired', 'oauth',
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready',
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready', 'allowed', null, $1,
$1, $1, $1),
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
$1, $1, $1),
('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'usage_limit_reached', $1,
$1, $1, $1),
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned', 'allowed', null, $1,
$1, $1, $1),
('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready',
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', 'allowed', null, $1,
$1, $1, $1)",
)
.bind(now)
@@ -173,22 +174,22 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
assert_eq!(
(
metrics.total,
metrics.enabled,
metrics.active,
metrics.unavailable,
metrics.expired,
metrics.normal,
metrics.quota_exhausted,
metrics.rate_limited,
metrics.disabled,
metrics.banned,
metrics.error,
),
(6, 5, 2, 4, 1, 1, 1, 1),
(6, 2, 1, 0, 1, 2),
);
// 首页按调度可用性聚合:配额耗尽也属于不可用。
assert_eq!(
metrics.unavailable,
metrics.quota_exhausted + metrics.expired + metrics.disabled + metrics.banned
metrics.total,
metrics.normal
+ metrics.quota_exhausted
+ metrics.rate_limited
+ metrics.disabled
+ metrics.error
);
assert_eq!(metrics.total, metrics.active + metrics.unavailable);
database.close().await;
}
@@ -202,35 +203,36 @@ async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligibl
"insert into provider_accounts (
id, provider_kind, name, upstream_user_id, authentication_kind,
provider_credentials_json, credential_revision, has_refresh_token,
access_token_expires_at, enabled, availability, quota_limit_reached,
availability_observed_at, created_at, updated_at
access_token_expires_at, enabled, credential_state,
quota_access_state, quota_evidence, quota_access_observed_at,
credential_observed_at, created_at, updated_at
) values
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
$1, $1, $1),
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
$1, $1, $1),
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1,
$1, $1, $1),
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1,
$1, $1, $1),
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
$1, $1, $1),
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1,
$1, $1, $1),
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', 'exhausted', 'provider_denied', $1,
$1, $1, $1),
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', 'unknown', null, null,
$1, $1, $1),
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', 'allowed', null, $1,
$1, $1, $1)",
)
.bind(now)
@@ -258,36 +260,21 @@ async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligibl
assert_eq!(
(
metrics.total,
metrics.enabled,
metrics.active,
metrics.normal,
metrics.rate_limited,
metrics.quota_exhausted,
metrics.expired,
metrics.invalid,
metrics.disabled,
metrics.banned,
metrics.unavailable,
metrics.error,
),
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
(9, 1, 2, 3, 1, 2),
);
assert_eq!(
metrics.total,
metrics.active
metrics.normal
+ metrics.rate_limited
+ metrics.quota_exhausted
+ metrics.expired
+ metrics.invalid
+ metrics.disabled
+ metrics.banned
);
assert_eq!(
metrics.unavailable,
metrics.rate_limited
+ metrics.quota_exhausted
+ metrics.expired
+ metrics.invalid
+ metrics.disabled
+ metrics.banned
+ metrics.error
);
database.close().await;
}
@@ -1013,8 +1000,8 @@ async fn seed_observability_facts(
id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind,
provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, enabled, availability,
availability_observed_at, created_at, updated_at
has_refresh_token, access_token_expires_at, enabled, credential_state,
credential_observed_at, created_at, updated_at
) values (
'acct_observe', 'openai', 'primary', 'account@example.invalid',
'user-observe', null, 'pro', 'oauth', '{}'::jsonb, 1, false, $1 + interval '1 day',
@@ -10,16 +10,15 @@ use gateway_admin::{
SortDirection,
},
observability::TimeRange,
provider_credentials::{
CredentialAvailabilityFilter, CredentialListQuery, CredentialListWindow,
},
provider_credentials::{CredentialListQuery, CredentialListWindow, CredentialStateFilter},
},
ports::store::AccountStore,
};
use gateway_core::engine::credential::{
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, OpaqueProviderData, PlaintextCredential, ProviderAccountId,
ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, CredentialState, OpaqueProviderData, PlaintextCredential,
ProviderAccountId, ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate,
QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaState,
QuotaWriteOutcome,
};
use gateway_core::routing::ProviderKind;
@@ -59,7 +58,7 @@ async fn provider_account_should_allow_missing_upstream_user_id() {
let mut pending = account("acct_pending_identity", "unused");
pending.upstream_user_id = None;
pending.email = None;
pending.availability = AccountAvailability::Ready;
pending.credential_state = CredentialState::Ready;
repository
.insert_provider_account(pending)
.await
@@ -71,8 +70,11 @@ async fn provider_account_should_allow_missing_upstream_user_id() {
.expect("load account")
.expect("stored account");
assert_eq!(
(stored.summary.upstream_user_id, stored.summary.availability,),
(None, AccountAvailability::Unknown)
(
stored.summary.upstream_user_id,
stored.summary.credential_state,
),
(None, CredentialState::Unknown)
);
database.close().await;
@@ -92,18 +94,19 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
}
let revision = CredentialRevision::new(1).expect("revision");
for (id, remaining) in [("acct_quota_a", 20), ("acct_quota_b", 80)] {
let observed_at = SystemTime::now();
let outcome = repository
.compare_and_swap_quota(QuotaObservation {
account_id: ProviderAccountId::new(id).expect("account id"),
expected_revision: revision,
quota: Some(OpaqueProviderData::new(
quota: OpaqueProviderData::new(
json!({"remaining": remaining})
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
),
observed_at,
state: QuotaState::allowed(observed_at),
})
.await
.expect("persist quota");
@@ -123,15 +126,12 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
assert_eq!(observations.len(), 2);
assert_eq!(observations[0].account_id.as_str(), "acct_quota_a");
assert_eq!(observations[0].expected_revision, revision);
assert_eq!(
observations[0]
.quota
.as_ref()
.expect("quota")
.expose_to_provider()["remaining"],
20
assert_eq!(observations[0].quota.expose_to_provider()["remaining"], 20);
assert!(
observations
.iter()
.all(|value| value.state.access() == QuotaAccessState::Allowed)
);
assert!(observations.iter().all(|value| value.observed_at.is_some()));
assert_eq!(
current_revision(&database.pool).await,
1,
@@ -163,14 +163,14 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
let quota = |marker: &str, observed_at| QuotaObservation {
account_id: account_id.clone(),
expected_revision: revision,
quota: Some(OpaqueProviderData::new(
quota: OpaqueProviderData::new(
json!({"marker": marker})
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(observed_at),
limit_reached: None,
),
observed_at,
state: QuotaState::allowed(observed_at),
};
assert_eq!(
repository
@@ -192,8 +192,9 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::QuotaExhausted,
credential_state: CredentialState::Invalid,
observed_at: newer,
error_reason: Some(AccountErrorReason::CredentialInvalid),
})
.await
.expect("persist newer account state");
@@ -203,8 +204,9 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::Ready,
credential_state: CredentialState::Ready,
observed_at: older,
error_reason: None,
})
.await
.is_err(),
@@ -216,23 +218,20 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
.await
.expect("load fenced account")
.expect("fenced account");
assert_eq!(current.availability(), AccountAvailability::QuotaExhausted);
assert_eq!(current.credential_state(), CredentialState::Invalid);
let observed = repository
.get_quotas(std::slice::from_ref(&account_id))
.await
.expect("load fenced quota")
.pop()
.expect("fenced quota");
assert_eq!(
observed.quota.expect("quota document").expose_to_provider()["marker"],
"newer"
);
assert_eq!(observed.quota.expose_to_provider()["marker"], "newer");
database.close().await;
}
#[tokio::test]
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
async fn credential_error_message_is_persisted_and_cleared_on_recovery() {
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
return;
};
@@ -249,8 +248,9 @@ async fn account_state_message_is_persisted_and_cleared_on_recovery() {
message: Some("upstream returned 402 payment required".to_owned()),
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::QuotaExhausted,
credential_state: CredentialState::Invalid,
observed_at: SystemTime::now(),
error_reason: Some(AccountErrorReason::CredentialInvalid),
})
.await
.expect("persist error message");
@@ -270,8 +270,9 @@ async fn account_state_message_is_persisted_and_cleared_on_recovery() {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::Ready,
credential_state: CredentialState::Ready,
observed_at: SystemTime::now(),
error_reason: None,
})
.await
.expect("recover account");
@@ -299,39 +300,34 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
let mut beta = account("acct_beta", "user-beta");
beta.provider_kind = "xai".to_owned();
beta.email = Some("beta@example.invalid".to_owned());
beta.availability = AccountAvailability::Banned;
beta.credential_state = CredentialState::Banned;
let mut charlie = account("acct_charlie", "user-charlie");
charlie.email = Some("charlie@example.invalid".to_owned());
let mut invalid = account("acct_invalid", "user-invalid");
invalid.email = Some("invalid@example.invalid".to_owned());
invalid.availability = AccountAvailability::Invalid;
invalid.credential_state = CredentialState::Invalid;
let mut disabled = account("acct_disabled", "user-disabled");
disabled.email = Some("disabled@example.invalid".to_owned());
disabled.enabled = false;
let mut quota_exhausted = account("acct_quota_exhausted", "user-quota-exhausted");
quota_exhausted.email = Some("quota-exhausted@example.invalid".to_owned());
quota_exhausted.availability = AccountAvailability::QuotaExhausted;
for account in [alpha, beta, charlie, invalid, disabled, quota_exhausted] {
repository
.insert_provider_account(account)
.await
.expect("insert account list fixture");
}
repository
.compare_and_swap_quota(QuotaObservation {
account_id: ProviderAccountId::new("acct_charlie").expect("account id"),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
quota: Some(OpaqueProviderData::new(
json!({"rate_limit": {"limit_reached": true}})
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: Some(true),
})
.await
.expect("seed rate-limited account");
for account_id in ["acct_charlie", "acct_quota_exhausted"] {
let observed_at = SystemTime::now();
repository
.apply_quota_access(QuotaAccessChange {
account_id: ProviderAccountId::new(account_id).expect("account id"),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
})
.await
.expect("seed exhausted account");
}
seed_model_request(
&database.pool,
@@ -425,7 +421,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
usage_page
.items
.iter()
.map(|account| account.id.as_str())
.map(|item| item.account.id.as_str())
.collect::<Vec<_>>(),
["acct_charlie", "acct_beta"]
);
@@ -448,7 +444,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
last_used_page
.items
.iter()
.map(|account| account.id.as_str())
.map(|item| item.account.id.as_str())
.collect::<Vec<_>>(),
["acct_beta", "acct_charlie"]
);
@@ -466,8 +462,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
.expect("filter account directory");
assert_eq!(filtered.total, 1);
assert_eq!(filtered.summary, usage_page.summary);
assert_eq!(filtered.items[0].id, "acct_alpha");
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
assert_eq!(filtered.items[0].account.id, "acct_alpha");
assert_eq!(filtered.items[0].account.provider_kind.as_str(), "openai");
let error_accounts = store
.list_accounts(AccountListQuery {
@@ -481,8 +477,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
.await
.expect("filter error accounts");
assert_eq!(error_accounts.items.len(), 2);
assert_eq!(error_accounts.items[0].id, "acct_beta");
assert_eq!(error_accounts.items[1].id, "acct_invalid");
assert_eq!(error_accounts.items[0].account.id, "acct_beta");
assert_eq!(error_accounts.items[1].account.id, "acct_invalid");
database.close().await;
}
@@ -498,11 +494,11 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
&format!("user-xai-{index:03}"),
);
credential.provider_kind = "xai".to_owned();
credential.availability = match index {
0 => AccountAvailability::Expired,
1 => AccountAvailability::Banned,
2 => AccountAvailability::Invalid,
_ => AccountAvailability::Ready,
credential.credential_state = match index {
0 => CredentialState::Expired,
1 => CredentialState::Banned,
2 => CredentialState::Invalid,
_ => CredentialState::Ready,
};
repository
.insert_provider_account(credential)
@@ -516,7 +512,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
.list_credentials(
&provider,
CredentialListQuery {
availability: None,
credential_state: None,
enabled: None,
window: CredentialListWindow::All,
},
@@ -530,7 +526,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
.list_credentials(
&provider,
CredentialListQuery {
availability: None,
credential_state: None,
enabled: None,
window: CredentialListWindow::Page {
cursor: None,
@@ -547,10 +543,10 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
.list_credentials(
&provider,
CredentialListQuery {
availability: Some(CredentialAvailabilityFilter::AnyOf(vec![
gateway_admin::model::accounts::AccountAvailability::Expired,
gateway_admin::model::accounts::AccountAvailability::Banned,
gateway_admin::model::accounts::AccountAvailability::Invalid,
credential_state: Some(CredentialStateFilter::AnyOf(vec![
CredentialState::Expired,
CredentialState::Banned,
CredentialState::Invalid,
])),
enabled: None,
window: CredentialListWindow::All,
@@ -851,7 +847,7 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
let mut updated = account("acct_admin_reimport", "user-admin-upsert");
updated.name = "updated import".to_owned();
updated.provider_credentials_json = credential_json("updated-import-secret");
updated.availability = AccountAvailability::Banned;
updated.credential_state = CredentialState::Banned;
let imported = repository
.import_provider_accounts(ImportProviderAccounts {
scope: scope.clone(),
@@ -871,7 +867,7 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
String,
) = sqlx::query_as(
"select name, provider_credentials_json, credential_revision, provider_quota_json,
availability
credential_state
from provider_accounts where id = 'acct_admin_upsert'",
)
.fetch_one(&database.pool)
@@ -920,8 +916,8 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() {
access_token_expires_at: Some(Utc::now() + TimeDelta::minutes(5)),
next_refresh_at: None,
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: Utc::now(),
credential_state: CredentialState::Ready,
credential_observed_at: Utc::now(),
})
.await
.expect("seed provider account");
@@ -1041,32 +1037,32 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
provider_kind: "openai".to_owned(),
};
let mut cooldown_account = account("acct_admin_b", "user-admin-b");
cooldown_account.availability = AccountAvailability::Ready;
let imported = repository
.import_provider_accounts(ImportProviderAccounts {
scope: scope.clone(),
accounts: vec![account("acct_admin_a", "user-admin-a"), cooldown_account],
accounts: vec![
account("acct_admin_a", "user-admin-a"),
account("acct_admin_b", "user-admin-b"),
],
audit: audit("audit_account_batch", "import_batch", "provider_accounts"),
})
.await
.expect("import provider accounts");
assert_eq!(imported.config_revision.get(), 2);
let ready: (bool, String) =
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
sqlx::query_as("select enabled, credential_state from provider_accounts where id = $1")
.bind("acct_admin_a")
.fetch_one(&database.pool)
.await
.expect("load ready imported account");
assert_eq!(ready, (true, "ready".to_owned()));
let cooldown: (bool, String) =
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
let second: (bool, String) =
sqlx::query_as("select enabled, credential_state from provider_accounts where id = $1")
.bind("acct_admin_b")
.fetch_one(&database.pool)
.await
.expect("load cooldown imported account");
assert!(cooldown.0);
assert_eq!(cooldown.1, "ready");
assert_eq!(second, (true, "ready".to_owned()));
repository
.import_provider_accounts(ImportProviderAccounts {
@@ -1113,7 +1109,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
sqlx::query(
"update provider_accounts
set availability = 'expired'
set credential_state = 'expired'
where id = $1",
)
.bind("acct_admin_a")
@@ -1137,7 +1133,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
assert_eq!(rotation.config_revision.get(), 3);
assert_eq!(rotation.credential_revision.get(), 2);
let restored: (String, String, Option<String>) = sqlx::query_as(
"select availability, upstream_user_id, upstream_account_id
"select credential_state, upstream_user_id, upstream_account_id
from provider_accounts where id = $1",
)
.bind("acct_admin_a")
@@ -1243,12 +1239,10 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
let scope = ProviderAccountAdminScope {
provider_kind: "openai".to_owned(),
};
let mut exhausted = account("acct_rotation_quota", "user-rotation-quota");
exhausted.availability = AccountAvailability::QuotaExhausted;
repository
.import_provider_accounts(ImportProviderAccounts {
scope: scope.clone(),
accounts: vec![exhausted],
accounts: vec![account("acct_rotation_quota", "user-rotation-quota")],
audit: audit(
"audit_rotation_quota_import",
"import",
@@ -1257,6 +1251,15 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
})
.await
.expect("import exhausted account");
let observed_at = SystemTime::now();
repository
.apply_quota_access(QuotaAccessChange {
account_id: ProviderAccountId::new("acct_rotation_quota").expect("account ID"),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
})
.await
.expect("persist quota exhaustion");
repository
.rotate_provider_account(RotateProviderAccount {
@@ -1273,13 +1276,13 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
.await
.expect("rotate exhausted account credential");
let availability: String =
sqlx::query_scalar("select availability from provider_accounts where id = $1")
let quota_access_state: String =
sqlx::query_scalar("select quota_access_state from provider_accounts where id = $1")
.bind("acct_rotation_quota")
.fetch_one(&database.pool)
.await
.expect("load exhausted account after credential rotation");
assert_eq!(availability, "quota_exhausted".to_owned());
assert_eq!(quota_access_state, "exhausted".to_owned());
database.close().await;
}
@@ -1298,26 +1301,25 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
.await
.expect("insert xAI account");
let observed_at = SystemTime::now();
repository
.apply_state_change(AccountStateChange {
message: None,
.apply_quota_access(QuotaAccessChange {
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
state: QuotaState::exhausted(QuotaEvidence::AccountLimitReached, observed_at, None),
})
.await
.expect("persist xAI free quota exhaustion");
let current: (String, String) = sqlx::query_as(
"select provider_kind, availability
"select provider_kind, quota_access_state
from provider_accounts where id = $1",
)
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("load persisted xAI quota state");
assert_eq!(current, ("xai".to_owned(), "quota_exhausted".to_owned()),);
assert_eq!(current, ("xai".to_owned(), "exhausted".to_owned()),);
database.close().await;
}
@@ -1338,25 +1340,29 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
let observed_at = SystemTime::now();
repository
.apply_state_change(AccountStateChange {
message: None,
.apply_quota_access(QuotaAccessChange {
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Ready,
observed_at,
state: QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
observed_at,
Some(observed_at + Duration::from_secs(60)),
),
})
.await
.expect("persist xAI resettable usage limit");
let current: (String, String) = sqlx::query_as(
"select provider_kind, availability
let current: (String, String, Option<chrono::DateTime<Utc>>) = sqlx::query_as(
"select provider_kind, quota_access_state, quota_reset_at
from provider_accounts where id = $1",
)
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("load persisted xAI usage limit state");
assert_eq!(current, ("xai".to_owned(), "ready".to_owned()));
assert_eq!(current.0, "xai");
assert_eq!(current.1, "exhausted");
assert!(current.2.is_some());
database.close().await;
}
@@ -1373,7 +1379,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
let account_id = ProviderAccountId::new("acct_disabled_refresh").expect("account ID");
let mut disabled = account(account_id.as_str(), "user-disabled-refresh");
disabled.enabled = false;
disabled.availability = AccountAvailability::Expired;
disabled.credential_state = CredentialState::Expired;
repository
.import_provider_accounts(ImportProviderAccounts {
scope: scope.clone(),
@@ -1392,8 +1398,9 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Ready,
credential_state: CredentialState::Ready,
observed_at: SystemTime::now(),
error_reason: None,
})
.await
.expect("disabled state write is a no-op");
@@ -1413,7 +1420,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
.expect("refresh disabled account credential");
let current: (bool, String, serde_json::Value, i64) = sqlx::query_as(
"select enabled, availability, provider_credentials_json, credential_revision
"select enabled, credential_state, provider_credentials_json, credential_revision
from provider_accounts where id = $1",
)
.bind(account_id.as_str())
@@ -1443,8 +1450,8 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
access_token_expires_at: Some(Utc::now() + TimeDelta::hours(1)),
next_refresh_at: None,
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: Utc::now(),
credential_state: CredentialState::Ready,
credential_observed_at: Utc::now(),
}
}
@@ -1570,13 +1577,6 @@ fn provider_credentials_are_redacted_from_debug() {
assert!(!format!("{object:?}").contains(secret));
}
#[test]
fn quota_exhausted_account_accepts_cooldown_runtime_facts() {
let mut imported = account("acct_quota_cooldown", "user-quota-cooldown");
imported.availability = AccountAvailability::QuotaExhausted;
assert!(imported.validate().is_ok());
}
#[test]
fn imported_ready_account_passes_validation() {
let imported = account("acct_ready_import", "user-ready-import");
@@ -108,7 +108,7 @@ async fn insert_refreshable_account(
"insert into provider_accounts (
id, provider_kind, name, upstream_user_id, authentication_kind,
provider_credentials_json, has_refresh_token, access_token_expires_at,
next_refresh_at, availability, availability_observed_at, created_at, updated_at
next_refresh_at, credential_state, credential_observed_at, created_at, updated_at
) values ($1, 'openai', $1, $1, 'oauth', '{}'::jsonb, true, $2, $3,
'ready', now(), now(), now())",
)
@@ -722,8 +722,8 @@ async fn seed_account(
id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind,
provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, enabled, availability,
availability_observed_at, created_at, updated_at
has_refresh_token, access_token_expires_at, enabled, credential_state,
credential_observed_at, created_at, updated_at
) values (
$1, 'openai', $2, $3, 'user-' || $1, null, null, $4,
'{}'::jsonb, 1, false, $5 + interval '1 day', true, 'ready',
@@ -19,7 +19,7 @@ fn credential_state_rejects_provider_specific_status() {
provider_account_id: "account-1".to_owned(),
revision: Revision::new(1).expect("positive revision"),
enabled: true,
availability: "codex_special".to_owned(),
credential_state: "codex_special".to_owned(),
observed_at: Utc::now(),
};
assert!(state.validate().is_err());
+15 -69
View File
@@ -25,9 +25,8 @@ use gateway_admin::model::{MutationActor, MutationContext, Revision};
use gateway_admin::ports::provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind};
use gateway_core::accounting::Money;
use gateway_core::engine::credential::{
AccountAvailability, CredentialRevision, LoadedCredential, NewProviderAccount,
OpaqueProviderData, PlaintextCredential, ProviderAccount, ProviderAccountId,
ProviderAccountStore,
CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData,
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
};
use gateway_core::error::StoreErrorKind;
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
@@ -424,14 +423,7 @@ impl ProviderAdmin for OpenAiAdminProvider {
.await
.map_err(map_quota_error)?
};
let mut quota = project_quota(snapshot, &account);
quota.rate_limited_until = self
.quota
.rate_limited_until(account.id(), account.revision())
.await
.map_err(map_quota_error)?
.map(DateTime::<Utc>::from);
Ok(quota)
Ok(project_quota(snapshot, &account))
}
async fn models(
@@ -519,7 +511,6 @@ fn prepared_create(
account,
credential,
} = prepared;
let availability = account.availability();
Ok(PreparedCredentialCreate {
account_id: account.id().clone(),
provider_kind: account.provider().clone(),
@@ -534,8 +525,8 @@ fn prepared_create(
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
enabled: account.enabled(),
availability,
availability_observed_at: observed_at,
credential_state: account.credential_state(),
credential_observed_at: observed_at,
})
}
@@ -627,7 +618,13 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
account.upstream_account_id.clone(),
account.plan_type.clone(),
)
.with_runtime_state(account.enabled, account.availability)
.with_account_facts(
account.enabled,
account.credential_state,
account.quota,
account.last_error_reason,
account.last_error_message.clone(),
)
.with_refresh_schedule(
account.has_refresh_token,
account.next_refresh_at.map(SystemTime::from),
@@ -687,7 +684,6 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
@@ -696,13 +692,10 @@ fn project_quota(
snapshot: Option<CodexAccountQuotaSnapshot>,
account: &ProviderAccount,
) -> ProviderQuota {
// 展示前先滚动已过期窗口,避免 reset 后的旧窗口值悬挂。
let mut quota = snapshot
.map(|snapshot| project_quota_snapshot(snapshot.roll_expired_windows(SystemTime::now())))
.map(project_quota_snapshot)
.unwrap_or_else(empty_quota);
if account.availability() == AccountAvailability::QuotaExhausted {
force_confirmed_exhaustion_projection(&mut quota);
}
quota.limit_reached = account.quota().is_exhausted();
quota
}
@@ -717,7 +710,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
);
provider_data.insert(
"exhausted".to_owned(),
Value::Bool(snapshot.fact().exhausted()),
Value::Bool(snapshot.quota().is_exhausted()),
);
let windows: Vec<ProviderQuotaWindow> = snapshot
.windows()
@@ -766,7 +759,6 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
refresh_token_expires_at: None,
windows,
limit_reached,
rate_limited_until: None,
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
provider_data,
))),
@@ -788,52 +780,6 @@ fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool {
windows.iter().any(|window| window.limit_reached)
}
fn force_confirmed_exhaustion_projection(quota: &mut ProviderQuota) {
// 402 已确认耗尽:只覆盖 Admin 展示投影,原始 Provider JSON 仍作为
// 后续判断窗口是否真正恢复的基线。
quota.limit_reached = true;
quota.provider_data = Some(ProviderDocument::new(OpaqueProviderData::new(
Map::from_iter([
(
"remaining_percent".to_owned(),
Value::Number(Number::from(0)),
),
("exhausted".to_owned(), Value::Bool(true)),
]),
)));
if quota.windows.is_empty() {
quota.windows.push(ProviderQuotaWindow {
key: "confirmed-quota-exhaustion".to_owned(),
group: "other".to_owned(),
label: "额度".to_owned(),
source: None,
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
window_seconds: None,
used_percent: Some(100.0),
reset_at: None,
limit_reached: true,
local_usage: None,
provider_data: None,
});
return;
}
// 选已用比例最高的窗口强制 100%,保留其自身 reset_at。
let index = quota
.windows
.iter()
.enumerate()
.max_by(|(_, left), (_, right)| {
left.used_percent
.unwrap_or(-1.0)
.total_cmp(&right.used_percent.unwrap_or(-1.0))
})
.map(|(index, _)| index)
.unwrap_or(0);
quota.windows[index].used_percent = Some(100.0);
quota.windows[index].limit_reached = true;
}
const fn quota_group(kind: CodexQuotaWindowKind) -> &'static str {
match kind {
CodexQuotaWindowKind::Monthly => "monthly",
@@ -68,11 +68,6 @@ impl OpenAiConfig {
self.quota.refresh_policy()
}
#[must_use]
pub const fn quota_skip_exhausted(&self) -> bool {
self.quota.skip_exhausted
}
#[must_use]
pub fn token_client_config(&self) -> TokenClientConfig {
TokenClientConfig {
@@ -209,14 +204,12 @@ impl CodexWebSocketPoolSettings {
#[serde(deny_unknown_fields)]
pub struct CodexQuotaSettings {
pub refresh_interval_minutes: u64,
pub skip_exhausted: bool,
}
impl Default for CodexQuotaSettings {
fn default() -> Self {
Self {
refresh_interval_minutes: 15,
skip_exhausted: true,
}
}
}
@@ -10,9 +10,9 @@ use std::time::SystemTime;
use chrono::{DateTime, FixedOffset, Utc};
use gateway_core::engine::credential::{
AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential,
AccountErrorReason, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential,
NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountIdentity,
ProviderAccountUpdate,
ProviderAccountUpdate, QuotaEvidence, QuotaState,
};
use gateway_core::provider_ports::{
ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest,
@@ -433,7 +433,13 @@ impl CodexCredentialAdmin {
Some(input.verified_account.chatgpt_account_id),
input.verified_account.plan_type,
)
.with_runtime_state(input.enabled, AccountAvailability::Ready)
.with_account_facts(
input.enabled,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
)
.with_refresh_schedule(
input.secret.refresh_token.is_some(),
optional_time(input.next_refresh_at),
@@ -479,13 +485,16 @@ impl CodexCredentialAdmin {
optional_time(input.access_token_expires_at),
)
.with_profile(email, upstream_account_id, plan_type)
.with_runtime_state(
.with_account_facts(
input.enabled,
if has_upstream_user_id {
AccountAvailability::Ready
CredentialState::Ready
} else {
AccountAvailability::Unknown
CredentialState::Unknown
},
QuotaState::unknown(),
(!has_upstream_user_id).then_some(AccountErrorReason::AccountUnverified),
None,
)
.with_refresh_schedule(
input.secret.refresh_token.is_some(),
@@ -898,10 +907,10 @@ impl CodexCredentialAdminService {
if !account_ids.insert(account_id.clone()) {
return Err(CodexCredentialAdminError::InvalidInput);
}
let (enabled, availability) =
import_runtime_state(candidate.status.as_deref())?;
let facts =
parse_cpr_account_facts(candidate.status.as_deref(), SystemTime::now())?;
let mut account =
self.prepare_agent_identity_import(&candidate, account_id, enabled)?;
self.prepare_agent_identity_import(&candidate, account_id, facts.enabled)?;
let upstream_user_id = account
.account
.upstream_user_id()
@@ -915,7 +924,7 @@ impl CodexCredentialAdminService {
if !upstream_identities.insert((upstream_user_id, upstream_account_id)) {
return Err(CodexCredentialAdminError::InvalidInput);
}
account.account = account.account.with_runtime_state(enabled, availability);
account.account = facts.apply(account.account);
account
}
};
@@ -1048,7 +1057,13 @@ impl CodexCredentialAdminService {
Some(upstream_account_id),
candidate.plan_type.clone(),
)
.with_runtime_state(enabled, AccountAvailability::Ready)
.with_account_facts(
enabled,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
)
.with_refresh_schedule(false, None);
Ok(NewProviderAccount {
account,
@@ -1065,11 +1080,13 @@ fn cpr_status(account: &ProviderAccount) -> &'static str {
if !account.enabled() {
return "disabled";
}
match account.availability() {
AccountAvailability::QuotaExhausted => "quota_exhausted",
AccountAvailability::Expired | AccountAvailability::Invalid => "expired",
AccountAvailability::Banned => "banned",
AccountAvailability::Unknown | AccountAvailability::Ready => "active",
if account.quota().is_exhausted() {
return "quota_exhausted";
}
match account.credential_state() {
CredentialState::Expired | CredentialState::Invalid => "expired",
CredentialState::Banned => "banned",
CredentialState::Unknown | CredentialState::Ready => "active",
}
}
@@ -1292,20 +1309,65 @@ fn normalize_import_text(value: &str, max_bytes: usize) -> Option<String> {
.then(|| value.to_owned())
}
fn import_runtime_state(
struct ImportedCprAccountFacts {
enabled: bool,
credential_state: CredentialState,
quota: QuotaState,
error_reason: Option<AccountErrorReason>,
}
impl ImportedCprAccountFacts {
fn apply(self, account: ProviderAccount) -> ProviderAccount {
account.with_account_facts(
self.enabled,
self.credential_state,
self.quota,
self.error_reason,
None,
)
}
}
fn parse_cpr_account_facts(
status: Option<&str>,
) -> Result<(bool, AccountAvailability), CodexCredentialAdminError> {
observed_at: SystemTime,
) -> Result<ImportedCprAccountFacts, CodexCredentialAdminError> {
match status
.map(str::trim)
.unwrap_or("active")
.to_ascii_lowercase()
.as_str()
{
"active" => Ok((true, AccountAvailability::Ready)),
"disabled" => Ok((false, AccountAvailability::Ready)),
"expired" => Ok((true, AccountAvailability::Expired)),
"quota_exhausted" => Ok((true, AccountAvailability::QuotaExhausted)),
"banned" => Ok((true, AccountAvailability::Banned)),
"active" => Ok(ImportedCprAccountFacts {
enabled: true,
credential_state: CredentialState::Ready,
quota: QuotaState::unknown(),
error_reason: None,
}),
"disabled" => Ok(ImportedCprAccountFacts {
enabled: false,
credential_state: CredentialState::Ready,
quota: QuotaState::unknown(),
error_reason: None,
}),
"expired" => Ok(ImportedCprAccountFacts {
enabled: true,
credential_state: CredentialState::Expired,
quota: QuotaState::unknown(),
error_reason: Some(AccountErrorReason::CredentialExpired),
}),
"quota_exhausted" => Ok(ImportedCprAccountFacts {
enabled: true,
credential_state: CredentialState::Ready,
quota: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
error_reason: None,
}),
"banned" => Ok(ImportedCprAccountFacts {
enabled: true,
credential_state: CredentialState::Banned,
quota: QuotaState::unknown(),
error_reason: Some(AccountErrorReason::AccountBanned),
}),
_ => Err(CodexCredentialAdminError::InvalidInput),
}
}
@@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime};
use chrono::{DateTime, Utc};
use gateway_core::engine::credential::{
AccountAvailability, OpaqueProviderData, ProviderAccount, ProviderAccountId,
CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
};
use gateway_core::engine::provider::ProviderCatalogGeneration;
use gateway_core::provider_ports::{
@@ -780,12 +780,8 @@ fn eligible_catalog_account(account: &ProviderAccount, now: SystemTime) -> bool
&& account
.access_token_expires_at()
.is_none_or(|expires_at| expires_at > now)
&& match account.availability() {
AccountAvailability::Unknown
| AccountAvailability::Ready
| AccountAvailability::QuotaExhausted => true,
AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid => false,
}
&& matches!(
account.credential_state(),
CredentialState::Unknown | CredentialState::Ready
)
}
@@ -0,0 +1,46 @@
//! 额度接口失败到规范化账号事实的唯一分类边界。
use gateway_core::engine::credential::{AccountErrorReason, CredentialState, QuotaEvidence};
use reqwest::StatusCode;
use serde_json::Value;
use crate::transport::CodexClientError;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum QuotaEndpointFailure {
Credential {
state: CredentialState,
reason: AccountErrorReason,
},
Exhausted(QuotaEvidence),
}
/// 额度 endpoint 只有明确 402 能改变账号事实。401/403、429、5xx 与传输失败
/// 均不足以判断凭据或账号状态,留给 OAuth 与真实推理请求确认。
pub(crate) fn classify_quota_endpoint_failure(
error: &CodexClientError,
) -> Option<QuotaEndpointFailure> {
let CodexClientError::Upstream { status, body, .. } = error else {
return None;
};
if *status != StatusCode::PAYMENT_REQUIRED {
return None;
}
if is_deactivated_workspace(body) {
return Some(QuotaEndpointFailure::Credential {
state: CredentialState::Banned,
reason: AccountErrorReason::AccountBanned,
});
}
Some(QuotaEndpointFailure::Exhausted(
QuotaEvidence::PaymentRequired,
))
}
fn is_deactivated_workspace(body: &str) -> bool {
serde_json::from_str::<Value>(body)
.ok()
.is_some_and(|value| {
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
})
}
@@ -1,9 +1,9 @@
//! Codex quota 服务编排:请求级额度投影、被动/主动同步与 429 合成。
//! Codex quota 服务编排:权威额度事实、展示投影、主动/被动同步与 429 冷却。
//!
//! - [`snapshot`]:快照/窗口解析、聚合、O(1) 滚动与调度信号。
//! - [`recovery`]:402 恢复证据与 availability 转换。
//! - [`evidence`]:额度接口错误到凭据/额度事实的分类。
pub(crate) mod recovery;
pub(crate) mod evidence;
pub(crate) mod snapshot;
pub use snapshot::{
@@ -13,12 +13,14 @@ pub use snapshot::{
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::time::{Duration, Instant, SystemTime};
use chrono::Utc;
use gateway_core::engine::credential::{
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, QuotaWriteOutcome,
AccountErrorReason, AccountQuotaSignals, CredentialRevision, CredentialState,
OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore,
QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaState,
QuotaWriteOutcome,
};
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
use gateway_protocol::openai::events::{
@@ -36,10 +38,7 @@ use crate::transport::{CodexBackendClient, CodexClientError, CodexRequestContext
use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeCredential};
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
use recovery::{
authoritative_quota_snapshot_state, quota_state_reason, quota_state_transition,
quota_success_state,
};
use evidence::{QuotaEndpointFailure, classify_quota_endpoint_failure};
use snapshot::{
parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation,
scheduling_signals_from_snapshot,
@@ -48,7 +47,8 @@ use snapshot::{
const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60);
pub(crate) const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60);
const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5);
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60);
const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(30 * 60);
const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60);
/// 首次 OAuth 异步观察失败时,由既有 quota worker 兜底重试的单轮上限。
const INITIAL_QUOTA_SYNC_BATCH: usize = 100;
// 5xx 上游拒绝的短退避重试预算;指数退避 1s/2s,吞掉瞬时抖动。
@@ -130,6 +130,12 @@ pub struct CodexCredentialQuotaService {
scheduling: CodexQuotaSchedulingProjection,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum QuotaRefreshAuthority {
ObserveAccess,
PreserveAccess,
}
#[derive(Clone, Default)]
struct CodexQuotaSchedulingProjection {
state: Arc<RwLock<CodexQuotaProjectionState>>,
@@ -299,12 +305,7 @@ impl CodexQuotaSchedulingProjection {
) -> Vec<ProviderAccount> {
let candidates = accounts
.into_iter()
.filter_map(|account| {
let limit_reached = self
.signals(&account)
.is_some_and(|signals| signals.limit_reached());
quota_refresh_candidate(account, now, limit_reached)
})
.filter_map(|account| quota_refresh_candidate(account, now))
.collect::<Vec<_>>();
let candidate_ids = candidates
.iter()
@@ -319,8 +320,8 @@ impl CodexQuotaSchedulingProjection {
.last_periodic_refresh_at
.retain(|account_id, _| candidate_ids.contains(account_id));
// 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只兜底
// 已耗尽账号,且不依据 reset 时间跳过复核;每个账号仍至少间隔 30 分钟。
// 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只复核
// 已耗尽且已到 reset/fallback 时刻的账号;重复复核至少间隔 30 分钟。
let mut reserved = Vec::new();
for account in candidates {
if !periodic_quota_refresh_due(&state, account.id(), refreshed_at) {
@@ -335,16 +336,13 @@ impl CodexQuotaSchedulingProjection {
}
}
fn quota_refresh_candidate(
account: ProviderAccount,
now: SystemTime,
limit_reached: bool,
) -> Option<ProviderAccount> {
// 定时器只兜底已耗尽或任一窗口触顶的账号;
// 正常账号由真实请求的响应头被动同步。
fn quota_refresh_candidate(account: ProviderAccount, now: SystemTime) -> Option<ProviderAccount> {
// reset 到期是“应刷新”的信号,不是“已恢复”的证据。
(eligible_periodic_quota_refresh(&account, now)
&& (account.availability() == AccountAvailability::QuotaExhausted || limit_reached))
.then_some(account)
&& account
.quota()
.exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL))
.then_some(account)
}
fn periodic_quota_refresh_due(
@@ -356,7 +354,7 @@ fn periodic_quota_refresh_due(
.last_periodic_refresh_at
.get(account_id)
.is_none_or(|last| {
now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL
now.saturating_duration_since(*last) >= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL
})
}
@@ -400,70 +398,55 @@ impl CodexCredentialQuotaService {
}
}
/// 写入新快照前读取 core primary 已确认耗尽窗口的 reset,用于判断该窗口是否真的推进。
async fn previous_confirmed_reset(&self, account: &ProviderAccount) -> Option<SystemTime> {
let observations = self
.store
.get_quotas(std::slice::from_ref(account.id()))
.await
.ok()?;
let observation = observations.into_iter().find(|observation| {
observation.account_id == *account.id()
&& observation.expected_revision == account.revision()
})?;
let snapshot = quota_snapshot_from_observation(&observation)?;
snapshot
.core_primary_window()
.and_then(|window| window.reset_at())
.map(SystemTime::from)
}
/// 将已确认的额度拒绝投影为 100%,使展示、调度和恢复共用同一窗口事实。
///
/// `reset_at` 来自官方 `usage_limit_reached` 错误体;缺失时保留当前快照的 reset。
/// 真实推理错误只更新额度访问事实,不伪造 Provider JSON 或展示百分比。
pub(crate) async fn record_confirmed_exhaustion(
&self,
account: &ProviderAccount,
reset_at: Option<i64>,
evidence: QuotaEvidence,
reset_at: Option<SystemTime>,
observed_at: SystemTime,
) -> Result<(), CodexCredentialQuotaError> {
let existing = self
.store
.get_quotas(std::slice::from_ref(account.id()))
.await?
.into_iter()
.find(|observation| {
observation.account_id == *account.id()
&& observation.expected_revision == account.revision()
})
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.unwrap_or_default();
let quota = confirmed_exhaustion_projection(
normalize_quota_window_placeholders(existing),
reset_at,
);
let observed_at = SystemTime::now();
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
.apply_quota_access(QuotaAccessChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(quota.clone())),
observed_at: Some(observed_at),
limit_reached: Some(true),
state: QuotaState::exhausted(evidence, observed_at, reset_at),
})
.await?;
if outcome == QuotaWriteOutcome::Conflict {
return Err(CodexCredentialQuotaError::RevisionConflict);
}
if let Ok(snapshot) = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(quota),
) {
self.scheduling.observe(&snapshot);
Ok(())
}
/// 成功推理是额度可访问的权威证据,同时解除账号级 429 冷却。
pub(crate) async fn record_successful_inference(
&self,
account: &ProviderAccount,
observed_at: SystemTime,
) -> Result<(), CodexCredentialQuotaError> {
// 已经发出的并发请求可能晚于耗尽事实成功返回;真实成功仍是独立的
// Allowed 权威证据,但调度不会为了试探恢复而放行耗尽账号。
if account.quota().access() == QuotaAccessState::Exhausted {
let outcome = self
.store
.apply_quota_access(QuotaAccessChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
state: QuotaState::allowed(observed_at),
})
.await?;
if outcome == QuotaWriteOutcome::Conflict {
return Err(CodexCredentialQuotaError::RevisionConflict);
}
}
self.cooldowns
.clear(account.id(), account.revision())
.await
.map_err(|error| CodexCredentialQuotaError::Store {
detail: error.to_string(),
})?;
Ok(())
}
@@ -562,30 +545,27 @@ impl CodexCredentialQuotaService {
);
}
Err(CodexQuotaFetchError::Upstream { account, error }) => {
match quota_state_transition(&error) {
// 402:真额度耗尽,写入 QuotaExhausted(worker 按 reset_at 自动恢复)。
Some(AccountAvailability::QuotaExhausted) => {
match classify_quota_endpoint_failure(&error) {
Some(QuotaEndpointFailure::Exhausted(evidence)) => {
summary.exhausted += 1;
self.persist_upstream_state(
&account,
AccountAvailability::QuotaExhausted,
&error,
observed_at,
)
.await;
if let Err(write_error) = self
.record_confirmed_exhaustion(&account, evidence, None, observed_at)
.await
{
summary.transient += 1;
tracing::warn!(
account_id = %account.id(),
error = %write_error,
"OpenAI quota exhaustion fact write failed"
);
}
}
Some(AccountAvailability::Banned) => {
Some(QuotaEndpointFailure::Credential { state, reason }) => {
summary.banned += 1;
self.persist_upstream_state(
&account,
AccountAvailability::Banned,
&error,
observed_at,
)
.await;
self.persist_credential_failure(&account, state, reason, observed_at)
.await;
}
// usage 401/403 与 429/503 都不写 availability;前者等待 OAuth 或真实请求确认。
None | Some(_) => {
None => {
summary.transient += 1;
tracing::warn!(
account_id = %account.id(),
@@ -613,7 +593,7 @@ impl CodexCredentialQuotaService {
let observed = self.store.get_quotas(&account_ids).await?;
let observed_ids = observed
.into_iter()
.filter_map(|observation| observation.observed_at.map(|_| observation.account_id))
.map(|observation| observation.account_id)
.collect::<BTreeSet<_>>();
Ok(accounts
.iter()
@@ -633,79 +613,40 @@ impl CodexCredentialQuotaService {
observed_at: SystemTime,
summary: &mut CodexQuotaSyncSummary,
) -> Result<(), CodexCredentialQuotaError> {
let refreshed_snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
value,
)?;
let previous_reset_at = self.previous_confirmed_reset(account).await;
let object = normalize_quota_window_placeholders(
value
.as_object()
.cloned()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
);
let object = if account.availability() == AccountAvailability::QuotaExhausted
&& quota_success_state(
account.availability(),
&refreshed_snapshot,
previous_reset_at,
)
.is_none()
{
confirmed_exhaustion_projection(
object,
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
)
} else {
object
};
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(object.clone()),
)?;
let fact = snapshot.fact();
let state = account.quota().merge_observation(snapshot.quota());
let snapshot = snapshot.with_quota_state(state);
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(object)),
observed_at: Some(observed_at),
limit_reached: Some(fact.exhausted()),
quota: OpaqueProviderData::new(object),
observed_at,
state,
})
.await?;
if outcome == QuotaWriteOutcome::Conflict {
summary.stale += 1;
return Ok(());
}
let Some(current) = self.store.get_account(account.id()).await? else {
summary.stale += 1;
return Ok(());
};
if current.revision() != account.revision() {
summary.stale += 1;
return Ok(());
}
self.scheduling.observe(&snapshot);
if fact.exhausted() {
if snapshot.quota().is_exhausted() {
summary.exhausted += 1;
} else {
summary.updated += 1;
}
if let Some(availability) = authoritative_quota_snapshot_state(
current.availability(),
&refreshed_snapshot,
previous_reset_at,
) {
let _ = self
.repository
.apply_state(&current, availability, observed_at)
.await;
}
Ok(())
}
@@ -722,11 +663,6 @@ impl CodexCredentialQuotaService {
.limits
.values()
.any(|details| passive_rate_limit_snapshot(details).is_some());
let previous_reset_at = if account.availability() == AccountAvailability::QuotaExhausted {
self.previous_confirmed_reset(account).await
} else {
None
};
let existing = self
.store
.get_quotas(std::slice::from_ref(account.id()))
@@ -736,17 +672,14 @@ impl CodexCredentialQuotaService {
observation.account_id == *account.id()
&& observation.expected_revision == account.revision()
});
let existing_observed_at = existing
.as_ref()
.and_then(|observation| observation.observed_at);
let existing_state = existing.as_ref().map(|observation| observation.state);
let existing = existing
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.map(|observation| observation.quota.into_inner())
.unwrap_or_default();
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
// 也不能借旧快照重新推导 quota state。
if !has_quota_facts {
let Some(observed_at) = existing_observed_at else {
let Some(state) = existing_state else {
return Ok(false);
};
let outcome = self
@@ -754,62 +687,42 @@ impl CodexCredentialQuotaService {
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(merge_passive_quota(
existing,
&rate_limits,
))),
observed_at: Some(observed_at),
limit_reached: None,
quota: OpaqueProviderData::new(merge_passive_quota(existing, &rate_limits)),
observed_at: SystemTime::now(),
state,
})
.await?;
return Ok(outcome != QuotaWriteOutcome::Conflict);
}
let quota = merge_passive_quota(existing, &rate_limits);
let observed_at = SystemTime::now();
let refreshed_snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(quota.clone()),
)?;
let quota = if account.availability() == AccountAvailability::QuotaExhausted {
// 被动成功响应不能解除确认额度耗尽;继续展示 100%,直到权威 usage 快照
// 证明旧 reset 已跨过且新窗口已经推进。
confirmed_exhaustion_projection(
quota,
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
)
} else {
quota
};
let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(quota.clone()),
);
)?;
// 这些 headers 来自一次成功推理,访问结论优先于可能滞后的百分比。
let state = QuotaState::allowed(observed_at);
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(quota)),
observed_at: Some(observed_at),
limit_reached: Some(fact.exhausted()),
quota: OpaqueProviderData::new(quota),
observed_at,
state,
})
.await?;
if outcome == QuotaWriteOutcome::Conflict {
return Ok(false);
}
if let Ok(snapshot) = snapshot {
self.scheduling.observe(&snapshot);
}
self.scheduling.observe(&snapshot);
Ok(true)
}
/// 真实 429 的单一事实入口:写入 Redis 临时限流冷却(`until = now + retry_after`)。
/// availability 与 quota 主窗口(额度重置时间)都不改变——临时限流是独立维度,
/// 凭据与额度主窗口(额度重置时间)都不改变——临时限流是独立维度,
/// 到期由 Redis key 过期自动解除,不污染配额耗尽状态。
/// 已有更晚的冷却不会被缩短(put_if_later)。
pub async fn apply_rate_limit_429(
@@ -838,7 +751,6 @@ impl CodexCredentialQuotaService {
pub async fn rate_limited_until(
&self,
account_id: &ProviderAccountId,
credential_revision: CredentialRevision,
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
CodexCredentialQuotaError::Store {
@@ -848,9 +760,6 @@ impl CodexCredentialQuotaService {
else {
return Ok(None);
};
if cooldown.credential_revision() != credential_revision {
return Ok(None);
}
let until = cooldown.until();
if until <= SystemTime::now() {
return Ok(None);
@@ -894,18 +803,17 @@ impl CodexCredentialQuotaService {
{
return Err(CodexCredentialQuotaError::RevisionConflict);
}
let Some(data) = observation.quota else {
if observation.quota.expose_to_provider().is_empty() {
return Ok(None);
};
let observed_at = observation
.observed_at
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
}
let observed_at = observation.observed_at;
let snapshot = parse_account_quota_snapshot(
account_id.clone(),
account.revision(),
observed_at,
&Value::Object(data.expose_to_provider().clone()),
)?;
&Value::Object(observation.quota.expose_to_provider().clone()),
)?
.with_quota_state(observation.state);
self.scheduling.observe(&snapshot);
Ok(Some(snapshot))
}
@@ -915,7 +823,8 @@ impl CodexCredentialQuotaService {
&self,
account_id: &ProviderAccountId,
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
self.refresh_account_snapshot(account_id).await
self.refresh_account_snapshot(account_id, QuotaRefreshAuthority::ObserveAccess)
.await
}
/// 真实限额失败后的异步刷新只补齐展示快照,不允许 usage 快照撤销已确认的失败状态。
@@ -923,12 +832,14 @@ impl CodexCredentialQuotaService {
&self,
account_id: &ProviderAccountId,
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
self.refresh_account_snapshot(account_id).await
self.refresh_account_snapshot(account_id, QuotaRefreshAuthority::PreserveAccess)
.await
}
async fn refresh_account_snapshot(
&self,
account_id: &ProviderAccountId,
authority: QuotaRefreshAuthority,
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
let account = self
.store
@@ -957,104 +868,77 @@ impl CodexCredentialQuotaService {
});
}
Err(CodexQuotaFetchError::Upstream { account, error }) => {
// 402 与明确的停用错误写对应状态;usage 401/403 不写 availability。
if let Some(availability) = quota_state_transition(&error) {
self.persist_upstream_state(&account, availability, &error, observed_at)
.await;
match classify_quota_endpoint_failure(&error) {
Some(QuotaEndpointFailure::Exhausted(evidence)) => {
self.record_confirmed_exhaustion(&account, evidence, None, observed_at)
.await?;
}
Some(QuotaEndpointFailure::Credential { state, reason }) => {
self.persist_credential_failure(&account, state, reason, observed_at)
.await;
}
None => {}
}
return Err(CodexCredentialQuotaError::Upstream {
detail: error.to_string(),
});
}
};
let refreshed_snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&value,
)?;
let previous_reset_at = self.previous_confirmed_reset(&account).await;
let object = normalize_quota_window_placeholders(
value
.as_object()
.cloned()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
);
let object = if account.availability() == AccountAvailability::QuotaExhausted
&& quota_success_state(
account.availability(),
&refreshed_snapshot,
previous_reset_at,
)
.is_none()
{
confirmed_exhaustion_projection(
object,
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
)
} else {
object
};
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(object.clone()),
)?;
let state = match authority {
QuotaRefreshAuthority::ObserveAccess => {
account.quota().merge_observation(snapshot.quota())
}
QuotaRefreshAuthority::PreserveAccess => account.quota(),
};
let snapshot = snapshot.with_quota_state(state);
if self
.store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(object)),
observed_at: Some(observed_at),
limit_reached: Some(snapshot.fact().exhausted()),
quota: OpaqueProviderData::new(object),
observed_at,
state,
})
.await?
== QuotaWriteOutcome::Conflict
{
return Err(CodexCredentialQuotaError::RevisionConflict);
}
let current = self
.store
.get_account(account.id())
.await?
.ok_or(CodexCredentialQuotaError::NotFound)?;
if current.revision() != account.revision() {
return Err(CodexCredentialQuotaError::RevisionConflict);
}
self.scheduling.observe(&snapshot);
if let Some(availability) = authoritative_quota_snapshot_state(
current.availability(),
&refreshed_snapshot,
previous_reset_at,
) {
self.repository
.apply_state(&current, availability, observed_at)
.await?;
}
Ok(snapshot)
}
async fn persist_upstream_state(
async fn persist_credential_failure(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
upstream_error: &CodexClientError,
credential_state: CredentialState,
reason: AccountErrorReason,
observed_at: SystemTime,
) {
let reason = quota_state_reason(upstream_error).map(str::to_owned);
if let Err(error) = self
.repository
.apply_state_with_message(account, availability, observed_at, reason.clone())
.apply_state_with_reason(account, credential_state, observed_at, Some(reason), None)
.await
{
tracing::warn!(
account_id = %account.id(),
?availability,
reason = reason.as_deref().unwrap_or_default(),
?credential_state,
reason = reason.as_str(),
error = %error,
"OpenAI quota upstream state write failed"
"OpenAI quota credential fact write failed"
);
}
}
@@ -1163,10 +1047,8 @@ fn eligible_periodic_quota_refresh(account: &ProviderAccount, now: SystemTime) -
fn eligible_initial_quota_sync(account: &ProviderAccount, now: SystemTime) -> bool {
// 首次观察只兜底刚入库、尚无 quota 快照的账号。已耗尽等运行时状态必须由
// periodic 路径处理,才能保留同一账号的最小复核间隔。
matches!(
account.availability(),
AccountAvailability::Ready | AccountAvailability::Unknown
) && eligible_periodic_quota_refresh(account, now)
account.credential_state() == CredentialState::Ready
&& eligible_periodic_quota_refresh(account, now)
}
fn access_token_is_current(account: &ProviderAccount, now: SystemTime) -> bool {
@@ -1297,71 +1179,6 @@ fn secondary_window_is_placeholder(window: &Map<String, Value>) -> bool {
})
}
/// 用已确认的上游额度拒绝覆盖 core 限额投影。
///
/// 上游 usage 快照可能在结算期间短暂滞后为 98/99% 或 `allowed=true`;保留其他
/// 原始 metadata,但把 core 窗口固定为 100%,直到权威快照证明进入下一窗口。
fn confirmed_exhaustion_projection(
mut quota: Map<String, Value>,
reset_at: Option<i64>,
) -> Map<String, Value> {
let mut rate_limit = quota
.remove("rate_limit")
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
for field in ["primary_window", "secondary_window"] {
let existed = rate_limit.contains_key(field);
let mut window = rate_limit
.remove(field)
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
if field == "primary_window" || existed {
window.insert("used_percent".to_owned(), Value::from(100_u64));
if field == "primary_window"
&& let Some(reset_at) = reset_at
{
window.insert("reset_at".to_owned(), Value::from(reset_at));
}
rate_limit.insert(field.to_owned(), Value::Object(window));
}
}
quota.insert("rate_limit".to_owned(), Value::Object(rate_limit));
quota
}
/// 选择应继续锁定的 reset:新鲜快照仍确认耗尽时可把锁推进到更晚窗口;
/// 否则保留已确认失败的旧窗口,避免 98/99% 快照反向解锁。
fn confirmed_exhaustion_reset_at(
previous_reset_at: Option<SystemTime>,
refreshed_snapshot: &CodexAccountQuotaSnapshot,
) -> Option<i64> {
let refreshed_reset_at = refreshed_snapshot
.core_primary_window()
.and_then(|window| window.reset_at())
.map(SystemTime::from);
let reset_at = match (previous_reset_at, refreshed_reset_at) {
(Some(previous), Some(refreshed))
if refreshed_snapshot.fact().exhausted() && refreshed > previous =>
{
refreshed
}
(Some(previous), _) => previous,
(None, refreshed) => refreshed?,
};
system_time_to_unix_seconds(reset_at)
}
fn system_time_to_unix_seconds(value: SystemTime) -> Option<i64> {
value
.duration_since(UNIX_EPOCH)
.ok()?
.as_secs()
.try_into()
.ok()
}
fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option<Map<String, Value>> {
// 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。
let mut snapshot = Map::new();
@@ -1,102 +0,0 @@
//! 权威额度快照的 availability 转换。
//!
//! 已触顶的 core primary 会立即把可调度账号标为 `QuotaExhausted`;
//! `QuotaExhausted -> Ready` 则只能由同一 credential revision 的权威 usage 快照证明:
//! 新快照未耗尽、core primary reset 已推进到下一窗口,且该窗口用量低于 10%。
use std::time::SystemTime;
use gateway_core::engine::credential::AccountAvailability;
use reqwest::StatusCode;
use serde_json::Value;
use super::snapshot::CodexAccountQuotaSnapshot;
use crate::transport::CodexClientError;
const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0;
/// 基于成功的权威 usage 快照收敛可调度账号的额度状态。
///
/// 只接受上游明确拒绝使用且 core primary 触顶的证据;secondary、附加桶、
/// credits 与 spend control 都不能单独把整个账号标成额度耗尽。
pub(crate) fn authoritative_quota_snapshot_state(
current: AccountAvailability,
snapshot: &CodexAccountQuotaSnapshot,
previous_reset_at: Option<SystemTime>,
) -> Option<AccountAvailability> {
match current {
AccountAvailability::Ready | AccountAvailability::Unknown
if snapshot.authoritative_core_primary_exhausted() =>
{
Some(AccountAvailability::QuotaExhausted)
}
AccountAvailability::QuotaExhausted => {
quota_success_state(current, snapshot, previous_reset_at)
}
AccountAvailability::Ready | AccountAvailability::Unknown => None,
AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid => None,
}
}
/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独覆盖
/// 一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。
///
/// `previous_reset_at` 是写入新快照前读取的 core primary 已确认耗尽窗口。上游可能只
/// 调整 reset 时间而尚未重置额度,因此还要求同一新鲜 primary 窗口的用量低于 10%。
pub(crate) fn quota_success_state(
current: AccountAvailability,
snapshot: &CodexAccountQuotaSnapshot,
previous_reset_at: Option<SystemTime>,
) -> Option<AccountAvailability> {
if current != AccountAvailability::QuotaExhausted || snapshot.fact().exhausted() {
return None;
}
let previous_reset_at = previous_reset_at?;
let primary_window = snapshot.core_primary_window()?;
let refreshed_reset_at = primary_window.reset_at().map(SystemTime::from)?;
let used_percent = primary_window.used_percent()?;
(refreshed_reset_at > previous_reset_at && used_percent < RESET_RECOVERY_MAX_USED_PERCENT)
.then_some(AccountAvailability::Ready)
}
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
let CodexClientError::Upstream { status, body, .. } = error else {
return None;
};
if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) {
return Some(AccountAvailability::Banned);
}
match *status {
// 402:真额度耗尽 → QuotaExhausted(worker 按 reset_at 自动恢复)。
StatusCode::PAYMENT_REQUIRED => Some(AccountAvailability::QuotaExhausted),
// usage 401/403 不能单独判定账号不可用:保持现有状态,交给 OAuth 或真实请求确认。
// 429/503 同样不写 availability,仅记失败待重试。
StatusCode::UNAUTHORIZED
| StatusCode::FORBIDDEN
| StatusCode::TOO_MANY_REQUESTS
| StatusCode::SERVICE_UNAVAILABLE => None,
_ => None,
}
}
/// 配额接口将账号变为非 ready 状态时可安全持久化的上游原因码。
pub(crate) fn quota_state_reason(error: &CodexClientError) -> Option<&'static str> {
match quota_state_transition(error)? {
AccountAvailability::Banned => Some("deactivated_workspace"),
AccountAvailability::QuotaExhausted => Some("payment_required"),
AccountAvailability::Unknown
| AccountAvailability::Ready
| AccountAvailability::Expired
| AccountAvailability::Invalid => None,
}
}
fn is_deactivated_workspace(body: &str) -> bool {
serde_json::from_str::<Value>(body)
.ok()
.is_some_and(|value| {
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
})
}
@@ -7,19 +7,17 @@ use std::time::{Duration, SystemTime};
use chrono::{DateTime, Utc};
use gateway_core::engine::credential::{
AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaObservation,
AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaEvidence, QuotaObservation,
QuotaState,
};
use serde_json::{Map, Value};
use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL};
const CORE_PRIMARY_WINDOW_KEY: &str = "core-primary";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CodexQuotaFact {
remaining_percent: Option<u8>,
resets_at: Option<DateTime<Utc>>,
exhausted: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -112,7 +110,7 @@ pub struct CodexAccountQuotaSnapshot {
credential_revision: CredentialRevision,
observed_at: SystemTime,
fact: CodexQuotaFact,
authoritative_core_primary_exhausted: bool,
quota: QuotaState,
windows: Vec<CodexQuotaWindow>,
}
@@ -143,54 +141,14 @@ impl CodexAccountQuotaSnapshot {
}
#[must_use]
pub(crate) fn core_primary_window(&self) -> Option<&CodexQuotaWindow> {
self.windows
.iter()
.find(|window| window.key() == CORE_PRIMARY_WINDOW_KEY)
pub const fn quota(&self) -> QuotaState {
self.quota
}
/// 上游成功 usage 明确拒绝请求,且 canonical core primary 已确认触顶。
/// 将已持久化的规范化事实覆盖到 raw JSON 展示快照上。
#[must_use]
pub(crate) const fn authoritative_core_primary_exhausted(&self) -> bool {
self.authoritative_core_primary_exhausted
}
/// 投影滚动已过期窗口:`reset_at` 已过的窗口 `used_percent=0`、
/// `limit_reached=false`、`reset_at` 滚到下一个周期(reset 到期自动归零)。
/// 只作用于展示/调度投影,不写回持久化 raw JSON。
///
/// O(1) 计算:`next_reset = reset + (floor((now - reset) / window) + 1) * window`,
/// 不做逐周期线性循环。
#[must_use]
pub fn roll_expired_windows(mut self, now: SystemTime) -> Self {
for window in &mut self.windows {
let Some(reset_at) = window.reset_at else {
continue;
};
let reset = SystemTime::from(reset_at);
if reset > now {
continue;
}
window.used_percent = Some(0.0);
window.limit_reached = false;
let Some(window_seconds) = window.window_seconds.filter(|seconds| *seconds > 0) else {
// 没有合法窗口时长:清除已过期的 reset,等待下一次权威观测。
window.reset_at = None;
continue;
};
let elapsed = now
.duration_since(reset)
.unwrap_or(Duration::ZERO)
.as_secs();
let periods = elapsed / window_seconds + 1;
let Some(after) =
reset.checked_add(Duration::from_secs(periods.saturating_mul(window_seconds)))
else {
window.reset_at = None;
continue;
};
window.reset_at = Some(DateTime::<Utc>::from(after));
}
pub(crate) const fn with_quota_state(mut self, quota: QuotaState) -> Self {
self.quota = quota;
self
}
}
@@ -205,11 +163,6 @@ impl CodexQuotaFact {
pub const fn resets_at(&self) -> Option<DateTime<Utc>> {
self.resets_at
}
#[must_use]
pub const fn exhausted(&self) -> bool {
self.exhausted
}
}
pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCredentialQuotaError> {
@@ -220,13 +173,12 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCre
for limit in canonical_rate_limits(object)? {
aggregate.observe_rate_limit(limit.rate_limit)?;
}
// spend_control 只提供 exhaustion 信号(`reached`),不生成窗口、
// 不参与剩余量聚合(`individual_limit` 等字段无窗口语义)。
// credits / spend_control 只证明文档结构已识别,不参与账号级访问结论。
if let Some(spend_control) = object.get("spend_control") {
aggregate.observe_exhaustion_object(spend_control, "reached")?;
aggregate.observe_metadata_object(spend_control, "reached")?;
}
if let Some(credits) = object.get("credits") {
aggregate.observe_exhaustion_object(credits, "overage_limit_reached")?;
aggregate.observe_metadata_object(credits, "overage_limit_reached")?;
}
if !aggregate.recognized {
return Err(CodexCredentialQuotaError::InvalidCredentialData);
@@ -234,22 +186,24 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCre
Ok(CodexQuotaFact {
remaining_percent: aggregate.remaining_percent,
resets_at: aggregate.resets_at,
exhausted: aggregate.exhausted,
})
}
pub(crate) fn quota_snapshot_from_observation(
observation: &QuotaObservation,
) -> Option<CodexAccountQuotaSnapshot> {
let observed_at = observation.observed_at?;
let quota = observation.quota.as_ref()?;
parse_account_quota_snapshot(
let observed_at = observation.observed_at;
let quota = &observation.quota;
let mut snapshot = parse_account_quota_snapshot(
observation.account_id.clone(),
observation.expected_revision,
observed_at,
&Value::Object(quota.expose_to_provider().clone()),
)
.ok()
.ok()?;
// 归一化列是持久化权威事实;raw JSON 只负责 Provider 展示结构。
snapshot.quota = observation.state;
Some(snapshot)
}
/// 调度投影缓存 TTL:`min(常规 TTL, 最近 active reset - now)`。
@@ -275,46 +229,36 @@ pub(crate) fn quota_projection_ttl(snapshot: &CodexAccountQuotaSnapshot) -> Opti
(!ttl.is_zero()).then_some(ttl)
}
/// 从快照的滚动窗口派生中立调度事实(`limit_reached` = 任一窗口触顶)。
/// 从快照窗口派生容量排序事实;额度访问资格由 `QuotaState` 统一判断。
pub(crate) fn scheduling_signals_from_snapshot(
snapshot: &CodexAccountQuotaSnapshot,
) -> Option<AccountQuotaSignals> {
let rolled = snapshot.clone().roll_expired_windows(SystemTime::now());
quota_scheduling_signals(&rolled)
quota_scheduling_signals(snapshot)
}
pub(crate) fn quota_scheduling_signals(
snapshot: &CodexAccountQuotaSnapshot,
) -> Option<AccountQuotaSignals> {
let now = SystemTime::now();
let mut limit_reached = false;
let mut remaining_rank: Option<u64> = None;
let mut reset_at: Option<SystemTime> = None;
for window in &snapshot.windows {
limit_reached |= window.limit_reached();
if let Some(used) = window
.used_percent()
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
let window_reset_at = window.reset_at().map(SystemTime::from);
let observation_expired = window_reset_at.is_some_and(|reset_at| reset_at <= now);
if !observation_expired
&& let Some(used) = window
.used_percent()
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
{
let rank = ((100.0 - used) * 100.0).round() as u64;
remaining_rank = Some(remaining_rank.map_or(rank, |current| current.min(rank)));
}
if let Some(reset) = window
.reset_at()
.map(SystemTime::from)
.filter(|reset| *reset > now)
{
if let Some(reset) = window_reset_at.filter(|reset| *reset > now) {
reset_at = Some(reset_at.map_or(reset, |current| current.min(reset)));
}
}
// 快照级 limit_reached 是调度排除的充分条件:即使没有任何窗口带
// percent/reset(如顶层限流标记),也必须产出信号;不能返回 None 丢失排除。
if limit_reached || remaining_rank.is_some() || reset_at.is_some() {
Some(AccountQuotaSignals::new(
reset_at,
remaining_rank,
limit_reached,
))
if remaining_rank.is_some() || reset_at.is_some() {
Some(AccountQuotaSignals::new(reset_at, remaining_rank))
} else {
None
}
@@ -330,7 +274,7 @@ pub(crate) fn parse_account_quota_snapshot(
let object = usage
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
let authoritative_core_primary_exhausted = authoritative_core_primary_exhausted(object)?;
let quota = authoritative_quota_state(object, observed_at)?;
let mut windows = Vec::new();
for limit in canonical_rate_limits(object)? {
parse_rate_limit_windows(
@@ -352,7 +296,7 @@ pub(crate) fn parse_account_quota_snapshot(
credential_revision,
observed_at,
fact,
authoritative_core_primary_exhausted,
quota,
windows,
})
}
@@ -365,48 +309,73 @@ fn quota_source_order(source: &str) -> u8 {
}
}
/// 账号状态只能由顶层 canonical Codex 额度桶的明确拒绝来改变。
///
/// `fact.exhausted()` 和窗口 `limit_reached` 都会聚合或继承 secondary、附加桶等事实,
/// 因此不能单独用来将整个账号标为 `QuotaExhausted`。
fn authoritative_core_primary_exhausted(
/// 顶层 account-wide Codex 桶是唯一可改变账号额度访问结论的 quota 响应字段。
fn authoritative_quota_state(
usage: &Map<String, Value>,
) -> Result<bool, CodexCredentialQuotaError> {
observed_at: SystemTime,
) -> Result<QuotaState, CodexCredentialQuotaError> {
let Some(rate_limit) = usage.get("rate_limit") else {
return Ok(false);
return Ok(QuotaState::observed_unknown(observed_at));
};
if rate_limit.is_null() {
return Ok(false);
return Ok(QuotaState::observed_unknown(observed_at));
}
let rate_limit = rate_limit
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
if optional_bool(rate_limit, "allowed")? != Some(false) {
return Ok(false);
let allowed = optional_bool(rate_limit, "allowed")?;
let top_level_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false);
let primary_window = rate_limit
.get("primary_window")
.filter(|value| !value.is_null())
.map(|value| {
value
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
})
.transpose()?;
let primary_reached = primary_window
.map(|window| optional_bool(window, "limit_reached"))
.transpose()?
.flatten()
.unwrap_or(false);
let reset_at = primary_window
.and_then(|window| window.get("reset_at"))
.map(|value| {
value
.as_i64()
.filter(|value| *value > 0)
.and_then(|value| DateTime::<Utc>::from_timestamp(value, 0))
.map(SystemTime::from)
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
})
.transpose()?;
if allowed == Some(true) {
if top_level_reached || primary_reached {
tracing::warn!(
top_level_reached,
primary_reached,
"OpenAI quota returned contradictory access facts; explicit allowed=true wins"
);
}
return Ok(QuotaState::allowed(observed_at));
}
let Some(primary_window) = rate_limit.get("primary_window") else {
return Ok(false);
};
if primary_window.is_null() {
return Ok(false);
if allowed == Some(false) {
return Ok(QuotaState::exhausted(
QuotaEvidence::ProviderDenied,
observed_at,
reset_at,
));
}
let primary_window = primary_window
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
let primary_window_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false)
|| optional_bool(primary_window, "limit_reached")?.unwrap_or(false)
|| primary_window
.get("used_percent")
.map(|value| {
value
.as_f64()
.filter(|value| value.is_finite() && *value >= 0.0)
.map(|value| value >= 100.0)
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
})
.transpose()?
.unwrap_or(false);
Ok(primary_window_reached)
if top_level_reached || primary_reached {
return Ok(QuotaState::exhausted(
QuotaEvidence::AccountLimitReached,
observed_at,
reset_at,
));
}
Ok(QuotaState::observed_unknown(observed_at))
}
/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。
@@ -523,10 +492,9 @@ fn parse_rate_limit_windows(
let object = value
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
// 顶层限流标记是快照级事实:任一窗口都要继承所属
// rate_limit 的 `limit_reached=true` / `allowed=false`。
let top_level_reached = optional_bool(object, "limit_reached")?.unwrap_or(false)
|| optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
let allowed = optional_bool(object, "allowed")?;
let top_level_reached = allowed != Some(true)
&& (optional_bool(object, "limit_reached")?.unwrap_or(false) || allowed == Some(false));
for (name, role) in [
("primary_window", CodexQuotaWindowRole::Primary),
("secondary_window", CodexQuotaWindowRole::Secondary),
@@ -564,12 +532,12 @@ fn parse_rate_limit_windows(
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
})
.transpose()?;
let limit_reached = top_level_reached
|| window
.get("limit_reached")
.and_then(Value::as_bool)
.unwrap_or(false)
|| used_percent.is_some_and(|used| used >= 100.0);
let limit_reached = allowed != Some(true)
&& (top_level_reached
|| window
.get("limit_reached")
.and_then(Value::as_bool)
.unwrap_or(false));
let kind = quota_window_kind(window_seconds);
output.push(CodexQuotaWindow {
key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)),
@@ -649,7 +617,6 @@ const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind)
#[derive(Default)]
struct QuotaAggregate {
recognized: bool,
exhausted: bool,
remaining_percent: Option<u8>,
resets_at: Option<DateTime<Utc>>,
}
@@ -660,8 +627,8 @@ impl QuotaAggregate {
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
self.recognized = true;
self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false);
self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
let _ = optional_bool(object, "limit_reached")?;
let _ = optional_bool(object, "allowed")?;
for key in ["primary_window", "secondary_window"] {
if let Some(window) = object.get(key) {
self.observe_window(window)?;
@@ -682,8 +649,6 @@ impl QuotaAggregate {
.as_f64()
.filter(|value| value.is_finite() && *value >= 0.0)
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
// 子窗口触顶(used>=100)是快照级 exhaustion 事实。
self.exhausted |= used >= 100.0;
let remaining = (100.0 - used).clamp(0.0, 100.0).round() as u8;
self.remaining_percent = Some(
self.remaining_percent
@@ -702,7 +667,7 @@ impl QuotaAggregate {
Ok(())
}
fn observe_exhaustion_object(
fn observe_metadata_object(
&mut self,
value: &Value,
key: &str,
@@ -714,7 +679,7 @@ impl QuotaAggregate {
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
self.recognized = true;
self.exhausted |= optional_bool(object, key)?.unwrap_or(false);
let _ = optional_bool(object, key)?;
Ok(())
}
}
@@ -6,7 +6,9 @@ use std::time::{Duration, SystemTime};
use chrono::{DateTime, Utc};
use futures::{StreamExt as _, stream};
use gateway_core::engine::credential::{AccountAvailability, ProviderAccount, ProviderAccountId};
use gateway_core::engine::credential::{
AccountErrorReason, CredentialState, ProviderAccount, ProviderAccountId,
};
use gateway_core::provider_ports::{
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
@@ -253,7 +255,8 @@ impl CodexCredentialRefreshService {
return self
.persist_terminal(
&due.account,
AccountAvailability::Expired,
CredentialState::Expired,
AccountErrorReason::CredentialExpired,
"refresh_recovery_window_exhausted",
CodexCredentialRefreshOutcome::Invalidated { account_id },
)
@@ -293,7 +296,8 @@ impl CodexCredentialRefreshService {
Err(RefreshFailure::InvalidGrant) => {
self.persist_terminal(
&due.account,
AccountAvailability::Expired,
CredentialState::Expired,
AccountErrorReason::CredentialExpired,
"refresh_invalid_grant",
CodexCredentialRefreshOutcome::Invalidated { account_id },
)
@@ -302,7 +306,8 @@ impl CodexCredentialRefreshService {
Err(RefreshFailure::Banned) => {
self.persist_terminal(
&due.account,
AccountAvailability::Banned,
CredentialState::Banned,
AccountErrorReason::AccountBanned,
"account_banned",
CodexCredentialRefreshOutcome::Banned { account_id },
)
@@ -353,11 +358,9 @@ impl CodexCredentialRefreshService {
&& account.enabled()
&& account.has_refresh_token()
&& !excluded.contains(account.id())
&& !matches!(
account.availability(),
AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid
&& matches!(
account.credential_state(),
CredentialState::Unknown | CredentialState::Ready
)
&& refresh_due_at(account, policy, now).is_some()
});
@@ -463,7 +466,8 @@ impl CodexCredentialRefreshService {
async fn persist_terminal(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
credential_state: CredentialState,
error_reason: AccountErrorReason,
reason: &'static str,
outcome: CodexCredentialRefreshOutcome,
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
@@ -478,10 +482,11 @@ impl CodexCredentialRefreshService {
}
match self
.repository
.apply_state_with_message(
.apply_state_with_reason(
account,
availability,
credential_state,
SystemTime::now(),
Some(error_reason),
Some(reason.to_owned()),
)
.await
@@ -489,7 +494,7 @@ impl CodexCredentialRefreshService {
Ok(()) => {
tracing::warn!(
account_id = %account.id(),
?availability,
?credential_state,
reason,
access_token_expires_at = ?account.access_token_expires_at()
.map(DateTime::<Utc>::from),
@@ -5,8 +5,8 @@ use std::time::SystemTime;
use chrono::{DateTime, Utc};
use gateway_core::engine::credential::{
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, ProviderAccount, ProviderAccountId, ProviderAccountStore,
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, CredentialState, ProviderAccount, ProviderAccountId, ProviderAccountStore,
ProviderAccountUpdate,
};
use gateway_core::routing::ProviderKind;
@@ -220,34 +220,46 @@ impl CodexCredentialRepository {
pub async fn apply_state(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
credential_state: CredentialState,
observed_at: SystemTime,
) -> Result<(), CredentialRepositoryError> {
self.apply_state_with_message(account, availability, observed_at, None)
.await
self.apply_state_with_reason(
account,
credential_state,
observed_at,
credential_state.error_reason(),
None,
)
.await
}
/// 带上游错误描述的状态写入;上游 message 优先,终态缺失 message 时写稳定兜底,
/// 恢复 `Ready` 时清空。
pub async fn apply_state_with_message(
/// 写入凭据事实及其稳定错误原因;额度事实不经过此入口。
pub async fn apply_state_with_reason(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
credential_state: CredentialState,
observed_at: SystemTime,
error_reason: Option<AccountErrorReason>,
message: Option<String>,
) -> Result<(), CredentialRepositoryError> {
if !account.enabled() {
return Ok(());
}
let message = message
.filter(|value| !value.trim().is_empty())
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
let (error_reason, message) = if credential_state == CredentialState::Ready {
(None, None)
} else {
(
error_reason.or_else(|| credential_state.error_reason()),
message.filter(|value| !value.trim().is_empty()),
)
};
self.store
.apply_state_change(AccountStateChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
availability,
credential_state,
observed_at,
error_reason,
message,
})
.await?;
@@ -6,9 +6,10 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
use gateway_core::engine::credential::{
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
AccountCandidate, AccountEligibilityPolicy, AccountErrorReason, AccountFeedbackStats,
AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, AccountSelector,
PreferredAccountSelection, ProviderAccount, ProviderAccountId,
AccountStatus, CredentialState, PreferredAccountSelection, ProviderAccount, ProviderAccountId,
QuotaEvidence,
};
use gateway_core::engine::{AttemptContext, ContinuationAttempt};
use gateway_core::provider_ports::{
@@ -57,8 +58,8 @@ pub enum CodexAccountFailure {
QuotaExhausted,
/// 当前用量窗口已耗尽;到重置时间后可自动恢复。
UsageLimitExhausted {
/// 上游返回的窗口重置时长。
retry_after: Option<Duration>,
/// 上游返回的窗口绝对重置时刻。
reset_at: Option<SystemTime>,
},
/// 账号触发临时用量限制。
RateLimited {
@@ -74,21 +75,6 @@ pub enum CodexAccountFailure {
CloudflarePathBlocked,
}
impl CodexAccountFailure {
/// 上游未提供可展示错误文本时,保留已分类的稳定原因。
const fn fallback_error_reason(self) -> Option<&'static str> {
match self {
Self::CredentialExpired => Some("upstream_credential_expired"),
Self::IdentityVerificationRequired => Some("upstream_identity_verification_required"),
Self::Banned => Some("upstream_account_banned"),
Self::QuotaExhausted => Some("upstream_quota_exhausted"),
Self::UsageLimitExhausted { .. } => Some("upstream_usage_limit_exhausted"),
Self::CloudflarePathBlocked => Some("upstream_cloudflare_path_blocked"),
Self::RateLimited { .. } | Self::CloudflareChallenge { .. } => None,
}
}
}
#[derive(Debug, Clone, Copy)]
struct RiskRecoveryState {
challenge_count: u32,
@@ -127,7 +113,6 @@ pub struct CodexCredentialSelector {
cookie_policy: CodexCookiePolicy,
risk_recovery: Mutex<HashMap<String, RiskRecoveryState>>,
account_feedback: Arc<AccountFeedbackStats>,
skip_exhausted: bool,
}
enum SessionAffinityLookup {
@@ -263,7 +248,6 @@ impl CodexCredentialSelector {
agent_identity: Arc<CodexAgentIdentityTaskService>,
account_feedback: Arc<AccountFeedbackStats>,
cookie_policy: CodexCookiePolicy,
skip_exhausted: bool,
) -> Self {
Self {
provider_kind,
@@ -277,7 +261,6 @@ impl CodexCredentialSelector {
cookie_policy,
risk_recovery: Mutex::new(HashMap::new()),
account_feedback,
skip_exhausted,
}
}
@@ -310,24 +293,17 @@ impl CodexCredentialSelector {
if !diagnostic {
self.quota.prepare_scheduling(&accounts).await;
}
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
let accounts = if diagnostic {
accounts
} else {
let mut retained = Vec::with_capacity(accounts.len());
for account in accounts {
let cooling = self
let mut rate_limits = HashMap::with_capacity(accounts.len());
if !diagnostic {
for account in &accounts {
let until = self
.quota
.rate_limited_until(account.id(), account.revision())
.rate_limited_until(account.id())
.await
.unwrap_or(None);
if cooling.is_none() {
retained.push(account);
}
rate_limits.insert(account.id().clone(), until);
}
retained
};
}
let account_ids = accounts
.iter()
.map(|account| account.id().clone())
@@ -352,11 +328,12 @@ impl CodexCredentialSelector {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
})
.with_provider_quota(self.quota.scheduling_signals(&account))
.with_rate_limit(rate_limits.get(account.id()).copied().flatten())
.with_runtime_health(health.0, health.1);
AccountCandidate { account, signals }
})
@@ -383,41 +360,6 @@ impl CodexCredentialSelector {
return Err(CredentialSelectionError::NoEligibleCredential);
}
let pinned_account = required_account.or_else(|| continuation_account.clone());
let preserve_native_continuation_state = matches!(
request.attempt.continuation_attempt(),
ContinuationAttempt::Native
) && continuation_account.is_some();
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
let scheduling_candidates = if diagnostic {
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
candidates.clone()
} else if self.skip_exhausted {
candidates
.iter()
.filter(|candidate| !candidate.signals.quota_limit_reached)
.cloned()
.collect::<Vec<_>>()
} else {
candidates
.iter()
.cloned()
.map(|mut candidate| {
// 已命中 previous_response_id 的 native continuation 先检查原账号。
// 即使常规调度允许探测额度耗尽账号,也必须保留原账号真实状态,
// 由 Coordinator 据此切换到可携带重放,而非伪装为 Ready。
if !preserve_native_continuation_state
|| pinned_account.as_ref() != Some(candidate.account.id())
{
candidate.account =
quota_exhausted_scheduling_projection(&candidate.account);
}
candidate
})
.collect::<Vec<_>>()
};
let scheduling_candidates = scheduling_candidates.as_slice();
let mut affinity = if diagnostic {
AffinitySelection::default()
} else {
@@ -432,18 +374,6 @@ impl CodexCredentialSelector {
.prepare_cyber_policy_scope(cyber_policy_session_key)
.await;
let mut excluded = request.attempt.excluded_accounts().clone();
if !diagnostic
&& preserve_native_continuation_state
&& let Some(pinned) = pinned_account.as_ref()
&& candidates.iter().any(|candidate| {
candidate.account.id() == pinned && candidate.signals.quota_limit_reached
})
{
// `skip_exhausted = false` 只允许普通新请求探测可疑账号;已绑定的
// native response handle 先将真实触顶状态交给 Coordinator,随后才可
// 通过 ReplayAny 按当前调度策略换号。
excluded.insert(pinned.clone());
}
if let Some(state) = cyber_policy_scope
.as_ref()
.and_then(|scope| scope.state.as_ref())
@@ -476,13 +406,13 @@ impl CodexCredentialSelector {
excluded_accounts: excluded.clone(),
preferred_account: preferred.clone(),
round_robin_cursor,
availability: if diagnostic {
AccountAvailabilityPolicy::BypassForDiagnostic
eligibility: if diagnostic {
AccountEligibilityPolicy::BypassForDiagnostic
} else {
AccountAvailabilityPolicy::Enforce
AccountEligibilityPolicy::Enforce
},
};
let Some(selection) = AccountSelector.select(scheduling_candidates, &context) else {
let Some(selection) = AccountSelector.select(&candidates, &context) else {
return match shortest_retry {
Some(retry_after) => Err(CredentialSelectionError::CapacityUnavailable {
retry_after: Some(retry_after),
@@ -597,10 +527,15 @@ impl CodexCredentialSelector {
self.clear_session_affinity(key).await;
return AffinitySelection::escaped(account_id, AffinityEscapeReason::HardUnavailable);
};
if candidate.account.is_schedulable(now) {
if candidate
.account
.status_projection(now, candidate.signals.rate_limited_until)
.status
== AccountStatus::Normal
{
return AffinitySelection::preferred(account_id);
}
let reason = affinity_unavailable_reason(&candidate.account, now);
let reason = affinity_unavailable_reason(candidate, now);
self.clear_session_affinity(key).await;
AffinitySelection::escaped(account_id, reason)
}
@@ -749,9 +684,7 @@ impl CodexCredentialSelector {
message: Option<String>,
) -> Result<(), CredentialSelectionError> {
let now = SystemTime::now();
let message = message
.filter(|value| !value.trim().is_empty())
.or_else(|| failure.fallback_error_reason().map(str::to_owned));
let message = message.filter(|value| !value.trim().is_empty());
match failure {
CodexAccountFailure::CredentialExpired => {
if account.authentication_kind() == CODEX_AUTHENTICATION_KIND_OAUTH
@@ -772,29 +705,51 @@ impl CodexCredentialSelector {
);
return Ok(());
}
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
.await
}
CodexAccountFailure::IdentityVerificationRequired => {
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
.await
}
CodexAccountFailure::Banned => {
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
.await
}
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
CodexAccountFailure::QuotaExhausted
| CodexAccountFailure::UsageLimitExhausted { .. } => {
self.apply_state_with_message(
self.apply_credential_state(
account,
AccountAvailability::QuotaExhausted,
CredentialState::Expired,
AccountErrorReason::AccessTokenExpired,
now,
message,
)
.await
}
// 429:临时限流只写 quota,availability 不改变。
CodexAccountFailure::IdentityVerificationRequired => {
self.apply_credential_state(
account,
CredentialState::Invalid,
AccountErrorReason::AccountUnverified,
now,
message,
)
.await
}
CodexAccountFailure::Banned => {
self.apply_credential_state(
account,
CredentialState::Banned,
AccountErrorReason::AccountBanned,
now,
message,
)
.await
}
CodexAccountFailure::QuotaExhausted => self
.quota
.record_confirmed_exhaustion(account, QuotaEvidence::PaymentRequired, None, now)
.await
.map_err(|_| CredentialSelectionError::Store),
CodexAccountFailure::UsageLimitExhausted { reset_at } => self
.quota
.record_confirmed_exhaustion(
account,
QuotaEvidence::UsageLimitReached,
reset_at,
now,
)
.await
.map_err(|_| CredentialSelectionError::Store),
// 429:临时限流只写运行时冷却,不改变凭据或额度事实。
CodexAccountFailure::RateLimited { retry_after } => {
self.quota
.apply_rate_limit_429(account, retry_after, now)
@@ -802,7 +757,7 @@ impl CodexCredentialSelector {
.map_err(|_| CredentialSelectionError::Store)?;
Ok(())
}
// Cloudflare 挑战:内存退避表(记录风险计数),不写 availability。
// Cloudflare 挑战:内存退避表(记录风险计数),不写账号事实。
CodexAccountFailure::CloudflareChallenge { retry_after } => {
let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after);
let recovery = now
@@ -815,9 +770,10 @@ impl CodexCredentialSelector {
CodexAccountFailure::CloudflarePathBlocked => {
let blocked = self.record_cloudflare_path_block(account.id(), now);
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
self.apply_state_with_message(
self.apply_credential_state(
account,
AccountAvailability::Invalid,
CredentialState::Invalid,
AccountErrorReason::CredentialInvalid,
now,
message,
)
@@ -830,15 +786,22 @@ impl CodexCredentialSelector {
}
}
async fn apply_state_with_message(
async fn apply_credential_state(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
credential_state: CredentialState,
error_reason: AccountErrorReason,
observed_at: SystemTime,
message: Option<String>,
) -> Result<(), CredentialSelectionError> {
self.repository
.apply_state_with_message(account, availability, observed_at, message)
.apply_state_with_reason(
account,
credential_state,
observed_at,
Some(error_reason),
message,
)
.await?;
Ok(())
}
@@ -895,29 +858,34 @@ impl CodexCredentialSelector {
let Ok(current) = self.current_account(account.id()).await else {
return;
};
// QuotaExhausted 只能由结构化额度事实解除;一次成功响应本身不能证明额度已经恢复。
if current.provider() != &self.provider_kind
|| current.revision() != account.revision()
|| !current.enabled()
|| !matches!(
current.availability(),
AccountAvailability::Unknown
| AccountAvailability::Expired
| AccountAvailability::Invalid
| AccountAvailability::Banned
)
{
return;
}
let observed_at = SystemTime::now();
if current.credential_state() != CredentialState::Ready
&& let Err(error) = self
.repository
.apply_state(&current, CredentialState::Ready, observed_at)
.await
{
tracing::warn!(
account_id = %account.id(),
error = %error,
"OpenAI credential recovery after successful upstream response failed"
);
}
if let Err(error) = self
.repository
.apply_state(&current, AccountAvailability::Ready, SystemTime::now())
.quota
.record_successful_inference(&current, observed_at)
.await
{
tracing::warn!(
account_id = %account.id(),
error = %error,
"OpenAI account state recovery after successful upstream response failed"
"OpenAI quota recovery after successful upstream response failed"
);
}
}
@@ -1045,30 +1013,19 @@ impl CodexCredentialSelector {
}
}
fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderAccount {
if account.availability() == AccountAvailability::QuotaExhausted {
return account
.clone()
.with_runtime_state(account.enabled(), AccountAvailability::Ready);
}
account.clone()
}
fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> AffinityEscapeReason {
if !account.enabled()
|| account
.access_token_expires_at()
.is_some_and(|expires_at| expires_at <= now)
fn affinity_unavailable_reason(
candidate: &AccountCandidate,
now: SystemTime,
) -> AffinityEscapeReason {
match candidate
.account
.status_projection(now, candidate.signals.rate_limited_until)
.status
{
return AffinityEscapeReason::HardUnavailable;
}
match account.availability() {
AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
AccountAvailability::Unknown
| AccountAvailability::Ready
| AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid => AffinityEscapeReason::HardUnavailable,
AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
AccountStatus::RateLimited => AffinityEscapeReason::Cooldown,
AccountStatus::Normal => AffinityEscapeReason::SelectionInvariant,
AccountStatus::Disabled | AccountStatus::Error => AffinityEscapeReason::HardUnavailable,
}
}
@@ -110,7 +110,6 @@ pub async fn initialize(
Arc::clone(&agent_identity),
ports.cooldowns(),
));
let quota_skip_exhausted = config.quota_skip_exhausted();
let selector = Arc::new(CodexCredentialSelector::new(
provider_kind.clone(),
repository.clone(),
@@ -122,7 +121,6 @@ pub async fn initialize(
Arc::clone(&agent_identity),
Arc::clone(&account_feedback),
CodexCookiePolicy::official().map_err(|_| OpenAiInitializeError::CookiePolicy)?,
quota_skip_exhausted,
));
let core_provider: Arc<dyn Provider> = Arc::new(
CodexProvider::new(
+12 -19
View File
@@ -1704,13 +1704,11 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
struct MappedProviderFailure {
error: ProviderError,
account_failure: Option<CodexAccountFailure>,
/// 原始上游错误描述,随 availability 状态一并持久化。
/// 原始上游错误描述,仅在凭据错误状态下持久化。
error_message: Option<String>,
cyber_policy_failure: bool,
set_cookie_headers: Vec<String>,
rate_limit_headers: Vec<(String, String)>,
/// 结构化 usage-limit 错误给出的窗口 reset(Unix 秒)。
confirmed_exhaustion_reset_at: Option<i64>,
observation: Option<ProviderResponseObservation>,
capture_response_cookies: bool,
}
@@ -1724,7 +1722,6 @@ impl MappedProviderFailure {
cyber_policy_failure: false,
set_cookie_headers: Vec::new(),
rate_limit_headers: Vec::new(),
confirmed_exhaustion_reset_at: None,
observation: None,
capture_response_cookies: false,
}
@@ -1882,18 +1879,6 @@ async fn apply_failure(
failure.account_failure,
Some(CodexAccountFailure::QuotaExhausted | CodexAccountFailure::UsageLimitExhausted { .. })
);
if needs_authoritative_quota_refresh
&& let Err(error) = context
.quota
.record_confirmed_exhaustion(account, failure.confirmed_exhaustion_reset_at)
.await
{
tracing::warn!(
account_id = %account.id(),
error = %error,
"Failed to project confirmed OpenAI quota exhaustion"
);
}
if failure.cyber_policy_failure {
context
.selector
@@ -2186,12 +2171,15 @@ fn map_upstream_failure(
}
MappedProviderFailure {
error,
account_failure: account_failure(category, failure.retry_after_seconds),
account_failure: account_failure(
category,
failure.retry_after_seconds,
failure.usage_limit_resets_at,
),
error_message: failure.client_message,
cyber_policy_failure,
set_cookie_headers: failure.set_cookie_headers,
rate_limit_headers: failure.rate_limit_headers,
confirmed_exhaustion_reset_at: failure.usage_limit_resets_at,
observation,
capture_response_cookies: !matches!(
category,
@@ -2245,6 +2233,7 @@ const fn upstream_send_state(phase: CodexUpstreamSendPhase) -> UpstreamSendState
fn account_failure(
category: CodexFailureCategory,
retry_after_seconds: Option<u64>,
usage_limit_resets_at: Option<i64>,
) -> Option<CodexAccountFailure> {
match category {
CodexFailureCategory::CredentialExpired => Some(CodexAccountFailure::CredentialExpired),
@@ -2254,7 +2243,11 @@ fn account_failure(
CodexFailureCategory::Banned => Some(CodexAccountFailure::Banned),
CodexFailureCategory::UsageLimitExhausted => {
Some(CodexAccountFailure::UsageLimitExhausted {
retry_after: retry_after_seconds.map(Duration::from_secs),
reset_at: usage_limit_resets_at
.and_then(|seconds| u64::try_from(seconds).ok())
.and_then(|seconds| {
SystemTime::UNIX_EPOCH.checked_add(Duration::from_secs(seconds))
}),
})
}
CodexFailureCategory::RateLimited => Some(CodexAccountFailure::RateLimited {
+24 -29
View File
@@ -5,9 +5,7 @@ use std::time::{Duration, SystemTime};
use chrono::{DateTime, TimeZone as _, Utc};
use futures::future::BoxFuture;
use gateway_admin::model::accounts::{
AccountAvailability as AdminAccountAvailability, AccountRecord,
};
use gateway_admin::model::accounts::AccountRecord;
use gateway_admin::model::observability::{
CurrencyCost, DesktopReleaseStatus, ProviderBillingInput,
};
@@ -20,8 +18,8 @@ use gateway_admin::model::provider_credentials::{
use gateway_admin::model::{MutationActor, MutationContext, Revision};
use gateway_admin::ports::provider::ProviderAdminErrorKind;
use gateway_core::engine::credential::{
AccountAvailability as CoreAccountAvailability, AccountStateChange, CredentialRevision,
OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation,
CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
ProviderAccountStore, QuotaAccessChange, QuotaEvidence, QuotaObservation, QuotaState,
};
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
use gateway_core::provider_ports::{
@@ -515,15 +513,14 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc
}
}]
});
let observed_at = SystemTime::now();
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at,
state: QuotaState::observed_unknown(observed_at),
})
.await
.expect("persist quota");
@@ -596,7 +593,7 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc
}
#[tokio::test]
async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutating_raw_usage() {
async fn openai_admin_keeps_confirmed_exhaustion_separate_from_raw_usage_display() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_admin_confirmed_exhaustion";
store
@@ -611,11 +608,12 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
.await;
let account = store.account(account_id).expect("stored account");
let reset_at = 1_900_000_000_u64;
let observed_at = SystemTime::now();
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
quota: OpaqueProviderData::new(
json!({
"rate_limit": {
"allowed": true,
@@ -626,19 +624,17 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
),
observed_at,
state: QuotaState::allowed(observed_at),
})
.await
.expect("persist raw quota");
store
.apply_state_change(AccountStateChange {
message: None,
.apply_quota_access(QuotaAccessChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: CoreAccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
state: QuotaState::exhausted(QuotaEvidence::UsageLimitReached, SystemTime::now(), None),
})
.await
.expect("mark account exhausted");
@@ -661,7 +657,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
.expect("project exhausted quota");
assert_eq!(exhausted.windows.len(), 1);
assert_eq!(exhausted.windows[0].used_percent, Some(100.0));
assert_eq!(exhausted.windows[0].used_percent, Some(86.0));
let raw = store
.get_quotas(std::slice::from_ref(account.id()))
.await
@@ -669,17 +665,15 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
.pop()
.expect("raw quota");
assert_eq!(
raw.quota.expect("raw document").expose_to_provider()["rate_limit"]["primary_window"]["used_percent"],
raw.quota.expose_to_provider()["rate_limit"]["primary_window"]["used_percent"],
86
);
store
.apply_state_change(AccountStateChange {
message: None,
.apply_quota_access(QuotaAccessChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: CoreAccountAvailability::Ready,
observed_at: SystemTime::now(),
state: QuotaState::allowed(SystemTime::now()),
})
.await
.expect("recover account");
@@ -735,7 +729,7 @@ async fn openai_admin_provider_rejects_unprepared_mutations_before_store_commit(
stale_record.access_token_expires_at = None;
stale_record.next_refresh_at = None;
stale_record.enabled = false;
stale_record.availability = AdminAccountAvailability::Banned;
stale_record.credential_state = CredentialState::Banned;
let rotation_error = admin
.prepare_rotation(PrepareCredentialRotation {
account: stale_record,
@@ -801,10 +795,11 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
enabled: account.enabled(),
availability: AdminAccountAvailability::Ready,
availability_observed_at: now,
credential_state: account.credential_state(),
credential_observed_at: now,
quota: account.quota(),
last_error_reason: account.last_error_reason(),
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
}
@@ -82,14 +82,6 @@ fn openai_config_restricts_upstream_base_url_to_https_or_loopback_http() {
}
}
#[test]
fn openai_config_keeps_the_quota_exhaustion_scheduling_switch() {
let mut config = valid_config();
config.quota.skip_exhausted = false;
assert!(!config.quota_skip_exhausted());
}
#[test]
fn openai_config_defaults_to_the_provider_owned_operating_values() {
let config = OpenAiConfig::default();
@@ -104,7 +96,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
config.ws_pool.max_connecting,
config.ws_pool.initial_event_timeout_ms,
config.quota.refresh_interval_minutes,
config.quota.skip_exhausted,
config.auth.refresh_enabled,
config.auth.oauth_client_id.as_str(),
config.auth.oauth_token_endpoint.as_str(),
@@ -119,7 +110,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
20_000,
15,
true,
true,
"app_EMoamEEZ73f0CkXaXp7hrann",
"https://auth.openai.com/oauth/token",
)
@@ -9,8 +9,8 @@ use chrono::{TimeZone, Utc};
use ed25519_dalek::pkcs8::EncodePrivateKey as _;
use ed25519_dalek::{Signature, SigningKey, Verifier as _};
use gateway_core::engine::credential::{
AccountAvailability, CredentialRevision, NewProviderAccount, ProviderAccount,
ProviderAccountId, ProviderAccountStore,
CredentialRevision, CredentialState, NewProviderAccount, ProviderAccount, ProviderAccountId,
ProviderAccountStore, QuotaState,
};
use gateway_core::routing::ProviderKind;
use provider_openai::credential::{
@@ -65,7 +65,13 @@ fn agent_account(
Some("agent-account".to_owned()),
Some("pro".to_owned()),
)
.with_runtime_state(true, AccountAvailability::Ready)
.with_account_facts(
true,
CredentialState::Ready,
QuotaState::unknown(),
None,
None,
)
.with_refresh_schedule(false, None);
NewProviderAccount {
account,
@@ -8,9 +8,10 @@ use gateway_core::engine::continuation::{
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
};
use gateway_core::engine::credential::{
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy,
AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _,
QuotaObservation, QuotaWriteOutcome, RotationStrategy,
AccountAttemptFeedback, AccountErrorReason, AccountFeedbackStats, AccountSelectionPolicy,
AccountStateChange, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
ProviderAccountStore as _, QuotaAccessChange, QuotaAccessState, QuotaEvidence,
QuotaObservation, QuotaState, QuotaWriteOutcome, RotationStrategy,
};
use gateway_core::engine::{
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
@@ -105,7 +106,6 @@ fn selector_with_affinity(
leases,
session_affinity,
Arc::new(AccountFeedbackStats::default()),
true,
Arc::new(MemoryCooldownPort::new()),
)
}
@@ -115,7 +115,6 @@ fn selector_with_runtime(
leases: Arc<TestLeaseCoordinator>,
session_affinity: Arc<MemorySessionAffinity>,
account_feedback: Arc<AccountFeedbackStats>,
skip_exhausted: bool,
cooldowns: Arc<dyn ProviderCooldownPort>,
) -> CodexCredentialSelector {
let profile = CodexWireProfileState::new(CodexWireProfile {
@@ -158,10 +157,42 @@ fn selector_with_runtime(
agent_identity,
account_feedback,
CodexCookiePolicy::official().expect("official cookie policy"),
skip_exhausted,
)
}
fn persist_credential_state(
store: &MemoryAccountStore,
account: &ProviderAccount,
credential_state: CredentialState,
) {
block_on(store.apply_state_change(AccountStateChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
credential_state,
observed_at: SystemTime::now(),
error_reason: credential_state.error_reason(),
message: None,
}))
.expect("persist credential state");
}
fn persist_quota_exhaustion(
store: &MemoryAccountStore,
account: &ProviderAccount,
reset_at: Option<SystemTime>,
) {
block_on(store.apply_quota_access(QuotaAccessChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
state: QuotaState::exhausted(
QuotaEvidence::UsageLimitReached,
SystemTime::now(),
reset_at,
),
}))
.expect("persist quota exhaustion");
}
#[test]
fn codec_persists_tokens_as_plaintext_provider_json() {
let encoded = CodexCredentialCodec::encode_new(
@@ -485,7 +516,6 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
Arc::new(TestLeaseCoordinator::default()),
affinity,
Arc::clone(&account_feedback),
true,
Arc::new(MemoryCooldownPort::new()),
);
for _ in 0..4 {
@@ -528,16 +558,7 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
create_account(&store, "acct_first", "at-first");
create_account(&store, "acct_second", "at-second");
let first = store.account("acct_first").expect("first account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: first.id().clone(),
expected_revision: first.revision(),
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
})
.await
.expect("mark first account exhausted");
persist_quota_exhaustion(&store, &first, None);
let affinity = Arc::new(MemorySessionAffinity::default());
let provider = ProviderKind::new("openai").expect("provider");
let key = ProviderSessionAffinityKey::try_new("quota-session").expect("affinity key");
@@ -697,13 +718,15 @@ fn credential_expired_failure_marks_unified_account_expired() {
store
.account("acct_primary")
.expect("account")
.availability(),
AccountAvailability::Expired
.credential_state(),
CredentialState::Expired
);
let account = store.account("acct_primary").expect("expired account");
assert_eq!(
store.last_error_message("acct_primary").as_deref(),
Some("upstream_credential_expired")
account.last_error_reason(),
Some(AccountErrorReason::AccessTokenExpired)
);
assert_eq!(account.last_error_message(), None);
}
#[test]
@@ -734,13 +757,13 @@ fn credential_expired_failure_keeps_expired_oauth_for_bounded_refresh_recovery()
store
.account("acct_primary")
.expect("account retained for refresh")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
}
#[test]
fn rate_limited_failure_records_quota_window_without_changing_availability() {
fn rate_limited_failure_records_runtime_cooldown_without_changing_persisted_facts() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let cooldowns = Arc::new(MemoryCooldownPort::new());
@@ -749,7 +772,6 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
true,
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
);
let attempt = attempt(BTreeSet::new());
@@ -774,10 +796,9 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
))
.expect("record rate-limit failure");
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
let account = store.account("acct_primary").expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
assert_eq!(account.quota().access(), QuotaAccessState::Unknown);
assert!(
store.quota_json("acct_primary").is_none(),
"429 must not synthesize quota window"
@@ -800,7 +821,7 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
.record_failure(
&account,
CodexAccountFailure::UsageLimitExhausted {
retry_after: Some(Duration::from_secs(30)),
reset_at: Some(SystemTime::now() + Duration::from_secs(30)),
},
None,
)
@@ -808,7 +829,11 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
.expect("record usage-limit exhaustion");
let account = store.account("acct_primary").expect("persisted account");
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
assert_eq!(account.quota().access(), QuotaAccessState::Exhausted);
assert_eq!(
account.quota().evidence(),
Some(QuotaEvidence::UsageLimitReached)
);
assert_eq!(store.quota_reads(), 0);
}
@@ -817,14 +842,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
}))
.expect("mark exhausted");
persist_quota_exhaustion(&store, &account, None);
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
block_on(selector.record_failure(
@@ -836,13 +854,13 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
))
.expect("record rate-limit failure");
// 429 不写 availability,402 的 QuotaExhausted 保持不变。
assert_eq!(
store
.account("acct_primary")
.expect("persisted account")
.availability(),
AccountAvailability::QuotaExhausted
.quota()
.access(),
QuotaAccessState::Exhausted
);
}
@@ -857,7 +875,6 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
true,
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
);
@@ -874,8 +891,8 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
store
.account("acct_primary")
.expect("persisted account")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
assert_eq!(store.quota_reads(), 0);
@@ -891,14 +908,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Invalid,
observed_at: SystemTime::now(),
}))
.expect("mark account invalid");
persist_credential_state(&store, &account, CredentialState::Invalid);
let current = store.account("acct_primary").expect("invalid account");
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
@@ -911,34 +921,26 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
))
.expect("record rate-limit failure");
// 429 不写 availability(限流不改变账号可用性),认证终态保持不变。
assert_eq!(
store
.account("acct_primary")
.expect("persisted account")
.availability(),
AccountAvailability::Invalid
.credential_state(),
CredentialState::Invalid
);
}
#[test]
fn successful_upstream_response_recovers_non_quota_terminal_states() {
for stale in [
AccountAvailability::Expired,
AccountAvailability::Invalid,
AccountAvailability::Banned,
CredentialState::Expired,
CredentialState::Invalid,
CredentialState::Banned,
] {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: stale,
observed_at: SystemTime::now(),
}))
.expect("seed stale state");
persist_credential_state(&store, &account, stale);
let current = store.account("acct_primary").expect("stale account");
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
@@ -948,13 +950,34 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
store
.account("acct_primary")
.expect("recovered account")
.availability(),
AccountAvailability::Ready,
.credential_state(),
CredentialState::Ready,
"stale state {stale:?}",
);
}
}
#[test]
fn elapsed_quota_reset_remains_blocked_until_authoritative_recovery() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
persist_quota_exhaustion(
&store,
&account,
Some(SystemTime::now() - Duration::from_secs(1)),
);
let blocked = store.account("acct_primary").expect("blocked account");
assert!(blocked.quota().is_exhausted());
assert_eq!(
blocked
.status_projection(SystemTime::now(), None)
.status
.as_str(),
"quota_exhausted"
);
}
#[test]
fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
let store = Arc::new(MemoryAccountStore::default());
@@ -990,52 +1013,13 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
store
.account("acct_first")
.expect("persisted first account")
.availability(),
.credential_state(),
store
.account("acct_second")
.expect("persisted second account")
.availability(),
.credential_state(),
],
[AccountAvailability::Ready; 2]
);
}
#[test]
fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip_it() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
let account = store.account("acct_primary").expect("account");
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
.expect("mark account exhausted");
let selector = selector_with_runtime(
&store,
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
false,
Arc::new(MemoryCooldownPort::new()),
);
let request_url =
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
let request_attempt = attempt(BTreeSet::new());
let lease = block_on(selector.select(&SelectCodexCredential {
upstream_model: "gpt-5.4",
request_url: &request_url,
attempt: &request_attempt,
session_affinity_key: None,
}))
.expect("select quota exhausted account when skipping is disabled");
assert_eq!(lease.account_id().as_str(), "acct_primary");
assert_eq!(
store
.account("acct_primary")
.expect("persisted account")
.availability(),
AccountAvailability::QuotaExhausted
[CredentialState::Ready; 2]
);
}
@@ -1054,7 +1038,6 @@ fn native_continuation_surfaces_the_original_accounts_quota_status_to_the_coordi
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
false,
Arc::new(MemoryCooldownPort::new()),
);
let continuation = NativeContinuationPin::new(
@@ -1108,14 +1091,13 @@ fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordi
"primary_window": {"used_percent": 98}
}
});
let observed_at = SystemTime::now();
let outcome = block_on(store.compare_and_swap_quota(QuotaObservation {
account_id: original.id().clone(),
expected_revision: original.revision(),
quota: Some(OpaqueProviderData::new(
quota.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(quota.as_object().expect("quota object").clone()),
observed_at,
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
}))
.expect("persist quota signal");
assert!(matches!(outcome, QuotaWriteOutcome::Updated));
@@ -1125,7 +1107,6 @@ fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordi
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
false,
Arc::new(MemoryCooldownPort::new()),
);
let continuation = NativeContinuationPin::new(
@@ -1194,8 +1175,8 @@ fn identity_verification_failure_isolates_only_selected_account() {
store
.account(lease.account_id().as_str())
.expect("selected account")
.availability(),
AccountAvailability::Invalid
.credential_state(),
CredentialState::Invalid
);
let other = if lease.account_id().as_str() == "acct_primary" {
"acct_other"
@@ -1203,13 +1184,16 @@ fn identity_verification_failure_isolates_only_selected_account() {
"acct_primary"
};
assert_eq!(
store.account(other).expect("other account").availability(),
AccountAvailability::Ready
store
.account(other)
.expect("other account")
.credential_state(),
CredentialState::Ready
);
}
#[test]
fn cloudflare_challenge_does_not_change_availability() {
fn cloudflare_challenge_does_not_change_persisted_account_facts() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
@@ -1226,7 +1210,6 @@ fn cloudflare_challenge_does_not_change_availability() {
)
.expect("select account");
// Cloudflare 挑战只走内存退避表,不写 availability(限流不改变账号可用性)。
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CloudflareChallenge { retry_after: None },
@@ -1238,8 +1221,8 @@ fn cloudflare_challenge_does_not_change_availability() {
store
.account("acct_primary")
.expect("account")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
block_on(selector.record_success(lease.account(), None));
}
@@ -1279,15 +1262,15 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
store
.account("acct_primary")
.expect("affected account")
.availability(),
AccountAvailability::Invalid
.credential_state(),
CredentialState::Invalid
);
assert_eq!(
store
.account("acct_other")
.expect("other account")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
}
@@ -1416,7 +1399,8 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
store
.account("acct_primary")
.expect("updated account")
.availability(),
AccountAvailability::QuotaExhausted
.quota()
.access(),
QuotaAccessState::Exhausted
);
}
@@ -7,6 +7,7 @@ use gateway_admin::model::provider_credentials::{
AuthorizationMutationTarget, AuthorizationOwnerBinding, PendingAuthorizationMutation,
};
use gateway_admin::model::{MutationActor, MutationContext};
use gateway_core::engine::credential::AccountStatus;
use gateway_core::routing::ProviderKind;
use provider_openai::credential::token_client::{
AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant,
@@ -182,7 +183,13 @@ async fn first_exchange_uses_official_id_token_claim_mapping_without_signature_v
assert_eq!(account.account.upstream_user_id(), Some("chatgpt-user"));
assert_eq!(account.account.upstream_account_id(), Some("workspace-id"));
assert_eq!(account.account.plan_type(), Some("pro"));
assert!(account.account.is_schedulable(SystemTime::now()));
assert_eq!(
account
.account
.status_projection(SystemTime::now(), None)
.status,
AccountStatus::Normal
);
assert!(
account.account.next_refresh_at().is_none(),
"normal OAuth creation must not persist a margin-derived refresh time"
@@ -201,7 +208,13 @@ async fn first_exchange_permits_missing_identity_claims_but_requires_a_parseable
};
assert_eq!(account.account.email(), Some("profile@example.com"));
assert_eq!(account.account.upstream_user_id(), None);
assert!(!account.account.is_schedulable(SystemTime::now()));
assert_eq!(
account
.account
.status_projection(SystemTime::now(), None)
.status,
AccountStatus::Error
);
let error = complete("header.not-base64.signature".to_owned())
.await
File diff suppressed because it is too large Load Diff
@@ -1,25 +1,15 @@
//! 402 恢复回归。
//!
//! 覆盖 `QuotaExhausted -> Ready` 的证据规则:新 core primary reset 后移 + 新快照用量低于 10%。
//! 额度刷新与凭证事实相互独立的回归测试。
use super::*;
#[tokio::test]
async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_reset() {
async fn quota_refresh_updates_access_fact_without_recovering_credential_error() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_manual_quota_state";
let account_id = "acct_independent_quota_and_credential";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Expired,
observed_at: SystemTime::now(),
})
.await
.expect("seed stale expired state");
persist_credential_state(&store, &account, CredentialState::Expired).await;
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
@@ -45,20 +35,15 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
server.uri(),
);
let exhausted = service
let snapshot = service
.refresh_account(account.id())
.await
.expect("refresh exhausted quota");
let current = store.account(account_id).expect("updated account");
assert!(exhausted.fact().exhausted());
assert_eq!(
store
.account(account_id)
.expect("exhausted account")
.availability(),
AccountAvailability::Expired,
"manual refresh never changes non-QuotaExhausted availability"
);
assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted);
assert_eq!(current.quota().access(), QuotaAccessState::Exhausted);
assert_eq!(current.credential_state(), CredentialState::Expired);
server.reset().await;
Mock::given(method("GET"))
@@ -77,111 +62,39 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
.mount(&server)
.await;
let stale = service
let snapshot = service
.refresh_account(account.id())
.await
.expect("refresh contradictory full quota");
.expect("refresh allowed quota");
let current = store.account(account_id).expect("recovered quota account");
// used=100 是快照级 exhaustion。
assert!(stale.fact().exhausted());
assert_eq!(
store
.account(account_id)
.expect("account after contradictory quota")
.availability(),
AccountAvailability::Expired,
"an unchanged usage window cannot override an expired credential state"
);
server.reset().await;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.and(header(
"authorization",
format!("Bearer token-{account_id}"),
))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"rate_limit": {
"allowed": true,
"limit_reached": false,
"primary_window": {"used_percent": 99, "reset_at": 1_900_000_000}
}
})))
.mount(&server)
.await;
let unchanged_reset = service
.refresh_account(account.id())
.await
.expect("refresh lower usage quota");
assert!(!unchanged_reset.fact().exhausted());
assert_eq!(
store
.account(account_id)
.expect("account after lower usage")
.availability(),
AccountAvailability::Expired,
"a lower used_percent before the recorded reset is not quota recovery for an expired credential"
);
assert_eq!(snapshot.fact().remaining_percent(), Some(0));
assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed);
assert_eq!(current.quota().access(), QuotaAccessState::Allowed);
assert_eq!(current.credential_state(), CredentialState::Expired);
}
#[tokio::test]
async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() {
// 旧 reset 仍在未来;同一主窗口 reset 后移且用量低于 10% 时才恢复 Ready。
async fn explicit_allowance_replaces_confirmed_exhaustion_without_usage_heuristics() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_cross_window_recovery";
let account_id = "acct_authoritative_quota_recovery";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("created account");
let old_reset = Utc::now().timestamp() + 3_600;
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
json!({
"rate_limit": {
"allowed": false,
"limit_reached": true,
"primary_window": {
"used_percent": 100,
"reset_at": old_reset,
"limit_window_seconds": 3600
}
}
})
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
})
.await
.expect("seed old exhausted snapshot");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
})
.await
.expect("mark account exhausted");
persist_quota_state(
&store,
&account,
exhausted_quota(Some(SystemTime::now() + Duration::from_secs(3_600))),
)
.await;
let server = MockServer::start().await;
let future_reset = old_reset + 3_600;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.and(header(
"authorization",
format!("Bearer token-{account_id}"),
))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"rate_limit": {
"allowed": true,
"limit_reached": false,
"primary_window": {"used_percent": 9.9, "reset_at": future_reset}
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
}
})))
.mount(&server)
@@ -195,77 +108,36 @@ async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() {
server.uri(),
);
let recovered = service
let snapshot = service
.refresh_account(account.id())
.await
.expect("refresh new window quota");
.expect("refresh authoritative allowance");
assert!(!recovered.fact().exhausted());
assert_eq!(snapshot.fact().remaining_percent(), Some(0));
assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed);
assert_eq!(
store
.account(account_id)
.expect("account after cross-window recovery")
.availability(),
AccountAvailability::Ready,
"an advanced reset with a nearly unused primary window must recover QuotaExhausted"
.expect("updated account")
.quota()
.access(),
QuotaAccessState::Allowed
);
}
#[tokio::test]
async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaustion() {
async fn inconclusive_quota_refresh_preserves_confirmed_exhaustion() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_cross_window_ten_percent";
let account_id = "acct_inconclusive_quota_refresh";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("created account");
let old_reset = Utc::now().timestamp() + 3_600;
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
json!({
"rate_limit": {
"allowed": false,
"limit_reached": true,
"primary_window": {
"used_percent": 100,
"reset_at": old_reset,
"limit_window_seconds": 3600
}
}
})
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
})
.await
.expect("seed old exhausted snapshot");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
})
.await
.expect("mark account exhausted");
persist_quota_state(&store, &account, exhausted_quota(None)).await;
let server = MockServer::start().await;
let future_reset = old_reset + 3_600;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.and(header(
"authorization",
format!("Bearer token-{account_id}"),
))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"rate_limit": {
"allowed": true,
"limit_reached": false,
"primary_window": {"used_percent": 10.0, "reset_at": future_reset}
"primary_window": {"used_percent": 12, "reset_at": 1_900_000_000}
}
})))
.mount(&server)
@@ -279,24 +151,24 @@ async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaus
server.uri(),
);
let held = service
let snapshot = service
.refresh_account(account.id())
.await
.expect("refresh non-empty new window quota");
.expect("refresh inconclusive quota");
assert!(held.fact().exhausted());
assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted);
assert_eq!(
store
.account(account_id)
.expect("account after ten-percent snapshot")
.availability(),
AccountAvailability::QuotaExhausted,
"a reset advance at exactly 10% must not release confirmed exhaustion"
.expect("preserved account")
.quota()
.access(),
QuotaAccessState::Exhausted
);
}
#[tokio::test]
async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
async fn deactivated_workspace_quota_response_persists_credential_error_reason() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_deactivated_workspace";
create_account(&store, account_id).await;
@@ -304,10 +176,6 @@ async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.and(header(
"authorization",
format!("Bearer token-{account_id}"),
))
.respond_with(ResponseTemplate::new(402).set_body_json(json!({
"detail": {
"code": "deactivated_workspace",
@@ -326,15 +194,12 @@ async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
);
assert!(service.refresh_account(account.id()).await.is_err());
let current = store
.account(account_id)
.expect("account after deactivation");
assert_eq!(current.credential_state(), CredentialState::Banned);
assert_eq!(
store
.account(account_id)
.expect("account after deactivation")
.availability(),
AccountAvailability::Banned
);
assert_eq!(
store.last_error_message(account_id).as_deref(),
Some("deactivated_workspace")
current.last_error_reason(),
Some(gateway_core::engine::credential::AccountErrorReason::AccountBanned)
);
}
@@ -7,7 +7,8 @@ use std::time::{Duration, SystemTime};
use futures::future::join_all;
use gateway_core::engine::credential::{
OpaqueProviderData, ProviderAccountStore as _, QuotaObservation, QuotaWriteOutcome,
OpaqueProviderData, ProviderAccountStore as _, QuotaAccessState, QuotaObservation, QuotaState,
QuotaWriteOutcome,
};
use provider_openai::credential::{CodexQuotaWindowKind, parse_codex_quota_usage};
use serde_json::json;
@@ -34,11 +35,11 @@ fn parser_extracts_dynamic_windows_without_a_fixed_database_shape() {
fact.resets_at().map(|value| value.timestamp()),
Some(1_800_000_100)
);
assert!(!fact.exhausted());
assert_eq!(fact.remaining_percent(), Some(20));
}
#[test]
fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
fn parser_keeps_additional_limit_as_display_data() {
let fact = parse_codex_quota_usage(&json!({
"rate_limit": {"primary_window": {"used_percent": 10}},
"additional_rate_limits": [{
@@ -47,7 +48,7 @@ fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
}))
.expect("valid quota");
assert!(fact.exhausted());
assert_eq!(fact.remaining_percent(), Some(0));
}
#[test]
@@ -61,14 +62,12 @@ fn parser_keeps_full_percent_as_display_without_provider_exhaustion_signal() {
}))
.expect("valid full-percent quota");
// 任一子窗口 used>=100 是快照级 exhaustion 事实,
// 即使顶层 allowed=true。
// 百分比只负责展示;额度访问结论由规范化 QuotaState 承载。
assert_eq!(fact.remaining_percent(), Some(0));
assert!(fact.exhausted());
}
#[test]
fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() {
fn parser_projects_percent_when_access_is_denied() {
let fact = parse_codex_quota_usage(&json!({
"rate_limit": {
"allowed": false,
@@ -79,7 +78,6 @@ fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent()
.expect("valid provider-confirmed quota");
assert_eq!(fact.remaining_percent(), Some(2));
assert!(fact.exhausted());
}
#[test]
@@ -94,7 +92,7 @@ fn parser_does_not_infer_exhaustion_from_unknown_credit_fields() {
}))
.expect("recognized credits object");
assert!(!fact.exhausted());
assert_eq!(fact.remaining_percent(), None);
}
#[test]
@@ -129,7 +127,6 @@ fn parser_accepts_official_null_additional_rate_limits() {
.expect("official null additional quota");
assert_eq!(fact.remaining_percent(), Some(70));
assert!(!fact.exhausted());
}
#[test]
@@ -192,11 +189,9 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io()
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::allowed(SystemTime::now()),
})
.await
.expect("persist quota");
@@ -218,8 +213,7 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io()
}
#[tokio::test]
async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
// 过去数年的 1 秒窗口:O(1) 完成,不能线性循环。
async fn expired_window_remains_the_last_observation_until_refresh() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_o1_roll").await;
let account = store.account("acct_o1_roll").expect("account");
@@ -240,11 +234,9 @@ async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::observed_unknown(SystemTime::now()),
})
.await
.expect("persist quota");
@@ -254,20 +246,18 @@ async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
.await
.expect("read quota")
.expect("quota snapshot");
let rolled = snapshot.roll_expired_windows(SystemTime::now());
let window = &rolled.windows()[0];
assert_eq!(window.used_percent(), Some(0.0));
assert!(!window.limit_reached());
// 下一 reset 必须落在 now 之后(O(1) 公式而非线性累加)。
let window = &snapshot.windows()[0];
assert_eq!(window.used_percent(), Some(100.0));
assert!(
window
.reset_at()
.is_some_and(|reset| SystemTime::from(reset) > SystemTime::now())
.is_some_and(|reset| { SystemTime::from(reset) < SystemTime::now() })
);
assert_eq!(service.scheduling_signals(&account), None);
}
#[tokio::test]
async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
async fn persisted_access_fact_survives_limit_without_percent_or_reset() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_signal_only_limit").await;
let account = store.account("acct_signal_only_limit").expect("account");
@@ -282,11 +272,13 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::exhausted(
gateway_core::engine::credential::QuotaEvidence::ProviderDenied,
SystemTime::now(),
None,
),
})
.await
.expect("persist quota");
@@ -296,12 +288,7 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
.await
.expect("read quota")
.expect("quota snapshot");
assert!(snapshot.fact().exhausted());
let signals = service.scheduling_signals(&account);
assert!(
signals.is_some_and(|signals| signals.limit_reached()),
"limit_reached without percent/reset must still produce scheduling exclusion"
);
assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted);
}
#[tokio::test]
@@ -334,11 +321,9 @@ async fn persisted_codex_alias_keeps_the_top_level_rate_limit_canonical() {
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::observed_unknown(SystemTime::now()),
})
.await
.expect("persist quota");
@@ -393,11 +378,9 @@ async fn persisted_quota_orders_core_window_before_additional_limit() {
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::observed_unknown(SystemTime::now()),
})
.await
.expect("persist quota");
@@ -459,11 +442,9 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() {
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::observed_unknown(SystemTime::now()),
})
.await
.expect("persist quota");
@@ -490,5 +471,5 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() {
}));
assert_ne!(review[0].key(), review[1].key());
assert_eq!(snapshot.windows().len(), 3);
assert!(!snapshot.fact().exhausted());
assert_eq!(snapshot.quota().access(), QuotaAccessState::Unknown);
}
@@ -7,8 +7,9 @@ use base64::{Engine as _, engine::general_purpose::STANDARD};
use chrono::Utc;
use futures::{SinkExt, StreamExt};
use gateway_core::engine::credential::{
AccountAvailability, AccountFeedbackStats, OpaqueProviderData, ProviderAccountId,
ProviderAccountStore as _, QuotaObservation,
AccountFeedbackStats, CredentialState, OpaqueProviderData, ProviderAccountId,
ProviderAccountStore as _, QuotaAccessChange, QuotaAccessState, QuotaEvidence,
QuotaObservation, QuotaState,
};
use gateway_core::engine::provider::{Provider as _, ProviderRequest};
use gateway_core::engine::{
@@ -166,7 +167,6 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
Arc::clone(&agent_identity),
Arc::clone(&account_feedback),
CodexCookiePolicy::official().expect("cookie policy"),
true,
));
let provider = CodexProvider::new(
@@ -1414,25 +1414,26 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc
vec!["response.created", "response.failed"]
);
let account = store.account(account_id).expect("rate-limited account");
// 429 只记录限流窗口,不改变 availability(限流不改变账号可用性)。
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
assert_eq!(account.quota().access(), QuotaAccessState::Unknown);
let _ = release.send(());
server.abort();
let _ = server.await;
}
#[tokio::test]
async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset_advances() {
async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_usage_limit_request_path";
let reset_at = 1_900_000_000;
create_account(&store, account_id).await;
let account = store.account(account_id).expect("created account");
let observed_at = SystemTime::now();
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
quota: OpaqueProviderData::new(
json!({
"rate_limit": {
"allowed": true,
@@ -1444,9 +1445,9 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
.as_object()
.expect("stale quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
),
observed_at,
state: QuotaState::allowed(observed_at),
})
.await
.expect("seed stale passive quota");
@@ -1505,7 +1506,11 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted);
let account = store.account(account_id).expect("usage-limit account");
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
assert_eq!(account.quota().access(), QuotaAccessState::Exhausted);
assert_eq!(
account.quota().evidence(),
Some(QuotaEvidence::UsageLimitReached)
);
let projected = store
.get_quotas(&[account.id().clone()])
.await
@@ -1513,22 +1518,20 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
.into_iter()
.next()
.expect("confirmed quota observation");
let projected_observed_at = projected.observed_at.expect("projection observed at");
let projected = Value::Object(
projected
.quota
.map(OpaqueProviderData::into_inner)
.expect("confirmed quota projection"),
);
let projected_observed_at = projected.observed_at;
let projected = Value::Object(projected.quota.into_inner());
assert_eq!(
projected
.pointer("/rate_limit/primary_window/used_percent")
.and_then(Value::as_u64),
Some(100)
Some(99)
);
assert!(
projected.pointer("/rate_limit/secondary_window").is_none(),
"request failure must not manufacture a secondary quota"
assert_eq!(
projected
.pointer("/rate_limit/secondary_window/used_percent")
.and_then(Value::as_u64),
Some(0),
"request failure must not rewrite the raw display document"
);
tokio::time::sleep(Duration::from_millis(200)).await;
let requests = server.received_requests().await.expect("received requests");
@@ -1549,7 +1552,7 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
.expect("read refreshed quota observation")
.into_iter()
.next()
.and_then(|observation| observation.observed_at);
.map(|observation| observation.observed_at);
if observed_at.is_some_and(|observed_at| observed_at > projected_observed_at) {
break;
}
@@ -1564,15 +1567,14 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
.expect("read refreshed quota")
.into_iter()
.next()
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.map(|observation| observation.quota.into_inner())
.map(Value::Object)
.expect("authoritative quota projection");
assert_eq!(
quota
.pointer("/rate_limit/primary_window/used_percent")
.and_then(Value::as_u64),
Some(100)
Some(99)
);
assert_eq!(
quota
@@ -1582,7 +1584,7 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
);
assert!(
quota.pointer("/rate_limit/secondary_window").is_none(),
"background quota refresh must retain the same window projection"
"background refresh may normalize empty display windows"
);
let requests = server.received_requests().await.expect("received requests");
assert_eq!(
@@ -1596,8 +1598,9 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
store
.account(account_id)
.expect("account after stale full usage refresh")
.availability(),
AccountAvailability::QuotaExhausted,
.quota()
.access(),
QuotaAccessState::Exhausted,
"a contradictory full usage snapshot must not unlock a confirmed exhausted account"
);
}
@@ -1976,12 +1979,11 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
create_account(&store, account_id).await;
let account = store.account(account_id).expect("test account");
store
.repository()
.apply_state(
&account,
AccountAvailability::QuotaExhausted,
SystemTime::now(),
)
.apply_quota_access(QuotaAccessChange {
account_id: account.id().clone(),
expected_revision: account.revision(),
state: QuotaState::exhausted(QuotaEvidence::UsageLimitReached, SystemTime::now(), None),
})
.await
.expect("seed quota-exhausted state");
store
@@ -2033,7 +2035,7 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
.account(account_id)
.expect("disabled test account after test");
assert!(!account.enabled());
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
assert_eq!(account.quota().access(), QuotaAccessState::Exhausted);
assert!(!store.has_quota(account_id));
assert_eq!(affinity.binding_count(), 0);
}
@@ -2051,18 +2053,19 @@ async fn quota_limited_account_diagnostic_uses_upstream() {
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
}
});
let observed_at = SystemTime::now();
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
quota: OpaqueProviderData::new(
raw_quota
.as_object()
.expect("quota snapshot object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: Some(true),
),
observed_at,
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
})
.await
.expect("seed quota-limited snapshot");
@@ -2118,7 +2121,7 @@ async fn quota_limited_account_diagnostic_uses_upstream() {
}
#[tokio::test]
async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() {
async fn successful_response_treats_inference_success_as_authoritative_allowance() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_success_exhausted";
create_account(&store, account_id).await;
@@ -2164,8 +2167,9 @@ async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers(
store
.account(account_id)
.expect("account after successful response")
.availability(),
AccountAvailability::Ready
.quota()
.access(),
QuotaAccessState::Allowed
);
assert!(store.has_quota(account_id));
}
@@ -2226,10 +2230,9 @@ async fn successful_http_sse_rate_limit_event_persists_structured_exhaustion() {
let account = store
.account(account_id)
.expect("account after HTTP SSE response");
assert_eq!(
(account.availability(), store.has_quota(account_id),),
(AccountAvailability::Ready, true,)
);
assert_eq!(account.credential_state(), CredentialState::Ready);
assert_eq!(account.quota().access(), QuotaAccessState::Allowed);
assert!(store.has_quota(account_id));
}
#[test]
@@ -9,10 +9,10 @@ use std::time::{Duration, SystemTime};
use async_trait::async_trait;
use futures::future::BoxFuture;
use gateway_core::engine::credential::{
AccountAvailability, AccountRuntimeSignals, AccountStateChange, CredentialCasOutcome,
CredentialCasUpdate, CredentialRevision, LoadedCredential, NewProviderAccount,
AccountErrorReason, AccountRuntimeSignals, AccountStateChange, CredentialCasOutcome,
CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount,
OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore,
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
ProviderAccountUpdate, QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome,
};
use gateway_core::error::{StoreError, StoreErrorKind};
use gateway_core::provider_ports::{
@@ -38,7 +38,6 @@ struct StoredAccount {
credential: gateway_core::engine::credential::PlaintextCredential,
quota: Option<QuotaObservation>,
state_observed_at: Option<SystemTime>,
last_error_message: Option<String>,
}
#[derive(Default)]
@@ -71,15 +70,6 @@ impl MemoryAccountStore {
.map(|stored| stored.account.clone())
}
pub(crate) fn last_error_message(&self, id: &str) -> Option<String> {
let id = ProviderAccountId::new(id).ok()?;
self.accounts
.lock()
.expect("account store lock")
.get(&id)
.and_then(|stored| stored.last_error_message.clone())
}
pub(crate) fn quota_reads(&self) -> usize {
self.quota_reads.load(Ordering::SeqCst)
}
@@ -102,8 +92,9 @@ impl MemoryAccountStore {
.expect("account store lock")
.get(&id)
.and_then(|stored| stored.quota.as_ref())
.and_then(|observation| observation.quota.as_ref())
.map(|data| serde_json::Value::Object(data.expose_to_provider().clone()))
.map(|observation| {
serde_json::Value::Object(observation.quota.expose_to_provider().clone())
})
}
pub(crate) fn fail_provider_listing(&self) {
@@ -125,7 +116,6 @@ impl ProviderAccountStore for MemoryAccountStore {
credential: input.credential,
quota: None,
state_observed_at: None,
last_error_message: None,
},
);
Ok(())
@@ -224,7 +214,10 @@ impl ProviderAccountStore for MemoryAccountStore {
AccountRebuild {
revision: next,
enabled: stored.account.enabled(),
availability: stored.account.availability(),
credential_state: stored.account.credential_state(),
quota: stored.account.quota(),
last_error_reason: stored.account.last_error_reason(),
last_error_message: stored.account.last_error_message().map(str::to_owned),
access_token_expires_at,
has_refresh_token,
next_refresh_at,
@@ -262,13 +255,37 @@ impl ProviderAccountStore for MemoryAccountStore {
if stored
.quota
.as_ref()
.and_then(|quota| quota.observed_at)
.zip(observation.observed_at)
.is_some_and(|(current, incoming)| current > incoming)
.is_some_and(|quota| quota.observed_at > observation.observed_at)
{
return Ok(QuotaWriteOutcome::Conflict);
}
let quota = observation.state;
stored.quota = Some(observation);
stored.account = rebuild_account(
&stored.account,
AccountRebuild::preserving(&stored.account).with_quota(quota),
);
Ok(QuotaWriteOutcome::Updated)
}
async fn apply_quota_access(
&self,
change: QuotaAccessChange,
) -> Result<QuotaWriteOutcome, StoreError> {
let mut accounts = self.accounts.lock().expect("account store lock");
let stored = accounts
.get_mut(&change.account_id)
.ok_or_else(|| store_error(StoreErrorKind::InvalidData))?;
if stored.account.revision() != change.expected_revision {
return Ok(QuotaWriteOutcome::Conflict);
}
stored.account = rebuild_account(
&stored.account,
AccountRebuild::preserving(&stored.account).with_quota(change.state),
);
if let Some(observation) = &mut stored.quota {
observation.state = change.state;
}
Ok(QuotaWriteOutcome::Updated)
}
@@ -291,7 +308,10 @@ impl ProviderAccountStore for MemoryAccountStore {
AccountRebuild {
revision: stored.account.revision(),
enabled: stored.account.enabled(),
availability: change.availability,
credential_state: change.credential_state,
quota: stored.account.quota(),
last_error_reason: change.error_reason,
last_error_message: change.message,
access_token_expires_at: stored.account.access_token_expires_at(),
has_refresh_token: stored.account.has_refresh_token(),
next_refresh_at: stored.account.next_refresh_at(),
@@ -299,11 +319,6 @@ impl ProviderAccountStore for MemoryAccountStore {
},
);
stored.state_observed_at = Some(change.observed_at);
if change.availability == AccountAvailability::Ready {
stored.last_error_message = None;
} else if let Some(message) = change.message {
stored.last_error_message = Some(message);
}
Ok(())
}
@@ -317,7 +332,10 @@ impl ProviderAccountStore for MemoryAccountStore {
AccountRebuild {
revision: stored.account.revision(),
enabled: stored.account.enabled(),
availability: stored.account.availability(),
credential_state: stored.account.credential_state(),
quota: stored.account.quota(),
last_error_reason: stored.account.last_error_reason(),
last_error_message: stored.account.last_error_message().map(str::to_owned),
access_token_expires_at: stored.account.access_token_expires_at(),
has_refresh_token: stored.account.has_refresh_token(),
next_refresh_at: stored.account.next_refresh_at(),
@@ -341,7 +359,10 @@ impl ProviderAccountStore for MemoryAccountStore {
AccountRebuild {
revision: stored.account.revision(),
enabled,
availability: stored.account.availability(),
credential_state: stored.account.credential_state(),
quota: stored.account.quota(),
last_error_reason: stored.account.last_error_reason(),
last_error_message: stored.account.last_error_message().map(str::to_owned),
access_token_expires_at: stored.account.access_token_expires_at(),
has_refresh_token: stored.account.has_refresh_token(),
next_refresh_at: stored.account.next_refresh_at(),
@@ -368,13 +389,38 @@ const fn store_error(kind: StoreErrorKind) -> StoreError {
struct AccountRebuild {
revision: CredentialRevision,
enabled: bool,
availability: AccountAvailability,
credential_state: CredentialState,
quota: QuotaState,
last_error_reason: Option<AccountErrorReason>,
last_error_message: Option<String>,
access_token_expires_at: Option<SystemTime>,
has_refresh_token: bool,
next_refresh_at: Option<SystemTime>,
profile: Option<(String, Option<String>, Option<String>)>,
}
impl AccountRebuild {
fn preserving(account: &ProviderAccount) -> Self {
Self {
revision: account.revision(),
enabled: account.enabled(),
credential_state: account.credential_state(),
quota: account.quota(),
last_error_reason: account.last_error_reason(),
last_error_message: account.last_error_message().map(str::to_owned),
access_token_expires_at: account.access_token_expires_at(),
has_refresh_token: account.has_refresh_token(),
next_refresh_at: account.next_refresh_at(),
profile: None,
}
}
fn with_quota(mut self, quota: QuotaState) -> Self {
self.quota = quota;
self
}
}
fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> ProviderAccount {
let (name, email, plan_type) = rebuild.profile.unwrap_or_else(|| {
(
@@ -397,7 +443,13 @@ fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> Provid
current.upstream_account_id().map(str::to_owned),
plan_type,
)
.with_runtime_state(rebuild.enabled, rebuild.availability)
.with_account_facts(
rebuild.enabled,
rebuild.credential_state,
rebuild.quota,
rebuild.last_error_reason,
rebuild.last_error_message,
)
.with_refresh_schedule(rebuild.has_refresh_token, rebuild.next_refresh_at)
}
@@ -450,7 +502,7 @@ impl ProviderLeasePort for TestLeaseCoordinator {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
+18 -12
View File
@@ -660,19 +660,21 @@ impl ProviderAdmin for XaiAdminProvider {
refresh,
rolling_usage,
} = request;
self.account(&account_id).await?;
let mut account = self.account(&account_id).await?;
let lifecycle = self
.repository
.read_lifecycle(&account_id)
.await
.map_err(map_repository_error)?;
let snapshot = if refresh {
Some(
let snapshot = Some(
self.quota
.refresh_account(&account_id)
.await
.map_err(map_quota_error)?,
)
);
account = self.account(&account_id).await?;
snapshot
} else {
self.quota
.read_account(&account_id)
@@ -681,6 +683,7 @@ impl ProviderAdmin for XaiAdminProvider {
};
Ok(project_quota(
snapshot,
account.quota().is_exhausted(),
lifecycle.refresh_token_expires_at().copied(),
rolling_usage,
))
@@ -891,8 +894,8 @@ fn prepared_create(
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
next_refresh_at: account.next_refresh_at().map(Into::into),
enabled: account.enabled(),
availability: account.availability(),
availability_observed_at: observed_at,
credential_state: account.credential_state(),
credential_observed_at: observed_at,
})
}
@@ -995,7 +998,13 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
account.upstream_account_id.clone(),
account.plan_type.clone(),
)
.with_runtime_state(account.enabled, account.availability)
.with_account_facts(
account.enabled,
account.credential_state,
account.quota,
account.last_error_reason,
account.last_error_message.clone(),
)
.with_refresh_schedule(
account.has_refresh_token,
account.next_refresh_at.map(Into::into),
@@ -1012,6 +1021,7 @@ fn account_matches_record(account: &ProviderAccount, record: &AccountRecord) ->
fn project_quota(
snapshot: Option<GrokQuotaSnapshot>,
quota_exhausted: bool,
refresh_token_expires_at: Option<DateTime<Utc>>,
rolling_usage: Option<gateway_admin::model::accounts::AccountUsage>,
) -> ProviderQuota {
@@ -1020,8 +1030,7 @@ fn project_quota(
observed_at: None,
refresh_token_expires_at,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
limit_reached: quota_exhausted,
provider_data: None,
};
};
@@ -1088,10 +1097,7 @@ fn project_quota(
observed_at: Some(snapshot.observed_at()),
refresh_token_expires_at,
windows: vec![window],
limit_reached: billing
.used_percent()
.is_some_and(|used| used.is_finite() && used >= 100.0),
rate_limited_until: None,
limit_reached: quota_exhausted,
provider_data: None,
}
}
@@ -8,8 +8,8 @@ use std::time::{Duration, Instant, SystemTime};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use gateway_core::engine::credential::{
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
ProviderAccount, ProviderAccountId, QuotaObservation,
AccountQuotaSignals, CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount,
ProviderAccountId, QuotaObservation, QuotaState,
};
use gateway_core::engine::provider::ProviderCatalogGeneration;
use gateway_core::provider_ports::{
@@ -19,7 +19,6 @@ use gateway_core::routing::ProviderKind;
use tokio::sync::Mutex;
use super::repository::{GrokCredentialRepository, LoadedGrokCredential};
use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState};
use crate::XaiWireProfileState;
use crate::transport::catalog::{MAX_CATALOG_MODELS, valid_model_slug, validate_etag};
use crate::{
@@ -445,15 +444,17 @@ impl GrokCredentialQuotaService {
let observed_at = Utc::now();
let document = billing.into_document();
let presentation = billing_presentation(&document)?;
let quota = loaded
.account
.quota()
.merge_observation(quota_access_observation(&presentation, observed_at.into()));
self.repository
.replace_quota(
loaded.account.id().clone(),
loaded.account.revision(),
document,
observed_at.into(),
presentation
.used_percent()
.is_some_and(|used| used.is_finite() && used >= 100.0),
quota,
)
.await
.map_err(map_quota_repository_error)?;
@@ -464,32 +465,6 @@ impl GrokCredentialQuotaService {
billing: presentation,
};
self.scheduling.observe(&snapshot);
let current = self
.repository
.load_current(account_id)
.await
.map_err(map_quota_repository_error)?;
if quota_refresh_may_update_state(&loaded.account, &current.account)
&& let Some(availability) = quota_refresh_availability(
current.account.availability(),
authoritative_quota_exhaustion(snapshot.billing()),
)
{
self.repository
.update_state(&UpdateGrokCredentialState {
account_id: current.account.id().clone(),
expected_revision: current.account.revision(),
availability,
availability_reason: matches!(
availability,
GrokCredentialAvailability::QuotaExhausted
)
.then_some("quota_exhausted".to_owned()),
observed_at,
})
.await
.map_err(map_quota_repository_error)?;
}
Ok(snapshot)
}
@@ -506,10 +481,8 @@ impl GrokCredentialQuotaService {
else {
return Ok(None);
};
let (Some(quota), Some(observed_at)) = (observation.quota, observation.observed_at) else {
return Err(GrokQuotaError::InvalidData);
};
let document = quota.into_inner();
let observed_at = observation.observed_at;
let document = observation.quota.into_inner();
let snapshot = GrokQuotaSnapshot {
account_id: observation.account_id,
credential_revision: observation.expected_revision,
@@ -1150,14 +1123,10 @@ fn eligible_catalog_candidate(candidate: &LoadedGrokCredential) -> bool {
&& candidate
.refresh_token_expires_at
.is_none_or(|expires_at| expires_at > Utc::now())
&& match account.availability() {
AccountAvailability::Unknown
| AccountAvailability::Ready
| AccountAvailability::QuotaExhausted => true,
AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid => false,
}
&& matches!(
account.credential_state(),
CredentialState::Unknown | CredentialState::Ready
)
}
async fn fetch_candidate_catalog(
@@ -1273,8 +1242,8 @@ fn billing_presentation(
}
fn quota_snapshot_from_observation(observation: &QuotaObservation) -> Option<GrokQuotaSnapshot> {
let observed_at = DateTime::<Utc>::from(observation.observed_at?);
let document = observation.quota.as_ref()?.expose_to_provider();
let observed_at = DateTime::<Utc>::from(observation.observed_at);
let document = observation.quota.expose_to_provider();
Some(GrokQuotaSnapshot {
account_id: observation.account_id.clone(),
credential_revision: observation.expected_revision,
@@ -1293,29 +1262,54 @@ fn quota_projection_ttl(observed_at: DateTime<Utc>) -> Option<Duration> {
}
fn quota_scheduling_signals(billing: &GrokBillingPresentation) -> Option<AccountQuotaSignals> {
let remaining_rank = billing
.used_percent()
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
.map(|used| ((100.0 - used) * 100.0).round() as u64);
let now = SystemTime::now();
let reset_at = billing
let period_end = billing
.period_end()
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
.map(|value| SystemTime::from(value.to_utc()));
let observation_expired = period_end.is_some_and(|period_end| period_end <= now);
let remaining_rank = (!observation_expired)
.then(|| {
billing
.used_percent()
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
.map(|used| ((100.0 - used) * 100.0).round() as u64)
})
.flatten();
let reset_at = period_end.filter(|reset_at| *reset_at > now);
(remaining_rank.is_some() || reset_at.is_some())
.then(|| AccountQuotaSignals::new(reset_at, remaining_rank))
}
/// 把 xAI billing 文档归一化为访问结论。缺少正反证据时返回 Unknown,
/// 由 `QuotaState::merge_observation` 保留既有已确认结论。
fn quota_access_observation(
billing: &GrokBillingPresentation,
observed_at: SystemTime,
) -> QuotaState {
if !billing.has_authoritative_quota() {
return QuotaState::observed_unknown(observed_at);
}
if billing_proves_access_allowed(billing, observed_at) {
QuotaState::allowed(observed_at)
} else {
QuotaState::observed_unknown(observed_at)
}
}
fn billing_proves_access_allowed(
billing: &GrokBillingPresentation,
observed_at: SystemTime,
) -> bool {
if billing
.period_end()
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
.map(|value| SystemTime::from(value.to_utc()))
.filter(|reset_at| *reset_at > now);
(remaining_rank.is_some() || reset_at.is_some()).then(|| {
AccountQuotaSignals::new(
reset_at,
remaining_rank,
billing
.used_percent()
.is_some_and(|used| used.is_finite() && used >= 100.0),
)
})
}
fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool {
if billing
.is_some_and(|period_end| period_end <= observed_at)
{
return false;
}
billing
.prepaid_balance_cents()
.is_some_and(|balance| balance > 0)
|| billing
@@ -1329,60 +1323,6 @@ fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool {
(billing.included_used_cents(), billing.monthly_limit_cents()),
(Some(used), Some(limit)) if limit > 0 && used < limit
)
{
return false;
}
billing
.used_percent()
.is_some_and(|used| used.is_finite() && used >= 100.0)
|| matches!(
(billing.included_used_cents(), billing.monthly_limit_cents()),
(Some(used), Some(limit)) if limit > 0 && used >= limit
)
|| matches!(
(billing.on_demand_used_cents(), billing.on_demand_cap_cents()),
(Some(used), Some(cap)) if cap > 0 && used >= cap
)
}
/// `None` means the billing document cannot prove either quota state.
fn authoritative_quota_exhaustion(billing: &GrokBillingPresentation) -> Option<bool> {
billing
.has_authoritative_quota()
.then(|| quota_is_exhausted(billing))
}
fn quota_refresh_may_update_state(before: &ProviderAccount, current: &ProviderAccount) -> bool {
current.enabled()
&& current
.access_token_expires_at()
.is_some_and(|expires_at| expires_at > SystemTime::now())
&& current.revision() == before.revision()
&& current.availability() == before.availability()
}
fn quota_refresh_availability(
current: AccountAvailability,
exhausted: Option<bool>,
) -> Option<GrokCredentialAvailability> {
match current {
AccountAvailability::Invalid
| AccountAvailability::Expired
| AccountAvailability::Banned => None,
AccountAvailability::QuotaExhausted => {
matches!(exhausted, Some(false)).then_some(GrokCredentialAvailability::Ready)
}
AccountAvailability::Unknown => exhausted.map(|exhausted| {
if exhausted {
GrokCredentialAvailability::QuotaExhausted
} else {
GrokCredentialAvailability::Ready
}
}),
AccountAvailability::Ready => {
matches!(exhausted, Some(true)).then_some(GrokCredentialAvailability::QuotaExhausted)
}
}
}
fn dynamic_string(
@@ -63,10 +63,9 @@ pub use token::{
parse_refresh_success,
};
pub use types::{
CreateGrokCredential, GrokAccountProfile, GrokCredentialAvailability, GrokCredentialRecord,
GrokOAuthSecret, PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard,
RotateGrokCredential, RotateManagedGrokCredential, UpdateGrokCredentialState,
XAI_AUTHENTICATION_KIND_OAUTH,
CreateGrokCredential, GrokAccountProfile, GrokCredentialRecord, GrokOAuthSecret,
PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, RotateGrokCredential,
RotateManagedGrokCredential, UpdateGrokCredentialState, XAI_AUTHENTICATION_KIND_OAUTH,
};
pub use verification::{
FailClosedTokenVerifier, TokenCandidate, TokenVerificationContext, TokenVerifier,
@@ -9,7 +9,7 @@ use async_trait::async_trait;
use chrono::{DateTime, Utc};
use futures::{StreamExt as _, stream};
use gateway_core::engine::credential::{
AccountAvailability, CredentialRevision, LoadedCredential, ProviderAccountId,
AccountErrorReason, CredentialRevision, CredentialState, LoadedCredential, ProviderAccountId,
};
use gateway_core::provider_ports::{
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
@@ -23,9 +23,8 @@ use super::repository::{
GrokCredentialAdmin, GrokCredentialRepository, GrokCredentialRepositoryError,
};
use super::types::{
GrokAccountProfile, GrokCredentialAvailability, GrokOAuthSecret,
PreparedGrokCredentialRotation, RotateGrokCredential, RotateManagedGrokCredential,
UpdateGrokCredentialState,
GrokAccountProfile, GrokOAuthSecret, PreparedGrokCredentialRotation, RotateGrokCredential,
RotateManagedGrokCredential, UpdateGrokCredentialState,
};
use crate::{
DiscoveryDocument, FailureClass, GrokOAuthClient, OAuthError, RefreshTokenGrant,
@@ -540,7 +539,8 @@ impl GrokCredentialRefreshService {
return self
.persist_terminal_failure(
credential,
GrokCredentialAvailability::Expired,
CredentialState::Expired,
AccountErrorReason::CredentialExpired,
"refresh_token_expired",
)
.await;
@@ -549,7 +549,8 @@ impl GrokCredentialRefreshService {
return self
.persist_terminal_failure(
credential,
GrokCredentialAvailability::Expired,
CredentialState::Expired,
AccountErrorReason::CredentialExpired,
"refresh_recovery_window_exhausted",
)
.await;
@@ -587,7 +588,8 @@ impl GrokCredentialRefreshService {
Err(GrokRefreshFailure::InvalidGrant) => {
self.persist_terminal_failure(
credential,
GrokCredentialAvailability::Expired,
CredentialState::Expired,
AccountErrorReason::CredentialExpired,
"refresh_invalid_grant",
)
.await
@@ -595,7 +597,8 @@ impl GrokCredentialRefreshService {
Err(GrokRefreshFailure::Banned) => {
self.persist_terminal_failure(
credential,
GrokCredentialAvailability::Banned,
CredentialState::Banned,
AccountErrorReason::AccountBanned,
"account_banned",
)
.await
@@ -717,7 +720,8 @@ impl GrokCredentialRefreshService {
async fn persist_terminal_failure(
&self,
credential: DueGrokCredential,
availability: GrokCredentialAvailability,
credential_state: CredentialState,
error_reason: AccountErrorReason,
reason: &str,
) -> Result<GrokCredentialRefreshOutcome, GrokCredentialRefreshError> {
let account_id = credential.account_id.clone();
@@ -737,8 +741,9 @@ impl GrokCredentialRefreshService {
.update_state(&UpdateGrokCredentialState {
account_id: account_id.clone(),
expected_revision: credential.credential_revision,
availability,
availability_reason: Some(reason.to_owned()),
credential_state,
error_reason: Some(error_reason),
error_message: Some(reason.to_owned()),
observed_at: Utc::now(),
})
.await
@@ -746,7 +751,7 @@ impl GrokCredentialRefreshService {
Ok(()) => {
tracing::warn!(
account_id = %account_id,
?availability,
?credential_state,
reason,
access_token_expires_at = ?credential.access_token_expires_at
.map(DateTime::<Utc>::from),
@@ -915,14 +920,11 @@ fn account_due(
policy: ProviderRefreshPolicy,
now: SystemTime,
) -> bool {
let availability = account.availability();
account.enabled()
&& account.has_refresh_token()
&& !matches!(
availability,
AccountAvailability::Expired
| AccountAvailability::Banned
| AccountAvailability::Invalid
account.credential_state(),
CredentialState::Expired | CredentialState::Banned | CredentialState::Invalid
)
&& (refresh_recovery_window_exhausted(account.access_token_expires_at(), now)
|| (access_token_refresh_due(account.access_token_expires_at(), policy, now)
@@ -6,10 +6,10 @@ use std::time::SystemTime;
use chrono::{DateTime, Utc};
use gateway_core::engine::credential::{
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData,
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
CredentialState, LoadedCredential, NewProviderAccount, OpaqueProviderData, PlaintextCredential,
ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome,
};
use gateway_core::error::StoreErrorKind;
use gateway_core::routing::ProviderKind;
@@ -17,8 +17,8 @@ use serde_json::Value;
use thiserror::Error;
use super::types::{
CreateGrokCredential, GrokAccountProfile, GrokCredentialAvailability, GrokCredentialRecord,
GrokOAuthSecret, GrokOAuthSecretWire, OwnedGrokOAuthSecretWire, PreparedGrokCredentialRotation,
CreateGrokCredential, GrokAccountProfile, GrokCredentialRecord, GrokOAuthSecret,
GrokOAuthSecretWire, OwnedGrokOAuthSecretWire, PreparedGrokCredentialRotation,
RotateGrokCredential, RotateManagedGrokCredential, UpdateGrokCredentialState,
XAI_AUTHENTICATION_KIND_OAUTH,
};
@@ -167,7 +167,13 @@ impl GrokCredentialAdmin {
input.account.upstream_account_id.clone(),
input.account.plan_type.clone(),
)
.with_runtime_state(input.enabled, input.initial_availability.into())
.with_account_facts(
input.enabled,
input.initial_credential_state,
QuotaState::unknown(),
input.initial_error_reason,
input.initial_error_message.clone(),
)
.with_refresh_schedule(true, None);
Ok(NewProviderAccount {
account,
@@ -210,8 +216,9 @@ impl GrokCredentialAdmin {
refresh_token_expires_at: None,
},
enabled: input.enabled,
initial_availability: GrokCredentialAvailability::Ready,
initial_availability_reason: None,
initial_credential_state: CredentialState::Ready,
initial_error_reason: None,
initial_error_message: None,
})
}
@@ -420,21 +427,30 @@ impl GrokCredentialRepository {
&self,
input: &UpdateGrokCredentialState,
) -> Result<(), GrokCredentialRepositoryError> {
let availability = AccountAvailability::from(input.availability);
let availability_reason = input
.availability_reason
let error_message = input
.error_message
.clone()
.filter(|value| !value.trim().is_empty())
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
validate_reason(availability_reason.as_deref())?;
.filter(|value| !value.trim().is_empty());
validate_reason(error_message.as_deref())?;
let (error_reason, error_message) = if input.credential_state == CredentialState::Ready {
(None, None)
} else {
(
input
.error_reason
.or_else(|| input.credential_state.error_reason()),
error_message,
)
};
self.ensure_xai_account(&input.account_id).await?;
self.store
.apply_state_change(AccountStateChange {
account_id: input.account_id.clone(),
expected_revision: input.expected_revision,
availability,
credential_state: input.credential_state,
observed_at: to_system_time(input.observed_at),
message: availability_reason,
error_reason,
message: error_message,
})
.await
.map_err(map_store_error)
@@ -518,16 +534,39 @@ impl GrokCredentialRepository {
expected_revision: CredentialRevision,
document: serde_json::Map<String, Value>,
observed_at: SystemTime,
limit_reached: bool,
state: QuotaState,
) -> Result<(), GrokCredentialRepositoryError> {
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
account_id,
expected_revision,
quota: Some(OpaqueProviderData::new(document)),
observed_at: Some(observed_at),
limit_reached: Some(limit_reached),
quota: OpaqueProviderData::new(document),
observed_at,
state,
})
.await
.map_err(map_store_error)?;
match outcome {
QuotaWriteOutcome::Updated => Ok(()),
QuotaWriteOutcome::Conflict => {
Err(GrokCredentialRepositoryError::StaleCredentialRevision)
}
}
}
pub(crate) async fn update_quota_access(
&self,
account_id: ProviderAccountId,
expected_revision: CredentialRevision,
state: QuotaState,
) -> Result<(), GrokCredentialRepositoryError> {
let outcome = self
.store
.apply_quota_access(QuotaAccessChange {
account_id,
expected_revision,
state,
})
.await
.map_err(map_store_error)?;
@@ -698,7 +737,7 @@ fn validate_create(input: &CreateGrokCredential) -> Result<(), GrokCredentialRep
validate_name(&input.name)?;
validate_profile(&input.account)?;
validate_secret(&input.secret)?;
validate_reason(input.initial_availability_reason.as_deref())
validate_reason(input.initial_error_message.as_deref())
}
fn validate_profile(profile: &GrokAccountProfile) -> Result<(), GrokCredentialRepositoryError> {
@@ -786,9 +825,7 @@ fn validate_reason(reason: Option<&str>) -> Result<(), GrokCredentialRepositoryE
if reason
.is_some_and(|value| value.len() > MAX_REASON_BYTES || value.chars().any(char::is_control))
{
return Err(GrokCredentialRepositoryError::InvalidInput(
"availability_reason",
));
return Err(GrokCredentialRepositoryError::InvalidInput("error_message"));
}
Ok(())
}
@@ -6,8 +6,9 @@ use std::time::{Duration, SystemTime};
use chrono::Utc;
use gateway_core::engine::credential::{
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId,
AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats, AccountSelectionContext,
AccountSelector, CredentialState, ProviderAccount, ProviderAccountId, QuotaAccessState,
QuotaEvidence, QuotaState,
};
use gateway_core::provider_ports::{
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition,
@@ -18,7 +19,7 @@ use gateway_core::routing::{ProviderKind, UpstreamModelId};
use super::catalog::{GrokCatalogScope, GrokCredentialCatalogCache, GrokCredentialQuotaService};
use super::repository::{GrokCredentialRepository, GrokCredentialRepositoryError};
use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState};
use super::types::UpdateGrokCredentialState;
use crate::{
GrokCredentialFailure, GrokCredentialFeedbackFuture, GrokSessionBinding, GrokSessionSelection,
GrokSessionSelector, GrokSessionSelectorError, GrokSessionSelectorFuture, SecretValue,
@@ -82,7 +83,7 @@ impl GrokAccountSessionSelector {
&self,
request: GrokSessionSelection,
) -> Result<SelectedGrokSession, GrokSessionSelectorError> {
let diagnostic = request.availability() == AccountAvailabilityPolicy::BypassForDiagnostic;
let diagnostic = request.eligibility() == AccountEligibilityPolicy::BypassForDiagnostic;
let accounts = self
.repository
.list_accounts_for_provider()
@@ -177,7 +178,7 @@ impl GrokAccountSessionSelector {
excluded_accounts: request.excluded_accounts().clone(),
preferred_account: request.required_account().cloned().or(affinity_account),
round_robin_cursor: scheduling.round_robin_cursor(),
availability: request.availability(),
eligibility: request.eligibility(),
};
let mut capacity_denied = false;
let mut retry_after = None;
@@ -363,23 +364,18 @@ impl GrokAccountSessionSelector {
.await;
}
async fn persist_account_failure(
async fn persist_quota_failure(
&self,
session: &SelectedGrokSession,
availability: GrokCredentialAvailability,
reason: &'static str,
observed_at: chrono::DateTime<Utc>,
) {
let update = |expected_revision| UpdateGrokCredentialState {
account_id: session.account_id().clone(),
expected_revision,
availability,
availability_reason: Some(reason.to_owned()),
observed_at,
};
match self
.repository
.update_state(&update(session.credential_revision()))
.update_quota_access(
session.account_id().clone(),
session.credential_revision(),
QuotaState::exhausted(QuotaEvidence::UsageLimitReached, observed_at.into(), None),
)
.await
{
Ok(()) => return,
@@ -410,7 +406,11 @@ impl GrokAccountSessionSelector {
};
if let Err(error) = self
.repository
.update_state(&update(current.account.revision()))
.update_quota_access(
session.account_id().clone(),
current.account.revision(),
QuotaState::exhausted(QuotaEvidence::UsageLimitReached, observed_at.into(), None),
)
.await
{
tracing::warn!(
@@ -450,20 +450,15 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
return;
}
let observed_at = Utc::now();
let persistent = match failure {
match failure {
GrokCredentialFailure::Unauthorized => {
self.record_runtime_cooldown(session, UNAUTHORIZED_COOLDOWN)
.await;
return;
}
GrokCredentialFailure::QuotaExhausted => (
GrokCredentialAvailability::QuotaExhausted,
"upstream_quota_exhausted",
),
GrokCredentialFailure::FreeQuotaExhausted => (
GrokCredentialAvailability::QuotaExhausted,
"upstream_free_quota_exhausted",
),
GrokCredentialFailure::QuotaExhausted
| GrokCredentialFailure::FreeQuotaExhausted => {
self.persist_quota_failure(session, observed_at).await;
}
// bare 402 没有结构化 quota code,不能证明账号额度耗尽:
// 只写短期账号级 runtime cooldown(TransientBackoff),
// 不持久化 QuotaExhausted(结构化 QuotaExhausted/FreeQuotaExhausted 走上面)。
@@ -472,7 +467,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
.min(MAX_RATE_LIMIT_COOLDOWN);
self.record_runtime_cooldown(session, retry_after).await;
return;
}
// xAI 免费模型额度按该模型滚动窗口恢复:限流写 model-scoped
// runtime cooldown(Redis 跨重启保留),不进入持久化账号状态,
@@ -485,7 +479,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN);
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
.await;
return;
}
GrokCredentialFailure::ModelAccessDenied {
upstream_model,
@@ -498,24 +491,18 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
);
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
.await;
return;
}
GrokCredentialFailure::RateLimited { retry_after } => {
let retry_after = retry_after
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
.min(MAX_RATE_LIMIT_COOLDOWN);
self.record_runtime_cooldown(session, retry_after).await;
return;
}
GrokCredentialFailure::StreamInterrupted => {
self.record_runtime_cooldown(session, STREAM_INTERRUPTION_COOLDOWN)
.await;
return;
}
};
let (availability, reason) = persistent;
self.persist_account_failure(session, availability, reason, observed_at)
.await;
}
})
}
@@ -531,25 +518,22 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
return;
};
let account = &current.account;
if account.revision() != session.credential_revision()
|| !account.enabled()
|| !matches!(
account.availability(),
AccountAvailability::Unknown | AccountAvailability::Expired
)
{
if account.revision() != session.credential_revision() || !account.enabled() {
return;
}
if let Err(error) = self
.repository
.update_state(&UpdateGrokCredentialState {
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: GrokCredentialAvailability::Ready,
availability_reason: None,
observed_at: Utc::now(),
})
.await
let observed_at = Utc::now();
if account.credential_state() != CredentialState::Ready
&& let Err(error) = self
.repository
.update_state(&UpdateGrokCredentialState {
account_id: account.id().clone(),
expected_revision: account.revision(),
credential_state: CredentialState::Ready,
error_reason: None,
error_message: None,
observed_at,
})
.await
{
tracing::warn!(
account_id = %account.id(),
@@ -557,6 +541,24 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
"xAI account state recovery after successful upstream response failed"
);
}
// 已经发出的并发请求可能晚于耗尽事实成功返回;真实成功可以收敛
// 该事实,但 selector 不会为恢复探测而放行耗尽账号。
if account.quota().access() == QuotaAccessState::Exhausted
&& let Err(error) = self
.repository
.update_quota_access(
account.id().clone(),
account.revision(),
QuotaState::allowed(observed_at.into()),
)
.await
{
tracing::warn!(
account_id = %account.id(),
error = %error,
"xAI quota recovery after successful upstream response failed"
);
}
})
}
}
@@ -4,7 +4,7 @@ use std::fmt;
use chrono::{DateTime, Utc};
use gateway_core::engine::credential::{
AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential,
AccountErrorReason, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential,
ProviderAccountId, ProviderAccountUpdate,
};
use gateway_core::provider_ports::ProviderLeaseGuard;
@@ -63,34 +63,6 @@ impl fmt::Debug for GrokAccountProfile {
}
}
/// 与 `provider_accounts.availability` 一一对应的 xAI 状态。
///
/// 限流(429/滚动窗口)不进入持久化状态:由 `ProviderCooldownPort`(Redis 跨重启
/// 保留)驱动调度排除,账号状态保持 `Ready`。
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum GrokCredentialAvailability {
Unknown,
Ready,
QuotaExhausted,
Expired,
Banned,
Invalid,
}
/// Grok 可用性语义翻入内核标准态的唯一汇入点。
impl From<GrokCredentialAvailability> for AccountAvailability {
fn from(value: GrokCredentialAvailability) -> Self {
match value {
GrokCredentialAvailability::Unknown => Self::Unknown,
GrokCredentialAvailability::Ready => Self::Ready,
GrokCredentialAvailability::QuotaExhausted => Self::QuotaExhausted,
GrokCredentialAvailability::Expired => Self::Expired,
GrokCredentialAvailability::Banned => Self::Banned,
GrokCredentialAvailability::Invalid => Self::Invalid,
}
}
}
/// 创建一个明文 OAuth Provider account。
pub struct CreateGrokCredential {
pub account_id: ProviderAccountId,
@@ -98,8 +70,9 @@ pub struct CreateGrokCredential {
pub secret: GrokOAuthSecret,
pub account: GrokAccountProfile,
pub enabled: bool,
pub initial_availability: GrokCredentialAvailability,
pub initial_availability_reason: Option<String>,
pub initial_credential_state: CredentialState,
pub initial_error_reason: Option<AccountErrorReason>,
pub initial_error_message: Option<String>,
}
impl fmt::Debug for CreateGrokCredential {
@@ -111,7 +84,7 @@ impl fmt::Debug for CreateGrokCredential {
.field("secret", &"[REDACTED]")
.field("account", &self.account)
.field("enabled", &self.enabled)
.field("initial_availability", &self.initial_availability)
.field("initial_credential_state", &self.initial_credential_state)
.finish_non_exhaustive()
}
}
@@ -235,8 +208,9 @@ impl fmt::Debug for RotateGrokCredential {
pub struct UpdateGrokCredentialState {
pub account_id: ProviderAccountId,
pub expected_revision: CredentialRevision,
pub availability: GrokCredentialAvailability,
pub availability_reason: Option<String>,
pub credential_state: CredentialState,
pub error_reason: Option<AccountErrorReason>,
pub error_message: Option<String>,
pub observed_at: DateTime<Utc>,
}
+19 -19
View File
@@ -30,25 +30,25 @@ pub use credential::{
FailClosedTokenVerifier, FailureClass, FormField, FormValue, GROK_FREE_ROLLING_WINDOW_SECONDS,
GrokAccountExport, GrokAccountProfile, GrokAccountSessionSelector, GrokBillingPresentation,
GrokCatalogCache, GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin,
GrokCredentialAvailability, GrokCredentialCatalogCache, GrokCredentialCatalogError,
GrokCredentialCatalogSeed, GrokCredentialCatalogService, GrokCredentialLifecycle,
GrokCredentialQuotaService, GrokCredentialRecord, GrokCredentialRecovery,
GrokCredentialRecoveryOutcome, GrokCredentialRefreshError, GrokCredentialRefreshOutcome,
GrokCredentialRefreshService, GrokCredentialRefresher, GrokCredentialRepository,
GrokCredentialRepositoryError, GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate,
GrokOAuthImportDocument, GrokOAuthImportEntry, GrokOAuthImportError, GrokOAuthImportMetadata,
GrokOAuthImportTokens, GrokOAuthRefreshClient, GrokOAuthSecret, GrokPlanCatalog,
GrokQuotaError, GrokQuotaPeriodKind, GrokQuotaSnapshot, GrokRefreshFailure, GrokRefreshTokens,
HttpHeader, HttpMethod, OAuthError, OAuthErrorCode, OAuthHttpRequest, OAuthHttpResponse,
OAuthHttpTransport, OAuthOperation, OAuthPrincipal, OFFICIAL_CLIENT_ID, OFFICIAL_ISSUER,
OFFICIAL_REDIRECT_URI, OFFICIAL_SCOPES, PendingAuthorization, Pkce,
PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, ProtocolViolation,
RedirectUriAllowlist, RefreshTokenGrant, RefreshedTokenSet, ReqwestOidcTokenVerifier,
RotateGrokCredential, RotateManagedGrokCredential, SecretValue, TokenCandidate,
TokenVerificationContext, TokenVerifier, TransportFailure, TransportFailureKind,
TransportFuture, UpdateGrokCredentialState, VerificationEvidence, VerificationFailure,
VerificationFlow, VerificationFuture, VerificationMethod, VerifiedGrokAccount,
VerifiedTokenSet, parse_oauth_error, parse_refresh_success,
GrokCredentialCatalogCache, GrokCredentialCatalogError, GrokCredentialCatalogSeed,
GrokCredentialCatalogService, GrokCredentialLifecycle, GrokCredentialQuotaService,
GrokCredentialRecord, GrokCredentialRecovery, GrokCredentialRecoveryOutcome,
GrokCredentialRefreshError, GrokCredentialRefreshOutcome, GrokCredentialRefreshService,
GrokCredentialRefresher, GrokCredentialRepository, GrokCredentialRepositoryError,
GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, GrokOAuthImportDocument,
GrokOAuthImportEntry, GrokOAuthImportError, GrokOAuthImportMetadata, GrokOAuthImportTokens,
GrokOAuthRefreshClient, GrokOAuthSecret, GrokPlanCatalog, GrokQuotaError, GrokQuotaPeriodKind,
GrokQuotaSnapshot, GrokRefreshFailure, GrokRefreshTokens, HttpHeader, HttpMethod, OAuthError,
OAuthErrorCode, OAuthHttpRequest, OAuthHttpResponse, OAuthHttpTransport, OAuthOperation,
OAuthPrincipal, OFFICIAL_CLIENT_ID, OFFICIAL_ISSUER, OFFICIAL_REDIRECT_URI, OFFICIAL_SCOPES,
PendingAuthorization, Pkce, PreparedGrokCredentialRotation,
PreparedGrokCredentialRotationGuard, ProtocolViolation, RedirectUriAllowlist,
RefreshTokenGrant, RefreshedTokenSet, ReqwestOidcTokenVerifier, RotateGrokCredential,
RotateManagedGrokCredential, SecretValue, TokenCandidate, TokenVerificationContext,
TokenVerifier, TransportFailure, TransportFailureKind, TransportFuture,
UpdateGrokCredentialState, VerificationEvidence, VerificationFailure, VerificationFlow,
VerificationFuture, VerificationMethod, VerifiedGrokAccount, VerifiedTokenSet,
parse_oauth_error, parse_refresh_success,
};
pub use provider::GrokBuildProvider;
pub use transport::profile::{
+12 -8
View File
@@ -8,8 +8,8 @@ use async_trait::async_trait;
use futures::{StreamExt, future::BoxFuture};
use gateway_core::engine::continuation::ContinuationBinding;
use gateway_core::engine::credential::{
AccountAvailability, AccountAvailabilityPolicy, AccountFeedbackStats, ProviderAccount,
ProviderAccountId, ProviderAccountStore,
AccountEligibilityPolicy, AccountFeedbackStats, ProviderAccount, ProviderAccountId,
ProviderAccountStore,
};
use gateway_core::engine::provider::{
EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration,
@@ -392,10 +392,10 @@ async fn select_grok_session(
context.account_selection_policy(),
context.deadline(),
)
.with_availability_policy(if context.is_diagnostic_required_account() {
AccountAvailabilityPolicy::BypassForDiagnostic
.with_eligibility_policy(if context.is_diagnostic_required_account() {
AccountEligibilityPolicy::BypassForDiagnostic
} else {
AccountAvailabilityPolicy::Enforce
AccountEligibilityPolicy::Enforce
})
.with_affinity(affinity);
let selection_deadline = remaining(context.deadline())
@@ -1981,7 +1981,8 @@ const OAUTH_REFRESH_INTERVAL: Duration = Duration::from_secs(30);
const QUOTA_CATALOG_INTERVAL: Duration = Duration::from_secs(5 * 60);
// grok2api 对 Free/未知额度耗尽采用 24 小时恢复探测;当前公共账号状态没有
// 独立的 paid period-end 字段,因此 Build 账号统一使用同一保守下限。
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = QUOTA_CATALOG_INTERVAL;
const CLI_RELEASE_WORKER_OWNER: &str = "xai-cli-release";
pub(crate) fn worker_contributions(
@@ -2175,7 +2176,8 @@ impl XaiQuotaCatalogTask {
let due = last_periodic_refresh_at
.get(account.id())
.is_none_or(|last| {
now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL
now.saturating_duration_since(*last)
>= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL
});
if due {
last_periodic_refresh_at.insert(account.id().clone(), now);
@@ -2191,5 +2193,7 @@ fn eligible_quota_worker_account(account: &ProviderAccount, now: SystemTime) ->
&& account
.access_token_expires_at()
.is_some_and(|expires_at| expires_at > now)
&& account.availability() == AccountAvailability::QuotaExhausted
&& account
.quota()
.exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL)
}
@@ -5,7 +5,7 @@ use std::pin::Pin;
use std::time::{Duration, SystemTime};
use gateway_core::engine::credential::{
AccountAvailabilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId,
AccountEligibilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId,
};
use gateway_core::engine::provider::ProviderResource;
use gateway_core::routing::UpstreamModelId;
@@ -210,7 +210,7 @@ pub struct GrokSessionSelection {
excluded_accounts: BTreeSet<ProviderAccountId>,
required_account: Option<ProviderAccountId>,
account_selection_policy: AccountSelectionPolicy,
availability: AccountAvailabilityPolicy,
eligibility: AccountEligibilityPolicy,
affinity: Option<GrokSessionAffinityKey>,
deadline: SystemTime,
}
@@ -230,7 +230,7 @@ impl GrokSessionSelection {
excluded_accounts,
required_account,
account_selection_policy,
availability: AccountAvailabilityPolicy::Enforce,
eligibility: AccountEligibilityPolicy::Enforce,
affinity: None,
deadline,
}
@@ -245,11 +245,11 @@ impl GrokSessionSelection {
/// 为固定账号的管理端诊断指定本地可用性判定策略。
#[must_use]
pub(crate) const fn with_availability_policy(
pub(crate) const fn with_eligibility_policy(
mut self,
availability: AccountAvailabilityPolicy,
eligibility: AccountEligibilityPolicy,
) -> Self {
self.availability = availability;
self.eligibility = eligibility;
self
}
@@ -278,8 +278,8 @@ impl GrokSessionSelection {
}
#[must_use]
pub const fn availability(&self) -> AccountAvailabilityPolicy {
self.availability
pub const fn eligibility(&self) -> AccountEligibilityPolicy {
self.eligibility
}
/// 返回不含原始客户端会话值的账号亲和键。
+6 -5
View File
@@ -471,10 +471,11 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
enabled: account.enabled(),
availability: account.availability(),
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
credential_state: account.credential_state(),
credential_observed_at: now,
quota: account.quota(),
last_error_reason: account.last_error_reason(),
last_error_message: account.last_error_message().map(str::to_owned),
created_at: now,
updated_at: now,
}
@@ -503,7 +504,7 @@ impl ProviderLeasePort for TestLeases {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -5,9 +5,9 @@ use std::time::{Duration, SystemTime};
use futures::future::{BoxFuture, join_all};
use gateway_core::engine::credential::{
AccountAvailability, AccountStateChange, CredentialCasUpdate, CredentialRevision,
AccountStateChange, CredentialCasUpdate, CredentialRevision, CredentialState,
OpaqueProviderData, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
QuotaObservation,
QuotaAccessChange, QuotaEvidence, QuotaObservation, QuotaState,
};
use gateway_core::provider_ports::{
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderStoreError,
@@ -200,14 +200,15 @@ async fn repository_with_accounts(
async fn set_account_state(
store: &MemoryProviderAccountStore,
id: &ProviderAccountId,
availability: AccountAvailability,
credential_state: CredentialState,
) {
store
.apply_state_change(AccountStateChange {
message: None,
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability,
credential_state,
error_reason: credential_state.error_reason(),
observed_at: SystemTime::now(),
})
.await
@@ -485,12 +486,10 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() {
repository_with_accounts(&[("quota-exhausted", "subject-quota-exhausted")]).await;
let id = account_id("quota-exhausted");
store
.apply_state_change(AccountStateChange {
message: None,
.apply_quota_access(QuotaAccessChange {
account_id: id,
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
})
.await
.expect("mark quota exhausted");
@@ -662,7 +661,7 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie
.expect("read persisted quota")
.pop()
.expect("quota exists");
let document = persisted.quota.expect("provider quota");
let document = persisted.quota;
assert_eq!(transport.calls.load(Ordering::SeqCst), 1);
assert_eq!(snapshot.billing().used_percent(), Some(37.5));
@@ -683,33 +682,18 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie
}
#[tokio::test]
async fn recovered_quota_refresh_releases_existing_quota_isolation() {
for (suffix, availability) in [
("recover-quota", AccountAvailability::QuotaExhausted),
("recover-ready", AccountAvailability::Ready),
] {
let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await;
let id = account_id(suffix);
set_account_state(&store, &id, availability).await;
crate::support::grok_quota_service(
repository,
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":25}}"#),
)
.refresh_account(&id)
.await
.expect("refresh recovered quota");
let account = store.account(&id).expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
}
}
#[tokio::test]
async fn authoritative_exhaustion_preserves_quota_exhausted_state() {
async fn billing_percent_does_not_rewrite_quota_access_fact() {
let (store, repository) =
repository_with_accounts(&[("still-exhausted", "still-exhausted")]).await;
let id = account_id("still-exhausted");
set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await;
store
.apply_quota_access(QuotaAccessChange {
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
})
.await
.expect("mark quota exhausted");
crate::support::grok_quota_service(
repository,
@@ -720,21 +704,75 @@ async fn authoritative_exhaustion_preserves_quota_exhausted_state() {
.expect("refresh exhausted quota");
assert_eq!(
store.account(&id).expect("account").availability(),
AccountAvailability::QuotaExhausted
store.account(&id).expect("account").quota().access(),
gateway_core::engine::credential::QuotaAccessState::Exhausted
);
}
#[tokio::test]
async fn authoritative_billing_refresh_clears_existing_quota_exhaustion() {
let (store, repository) =
repository_with_accounts(&[("recovered-quota", "recovered-quota")]).await;
let id = account_id("recovered-quota");
store
.apply_quota_access(QuotaAccessChange {
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
})
.await
.expect("mark quota exhausted");
crate::support::grok_quota_service(
repository,
QueueBillingTransport::success(
br#"{"config":{"creditUsagePercent":12.5,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2099-07-22T00:00:00Z"}}}"#,
),
)
.refresh_account(&id)
.await
.expect("refresh recovered quota");
assert_eq!(
store.account(&id).expect("account").quota().access(),
gateway_core::engine::credential::QuotaAccessState::Allowed
);
}
#[tokio::test]
async fn full_usage_display_does_not_invent_quota_exhaustion() {
let (store, repository) =
repository_with_accounts(&[("exhausted-quota", "exhausted-quota")]).await;
let id = account_id("exhausted-quota");
crate::support::grok_quota_service(
repository,
QueueBillingTransport::success(
br#"{"config":{"creditUsagePercent":100,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2099-07-22T00:00:00Z"}}}"#,
),
)
.refresh_account(&id)
.await
.expect("refresh exhausted quota");
let quota = store.account(&id).expect("account").quota();
assert_eq!(
quota.access(),
gateway_core::engine::credential::QuotaAccessState::Unknown
);
assert!(quota.reset_at().is_none());
}
#[tokio::test]
async fn recovered_quota_does_not_clear_terminal_account_states() {
for (suffix, availability) in [
("keep-banned", AccountAvailability::Banned),
("keep-expired", AccountAvailability::Expired),
("keep-invalid", AccountAvailability::Invalid),
for (suffix, credential_state) in [
("keep-banned", CredentialState::Banned),
("keep-expired", CredentialState::Expired),
("keep-invalid", CredentialState::Invalid),
] {
let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await;
let id = account_id(suffix);
set_account_state(&store, &id, availability).await;
set_account_state(&store, &id, credential_state).await;
crate::support::grok_quota_service(
repository,
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":10}}"#),
@@ -744,14 +782,14 @@ async fn recovered_quota_does_not_clear_terminal_account_states() {
.expect("refresh terminal account quota");
assert_eq!(
store.account(&id).expect("account").availability(),
availability
store.account(&id).expect("account").credential_state(),
credential_state
);
}
}
#[tokio::test]
async fn quota_refresh_preserves_ready_availability_across_concurrent_state_write() {
async fn quota_refresh_preserves_ready_credential_state_across_concurrent_write() {
let (store, repository) = repository_with_accounts(&[("new-cooldown", "new-cooldown")]).await;
let id = account_id("new-cooldown");
let transport = Arc::new(MutatingBillingTransport {
@@ -760,7 +798,8 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
message: None,
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: AccountAvailability::Ready,
credential_state: CredentialState::Ready,
error_reason: None,
observed_at: SystemTime::now(),
}))),
body: br#"{"config":{"creditUsagePercent":10}}"#.to_vec(),
@@ -772,7 +811,7 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
.expect("refresh around concurrent state write");
let account = store.account(&id).expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
}
#[tokio::test]
@@ -860,12 +899,48 @@ async fn quota_projection_preserves_unknown_period_for_dynamic_duration_fallback
);
}
#[tokio::test]
async fn expired_billing_window_does_not_participate_in_scheduling_rank() {
let (store, repository) =
repository_with_accounts(&[("expired-billing", "expired-billing")]).await;
let id = account_id("expired-billing");
let account = store.account(&id).expect("account");
let service = crate::support::grok_quota_service(
repository,
QueueBillingTransport::success(
br#"{"config":{"creditUsagePercent":25,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2000-01-01T00:00:00Z","end":"2000-01-08T00:00:00Z"}}}"#,
),
);
service
.refresh_account(&id)
.await
.expect("refresh expired billing window");
assert_eq!(service.scheduling_signals(&account), None);
assert_eq!(
store
.account(&id)
.expect("refreshed account")
.quota()
.access(),
gateway_core::engine::credential::QuotaAccessState::Unknown
);
}
#[tokio::test]
async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustion() {
let (store, repository) =
repository_with_accounts(&[("free-quota", "subject-free-quota")]).await;
let id = account_id("free-quota");
set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await;
store
.apply_quota_access(QuotaAccessChange {
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
})
.await
.expect("mark quota exhausted");
let transport = QueueBillingTransport::success(
br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"onDemandCap":{"val":0},"onDemandUsed":{"val":0},"prepaidBalance":{"val":0}}}"#,
);
@@ -876,8 +951,8 @@ async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustio
assert!(!snapshot.billing().has_authoritative_quota());
assert_eq!(
store.account(&id).expect("account").availability(),
AccountAvailability::QuotaExhausted
store.account(&id).expect("account").quota().access(),
gateway_core::engine::credential::QuotaAccessState::Exhausted
);
}
@@ -920,9 +995,9 @@ async fn quota_read_rejects_corrupt_provider_document() {
.compare_and_swap_quota(QuotaObservation {
account_id: account_id("corrupt"),
expected_revision: CredentialRevision::new(1).expect("revision"),
quota: Some(OpaqueProviderData::new(document)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(document),
observed_at: SystemTime::now(),
state: QuotaState::unknown(),
})
.await
.expect("seed corrupt quota");
@@ -975,8 +1050,7 @@ async fn failed_quota_refresh_does_not_replace_last_good_observation() {
.expect("read quota")
.pop()
.expect("quota remains")
.quota
.expect("provider document");
.quota;
assert_eq!(
persisted.expose_to_provider()["config"]["creditUsagePercent"].as_f64(),
Some(10.0),
@@ -4,7 +4,7 @@ use std::collections::VecDeque;
use std::future::ready;
use std::sync::{Arc, Mutex};
use gateway_core::engine::credential::AccountAvailability;
use gateway_core::engine::credential::CredentialState;
use provider_xai::{
AuthorizationCallback, FailClosedTokenVerifier, FailureClass, FormValue, GrokCredentialAdmin,
GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, GrokOAuthImportMetadata,
@@ -128,7 +128,7 @@ async fn credential_import_should_require_exact_metadata_and_official_userinfo()
enabled: true,
})
.expect("Provider projects verified token lifetime");
assert_eq!(prepared.account.availability(), AccountAvailability::Ready);
assert_eq!(prepared.account.credential_state(), CredentialState::Ready);
assert!(prepared.account.next_refresh_at().is_none());
assert!(
prepared
@@ -2,15 +2,14 @@ use std::sync::Arc;
use chrono::Utc;
use gateway_core::engine::credential::{
AccountAvailability, CredentialCasOutcome, CredentialRevision, NewProviderAccount,
PlaintextCredential, ProviderAccount, ProviderAccountStore,
AccountErrorReason, CredentialCasOutcome, CredentialRevision, CredentialState,
NewProviderAccount, PlaintextCredential, ProviderAccount, ProviderAccountStore,
};
use gateway_core::error::StoreErrorKind;
use gateway_core::routing::ProviderKind;
use provider_xai::{
GrokCredentialAdmin, GrokCredentialAvailability, GrokCredentialRepository,
GrokCredentialRepositoryError, GrokOAuthSecret, RotateManagedGrokCredential, SecretValue,
UpdateGrokCredentialState,
GrokCredentialAdmin, GrokCredentialRepository, GrokCredentialRepositoryError, GrokOAuthSecret,
RotateManagedGrokCredential, SecretValue, UpdateGrokCredentialState,
};
use crate::support::{
@@ -219,14 +218,15 @@ async fn state_update_uses_credential_revision_fence() {
.update_state(&UpdateGrokCredentialState {
account_id: input.account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: GrokCredentialAvailability::Ready,
availability_reason: Some("upstream_rate_limited".to_owned()),
credential_state: CredentialState::Ready,
error_reason: None,
error_message: Some("upstream_rate_limited".to_owned()),
observed_at: Utc::now(),
})
.await
.expect("state update");
let account = store.account(&input.account_id).expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
}
#[tokio::test]
@@ -239,16 +239,20 @@ async fn terminal_state_without_upstream_message_uses_stable_error_reason() {
.update_state(&UpdateGrokCredentialState {
account_id: input.account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: GrokCredentialAvailability::Banned,
availability_reason: None,
credential_state: CredentialState::Banned,
error_reason: Some(AccountErrorReason::AccountBanned),
error_message: None,
observed_at: Utc::now(),
})
.await
.expect("state update");
assert_eq!(
store.last_error_message(&input.account_id).as_deref(),
Some("account_banned")
store
.account(&input.account_id)
.expect("account")
.last_error_reason(),
Some(AccountErrorReason::AccountBanned)
);
}
@@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime};
use async_trait::async_trait;
use chrono::Utc;
use gateway_core::engine::credential::{
AccountAvailability, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
CredentialCasOutcome, CredentialCasUpdate, CredentialRevision, CredentialState,
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
};
use gateway_core::provider_ports::{
@@ -514,8 +514,8 @@ async fn unauthorized_recovery_marks_a_permanently_rejected_refresh_expired() {
assert_eq!(outcome, GrokCredentialRecoveryOutcome::Rejected);
assert_eq!(
store.account(&id).expect("account").availability(),
AccountAvailability::Expired
store.account(&id).expect("account").credential_state(),
CredentialState::Expired
);
assert_eq!(
store.last_error_message(&id).as_deref(),
@@ -664,8 +664,8 @@ async fn invalid_grant_marks_account_expired() {
fixture(input, [Err(GrokRefreshFailure::InvalidGrant)], true).await;
service.refresh_due().await.expect("refresh");
assert_eq!(
store.account(&id).expect("account").availability(),
AccountAvailability::Expired
store.account(&id).expect("account").credential_state(),
CredentialState::Expired
);
}
@@ -676,8 +676,8 @@ async fn banned_failure_marks_account_banned() {
let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Banned)], true).await;
service.refresh_due().await.expect("refresh");
assert_eq!(
store.account(&id).expect("account").availability(),
AccountAvailability::Banned
store.account(&id).expect("account").credential_state(),
CredentialState::Banned
);
assert_eq!(
store.last_error_message(&id).as_deref(),
@@ -704,8 +704,8 @@ async fn ambiguous_refresh_applies_bounded_oauth_backoff() {
[GrokCredentialRefreshOutcome::Ambiguous { account_id }] if account_id == &id
));
assert_eq!(
store.account(&id).expect("account").availability(),
AccountAvailability::Unknown
store.account(&id).expect("account").credential_state(),
CredentialState::Unknown
);
let account = store.account(&id).expect("account");
assert_eq!(account.revision().get(), 2);
@@ -736,7 +736,7 @@ async fn pre_send_transient_failure_applies_bounded_oauth_backoff() {
let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Transient)], true).await;
service.refresh_due().await.expect("refresh");
let account = store.account(&id).expect("account");
assert_eq!(account.availability(), AccountAvailability::Unknown);
assert_eq!(account.credential_state(), CredentialState::Unknown);
assert_eq!(account.revision().get(), 2);
assert!(
account
@@ -908,8 +908,11 @@ async fn recovery_window_exhaustion_marks_account_expired_without_token_exchange
[GrokCredentialRefreshOutcome::Invalidated { account_id }] if account_id == &id
));
assert_eq!(
store.account(&id).expect("expired account").availability(),
AccountAvailability::Expired
store
.account(&id)
.expect("expired account")
.credential_state(),
CredentialState::Expired
);
assert_eq!(refresher.prepare_calls.load(Ordering::SeqCst), 1);
assert_eq!(refresher.responses.lock().expect("queue").len(), 1);
@@ -4,10 +4,10 @@ use std::time::{Duration, SystemTime};
use chrono::Utc;
use gateway_core::engine::credential::{
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals,
AccountSelectionPolicy, CredentialCasOutcome, CredentialRevision, OpaqueProviderData,
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
QuotaWriteOutcome, RotationStrategy,
AccountAttemptFeedback, AccountFeedbackStats, AccountRuntimeSignals, AccountSelectionPolicy,
CredentialCasOutcome, CredentialRevision, CredentialState, OpaqueProviderData,
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaAccessState,
QuotaObservation, QuotaState, QuotaWriteOutcome, RotationStrategy,
};
use gateway_core::provider_ports::{
ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort,
@@ -17,10 +17,10 @@ use gateway_core::routing::UpstreamModelId;
use provider_xai::{
GrokAccountSessionSelector, GrokBillingRequest, GrokBillingTransport,
GrokBillingTransportError, GrokBillingTransportErrorKind, GrokBillingTransportFuture,
GrokCatalogScope, GrokCredentialAdmin, GrokCredentialAvailability, GrokCredentialCatalogCache,
GrokCredentialCatalogSeed, GrokCredentialFailure, GrokCredentialRepository, GrokPlanCatalog,
GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError,
RotateManagedGrokCredential, UpdateGrokCredentialState,
GrokCatalogScope, GrokCredentialAdmin, GrokCredentialCatalogCache, GrokCredentialCatalogSeed,
GrokCredentialFailure, GrokCredentialRepository, GrokPlanCatalog, GrokSessionSelection,
GrokSessionSelector, GrokSessionSelectorError, RotateManagedGrokCredential,
UpdateGrokCredentialState,
};
use crate::support::{
@@ -108,8 +108,9 @@ impl SelectorFixture {
.update_state(&UpdateGrokCredentialState {
account_id: input.account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: GrokCredentialAvailability::Ready,
availability_reason: None,
credential_state: CredentialState::Ready,
error_reason: None,
error_message: None,
observed_at: Utc::now(),
})
.await
@@ -131,7 +132,7 @@ impl SelectorFixture {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
quota_limit_reached: false,
rate_limited_until: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -241,11 +242,9 @@ impl SelectorFixture {
.compare_and_swap_quota(QuotaObservation {
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
quota: Some(OpaqueProviderData::new(
document.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
quota: OpaqueProviderData::new(document.as_object().expect("quota object").clone()),
observed_at: SystemTime::now(),
state: QuotaState::unknown(),
})
.await
.expect("persist quota");
@@ -305,16 +304,20 @@ async fn unauthorized_feedback_records_runtime_cooldown_without_persisting_accou
.store
.account(&selected)
.expect("selected")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
let other = [account_id("feedback-a"), account_id("feedback-b")]
.into_iter()
.find(|id| id != &selected)
.expect("other account");
assert_eq!(
fixture.store.account(&other).expect("other").availability(),
AccountAvailability::Ready
fixture
.store
.account(&other)
.expect("other")
.credential_state(),
CredentialState::Ready
);
assert!(
fixture
@@ -379,7 +382,7 @@ async fn account_scoped_cooldown_survives_credential_rotation() {
));
let account = fixture.store.account(&id).expect("rotated account");
assert_eq!(account.revision().get(), 2);
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
}
#[tokio::test]
@@ -401,7 +404,7 @@ async fn rate_limit_feedback_persists_the_grok2api_cooldown_state() {
)
.await;
let account = fixture.store.account(&selected).expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
assert!(
fixture
.cooldowns
@@ -440,7 +443,7 @@ async fn successful_request_clears_the_persisted_cooldown_state() {
.store
.account(session.account_id())
.expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
}
#[tokio::test]
@@ -469,8 +472,8 @@ async fn payment_required_feedback_writes_short_account_cooldown_without_persist
.store
.account(&selected)
.expect("selected account")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
assert!(
fixture
@@ -508,7 +511,7 @@ async fn model_quota_feedback_writes_model_scoped_cooldown_without_blocking_othe
.await;
let account = fixture.store.account(&selected).expect("selected account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60);
let scope = ProviderCooldownScope::upstream_model(failed_model);
// model-scoped cooldown:目标模型被排除,账号级无 cooldown。
@@ -563,8 +566,8 @@ async fn model_access_feedback_reports_model_cooldown_without_blocking_the_accou
.store
.account(session.account_id())
.expect("account")
.availability(),
AccountAvailability::Ready
.credential_state(),
CredentialState::Ready
);
assert!(matches!(
fixture
@@ -594,7 +597,7 @@ async fn interrupted_stream_feedback_persists_the_grok2api_cooldown_state() {
.store
.account(session.account_id())
.expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
assert_eq!(account.credential_state(), CredentialState::Ready);
assert!(
fixture
.cooldowns
@@ -630,8 +633,9 @@ async fn quota_feedback_uses_common_quota_exhausted_state() {
.store
.account(session.account_id())
.expect("account")
.availability(),
AccountAvailability::QuotaExhausted
.quota()
.access(),
QuotaAccessState::Exhausted
);
}
@@ -673,13 +677,18 @@ async fn quota_feedback_should_follow_the_account_across_a_credential_rotation()
.await;
assert_eq!(
fixture.store.account(&id).expect("account").availability(),
AccountAvailability::QuotaExhausted
fixture
.store
.account(&id)
.expect("account")
.quota()
.access(),
QuotaAccessState::Exhausted
);
}
#[tokio::test]
async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
async fn ordinary_success_clears_confirmed_exhaustion() {
let fixture = SelectorFixture::new(&["quota-success"]).await;
let session = fixture
.selector
@@ -690,7 +699,6 @@ async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
.selector
.record_failure(&session, GrokCredentialFailure::FreeQuotaExhausted)
.await;
fixture.selector.record_success(&session).await;
assert_eq!(
@@ -698,8 +706,9 @@ async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
.store
.account(session.account_id())
.expect("account")
.availability(),
AccountAvailability::QuotaExhausted
.quota()
.access(),
QuotaAccessState::Allowed
);
}
@@ -1061,8 +1070,8 @@ async fn bare_402_cooldown_expires_and_account_recovers_without_persisted_exhaus
.store
.account(&selected)
.expect("account")
.availability(),
AccountAvailability::Ready,
.credential_state(),
CredentialState::Ready,
"bare 402 must not persist QuotaExhausted"
);
let next = fixture
+87 -25
View File
@@ -8,10 +8,10 @@ use std::time::{Duration, SystemTime};
use async_trait::async_trait;
use chrono::Utc;
use gateway_core::engine::credential::{
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, LoadedCredential, NewProviderAccount, PlaintextCredential, ProviderAccount,
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
QuotaWriteOutcome,
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount, PlaintextCredential,
ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome,
};
use gateway_core::error::{StoreError, StoreErrorKind};
use gateway_core::provider_ports::{
@@ -23,11 +23,10 @@ use gateway_core::provider_ports::{
use gateway_core::routing::ProviderKind;
use provider_xai::{
CreateGrokCredential, GROK_BILLING_URL, GrokAccountProfile, GrokBillingTransport,
GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, GrokCredentialAvailability,
GrokCredentialCatalogCache, GrokCredentialCatalogService, GrokCredentialQuotaService,
GrokCredentialRepository, GrokCredentialRepositoryError, GrokEndpointPolicy,
GrokModelCatalogTransport, GrokOAuthSecret, GrokPlanCatalog, GrokReqwestTransportBuildError,
SecretValue, XaiConfig, XaiWireProfileState,
GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, GrokCredentialCatalogCache,
GrokCredentialCatalogService, GrokCredentialQuotaService, GrokCredentialRepository,
GrokCredentialRepositoryError, GrokEndpointPolicy, GrokModelCatalogTransport, GrokOAuthSecret,
GrokPlanCatalog, GrokReqwestTransportBuildError, SecretValue, XaiConfig, XaiWireProfileState,
};
use reqwest::Client;
use reqwest::redirect::Policy;
@@ -194,7 +193,6 @@ struct StoredAccount {
account: ProviderAccount,
credential: PlaintextCredential,
quota: Option<QuotaObservation>,
last_error_message: Option<String>,
}
#[derive(Default)]
@@ -218,7 +216,7 @@ impl MemoryProviderAccountStore {
pub fn last_error_message(&self, id: &ProviderAccountId) -> Option<String> {
lock(&self.accounts)
.get(id)
.and_then(|stored| stored.last_error_message.clone())
.and_then(|stored| stored.account.last_error_message().map(str::to_owned))
}
pub fn credential(&self, id: &ProviderAccountId) -> Option<PlaintextCredential> {
@@ -253,7 +251,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
account: account.account,
credential: account.credential,
quota: None,
last_error_message: None,
},
);
Ok(())
@@ -335,7 +332,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
AccountReplacement {
revision: next,
access_token_expires_at,
availability: stored.account.availability(),
credential_state: stored.account.credential_state(),
quota: stored.account.quota(),
last_error_reason: stored.account.last_error_reason(),
last_error_message: stored.account.last_error_message().map(str::to_owned),
enabled: stored.account.enabled(),
has_refresh_token,
next_refresh_at,
@@ -372,7 +372,31 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
if stored.account.revision() != observation.expected_revision {
return Ok(QuotaWriteOutcome::Conflict);
}
let quota = observation.state;
stored.quota = Some(observation);
stored.account = rebuild_account(
&stored.account,
AccountReplacement::preserving(&stored.account).with_quota(quota),
);
Ok(QuotaWriteOutcome::Updated)
}
async fn apply_quota_access(
&self,
change: QuotaAccessChange,
) -> Result<QuotaWriteOutcome, StoreError> {
let mut accounts = lock(&self.accounts);
let stored = accounts.get_mut(&change.account_id).ok_or_else(invalid)?;
if stored.account.revision() != change.expected_revision {
return Ok(QuotaWriteOutcome::Conflict);
}
stored.account = rebuild_account(
&stored.account,
AccountReplacement::preserving(&stored.account).with_quota(change.state),
);
if let Some(observation) = &mut stored.quota {
observation.state = change.state;
}
Ok(QuotaWriteOutcome::Updated)
}
@@ -387,7 +411,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
AccountReplacement {
revision: stored.account.revision(),
access_token_expires_at: stored.account.access_token_expires_at(),
availability: change.availability,
credential_state: change.credential_state,
quota: stored.account.quota(),
last_error_reason: change.error_reason,
last_error_message: change.message.clone(),
enabled: stored.account.enabled(),
has_refresh_token: stored.account.has_refresh_token(),
next_refresh_at: stored.account.next_refresh_at(),
@@ -396,11 +423,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
plan_type: stored.account.plan_type().map(str::to_owned),
},
);
if change.availability == AccountAvailability::Ready {
stored.last_error_message = None;
} else if let Some(message) = change.message {
stored.last_error_message = Some(message);
}
Ok(())
}
@@ -412,7 +434,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
AccountReplacement {
revision: stored.account.revision(),
access_token_expires_at: stored.account.access_token_expires_at(),
availability: stored.account.availability(),
credential_state: stored.account.credential_state(),
quota: stored.account.quota(),
last_error_reason: stored.account.last_error_reason(),
last_error_message: stored.account.last_error_message().map(str::to_owned),
enabled: stored.account.enabled(),
has_refresh_token: stored.account.has_refresh_token(),
next_refresh_at: stored.account.next_refresh_at(),
@@ -436,7 +461,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
AccountReplacement {
revision: stored.account.revision(),
access_token_expires_at: stored.account.access_token_expires_at(),
availability: stored.account.availability(),
credential_state: stored.account.credential_state(),
quota: stored.account.quota(),
last_error_reason: stored.account.last_error_reason(),
last_error_message: stored.account.last_error_message().map(str::to_owned),
enabled,
has_refresh_token: stored.account.has_refresh_token(),
next_refresh_at: stored.account.next_refresh_at(),
@@ -457,7 +485,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
struct AccountReplacement {
revision: CredentialRevision,
access_token_expires_at: Option<SystemTime>,
availability: AccountAvailability,
credential_state: CredentialState,
quota: QuotaState,
last_error_reason: Option<AccountErrorReason>,
last_error_message: Option<String>,
enabled: bool,
has_refresh_token: bool,
next_refresh_at: Option<SystemTime>,
@@ -466,6 +497,30 @@ struct AccountReplacement {
plan_type: Option<String>,
}
impl AccountReplacement {
fn preserving(account: &ProviderAccount) -> Self {
Self {
revision: account.revision(),
access_token_expires_at: account.access_token_expires_at(),
credential_state: account.credential_state(),
quota: account.quota(),
last_error_reason: account.last_error_reason(),
last_error_message: account.last_error_message().map(str::to_owned),
enabled: account.enabled(),
has_refresh_token: account.has_refresh_token(),
next_refresh_at: account.next_refresh_at(),
name: account.name().to_owned(),
email: account.email().map(str::to_owned),
plan_type: account.plan_type().map(str::to_owned),
}
}
fn with_quota(mut self, quota: QuotaState) -> Self {
self.quota = quota;
self
}
}
fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement) -> ProviderAccount {
ProviderAccount::new(
previous.id().clone(),
@@ -481,7 +536,13 @@ fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement)
previous.upstream_account_id().map(str::to_owned),
replacement.plan_type,
)
.with_runtime_state(replacement.enabled, replacement.availability)
.with_account_facts(
replacement.enabled,
replacement.credential_state,
replacement.quota,
replacement.last_error_reason,
replacement.last_error_message,
)
.with_refresh_schedule(replacement.has_refresh_token, replacement.next_refresh_at)
}
@@ -549,8 +610,9 @@ pub fn create_input(suffix: &str, subject: &str) -> CreateGrokCredential {
},
account: profile(subject),
enabled: true,
initial_availability: GrokCredentialAvailability::Unknown,
initial_availability_reason: None,
initial_credential_state: CredentialState::Unknown,
initial_error_reason: None,
initial_error_message: None,
}
}
+1
View File
@@ -1,3 +1,4 @@
5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql
f4168f7e23d42cf9d37096339e931b0e0c823f7a8dbfc6580a04f096c4c3d8e3 0002_provider_accounts_nullable_upstream_user_id.sql
4163f61183d9e90f1368c08f614040d6a1a0048de3886a77dd708ce21430b2d0 0003_backfill_openai_access_token_expiry.sql
b41bb3be5a84b95852158d82267456f8415811a1137025581eefd1c96c0882d1 0004_normalize_account_state_facts.sql
@@ -0,0 +1,165 @@
-- Separate credential identity facts from quota access facts. The public account
-- status remains derived at read time because Redis cooldowns and reset times are
-- time-dependent.
alter table provider_accounts
rename column availability to credential_state;
alter table provider_accounts
rename column availability_observed_at to credential_observed_at;
alter table provider_accounts
drop constraint provider_accounts_availability_ck;
alter table provider_accounts
add constraint provider_accounts_credential_state_ck check (
credential_state in ('unknown', 'ready', 'expired', 'banned', 'invalid')
) not valid;
alter table provider_accounts
add column quota_access_state text not null default 'unknown',
add column quota_evidence text,
add column quota_access_observed_at timestamptz,
add column quota_reset_at timestamptz,
add column last_error_reason text;
alter table provider_accounts
drop constraint provider_accounts_quota_observation_ck;
-- Old quota_exhausted credential states are quota facts. Re-derive OpenAI quota
-- access from the raw authoritative response and never trust the old percentage-
-- derived quota_limit_reached boolean.
update provider_accounts
set
quota_access_state = case
when provider_kind = 'openai'
and provider_quota_json #>> '{rate_limit,allowed}' = 'true'
then 'allowed'
when provider_kind = 'openai'
and (
provider_quota_json #>> '{rate_limit,allowed}' = 'false'
or (
coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true'
and (
provider_quota_json #>> '{rate_limit,limit_reached}' = 'true'
or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true'
)
)
)
then 'exhausted'
when provider_kind = 'xai' and credential_state = 'quota_exhausted'
then 'exhausted'
when provider_quota_json is not null then 'unknown'
else 'unknown'
end,
quota_evidence = case
when provider_kind = 'openai'
and provider_quota_json #>> '{rate_limit,allowed}' = 'false'
then 'provider_denied'
when provider_kind = 'openai'
and coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true'
and (
provider_quota_json #>> '{rate_limit,limit_reached}' = 'true'
or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true'
)
then 'account_limit_reached'
when provider_kind = 'xai' and credential_state = 'quota_exhausted'
then 'usage_limit_reached'
else null
end,
quota_access_observed_at = case
when provider_kind = 'openai' and provider_quota_json is not null
then quota_observed_at
when provider_kind = 'xai' and credential_state = 'quota_exhausted'
then credential_observed_at
else null
end,
quota_reset_at = case
when provider_kind = 'openai'
and jsonb_typeof(provider_quota_json #> '{rate_limit,primary_window,reset_at}') = 'number'
then to_timestamp((provider_quota_json #>> '{rate_limit,primary_window,reset_at}')::double precision)
else null
end,
credential_state = case
when credential_state = 'quota_exhausted' and upstream_user_id is not null then 'ready'
when credential_state = 'quota_exhausted' then 'unknown'
else credential_state
end,
last_error_reason = case credential_state
when 'unknown' then 'account_unverified'
when 'expired' then 'credential_expired'
when 'invalid' then 'credential_invalid'
when 'banned' then 'account_banned'
else null
end,
last_error_message = case
when credential_state in ('unknown', 'expired', 'invalid', 'banned') then last_error_message
else null
end;
alter table provider_accounts
validate constraint provider_accounts_credential_state_ck;
alter table provider_accounts
add constraint provider_accounts_quota_access_state_ck check (
quota_access_state in ('unknown', 'allowed', 'exhausted')
),
add constraint provider_accounts_quota_evidence_ck check (
quota_evidence is null
or quota_evidence in (
'provider_denied',
'account_limit_reached',
'usage_limit_reached',
'payment_required'
)
),
add constraint provider_accounts_quota_fact_ck check (
(quota_access_state = 'unknown' and quota_evidence is null and quota_reset_at is null)
or (
quota_access_state = 'allowed'
and quota_evidence is null
and quota_access_observed_at is not null
and quota_reset_at is null
)
or (
quota_access_state = 'exhausted'
and quota_evidence is not null
and quota_access_observed_at is not null
)
),
add constraint provider_accounts_quota_observation_ck check (
(provider_quota_json is null) = (quota_observed_at is null)
),
add constraint provider_accounts_error_reason_ck check (
last_error_reason is null
or last_error_reason in (
'account_unverified',
'access_token_expired',
'credential_expired',
'credential_invalid',
'account_banned'
)
);
drop index provider_accounts_availability_idx;
create index provider_accounts_credential_state_idx
on provider_accounts (credential_state, id);
create index provider_accounts_quota_access_idx
on provider_accounts (quota_access_state, quota_reset_at, id);
alter table provider_accounts
drop column quota_limit_reached;
alter table provider_accounts
rename constraint provider_accounts_time_ck to provider_accounts_time_ck_old;
alter table provider_accounts
drop constraint provider_accounts_time_ck_old;
alter table provider_accounts
add constraint provider_accounts_time_ck check (
created_at <= updated_at
and credential_observed_at <= updated_at
and (quota_access_observed_at is null or quota_access_observed_at <= updated_at)
and (quota_observed_at is null or quota_observed_at <= updated_at)
);
+12 -6
View File
@@ -112,7 +112,7 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段
账号列表支持以下稳定值:
- `provider`: `all`、`openai`、`xai`;
- `status`: `active`、`expired`、`quota_exhausted`、`disabled`、`banned`;
- `status`: `normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`;
- `sortBy`: `email`、`status`、`planType`、`usage`、`lastUsedAt`、`expiresAt`;
- `sortDirection`: `asc`、`desc`。
@@ -184,13 +184,19 @@ OAuth start 使用:
但上游 authorization code 本身通常只能交换一次;已完成过 token exchange 时应重新创建 OAuth flow。
- `GET /accounts/quota` 只读取最后一次落库快照;`POST /accounts/quota/refresh` 才访问上游。access token
已过期时,额度刷新要求先走 credential 刷新或重新授权,不会拿过期 token 探测额度。
- 成功额度观测会 revision-fenced 写入 quota,并在 `active` 与 `quota_exhausted` 等额度所属状态间同步;
不会清除 `expired`、`banned` 等 credential/封禁终态。额度接口的 401/403 也不足以判定 refresh token
永久失效,credential 终态只由 OAuth refresh 的明确永久错误写入。
- 成功额度观测会 revision-fenced 写入 quota;明确 `Allowed` 投影为 `normal`,明确耗尽投影为
`quota_exhausted`。额度观测不会清除凭据过期、无效或封禁事实;这些事实统一投影为 `error`,并由
`errorReason` 区分。额度接口的 401/403 也不足以判定 refresh token 永久失效,credential 终态只由
OAuth refresh 的明确永久错误写入。
- 正常 Responses 请求会解析上游响应的 rate-limit headers,合并进同一 quota 快照并同步状态。Free、
K12 等套餐共用该状态机;套餐只参与账号展示和按套餐隔离的模型目录 cache,不存在 K12 专属额度路径。
- 账号页没有定时静默轮询。手工额度刷新完成后页面会重新读取该账号;请求驱动或后台任务产生的状态
变化,需要下一次显式查询账号列表后才会显示。
- 账号展开区的 Token 结构和模型排行使用代表性账号级额度窗口聚合,查询边界严格为
`[resetAt - windowSeconds, resetAt)`;额度刷新若返回了更早的重置时间,会按新边界重新聚合。无法取得
完整窗口边界或只有模型专属额度时显示无数据,不回退成历史累计。金额原值保持完整精度,USD 展示值
小于 1 美元时最多保留四位小数,其余保留两位。
- 账号页没有定时静默轮询。手工额度刷新只替换响应中的账号行并同步状态汇总,不触发整页 loading;若
新状态不符合当前筛选,该行从当前页移除。请求驱动或后台任务产生的状态变化,需要下一次显式查询账号
列表后才会显示。
## 6. Client Key
+24 -10
View File
@@ -116,7 +116,17 @@ Provider 独占以下职责:
- OpenAI 是逐字节透明代理:请求 body 的未知字段和字段顺序保持不变(受控模型映射除外),上游 wire 是客户端可见的事实来源,HTTP SSE 帧以原始字节原样下发,WebSocket 上游 JSON 文本原样下发;response ID 以 opaque bytes 存储,不假设 UUID 或固定长度。canonical facts 只从同一帧旁路解析,用于观测、亲和与计费。仅网关内部帧(rate limits、metadata)被就地消费,不向客户端转发。
- xAI 是翻译层:把 Grok wire 解码后重新合成 Responses wire 事件,不承诺字节透明;上游错误按结构化 message/code/type 透出,message 中的 UUID 账号指纹替换为占位符。
credential 更新使用 `credential_revision` CAS。认证永久失败、封禁、额度耗尽和带截止时间的 cooldown 是账号运行时事实。cooldown 到期后,Core 的有效调度谓词会自然允许账号重新参与选择;Redis cooldown 只是热缓存。成功调用或成功额度观测才可把账号状态重新观测为 `ready`。推理请求的账号/model cooldown 只约束数据面调度,不参与 OAuth refresh;OpenAI 与 xAI 的 OAuth refresh 请求状态不明确时只按 OAuth 自身的有界退避推进该 credential 的 `next_refresh_at`:前五次约为 5/15/45/135/300 秒,之后约每 10 分钟恢复;仍有效 AT 时不晚于其到期时刻,AT 到期后仍按该节奏恢复,至 `AT 到期 + 24h` 才终态化。自动 refresh 的瞬态失败(限流、5xx、超时、畸形响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;上游明确的永久错误(`invalid_grant`、封禁)仍立即写入终态失效。
credential 更新使用 `credential_revision` CAS。PostgreSQL 分别保存凭据事实与额度访问事实;带截止时间的
429 cooldown 只保存在 Redis,是唯一的临时冷却运行时事实,不进入 Provider quota 展示模型。Store 把三类
事实交给 Core 的唯一解析器,投影为 `normal`、`quota_exhausted`、`rate_limited`、`disabled` 或 `error`;
投影为 `rate_limited` 时同时携带有效的 `rate_limited_until`,到期重算后该时间和状态一起消失。若没有其它
阻塞事实,账号才恢复为 `normal`。成功推理或明确的成功额度观测可以写入 `Allowed`;本地时钟与不确定响应
不能伪造恢复。推理请求的账号/model cooldown 只约束数据面调度,不参与 OAuth refresh;OpenAI 与 xAI 的
OAuth refresh 请求状态不明确时只按 OAuth 自身的有界退避推进该 credential 的 `next_refresh_at`:前五次
约为 5/15/45/135/300 秒,之后约每 10 分钟恢复;仍有效 Access Token 时不晚于其到期时刻,Access Token
到期后仍按该节奏恢复,至“到期 + 24 小时”才终态化。自动 refresh 的瞬态失败(限流、5xx、超时、畸形
响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;上游明确的永久错误
(`invalid_grant`、封禁)仍立即写入终态失效。
`refresh_token_expires_at` 不是公共 SQL 列或 Core 权威状态。xAI 可在 `provider_credentials_json` 内保存它作为 Provider 私有提示;真正失效以 refresh endpoint 返回的永久错误为准。
@@ -127,12 +137,15 @@ exchange 成功后执行同一补全。两条路径不执行 JWKS 验签,不
credential revision;complete 阶段重新读取目标账号及其当前 revision,并以当前 revision 做最终 CAS。
重新授权和手工或后台 RT refresh 只轮换目标账号的 token,保留既有账号资料与 OAuth principal。
credential 与 quota 是两套状态机。主动 quota refresh 会拒绝过期 access token;quota 探测的
401/403 不具备判定 refresh token 永久失效的证据。成功主动观测和正常推理响应中的 rate-limit header
观测都 revision-fenced 写入同一 quota,并转换 readiness、`QuotaExhausted` 或 `Cooldown` 等额度事实;
只有明确的 deactivated workspace 才能由额度链路写入 `Banned`,且额度观测不能清除 `Invalid`、
`Expired`、`Banned`。Free、K12 等套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider
事实并用于隔离不同套餐的 catalog cache。
credential 与 quota 是两组独立事实,不是两套对外状态。主动 quota refresh 会拒绝过期 Access Token;
quota 探测的 401/403 不具备判定 Refresh Token 永久失效的证据。成功主动观测和正常推理响应中的
rate-limit header 观测都 revision-fenced 写入同一 quota;只有明确的访问证据才写 `Allowed` 或
`Exhausted`。只有明确的 deactivated workspace 才能由额度链路写入 `Banned` 凭据事实,且额度观测不能
清除 `Invalid`、`Expired`、`Banned`。这些凭据事实对外统一投影为 `error` 并携带稳定原因。Free、K12 等
套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider 事实并用于隔离不同套餐的 catalog cache。
账号展开用量不读取 retention 范围累计值,而是选择带完整边界且可归属到账号的代表性额度窗口,并按
`[reset_at - window_seconds, reset_at)` 聚合请求、Token、模型和成本;额度重置边界变化后,同一查询随刷新
结果重算。模型专属窗口不会错误复用账号总用量,边界不完整时也不伪造“当前窗口”统计。
## 7. 控制面与内部 revision
@@ -156,7 +169,7 @@ Admin API 不要求客户端提交配置 revision,也不向客户端暴露配
| `admin_audit_events` | 管理 mutation 审计 |
| `client_api_keys` | 客户端鉴权、限额与授权范围 |
| `runtime_settings` | 全局配置与 `config_revision` |
| `provider_accounts` | 账号资料、Provider-owned 明文 credential JSON、revision、quota、cooldown |
| `provider_accounts` | 账号资料、Provider-owned 明文 credential JSON、revision、凭据事实与规范化 quota 事实 |
| `model_requests` | 请求、attempt、计费、交付与恢复事实 |
| `ops_events` | 脱敏运行事件 |
| `backup_settings` | S3/R2 存储、Cron 计划与保留策略单例配置 |
@@ -184,13 +197,14 @@ heartbeat;计划时间点冲突只记录日志/指标并推进游标,不产
业务 schema 由 `backend/migrations/0001_initial.sql` 直接定义当前大版本的完整新库基线,其中
response ID 使用无格式假设的 UTF-8 bytes,`model_requests.service_tier` 保存响应观测到的服务档位,
且额度耗尽账号可以携带冷却截止时间。同一大版本内已合入的迁移永久冻结——
账号对外五态由 PostgreSQL 凭据/额度事实与 Redis 429 冷却通过 Core 唯一解析器动态派生,
不保存重复的 `status` 列。同一大版本内已合入的迁移永久冻结——
`.frozen-sha256` 是冻结清单,CI 校验清单相对 PR base 只增不改,启动时 sqlx 重校验已应用
迁移的 checksum,不一致直接拒绝启动。规则见 `backend/migrations/README.md`。
## 9. Redis 终态
Redis 保存的协调状态恰好是:客户端 admission 热状态、credential lease/fencing 与调度信号、账号 cooldown 与 availability 热缓存、catalog 缓存、Provider circuit、会话亲和与会话级账号排除、continuation pin、OAuth pending flow、worker leader lease 以及 runtime change 通知。Redis 丢失后必须从 PostgreSQL 或 Provider 事实恢复;恢复完成前需要保护的 acquire 路径 fail closed。
Redis 保存的协调状态恰好是:客户端 admission 热状态、credential lease/fencing 与调度信号、账号 429 cooldown 与 credential state 可重建缓存、catalog 缓存、Provider circuit、会话亲和与会话级账号排除、continuation pin、OAuth pending flow、worker leader lease以及 runtime change 通知。Redis 丢失后必须从 PostgreSQL 或 Provider 事实恢复;恢复完成前需要保护的 acquire 路径 fail closed。
多副本部署时,跨副本协调面只有上述 Redis 状态与 PostgreSQL 事实;WebSocket 连接池、请求画像与 RuntimeSnapshot 副本都是进程内状态,其中 RuntimeSnapshot 依靠周期对账收敛,请求画像依靠各自的官方版本检查收敛。
+12 -4
View File
@@ -1,5 +1,15 @@
import request from '../request'
export type AccountStatus
= 'normal' | 'quota_exhausted' | 'rate_limited' | 'disabled' | 'error'
export type AccountErrorReason
= 'account_unverified'
| 'access_token_expired'
| 'credential_expired'
| 'credential_invalid'
| 'account_banned'
export interface AccountQuotaWindow {
key: string
group: string
@@ -104,11 +114,9 @@ export interface Account {
planType: string | null
authenticationKind: string
hasRefreshToken: boolean
status: string
displayStatus: string
errorReason: string | null
status: AccountStatus
errorReason: AccountErrorReason | null
errorMessage: string | null
availability: string
enabled: boolean
accessTokenExpiresAt: string | null
accessTokenExpiresAtDisplay: string | null
+3 -6
View File
@@ -143,14 +143,11 @@ export interface DashboardAccountUsage {
export interface DashboardPoolSummary {
total: number
active: number
rateLimited: number
expired: number
invalid: number
normal: number
quotaExhausted: number
refreshing: number | null
rateLimited: number
disabled: number
banned: number
error: number
}
export interface DashboardCapacityInfo {
@@ -1,4 +1,5 @@
<script setup lang="ts">
import type { AccountErrorReason, AccountStatus } from '@/api'
import { AlertTriangle, CircleCheck, Gauge, Power, Timer } from '@lucide/vue'
import { computed } from 'vue'
@@ -8,8 +9,8 @@ import { errorReasonLabels, statusLabels, statusTones } from '../constants'
const props = withDefaults(
defineProps<{
status: string
errorReason?: string | null
status: AccountStatus
errorReason?: AccountErrorReason | null
errorMessage?: string | null
rateLimitedUntil?: string | null
variant?: 'inline' | 'pill'
@@ -55,12 +56,12 @@ const statusStyles = {
},
} as const
const tone = computed(() => statusTones[props.status] || 'normal')
const tone = computed(() => statusTones[props.status])
const statusStyle = computed(() => statusStyles[tone.value])
const label = computed(() => statusLabels[props.status] || props.status)
const label = computed(() => statusLabels[props.status])
const reasonLabel = computed(() =>
props.errorReason
? errorReasonLabels[props.errorReason] || props.errorReason
? errorReasonLabels[props.errorReason]
: null,
)
const errorText = computed(() => props.errorMessage || null)
@@ -102,16 +103,16 @@ const recoveryHint = computed(() => {
return '冷却结束后,系统会自动恢复调度。'
switch (props.errorReason) {
case 'expired':
case 'token_expired':
case 'account_unverified':
case 'access_token_expired':
case 'credential_expired':
return '重新授权后,账号会重新参与调度。'
case 'invalid':
case 'credential_invalid':
return '请更新或重新导入凭据,再次测试连接。'
case 'banned':
case 'account_banned':
return '请确认上游账号状态,解除限制后再启用。'
default:
return '重新测试连接以获取最新状态。'
}
return '重新测试连接以获取最新状态。'
})
const triggerLabel = computed(() =>
@@ -11,9 +11,12 @@ defineProps<{
class="grid gap-4 rounded-lg bg-(--cp-bg-surface) p-4 shadow-(--cp-shadow-control) xl:grid-cols-[0.52fr_1.48fr]"
>
<div>
<h3 class="m-0 mb-3 text-[14px] font-heavy text-(--cp-text-primary)">
Token 结构
</h3>
<div class="mb-3 flex items-baseline justify-between gap-3">
<h3 class="m-0 text-[14px] font-heavy text-(--cp-text-primary)">
Token 结构
</h3>
<span class="text-[11px] font-emphasis text-(--cp-text-muted)">当前额度窗口</span>
</div>
<div class="grid gap-2">
<div class="flex items-center justify-between rounded-lg bg-(--cp-success-bg) px-3 py-2">
<span class="text-[12px] font-bold text-(--cp-success-text)">输入 Tokens</span>
@@ -55,10 +58,11 @@ defineProps<{
<h3 class="m-0 text-[14px] font-heavy text-(--cp-text-primary)">
模型使用排行
</h3>
<span class="text-[11px] font-emphasis text-(--cp-text-muted)">当前额度窗口</span>
</div>
<div
class="grid grid-cols-[1.2fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pb-2 text-[11px] font-heavy text-(--cp-text-secondary) shadow-[inset_0_-1px_0_var(--cp-divider-subtle)]"
class="grid grid-cols-[1.5fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pb-2 text-[11px] font-heavy text-(--cp-text-secondary) shadow-[inset_0_-1px_0_var(--cp-divider-subtle)]"
>
<span>模型</span>
<span>调用</span>
@@ -66,9 +70,9 @@ defineProps<{
<span>输入</span>
<span>输出</span>
<span>缓存</span>
<span>总TOKEN</span>
<span>计费金额</span>
<span>最近请求时间</span>
<span>总计</span>
<span>计费</span>
<span>最近请求</span>
</div>
<div
v-if="account.usage.models.length === 0"
@@ -80,7 +84,7 @@ defineProps<{
<div
v-for="model in account.usage.models"
:key="model.model"
class="grid grid-cols-[1.2fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pt-3 text-[12px] font-emphasis text-(--cp-text-primary)"
class="grid grid-cols-[1.5fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pt-3 text-[12px] font-emphasis text-(--cp-text-primary)"
>
<span class="truncate">{{ model.model }}</span>
<span>{{ model.requestCountDisplay }}</span>
@@ -42,16 +42,11 @@ interface ConnectionTestEvent {
model?: string
payload?: ConnectionTestRequestPayload
success?: boolean
accountStatus?: string
upstreamBody?: string
upstreamStatus?: number
error?: string
}
interface AccountConnectionTestOptions {
onAccountStatus: (accountId: string, status: string) => void
}
function connectionTestErrorText(event: ConnectionTestEvent) {
if (typeof event.upstreamBody === 'string' && event.upstreamBody.length > 0)
return event.upstreamBody
@@ -69,7 +64,7 @@ function connectionTestFailureLabel(event: ConnectionTestEvent) {
: '未能向上游发起请求'
}
export function useAccountConnectionTest(options: AccountConnectionTestOptions) {
export function useAccountConnectionTest(options: { reload: () => Promise<unknown> }) {
const showConnectionTestModal = shallowRef(false)
const testingAccount = shallowRef<Account | null>(null)
const connectionTestStatus = shallowRef<ConnectionTestStatus>('idle')
@@ -232,18 +227,6 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
)
}
function applyAccountStatus(status?: string) {
const account = testingAccount.value
if (!account || !status)
return
testingAccount.value = {
...account,
status,
displayStatus: status,
}
options.onAccountStatus(account.id, status)
}
function clearConnectionTestRun() {
const run = connectionTestRun
connectionTestRun = undefined
@@ -283,7 +266,6 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
return
}
if (event.type === 'test_complete') {
applyAccountStatus(event.accountStatus)
if (event.success) {
if (!connectionTestContent.value) {
setConnectionTestLog('response', '响应完成', 'success', '上游已完成,没有返回文本内容')
@@ -297,10 +279,10 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
finishConnectionTest('error')
}
clearConnectionTestRun()
void options.reload()
return
}
if (event.type === 'error') {
applyAccountStatus(event.accountStatus)
connectionTestError.value = connectionTestErrorText(event)
appendConnectionTestLog(
connectionTestFailureLabel(event),
@@ -309,6 +291,7 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
)
finishConnectionTest('error')
clearConnectionTestRun()
void options.reload()
}
}
@@ -1,7 +1,5 @@
import type { Ref } from 'vue'
import type { getAccounts } from '@/api'
import type { BaseTableSort } from '@/components/base/BaseTable/columns'
import dayjs from 'dayjs'
import { ref, watch } from 'vue'
import {
@@ -9,7 +7,6 @@ import {
disableAccount,
enableAccount,
exportAccounts,
getAccountQuota,
refreshAccount,
refreshAccountQuota,
} from '@/api'
@@ -23,22 +20,11 @@ import { useAccountOnboarding } from './useAccountOnboarding'
type AccountRow = Awaited<ReturnType<typeof getAccounts>>['items'][number]
const accountStatusSortRank: Record<string, number> = {
normal: 0,
quota_exhausted: 1,
rate_limited: 2,
disabled: 3,
error: 4,
}
export function useAccountMutations(options: {
accounts: Ref<AccountRow[]>
accountSummary: Ref<Awaited<ReturnType<typeof getAccounts>>['summary']>
statusQuery: Ref<string>
sort: Ref<BaseTableSort | undefined>
selectedIds: Ref<Set<string>>
totalAccounts: Ref<number>
reload: () => Promise<unknown>
replaceAccount: (account: AccountRow) => boolean
}) {
const loadAccounts = options.reload
const { downloadJson } = useDownload()
@@ -188,13 +174,13 @@ export function useAccountMutations(options: {
async function handleRefreshQuota(accountId: string) {
await refreshingQuotaAccounts.run(accountId, async () => {
try {
await withMinimumDuration(async () => {
await refreshAccountQuota({ accountId })
const result = await getAccountQuota({ accountId })
options.accounts.value = options.accounts.value.map(account =>
account.id === accountId ? { ...account, ...result.account } : account,
)
})
const result = await withMinimumDuration(() => refreshAccountQuota({ accountId }))
const remainsVisible = options.replaceAccount(result.account)
if (!remainsVisible) {
const selectedIds = new Set(options.selectedIds.value)
selectedIds.delete(accountId)
options.selectedIds.value = selectedIds
}
toast.success('额度已刷新')
}
catch (error: unknown) {
@@ -203,81 +189,6 @@ export function useAccountMutations(options: {
})
}
function patchAccountStatus(accountId: string, status: string) {
const current = options.accounts.value.find(account => account.id === accountId)
if (!current)
return
if (options.statusQuery.value && options.statusQuery.value !== status) {
options.accounts.value = options.accounts.value.filter(account => account.id !== accountId)
options.totalAccounts.value = Math.max(0, options.totalAccounts.value - 1)
options.selectedIds.value = new Set(
[...options.selectedIds.value].filter(id => id !== accountId),
)
}
else {
const rows = options.accounts.value.map(account =>
account.id === accountId
? {
...account,
status,
displayStatus: status,
}
: account,
)
options.accounts.value = sortAccountsByStatus(rows)
}
if (current.status === status)
return
options.accountSummary.value = {
...options.accountSummary.value,
normal: Math.max(
0,
options.accountSummary.value.normal
+ Number(status === 'normal')
- Number(current.status === 'normal'),
),
quotaExhausted: Math.max(
0,
options.accountSummary.value.quotaExhausted
+ Number(status === 'quota_exhausted')
- Number(current.status === 'quota_exhausted'),
),
rateLimited: Math.max(
0,
options.accountSummary.value.rateLimited
+ Number(status === 'rate_limited')
- Number(current.status === 'rate_limited'),
),
disabled: Math.max(
0,
options.accountSummary.value.disabled
+ Number(status === 'disabled')
- Number(current.status === 'disabled'),
),
error: Math.max(
0,
options.accountSummary.value.error
+ Number(status === 'error')
- Number(current.status === 'error'),
),
}
}
function sortAccountsByStatus(rows: AccountRow[]) {
if (options.sort.value?.key !== 'status')
return rows
const direction = options.sort.value.direction === 'asc' ? 1 : -1
return [...rows].sort((left, right) => {
const rankDifference
= (accountStatusSortRank[left.status] ?? 7) - (accountStatusSortRank[right.status] ?? 7)
return rankDifference === 0
? left.id.localeCompare(right.id) * direction
: rankDifference * direction
})
}
async function handleToggleSchedule(account: AccountRow) {
const action = account.enabled ? 'disable' : 'enable'
await updatingStatusAccounts.run(account.id, async () => {
@@ -362,7 +273,6 @@ export function useAccountMutations(options: {
handleExportAccounts,
handleRefresh,
handleRefreshQuota,
patchAccountStatus,
handleToggleSchedule,
scheduleActionLabel,
}
@@ -7,6 +7,18 @@ import { toast } from '@/components/base/BaseToast'
import { usePagedQuery } from '@/composables/usePagedQuery'
import { errorMessage } from '@/utils/async'
type AccountRow = Awaited<ReturnType<typeof getAccounts>>['items'][number]
type AccountSummary = Awaited<ReturnType<typeof getAccounts>>['summary']
type AccountSummaryStatusKey = Exclude<keyof AccountSummary, 'total'>
const summaryKeyByStatus = {
normal: 'normal',
quota_exhausted: 'quotaExhausted',
rate_limited: 'rateLimited',
disabled: 'disabled',
error: 'error',
} as const satisfies Record<AccountRow['status'], AccountSummaryStatusKey>
export function useAccountsQuery() {
const searchQuery = shallowRef('')
const providerQuery = shallowRef('')
@@ -64,6 +76,44 @@ export function useAccountsQuery() {
void query.execute()
}
function replaceAccount(updated: AccountRow) {
const index = query.items.value.findIndex(account => account.id === updated.id)
if (index < 0)
return false
const current = query.items.value[index]
if (!current)
return false
updateAccountSummary(current.status, updated.status)
if (statusQuery.value && statusQuery.value !== updated.status) {
query.items.value = query.items.value.filter(account => account.id !== updated.id)
query.total.value = Math.max(0, query.total.value - 1)
return false
}
const accounts = [...query.items.value]
accounts[index] = updated
query.items.value = accounts
return true
}
function updateAccountSummary(
previousStatus: AccountRow['status'],
nextStatus: AccountRow['status'],
) {
if (previousStatus === nextStatus)
return
const previousKey = summaryKeyByStatus[previousStatus]
const nextKey = summaryKeyByStatus[nextStatus]
const summary = { ...accountSummary.value }
summary[previousKey] = Math.max(0, summary[previousKey] - 1)
summary[nextKey] += 1
accountSummary.value = summary
}
watchDebounced(
searchQuery,
() => {
@@ -95,6 +145,7 @@ export function useAccountsQuery() {
sort,
accountSummary,
accountPagination,
replaceAccount,
handlePageChange,
handlePageSizeChange,
handleSortChange,
+10 -10
View File
@@ -1,4 +1,4 @@
import type { getAccounts } from '@/api'
import type { AccountErrorReason, AccountStatus, getAccounts } from '@/api'
import type { BaseTableColumn } from '@/components/base/BaseTable/columns'
import { clamp } from 'es-toolkit'
import { providerDisplayName } from '@/utils/providers'
@@ -118,7 +118,7 @@ export const accountColumns = [
},
] satisfies BaseTableColumn<AccountRow>[]
export const statusLabels: Record<string, string> = {
export const statusLabels: Record<AccountStatus, string> = {
normal: '正常',
quota_exhausted: '配额耗尽',
rate_limited: '限流中',
@@ -126,7 +126,7 @@ export const statusLabels: Record<string, string> = {
error: '错误',
}
export const statusTones: Record<string, 'success' | 'danger' | 'warning' | 'info' | 'normal'> = {
export const statusTones: Record<AccountStatus, 'success' | 'danger' | 'warning' | 'info' | 'normal'> = {
normal: 'success',
quota_exhausted: 'warning',
rate_limited: 'warning',
@@ -144,19 +144,19 @@ export const accountStatusFilterOptions = [
]
/** `error` 分类下具体原因的展示文案(对应后端 `errorReason`)。 */
export const errorReasonLabels: Record<string, string> = {
expired: '账号已过期',
token_expired: '访问令牌已过期',
invalid: '凭据已失效',
banned: '账号已封禁',
unknown: '状态未知',
export const errorReasonLabels: Record<AccountErrorReason, string> = {
account_unverified: '账号身份尚未确认',
access_token_expired: 'Access Token 已过期',
credential_expired: '凭据已过期',
credential_invalid: '凭据无效',
account_banned: '账号不可用或已被封禁',
}
/**
* 后端已派生互斥状态(正常 / 配额耗尽 / 限流中 / 已停用 / 错误);
* 前端只渲染,不再独立派生。
*/
export function derivedAccountStatus(row: AccountRow): string {
export function derivedAccountStatus(row: AccountRow): AccountStatus {
return row.status
}
+3 -9
View File
@@ -27,7 +27,6 @@ import { accountColumns, derivedAccountStatus } from './constants'
const selectedIds = ref<Set<string>>(new Set())
const {
totalAccounts,
loading,
accounts,
loadAccounts,
@@ -37,6 +36,7 @@ const {
sort,
accountSummary,
accountPagination,
replaceAccount,
handlePageChange,
handlePageSizeChange,
handleSortChange,
@@ -67,17 +67,13 @@ const {
handleExportAccounts,
handleRefresh,
handleRefreshQuota,
patchAccountStatus,
handleToggleSchedule,
scheduleActionLabel,
} = useAccountMutations({
accounts,
accountSummary,
statusQuery,
sort,
selectedIds,
totalAccounts,
reload: loadAccounts,
replaceAccount,
})
const {
@@ -99,9 +95,7 @@ const {
openConnectionTest,
handleRefreshConnectionTestModels,
handleTestConnection,
} = useAccountConnectionTest({
onAccountStatus: patchAccountStatus,
})
} = useAccountConnectionTest({ reload: loadAccounts })
const {
expandedAccountIds,
@@ -1,6 +1,6 @@
<script setup lang="ts">
import type { dashboardSnapshotView, MetricTone } from '../composables/presenter'
import { CircleCheck, RefreshCw, ShieldAlert, TriangleAlert } from '@lucide/vue'
import { CircleCheck, CircleOff, RefreshCw, ShieldAlert, TriangleAlert } from '@lucide/vue'
import { clamp } from 'es-toolkit'
import { computed } from 'vue'
@@ -82,17 +82,19 @@ const accountStatusCounts = computed(() => {
const p = props.pool
if (!p)
return null
const active = p.active ?? 0
const refreshing = p.refreshing ?? 0
const quotaLimited = (p.rateLimited ?? 0) + (p.quotaExhausted ?? 0)
const pending = (p.expired ?? 0) + (p.invalid ?? 0) + (p.disabled ?? 0) + (p.banned ?? 0)
const normal = p.normal ?? 0
const quotaExhausted = p.quotaExhausted ?? 0
const rateLimited = p.rateLimited ?? 0
const disabled = p.disabled ?? 0
const error = p.error ?? 0
return {
total: p.total,
active,
refreshing,
quotaLimited,
pending,
unavailable: quotaLimited + pending,
normal,
quotaExhausted,
rateLimited,
disabled,
error,
unavailable: quotaExhausted + rateLimited + disabled + error,
}
})
@@ -101,54 +103,58 @@ const statusRows = computed(() => {
const counts = accountStatusCounts.value
if (!p || !counts) {
return [
{ label: '活跃账号', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
{ label: '刷新中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
{ label: '额度受限', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
{ label: '待处理', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
{ label: '正常', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
{ label: '配额耗尽', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
{ label: '限流中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
{ label: '已停用', description: '暂无账号池观测', value: '—', tone: 'normal', icon: CircleOff },
{ label: '错误', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
]
}
const expired = p?.expired ?? 0
const invalid = p?.invalid ?? 0
const disabled = p?.disabled ?? 0
const banned = p?.banned ?? 0
return [
{
label: '活跃账号',
description: '可直接参与调度',
value: String(counts.active),
label: '正常',
description: '账号当前可用',
value: String(counts.normal),
tone: 'success',
icon: CircleCheck,
},
{
label: '刷新中',
description: '令牌自动刷新中',
value: String(counts.refreshing),
tone: 'normal',
icon: RefreshCw,
},
{
label: '额度受限',
description: '配额耗尽与限流中',
value: String(counts.quotaLimited),
label: '配额耗尽',
description: '账号额度已耗尽',
value: String(counts.quotaExhausted),
tone: 'warning',
icon: TriangleAlert,
},
{
label: '待处理',
description: `过期 ${expired} · 失效 ${invalid} · 禁用 ${disabled} · 封禁 ${banned}`,
value: String(counts.pending),
label: '限流中',
description: '账号请求暂时受限',
value: String(counts.rateLimited),
tone: 'normal',
icon: RefreshCw,
},
{
label: '已停用',
description: '账号已停用',
value: String(counts.disabled),
tone: 'normal',
icon: CircleOff,
},
{
label: '错误',
description: '账号状态异常',
value: String(counts.error),
tone: 'danger',
icon: ShieldAlert,
},
]
})
const availabilityRate = computed(() => {
const normalRate = computed(() => {
const p = props.pool
if (!p || p.total === 0)
return '—'
return `${((p.active / p.total) * 100).toFixed(1)}%`
return `${((p.normal / p.total) * 100).toFixed(1)}%`
})
const statusBars = computed(() => {
@@ -156,10 +162,11 @@ const statusBars = computed(() => {
if (!counts || counts.total === 0)
return []
return [
{ pct: (counts.active / counts.total) * 100, cls: 'bg-(--cp-success)' },
{ pct: (counts.refreshing / counts.total) * 100, cls: 'bg-(--cp-normal)' },
{ pct: (counts.quotaLimited / counts.total) * 100, cls: 'bg-(--cp-warning)' },
{ pct: (counts.pending / counts.total) * 100, cls: 'bg-(--cp-danger)' },
{ pct: (counts.normal / counts.total) * 100, cls: 'bg-(--cp-success)' },
{ pct: (counts.quotaExhausted / counts.total) * 100, cls: 'bg-(--cp-warning)' },
{ pct: (counts.rateLimited / counts.total) * 100, cls: 'bg-(--cp-normal)' },
{ pct: (counts.disabled / counts.total) * 100, cls: 'bg-(--cp-text-secondary)' },
{ pct: (counts.error / counts.total) * 100, cls: 'bg-(--cp-danger)' },
].filter(b => b.pct > 0)
})
</script>
@@ -305,8 +312,8 @@ const statusBars = computed(() => {
<div class="grid justify-items-end">
<strong
class="font-mono text-2xl leading-[1.05] font-heavy tabular-nums text-(--cp-success-text)"
>{{ availabilityRate }}</strong>
<span class="mt-0.5 text-xs leading-[1.15] font-bold text-(--cp-text-secondary)">可用率</span>
>{{ normalRate }}</strong>
<span class="mt-0.5 text-xs leading-[1.15] font-bold text-(--cp-text-secondary)">正常率</span>
</div>
</header>
@@ -326,11 +333,11 @@ const statusBars = computed(() => {
</div>
</div>
<div class="mt-6.5 grid h-65.5 w-full gap-2.5">
<div class="mt-5 grid h-67 w-full grid-rows-5 gap-2">
<div
v-for="row in statusRows"
:key="row.label"
class="grid h-14.5 grid-cols-[28px_14px_minmax(0,1fr)_76px] items-center rounded-[14px] bg-(--cp-bg-subtle) px-3.5"
class="grid min-h-0 grid-cols-[28px_14px_minmax(0,1fr)_76px] items-center rounded-[14px] bg-(--cp-bg-subtle) px-3.5"
>
<span
class="inline-flex size-7 items-center justify-center rounded-[9px]"