mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-15 08:06:21 +00:00
refactor(accounts): unify five-state account status
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -255,7 +255,6 @@ pub async fn initialize(
|
||||
|
||||
let accounts = Arc::new(DefaultAccountsService::new(
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
use std::pin::Pin;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -9,151 +9,10 @@ use gateway_core::routing::ProviderKind;
|
||||
|
||||
use super::{PageSize, Revision, observability::TimeRange};
|
||||
|
||||
/// Provider 账号当前可用性;权威定义与字符串编解码在 gateway-core,此处仅复用。
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
///
|
||||
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
|
||||
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
/// 正常,可调度。
|
||||
Normal,
|
||||
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
|
||||
QuotaExhausted,
|
||||
/// 限流中(quota 快照级信号),自动恢复,无需干预。
|
||||
RateLimited,
|
||||
/// 管理员手动停用(`enabled = false`)。
|
||||
Disabled,
|
||||
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
|
||||
#[must_use]
|
||||
pub const fn derive(self) -> AccountStatus {
|
||||
if !self.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if self.access_token_expired
|
||||
|| matches!(
|
||||
self.availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
|
||||
///
|
||||
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountErrorReason {
|
||||
Expired,
|
||||
TokenExpired,
|
||||
Invalid,
|
||||
Banned,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Expired => "expired",
|
||||
Self::TokenExpired => "token_expired",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Banned => "banned",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"expired" => Some(Self::Expired),
|
||||
"token_expired" => Some(Self::TokenExpired),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
"banned" => Some(Self::Banned),
|
||||
"unknown" => Some(Self::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
match signals.availability {
|
||||
AccountAvailability::QuotaExhausted => None,
|
||||
AccountAvailability::Ready => signals
|
||||
.access_token_expired
|
||||
.then_some(AccountErrorReason::TokenExpired),
|
||||
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
|
||||
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
|
||||
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
|
||||
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
|
||||
}
|
||||
}
|
||||
pub use gateway_core::engine::credential::{
|
||||
AccountErrorReason, AccountStatus, AccountStatusFacts, AccountStatusProjection,
|
||||
CredentialState, QuotaAccessState, QuotaEvidence, QuotaState, resolve_account_status,
|
||||
};
|
||||
|
||||
/// 账号列表排序字段。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -207,11 +66,11 @@ pub struct AccountRecord {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub quota: QuotaState,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -296,11 +155,18 @@ pub struct AccountUsageWindowResult {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountPage {
|
||||
pub config_revision: Revision,
|
||||
pub items: Vec<AccountRecord>,
|
||||
pub items: Vec<AccountPageItem>,
|
||||
pub total: u64,
|
||||
pub summary: AccountSummary,
|
||||
}
|
||||
|
||||
/// 同一状态快照下的账号事实与唯一状态投影。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountPageItem {
|
||||
pub account: AccountRecord,
|
||||
pub projection: AccountStatusProjection,
|
||||
}
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
@@ -342,9 +208,7 @@ pub enum AccountConnectionTestEvent {
|
||||
Content {
|
||||
text: String,
|
||||
},
|
||||
Completed {
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
Completed,
|
||||
Failed {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
@@ -352,7 +216,6 @@ pub enum AccountConnectionTestEvent {
|
||||
upstream_status: Option<u16>,
|
||||
upstream_content_type: Option<String>,
|
||||
upstream_body: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -439,25 +439,15 @@ pub struct RequestMetricPoint {
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
}
|
||||
|
||||
/// 账号池容量统计。
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
|
||||
/// 以便状态详情展示和诊断。
|
||||
/// 账号池的统一五态统计。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 中一个账号的模型级用量事实。
|
||||
@@ -488,7 +478,7 @@ pub struct DashboardAccountRequestBucket {
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 中一个账号的完整用量与公共状态事实。
|
||||
/// Dashboard 中一个账号的完整用量事实。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardAccountUsage {
|
||||
pub account_id: String,
|
||||
@@ -497,8 +487,6 @@ pub struct DashboardAccountUsage {
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub enabled: bool,
|
||||
pub availability: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
|
||||
@@ -11,10 +11,7 @@ use gateway_core::{
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{
|
||||
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
|
||||
AccountUsage,
|
||||
},
|
||||
accounts::{AccountRecord, AccountSummary, AccountUsage, CredentialState},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -55,24 +52,24 @@ pub struct CredentialCursor {
|
||||
/// Provider credential 列表查询。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CredentialListQuery {
|
||||
pub availability: Option<CredentialAvailabilityFilter>,
|
||||
pub credential_state: Option<CredentialStateFilter>,
|
||||
pub enabled: Option<bool>,
|
||||
pub window: CredentialListWindow,
|
||||
}
|
||||
|
||||
/// Credential 可用性筛选;支持一个 wire 值对应多个持久状态。
|
||||
/// Credential 状态筛选。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CredentialAvailabilityFilter {
|
||||
Exact(AccountAvailability),
|
||||
AnyOf(Vec<AccountAvailability>),
|
||||
pub enum CredentialStateFilter {
|
||||
Exact(CredentialState),
|
||||
AnyOf(Vec<CredentialState>),
|
||||
}
|
||||
|
||||
impl CredentialAvailabilityFilter {
|
||||
impl CredentialStateFilter {
|
||||
#[must_use]
|
||||
pub fn matches(&self, availability: AccountAvailability) -> bool {
|
||||
pub fn matches(&self, credential_state: CredentialState) -> bool {
|
||||
match self {
|
||||
Self::Exact(expected) => *expected == availability,
|
||||
Self::AnyOf(expected) => expected.contains(&availability),
|
||||
Self::Exact(expected) => *expected == credential_state,
|
||||
Self::AnyOf(expected) => expected.contains(&credential_state),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -155,8 +152,8 @@ pub struct PreparedCredentialCreate {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Provider 对一份导入文档的完整验证结果。
|
||||
@@ -603,15 +600,31 @@ pub struct ProviderQuota {
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
/// 展示用快照级触顶事实(顶层或任一窗口触顶);不参与账号五态派生。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
impl ProviderQuota {
|
||||
/// 返回账号展开面板使用的当前额度窗口用量。
|
||||
///
|
||||
/// 只选择具备完整时间边界且可按账号归属的窗口;优先级与 Dashboard
|
||||
/// 代表性额度窗口一致,同一优先级保持 Provider 投影顺序。
|
||||
#[must_use]
|
||||
pub fn representative_window_usage(&self) -> Option<&AccountUsage> {
|
||||
self.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, window)| {
|
||||
window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
&& window.window_seconds.is_some()
|
||||
&& window.reset_at.is_some()
|
||||
&& window.local_usage.is_some()
|
||||
})
|
||||
.min_by_key(|(index, window)| (quota_usage_priority(window), *index))
|
||||
.and_then(|(_, window)| window.local_usage.as_ref())
|
||||
}
|
||||
|
||||
/// 返回 Dashboard 使用的代表性额度比例。
|
||||
///
|
||||
/// 优先使用短周期窗口,再依次使用周、月和其它窗口;同一优先级取较高的已用比例。
|
||||
@@ -703,9 +716,7 @@ impl fmt::Debug for ProviderExport {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
/// `status == Error` 时的具体原因;其余状态为 `None`。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub projection: gateway_core::engine::credential::AccountStatusProjection,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use super::backup::BackupStorePorts;
|
||||
use crate::model::{
|
||||
MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountListQuery, AccountPage, AccountPageItem, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
},
|
||||
auth::{AdminAuditEvent, AdminSession},
|
||||
@@ -83,7 +83,7 @@ pub type AdminStoreResult<T> = Result<T, AdminStoreError>;
|
||||
pub trait AccountStore: Send + Sync {
|
||||
async fn list_accounts(&self, query: AccountListQuery) -> AdminStoreResult<AccountPage>;
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>>;
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>>;
|
||||
|
||||
async fn load_account_usage(
|
||||
&self,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -18,8 +18,7 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
|
||||
derive_error_reason,
|
||||
AccountPageItem, AccountUsageWindowQuery,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -28,10 +27,7 @@ use crate::{
|
||||
ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
store::{AccountStore, SettingsStore},
|
||||
},
|
||||
ports::{provider::ProviderAdminRegistry, store::AccountStore},
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -79,7 +75,6 @@ pub trait AccountsService: Send + Sync {
|
||||
|
||||
pub(crate) struct DefaultAccountsService {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
@@ -89,14 +84,12 @@ impl DefaultAccountsService {
|
||||
#[must_use]
|
||||
pub(crate) const fn new(
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
settings,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
@@ -106,7 +99,7 @@ impl DefaultAccountsService {
|
||||
async fn load_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<AccountRecord, AdminError> {
|
||||
) -> Result<AccountPageItem, AdminError> {
|
||||
self.accounts
|
||||
.load_account(account_id.as_str())
|
||||
.await
|
||||
@@ -119,32 +112,32 @@ impl DefaultAccountsService {
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<
|
||||
(
|
||||
AccountRecord,
|
||||
AccountPageItem,
|
||||
Arc<dyn crate::ports::provider::ProviderAdmin>,
|
||||
),
|
||||
AdminError,
|
||||
> {
|
||||
let account = self.load_account(account_id).await?;
|
||||
let item = self.load_account(account_id).await?;
|
||||
let provider = self
|
||||
.providers
|
||||
.require(&account.provider_kind)
|
||||
.require(&item.account.provider_kind)
|
||||
.map_err(|error| map_provider_error(error, "provider account"))?;
|
||||
Ok((account, provider))
|
||||
Ok((item, provider))
|
||||
}
|
||||
|
||||
async fn attach_quota_local_usage(
|
||||
&self,
|
||||
accounts: &[AccountRecord],
|
||||
accounts: &[AccountPageItem],
|
||||
quotas: &mut [ProviderQuota],
|
||||
) -> Result<(), AdminError> {
|
||||
let windows = accounts
|
||||
.iter()
|
||||
.zip(quotas.iter())
|
||||
.flat_map(|(account, quota)| {
|
||||
.flat_map(|(item, quota)| {
|
||||
quota
|
||||
.windows
|
||||
.iter()
|
||||
.filter_map(|window| quota_usage_window(&account.id, window))
|
||||
.filter_map(|window| quota_usage_window(&item.account.id, window))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if windows.is_empty() {
|
||||
@@ -158,12 +151,12 @@ impl DefaultAccountsService {
|
||||
.into_iter()
|
||||
.map(|result| ((result.account_id, result.key), result.usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (account, quota) in accounts.iter().zip(quotas) {
|
||||
for (item, quota) in accounts.iter().zip(quotas) {
|
||||
for window in &mut quota.windows {
|
||||
if window.local_usage.is_none()
|
||||
&& window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
let key = (account.id.clone(), window.key.clone());
|
||||
let key = (item.account.id.clone(), window.key.clone());
|
||||
if let Some(usage) = usage_by_window.get(&key) {
|
||||
window.local_usage = Some(usage.clone());
|
||||
}
|
||||
@@ -178,33 +171,19 @@ impl DefaultAccountsService {
|
||||
account_id: &ProviderAccountId,
|
||||
refresh_quota: bool,
|
||||
) -> Result<AccountDirectoryItem, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(account_id).await?;
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "runtime settings"))?;
|
||||
let (stored, provider) = self.provider_for_account(account_id).await?;
|
||||
let account = &stored.account;
|
||||
let now = Utc::now();
|
||||
let retained_range = retained_usage_range(now, settings.usage_retention_days);
|
||||
let rolling_range = TimeRange {
|
||||
start: now - Duration::hours(24),
|
||||
end: now,
|
||||
};
|
||||
let ids = vec![account.id.clone()];
|
||||
let (usage, rolling_usage) = futures::try_join!(
|
||||
async {
|
||||
self.accounts
|
||||
.load_account_usage(retained_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account usage"))
|
||||
},
|
||||
async {
|
||||
self.accounts
|
||||
.load_account_usage(rolling_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "rolling account usage"))
|
||||
},
|
||||
)?;
|
||||
let rolling_usage = self
|
||||
.accounts
|
||||
.load_account_usage(rolling_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "rolling account usage"))?;
|
||||
let rolling_usage = rolling_usage.into_iter().next();
|
||||
let mut quota = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
@@ -215,17 +194,15 @@ impl DefaultAccountsService {
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
self.attach_quota_local_usage(
|
||||
std::slice::from_ref(&account),
|
||||
std::slice::from_ref(&stored),
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
let usage = quota.representative_window_usage().cloned();
|
||||
Ok(AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
projection: stored.projection,
|
||||
usage,
|
||||
account: stored.account,
|
||||
quota,
|
||||
})
|
||||
}
|
||||
@@ -234,13 +211,12 @@ impl DefaultAccountsService {
|
||||
#[async_trait]
|
||||
impl AccountsService for DefaultAccountsService {
|
||||
async fn list(&self, query: AccountListQuery) -> Result<AccountDirectoryPage, AdminError> {
|
||||
let (page, settings) = futures::try_join!(
|
||||
self.accounts.list_accounts(query),
|
||||
self.settings.load_runtime_settings(),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "account directory"))?;
|
||||
let page = self
|
||||
.accounts
|
||||
.list_accounts(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account directory"))?;
|
||||
let now = Utc::now();
|
||||
let retained_range = retained_usage_range(now, settings.usage_retention_days);
|
||||
let rolling_range = TimeRange {
|
||||
start: now - Duration::hours(24),
|
||||
end: now,
|
||||
@@ -248,22 +224,19 @@ impl AccountsService for DefaultAccountsService {
|
||||
let ids = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|account| account.id.clone())
|
||||
.map(|item| item.account.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let (usage, rolling_usage) = futures::try_join!(
|
||||
self.accounts.load_account_usage(retained_range, &ids),
|
||||
self.accounts.load_account_usage(rolling_range, &ids),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "account usage"))?;
|
||||
let usage = usage
|
||||
.into_iter()
|
||||
.map(|usage| (usage.account_id.clone(), usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let rolling_usage = self
|
||||
.accounts
|
||||
.load_account_usage(rolling_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "rolling account usage"))?;
|
||||
let rolling_usage = rolling_usage
|
||||
.into_iter()
|
||||
.map(|usage| (usage.account_id.clone(), usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|item| async {
|
||||
let account = &item.account;
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
@@ -307,14 +280,12 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
.map(|(item, quota)| {
|
||||
let usage = quota.representative_window_usage().cloned();
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
usage,
|
||||
account: item.account,
|
||||
projection: item.projection,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
@@ -342,7 +313,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
for account_id in account_ids {
|
||||
let account = self.load_account(&account_id).await?;
|
||||
grouped
|
||||
.entry(account.provider_kind)
|
||||
.entry(account.account.provider_kind)
|
||||
.or_default()
|
||||
.push(account_id);
|
||||
}
|
||||
@@ -385,7 +356,8 @@ impl AccountsService for DefaultAccountsService {
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let (stored, provider) = self.provider_for_account(&account_id).await?;
|
||||
let account = stored.account;
|
||||
let prepared = provider
|
||||
.prepare_refresh(PrepareCredentialRefresh {
|
||||
account: account.clone(),
|
||||
@@ -436,8 +408,8 @@ impl AccountsService for DefaultAccountsService {
|
||||
account_id: ProviderAccountId,
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let initial_status = account_status(&account, false, Utc::now());
|
||||
let (stored, provider) = self.provider_for_account(&account_id).await?;
|
||||
let account = stored.account;
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
|
||||
@@ -454,8 +426,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
},
|
||||
];
|
||||
let probe = Arc::clone(&self.probe);
|
||||
let accounts = Arc::clone(&self.accounts);
|
||||
let stored_account_id = account.id.clone();
|
||||
let terminal = futures::stream::once(async move {
|
||||
let result = probe
|
||||
.probe(AccountProbeRequest {
|
||||
@@ -465,22 +435,12 @@ impl AccountsService for DefaultAccountsService {
|
||||
operation,
|
||||
})
|
||||
.await;
|
||||
let status = accounts
|
||||
.load_account(&stored_account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_or(initial_status, |account| {
|
||||
account_status(&account, false, Utc::now())
|
||||
});
|
||||
match result {
|
||||
Ok(result) => result
|
||||
.text
|
||||
.into_iter()
|
||||
.map(|text| AccountConnectionTestEvent::Content { text })
|
||||
.chain(std::iter::once(AccountConnectionTestEvent::Completed {
|
||||
account_status: status,
|
||||
}))
|
||||
.chain(std::iter::once(AccountConnectionTestEvent::Completed))
|
||||
.collect(),
|
||||
Err(error) => {
|
||||
let upstream_status = error
|
||||
@@ -506,7 +466,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status: status,
|
||||
}]
|
||||
}
|
||||
}
|
||||
@@ -523,49 +482,10 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号运营分类(互斥展示状态)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
|
||||
TimeRange {
|
||||
start: now - Duration::days(i64::from(retention_days)),
|
||||
end: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_usage_window(
|
||||
account_id: &str,
|
||||
window: &ProviderQuotaWindow,
|
||||
|
||||
@@ -127,7 +127,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let max_concurrent_per_account = u64::from(settings.max_concurrent_per_account);
|
||||
let active_accounts = runtime_slots
|
||||
.as_ref()
|
||||
.map_or(observation.provider_accounts.active, |slots| {
|
||||
.map_or(observation.provider_accounts.normal, |slots| {
|
||||
slots.active_accounts
|
||||
});
|
||||
let total_slots = active_accounts.saturating_mul(max_concurrent_per_account);
|
||||
@@ -484,8 +484,7 @@ impl DefaultObservabilityService {
|
||||
});
|
||||
let quota_used = futures::future::join_all(quota_reads).await;
|
||||
for (account, used_percent) in accounts.iter_mut().zip(quota_used) {
|
||||
account.quota_used_percent = used_percent
|
||||
.or_else(|| (account.availability == "quota_exhausted").then_some(100.0));
|
||||
account.quota_used_percent = used_percent;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -129,7 +129,6 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
|
||||
@@ -5,7 +5,10 @@ use chrono::{TimeDelta, Utc};
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
credential::{
|
||||
AccountStatusFacts, CredentialState, OpaqueProviderData, ProviderAccountId,
|
||||
QuotaEvidence, QuotaState, resolve_account_status,
|
||||
},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
@@ -18,7 +21,7 @@ use gateway_admin::{
|
||||
model::{
|
||||
AdminError, MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
|
||||
AccountConnectionTestEvent, AccountListQuery, AccountPage, AccountPageItem,
|
||||
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
},
|
||||
@@ -378,6 +381,7 @@ pub(super) struct FakeAccountStore {
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
@@ -393,6 +397,7 @@ impl FakeAccountStore {
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
quota_window_queries: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -408,6 +413,13 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
fn quota_window_queries(&self) -> Vec<AccountUsageWindowQuery> {
|
||||
self.quota_window_queries
|
||||
.lock()
|
||||
.expect("quota window queries")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
|
||||
*self.accounts.lock().expect("accounts") = accounts;
|
||||
}
|
||||
@@ -438,6 +450,22 @@ impl FakeAccountStore {
|
||||
.expect("audit requests")
|
||||
.push(context.request_id.clone());
|
||||
}
|
||||
|
||||
fn page_item(account: AccountRecord) -> AccountPageItem {
|
||||
let facts = AccountStatusFacts {
|
||||
enabled: account.enabled,
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
};
|
||||
AccountPageItem {
|
||||
account,
|
||||
projection: resolve_account_status(&facts, std::time::SystemTime::now()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -445,12 +473,13 @@ impl AccountStore for FakeAccountStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
self.record("store.list_accounts");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
let total = accounts.len() as u64;
|
||||
Ok(AccountPage {
|
||||
config_revision: revision(1),
|
||||
items: accounts.clone(),
|
||||
total: accounts.len() as u64,
|
||||
items: accounts.into_iter().map(Self::page_item).collect(),
|
||||
total,
|
||||
summary: AccountSummary {
|
||||
total: accounts.len() as u64,
|
||||
total,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
rate_limited: 0,
|
||||
@@ -460,7 +489,7 @@ impl AccountStore for FakeAccountStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>> {
|
||||
self.record("store.load_account");
|
||||
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
|
||||
let account = self
|
||||
@@ -477,7 +506,7 @@ impl AccountStore for FakeAccountStore {
|
||||
.find(|account| account.id == account_id)
|
||||
.cloned()
|
||||
});
|
||||
Ok(account)
|
||||
Ok(account.map(Self::page_item))
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -490,8 +519,12 @@ impl AccountStore for FakeAccountStore {
|
||||
|
||||
async fn load_account_usage_by_windows(
|
||||
&self,
|
||||
_: &[AccountUsageWindowQuery],
|
||||
windows: &[AccountUsageWindowQuery],
|
||||
) -> AdminStoreResult<Vec<AccountUsageWindowResult>> {
|
||||
*self
|
||||
.quota_window_queries
|
||||
.lock()
|
||||
.expect("quota window queries") = windows.to_vec();
|
||||
Ok(self
|
||||
.quota_window_usage
|
||||
.lock()
|
||||
@@ -815,7 +848,11 @@ fn provider_registry_should_reject_duplicate_kind() {
|
||||
async fn connection_test_should_probe_unavailable_account() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.quota = QuotaState::exhausted(
|
||||
QuotaEvidence::UsageLimitReached,
|
||||
std::time::SystemTime::now(),
|
||||
None,
|
||||
);
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
@@ -837,7 +874,6 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
message,
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
@@ -846,7 +882,7 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
async fn connection_test_rate_limited_probe_returns_provider_failure() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
@@ -869,9 +905,10 @@ async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
message,
|
||||
..
|
||||
})
|
||||
if message == "xAI free model quota is exhausted"
|
||||
));
|
||||
}
|
||||
|
||||
@@ -897,10 +934,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
|
||||
..
|
||||
})
|
||||
Some(AccountConnectionTestEvent::Failed { .. })
|
||||
));
|
||||
}
|
||||
|
||||
@@ -949,7 +983,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.config_revision, revision(2));
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
result.account.projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
@@ -990,7 +1024,7 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
account.projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
@@ -1016,7 +1050,6 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
@@ -1064,10 +1097,14 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
|
||||
async fn accounts_list_should_prefer_credential_error_over_quota_exhaustion() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.quota = QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
std::time::SystemTime::now(),
|
||||
None,
|
||||
);
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
@@ -1086,17 +1123,17 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
|
||||
.expect("usage-limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
async fn accounts_list_should_map_unknown_credential_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
account.credential_state = CredentialState::Unknown;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
@@ -1114,15 +1151,13 @@ async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
.expect("unknown account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
async fn accounts_list_should_not_derive_rate_limited_from_provider_quota_view() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
@@ -1141,7 +1176,6 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1158,17 +1192,16 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("rate limited account directory");
|
||||
.expect("account directory with display quota cooldown");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::RateLimited,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
async fn accounts_list_should_not_derive_exhaustion_from_provider_quota_view() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
@@ -1187,7 +1220,6 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1204,11 +1236,11 @@ async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
.expect("account directory with display quota limit");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1233,7 +1265,6 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1243,7 +1274,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
usage: quota_local_usage("acct_test", 4_330_000),
|
||||
}]);
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
let page = accounts_service(provider, store.clone())
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
@@ -1257,11 +1288,23 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
.await
|
||||
.expect("quota window usage");
|
||||
|
||||
let usage = page.items[0].quota.windows[0]
|
||||
let item = &page.items[0];
|
||||
let usage = item.quota.windows[0]
|
||||
.local_usage
|
||||
.as_ref()
|
||||
.expect("quota window local usage");
|
||||
assert_eq!(usage.total_tokens, Some(4_330_000));
|
||||
assert_eq!(
|
||||
item.usage.as_ref().and_then(|usage| usage.total_tokens),
|
||||
Some(4_330_000),
|
||||
);
|
||||
|
||||
let queries = store.quota_window_queries();
|
||||
assert_eq!(queries.len(), 1);
|
||||
assert_eq!(queries[0].account_id, "acct_test");
|
||||
assert_eq!(queries[0].key, "primary");
|
||||
assert_eq!(queries[0].range.start, reset_at - TimeDelta::hours(5));
|
||||
assert_eq!(queries[0].range.end, reset_at);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1300,7 +1343,6 @@ async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_w
|
||||
},
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1352,6 +1394,13 @@ async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_w
|
||||
("codex-bengalfox-primary", None),
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
page.items[0]
|
||||
.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.total_tokens),
|
||||
Some(12_818_806),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1432,7 +1481,6 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -1476,10 +1524,11 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
access_token_expires_at: Some(now + TimeDelta::hours(1)),
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
credential_state: CredentialState::Ready,
|
||||
credential_observed_at: now,
|
||||
quota: QuotaState::allowed(now.into()),
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
@@ -1513,8 +1562,8 @@ fn prepared_create_with_id(
|
||||
access_token_expires_at: Some(now + TimeDelta::hours(1)),
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
credential_state: CredentialState::Ready,
|
||||
credential_observed_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1597,9 +1646,7 @@ impl AccountProbe for FreeModelQuotaProbe {
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Ready;
|
||||
self.store.set_account_after_probe(account);
|
||||
self.store.set_account_after_probe(account_record("xai"));
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"xAI free model quota is exhausted",
|
||||
|
||||
@@ -20,7 +20,7 @@ use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountListQuery, AccountPage, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
},
|
||||
auth::{AdminAuditEvent, AdminSession},
|
||||
@@ -240,7 +240,10 @@ impl AccountStore for UnavailableStore {
|
||||
Err(unavailable("accounts"))
|
||||
}
|
||||
|
||||
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
async fn load_account(
|
||||
&self,
|
||||
_: &str,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountPageItem>> {
|
||||
Err(unavailable("account"))
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ use gateway_core::{
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError,
|
||||
@@ -185,12 +185,10 @@ pub struct AccountView {
|
||||
pub authentication_kind: String,
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
pub access_token_expires_at_display: Option<String>,
|
||||
@@ -522,10 +520,9 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Content { text } => {
|
||||
serde_json::json!({ "type": "content", "text": text })
|
||||
}
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
DomainConnectionTestEvent::Completed => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status.as_str()
|
||||
"success": true
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
@@ -534,7 +531,6 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
@@ -542,8 +538,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
"providerErrorType": provider_error_type,
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status.as_str()
|
||||
"upstreamBody": upstream_body
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -1365,16 +1360,16 @@ fn account_models_data(result: ProviderModels) -> AccountModelsData {
|
||||
fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
error_reason,
|
||||
projection,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = status.as_str().to_owned();
|
||||
let status = projection.status.as_str().to_owned();
|
||||
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, now);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
|
||||
AccountView {
|
||||
id: account.id.clone(),
|
||||
name: account.name,
|
||||
@@ -1387,11 +1382,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
plan_type: account.plan_type,
|
||||
authentication_kind: account.authentication_kind,
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: account.last_error_message,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
status,
|
||||
error_reason: projection
|
||||
.error_reason
|
||||
.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: projection.error_message,
|
||||
enabled: account.enabled,
|
||||
access_token_expires_at: expires_at,
|
||||
access_token_expires_at_display: account
|
||||
@@ -1411,6 +1406,7 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
|
||||
fn account_quota_view(
|
||||
quota: ProviderQuota,
|
||||
rate_limited_until: Option<DateTime<Utc>>,
|
||||
now: DateTime<Utc>,
|
||||
) -> (AccountQuotaView, Option<String>) {
|
||||
let refresh_token_expires_at = quota
|
||||
@@ -1420,7 +1416,7 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
@@ -1590,7 +1586,7 @@ fn account_model_usage_view(usage: AccountModelUsage, now: DateTime<Utc>) -> Mod
|
||||
billing_amount_usd: usd.map(|cost| cost.amount.as_str().to_owned()),
|
||||
billing_amount_usd_display: usd.map_or_else(
|
||||
|| "—".to_owned(),
|
||||
|cost| format!("${}", cost.amount.as_str()),
|
||||
|cost| format_decimal_currency(cost.amount.as_str(), "USD"),
|
||||
),
|
||||
cost_estimate_status: cost_estimate_status.to_owned(),
|
||||
known_cost_count: known_count,
|
||||
|
||||
@@ -15,7 +15,7 @@ use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
|
||||
@@ -127,20 +127,6 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn format_decimal_currency(amount: &str, currency: &str) -> String {
|
||||
if currency == "USD" {
|
||||
let value = amount.parse::<f64>().map_or(0.0, |value| value);
|
||||
let precision = if value != 0.0 && value.abs() < 0.01 {
|
||||
4
|
||||
} else {
|
||||
2
|
||||
};
|
||||
format!("${value:.precision$}")
|
||||
} else {
|
||||
format!("{currency} {amount}")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn format_money(cost: &domain::CurrencyCost) -> String {
|
||||
format_decimal_currency(cost.amount.as_str(), &cost.currency)
|
||||
}
|
||||
@@ -747,10 +733,16 @@ pub(crate) fn dashboard_view(
|
||||
credentials: DashboardCredentialsCardView {
|
||||
total: format_compact_number(provider_accounts.total),
|
||||
total_value: provider_accounts.total,
|
||||
available: format_compact_number(provider_accounts.active),
|
||||
available_value: provider_accounts.active,
|
||||
unavailable: format_compact_number(provider_accounts.unavailable),
|
||||
unavailable_value: provider_accounts.unavailable,
|
||||
available: format_compact_number(provider_accounts.normal),
|
||||
available_value: provider_accounts.normal,
|
||||
unavailable: format_compact_number(
|
||||
provider_accounts
|
||||
.total
|
||||
.saturating_sub(provider_accounts.normal),
|
||||
),
|
||||
unavailable_value: provider_accounts
|
||||
.total
|
||||
.saturating_sub(provider_accounts.normal),
|
||||
},
|
||||
traffic: DashboardTrafficCardView {
|
||||
today_requests: format_compact_number(today.request_count),
|
||||
@@ -783,14 +775,11 @@ pub(crate) fn dashboard_view(
|
||||
usage_records: recent_requests.into_iter().map(usage_record_view).collect(),
|
||||
pool_summary: DashboardPoolSummaryView {
|
||||
total: provider_accounts.total,
|
||||
active: provider_accounts.active,
|
||||
rate_limited: provider_accounts.rate_limited,
|
||||
expired: provider_accounts.expired,
|
||||
invalid: provider_accounts.invalid,
|
||||
normal: provider_accounts.normal,
|
||||
quota_exhausted: provider_accounts.quota_exhausted,
|
||||
refreshing: provider_accounts.refreshing,
|
||||
rate_limited: provider_accounts.rate_limited,
|
||||
disabled: provider_accounts.disabled,
|
||||
banned: provider_accounts.banned,
|
||||
error: provider_accounts.error,
|
||||
},
|
||||
capacity_info: DashboardCapacityInfoView {
|
||||
max_concurrent_per_account: capacity.max_concurrent_per_account,
|
||||
|
||||
@@ -347,14 +347,11 @@ pub struct DashboardAccountRequestBucketView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardPoolSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
|
||||
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
|
||||
|
||||
use gateway_core::accounting::Decimal;
|
||||
|
||||
/// 千分位格式化。
|
||||
#[must_use]
|
||||
pub fn format_number(value: u64) -> String {
|
||||
@@ -37,3 +39,33 @@ pub fn format_compact_number(value: u64) -> String {
|
||||
}
|
||||
format_number(value)
|
||||
}
|
||||
|
||||
/// 货币展示格式化。USD 常规金额保留两位,小于 1 美元时最多保留四位。
|
||||
#[must_use]
|
||||
pub fn format_decimal_currency(amount: &str, currency: &str) -> String {
|
||||
if currency != "USD" {
|
||||
return format!("{currency} {amount}");
|
||||
}
|
||||
|
||||
const DECIMAL_SCALE: u128 = 10_000_000_000;
|
||||
let scaled = amount.parse::<Decimal>().unwrap_or_default().scaled();
|
||||
let precision = if scaled != 0 && scaled < DECIMAL_SCALE {
|
||||
4_u32
|
||||
} else {
|
||||
2_u32
|
||||
};
|
||||
let rounding_unit = 10_u128.pow(10 - precision);
|
||||
let rounded = (scaled + rounding_unit / 2) / rounding_unit;
|
||||
let display_scale = 10_u128.pow(precision);
|
||||
let whole = rounded / display_scale;
|
||||
let fraction = rounded % display_scale;
|
||||
let mut display = format!("{whole}.{fraction:0width$}", width = precision as usize);
|
||||
while display.ends_with('0')
|
||||
&& display
|
||||
.split_once('.')
|
||||
.is_some_and(|(_, fraction)| fraction.len() > 2)
|
||||
{
|
||||
display.pop();
|
||||
}
|
||||
format!("${display}")
|
||||
}
|
||||
|
||||
@@ -72,6 +72,7 @@ mod query {
|
||||
|
||||
mod response {
|
||||
use gateway_api::admin::accounts::AccountUsageView;
|
||||
use gateway_api::admin::presenter::format_decimal_currency;
|
||||
|
||||
#[test]
|
||||
fn account_usage_view_should_keep_unobserved_numbers_null() {
|
||||
@@ -112,12 +113,21 @@ mod response {
|
||||
assert!(value["totalTokens"].is_null());
|
||||
assert_eq!(value["createdTokensDisplay"], "-");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_usage_currency_should_limit_usd_to_four_fraction_digits() {
|
||||
assert_eq!(format_decimal_currency("0.1204956", "USD"), "$0.1205");
|
||||
assert_eq!(format_decimal_currency("0.99996", "USD"), "$1.00");
|
||||
assert_eq!(format_decimal_currency("0.0300", "USD"), "$0.03");
|
||||
assert_eq!(format_decimal_currency("12.3456", "USD"), "$12.35");
|
||||
assert_eq!(format_decimal_currency("0.1204956", "CNY"), "CNY 0.1204956");
|
||||
}
|
||||
}
|
||||
|
||||
mod actions {
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
|
||||
accounts::AccountConnectionTestEvent as DomainConnectionTestEvent,
|
||||
provider_credentials::{
|
||||
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
|
||||
},
|
||||
@@ -343,9 +353,7 @@ mod actions {
|
||||
DomainConnectionTestEvent::Content {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Normal,
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
@@ -353,7 +361,6 @@ mod actions {
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::QuotaExhausted,
|
||||
},
|
||||
]
|
||||
.map(|event| AccountConnectionTestEvent::from(event).data);
|
||||
@@ -378,7 +385,7 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
|
||||
json!({ "type": "test_complete", "success": true }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
@@ -386,8 +393,7 @@ mod actions {
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "quota_exhausted"
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
@@ -14,7 +14,7 @@ use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountListQuery, AccountPage, AccountPageItem, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
},
|
||||
auth::{AdminAuditEvent, AdminSession},
|
||||
@@ -426,7 +426,7 @@ impl AccountStore for UnusedStore {
|
||||
Err(unavailable("account list"))
|
||||
}
|
||||
|
||||
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountPageItem>> {
|
||||
Err(unavailable("account"))
|
||||
}
|
||||
|
||||
|
||||
@@ -222,16 +222,11 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics {
|
||||
total: 807,
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
rate_limited: 0,
|
||||
expired: 2,
|
||||
invalid: 0,
|
||||
normal: 15,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
rate_limited: 0,
|
||||
disabled: 3,
|
||||
banned: 2,
|
||||
error: 4,
|
||||
},
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
|
||||
@@ -187,45 +187,41 @@ impl fmt::Debug for OpaqueProviderData {
|
||||
}
|
||||
}
|
||||
|
||||
/// 数据库中固定的账号运行状态。
|
||||
/// 已持久化的凭据/账号身份状态。
|
||||
///
|
||||
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
|
||||
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
|
||||
/// 这里只保存凭据与账号身份事实。额度耗尽属于 [`QuotaState`],临时 429 属于
|
||||
/// 运行时冷却;二者都不得写入此枚举。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountAvailability {
|
||||
pub enum CredentialState {
|
||||
Unknown,
|
||||
Ready,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
Invalid,
|
||||
}
|
||||
|
||||
impl AccountAvailability {
|
||||
impl CredentialState {
|
||||
/// 返回数据库稳定值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Ready => "ready",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::Expired => "expired",
|
||||
Self::Banned => "banned",
|
||||
Self::Invalid => "invalid",
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回账号进入不可恢复错误态、但 Provider 未给出更具体上游原因时的稳定兜底。
|
||||
///
|
||||
/// Provider 应优先持久化已脱敏的上游 message 或原因码;该值只避免错误态沿用
|
||||
/// 上一次无关的错误信息。
|
||||
/// 返回该凭据状态对应的稳定错误原因。
|
||||
#[must_use]
|
||||
pub const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
pub const fn error_reason(self) -> Option<AccountErrorReason> {
|
||||
match self {
|
||||
Self::Expired => Some("credential_expired"),
|
||||
Self::Banned => Some("account_banned"),
|
||||
Self::Invalid => Some("credential_invalid"),
|
||||
Self::Unknown | Self::Ready | Self::QuotaExhausted => None,
|
||||
Self::Unknown => Some(AccountErrorReason::AccountUnverified),
|
||||
Self::Expired => Some(AccountErrorReason::CredentialExpired),
|
||||
Self::Banned => Some(AccountErrorReason::AccountBanned),
|
||||
Self::Invalid => Some(AccountErrorReason::CredentialInvalid),
|
||||
Self::Ready => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,7 +231,6 @@ impl AccountAvailability {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"ready" => Some(Self::Ready),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"expired" => Some(Self::Expired),
|
||||
"banned" => Some(Self::Banned),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
@@ -244,6 +239,373 @@ impl AccountAvailability {
|
||||
}
|
||||
}
|
||||
|
||||
/// 额度访问结论;百分比和余额等展示值不进入此枚举。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum QuotaAccessState {
|
||||
Unknown,
|
||||
Allowed,
|
||||
Exhausted,
|
||||
}
|
||||
|
||||
impl QuotaAccessState {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Allowed => "allowed",
|
||||
Self::Exhausted => "exhausted",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"allowed" => Some(Self::Allowed),
|
||||
"exhausted" => Some(Self::Exhausted),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 权威额度耗尽证据。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum QuotaEvidence {
|
||||
ProviderDenied,
|
||||
AccountLimitReached,
|
||||
UsageLimitReached,
|
||||
PaymentRequired,
|
||||
}
|
||||
|
||||
impl QuotaEvidence {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ProviderDenied => "provider_denied",
|
||||
Self::AccountLimitReached => "account_limit_reached",
|
||||
Self::UsageLimitReached => "usage_limit_reached",
|
||||
Self::PaymentRequired => "payment_required",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"provider_denied" => Some(Self::ProviderDenied),
|
||||
"account_limit_reached" => Some(Self::AccountLimitReached),
|
||||
"usage_limit_reached" => Some(Self::UsageLimitReached),
|
||||
"payment_required" => Some(Self::PaymentRequired),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 可审计的额度访问事实。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaState {
|
||||
access: QuotaAccessState,
|
||||
evidence: Option<QuotaEvidence>,
|
||||
observed_at: Option<SystemTime>,
|
||||
reset_at: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl Default for QuotaState {
|
||||
fn default() -> Self {
|
||||
Self::unknown()
|
||||
}
|
||||
}
|
||||
|
||||
impl QuotaState {
|
||||
#[must_use]
|
||||
pub const fn unknown() -> Self {
|
||||
Self {
|
||||
access: QuotaAccessState::Unknown,
|
||||
evidence: None,
|
||||
observed_at: None,
|
||||
reset_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn observed_unknown(observed_at: SystemTime) -> Self {
|
||||
Self {
|
||||
access: QuotaAccessState::Unknown,
|
||||
evidence: None,
|
||||
observed_at: Some(observed_at),
|
||||
reset_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn allowed(observed_at: SystemTime) -> Self {
|
||||
Self {
|
||||
access: QuotaAccessState::Allowed,
|
||||
evidence: None,
|
||||
observed_at: Some(observed_at),
|
||||
reset_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn exhausted(
|
||||
evidence: QuotaEvidence,
|
||||
observed_at: SystemTime,
|
||||
reset_at: Option<SystemTime>,
|
||||
) -> Self {
|
||||
Self {
|
||||
access: QuotaAccessState::Exhausted,
|
||||
evidence: Some(evidence),
|
||||
observed_at: Some(observed_at),
|
||||
reset_at,
|
||||
}
|
||||
}
|
||||
|
||||
/// 从持久化列恢复额度状态;非法组合返回 `None`。
|
||||
#[must_use]
|
||||
pub const fn from_persisted(
|
||||
access: QuotaAccessState,
|
||||
evidence: Option<QuotaEvidence>,
|
||||
observed_at: Option<SystemTime>,
|
||||
reset_at: Option<SystemTime>,
|
||||
) -> Option<Self> {
|
||||
match access {
|
||||
QuotaAccessState::Unknown if evidence.is_none() && reset_at.is_none() => Some(Self {
|
||||
access,
|
||||
evidence,
|
||||
observed_at,
|
||||
reset_at,
|
||||
}),
|
||||
QuotaAccessState::Allowed
|
||||
if evidence.is_none() && observed_at.is_some() && reset_at.is_none() =>
|
||||
{
|
||||
Some(Self {
|
||||
access,
|
||||
evidence,
|
||||
observed_at,
|
||||
reset_at,
|
||||
})
|
||||
}
|
||||
QuotaAccessState::Exhausted if evidence.is_some() && observed_at.is_some() => {
|
||||
Some(Self {
|
||||
access,
|
||||
evidence,
|
||||
observed_at,
|
||||
reset_at,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn access(self) -> QuotaAccessState {
|
||||
self.access
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn evidence(self) -> Option<QuotaEvidence> {
|
||||
self.evidence
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn observed_at(self) -> Option<SystemTime> {
|
||||
self.observed_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn reset_at(self) -> Option<SystemTime> {
|
||||
self.reset_at
|
||||
}
|
||||
|
||||
/// 返回当前已确认的额度耗尽结论。
|
||||
///
|
||||
/// `reset_at` 只表示何时应重新向 Provider 求证;时间到期本身不是额度恢复证据。
|
||||
#[must_use]
|
||||
pub fn is_exhausted(self) -> bool {
|
||||
self.access == QuotaAccessState::Exhausted
|
||||
}
|
||||
|
||||
/// 判断已耗尽额度是否到达 Provider 的下一次复核时间。
|
||||
///
|
||||
/// 有明确 `reset_at` 时严格等待该时刻;没有时由 Provider 传入自己的保守复核周期。
|
||||
#[must_use]
|
||||
pub fn exhaustion_refresh_due(self, now: SystemTime, fallback_interval: Duration) -> bool {
|
||||
if !self.is_exhausted() {
|
||||
return false;
|
||||
}
|
||||
let due_at = self.reset_at.or_else(|| {
|
||||
self.observed_at
|
||||
.and_then(|observed_at| observed_at.checked_add(fallback_interval))
|
||||
});
|
||||
due_at.is_none_or(|due_at| due_at <= now)
|
||||
}
|
||||
|
||||
/// 合并一次额度访问观察;`Unknown` 不能擦除已经确认的访问结论。
|
||||
#[must_use]
|
||||
pub fn merge_observation(self, observation: Self) -> Self {
|
||||
if observation.access == QuotaAccessState::Unknown
|
||||
&& self.access != QuotaAccessState::Unknown
|
||||
{
|
||||
self
|
||||
} else {
|
||||
observation
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 对外唯一的五态账号状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountStatus {
|
||||
Normal,
|
||||
QuotaExhausted,
|
||||
RateLimited,
|
||||
Disabled,
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理页稳定业务排序:正常、限流、耗尽、错误、停用。
|
||||
#[must_use]
|
||||
pub const fn sort_rank(self) -> u8 {
|
||||
match self {
|
||||
Self::Normal => 0,
|
||||
Self::RateLimited => 1,
|
||||
Self::QuotaExhausted => 2,
|
||||
Self::Error => 3,
|
||||
Self::Disabled => 4,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `error` 状态下的稳定原因码。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountErrorReason {
|
||||
AccountUnverified,
|
||||
AccessTokenExpired,
|
||||
CredentialExpired,
|
||||
CredentialInvalid,
|
||||
AccountBanned,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AccountUnverified => "account_unverified",
|
||||
Self::AccessTokenExpired => "access_token_expired",
|
||||
Self::CredentialExpired => "credential_expired",
|
||||
Self::CredentialInvalid => "credential_invalid",
|
||||
Self::AccountBanned => "account_banned",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"account_unverified" => Some(Self::AccountUnverified),
|
||||
"access_token_expired" => Some(Self::AccessTokenExpired),
|
||||
"credential_expired" => Some(Self::CredentialExpired),
|
||||
"credential_invalid" => Some(Self::CredentialInvalid),
|
||||
"account_banned" => Some(Self::AccountBanned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 唯一状态解析器的完整输入事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusFacts {
|
||||
pub enabled: bool,
|
||||
pub credential_state: CredentialState,
|
||||
pub access_token_expires_at: Option<SystemTime>,
|
||||
pub quota: QuotaState,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
}
|
||||
|
||||
/// 唯一状态解析器的输出。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusProjection {
|
||||
pub status: AccountStatus,
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub error_message: Option<String>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
/// 从独立事实派生唯一、互斥的对外状态。
|
||||
#[must_use]
|
||||
pub fn resolve_account_status(
|
||||
facts: &AccountStatusFacts,
|
||||
now: SystemTime,
|
||||
) -> AccountStatusProjection {
|
||||
if !facts.enabled {
|
||||
return status_projection(AccountStatus::Disabled);
|
||||
}
|
||||
let credential_error = facts.credential_state.error_reason().or_else(|| {
|
||||
facts
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
.then_some(AccountErrorReason::AccessTokenExpired)
|
||||
});
|
||||
if let Some(default_reason) = credential_error {
|
||||
return AccountStatusProjection {
|
||||
status: AccountStatus::Error,
|
||||
error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)),
|
||||
error_message: facts.last_error_message.clone(),
|
||||
rate_limited_until: None,
|
||||
};
|
||||
}
|
||||
if facts.quota.is_exhausted() {
|
||||
return status_projection(AccountStatus::QuotaExhausted);
|
||||
}
|
||||
if facts
|
||||
.rate_limited_until
|
||||
.is_some_and(|rate_limited_until| rate_limited_until > now)
|
||||
{
|
||||
return AccountStatusProjection {
|
||||
status: AccountStatus::RateLimited,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: facts.rate_limited_until,
|
||||
};
|
||||
}
|
||||
status_projection(AccountStatus::Normal)
|
||||
}
|
||||
|
||||
const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
|
||||
AccountStatusProjection {
|
||||
status,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 不含 secret 的账号持久事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccount {
|
||||
@@ -257,7 +619,10 @@ pub struct ProviderAccount {
|
||||
authentication_kind: String,
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
quota: QuotaState,
|
||||
last_error_reason: Option<AccountErrorReason>,
|
||||
last_error_message: Option<String>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
@@ -286,7 +651,10 @@ impl ProviderAccount {
|
||||
authentication_kind,
|
||||
revision,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Unknown,
|
||||
credential_state: CredentialState::Unknown,
|
||||
quota: QuotaState::unknown(),
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
@@ -307,17 +675,23 @@ impl ProviderAccount {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_runtime_state(
|
||||
pub fn with_account_facts(
|
||||
mut self,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
quota: QuotaState,
|
||||
last_error_reason: Option<AccountErrorReason>,
|
||||
last_error_message: Option<String>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.availability = if self.upstream_user_id.is_some() {
|
||||
availability
|
||||
self.credential_state = if self.upstream_user_id.is_some() {
|
||||
credential_state
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
CredentialState::Unknown
|
||||
};
|
||||
self.quota = quota;
|
||||
self.last_error_reason = last_error_reason;
|
||||
self.last_error_message = last_error_message;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -382,8 +756,23 @@ impl ProviderAccount {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn availability(&self) -> AccountAvailability {
|
||||
self.availability
|
||||
pub const fn credential_state(&self) -> CredentialState {
|
||||
self.credential_state
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn quota(&self) -> QuotaState {
|
||||
self.quota
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn last_error_reason(&self) -> Option<AccountErrorReason> {
|
||||
self.last_error_reason
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn last_error_message(&self) -> Option<&str> {
|
||||
self.last_error_message.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -407,23 +796,25 @@ impl ProviderAccount {
|
||||
self.has_refresh_token
|
||||
}
|
||||
|
||||
/// 判断账号当前能否进入同 target 的候选池。
|
||||
/// 组合持久事实与请求级冷却,交给唯一解析器派生状态。
|
||||
#[must_use]
|
||||
pub fn is_schedulable(&self, now: SystemTime) -> bool {
|
||||
let available = match self.availability {
|
||||
AccountAvailability::Ready => true,
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => false,
|
||||
};
|
||||
self.upstream_user_id.is_some()
|
||||
&& self.enabled
|
||||
&& available
|
||||
&& self
|
||||
.access_token_expires_at
|
||||
.is_none_or(|expires_at| expires_at > now)
|
||||
pub fn status_projection(
|
||||
&self,
|
||||
now: SystemTime,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AccountStatusProjection {
|
||||
resolve_account_status(
|
||||
&AccountStatusFacts {
|
||||
enabled: self.enabled,
|
||||
credential_state: self.credential_state,
|
||||
access_token_expires_at: self.access_token_expires_at,
|
||||
quota: self.quota,
|
||||
rate_limited_until,
|
||||
last_error_reason: self.last_error_reason,
|
||||
last_error_message: self.last_error_message.clone(),
|
||||
},
|
||||
now,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -600,10 +991,11 @@ pub enum CredentialCasOutcome {
|
||||
pub struct QuotaObservation {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: Option<OpaqueProviderData>,
|
||||
pub observed_at: Option<SystemTime>,
|
||||
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
|
||||
pub limit_reached: Option<bool>,
|
||||
pub quota: OpaqueProviderData,
|
||||
/// Provider 原始 quota document 的观察时间;不代表访问结论发生变化。
|
||||
pub observed_at: SystemTime,
|
||||
/// Provider 已从私有 JSON 归一化出的额度访问事实。
|
||||
pub state: QuotaState,
|
||||
}
|
||||
|
||||
impl fmt::Debug for QuotaObservation {
|
||||
@@ -614,6 +1006,7 @@ impl fmt::Debug for QuotaObservation {
|
||||
.field("expected_revision", &self.expected_revision)
|
||||
.field("quota", &self.quota)
|
||||
.field("observed_at", &self.observed_at)
|
||||
.field("state", &self.state)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -624,14 +1017,24 @@ pub enum QuotaWriteOutcome {
|
||||
Conflict,
|
||||
}
|
||||
|
||||
/// 不改 Provider 原始 quota JSON 的额度访问事实写入。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct QuotaAccessChange {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub state: QuotaState,
|
||||
}
|
||||
|
||||
/// 账号状态的 revision-fenced 写入。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStateChange {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: AccountAvailability,
|
||||
pub credential_state: CredentialState,
|
||||
pub observed_at: SystemTime,
|
||||
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
|
||||
/// 受控错误原因;恢复 Ready 时必须清空。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
/// 已脱敏的上游诊断信息;恢复 Ready 时必须清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
@@ -682,6 +1085,11 @@ pub trait ProviderAccountStore: Send + Sync {
|
||||
observation: QuotaObservation,
|
||||
) -> Result<QuotaWriteOutcome, StoreError>;
|
||||
|
||||
async fn apply_quota_access(
|
||||
&self,
|
||||
change: QuotaAccessChange,
|
||||
) -> Result<QuotaWriteOutcome, StoreError>;
|
||||
|
||||
async fn apply_state_change(&self, change: AccountStateChange) -> Result<(), StoreError>;
|
||||
|
||||
async fn update_account(&self, update: ProviderAccountUpdate) -> Result<(), StoreError>;
|
||||
@@ -772,7 +1180,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub last_started_at: Option<SystemTime>,
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub quota_limit_reached: bool,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -920,11 +1328,16 @@ impl AccountRuntimeSignals {
|
||||
if let Some(quota) = quota {
|
||||
self.quota_reset_at = quota.reset_at;
|
||||
self.quota_remaining_rank = quota.remaining_rank;
|
||||
self.quota_limit_reached = quota.limit_reached;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_rate_limit(mut self, rate_limited_until: Option<SystemTime>) -> Self {
|
||||
self.rate_limited_until = rate_limited_until;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_runtime_health(
|
||||
mut self,
|
||||
@@ -942,20 +1355,14 @@ impl AccountRuntimeSignals {
|
||||
pub struct AccountQuotaSignals {
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl AccountQuotaSignals {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
) -> Self {
|
||||
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
|
||||
Self {
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -968,12 +1375,6 @@ impl AccountQuotaSignals {
|
||||
pub const fn remaining_rank(self) -> Option<u64> {
|
||||
self.remaining_rank
|
||||
}
|
||||
|
||||
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号持久事实与可重建运行信号的请求级组合。
|
||||
@@ -991,24 +1392,24 @@ pub struct AccountSelectionContext {
|
||||
pub excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
pub preferred_account: Option<ProviderAccountId>,
|
||||
pub round_robin_cursor: u64,
|
||||
pub availability: AccountAvailabilityPolicy,
|
||||
pub eligibility: AccountEligibilityPolicy,
|
||||
}
|
||||
|
||||
/// 选择账号时是否信任本地可用性投影。
|
||||
/// 选择账号时是否执行本地调度资格投影。
|
||||
///
|
||||
/// 管理端对指定账号执行诊断时,会直接向上游确认实际状态;该模式跳过
|
||||
/// `enabled`、可用性、冷却和 token 到期等本地投影,仍保留租约、并发和
|
||||
/// 请求间隔约束,且只能与固定账号约束组合使用。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AccountAvailabilityPolicy {
|
||||
pub enum AccountEligibilityPolicy {
|
||||
#[default]
|
||||
Enforce,
|
||||
BypassForDiagnostic,
|
||||
}
|
||||
|
||||
impl AccountAvailabilityPolicy {
|
||||
impl AccountEligibilityPolicy {
|
||||
#[must_use]
|
||||
pub const fn bypasses_local_availability(self) -> bool {
|
||||
pub const fn bypasses_local_eligibility(self) -> bool {
|
||||
matches!(self, Self::BypassForDiagnostic)
|
||||
}
|
||||
}
|
||||
@@ -1132,10 +1533,14 @@ impl AccountSelector {
|
||||
candidate: &AccountCandidate,
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<AccountSchedulingBlocker> {
|
||||
if !context.availability.bypasses_local_availability()
|
||||
&& !candidate.account.is_schedulable(context.now)
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
if !context.eligibility.bypasses_local_eligibility() {
|
||||
let status = candidate
|
||||
.account
|
||||
.status_projection(context.now, candidate.signals.rate_limited_until)
|
||||
.status;
|
||||
if status != AccountStatus::Normal {
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
}
|
||||
}
|
||||
if context.excluded_accounts.contains(candidate.account.id()) {
|
||||
return Some(AccountSchedulingBlocker::Excluded);
|
||||
|
||||
@@ -9,7 +9,7 @@ use std::time::{Duration, SystemTime};
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::engine::credential::{
|
||||
AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision,
|
||||
AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision, CredentialState,
|
||||
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use crate::error::{IdentifierError, validate_text};
|
||||
@@ -411,7 +411,7 @@ pub struct ProviderCredentialState {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
}
|
||||
|
||||
@@ -421,14 +421,14 @@ impl ProviderCredentialState {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
enabled,
|
||||
availability,
|
||||
credential_state,
|
||||
observed_at,
|
||||
}
|
||||
}
|
||||
@@ -449,8 +449,8 @@ impl ProviderCredentialState {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn availability(&self) -> AccountAvailability {
|
||||
self.availability
|
||||
pub const fn credential_state(&self) -> CredentialState {
|
||||
self.credential_state
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
||||
@@ -5,12 +5,12 @@ use std::time::{Duration, SystemTime};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate,
|
||||
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
|
||||
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
|
||||
RotationStrategy,
|
||||
AccountAttemptFeedback, AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats,
|
||||
AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext,
|
||||
AccountSelectionPolicy, AccountSelector, AccountStatus, CredentialCasUpdate,
|
||||
CredentialRevision, CredentialState, OpaqueProviderData, PlaintextCredential,
|
||||
PreferredAccountSelection, ProviderAccount, ProviderAccountId, ProviderAccountIdentity,
|
||||
ProviderAccountUpdate, QuotaEvidence, QuotaState, RotationStrategy,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
@@ -24,7 +24,13 @@ fn account(id: &str) -> ProviderAccount {
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
|
||||
@@ -35,7 +41,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -53,7 +59,7 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
preferred_account: None,
|
||||
round_robin_cursor: 0,
|
||||
availability: AccountAvailabilityPolicy::Enforce,
|
||||
eligibility: AccountEligibilityPolicy::Enforce,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,43 +107,99 @@ fn provider_account_identity_debug_should_redact_upstream_ids() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn availability_should_round_trip_all_database_values() {
|
||||
fn account_fact_values_should_round_trip() {
|
||||
assert_eq!(
|
||||
AccountAvailability::parse(AccountAvailability::QuotaExhausted.as_str()),
|
||||
Some(AccountAvailability::QuotaExhausted)
|
||||
CredentialState::parse(CredentialState::Banned.as_str()),
|
||||
Some(CredentialState::Banned)
|
||||
);
|
||||
assert_eq!(
|
||||
QuotaEvidence::parse("provider_denied"),
|
||||
Some(QuotaEvidence::ProviderDenied)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
fn exhausted_quota_refresh_is_due_at_reset_or_provider_fallback() {
|
||||
let now = SystemTime::now();
|
||||
let observed_at = now - Duration::from_secs(60);
|
||||
let before_reset = QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
observed_at,
|
||||
Some(now + Duration::from_secs(1)),
|
||||
);
|
||||
let after_reset = QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
observed_at,
|
||||
Some(now - Duration::from_secs(1)),
|
||||
);
|
||||
let without_reset = QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None);
|
||||
|
||||
assert!(!before_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
|
||||
assert!(after_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
|
||||
assert!(without_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
|
||||
assert!(!without_reset.exhaustion_refresh_due(now, Duration::from_secs(120)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inconclusive_quota_observation_cannot_erase_confirmed_access() {
|
||||
let now = SystemTime::now();
|
||||
let exhausted = QuotaState::exhausted(QuotaEvidence::ProviderDenied, now, None);
|
||||
|
||||
assert_eq!(
|
||||
exhausted
|
||||
.merge_observation(QuotaState::observed_unknown(now + Duration::from_secs(1)))
|
||||
.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Exhausted
|
||||
);
|
||||
assert_eq!(
|
||||
exhausted
|
||||
.merge_observation(QuotaState::allowed(now + Duration::from_secs(1)))
|
||||
.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Allowed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
let observed_at = SystemTime::now();
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted")
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
|
||||
account: account("acct_exhausted").with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled")
|
||||
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
|
||||
account: account("acct_disabled").with_account_facts(
|
||||
false,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let mut context = context(RotationStrategy::Sticky);
|
||||
context.availability = AccountAvailabilityPolicy::BypassForDiagnostic;
|
||||
context.eligibility = AccountEligibilityPolicy::BypassForDiagnostic;
|
||||
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
@@ -155,9 +217,33 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
|
||||
let account = account("acct_disabled").with_account_facts(
|
||||
false,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(!account.is_schedulable(SystemTime::now()));
|
||||
assert_eq!(
|
||||
account.status_projection(SystemTime::now(), None).status,
|
||||
AccountStatus::Disabled
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let active_until = now + Duration::from_secs(60);
|
||||
let account = account("acct_rate_limited");
|
||||
|
||||
let active = account.status_projection(now, Some(active_until));
|
||||
assert_eq!(active.status, AccountStatus::RateLimited);
|
||||
assert_eq!(active.rate_limited_until, Some(active_until));
|
||||
|
||||
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
|
||||
assert_eq!(elapsed.status, AccountStatus::Normal);
|
||||
assert_eq!(elapsed.rate_limited_until, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -171,14 +257,20 @@ fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready);
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
account.availability(),
|
||||
account.is_schedulable(SystemTime::now())
|
||||
account.credential_state(),
|
||||
account.status_projection(SystemTime::now(), None).status
|
||||
),
|
||||
(AccountAvailability::Unknown, false)
|
||||
(CredentialState::Unknown, AccountStatus::Error)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -308,15 +400,11 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(
|
||||
Some(reset_at),
|
||||
Some(75),
|
||||
false,
|
||||
)));
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -633,21 +721,31 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
// availability 终态(Banned/Expired/Invalid)始终排除,
|
||||
// 不受 quota 信号或 skip_exhausted 放宽影响。
|
||||
fn enforced_eligibility_excludes_credential_errors_despite_quota_signals() {
|
||||
let mut banned = candidate("acct_banned", 0, Some(100));
|
||||
banned.account = banned
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Banned);
|
||||
banned.account = banned.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Banned,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let mut expired = candidate("acct_expired", 0, Some(100));
|
||||
expired.account = expired
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Expired);
|
||||
expired.account = expired.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Expired,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let mut invalid = candidate("acct_invalid", 0, Some(100));
|
||||
invalid.account = invalid
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Invalid);
|
||||
invalid.account = invalid.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Invalid,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![banned, expired, invalid, healthy];
|
||||
|
||||
@@ -659,14 +757,15 @@ fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
|
||||
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
|
||||
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
|
||||
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
|
||||
fn enforced_eligibility_excludes_authoritative_quota_exhaustion() {
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
|
||||
exhausted.account = exhausted.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![exhausted, healthy];
|
||||
|
||||
@@ -676,3 +775,31 @@ fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection()
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elapsed_quota_reset_does_not_fabricate_recovery() {
|
||||
let now = SystemTime::now();
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
now - Duration::from_secs(2),
|
||||
Some(now - Duration::from_secs(1)),
|
||||
),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
|
||||
assert_eq!(
|
||||
exhausted.account.status_projection(now, None).status,
|
||||
AccountStatus::QuotaExhausted
|
||||
);
|
||||
let candidates = [exhausted, healthy];
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::postgres::{
|
||||
PgProviderAccountRepository, ProviderAccountRepository, account_status_projection,
|
||||
load_rate_limited_until,
|
||||
};
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -21,73 +24,48 @@ impl PgObservabilityRepository {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
|
||||
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
/// 从账号事实和当前冷却一次性派生 Dashboard 五态;不在 SQL 中复制状态机。
|
||||
async fn account_status_snapshot(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let base = provider_account_metrics(&self.pool, observed_at).await?;
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
) -> StoreResult<(ProviderAccountMetrics, Vec<String>)> {
|
||||
let accounts = PgProviderAccountRepository::new(self.pool.clone())
|
||||
.list_provider_accounts(None, true)
|
||||
.await?;
|
||||
let now = observed_at.into();
|
||||
let rate_limited_until =
|
||||
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await;
|
||||
let mut metrics = ProviderAccountMetrics {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
..ProviderAccountMetrics::default()
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
|
||||
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
|
||||
let mut active = base.active;
|
||||
let mut rate_limited = base.rate_limited;
|
||||
let mut quota_exhausted = base.quota_exhausted;
|
||||
let now = SystemTime::now();
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let cooling = cooldowns
|
||||
.read(&account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > now)
|
||||
.is_some();
|
||||
if !cooling {
|
||||
continue;
|
||||
}
|
||||
if quota_reached {
|
||||
if quota_exhausted == 0 {
|
||||
continue;
|
||||
let mut normal_account_ids = Vec::new();
|
||||
for account in &accounts {
|
||||
let projection = account_status_projection(
|
||||
account,
|
||||
now,
|
||||
rate_limited_until.get(&account.id).copied(),
|
||||
);
|
||||
match projection.status {
|
||||
gateway_core::engine::credential::AccountStatus::Normal => {
|
||||
metrics.normal = metrics.normal.saturating_add(1);
|
||||
normal_account_ids.push(account.id.clone());
|
||||
}
|
||||
quota_exhausted -= 1;
|
||||
} else {
|
||||
if active == 0 {
|
||||
continue;
|
||||
gateway_core::engine::credential::AccountStatus::QuotaExhausted => {
|
||||
metrics.quota_exhausted = metrics.quota_exhausted.saturating_add(1);
|
||||
}
|
||||
gateway_core::engine::credential::AccountStatus::RateLimited => {
|
||||
metrics.rate_limited = metrics.rate_limited.saturating_add(1);
|
||||
}
|
||||
gateway_core::engine::credential::AccountStatus::Disabled => {
|
||||
metrics.disabled = metrics.disabled.saturating_add(1);
|
||||
}
|
||||
gateway_core::engine::credential::AccountStatus::Error => {
|
||||
metrics.error = metrics.error.saturating_add(1);
|
||||
}
|
||||
active -= 1;
|
||||
}
|
||||
rate_limited += 1;
|
||||
}
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
unavailable: rate_limited
|
||||
+ quota_exhausted
|
||||
+ base.expired
|
||||
+ base.invalid
|
||||
+ base.disabled
|
||||
+ base.banned,
|
||||
active,
|
||||
rate_limited,
|
||||
expired: base.expired,
|
||||
invalid: base.invalid,
|
||||
quota_exhausted,
|
||||
disabled: base.disabled,
|
||||
banned: base.banned,
|
||||
})
|
||||
Ok((metrics, normal_account_ids))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,9 +123,7 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
dashboard_totals(&self.pool),
|
||||
)?;
|
||||
let provider_accounts = self
|
||||
.provider_account_metrics_with_cooldowns(range.end)
|
||||
.await?;
|
||||
let (provider_accounts, _) = self.account_status_snapshot(range.end).await?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
@@ -252,12 +228,14 @@ impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<admin_observability::DashboardRuntimeSlots>> {
|
||||
let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at)
|
||||
let (_, normal_account_ids) = self
|
||||
.repository
|
||||
.account_status_snapshot(observed_at)
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
let active_accounts = u64::try_from(active_account_ids.len())
|
||||
let active_accounts = u64::try_from(normal_account_ids.len())
|
||||
.map_err(|_| observability_error(invalid("active account count overflows u64")))?;
|
||||
if active_account_ids.is_empty() {
|
||||
if normal_account_ids.is_empty() {
|
||||
return Ok(Some(admin_observability::DashboardRuntimeSlots {
|
||||
active_accounts,
|
||||
used_slots: Some(0),
|
||||
@@ -267,7 +245,7 @@ impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
return Ok(None);
|
||||
};
|
||||
let signals = match runtime_signals
|
||||
.credential_runtime_signals(&active_account_ids)
|
||||
.credential_runtime_signals(&normal_account_ids)
|
||||
.await
|
||||
{
|
||||
Ok(signals) => signals,
|
||||
|
||||
@@ -274,16 +274,11 @@ pub(crate) const fn admin_account_pool_metrics(
|
||||
) -> admin_observability::AccountPoolMetrics {
|
||||
admin_observability::AccountPoolMetrics {
|
||||
total: metrics.total,
|
||||
enabled: metrics.enabled,
|
||||
unavailable: metrics.unavailable,
|
||||
active: metrics.active,
|
||||
rate_limited: metrics.rate_limited,
|
||||
expired: metrics.expired,
|
||||
invalid: metrics.invalid,
|
||||
normal: metrics.normal,
|
||||
quota_exhausted: metrics.quota_exhausted,
|
||||
refreshing: None,
|
||||
rate_limited: metrics.rate_limited,
|
||||
disabled: metrics.disabled,
|
||||
banned: metrics.banned,
|
||||
error: metrics.error,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,8 +292,6 @@ pub(crate) fn admin_dashboard_account_usage(
|
||||
name: usage.name,
|
||||
email: usage.email,
|
||||
plan_type: usage.plan_type,
|
||||
enabled: usage.enabled,
|
||||
availability: usage.availability,
|
||||
request_count: usage.request_count,
|
||||
success_count: usage.success_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
|
||||
@@ -363,15 +363,11 @@ pub struct CalculatedUsageBillingFact {
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProviderAccountMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -388,8 +384,6 @@ pub struct ProviderAccountUsageObservation {
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub enabled: bool,
|
||||
pub availability: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
|
||||
@@ -15,7 +15,7 @@ pub(crate) async fn provider_account_usage(
|
||||
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
|
||||
pa.plan_type, pa.enabled, pa.availability,
|
||||
pa.plan_type,
|
||||
count(mr.id)::bigint as request_count,
|
||||
count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count,
|
||||
sum(mr.input_tokens)::bigint as input_tokens,
|
||||
@@ -52,7 +52,7 @@ pub(crate) async fn provider_account_usage(
|
||||
}
|
||||
statement.push(
|
||||
" group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
|
||||
pa.plan_type, pa.enabled, pa.availability
|
||||
pa.plan_type
|
||||
order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ",
|
||||
);
|
||||
statement.push_bind(i64::from(query.limit));
|
||||
@@ -313,8 +313,6 @@ pub(crate) fn provider_account_from_row(
|
||||
name: get(row, "name")?,
|
||||
email: get(row, "email")?,
|
||||
plan_type: get(row, "plan_type")?,
|
||||
enabled: get(row, "enabled")?,
|
||||
availability: get(row, "availability")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
|
||||
@@ -369,118 +369,6 @@ pub(crate) async fn request_costs(
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_account_metrics(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let row = sqlx::query(
|
||||
"with normalized as (
|
||||
select enabled, availability, quota_limit_reached,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability = 'invalid' then 'invalid'
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
when quota_limit_reached then 'quota_exhausted'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'rate_limited')::bigint as rate_limited,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
count(*) filter (where status = 'invalid')::bigint as invalid,
|
||||
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
|
||||
count(*) filter (where status = 'disabled')::bigint as disabled,
|
||||
count(*) filter (where status = 'banned')::bigint as banned
|
||||
from normalized",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account metrics"))?;
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: unsigned(&row, "total")?,
|
||||
enabled: unsigned(&row, "enabled")?,
|
||||
unavailable: unsigned(&row, "unavailable")?,
|
||||
active: unsigned(&row, "active")?,
|
||||
rate_limited: unsigned(&row, "rate_limited")?,
|
||||
expired: unsigned(&row, "expired")?,
|
||||
invalid: unsigned(&row, "invalid")?,
|
||||
quota_exhausted: unsigned(&row, "quota_exhausted")?,
|
||||
disabled: unsigned(&row, "disabled")?,
|
||||
banned: unsigned(&row, "banned")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn active_provider_account_ids(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<String>> {
|
||||
sqlx::query(
|
||||
"with normalized as (
|
||||
select id,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select id
|
||||
from normalized
|
||||
where status = 'active'
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load active provider account IDs"))?
|
||||
.iter()
|
||||
.map(|row| get(row, "id"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
|
||||
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(&row, "id")?,
|
||||
get(&row, "credential_revision")?,
|
||||
get(&row, "quota_limit_reached")?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
@@ -32,34 +31,6 @@ impl PgAdminAccountStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
|
||||
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
|
||||
async fn load_rate_limited_until(
|
||||
&self,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
let mut map = BTreeMap::new();
|
||||
for account in accounts {
|
||||
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
|
||||
continue;
|
||||
};
|
||||
if cooldown.credential_revision().get() != account.credential_revision.get() {
|
||||
continue;
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until > SystemTime::now() {
|
||||
map.insert(account.id.clone(), until);
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
async fn usage_observations(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
@@ -278,21 +249,18 @@ impl AccountStore for PgAdminAccountStore {
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let now = Utc::now();
|
||||
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
|
||||
let rate_limited_until = self
|
||||
.load_rate_limited_until(&accounts)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let rate_limited_until =
|
||||
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now.into()).await;
|
||||
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
|
||||
let mut items = accounts
|
||||
.into_iter()
|
||||
.filter_map(|account| {
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
let status = admin_account_status(&account, now, until);
|
||||
account_matches_admin_query(&account, status, &query).then_some(
|
||||
let projection = account_status_projection(&account, now.into(), until);
|
||||
account_matches_admin_query(&account, projection.status, &query).then_some(
|
||||
AdminAccountListItem {
|
||||
account,
|
||||
status,
|
||||
projection,
|
||||
usage: None,
|
||||
},
|
||||
)
|
||||
@@ -330,7 +298,12 @@ impl AccountStore for PgAdminAccountStore {
|
||||
.into_iter()
|
||||
.skip(offset)
|
||||
.take(page_size)
|
||||
.map(|item| admin_account_record(item.account))
|
||||
.map(|item| {
|
||||
Ok(AccountPageItem {
|
||||
account: admin_account_record(item.account)?,
|
||||
projection: item.projection,
|
||||
})
|
||||
})
|
||||
.collect::<AdminStoreResult<Vec<_>>>()?;
|
||||
Ok(AccountPage {
|
||||
config_revision: admin_revision(control_plane.settings.config_revision)?,
|
||||
@@ -340,13 +313,29 @@ impl AccountStore for PgAdminAccountStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.accounts
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>> {
|
||||
let record = self
|
||||
.accounts
|
||||
.load_provider_account(account_id)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?
|
||||
.map(|record| admin_account_record(record.summary))
|
||||
.transpose()
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let Some(record) = record else {
|
||||
return Ok(None);
|
||||
};
|
||||
let now = Utc::now();
|
||||
let rate_limited_until = load_rate_limited_until(
|
||||
self.cooldowns.as_deref(),
|
||||
std::slice::from_ref(&record.summary),
|
||||
now.into(),
|
||||
)
|
||||
.await
|
||||
.get(account_id)
|
||||
.copied();
|
||||
let projection = account_status_projection(&record.summary, now.into(), rate_limited_until);
|
||||
Ok(Some(AccountPageItem {
|
||||
account: admin_account_record(record.summary)?,
|
||||
projection,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -386,9 +375,9 @@ impl AccountStore for PgAdminAccountStore {
|
||||
.filter(|account| account.provider_kind == provider_kind.as_str())
|
||||
.filter(|account| {
|
||||
query
|
||||
.availability
|
||||
.credential_state
|
||||
.as_ref()
|
||||
.is_none_or(|expected| expected.matches(account.availability))
|
||||
.is_none_or(|expected| expected.matches(account.credential_state))
|
||||
})
|
||||
.filter(|account| {
|
||||
query
|
||||
|
||||
@@ -31,8 +31,8 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.map(DateTime::<Utc>::from),
|
||||
next_refresh_at: account.account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.account.enabled(),
|
||||
availability: account.account.availability(),
|
||||
availability_observed_at: Utc::now(),
|
||||
credential_state: account.account.credential_state(),
|
||||
credential_observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
@@ -164,9 +164,9 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.collect::<Vec<_>>();
|
||||
let rows = sqlx::query(
|
||||
"select id, credential_revision, provider_quota_json, quota_observed_at, \
|
||||
quota_limit_reached \
|
||||
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at \
|
||||
from provider_accounts \
|
||||
where id = any($1) and provider_quota_json is not null",
|
||||
where id = any($1) and quota_observed_at is not null",
|
||||
)
|
||||
.bind(account_ids)
|
||||
.fetch_all(&self.pool)
|
||||
@@ -184,24 +184,49 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.and_then(|value| CoreCredentialRevision::new(value).ok())
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let quota = row
|
||||
.try_get::<serde_json::Value, _>("provider_quota_json")
|
||||
.ok()
|
||||
.try_get::<Option<serde_json::Value>, _>("provider_quota_json")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.map(OpaqueProviderData::new)
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
.unwrap_or_else(|| OpaqueProviderData::new(serde_json::Map::new()));
|
||||
let observed_at = row
|
||||
.try_get::<DateTime<Utc>, _>("quota_observed_at")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let limit_reached = row
|
||||
.try_get::<bool, _>("quota_limit_reached")
|
||||
let access = row
|
||||
.try_get::<String, _>("quota_access_state")
|
||||
.ok()
|
||||
.and_then(|value| QuotaAccessState::parse(&value))
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let evidence = row
|
||||
.try_get::<Option<String>, _>("quota_evidence")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let evidence = match evidence {
|
||||
Some(value) => Some(
|
||||
QuotaEvidence::parse(&value)
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
let reset_at = row
|
||||
.try_get::<Option<DateTime<Utc>>, _>("quota_reset_at")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let access_observed_at = row
|
||||
.try_get::<Option<DateTime<Utc>>, _>("quota_access_observed_at")
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let state = QuotaState::from_persisted(
|
||||
access,
|
||||
evidence,
|
||||
access_observed_at.map(Into::into),
|
||||
reset_at.map(Into::into),
|
||||
)
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
Ok(QuotaObservation {
|
||||
account_id: CoreProviderAccountId::new(account_id)
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
expected_revision: revision,
|
||||
quota: Some(quota),
|
||||
observed_at: Some(observed_at.into()),
|
||||
limit_reached: Some(limit_reached),
|
||||
quota,
|
||||
observed_at: observed_at.into(),
|
||||
state,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -211,24 +236,38 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
&self,
|
||||
observation: QuotaObservation,
|
||||
) -> Result<QuotaWriteOutcome, CoreStoreError> {
|
||||
let quota = observation
|
||||
.quota
|
||||
.map(|value| {
|
||||
JsonObject::try_from_value(
|
||||
"provider_quota_json",
|
||||
serde_json::Value::Object(value.into_inner()),
|
||||
QUOTA_MAX_BYTES,
|
||||
)
|
||||
})
|
||||
.transpose()
|
||||
.map_err(core_store_error)?;
|
||||
let quota = JsonObject::try_from_value(
|
||||
"provider_quota_json",
|
||||
serde_json::Value::Object(observation.quota.into_inner()),
|
||||
QUOTA_MAX_BYTES,
|
||||
)
|
||||
.map_err(core_store_error)?;
|
||||
let updated = self
|
||||
.compare_and_swap_provider_quota(
|
||||
observation.account_id.as_str(),
|
||||
Revision::new(observation.expected_revision.get()).map_err(core_store_error)?,
|
||||
quota,
|
||||
observation.observed_at.map(DateTime::<Utc>::from),
|
||||
observation.limit_reached,
|
||||
DateTime::<Utc>::from(observation.observed_at),
|
||||
observation.state,
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
Ok(if updated {
|
||||
QuotaWriteOutcome::Updated
|
||||
} else {
|
||||
QuotaWriteOutcome::Conflict
|
||||
})
|
||||
}
|
||||
|
||||
async fn apply_quota_access(
|
||||
&self,
|
||||
change: QuotaAccessChange,
|
||||
) -> Result<QuotaWriteOutcome, CoreStoreError> {
|
||||
let updated = self
|
||||
.apply_provider_quota_access(
|
||||
change.account_id.as_str(),
|
||||
Revision::new(change.expected_revision.get()).map_err(core_store_error)?,
|
||||
change.state,
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
@@ -245,8 +284,9 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
account_id: change.account_id.as_str().to_owned(),
|
||||
expected_revision: Revision::new(change.expected_revision.get())
|
||||
.map_err(core_store_error)?,
|
||||
availability: change.availability,
|
||||
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
|
||||
credential_state: change.credential_state,
|
||||
credential_observed_at: DateTime::<Utc>::from(change.observed_at),
|
||||
error_reason: change.error_reason,
|
||||
message: change.message,
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
//! Admin 账号列表、用量与成本的领域映射。
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
use super::*;
|
||||
|
||||
use gateway_admin::model::accounts::AccountStatusSignals;
|
||||
|
||||
pub(crate) struct AdminAccountListItem {
|
||||
pub(crate) account: ProviderAccountSummary,
|
||||
pub(crate) status: AdminAccountStatus,
|
||||
pub(crate) projection: gateway_core::engine::credential::AccountStatusProjection,
|
||||
pub(crate) usage: Option<ProviderAccountUsageObservation>,
|
||||
}
|
||||
|
||||
@@ -57,24 +57,6 @@ pub(crate) fn account_matches_admin_query(
|
||||
provider_matches && search_matches && status_matches
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_status(
|
||||
account: &ProviderAccountSummary,
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AdminAccountStatus {
|
||||
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: account.quota_limit_reached,
|
||||
rate_limited_until,
|
||||
}
|
||||
.derive()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_summary(
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: DateTime<Utc>,
|
||||
@@ -90,7 +72,7 @@ pub(crate) fn admin_account_summary(
|
||||
};
|
||||
for account in accounts {
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
match admin_account_status(account, now, until) {
|
||||
match account_status_projection(account, now.into(), until).status {
|
||||
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
|
||||
AdminAccountStatus::QuotaExhausted => {
|
||||
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
|
||||
@@ -118,7 +100,11 @@ pub(crate) fn sort_admin_account_items(
|
||||
items.sort_by(|left, right| {
|
||||
let ordering = match sort.field {
|
||||
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
|
||||
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
|
||||
AdminAccountSortField::Status => left
|
||||
.projection
|
||||
.status
|
||||
.sort_rank()
|
||||
.cmp(&right.projection.status.sort_rank()),
|
||||
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
|
||||
AdminAccountSortField::Usage => left
|
||||
.usage
|
||||
@@ -171,10 +157,11 @@ pub(crate) fn admin_account_record(
|
||||
access_token_expires_at: summary.access_token_expires_at,
|
||||
next_refresh_at: summary.next_refresh_at,
|
||||
enabled: summary.enabled,
|
||||
availability: summary.availability,
|
||||
availability_observed_at: summary.availability_observed_at,
|
||||
credential_state: summary.credential_state,
|
||||
credential_observed_at: summary.credential_observed_at,
|
||||
quota: summary.quota,
|
||||
last_error_reason: summary.last_error_reason,
|
||||
last_error_message: summary.last_error_message,
|
||||
quota_observed_at: summary.quota_observed_at,
|
||||
created_at: summary.created_at,
|
||||
updated_at: summary.updated_at,
|
||||
})
|
||||
@@ -197,8 +184,8 @@ pub(crate) fn prepared_account(
|
||||
access_token_expires_at: credential.access_token_expires_at,
|
||||
next_refresh_at: credential.next_refresh_at,
|
||||
enabled: credential.enabled,
|
||||
availability: credential.availability,
|
||||
availability_observed_at: credential.availability_observed_at,
|
||||
credential_state: credential.credential_state,
|
||||
credential_observed_at: credential.credential_observed_at,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt,
|
||||
str::FromStr,
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -14,7 +13,7 @@ use gateway_admin::{
|
||||
MutationContext, Revision as AdminRevision,
|
||||
accounts::{
|
||||
AccountCost, AccountListQuery as AdminAccountListQuery, AccountModelUsage, AccountPage,
|
||||
AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort,
|
||||
AccountPageItem, AccountRecord, AccountRequestBucket, AccountSort as AdminAccountSort,
|
||||
AccountSortField as AdminAccountSortField, AccountStatus as AdminAccountStatus,
|
||||
AccountSummary, AccountUsage, AccountUsageWindowQuery, AccountUsageWindowResult,
|
||||
DeleteAccounts, SetAccountEnabled, SortDirection as AdminSortDirection,
|
||||
@@ -35,12 +34,13 @@ use gateway_admin::{
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision as CoreCredentialRevision, LoadedCredential,
|
||||
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision as CoreCredentialRevision, CredentialState, LoadedCredential,
|
||||
NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential,
|
||||
ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId,
|
||||
ProviderAccountIdentity, ProviderAccountStore,
|
||||
ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaAccessChange, QuotaAccessState,
|
||||
QuotaEvidence, QuotaObservation, QuotaState, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -63,8 +63,10 @@ mod core_adapter;
|
||||
mod mapping;
|
||||
mod repository;
|
||||
mod rows;
|
||||
mod runtime;
|
||||
|
||||
pub use admin_adapter::*;
|
||||
pub(crate) use mapping::*;
|
||||
pub use repository::*;
|
||||
pub use rows::*;
|
||||
pub(crate) use runtime::*;
|
||||
|
||||
@@ -25,9 +25,15 @@ pub trait ProviderAccountRepository: Send + Sync {
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected_revision: Revision,
|
||||
quota: Option<JsonObject>,
|
||||
observed_at: Option<DateTime<Utc>>,
|
||||
limit_reached: Option<bool>,
|
||||
quota: JsonObject,
|
||||
observed_at: DateTime<Utc>,
|
||||
state: QuotaState,
|
||||
) -> StoreResult<bool>;
|
||||
async fn apply_provider_quota_access(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected_revision: Revision,
|
||||
state: QuotaState,
|
||||
) -> StoreResult<bool>;
|
||||
async fn delete_provider_account(&self, id: &str) -> StoreResult<bool>;
|
||||
}
|
||||
@@ -93,9 +99,10 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
let rows = sqlx::query(
|
||||
"select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_observed_at,
|
||||
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
|
||||
access_token_expires_at, next_refresh_at, enabled, credential_state,
|
||||
credential_observed_at, quota_access_state, quota_evidence,
|
||||
quota_access_observed_at, quota_reset_at,
|
||||
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
|
||||
from provider_accounts
|
||||
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
|
||||
order by provider_kind, name, id",
|
||||
@@ -110,21 +117,21 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
|
||||
async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> {
|
||||
account.validate()?;
|
||||
let availability = if account.upstream_user_id.is_some() {
|
||||
account.availability
|
||||
let credential_state = if account.upstream_user_id.is_some() {
|
||||
account.credential_state
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
CredentialState::Unknown
|
||||
};
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
|
||||
availability, provider_quota_json,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
credential_state, provider_quota_json,
|
||||
credential_observed_at, quota_access_observed_at, quota_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
|
||||
$14, null, $15, null, now(), greatest(now(), $15)
|
||||
$14, null, $15, null, null, now(), greatest(now(), $15)
|
||||
)",
|
||||
)
|
||||
.bind(account.id)
|
||||
@@ -140,8 +147,8 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(account.access_token_expires_at)
|
||||
.bind(account.next_refresh_at)
|
||||
.bind(account.enabled)
|
||||
.bind(availability.as_str())
|
||||
.bind(account.availability_observed_at)
|
||||
.bind(credential_state.as_str())
|
||||
.bind(account.credential_observed_at)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("insert provider account"))?;
|
||||
@@ -214,27 +221,33 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
update.validate()?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = case
|
||||
set credential_state = case
|
||||
when enabled and upstream_user_id is not null then $3
|
||||
else availability
|
||||
else credential_state
|
||||
end,
|
||||
availability_observed_at = case
|
||||
credential_observed_at = case
|
||||
when enabled and upstream_user_id is not null then $4
|
||||
else availability_observed_at
|
||||
else credential_observed_at
|
||||
end,
|
||||
last_error_reason = case
|
||||
when enabled and upstream_user_id is not null then
|
||||
case when $3 = 'ready' then null else $5 end
|
||||
else last_error_reason
|
||||
end,
|
||||
last_error_message = case
|
||||
when enabled and upstream_user_id is not null then
|
||||
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
|
||||
case when $3 = 'ready' then null else $6 end
|
||||
else last_error_message
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $4) else updated_at end
|
||||
where id = $1 and credential_revision = $2
|
||||
and (availability_observed_at is null or availability_observed_at <= $4)",
|
||||
and (credential_observed_at is null or credential_observed_at <= $4)",
|
||||
)
|
||||
.bind(update.account_id)
|
||||
.bind(to_i64(update.expected_revision.get())?)
|
||||
.bind(update.availability.as_str())
|
||||
.bind(update.availability_observed_at)
|
||||
.bind(update.credential_state.as_str())
|
||||
.bind(update.credential_observed_at)
|
||||
.bind(update.error_reason.map(AccountErrorReason::as_str))
|
||||
.bind(update.message.as_deref())
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
@@ -259,36 +272,81 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected_revision: Revision,
|
||||
quota: Option<JsonObject>,
|
||||
observed_at: Option<DateTime<Utc>>,
|
||||
limit_reached: Option<bool>,
|
||||
quota: JsonObject,
|
||||
observed_at: DateTime<Utc>,
|
||||
state: QuotaState,
|
||||
) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "account_id", account_id)?;
|
||||
if quota.is_some() != observed_at.is_some() {
|
||||
return Err(invalid("quota JSON and observed_at must agree"));
|
||||
}
|
||||
if let Some(quota) = quota.as_ref() {
|
||||
validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?;
|
||||
}
|
||||
validate_object_size("provider_quota_json", "a, QUOTA_MAX_BYTES)?;
|
||||
let access_observed_at = state.observed_at().map(DateTime::<Utc>::from);
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set provider_quota_json = $3, quota_observed_at = $4,
|
||||
quota_limit_reached = coalesce($5, quota_limit_reached),
|
||||
updated_at = greatest(now(), coalesce($4, now()))
|
||||
quota_access_state = case
|
||||
when $7::timestamptz is not null
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
|
||||
then $5 else quota_access_state end,
|
||||
quota_evidence = case
|
||||
when $7::timestamptz is not null
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
|
||||
then $6 else quota_evidence end,
|
||||
quota_access_observed_at = case
|
||||
when $7::timestamptz is not null
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
|
||||
then $7 else quota_access_observed_at end,
|
||||
quota_reset_at = case
|
||||
when $7::timestamptz is not null
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= $7)
|
||||
then $8 else quota_reset_at end,
|
||||
updated_at = greatest(now(), $4)
|
||||
where id = $1 and credential_revision = $2
|
||||
and ($4 is null or quota_observed_at is null or quota_observed_at <= $4)",
|
||||
and (quota_observed_at is null or quota_observed_at <= $4)",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(to_i64(expected_revision.get())?)
|
||||
.bind(quota.map(|value| value.as_value()))
|
||||
.bind(quota.as_value())
|
||||
.bind(observed_at)
|
||||
.bind(limit_reached)
|
||||
.bind(state.access().as_str())
|
||||
.bind(state.evidence().map(QuotaEvidence::as_str))
|
||||
.bind(access_observed_at)
|
||||
.bind(state.reset_at().map(DateTime::<Utc>::from))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("compare and swap provider quota"))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
|
||||
async fn apply_provider_quota_access(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected_revision: Revision,
|
||||
state: QuotaState,
|
||||
) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "account_id", account_id)?;
|
||||
let observed_at = state
|
||||
.observed_at()
|
||||
.map(DateTime::<Utc>::from)
|
||||
.ok_or_else(|| invalid("quota access change requires observed_at"))?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set quota_access_observed_at = $3, quota_access_state = $4,
|
||||
quota_evidence = $5, quota_reset_at = $6,
|
||||
updated_at = greatest(now(), $3)
|
||||
where id = $1 and credential_revision = $2
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= $3)",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(to_i64(expected_revision.get())?)
|
||||
.bind(observed_at)
|
||||
.bind(state.access().as_str())
|
||||
.bind(state.evidence().map(QuotaEvidence::as_str))
|
||||
.bind(state.reset_at().map(DateTime::<Utc>::from))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("apply provider quota access"))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
|
||||
async fn delete_provider_account(&self, id: &str) -> StoreResult<bool> {
|
||||
require_nonempty(ENTITY, "id", id)?;
|
||||
let result = sqlx::query("delete from provider_accounts where id = $1 and not enabled")
|
||||
@@ -474,21 +532,21 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
account: &NewProviderAccount,
|
||||
) -> StoreResult<String> {
|
||||
account.validate()?;
|
||||
let availability = if account.upstream_user_id.is_some() {
|
||||
account.availability
|
||||
let credential_state = if account.upstream_user_id.is_some() {
|
||||
account.credential_state
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
CredentialState::Unknown
|
||||
};
|
||||
let imported_id = sqlx::query_scalar::<_, String>(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled,
|
||||
availability, provider_quota_json,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
credential_state, provider_quota_json,
|
||||
credential_observed_at, quota_access_observed_at, quota_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, 1, $10, $11, $12, $13,
|
||||
$14, null, $15, null, now(), greatest(now(), $15)
|
||||
$14, null, $15, null, null, now(), greatest(now(), $15)
|
||||
)
|
||||
on conflict (
|
||||
provider_kind,
|
||||
@@ -505,11 +563,17 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
access_token_expires_at = excluded.access_token_expires_at,
|
||||
next_refresh_at = excluded.next_refresh_at,
|
||||
enabled = excluded.enabled,
|
||||
availability = excluded.availability,
|
||||
credential_state = excluded.credential_state,
|
||||
provider_quota_json = null,
|
||||
availability_observed_at = excluded.availability_observed_at,
|
||||
quota_access_state = 'unknown',
|
||||
quota_evidence = null,
|
||||
quota_access_observed_at = null,
|
||||
quota_reset_at = null,
|
||||
credential_observed_at = excluded.credential_observed_at,
|
||||
quota_observed_at = null,
|
||||
updated_at = greatest(now(), excluded.availability_observed_at)
|
||||
last_error_reason = null,
|
||||
last_error_message = null,
|
||||
updated_at = greatest(now(), excluded.credential_observed_at)
|
||||
returning id",
|
||||
)
|
||||
.bind(&account.id)
|
||||
@@ -525,8 +589,8 @@ pub(crate) async fn upsert_provider_account_in_transaction(
|
||||
.bind(account.access_token_expires_at)
|
||||
.bind(account.next_refresh_at)
|
||||
.bind(account.enabled)
|
||||
.bind(availability.as_str())
|
||||
.bind(account.availability_observed_at)
|
||||
.bind(credential_state.as_str())
|
||||
.bind(account.credential_observed_at)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
@@ -574,17 +638,17 @@ pub(crate) async fn rotate_provider_account_in_transaction(
|
||||
next_refresh_at = $10,
|
||||
upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end,
|
||||
upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end,
|
||||
availability = case
|
||||
when not enabled then availability
|
||||
when availability = 'quota_exhausted' then availability
|
||||
credential_state = case
|
||||
when not enabled then credential_state
|
||||
when coalesce($12::text, upstream_user_id) is not null then 'ready'
|
||||
else 'unknown'
|
||||
end,
|
||||
availability_observed_at = case
|
||||
when not enabled then availability_observed_at
|
||||
when availability <> 'quota_exhausted' then now()
|
||||
else availability_observed_at
|
||||
credential_observed_at = case
|
||||
when not enabled then credential_observed_at
|
||||
else now()
|
||||
end,
|
||||
last_error_reason = case when enabled then null else last_error_reason end,
|
||||
last_error_message = case when enabled then null else last_error_message end,
|
||||
updated_at = now()
|
||||
where id = $1 and provider_kind = $2
|
||||
and credential_revision = $3
|
||||
|
||||
@@ -65,9 +65,29 @@ impl ProviderAccountAdminScope {
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析数据库 availability 列;字符串映射本体唯一归属 gateway-core。
|
||||
pub(crate) fn parse_availability(value: &str) -> StoreResult<AccountAvailability> {
|
||||
AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value"))
|
||||
pub(crate) fn parse_credential_state(value: &str) -> StoreResult<CredentialState> {
|
||||
CredentialState::parse(value).ok_or_else(|| invalid("unknown credential_state value"))
|
||||
}
|
||||
|
||||
pub(crate) fn parse_quota_access_state(value: &str) -> StoreResult<QuotaAccessState> {
|
||||
QuotaAccessState::parse(value).ok_or_else(|| invalid("unknown quota_access_state value"))
|
||||
}
|
||||
|
||||
pub(crate) fn parse_quota_evidence(value: Option<String>) -> StoreResult<Option<QuotaEvidence>> {
|
||||
value
|
||||
.map(|value| {
|
||||
QuotaEvidence::parse(&value).ok_or_else(|| invalid("unknown quota_evidence value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn parse_error_reason(value: Option<String>) -> StoreResult<Option<AccountErrorReason>> {
|
||||
value
|
||||
.map(|value| {
|
||||
AccountErrorReason::parse(&value)
|
||||
.ok_or_else(|| invalid("unknown last_error_reason value"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -85,12 +105,10 @@ pub struct ProviderAccountSummary {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub quota: QuotaState,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
@@ -132,8 +150,8 @@ pub struct NewProviderAccount {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -152,7 +170,7 @@ impl fmt::Debug for NewProviderAccount {
|
||||
.field("provider_kind", &self.provider_kind)
|
||||
.field("name", &self.name)
|
||||
.field("enabled", &self.enabled)
|
||||
.field("availability", &self.availability)
|
||||
.field("credential_state", &self.credential_state)
|
||||
.field("provider_credentials_json", &"[REDACTED]")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
@@ -297,24 +315,13 @@ impl fmt::Debug for ProviderCredentialUpdate {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProviderAccountObservation {
|
||||
pub account_id: String,
|
||||
pub availability: AccountAvailability,
|
||||
pub provider_quota_json: Option<JsonObject>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountStateUpdate {
|
||||
pub account_id: String,
|
||||
pub expected_revision: Revision,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
@@ -325,31 +332,20 @@ impl ProviderAccountStateUpdate {
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderAccountObservation {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() {
|
||||
return Err(invalid("quota JSON and quota_observed_at must agree"));
|
||||
}
|
||||
if let Some(quota) = &self.provider_quota_json {
|
||||
validate_object_size("provider_quota_json", quota, QUOTA_MAX_BYTES)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, credential_state,
|
||||
provider_quota_json, quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
|
||||
last_error_reason, last_error_message,
|
||||
credential_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts where id = $1";
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, credential_state,
|
||||
provider_quota_json, quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
|
||||
last_error_reason, last_error_message,
|
||||
credential_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts
|
||||
where id = any($1::text[]) and provider_kind = $2
|
||||
order by id";
|
||||
@@ -398,7 +394,13 @@ pub(crate) fn core_account_from_summary(
|
||||
summary.upstream_account_id,
|
||||
summary.plan_type,
|
||||
)
|
||||
.with_runtime_state(summary.enabled, summary.availability)
|
||||
.with_account_facts(
|
||||
summary.enabled,
|
||||
summary.credential_state,
|
||||
summary.quota,
|
||||
summary.last_error_reason,
|
||||
summary.last_error_message,
|
||||
)
|
||||
.with_refresh_schedule(
|
||||
summary.has_refresh_token,
|
||||
summary.next_refresh_at.map(Into::into),
|
||||
@@ -430,9 +432,20 @@ pub(crate) fn account_summary_from_row(
|
||||
let revision = row
|
||||
.try_get::<i64, _>("credential_revision")
|
||||
.map_err(|_| invalid("invalid credential revision"))?;
|
||||
let availability = row
|
||||
.try_get::<String, _>("availability")
|
||||
.map_err(|_| invalid("invalid availability"))?;
|
||||
let credential_state = row
|
||||
.try_get::<String, _>("credential_state")
|
||||
.map_err(|_| invalid("invalid credential_state"))?;
|
||||
let quota_access_state = parse_quota_access_state(&get::<String>(&row, "quota_access_state")?)?;
|
||||
let quota_evidence = parse_quota_evidence(get(&row, "quota_evidence")?)?;
|
||||
let quota_access_observed_at = get::<Option<DateTime<Utc>>>(&row, "quota_access_observed_at")?;
|
||||
let quota_reset_at = get::<Option<DateTime<Utc>>>(&row, "quota_reset_at")?;
|
||||
let quota = QuotaState::from_persisted(
|
||||
quota_access_state,
|
||||
quota_evidence,
|
||||
quota_access_observed_at.map(Into::into),
|
||||
quota_reset_at.map(Into::into),
|
||||
)
|
||||
.ok_or_else(|| invalid("invalid persisted quota fact"))?;
|
||||
Ok(ProviderAccountSummary {
|
||||
id: get(&row, "id")?,
|
||||
provider_kind: get(&row, "provider_kind")?,
|
||||
@@ -447,10 +460,10 @@ pub(crate) fn account_summary_from_row(
|
||||
access_token_expires_at: get(&row, "access_token_expires_at")?,
|
||||
next_refresh_at: get(&row, "next_refresh_at")?,
|
||||
enabled: get(&row, "enabled")?,
|
||||
availability: parse_availability(&availability)?,
|
||||
availability_observed_at: get(&row, "availability_observed_at")?,
|
||||
quota_observed_at: get(&row, "quota_observed_at")?,
|
||||
quota_limit_reached: get(&row, "quota_limit_reached")?,
|
||||
credential_state: parse_credential_state(&credential_state)?,
|
||||
credential_observed_at: get(&row, "credential_observed_at")?,
|
||||
quota,
|
||||
last_error_reason: parse_error_reason(get(&row, "last_error_reason")?)?,
|
||||
last_error_message: get(&row, "last_error_message")?,
|
||||
created_at: get(&row, "created_at")?,
|
||||
updated_at: get(&row, "updated_at")?,
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
//! 持久账号事实与可丢失冷却事实的统一状态投影。
|
||||
|
||||
use std::{collections::BTreeMap, time::SystemTime};
|
||||
|
||||
use futures::future::join_all;
|
||||
use gateway_core::{
|
||||
engine::credential::{
|
||||
AccountStatusFacts, AccountStatusProjection, ProviderAccountId, resolve_account_status,
|
||||
},
|
||||
provider_ports::ProviderCooldownPort,
|
||||
};
|
||||
|
||||
use super::ProviderAccountSummary;
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn account_status_projection(
|
||||
account: &ProviderAccountSummary,
|
||||
now: SystemTime,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AccountStatusProjection {
|
||||
resolve_account_status(
|
||||
&AccountStatusFacts {
|
||||
enabled: account.enabled,
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
},
|
||||
now,
|
||||
)
|
||||
}
|
||||
|
||||
/// 尽力读取账号当前有效的 429 冷却。
|
||||
///
|
||||
/// 冷却属于可丢失的账号级运行时事实:端口不可用、单条读取失败或脏账号 ID
|
||||
/// 都视为没有冷却,不污染持久账号状态。凭据轮换不会解除上游对同一账号的限流。
|
||||
pub(crate) async fn load_rate_limited_until(
|
||||
cooldowns: Option<&dyn ProviderCooldownPort>,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: SystemTime,
|
||||
) -> BTreeMap<String, SystemTime> {
|
||||
let Some(cooldowns) = cooldowns else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
let reads = accounts.iter().map(|account| async move {
|
||||
let account_id = ProviderAccountId::new(account.id.clone()).ok()?;
|
||||
let cooldown = cooldowns.read(&account_id).await.ok().flatten()?;
|
||||
(cooldown.until() > now).then(|| (account.id.clone(), cooldown.until()))
|
||||
});
|
||||
join_all(reads).await.into_iter().flatten().collect()
|
||||
}
|
||||
@@ -31,7 +31,7 @@ pub struct SnapshotRuntimeSettings {
|
||||
pub struct CredentialRevisionVectorEntry {
|
||||
pub provider_account_id: String,
|
||||
pub credential_revision: Revision,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
|
||||
|
||||
async fn credential_revision_vector(&self) -> StoreResult<Vec<CredentialRevisionVectorEntry>> {
|
||||
let rows = sqlx::query_as::<_, (String, i64, DateTime<Utc>, Option<DateTime<Utc>>)>(
|
||||
"select id, credential_revision, availability_observed_at, quota_observed_at
|
||||
"select id, credential_revision, credential_observed_at, quota_observed_at
|
||||
from provider_accounts order by id",
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
@@ -119,7 +119,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
|
||||
Ok(CredentialRevisionVectorEntry {
|
||||
provider_account_id: row.0,
|
||||
credential_revision: revision_from_i64(row.1)?,
|
||||
availability_observed_at: row.2,
|
||||
credential_observed_at: row.2,
|
||||
quota_observed_at: row.3,
|
||||
})
|
||||
})
|
||||
|
||||
@@ -512,7 +512,7 @@ impl RedisProviderLeaseCoordinator {
|
||||
last_started_at: signal.last_started_at.map(Into::into),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Provider account availability 的可重建 Redis cache fence。
|
||||
//! Provider credential state 的可重建 Redis cache fence。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -7,7 +7,7 @@ use chrono::{DateTime, Utc};
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, OpaqueProviderData, ProviderAccountId,
|
||||
CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCredentialState,
|
||||
@@ -24,7 +24,7 @@ if current >= tonumber(ARGV[1]) then return 0 end
|
||||
redis.call('HSET', KEYS[1],
|
||||
'revision', ARGV[1],
|
||||
'enabled', ARGV[2],
|
||||
'availability', ARGV[3],
|
||||
'credential_state', ARGV[3],
|
||||
'observed_at_ms', ARGV[4])
|
||||
redis.call('PEXPIRE', KEYS[1], ARGV[5])
|
||||
return 1
|
||||
@@ -44,7 +44,7 @@ pub struct CredentialStateCache {
|
||||
pub provider_account_id: String,
|
||||
pub revision: Revision,
|
||||
pub enabled: bool,
|
||||
pub availability: String,
|
||||
pub credential_state: String,
|
||||
pub observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -55,8 +55,8 @@ impl CredentialStateCache {
|
||||
"provider_account_id",
|
||||
&self.provider_account_id,
|
||||
)?;
|
||||
if AccountAvailability::parse(&self.availability).is_none() {
|
||||
return Err(invalid("availability is not registered"));
|
||||
if CredentialState::parse(&self.credential_state).is_none() {
|
||||
return Err(invalid("credential_state is not registered"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -172,7 +172,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
|
||||
.key(self.key(&state.provider_account_id)?)
|
||||
.arg(state.revision.get())
|
||||
.arg(u8::from(state.enabled))
|
||||
.arg(&state.availability)
|
||||
.arg(&state.credential_state)
|
||||
.arg(state.observed_at.timestamp_millis())
|
||||
.arg(ttl_ms)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
@@ -195,12 +195,12 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
|
||||
.arg(self.key(provider_account_id)?)
|
||||
.arg("revision")
|
||||
.arg("enabled")
|
||||
.arg("availability")
|
||||
.arg("credential_state")
|
||||
.arg("observed_at_ms")
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("read credential state"))?;
|
||||
let (Some(revision), Some(enabled), Some(availability), Some(observed_at_ms)) = values
|
||||
let (Some(revision), Some(enabled), Some(credential_state), Some(observed_at_ms)) = values
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -208,7 +208,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
|
||||
provider_account_id: provider_account_id.to_owned(),
|
||||
revision: Revision::new(revision)?,
|
||||
enabled: enabled == 1,
|
||||
availability,
|
||||
credential_state,
|
||||
observed_at: DateTime::from_timestamp_millis(observed_at_ms)
|
||||
.ok_or_else(|| invalid("observed_at is invalid"))?,
|
||||
};
|
||||
@@ -299,7 +299,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
|
||||
revision: Revision::new(state.credential_revision().get())
|
||||
.map_err(|_| provider_invalid("encode credential state"))?,
|
||||
enabled: state.enabled(),
|
||||
availability: state.availability().as_str().to_owned(),
|
||||
credential_state: state.credential_state().as_str().to_owned(),
|
||||
observed_at,
|
||||
};
|
||||
CredentialStateRepository::cache_credential_state(
|
||||
@@ -329,13 +329,13 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
|
||||
.map_err(|_| provider_invalid("decode credential state"))?;
|
||||
let revision = CredentialRevision::new(state.revision.get())
|
||||
.map_err(|_| provider_invalid("decode credential state"))?;
|
||||
let availability = AccountAvailability::parse(&state.availability)
|
||||
let credential_state = CredentialState::parse(&state.credential_state)
|
||||
.ok_or_else(|| provider_invalid("decode credential state"))?;
|
||||
Ok(ProviderCredentialState::new(
|
||||
account_id,
|
||||
revision,
|
||||
state.enabled,
|
||||
availability,
|
||||
credential_state,
|
||||
state.observed_at.into(),
|
||||
))
|
||||
})
|
||||
|
||||
@@ -55,8 +55,8 @@ async fn merged_first_attempt_insert_should_match_sequential_semantics() {
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind,
|
||||
provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, enabled, availability,
|
||||
availability_observed_at, created_at, updated_at
|
||||
has_refresh_token, access_token_expires_at, enabled, credential_state,
|
||||
credential_observed_at, created_at, updated_at
|
||||
) values (
|
||||
'acct_merged', 'openai', 'merged', null, 'user-merged', null, null, 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', $1, $1, $1
|
||||
|
||||
@@ -134,26 +134,27 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability,
|
||||
availability_observed_at, created_at, updated_at
|
||||
access_token_expires_at, enabled, credential_state,
|
||||
quota_access_state, quota_evidence, quota_access_observed_at,
|
||||
credential_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_available', 'openai', 'available', 'user-available', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready',
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_usage_limit', 'xai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'usage_limit_reached', $1,
|
||||
$1, $1, $1),
|
||||
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
@@ -173,22 +174,22 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
assert_eq!(
|
||||
(
|
||||
metrics.total,
|
||||
metrics.enabled,
|
||||
metrics.active,
|
||||
metrics.unavailable,
|
||||
metrics.expired,
|
||||
metrics.normal,
|
||||
metrics.quota_exhausted,
|
||||
metrics.rate_limited,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
metrics.error,
|
||||
),
|
||||
(6, 5, 2, 4, 1, 1, 1, 1),
|
||||
(6, 2, 1, 0, 1, 2),
|
||||
);
|
||||
// 首页按调度可用性聚合:配额耗尽也属于不可用。
|
||||
assert_eq!(
|
||||
metrics.unavailable,
|
||||
metrics.quota_exhausted + metrics.expired + metrics.disabled + metrics.banned
|
||||
metrics.total,
|
||||
metrics.normal
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.rate_limited
|
||||
+ metrics.disabled
|
||||
+ metrics.error
|
||||
);
|
||||
assert_eq!(metrics.total, metrics.active + metrics.unavailable);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -202,35 +203,36 @@ async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligibl
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability, quota_limit_reached,
|
||||
availability_observed_at, created_at, updated_at
|
||||
access_token_expires_at, enabled, credential_state,
|
||||
quota_access_state, quota_evidence, quota_access_observed_at,
|
||||
credential_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
|
||||
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
|
||||
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
|
||||
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
|
||||
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
|
||||
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
|
||||
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', 'exhausted', 'provider_denied', $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
|
||||
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', 'exhausted', 'provider_denied', $1,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
|
||||
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
|
||||
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', 'unknown', null, null,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', 'allowed', null, $1,
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
@@ -258,36 +260,21 @@ async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligibl
|
||||
assert_eq!(
|
||||
(
|
||||
metrics.total,
|
||||
metrics.enabled,
|
||||
metrics.active,
|
||||
metrics.normal,
|
||||
metrics.rate_limited,
|
||||
metrics.quota_exhausted,
|
||||
metrics.expired,
|
||||
metrics.invalid,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
metrics.unavailable,
|
||||
metrics.error,
|
||||
),
|
||||
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
|
||||
(9, 1, 2, 3, 1, 2),
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.total,
|
||||
metrics.active
|
||||
metrics.normal
|
||||
+ metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.unavailable,
|
||||
metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
+ metrics.error
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1013,8 +1000,8 @@ async fn seed_observability_facts(
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind,
|
||||
provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, enabled, availability,
|
||||
availability_observed_at, created_at, updated_at
|
||||
has_refresh_token, access_token_expires_at, enabled, credential_state,
|
||||
credential_observed_at, created_at, updated_at
|
||||
) values (
|
||||
'acct_observe', 'openai', 'primary', 'account@example.invalid',
|
||||
'user-observe', null, 'pro', 'oauth', '{}'::jsonb, 1, false, $1 + interval '1 day',
|
||||
|
||||
@@ -10,16 +10,15 @@ use gateway_admin::{
|
||||
SortDirection,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
CredentialAvailabilityFilter, CredentialListQuery, CredentialListWindow,
|
||||
},
|
||||
provider_credentials::{CredentialListQuery, CredentialListWindow, CredentialStateFilter},
|
||||
},
|
||||
ports::store::AccountStore,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, ProviderAccountId,
|
||||
ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, CredentialState, OpaqueProviderData, PlaintextCredential,
|
||||
ProviderAccountId, ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate,
|
||||
QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaState,
|
||||
QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -59,7 +58,7 @@ async fn provider_account_should_allow_missing_upstream_user_id() {
|
||||
let mut pending = account("acct_pending_identity", "unused");
|
||||
pending.upstream_user_id = None;
|
||||
pending.email = None;
|
||||
pending.availability = AccountAvailability::Ready;
|
||||
pending.credential_state = CredentialState::Ready;
|
||||
repository
|
||||
.insert_provider_account(pending)
|
||||
.await
|
||||
@@ -71,8 +70,11 @@ async fn provider_account_should_allow_missing_upstream_user_id() {
|
||||
.expect("load account")
|
||||
.expect("stored account");
|
||||
assert_eq!(
|
||||
(stored.summary.upstream_user_id, stored.summary.availability,),
|
||||
(None, AccountAvailability::Unknown)
|
||||
(
|
||||
stored.summary.upstream_user_id,
|
||||
stored.summary.credential_state,
|
||||
),
|
||||
(None, CredentialState::Unknown)
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
@@ -92,18 +94,19 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
|
||||
}
|
||||
let revision = CredentialRevision::new(1).expect("revision");
|
||||
for (id, remaining) in [("acct_quota_a", 20), ("acct_quota_b", 80)] {
|
||||
let observed_at = SystemTime::now();
|
||||
let outcome = repository
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: ProviderAccountId::new(id).expect("account id"),
|
||||
expected_revision: revision,
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota: OpaqueProviderData::new(
|
||||
json!({"remaining": remaining})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
),
|
||||
observed_at,
|
||||
state: QuotaState::allowed(observed_at),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -123,15 +126,12 @@ async fn core_quota_batch_reads_only_observed_accounts_in_one_contract_call() {
|
||||
assert_eq!(observations.len(), 2);
|
||||
assert_eq!(observations[0].account_id.as_str(), "acct_quota_a");
|
||||
assert_eq!(observations[0].expected_revision, revision);
|
||||
assert_eq!(
|
||||
observations[0]
|
||||
.quota
|
||||
.as_ref()
|
||||
.expect("quota")
|
||||
.expose_to_provider()["remaining"],
|
||||
20
|
||||
assert_eq!(observations[0].quota.expose_to_provider()["remaining"], 20);
|
||||
assert!(
|
||||
observations
|
||||
.iter()
|
||||
.all(|value| value.state.access() == QuotaAccessState::Allowed)
|
||||
);
|
||||
assert!(observations.iter().all(|value| value.observed_at.is_some()));
|
||||
assert_eq!(
|
||||
current_revision(&database.pool).await,
|
||||
1,
|
||||
@@ -163,14 +163,14 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
let quota = |marker: &str, observed_at| QuotaObservation {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota: OpaqueProviderData::new(
|
||||
json!({"marker": marker})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
),
|
||||
observed_at,
|
||||
state: QuotaState::allowed(observed_at),
|
||||
};
|
||||
assert_eq!(
|
||||
repository
|
||||
@@ -192,8 +192,9 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
credential_state: CredentialState::Invalid,
|
||||
observed_at: newer,
|
||||
error_reason: Some(AccountErrorReason::CredentialInvalid),
|
||||
})
|
||||
.await
|
||||
.expect("persist newer account state");
|
||||
@@ -203,8 +204,9 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
credential_state: CredentialState::Ready,
|
||||
observed_at: older,
|
||||
error_reason: None,
|
||||
})
|
||||
.await
|
||||
.is_err(),
|
||||
@@ -216,23 +218,20 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
.await
|
||||
.expect("load fenced account")
|
||||
.expect("fenced account");
|
||||
assert_eq!(current.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert_eq!(current.credential_state(), CredentialState::Invalid);
|
||||
let observed = repository
|
||||
.get_quotas(std::slice::from_ref(&account_id))
|
||||
.await
|
||||
.expect("load fenced quota")
|
||||
.pop()
|
||||
.expect("fenced quota");
|
||||
assert_eq!(
|
||||
observed.quota.expect("quota document").expose_to_provider()["marker"],
|
||||
"newer"
|
||||
);
|
||||
assert_eq!(observed.quota.expose_to_provider()["marker"], "newer");
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
|
||||
async fn credential_error_message_is_persisted_and_cleared_on_recovery() {
|
||||
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
|
||||
return;
|
||||
};
|
||||
@@ -249,8 +248,9 @@ async fn account_state_message_is_persisted_and_cleared_on_recovery() {
|
||||
message: Some("upstream returned 402 payment required".to_owned()),
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
credential_state: CredentialState::Invalid,
|
||||
observed_at: SystemTime::now(),
|
||||
error_reason: Some(AccountErrorReason::CredentialInvalid),
|
||||
})
|
||||
.await
|
||||
.expect("persist error message");
|
||||
@@ -270,8 +270,9 @@ async fn account_state_message_is_persisted_and_cleared_on_recovery() {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
credential_state: CredentialState::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
error_reason: None,
|
||||
})
|
||||
.await
|
||||
.expect("recover account");
|
||||
@@ -299,39 +300,34 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
let mut beta = account("acct_beta", "user-beta");
|
||||
beta.provider_kind = "xai".to_owned();
|
||||
beta.email = Some("beta@example.invalid".to_owned());
|
||||
beta.availability = AccountAvailability::Banned;
|
||||
beta.credential_state = CredentialState::Banned;
|
||||
let mut charlie = account("acct_charlie", "user-charlie");
|
||||
charlie.email = Some("charlie@example.invalid".to_owned());
|
||||
let mut invalid = account("acct_invalid", "user-invalid");
|
||||
invalid.email = Some("invalid@example.invalid".to_owned());
|
||||
invalid.availability = AccountAvailability::Invalid;
|
||||
invalid.credential_state = CredentialState::Invalid;
|
||||
let mut disabled = account("acct_disabled", "user-disabled");
|
||||
disabled.email = Some("disabled@example.invalid".to_owned());
|
||||
disabled.enabled = false;
|
||||
let mut quota_exhausted = account("acct_quota_exhausted", "user-quota-exhausted");
|
||||
quota_exhausted.email = Some("quota-exhausted@example.invalid".to_owned());
|
||||
quota_exhausted.availability = AccountAvailability::QuotaExhausted;
|
||||
for account in [alpha, beta, charlie, invalid, disabled, quota_exhausted] {
|
||||
repository
|
||||
.insert_provider_account(account)
|
||||
.await
|
||||
.expect("insert account list fixture");
|
||||
}
|
||||
repository
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: ProviderAccountId::new("acct_charlie").expect("account id"),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({"rate_limit": {"limit_reached": true}})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed rate-limited account");
|
||||
for account_id in ["acct_charlie", "acct_quota_exhausted"] {
|
||||
let observed_at = SystemTime::now();
|
||||
repository
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: ProviderAccountId::new(account_id).expect("account id"),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
})
|
||||
.await
|
||||
.expect("seed exhausted account");
|
||||
}
|
||||
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
@@ -425,7 +421,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
usage_page
|
||||
.items
|
||||
.iter()
|
||||
.map(|account| account.id.as_str())
|
||||
.map(|item| item.account.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["acct_charlie", "acct_beta"]
|
||||
);
|
||||
@@ -448,7 +444,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
last_used_page
|
||||
.items
|
||||
.iter()
|
||||
.map(|account| account.id.as_str())
|
||||
.map(|item| item.account.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["acct_beta", "acct_charlie"]
|
||||
);
|
||||
@@ -466,8 +462,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.expect("filter account directory");
|
||||
assert_eq!(filtered.total, 1);
|
||||
assert_eq!(filtered.summary, usage_page.summary);
|
||||
assert_eq!(filtered.items[0].id, "acct_alpha");
|
||||
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
|
||||
assert_eq!(filtered.items[0].account.id, "acct_alpha");
|
||||
assert_eq!(filtered.items[0].account.provider_kind.as_str(), "openai");
|
||||
|
||||
let error_accounts = store
|
||||
.list_accounts(AccountListQuery {
|
||||
@@ -481,8 +477,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("filter error accounts");
|
||||
assert_eq!(error_accounts.items.len(), 2);
|
||||
assert_eq!(error_accounts.items[0].id, "acct_beta");
|
||||
assert_eq!(error_accounts.items[1].id, "acct_invalid");
|
||||
assert_eq!(error_accounts.items[0].account.id, "acct_beta");
|
||||
assert_eq!(error_accounts.items[1].account.id, "acct_invalid");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -498,11 +494,11 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
&format!("user-xai-{index:03}"),
|
||||
);
|
||||
credential.provider_kind = "xai".to_owned();
|
||||
credential.availability = match index {
|
||||
0 => AccountAvailability::Expired,
|
||||
1 => AccountAvailability::Banned,
|
||||
2 => AccountAvailability::Invalid,
|
||||
_ => AccountAvailability::Ready,
|
||||
credential.credential_state = match index {
|
||||
0 => CredentialState::Expired,
|
||||
1 => CredentialState::Banned,
|
||||
2 => CredentialState::Invalid,
|
||||
_ => CredentialState::Ready,
|
||||
};
|
||||
repository
|
||||
.insert_provider_account(credential)
|
||||
@@ -516,7 +512,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.list_credentials(
|
||||
&provider,
|
||||
CredentialListQuery {
|
||||
availability: None,
|
||||
credential_state: None,
|
||||
enabled: None,
|
||||
window: CredentialListWindow::All,
|
||||
},
|
||||
@@ -530,7 +526,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.list_credentials(
|
||||
&provider,
|
||||
CredentialListQuery {
|
||||
availability: None,
|
||||
credential_state: None,
|
||||
enabled: None,
|
||||
window: CredentialListWindow::Page {
|
||||
cursor: None,
|
||||
@@ -547,10 +543,10 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.list_credentials(
|
||||
&provider,
|
||||
CredentialListQuery {
|
||||
availability: Some(CredentialAvailabilityFilter::AnyOf(vec![
|
||||
gateway_admin::model::accounts::AccountAvailability::Expired,
|
||||
gateway_admin::model::accounts::AccountAvailability::Banned,
|
||||
gateway_admin::model::accounts::AccountAvailability::Invalid,
|
||||
credential_state: Some(CredentialStateFilter::AnyOf(vec![
|
||||
CredentialState::Expired,
|
||||
CredentialState::Banned,
|
||||
CredentialState::Invalid,
|
||||
])),
|
||||
enabled: None,
|
||||
window: CredentialListWindow::All,
|
||||
@@ -851,7 +847,7 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
let mut updated = account("acct_admin_reimport", "user-admin-upsert");
|
||||
updated.name = "updated import".to_owned();
|
||||
updated.provider_credentials_json = credential_json("updated-import-secret");
|
||||
updated.availability = AccountAvailability::Banned;
|
||||
updated.credential_state = CredentialState::Banned;
|
||||
let imported = repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -871,7 +867,7 @@ async fn admin_import_updates_the_same_verified_identity_without_rebinding_it()
|
||||
String,
|
||||
) = sqlx::query_as(
|
||||
"select name, provider_credentials_json, credential_revision, provider_quota_json,
|
||||
availability
|
||||
credential_state
|
||||
from provider_accounts where id = 'acct_admin_upsert'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
@@ -920,8 +916,8 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() {
|
||||
access_token_expires_at: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: Utc::now(),
|
||||
credential_state: CredentialState::Ready,
|
||||
credential_observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
.expect("seed provider account");
|
||||
@@ -1041,32 +1037,32 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
provider_kind: "openai".to_owned(),
|
||||
};
|
||||
|
||||
let mut cooldown_account = account("acct_admin_b", "user-admin-b");
|
||||
cooldown_account.availability = AccountAvailability::Ready;
|
||||
let imported = repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
accounts: vec![account("acct_admin_a", "user-admin-a"), cooldown_account],
|
||||
accounts: vec![
|
||||
account("acct_admin_a", "user-admin-a"),
|
||||
account("acct_admin_b", "user-admin-b"),
|
||||
],
|
||||
audit: audit("audit_account_batch", "import_batch", "provider_accounts"),
|
||||
})
|
||||
.await
|
||||
.expect("import provider accounts");
|
||||
assert_eq!(imported.config_revision.get(), 2);
|
||||
let ready: (bool, String) =
|
||||
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
|
||||
sqlx::query_as("select enabled, credential_state from provider_accounts where id = $1")
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load ready imported account");
|
||||
assert_eq!(ready, (true, "ready".to_owned()));
|
||||
let cooldown: (bool, String) =
|
||||
sqlx::query_as("select enabled, availability from provider_accounts where id = $1")
|
||||
let second: (bool, String) =
|
||||
sqlx::query_as("select enabled, credential_state from provider_accounts where id = $1")
|
||||
.bind("acct_admin_b")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load cooldown imported account");
|
||||
assert!(cooldown.0);
|
||||
assert_eq!(cooldown.1, "ready");
|
||||
assert_eq!(second, (true, "ready".to_owned()));
|
||||
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
@@ -1113,7 +1109,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = 'expired'
|
||||
set credential_state = 'expired'
|
||||
where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
@@ -1137,7 +1133,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
assert_eq!(rotation.config_revision.get(), 3);
|
||||
assert_eq!(rotation.credential_revision.get(), 2);
|
||||
let restored: (String, String, Option<String>) = sqlx::query_as(
|
||||
"select availability, upstream_user_id, upstream_account_id
|
||||
"select credential_state, upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
@@ -1243,12 +1239,10 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
let scope = ProviderAccountAdminScope {
|
||||
provider_kind: "openai".to_owned(),
|
||||
};
|
||||
let mut exhausted = account("acct_rotation_quota", "user-rotation-quota");
|
||||
exhausted.availability = AccountAvailability::QuotaExhausted;
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
accounts: vec![exhausted],
|
||||
accounts: vec![account("acct_rotation_quota", "user-rotation-quota")],
|
||||
audit: audit(
|
||||
"audit_rotation_quota_import",
|
||||
"import",
|
||||
@@ -1257,6 +1251,15 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
})
|
||||
.await
|
||||
.expect("import exhausted account");
|
||||
let observed_at = SystemTime::now();
|
||||
repository
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: ProviderAccountId::new("acct_rotation_quota").expect("account ID"),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota exhaustion");
|
||||
|
||||
repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
@@ -1273,13 +1276,13 @@ async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
.await
|
||||
.expect("rotate exhausted account credential");
|
||||
|
||||
let availability: String =
|
||||
sqlx::query_scalar("select availability from provider_accounts where id = $1")
|
||||
let quota_access_state: String =
|
||||
sqlx::query_scalar("select quota_access_state from provider_accounts where id = $1")
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(availability, "quota_exhausted".to_owned());
|
||||
assert_eq!(quota_access_state, "exhausted".to_owned());
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1298,26 +1301,25 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
|
||||
.await
|
||||
.expect("insert xAI account");
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::exhausted(QuotaEvidence::AccountLimitReached, observed_at, None),
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI free quota exhaustion");
|
||||
|
||||
let current: (String, String) = sqlx::query_as(
|
||||
"select provider_kind, availability
|
||||
"select provider_kind, quota_access_state
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI quota state");
|
||||
assert_eq!(current, ("xai".to_owned(), "quota_exhausted".to_owned()),);
|
||||
assert_eq!(current, ("xai".to_owned(), "exhausted".to_owned()),);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1338,25 +1340,29 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at,
|
||||
state: QuotaState::exhausted(
|
||||
QuotaEvidence::UsageLimitReached,
|
||||
observed_at,
|
||||
Some(observed_at + Duration::from_secs(60)),
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("persist xAI resettable usage limit");
|
||||
|
||||
let current: (String, String) = sqlx::query_as(
|
||||
"select provider_kind, availability
|
||||
let current: (String, String, Option<chrono::DateTime<Utc>>) = sqlx::query_as(
|
||||
"select provider_kind, quota_access_state, quota_reset_at
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load persisted xAI usage limit state");
|
||||
assert_eq!(current, ("xai".to_owned(), "ready".to_owned()));
|
||||
assert_eq!(current.0, "xai");
|
||||
assert_eq!(current.1, "exhausted");
|
||||
assert!(current.2.is_some());
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1373,7 +1379,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
let account_id = ProviderAccountId::new("acct_disabled_refresh").expect("account ID");
|
||||
let mut disabled = account(account_id.as_str(), "user-disabled-refresh");
|
||||
disabled.enabled = false;
|
||||
disabled.availability = AccountAvailability::Expired;
|
||||
disabled.credential_state = CredentialState::Expired;
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
@@ -1392,8 +1398,9 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
credential_state: CredentialState::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
error_reason: None,
|
||||
})
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
@@ -1413,7 +1420,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
.expect("refresh disabled account credential");
|
||||
|
||||
let current: (bool, String, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, provider_credentials_json, credential_revision
|
||||
"select enabled, credential_state, provider_credentials_json, credential_revision
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
@@ -1443,8 +1450,8 @@ fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
|
||||
access_token_expires_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
next_refresh_at: None,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: Utc::now(),
|
||||
credential_state: CredentialState::Ready,
|
||||
credential_observed_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1570,13 +1577,6 @@ fn provider_credentials_are_redacted_from_debug() {
|
||||
assert!(!format!("{object:?}").contains(secret));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_exhausted_account_accepts_cooldown_runtime_facts() {
|
||||
let mut imported = account("acct_quota_cooldown", "user-quota-cooldown");
|
||||
imported.availability = AccountAvailability::QuotaExhausted;
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_ready_account_passes_validation() {
|
||||
let imported = account("acct_ready_import", "user-ready-import");
|
||||
|
||||
@@ -108,7 +108,7 @@ async fn insert_refreshable_account(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, has_refresh_token, access_token_expires_at,
|
||||
next_refresh_at, availability, availability_observed_at, created_at, updated_at
|
||||
next_refresh_at, credential_state, credential_observed_at, created_at, updated_at
|
||||
) values ($1, 'openai', $1, $1, 'oauth', '{}'::jsonb, true, $2, $3,
|
||||
'ready', now(), now(), now())",
|
||||
)
|
||||
|
||||
@@ -722,8 +722,8 @@ async fn seed_account(
|
||||
id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind,
|
||||
provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, enabled, availability,
|
||||
availability_observed_at, created_at, updated_at
|
||||
has_refresh_token, access_token_expires_at, enabled, credential_state,
|
||||
credential_observed_at, created_at, updated_at
|
||||
) values (
|
||||
$1, 'openai', $2, $3, 'user-' || $1, null, null, $4,
|
||||
'{}'::jsonb, 1, false, $5 + interval '1 day', true, 'ready',
|
||||
|
||||
@@ -19,7 +19,7 @@ fn credential_state_rejects_provider_specific_status() {
|
||||
provider_account_id: "account-1".to_owned(),
|
||||
revision: Revision::new(1).expect("positive revision"),
|
||||
enabled: true,
|
||||
availability: "codex_special".to_owned(),
|
||||
credential_state: "codex_special".to_owned(),
|
||||
observed_at: Utc::now(),
|
||||
};
|
||||
assert!(state.validate().is_err());
|
||||
|
||||
@@ -25,9 +25,8 @@ use gateway_admin::model::{MutationActor, MutationContext, Revision};
|
||||
use gateway_admin::ports::provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind};
|
||||
use gateway_core::accounting::Money;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, LoadedCredential, NewProviderAccount,
|
||||
OpaqueProviderData, PlaintextCredential, ProviderAccount, ProviderAccountId,
|
||||
ProviderAccountStore,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData,
|
||||
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
|
||||
@@ -424,14 +423,7 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
};
|
||||
let mut quota = project_quota(snapshot, &account);
|
||||
quota.rate_limited_until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
.map(DateTime::<Utc>::from);
|
||||
Ok(quota)
|
||||
Ok(project_quota(snapshot, &account))
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -519,7 +511,6 @@ fn prepared_create(
|
||||
account,
|
||||
credential,
|
||||
} = prepared;
|
||||
let availability = account.availability();
|
||||
Ok(PreparedCredentialCreate {
|
||||
account_id: account.id().clone(),
|
||||
provider_kind: account.provider().clone(),
|
||||
@@ -534,8 +525,8 @@ fn prepared_create(
|
||||
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability,
|
||||
availability_observed_at: observed_at,
|
||||
credential_state: account.credential_state(),
|
||||
credential_observed_at: observed_at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -627,7 +618,13 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
|
||||
account.upstream_account_id.clone(),
|
||||
account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(account.enabled, account.availability)
|
||||
.with_account_facts(
|
||||
account.enabled,
|
||||
account.credential_state,
|
||||
account.quota,
|
||||
account.last_error_reason,
|
||||
account.last_error_message.clone(),
|
||||
)
|
||||
.with_refresh_schedule(
|
||||
account.has_refresh_token,
|
||||
account.next_refresh_at.map(SystemTime::from),
|
||||
@@ -687,7 +684,6 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -696,13 +692,10 @@ fn project_quota(
|
||||
snapshot: Option<CodexAccountQuotaSnapshot>,
|
||||
account: &ProviderAccount,
|
||||
) -> ProviderQuota {
|
||||
// 展示前先滚动已过期窗口,避免 reset 后的旧窗口值悬挂。
|
||||
let mut quota = snapshot
|
||||
.map(|snapshot| project_quota_snapshot(snapshot.roll_expired_windows(SystemTime::now())))
|
||||
.map(project_quota_snapshot)
|
||||
.unwrap_or_else(empty_quota);
|
||||
if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
force_confirmed_exhaustion_projection(&mut quota);
|
||||
}
|
||||
quota.limit_reached = account.quota().is_exhausted();
|
||||
quota
|
||||
}
|
||||
|
||||
@@ -717,7 +710,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
);
|
||||
provider_data.insert(
|
||||
"exhausted".to_owned(),
|
||||
Value::Bool(snapshot.fact().exhausted()),
|
||||
Value::Bool(snapshot.quota().is_exhausted()),
|
||||
);
|
||||
let windows: Vec<ProviderQuotaWindow> = snapshot
|
||||
.windows()
|
||||
@@ -766,7 +759,6 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
refresh_token_expires_at: None,
|
||||
windows,
|
||||
limit_reached,
|
||||
rate_limited_until: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
provider_data,
|
||||
))),
|
||||
@@ -788,52 +780,6 @@ fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool {
|
||||
windows.iter().any(|window| window.limit_reached)
|
||||
}
|
||||
|
||||
fn force_confirmed_exhaustion_projection(quota: &mut ProviderQuota) {
|
||||
// 402 已确认耗尽:只覆盖 Admin 展示投影,原始 Provider JSON 仍作为
|
||||
// 后续判断窗口是否真正恢复的基线。
|
||||
quota.limit_reached = true;
|
||||
quota.provider_data = Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
Map::from_iter([
|
||||
(
|
||||
"remaining_percent".to_owned(),
|
||||
Value::Number(Number::from(0)),
|
||||
),
|
||||
("exhausted".to_owned(), Value::Bool(true)),
|
||||
]),
|
||||
)));
|
||||
if quota.windows.is_empty() {
|
||||
quota.windows.push(ProviderQuotaWindow {
|
||||
key: "confirmed-quota-exhaustion".to_owned(),
|
||||
group: "other".to_owned(),
|
||||
label: "额度".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
|
||||
window_seconds: None,
|
||||
used_percent: Some(100.0),
|
||||
reset_at: None,
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 选已用比例最高的窗口强制 100%,保留其自身 reset_at。
|
||||
let index = quota
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, left), (_, right)| {
|
||||
left.used_percent
|
||||
.unwrap_or(-1.0)
|
||||
.total_cmp(&right.used_percent.unwrap_or(-1.0))
|
||||
})
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
quota.windows[index].used_percent = Some(100.0);
|
||||
quota.windows[index].limit_reached = true;
|
||||
}
|
||||
|
||||
const fn quota_group(kind: CodexQuotaWindowKind) -> &'static str {
|
||||
match kind {
|
||||
CodexQuotaWindowKind::Monthly => "monthly",
|
||||
|
||||
@@ -68,11 +68,6 @@ impl OpenAiConfig {
|
||||
self.quota.refresh_policy()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn quota_skip_exhausted(&self) -> bool {
|
||||
self.quota.skip_exhausted
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn token_client_config(&self) -> TokenClientConfig {
|
||||
TokenClientConfig {
|
||||
@@ -209,14 +204,12 @@ impl CodexWebSocketPoolSettings {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CodexQuotaSettings {
|
||||
pub refresh_interval_minutes: u64,
|
||||
pub skip_exhausted: bool,
|
||||
}
|
||||
|
||||
impl Default for CodexQuotaSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
refresh_interval_minutes: 15,
|
||||
skip_exhausted: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@ use std::time::SystemTime;
|
||||
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential,
|
||||
AccountErrorReason, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential,
|
||||
NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountIdentity,
|
||||
ProviderAccountUpdate,
|
||||
ProviderAccountUpdate, QuotaEvidence, QuotaState,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest,
|
||||
@@ -433,7 +433,13 @@ impl CodexCredentialAdmin {
|
||||
Some(input.verified_account.chatgpt_account_id),
|
||||
input.verified_account.plan_type,
|
||||
)
|
||||
.with_runtime_state(input.enabled, AccountAvailability::Ready)
|
||||
.with_account_facts(
|
||||
input.enabled,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.with_refresh_schedule(
|
||||
input.secret.refresh_token.is_some(),
|
||||
optional_time(input.next_refresh_at),
|
||||
@@ -479,13 +485,16 @@ impl CodexCredentialAdmin {
|
||||
optional_time(input.access_token_expires_at),
|
||||
)
|
||||
.with_profile(email, upstream_account_id, plan_type)
|
||||
.with_runtime_state(
|
||||
.with_account_facts(
|
||||
input.enabled,
|
||||
if has_upstream_user_id {
|
||||
AccountAvailability::Ready
|
||||
CredentialState::Ready
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
CredentialState::Unknown
|
||||
},
|
||||
QuotaState::unknown(),
|
||||
(!has_upstream_user_id).then_some(AccountErrorReason::AccountUnverified),
|
||||
None,
|
||||
)
|
||||
.with_refresh_schedule(
|
||||
input.secret.refresh_token.is_some(),
|
||||
@@ -898,10 +907,10 @@ impl CodexCredentialAdminService {
|
||||
if !account_ids.insert(account_id.clone()) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let (enabled, availability) =
|
||||
import_runtime_state(candidate.status.as_deref())?;
|
||||
let facts =
|
||||
parse_cpr_account_facts(candidate.status.as_deref(), SystemTime::now())?;
|
||||
let mut account =
|
||||
self.prepare_agent_identity_import(&candidate, account_id, enabled)?;
|
||||
self.prepare_agent_identity_import(&candidate, account_id, facts.enabled)?;
|
||||
let upstream_user_id = account
|
||||
.account
|
||||
.upstream_user_id()
|
||||
@@ -915,7 +924,7 @@ impl CodexCredentialAdminService {
|
||||
if !upstream_identities.insert((upstream_user_id, upstream_account_id)) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
account.account = account.account.with_runtime_state(enabled, availability);
|
||||
account.account = facts.apply(account.account);
|
||||
account
|
||||
}
|
||||
};
|
||||
@@ -1048,7 +1057,13 @@ impl CodexCredentialAdminService {
|
||||
Some(upstream_account_id),
|
||||
candidate.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(enabled, AccountAvailability::Ready)
|
||||
.with_account_facts(
|
||||
enabled,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.with_refresh_schedule(false, None);
|
||||
Ok(NewProviderAccount {
|
||||
account,
|
||||
@@ -1065,11 +1080,13 @@ fn cpr_status(account: &ProviderAccount) -> &'static str {
|
||||
if !account.enabled() {
|
||||
return "disabled";
|
||||
}
|
||||
match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => "quota_exhausted",
|
||||
AccountAvailability::Expired | AccountAvailability::Invalid => "expired",
|
||||
AccountAvailability::Banned => "banned",
|
||||
AccountAvailability::Unknown | AccountAvailability::Ready => "active",
|
||||
if account.quota().is_exhausted() {
|
||||
return "quota_exhausted";
|
||||
}
|
||||
match account.credential_state() {
|
||||
CredentialState::Expired | CredentialState::Invalid => "expired",
|
||||
CredentialState::Banned => "banned",
|
||||
CredentialState::Unknown | CredentialState::Ready => "active",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1292,20 +1309,65 @@ fn normalize_import_text(value: &str, max_bytes: usize) -> Option<String> {
|
||||
.then(|| value.to_owned())
|
||||
}
|
||||
|
||||
fn import_runtime_state(
|
||||
struct ImportedCprAccountFacts {
|
||||
enabled: bool,
|
||||
credential_state: CredentialState,
|
||||
quota: QuotaState,
|
||||
error_reason: Option<AccountErrorReason>,
|
||||
}
|
||||
|
||||
impl ImportedCprAccountFacts {
|
||||
fn apply(self, account: ProviderAccount) -> ProviderAccount {
|
||||
account.with_account_facts(
|
||||
self.enabled,
|
||||
self.credential_state,
|
||||
self.quota,
|
||||
self.error_reason,
|
||||
None,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_cpr_account_facts(
|
||||
status: Option<&str>,
|
||||
) -> Result<(bool, AccountAvailability), CodexCredentialAdminError> {
|
||||
observed_at: SystemTime,
|
||||
) -> Result<ImportedCprAccountFacts, CodexCredentialAdminError> {
|
||||
match status
|
||||
.map(str::trim)
|
||||
.unwrap_or("active")
|
||||
.to_ascii_lowercase()
|
||||
.as_str()
|
||||
{
|
||||
"active" => Ok((true, AccountAvailability::Ready)),
|
||||
"disabled" => Ok((false, AccountAvailability::Ready)),
|
||||
"expired" => Ok((true, AccountAvailability::Expired)),
|
||||
"quota_exhausted" => Ok((true, AccountAvailability::QuotaExhausted)),
|
||||
"banned" => Ok((true, AccountAvailability::Banned)),
|
||||
"active" => Ok(ImportedCprAccountFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
quota: QuotaState::unknown(),
|
||||
error_reason: None,
|
||||
}),
|
||||
"disabled" => Ok(ImportedCprAccountFacts {
|
||||
enabled: false,
|
||||
credential_state: CredentialState::Ready,
|
||||
quota: QuotaState::unknown(),
|
||||
error_reason: None,
|
||||
}),
|
||||
"expired" => Ok(ImportedCprAccountFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Expired,
|
||||
quota: QuotaState::unknown(),
|
||||
error_reason: Some(AccountErrorReason::CredentialExpired),
|
||||
}),
|
||||
"quota_exhausted" => Ok(ImportedCprAccountFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
quota: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
error_reason: None,
|
||||
}),
|
||||
"banned" => Ok(ImportedCprAccountFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Banned,
|
||||
quota: QuotaState::unknown(),
|
||||
error_reason: Some(AccountErrorReason::AccountBanned),
|
||||
}),
|
||||
_ => Err(CodexCredentialAdminError::InvalidInput),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, OpaqueProviderData, ProviderAccount, ProviderAccountId,
|
||||
CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::engine::provider::ProviderCatalogGeneration;
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -780,12 +780,8 @@ fn eligible_catalog_account(account: &ProviderAccount, now: SystemTime) -> bool
|
||||
&& account
|
||||
.access_token_expires_at()
|
||||
.is_none_or(|expires_at| expires_at > now)
|
||||
&& match account.availability() {
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::QuotaExhausted => true,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => false,
|
||||
}
|
||||
&& matches!(
|
||||
account.credential_state(),
|
||||
CredentialState::Unknown | CredentialState::Ready
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
//! 额度接口失败到规范化账号事实的唯一分类边界。
|
||||
|
||||
use gateway_core::engine::credential::{AccountErrorReason, CredentialState, QuotaEvidence};
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::transport::CodexClientError;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum QuotaEndpointFailure {
|
||||
Credential {
|
||||
state: CredentialState,
|
||||
reason: AccountErrorReason,
|
||||
},
|
||||
Exhausted(QuotaEvidence),
|
||||
}
|
||||
|
||||
/// 额度 endpoint 只有明确 402 能改变账号事实。401/403、429、5xx 与传输失败
|
||||
/// 均不足以判断凭据或账号状态,留给 OAuth 与真实推理请求确认。
|
||||
pub(crate) fn classify_quota_endpoint_failure(
|
||||
error: &CodexClientError,
|
||||
) -> Option<QuotaEndpointFailure> {
|
||||
let CodexClientError::Upstream { status, body, .. } = error else {
|
||||
return None;
|
||||
};
|
||||
if *status != StatusCode::PAYMENT_REQUIRED {
|
||||
return None;
|
||||
}
|
||||
if is_deactivated_workspace(body) {
|
||||
return Some(QuotaEndpointFailure::Credential {
|
||||
state: CredentialState::Banned,
|
||||
reason: AccountErrorReason::AccountBanned,
|
||||
});
|
||||
}
|
||||
Some(QuotaEndpointFailure::Exhausted(
|
||||
QuotaEvidence::PaymentRequired,
|
||||
))
|
||||
}
|
||||
|
||||
fn is_deactivated_workspace(body: &str) -> bool {
|
||||
serde_json::from_str::<Value>(body)
|
||||
.ok()
|
||||
.is_some_and(|value| {
|
||||
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
|
||||
})
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
//! Codex quota 服务编排:请求级额度投影、被动/主动同步与 429 合成。
|
||||
//! Codex quota 服务编排:权威额度事实、展示投影、主动/被动同步与 429 冷却。
|
||||
//!
|
||||
//! - [`snapshot`]:快照/窗口解析、聚合、O(1) 滚动与调度信号。
|
||||
//! - [`recovery`]:402 恢复证据与 availability 转换。
|
||||
//! - [`evidence`]:额度接口错误到凭据/额度事实的分类。
|
||||
|
||||
pub(crate) mod recovery;
|
||||
pub(crate) mod evidence;
|
||||
pub(crate) mod snapshot;
|
||||
|
||||
pub use snapshot::{
|
||||
@@ -13,12 +13,14 @@ pub use snapshot::{
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, QuotaWriteOutcome,
|
||||
AccountErrorReason, AccountQuotaSignals, CredentialRevision, CredentialState,
|
||||
OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaState,
|
||||
QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
|
||||
use gateway_protocol::openai::events::{
|
||||
@@ -36,10 +38,7 @@ use crate::transport::{CodexBackendClient, CodexClientError, CodexRequestContext
|
||||
|
||||
use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeCredential};
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use recovery::{
|
||||
authoritative_quota_snapshot_state, quota_state_reason, quota_state_transition,
|
||||
quota_success_state,
|
||||
};
|
||||
use evidence::{QuotaEndpointFailure, classify_quota_endpoint_failure};
|
||||
use snapshot::{
|
||||
parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation,
|
||||
scheduling_signals_from_snapshot,
|
||||
@@ -48,7 +47,8 @@ use snapshot::{
|
||||
const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
pub(crate) const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5);
|
||||
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
/// 首次 OAuth 异步观察失败时,由既有 quota worker 兜底重试的单轮上限。
|
||||
const INITIAL_QUOTA_SYNC_BATCH: usize = 100;
|
||||
// 5xx 上游拒绝的短退避重试预算;指数退避 1s/2s,吞掉瞬时抖动。
|
||||
@@ -130,6 +130,12 @@ pub struct CodexCredentialQuotaService {
|
||||
scheduling: CodexQuotaSchedulingProjection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum QuotaRefreshAuthority {
|
||||
ObserveAccess,
|
||||
PreserveAccess,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CodexQuotaSchedulingProjection {
|
||||
state: Arc<RwLock<CodexQuotaProjectionState>>,
|
||||
@@ -299,12 +305,7 @@ impl CodexQuotaSchedulingProjection {
|
||||
) -> Vec<ProviderAccount> {
|
||||
let candidates = accounts
|
||||
.into_iter()
|
||||
.filter_map(|account| {
|
||||
let limit_reached = self
|
||||
.signals(&account)
|
||||
.is_some_and(|signals| signals.limit_reached());
|
||||
quota_refresh_candidate(account, now, limit_reached)
|
||||
})
|
||||
.filter_map(|account| quota_refresh_candidate(account, now))
|
||||
.collect::<Vec<_>>();
|
||||
let candidate_ids = candidates
|
||||
.iter()
|
||||
@@ -319,8 +320,8 @@ impl CodexQuotaSchedulingProjection {
|
||||
.last_periodic_refresh_at
|
||||
.retain(|account_id, _| candidate_ids.contains(account_id));
|
||||
|
||||
// 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只兜底
|
||||
// 已耗尽账号,且不依据 reset 时间跳过复核;每个账号仍至少间隔 30 分钟。
|
||||
// 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只复核
|
||||
// 已耗尽且已到 reset/fallback 时刻的账号;重复复核至少间隔 30 分钟。
|
||||
let mut reserved = Vec::new();
|
||||
for account in candidates {
|
||||
if !periodic_quota_refresh_due(&state, account.id(), refreshed_at) {
|
||||
@@ -335,16 +336,13 @@ impl CodexQuotaSchedulingProjection {
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_refresh_candidate(
|
||||
account: ProviderAccount,
|
||||
now: SystemTime,
|
||||
limit_reached: bool,
|
||||
) -> Option<ProviderAccount> {
|
||||
// 定时器只兜底已耗尽或任一窗口触顶的账号;
|
||||
// 正常账号由真实请求的响应头被动同步。
|
||||
fn quota_refresh_candidate(account: ProviderAccount, now: SystemTime) -> Option<ProviderAccount> {
|
||||
// reset 到期是“应刷新”的信号,不是“已恢复”的证据。
|
||||
(eligible_periodic_quota_refresh(&account, now)
|
||||
&& (account.availability() == AccountAvailability::QuotaExhausted || limit_reached))
|
||||
.then_some(account)
|
||||
&& account
|
||||
.quota()
|
||||
.exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL))
|
||||
.then_some(account)
|
||||
}
|
||||
|
||||
fn periodic_quota_refresh_due(
|
||||
@@ -356,7 +354,7 @@ fn periodic_quota_refresh_due(
|
||||
.last_periodic_refresh_at
|
||||
.get(account_id)
|
||||
.is_none_or(|last| {
|
||||
now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL
|
||||
now.saturating_duration_since(*last) >= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL
|
||||
})
|
||||
}
|
||||
|
||||
@@ -400,70 +398,55 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 写入新快照前读取 core primary 已确认耗尽窗口的 reset,用于判断该窗口是否真的推进。
|
||||
async fn previous_confirmed_reset(&self, account: &ProviderAccount) -> Option<SystemTime> {
|
||||
let observations = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.ok()?;
|
||||
let observation = observations.into_iter().find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})?;
|
||||
let snapshot = quota_snapshot_from_observation(&observation)?;
|
||||
snapshot
|
||||
.core_primary_window()
|
||||
.and_then(|window| window.reset_at())
|
||||
.map(SystemTime::from)
|
||||
}
|
||||
|
||||
/// 将已确认的额度拒绝投影为 100%,使展示、调度和恢复共用同一窗口事实。
|
||||
///
|
||||
/// `reset_at` 来自官方 `usage_limit_reached` 错误体;缺失时保留当前快照的 reset。
|
||||
/// 真实推理错误只更新额度访问事实,不伪造 Provider JSON 或展示百分比。
|
||||
pub(crate) async fn record_confirmed_exhaustion(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
reset_at: Option<i64>,
|
||||
evidence: QuotaEvidence,
|
||||
reset_at: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let existing = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
let quota = confirmed_exhaustion_projection(
|
||||
normalize_quota_window_placeholders(existing),
|
||||
reset_at,
|
||||
);
|
||||
let observed_at = SystemTime::now();
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(quota.clone())),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(true),
|
||||
state: QuotaState::exhausted(evidence, observed_at, reset_at),
|
||||
})
|
||||
.await?;
|
||||
if outcome == QuotaWriteOutcome::Conflict {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
if let Ok(snapshot) = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(quota),
|
||||
) {
|
||||
self.scheduling.observe(&snapshot);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 成功推理是额度可访问的权威证据,同时解除账号级 429 冷却。
|
||||
pub(crate) async fn record_successful_inference(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
// 已经发出的并发请求可能晚于耗尽事实成功返回;真实成功仍是独立的
|
||||
// Allowed 权威证据,但调度不会为了试探恢复而放行耗尽账号。
|
||||
if account.quota().access() == QuotaAccessState::Exhausted {
|
||||
let outcome = self
|
||||
.store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
state: QuotaState::allowed(observed_at),
|
||||
})
|
||||
.await?;
|
||||
if outcome == QuotaWriteOutcome::Conflict {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
}
|
||||
self.cooldowns
|
||||
.clear(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(|error| CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -562,30 +545,27 @@ impl CodexCredentialQuotaService {
|
||||
);
|
||||
}
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
match quota_state_transition(&error) {
|
||||
// 402:真额度耗尽,写入 QuotaExhausted(worker 按 reset_at 自动恢复)。
|
||||
Some(AccountAvailability::QuotaExhausted) => {
|
||||
match classify_quota_endpoint_failure(&error) {
|
||||
Some(QuotaEndpointFailure::Exhausted(evidence)) => {
|
||||
summary.exhausted += 1;
|
||||
self.persist_upstream_state(
|
||||
&account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
&error,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
if let Err(write_error) = self
|
||||
.record_confirmed_exhaustion(&account, evidence, None, observed_at)
|
||||
.await
|
||||
{
|
||||
summary.transient += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %write_error,
|
||||
"OpenAI quota exhaustion fact write failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Some(AccountAvailability::Banned) => {
|
||||
Some(QuotaEndpointFailure::Credential { state, reason }) => {
|
||||
summary.banned += 1;
|
||||
self.persist_upstream_state(
|
||||
&account,
|
||||
AccountAvailability::Banned,
|
||||
&error,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
self.persist_credential_failure(&account, state, reason, observed_at)
|
||||
.await;
|
||||
}
|
||||
// usage 401/403 与 429/503 都不写 availability;前者等待 OAuth 或真实请求确认。
|
||||
None | Some(_) => {
|
||||
None => {
|
||||
summary.transient += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
@@ -613,7 +593,7 @@ impl CodexCredentialQuotaService {
|
||||
let observed = self.store.get_quotas(&account_ids).await?;
|
||||
let observed_ids = observed
|
||||
.into_iter()
|
||||
.filter_map(|observation| observation.observed_at.map(|_| observation.account_id))
|
||||
.map(|observation| observation.account_id)
|
||||
.collect::<BTreeSet<_>>();
|
||||
Ok(accounts
|
||||
.iter()
|
||||
@@ -633,79 +613,40 @@ impl CodexCredentialQuotaService {
|
||||
observed_at: SystemTime,
|
||||
summary: &mut CodexQuotaSyncSummary,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let refreshed_snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
value,
|
||||
)?;
|
||||
let previous_reset_at = self.previous_confirmed_reset(account).await;
|
||||
let object = normalize_quota_window_placeholders(
|
||||
value
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
let object = if account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& quota_success_state(
|
||||
account.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
confirmed_exhaustion_projection(
|
||||
object,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
|
||||
)
|
||||
} else {
|
||||
object
|
||||
};
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(object.clone()),
|
||||
)?;
|
||||
let fact = snapshot.fact();
|
||||
let state = account.quota().merge_observation(snapshot.quota());
|
||||
let snapshot = snapshot.with_quota_state(state);
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(object)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(fact.exhausted()),
|
||||
quota: OpaqueProviderData::new(object),
|
||||
observed_at,
|
||||
state,
|
||||
})
|
||||
.await?;
|
||||
if outcome == QuotaWriteOutcome::Conflict {
|
||||
summary.stale += 1;
|
||||
return Ok(());
|
||||
}
|
||||
let Some(current) = self.store.get_account(account.id()).await? else {
|
||||
summary.stale += 1;
|
||||
return Ok(());
|
||||
};
|
||||
if current.revision() != account.revision() {
|
||||
summary.stale += 1;
|
||||
return Ok(());
|
||||
}
|
||||
self.scheduling.observe(&snapshot);
|
||||
if fact.exhausted() {
|
||||
if snapshot.quota().is_exhausted() {
|
||||
summary.exhausted += 1;
|
||||
} else {
|
||||
summary.updated += 1;
|
||||
}
|
||||
if let Some(availability) = authoritative_quota_snapshot_state(
|
||||
current.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
) {
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(¤t, availability, observed_at)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -722,11 +663,6 @@ impl CodexCredentialQuotaService {
|
||||
.limits
|
||||
.values()
|
||||
.any(|details| passive_rate_limit_snapshot(details).is_some());
|
||||
let previous_reset_at = if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
self.previous_confirmed_reset(account).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let existing = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
@@ -736,17 +672,14 @@ impl CodexCredentialQuotaService {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
});
|
||||
let existing_observed_at = existing
|
||||
.as_ref()
|
||||
.and_then(|observation| observation.observed_at);
|
||||
let existing_state = existing.as_ref().map(|observation| observation.state);
|
||||
let existing = existing
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.map(|observation| observation.quota.into_inner())
|
||||
.unwrap_or_default();
|
||||
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
|
||||
// 也不能借旧快照重新推导 quota state。
|
||||
if !has_quota_facts {
|
||||
let Some(observed_at) = existing_observed_at else {
|
||||
let Some(state) = existing_state else {
|
||||
return Ok(false);
|
||||
};
|
||||
let outcome = self
|
||||
@@ -754,62 +687,42 @@ impl CodexCredentialQuotaService {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(merge_passive_quota(
|
||||
existing,
|
||||
&rate_limits,
|
||||
))),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(merge_passive_quota(existing, &rate_limits)),
|
||||
observed_at: SystemTime::now(),
|
||||
state,
|
||||
})
|
||||
.await?;
|
||||
return Ok(outcome != QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
let quota = merge_passive_quota(existing, &rate_limits);
|
||||
let observed_at = SystemTime::now();
|
||||
let refreshed_snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(quota.clone()),
|
||||
)?;
|
||||
let quota = if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
// 被动成功响应不能解除确认额度耗尽;继续展示 100%,直到权威 usage 快照
|
||||
// 证明旧 reset 已跨过且新窗口已经推进。
|
||||
confirmed_exhaustion_projection(
|
||||
quota,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
|
||||
)
|
||||
} else {
|
||||
quota
|
||||
};
|
||||
let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(quota.clone()),
|
||||
);
|
||||
)?;
|
||||
// 这些 headers 来自一次成功推理,访问结论优先于可能滞后的百分比。
|
||||
let state = QuotaState::allowed(observed_at);
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(quota)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(fact.exhausted()),
|
||||
quota: OpaqueProviderData::new(quota),
|
||||
observed_at,
|
||||
state,
|
||||
})
|
||||
.await?;
|
||||
if outcome == QuotaWriteOutcome::Conflict {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Ok(snapshot) = snapshot {
|
||||
self.scheduling.observe(&snapshot);
|
||||
}
|
||||
self.scheduling.observe(&snapshot);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// 真实 429 的单一事实入口:写入 Redis 临时限流冷却(`until = now + retry_after`)。
|
||||
/// availability 与 quota 主窗口(额度重置时间)都不改变——临时限流是独立维度,
|
||||
/// 凭据与额度主窗口(额度重置时间)都不改变——临时限流是独立维度,
|
||||
/// 到期由 Redis key 过期自动解除,不污染配额耗尽状态。
|
||||
/// 已有更晚的冷却不会被缩短(put_if_later)。
|
||||
pub async fn apply_rate_limit_429(
|
||||
@@ -838,7 +751,6 @@ impl CodexCredentialQuotaService {
|
||||
pub async fn rate_limited_until(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
|
||||
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
|
||||
CodexCredentialQuotaError::Store {
|
||||
@@ -848,9 +760,6 @@ impl CodexCredentialQuotaService {
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if cooldown.credential_revision() != credential_revision {
|
||||
return Ok(None);
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until <= SystemTime::now() {
|
||||
return Ok(None);
|
||||
@@ -894,18 +803,17 @@ impl CodexCredentialQuotaService {
|
||||
{
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
let Some(data) = observation.quota else {
|
||||
if observation.quota.expose_to_provider().is_empty() {
|
||||
return Ok(None);
|
||||
};
|
||||
let observed_at = observation
|
||||
.observed_at
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
}
|
||||
let observed_at = observation.observed_at;
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account_id.clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(data.expose_to_provider().clone()),
|
||||
)?;
|
||||
&Value::Object(observation.quota.expose_to_provider().clone()),
|
||||
)?
|
||||
.with_quota_state(observation.state);
|
||||
self.scheduling.observe(&snapshot);
|
||||
Ok(Some(snapshot))
|
||||
}
|
||||
@@ -915,7 +823,8 @@ impl CodexCredentialQuotaService {
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
|
||||
self.refresh_account_snapshot(account_id).await
|
||||
self.refresh_account_snapshot(account_id, QuotaRefreshAuthority::ObserveAccess)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 真实限额失败后的异步刷新只补齐展示快照,不允许 usage 快照撤销已确认的失败状态。
|
||||
@@ -923,12 +832,14 @@ impl CodexCredentialQuotaService {
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
|
||||
self.refresh_account_snapshot(account_id).await
|
||||
self.refresh_account_snapshot(account_id, QuotaRefreshAuthority::PreserveAccess)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn refresh_account_snapshot(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
authority: QuotaRefreshAuthority,
|
||||
) -> Result<CodexAccountQuotaSnapshot, CodexCredentialQuotaError> {
|
||||
let account = self
|
||||
.store
|
||||
@@ -957,104 +868,77 @@ impl CodexCredentialQuotaService {
|
||||
});
|
||||
}
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
// 402 与明确的停用错误写对应状态;usage 401/403 不写 availability。
|
||||
if let Some(availability) = quota_state_transition(&error) {
|
||||
self.persist_upstream_state(&account, availability, &error, observed_at)
|
||||
.await;
|
||||
match classify_quota_endpoint_failure(&error) {
|
||||
Some(QuotaEndpointFailure::Exhausted(evidence)) => {
|
||||
self.record_confirmed_exhaustion(&account, evidence, None, observed_at)
|
||||
.await?;
|
||||
}
|
||||
Some(QuotaEndpointFailure::Credential { state, reason }) => {
|
||||
self.persist_credential_failure(&account, state, reason, observed_at)
|
||||
.await;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
return Err(CodexCredentialQuotaError::Upstream {
|
||||
detail: error.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
let refreshed_snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&value,
|
||||
)?;
|
||||
let previous_reset_at = self.previous_confirmed_reset(&account).await;
|
||||
let object = normalize_quota_window_placeholders(
|
||||
value
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
let object = if account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& quota_success_state(
|
||||
account.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
confirmed_exhaustion_projection(
|
||||
object,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
|
||||
)
|
||||
} else {
|
||||
object
|
||||
};
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(object.clone()),
|
||||
)?;
|
||||
let state = match authority {
|
||||
QuotaRefreshAuthority::ObserveAccess => {
|
||||
account.quota().merge_observation(snapshot.quota())
|
||||
}
|
||||
QuotaRefreshAuthority::PreserveAccess => account.quota(),
|
||||
};
|
||||
let snapshot = snapshot.with_quota_state(state);
|
||||
if self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(object)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(snapshot.fact().exhausted()),
|
||||
quota: OpaqueProviderData::new(object),
|
||||
observed_at,
|
||||
state,
|
||||
})
|
||||
.await?
|
||||
== QuotaWriteOutcome::Conflict
|
||||
{
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
let current = self
|
||||
.store
|
||||
.get_account(account.id())
|
||||
.await?
|
||||
.ok_or(CodexCredentialQuotaError::NotFound)?;
|
||||
if current.revision() != account.revision() {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
self.scheduling.observe(&snapshot);
|
||||
if let Some(availability) = authoritative_quota_snapshot_state(
|
||||
current.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
) {
|
||||
self.repository
|
||||
.apply_state(¤t, availability, observed_at)
|
||||
.await?;
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn persist_upstream_state(
|
||||
async fn persist_credential_failure(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
upstream_error: &CodexClientError,
|
||||
credential_state: CredentialState,
|
||||
reason: AccountErrorReason,
|
||||
observed_at: SystemTime,
|
||||
) {
|
||||
let reason = quota_state_reason(upstream_error).map(str::to_owned);
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state_with_message(account, availability, observed_at, reason.clone())
|
||||
.apply_state_with_reason(account, credential_state, observed_at, Some(reason), None)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
?availability,
|
||||
reason = reason.as_deref().unwrap_or_default(),
|
||||
?credential_state,
|
||||
reason = reason.as_str(),
|
||||
error = %error,
|
||||
"OpenAI quota upstream state write failed"
|
||||
"OpenAI quota credential fact write failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1163,10 +1047,8 @@ fn eligible_periodic_quota_refresh(account: &ProviderAccount, now: SystemTime) -
|
||||
fn eligible_initial_quota_sync(account: &ProviderAccount, now: SystemTime) -> bool {
|
||||
// 首次观察只兜底刚入库、尚无 quota 快照的账号。已耗尽等运行时状态必须由
|
||||
// periodic 路径处理,才能保留同一账号的最小复核间隔。
|
||||
matches!(
|
||||
account.availability(),
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown
|
||||
) && eligible_periodic_quota_refresh(account, now)
|
||||
account.credential_state() == CredentialState::Ready
|
||||
&& eligible_periodic_quota_refresh(account, now)
|
||||
}
|
||||
|
||||
fn access_token_is_current(account: &ProviderAccount, now: SystemTime) -> bool {
|
||||
@@ -1297,71 +1179,6 @@ fn secondary_window_is_placeholder(window: &Map<String, Value>) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// 用已确认的上游额度拒绝覆盖 core 限额投影。
|
||||
///
|
||||
/// 上游 usage 快照可能在结算期间短暂滞后为 98/99% 或 `allowed=true`;保留其他
|
||||
/// 原始 metadata,但把 core 窗口固定为 100%,直到权威快照证明进入下一窗口。
|
||||
fn confirmed_exhaustion_projection(
|
||||
mut quota: Map<String, Value>,
|
||||
reset_at: Option<i64>,
|
||||
) -> Map<String, Value> {
|
||||
let mut rate_limit = quota
|
||||
.remove("rate_limit")
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
|
||||
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
|
||||
for field in ["primary_window", "secondary_window"] {
|
||||
let existed = rate_limit.contains_key(field);
|
||||
let mut window = rate_limit
|
||||
.remove(field)
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
if field == "primary_window" || existed {
|
||||
window.insert("used_percent".to_owned(), Value::from(100_u64));
|
||||
if field == "primary_window"
|
||||
&& let Some(reset_at) = reset_at
|
||||
{
|
||||
window.insert("reset_at".to_owned(), Value::from(reset_at));
|
||||
}
|
||||
rate_limit.insert(field.to_owned(), Value::Object(window));
|
||||
}
|
||||
}
|
||||
quota.insert("rate_limit".to_owned(), Value::Object(rate_limit));
|
||||
quota
|
||||
}
|
||||
|
||||
/// 选择应继续锁定的 reset:新鲜快照仍确认耗尽时可把锁推进到更晚窗口;
|
||||
/// 否则保留已确认失败的旧窗口,避免 98/99% 快照反向解锁。
|
||||
fn confirmed_exhaustion_reset_at(
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
refreshed_snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<i64> {
|
||||
let refreshed_reset_at = refreshed_snapshot
|
||||
.core_primary_window()
|
||||
.and_then(|window| window.reset_at())
|
||||
.map(SystemTime::from);
|
||||
let reset_at = match (previous_reset_at, refreshed_reset_at) {
|
||||
(Some(previous), Some(refreshed))
|
||||
if refreshed_snapshot.fact().exhausted() && refreshed > previous =>
|
||||
{
|
||||
refreshed
|
||||
}
|
||||
(Some(previous), _) => previous,
|
||||
(None, refreshed) => refreshed?,
|
||||
};
|
||||
system_time_to_unix_seconds(reset_at)
|
||||
}
|
||||
|
||||
fn system_time_to_unix_seconds(value: SystemTime) -> Option<i64> {
|
||||
value
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()?
|
||||
.as_secs()
|
||||
.try_into()
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option<Map<String, Value>> {
|
||||
// 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。
|
||||
let mut snapshot = Map::new();
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
//! 权威额度快照的 availability 转换。
|
||||
//!
|
||||
//! 已触顶的 core primary 会立即把可调度账号标为 `QuotaExhausted`;
|
||||
//! `QuotaExhausted -> Ready` 则只能由同一 credential revision 的权威 usage 快照证明:
|
||||
//! 新快照未耗尽、core primary reset 已推进到下一窗口,且该窗口用量低于 10%。
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::engine::credential::AccountAvailability;
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::snapshot::CodexAccountQuotaSnapshot;
|
||||
use crate::transport::CodexClientError;
|
||||
|
||||
const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0;
|
||||
|
||||
/// 基于成功的权威 usage 快照收敛可调度账号的额度状态。
|
||||
///
|
||||
/// 只接受上游明确拒绝使用且 core primary 触顶的证据;secondary、附加桶、
|
||||
/// credits 与 spend control 都不能单独把整个账号标成额度耗尽。
|
||||
pub(crate) fn authoritative_quota_snapshot_state(
|
||||
current: AccountAvailability,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
) -> Option<AccountAvailability> {
|
||||
match current {
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown
|
||||
if snapshot.authoritative_core_primary_exhausted() =>
|
||||
{
|
||||
Some(AccountAvailability::QuotaExhausted)
|
||||
}
|
||||
AccountAvailability::QuotaExhausted => {
|
||||
quota_success_state(current, snapshot, previous_reset_at)
|
||||
}
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown => None,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独覆盖
|
||||
/// 一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。
|
||||
///
|
||||
/// `previous_reset_at` 是写入新快照前读取的 core primary 已确认耗尽窗口。上游可能只
|
||||
/// 调整 reset 时间而尚未重置额度,因此还要求同一新鲜 primary 窗口的用量低于 10%。
|
||||
pub(crate) fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
) -> Option<AccountAvailability> {
|
||||
if current != AccountAvailability::QuotaExhausted || snapshot.fact().exhausted() {
|
||||
return None;
|
||||
}
|
||||
let previous_reset_at = previous_reset_at?;
|
||||
let primary_window = snapshot.core_primary_window()?;
|
||||
let refreshed_reset_at = primary_window.reset_at().map(SystemTime::from)?;
|
||||
let used_percent = primary_window.used_percent()?;
|
||||
(refreshed_reset_at > previous_reset_at && used_percent < RESET_RECOVERY_MAX_USED_PERCENT)
|
||||
.then_some(AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
|
||||
let CodexClientError::Upstream { status, body, .. } = error else {
|
||||
return None;
|
||||
};
|
||||
if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) {
|
||||
return Some(AccountAvailability::Banned);
|
||||
}
|
||||
match *status {
|
||||
// 402:真额度耗尽 → QuotaExhausted(worker 按 reset_at 自动恢复)。
|
||||
StatusCode::PAYMENT_REQUIRED => Some(AccountAvailability::QuotaExhausted),
|
||||
// usage 401/403 不能单独判定账号不可用:保持现有状态,交给 OAuth 或真实请求确认。
|
||||
// 429/503 同样不写 availability,仅记失败待重试。
|
||||
StatusCode::UNAUTHORIZED
|
||||
| StatusCode::FORBIDDEN
|
||||
| StatusCode::TOO_MANY_REQUESTS
|
||||
| StatusCode::SERVICE_UNAVAILABLE => None,
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 配额接口将账号变为非 ready 状态时可安全持久化的上游原因码。
|
||||
pub(crate) fn quota_state_reason(error: &CodexClientError) -> Option<&'static str> {
|
||||
match quota_state_transition(error)? {
|
||||
AccountAvailability::Banned => Some("deactivated_workspace"),
|
||||
AccountAvailability::QuotaExhausted => Some("payment_required"),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_deactivated_workspace(body: &str) -> bool {
|
||||
serde_json::from_str::<Value>(body)
|
||||
.ok()
|
||||
.is_some_and(|value| {
|
||||
value.pointer("/detail/code").and_then(Value::as_str) == Some("deactivated_workspace")
|
||||
})
|
||||
}
|
||||
@@ -7,19 +7,17 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaObservation,
|
||||
AccountQuotaSignals, CredentialRevision, ProviderAccountId, QuotaEvidence, QuotaObservation,
|
||||
QuotaState,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL};
|
||||
|
||||
const CORE_PRIMARY_WINDOW_KEY: &str = "core-primary";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CodexQuotaFact {
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
exhausted: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -112,7 +110,7 @@ pub struct CodexAccountQuotaSnapshot {
|
||||
credential_revision: CredentialRevision,
|
||||
observed_at: SystemTime,
|
||||
fact: CodexQuotaFact,
|
||||
authoritative_core_primary_exhausted: bool,
|
||||
quota: QuotaState,
|
||||
windows: Vec<CodexQuotaWindow>,
|
||||
}
|
||||
|
||||
@@ -143,54 +141,14 @@ impl CodexAccountQuotaSnapshot {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn core_primary_window(&self) -> Option<&CodexQuotaWindow> {
|
||||
self.windows
|
||||
.iter()
|
||||
.find(|window| window.key() == CORE_PRIMARY_WINDOW_KEY)
|
||||
pub const fn quota(&self) -> QuotaState {
|
||||
self.quota
|
||||
}
|
||||
|
||||
/// 上游成功 usage 明确拒绝请求,且 canonical core primary 已确认触顶。
|
||||
/// 将已持久化的规范化事实覆盖到 raw JSON 展示快照上。
|
||||
#[must_use]
|
||||
pub(crate) const fn authoritative_core_primary_exhausted(&self) -> bool {
|
||||
self.authoritative_core_primary_exhausted
|
||||
}
|
||||
|
||||
/// 投影滚动已过期窗口:`reset_at` 已过的窗口 `used_percent=0`、
|
||||
/// `limit_reached=false`、`reset_at` 滚到下一个周期(reset 到期自动归零)。
|
||||
/// 只作用于展示/调度投影,不写回持久化 raw JSON。
|
||||
///
|
||||
/// O(1) 计算:`next_reset = reset + (floor((now - reset) / window) + 1) * window`,
|
||||
/// 不做逐周期线性循环。
|
||||
#[must_use]
|
||||
pub fn roll_expired_windows(mut self, now: SystemTime) -> Self {
|
||||
for window in &mut self.windows {
|
||||
let Some(reset_at) = window.reset_at else {
|
||||
continue;
|
||||
};
|
||||
let reset = SystemTime::from(reset_at);
|
||||
if reset > now {
|
||||
continue;
|
||||
}
|
||||
window.used_percent = Some(0.0);
|
||||
window.limit_reached = false;
|
||||
let Some(window_seconds) = window.window_seconds.filter(|seconds| *seconds > 0) else {
|
||||
// 没有合法窗口时长:清除已过期的 reset,等待下一次权威观测。
|
||||
window.reset_at = None;
|
||||
continue;
|
||||
};
|
||||
let elapsed = now
|
||||
.duration_since(reset)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs();
|
||||
let periods = elapsed / window_seconds + 1;
|
||||
let Some(after) =
|
||||
reset.checked_add(Duration::from_secs(periods.saturating_mul(window_seconds)))
|
||||
else {
|
||||
window.reset_at = None;
|
||||
continue;
|
||||
};
|
||||
window.reset_at = Some(DateTime::<Utc>::from(after));
|
||||
}
|
||||
pub(crate) const fn with_quota_state(mut self, quota: QuotaState) -> Self {
|
||||
self.quota = quota;
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -205,11 +163,6 @@ impl CodexQuotaFact {
|
||||
pub const fn resets_at(&self) -> Option<DateTime<Utc>> {
|
||||
self.resets_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn exhausted(&self) -> bool {
|
||||
self.exhausted
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCredentialQuotaError> {
|
||||
@@ -220,13 +173,12 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCre
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
aggregate.observe_rate_limit(limit.rate_limit)?;
|
||||
}
|
||||
// spend_control 只提供 exhaustion 信号(`reached`),不生成窗口、
|
||||
// 不参与剩余量聚合(`individual_limit` 等字段无窗口语义)。
|
||||
// credits / spend_control 只证明文档结构已识别,不参与账号级访问结论。
|
||||
if let Some(spend_control) = object.get("spend_control") {
|
||||
aggregate.observe_exhaustion_object(spend_control, "reached")?;
|
||||
aggregate.observe_metadata_object(spend_control, "reached")?;
|
||||
}
|
||||
if let Some(credits) = object.get("credits") {
|
||||
aggregate.observe_exhaustion_object(credits, "overage_limit_reached")?;
|
||||
aggregate.observe_metadata_object(credits, "overage_limit_reached")?;
|
||||
}
|
||||
if !aggregate.recognized {
|
||||
return Err(CodexCredentialQuotaError::InvalidCredentialData);
|
||||
@@ -234,22 +186,24 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCre
|
||||
Ok(CodexQuotaFact {
|
||||
remaining_percent: aggregate.remaining_percent,
|
||||
resets_at: aggregate.resets_at,
|
||||
exhausted: aggregate.exhausted,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn quota_snapshot_from_observation(
|
||||
observation: &QuotaObservation,
|
||||
) -> Option<CodexAccountQuotaSnapshot> {
|
||||
let observed_at = observation.observed_at?;
|
||||
let quota = observation.quota.as_ref()?;
|
||||
parse_account_quota_snapshot(
|
||||
let observed_at = observation.observed_at;
|
||||
let quota = &observation.quota;
|
||||
let mut snapshot = parse_account_quota_snapshot(
|
||||
observation.account_id.clone(),
|
||||
observation.expected_revision,
|
||||
observed_at,
|
||||
&Value::Object(quota.expose_to_provider().clone()),
|
||||
)
|
||||
.ok()
|
||||
.ok()?;
|
||||
// 归一化列是持久化权威事实;raw JSON 只负责 Provider 展示结构。
|
||||
snapshot.quota = observation.state;
|
||||
Some(snapshot)
|
||||
}
|
||||
|
||||
/// 调度投影缓存 TTL:`min(常规 TTL, 最近 active reset - now)`。
|
||||
@@ -275,46 +229,36 @@ pub(crate) fn quota_projection_ttl(snapshot: &CodexAccountQuotaSnapshot) -> Opti
|
||||
(!ttl.is_zero()).then_some(ttl)
|
||||
}
|
||||
|
||||
/// 从快照的滚动窗口派生中立调度事实(`limit_reached` = 任一窗口触顶)。
|
||||
/// 从快照窗口派生容量排序事实;额度访问资格由 `QuotaState` 统一判断。
|
||||
pub(crate) fn scheduling_signals_from_snapshot(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<AccountQuotaSignals> {
|
||||
let rolled = snapshot.clone().roll_expired_windows(SystemTime::now());
|
||||
quota_scheduling_signals(&rolled)
|
||||
quota_scheduling_signals(snapshot)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_scheduling_signals(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<AccountQuotaSignals> {
|
||||
let now = SystemTime::now();
|
||||
let mut limit_reached = false;
|
||||
let mut remaining_rank: Option<u64> = None;
|
||||
let mut reset_at: Option<SystemTime> = None;
|
||||
for window in &snapshot.windows {
|
||||
limit_reached |= window.limit_reached();
|
||||
if let Some(used) = window
|
||||
.used_percent()
|
||||
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
|
||||
let window_reset_at = window.reset_at().map(SystemTime::from);
|
||||
let observation_expired = window_reset_at.is_some_and(|reset_at| reset_at <= now);
|
||||
if !observation_expired
|
||||
&& let Some(used) = window
|
||||
.used_percent()
|
||||
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
|
||||
{
|
||||
let rank = ((100.0 - used) * 100.0).round() as u64;
|
||||
remaining_rank = Some(remaining_rank.map_or(rank, |current| current.min(rank)));
|
||||
}
|
||||
if let Some(reset) = window
|
||||
.reset_at()
|
||||
.map(SystemTime::from)
|
||||
.filter(|reset| *reset > now)
|
||||
{
|
||||
if let Some(reset) = window_reset_at.filter(|reset| *reset > now) {
|
||||
reset_at = Some(reset_at.map_or(reset, |current| current.min(reset)));
|
||||
}
|
||||
}
|
||||
// 快照级 limit_reached 是调度排除的充分条件:即使没有任何窗口带
|
||||
// percent/reset(如顶层限流标记),也必须产出信号;不能返回 None 丢失排除。
|
||||
if limit_reached || remaining_rank.is_some() || reset_at.is_some() {
|
||||
Some(AccountQuotaSignals::new(
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
))
|
||||
if remaining_rank.is_some() || reset_at.is_some() {
|
||||
Some(AccountQuotaSignals::new(reset_at, remaining_rank))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -330,7 +274,7 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
let object = usage
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let authoritative_core_primary_exhausted = authoritative_core_primary_exhausted(object)?;
|
||||
let quota = authoritative_quota_state(object, observed_at)?;
|
||||
let mut windows = Vec::new();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
parse_rate_limit_windows(
|
||||
@@ -352,7 +296,7 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
credential_revision,
|
||||
observed_at,
|
||||
fact,
|
||||
authoritative_core_primary_exhausted,
|
||||
quota,
|
||||
windows,
|
||||
})
|
||||
}
|
||||
@@ -365,48 +309,73 @@ fn quota_source_order(source: &str) -> u8 {
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号状态只能由顶层 canonical Codex 额度桶的明确拒绝来改变。
|
||||
///
|
||||
/// `fact.exhausted()` 和窗口 `limit_reached` 都会聚合或继承 secondary、附加桶等事实,
|
||||
/// 因此不能单独用来将整个账号标为 `QuotaExhausted`。
|
||||
fn authoritative_core_primary_exhausted(
|
||||
/// 顶层 account-wide Codex 桶是唯一可改变账号额度访问结论的 quota 响应字段。
|
||||
fn authoritative_quota_state(
|
||||
usage: &Map<String, Value>,
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
observed_at: SystemTime,
|
||||
) -> Result<QuotaState, CodexCredentialQuotaError> {
|
||||
let Some(rate_limit) = usage.get("rate_limit") else {
|
||||
return Ok(false);
|
||||
return Ok(QuotaState::observed_unknown(observed_at));
|
||||
};
|
||||
if rate_limit.is_null() {
|
||||
return Ok(false);
|
||||
return Ok(QuotaState::observed_unknown(observed_at));
|
||||
}
|
||||
let rate_limit = rate_limit
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
if optional_bool(rate_limit, "allowed")? != Some(false) {
|
||||
return Ok(false);
|
||||
let allowed = optional_bool(rate_limit, "allowed")?;
|
||||
let top_level_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false);
|
||||
let primary_window = rate_limit
|
||||
.get("primary_window")
|
||||
.filter(|value| !value.is_null())
|
||||
.map(|value| {
|
||||
value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let primary_reached = primary_window
|
||||
.map(|window| optional_bool(window, "limit_reached"))
|
||||
.transpose()?
|
||||
.flatten()
|
||||
.unwrap_or(false);
|
||||
let reset_at = primary_window
|
||||
.and_then(|window| window.get("reset_at"))
|
||||
.map(|value| {
|
||||
value
|
||||
.as_i64()
|
||||
.filter(|value| *value > 0)
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value, 0))
|
||||
.map(SystemTime::from)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
if allowed == Some(true) {
|
||||
if top_level_reached || primary_reached {
|
||||
tracing::warn!(
|
||||
top_level_reached,
|
||||
primary_reached,
|
||||
"OpenAI quota returned contradictory access facts; explicit allowed=true wins"
|
||||
);
|
||||
}
|
||||
return Ok(QuotaState::allowed(observed_at));
|
||||
}
|
||||
let Some(primary_window) = rate_limit.get("primary_window") else {
|
||||
return Ok(false);
|
||||
};
|
||||
if primary_window.is_null() {
|
||||
return Ok(false);
|
||||
if allowed == Some(false) {
|
||||
return Ok(QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
observed_at,
|
||||
reset_at,
|
||||
));
|
||||
}
|
||||
let primary_window = primary_window
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let primary_window_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false)
|
||||
|| optional_bool(primary_window, "limit_reached")?.unwrap_or(false)
|
||||
|| primary_window
|
||||
.get("used_percent")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.map(|value| value >= 100.0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(false);
|
||||
Ok(primary_window_reached)
|
||||
if top_level_reached || primary_reached {
|
||||
return Ok(QuotaState::exhausted(
|
||||
QuotaEvidence::AccountLimitReached,
|
||||
observed_at,
|
||||
reset_at,
|
||||
));
|
||||
}
|
||||
Ok(QuotaState::observed_unknown(observed_at))
|
||||
}
|
||||
|
||||
/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。
|
||||
@@ -523,10 +492,9 @@ fn parse_rate_limit_windows(
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
// 顶层限流标记是快照级事实:任一窗口都要继承所属
|
||||
// rate_limit 的 `limit_reached=true` / `allowed=false`。
|
||||
let top_level_reached = optional_bool(object, "limit_reached")?.unwrap_or(false)
|
||||
|| optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
let allowed = optional_bool(object, "allowed")?;
|
||||
let top_level_reached = allowed != Some(true)
|
||||
&& (optional_bool(object, "limit_reached")?.unwrap_or(false) || allowed == Some(false));
|
||||
for (name, role) in [
|
||||
("primary_window", CodexQuotaWindowRole::Primary),
|
||||
("secondary_window", CodexQuotaWindowRole::Secondary),
|
||||
@@ -564,12 +532,12 @@ fn parse_rate_limit_windows(
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let limit_reached = top_level_reached
|
||||
|| window
|
||||
.get("limit_reached")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
|| used_percent.is_some_and(|used| used >= 100.0);
|
||||
let limit_reached = allowed != Some(true)
|
||||
&& (top_level_reached
|
||||
|| window
|
||||
.get("limit_reached")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false));
|
||||
let kind = quota_window_kind(window_seconds);
|
||||
output.push(CodexQuotaWindow {
|
||||
key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)),
|
||||
@@ -649,7 +617,6 @@ const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind)
|
||||
#[derive(Default)]
|
||||
struct QuotaAggregate {
|
||||
recognized: bool,
|
||||
exhausted: bool,
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
@@ -660,8 +627,8 @@ impl QuotaAggregate {
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false);
|
||||
self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
let _ = optional_bool(object, "limit_reached")?;
|
||||
let _ = optional_bool(object, "allowed")?;
|
||||
for key in ["primary_window", "secondary_window"] {
|
||||
if let Some(window) = object.get(key) {
|
||||
self.observe_window(window)?;
|
||||
@@ -682,8 +649,6 @@ impl QuotaAggregate {
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
// 子窗口触顶(used>=100)是快照级 exhaustion 事实。
|
||||
self.exhausted |= used >= 100.0;
|
||||
let remaining = (100.0 - used).clamp(0.0, 100.0).round() as u8;
|
||||
self.remaining_percent = Some(
|
||||
self.remaining_percent
|
||||
@@ -702,7 +667,7 @@ impl QuotaAggregate {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_exhaustion_object(
|
||||
fn observe_metadata_object(
|
||||
&mut self,
|
||||
value: &Value,
|
||||
key: &str,
|
||||
@@ -714,7 +679,7 @@ impl QuotaAggregate {
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, key)?.unwrap_or(false);
|
||||
let _ = optional_bool(object, key)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::{StreamExt as _, stream};
|
||||
use gateway_core::engine::credential::{AccountAvailability, ProviderAccount, ProviderAccountId};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountErrorReason, CredentialState, ProviderAccount, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
|
||||
@@ -253,7 +255,8 @@ impl CodexCredentialRefreshService {
|
||||
return self
|
||||
.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
CredentialState::Expired,
|
||||
AccountErrorReason::CredentialExpired,
|
||||
"refresh_recovery_window_exhausted",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
@@ -293,7 +296,8 @@ impl CodexCredentialRefreshService {
|
||||
Err(RefreshFailure::InvalidGrant) => {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
CredentialState::Expired,
|
||||
AccountErrorReason::CredentialExpired,
|
||||
"refresh_invalid_grant",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
@@ -302,7 +306,8 @@ impl CodexCredentialRefreshService {
|
||||
Err(RefreshFailure::Banned) => {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Banned,
|
||||
CredentialState::Banned,
|
||||
AccountErrorReason::AccountBanned,
|
||||
"account_banned",
|
||||
CodexCredentialRefreshOutcome::Banned { account_id },
|
||||
)
|
||||
@@ -353,11 +358,9 @@ impl CodexCredentialRefreshService {
|
||||
&& account.enabled()
|
||||
&& account.has_refresh_token()
|
||||
&& !excluded.contains(account.id())
|
||||
&& !matches!(
|
||||
account.availability(),
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
&& matches!(
|
||||
account.credential_state(),
|
||||
CredentialState::Unknown | CredentialState::Ready
|
||||
)
|
||||
&& refresh_due_at(account, policy, now).is_some()
|
||||
});
|
||||
@@ -463,7 +466,8 @@ impl CodexCredentialRefreshService {
|
||||
async fn persist_terminal(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
error_reason: AccountErrorReason,
|
||||
reason: &'static str,
|
||||
outcome: CodexCredentialRefreshOutcome,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
@@ -478,10 +482,11 @@ impl CodexCredentialRefreshService {
|
||||
}
|
||||
match self
|
||||
.repository
|
||||
.apply_state_with_message(
|
||||
.apply_state_with_reason(
|
||||
account,
|
||||
availability,
|
||||
credential_state,
|
||||
SystemTime::now(),
|
||||
Some(error_reason),
|
||||
Some(reason.to_owned()),
|
||||
)
|
||||
.await
|
||||
@@ -489,7 +494,7 @@ impl CodexCredentialRefreshService {
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
?availability,
|
||||
?credential_state,
|
||||
reason,
|
||||
access_token_expires_at = ?account.access_token_expires_at()
|
||||
.map(DateTime::<Utc>::from),
|
||||
|
||||
@@ -5,8 +5,8 @@ use std::time::SystemTime;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, CredentialState, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -220,34 +220,46 @@ impl CodexCredentialRepository {
|
||||
pub async fn apply_state(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
self.apply_state_with_message(account, availability, observed_at, None)
|
||||
.await
|
||||
self.apply_state_with_reason(
|
||||
account,
|
||||
credential_state,
|
||||
observed_at,
|
||||
credential_state.error_reason(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 带上游错误描述的状态写入;上游 message 优先,终态缺失 message 时写稳定兜底,
|
||||
/// 恢复 `Ready` 时清空。
|
||||
pub async fn apply_state_with_message(
|
||||
/// 写入凭据事实及其稳定错误原因;额度事实不经过此入口。
|
||||
pub async fn apply_state_with_reason(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
error_reason: Option<AccountErrorReason>,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
let message = message
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
|
||||
let (error_reason, message) = if credential_state == CredentialState::Ready {
|
||||
(None, None)
|
||||
} else {
|
||||
(
|
||||
error_reason.or_else(|| credential_state.error_reason()),
|
||||
message.filter(|value| !value.trim().is_empty()),
|
||||
)
|
||||
};
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability,
|
||||
credential_state,
|
||||
observed_at,
|
||||
error_reason,
|
||||
message,
|
||||
})
|
||||
.await?;
|
||||
|
||||
@@ -6,9 +6,10 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
|
||||
AccountCandidate, AccountEligibilityPolicy, AccountErrorReason, AccountFeedbackStats,
|
||||
AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, AccountSelector,
|
||||
PreferredAccountSelection, ProviderAccount, ProviderAccountId,
|
||||
AccountStatus, CredentialState, PreferredAccountSelection, ProviderAccount, ProviderAccountId,
|
||||
QuotaEvidence,
|
||||
};
|
||||
use gateway_core::engine::{AttemptContext, ContinuationAttempt};
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -57,8 +58,8 @@ pub enum CodexAccountFailure {
|
||||
QuotaExhausted,
|
||||
/// 当前用量窗口已耗尽;到重置时间后可自动恢复。
|
||||
UsageLimitExhausted {
|
||||
/// 上游返回的窗口重置时长。
|
||||
retry_after: Option<Duration>,
|
||||
/// 上游返回的窗口绝对重置时刻。
|
||||
reset_at: Option<SystemTime>,
|
||||
},
|
||||
/// 账号触发临时用量限制。
|
||||
RateLimited {
|
||||
@@ -74,21 +75,6 @@ pub enum CodexAccountFailure {
|
||||
CloudflarePathBlocked,
|
||||
}
|
||||
|
||||
impl CodexAccountFailure {
|
||||
/// 上游未提供可展示错误文本时,保留已分类的稳定原因。
|
||||
const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::CredentialExpired => Some("upstream_credential_expired"),
|
||||
Self::IdentityVerificationRequired => Some("upstream_identity_verification_required"),
|
||||
Self::Banned => Some("upstream_account_banned"),
|
||||
Self::QuotaExhausted => Some("upstream_quota_exhausted"),
|
||||
Self::UsageLimitExhausted { .. } => Some("upstream_usage_limit_exhausted"),
|
||||
Self::CloudflarePathBlocked => Some("upstream_cloudflare_path_blocked"),
|
||||
Self::RateLimited { .. } | Self::CloudflareChallenge { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RiskRecoveryState {
|
||||
challenge_count: u32,
|
||||
@@ -127,7 +113,6 @@ pub struct CodexCredentialSelector {
|
||||
cookie_policy: CodexCookiePolicy,
|
||||
risk_recovery: Mutex<HashMap<String, RiskRecoveryState>>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
skip_exhausted: bool,
|
||||
}
|
||||
|
||||
enum SessionAffinityLookup {
|
||||
@@ -263,7 +248,6 @@ impl CodexCredentialSelector {
|
||||
agent_identity: Arc<CodexAgentIdentityTaskService>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
cookie_policy: CodexCookiePolicy,
|
||||
skip_exhausted: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider_kind,
|
||||
@@ -277,7 +261,6 @@ impl CodexCredentialSelector {
|
||||
cookie_policy,
|
||||
risk_recovery: Mutex::new(HashMap::new()),
|
||||
account_feedback,
|
||||
skip_exhausted,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,24 +293,17 @@ impl CodexCredentialSelector {
|
||||
if !diagnostic {
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
}
|
||||
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
|
||||
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
|
||||
let accounts = if diagnostic {
|
||||
accounts
|
||||
} else {
|
||||
let mut retained = Vec::with_capacity(accounts.len());
|
||||
for account in accounts {
|
||||
let cooling = self
|
||||
let mut rate_limits = HashMap::with_capacity(accounts.len());
|
||||
if !diagnostic {
|
||||
for account in &accounts {
|
||||
let until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.rate_limited_until(account.id())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
if cooling.is_none() {
|
||||
retained.push(account);
|
||||
}
|
||||
rate_limits.insert(account.id().clone(), until);
|
||||
}
|
||||
retained
|
||||
};
|
||||
}
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
@@ -352,11 +328,12 @@ impl CodexCredentialSelector {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
})
|
||||
.with_provider_quota(self.quota.scheduling_signals(&account))
|
||||
.with_rate_limit(rate_limits.get(account.id()).copied().flatten())
|
||||
.with_runtime_health(health.0, health.1);
|
||||
AccountCandidate { account, signals }
|
||||
})
|
||||
@@ -383,41 +360,6 @@ impl CodexCredentialSelector {
|
||||
return Err(CredentialSelectionError::NoEligibleCredential);
|
||||
}
|
||||
let pinned_account = required_account.or_else(|| continuation_account.clone());
|
||||
let preserve_native_continuation_state = matches!(
|
||||
request.attempt.continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
) && continuation_account.is_some();
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if diagnostic {
|
||||
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
|
||||
candidates.clone()
|
||||
} else if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
candidates
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut candidate| {
|
||||
// 已命中 previous_response_id 的 native continuation 先检查原账号。
|
||||
// 即使常规调度允许探测额度耗尽账号,也必须保留原账号真实状态,
|
||||
// 由 Coordinator 据此切换到可携带重放,而非伪装为 Ready。
|
||||
if !preserve_native_continuation_state
|
||||
|| pinned_account.as_ref() != Some(candidate.account.id())
|
||||
{
|
||||
candidate.account =
|
||||
quota_exhausted_scheduling_projection(&candidate.account);
|
||||
}
|
||||
candidate
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let scheduling_candidates = scheduling_candidates.as_slice();
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
@@ -432,18 +374,6 @@ impl CodexCredentialSelector {
|
||||
.prepare_cyber_policy_scope(cyber_policy_session_key)
|
||||
.await;
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if !diagnostic
|
||||
&& preserve_native_continuation_state
|
||||
&& let Some(pinned) = pinned_account.as_ref()
|
||||
&& candidates.iter().any(|candidate| {
|
||||
candidate.account.id() == pinned && candidate.signals.quota_limit_reached
|
||||
})
|
||||
{
|
||||
// `skip_exhausted = false` 只允许普通新请求探测可疑账号;已绑定的
|
||||
// native response handle 先将真实触顶状态交给 Coordinator,随后才可
|
||||
// 通过 ReplayAny 按当前调度策略换号。
|
||||
excluded.insert(pinned.clone());
|
||||
}
|
||||
if let Some(state) = cyber_policy_scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.state.as_ref())
|
||||
@@ -476,13 +406,13 @@ impl CodexCredentialSelector {
|
||||
excluded_accounts: excluded.clone(),
|
||||
preferred_account: preferred.clone(),
|
||||
round_robin_cursor,
|
||||
availability: if diagnostic {
|
||||
AccountAvailabilityPolicy::BypassForDiagnostic
|
||||
eligibility: if diagnostic {
|
||||
AccountEligibilityPolicy::BypassForDiagnostic
|
||||
} else {
|
||||
AccountAvailabilityPolicy::Enforce
|
||||
AccountEligibilityPolicy::Enforce
|
||||
},
|
||||
};
|
||||
let Some(selection) = AccountSelector.select(scheduling_candidates, &context) else {
|
||||
let Some(selection) = AccountSelector.select(&candidates, &context) else {
|
||||
return match shortest_retry {
|
||||
Some(retry_after) => Err(CredentialSelectionError::CapacityUnavailable {
|
||||
retry_after: Some(retry_after),
|
||||
@@ -597,10 +527,15 @@ impl CodexCredentialSelector {
|
||||
self.clear_session_affinity(key).await;
|
||||
return AffinitySelection::escaped(account_id, AffinityEscapeReason::HardUnavailable);
|
||||
};
|
||||
if candidate.account.is_schedulable(now) {
|
||||
if candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.rate_limited_until)
|
||||
.status
|
||||
== AccountStatus::Normal
|
||||
{
|
||||
return AffinitySelection::preferred(account_id);
|
||||
}
|
||||
let reason = affinity_unavailable_reason(&candidate.account, now);
|
||||
let reason = affinity_unavailable_reason(candidate, now);
|
||||
self.clear_session_affinity(key).await;
|
||||
AffinitySelection::escaped(account_id, reason)
|
||||
}
|
||||
@@ -749,9 +684,7 @@ impl CodexCredentialSelector {
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let now = SystemTime::now();
|
||||
let message = message
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| failure.fallback_error_reason().map(str::to_owned));
|
||||
let message = message.filter(|value| !value.trim().is_empty());
|
||||
match failure {
|
||||
CodexAccountFailure::CredentialExpired => {
|
||||
if account.authentication_kind() == CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
@@ -772,29 +705,51 @@ impl CodexCredentialSelector {
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::IdentityVerificationRequired => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::Banned => {
|
||||
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
|
||||
.await
|
||||
}
|
||||
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
|
||||
CodexAccountFailure::QuotaExhausted
|
||||
| CodexAccountFailure::UsageLimitExhausted { .. } => {
|
||||
self.apply_state_with_message(
|
||||
self.apply_credential_state(
|
||||
account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
CredentialState::Expired,
|
||||
AccountErrorReason::AccessTokenExpired,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// 429:临时限流只写 quota,availability 不改变。
|
||||
CodexAccountFailure::IdentityVerificationRequired => {
|
||||
self.apply_credential_state(
|
||||
account,
|
||||
CredentialState::Invalid,
|
||||
AccountErrorReason::AccountUnverified,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::Banned => {
|
||||
self.apply_credential_state(
|
||||
account,
|
||||
CredentialState::Banned,
|
||||
AccountErrorReason::AccountBanned,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::QuotaExhausted => self
|
||||
.quota
|
||||
.record_confirmed_exhaustion(account, QuotaEvidence::PaymentRequired, None, now)
|
||||
.await
|
||||
.map_err(|_| CredentialSelectionError::Store),
|
||||
CodexAccountFailure::UsageLimitExhausted { reset_at } => self
|
||||
.quota
|
||||
.record_confirmed_exhaustion(
|
||||
account,
|
||||
QuotaEvidence::UsageLimitReached,
|
||||
reset_at,
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CredentialSelectionError::Store),
|
||||
// 429:临时限流只写运行时冷却,不改变凭据或额度事实。
|
||||
CodexAccountFailure::RateLimited { retry_after } => {
|
||||
self.quota
|
||||
.apply_rate_limit_429(account, retry_after, now)
|
||||
@@ -802,7 +757,7 @@ impl CodexCredentialSelector {
|
||||
.map_err(|_| CredentialSelectionError::Store)?;
|
||||
Ok(())
|
||||
}
|
||||
// Cloudflare 挑战:内存退避表(记录风险计数),不写 availability。
|
||||
// Cloudflare 挑战:内存退避表(记录风险计数),不写账号事实。
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after } => {
|
||||
let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after);
|
||||
let recovery = now
|
||||
@@ -815,9 +770,10 @@ impl CodexCredentialSelector {
|
||||
CodexAccountFailure::CloudflarePathBlocked => {
|
||||
let blocked = self.record_cloudflare_path_block(account.id(), now);
|
||||
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
|
||||
self.apply_state_with_message(
|
||||
self.apply_credential_state(
|
||||
account,
|
||||
AccountAvailability::Invalid,
|
||||
CredentialState::Invalid,
|
||||
AccountErrorReason::CredentialInvalid,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
@@ -830,15 +786,22 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_state_with_message(
|
||||
async fn apply_credential_state(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
error_reason: AccountErrorReason,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
self.repository
|
||||
.apply_state_with_message(account, availability, observed_at, message)
|
||||
.apply_state_with_reason(
|
||||
account,
|
||||
credential_state,
|
||||
observed_at,
|
||||
Some(error_reason),
|
||||
message,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -895,29 +858,34 @@ impl CodexCredentialSelector {
|
||||
let Ok(current) = self.current_account(account.id()).await else {
|
||||
return;
|
||||
};
|
||||
// QuotaExhausted 只能由结构化额度事实解除;一次成功响应本身不能证明额度已经恢复。
|
||||
if current.provider() != &self.provider_kind
|
||||
|| current.revision() != account.revision()
|
||||
|| !current.enabled()
|
||||
|| !matches!(
|
||||
current.availability(),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
)
|
||||
{
|
||||
return;
|
||||
}
|
||||
let observed_at = SystemTime::now();
|
||||
if current.credential_state() != CredentialState::Ready
|
||||
&& let Err(error) = self
|
||||
.repository
|
||||
.apply_state(¤t, CredentialState::Ready, observed_at)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI credential recovery after successful upstream response failed"
|
||||
);
|
||||
}
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state(¤t, AccountAvailability::Ready, SystemTime::now())
|
||||
.quota
|
||||
.record_successful_inference(¤t, observed_at)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI account state recovery after successful upstream response failed"
|
||||
"OpenAI quota recovery after successful upstream response failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1045,30 +1013,19 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderAccount {
|
||||
if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
return account
|
||||
.clone()
|
||||
.with_runtime_state(account.enabled(), AccountAvailability::Ready);
|
||||
}
|
||||
account.clone()
|
||||
}
|
||||
|
||||
fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> AffinityEscapeReason {
|
||||
if !account.enabled()
|
||||
|| account
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
fn affinity_unavailable_reason(
|
||||
candidate: &AccountCandidate,
|
||||
now: SystemTime,
|
||||
) -> AffinityEscapeReason {
|
||||
match candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.rate_limited_until)
|
||||
.status
|
||||
{
|
||||
return AffinityEscapeReason::HardUnavailable;
|
||||
}
|
||||
match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => AffinityEscapeReason::HardUnavailable,
|
||||
AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
AccountStatus::RateLimited => AffinityEscapeReason::Cooldown,
|
||||
AccountStatus::Normal => AffinityEscapeReason::SelectionInvariant,
|
||||
AccountStatus::Disabled | AccountStatus::Error => AffinityEscapeReason::HardUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -110,7 +110,6 @@ pub async fn initialize(
|
||||
Arc::clone(&agent_identity),
|
||||
ports.cooldowns(),
|
||||
));
|
||||
let quota_skip_exhausted = config.quota_skip_exhausted();
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
provider_kind.clone(),
|
||||
repository.clone(),
|
||||
@@ -122,7 +121,6 @@ pub async fn initialize(
|
||||
Arc::clone(&agent_identity),
|
||||
Arc::clone(&account_feedback),
|
||||
CodexCookiePolicy::official().map_err(|_| OpenAiInitializeError::CookiePolicy)?,
|
||||
quota_skip_exhausted,
|
||||
));
|
||||
let core_provider: Arc<dyn Provider> = Arc::new(
|
||||
CodexProvider::new(
|
||||
|
||||
@@ -1704,13 +1704,11 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
|
||||
struct MappedProviderFailure {
|
||||
error: ProviderError,
|
||||
account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,随 availability 状态一并持久化。
|
||||
/// 原始上游错误描述,仅在凭据错误状态下持久化。
|
||||
error_message: Option<String>,
|
||||
cyber_policy_failure: bool,
|
||||
set_cookie_headers: Vec<String>,
|
||||
rate_limit_headers: Vec<(String, String)>,
|
||||
/// 结构化 usage-limit 错误给出的窗口 reset(Unix 秒)。
|
||||
confirmed_exhaustion_reset_at: Option<i64>,
|
||||
observation: Option<ProviderResponseObservation>,
|
||||
capture_response_cookies: bool,
|
||||
}
|
||||
@@ -1724,7 +1722,6 @@ impl MappedProviderFailure {
|
||||
cyber_policy_failure: false,
|
||||
set_cookie_headers: Vec::new(),
|
||||
rate_limit_headers: Vec::new(),
|
||||
confirmed_exhaustion_reset_at: None,
|
||||
observation: None,
|
||||
capture_response_cookies: false,
|
||||
}
|
||||
@@ -1882,18 +1879,6 @@ async fn apply_failure(
|
||||
failure.account_failure,
|
||||
Some(CodexAccountFailure::QuotaExhausted | CodexAccountFailure::UsageLimitExhausted { .. })
|
||||
);
|
||||
if needs_authoritative_quota_refresh
|
||||
&& let Err(error) = context
|
||||
.quota
|
||||
.record_confirmed_exhaustion(account, failure.confirmed_exhaustion_reset_at)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"Failed to project confirmed OpenAI quota exhaustion"
|
||||
);
|
||||
}
|
||||
if failure.cyber_policy_failure {
|
||||
context
|
||||
.selector
|
||||
@@ -2186,12 +2171,15 @@ fn map_upstream_failure(
|
||||
}
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
account_failure: account_failure(category, failure.retry_after_seconds),
|
||||
account_failure: account_failure(
|
||||
category,
|
||||
failure.retry_after_seconds,
|
||||
failure.usage_limit_resets_at,
|
||||
),
|
||||
error_message: failure.client_message,
|
||||
cyber_policy_failure,
|
||||
set_cookie_headers: failure.set_cookie_headers,
|
||||
rate_limit_headers: failure.rate_limit_headers,
|
||||
confirmed_exhaustion_reset_at: failure.usage_limit_resets_at,
|
||||
observation,
|
||||
capture_response_cookies: !matches!(
|
||||
category,
|
||||
@@ -2245,6 +2233,7 @@ const fn upstream_send_state(phase: CodexUpstreamSendPhase) -> UpstreamSendState
|
||||
fn account_failure(
|
||||
category: CodexFailureCategory,
|
||||
retry_after_seconds: Option<u64>,
|
||||
usage_limit_resets_at: Option<i64>,
|
||||
) -> Option<CodexAccountFailure> {
|
||||
match category {
|
||||
CodexFailureCategory::CredentialExpired => Some(CodexAccountFailure::CredentialExpired),
|
||||
@@ -2254,7 +2243,11 @@ fn account_failure(
|
||||
CodexFailureCategory::Banned => Some(CodexAccountFailure::Banned),
|
||||
CodexFailureCategory::UsageLimitExhausted => {
|
||||
Some(CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: retry_after_seconds.map(Duration::from_secs),
|
||||
reset_at: usage_limit_resets_at
|
||||
.and_then(|seconds| u64::try_from(seconds).ok())
|
||||
.and_then(|seconds| {
|
||||
SystemTime::UNIX_EPOCH.checked_add(Duration::from_secs(seconds))
|
||||
}),
|
||||
})
|
||||
}
|
||||
CodexFailureCategory::RateLimited => Some(CodexAccountFailure::RateLimited {
|
||||
|
||||
@@ -5,9 +5,7 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, TimeZone as _, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::model::accounts::{
|
||||
AccountAvailability as AdminAccountAvailability, AccountRecord,
|
||||
};
|
||||
use gateway_admin::model::accounts::AccountRecord;
|
||||
use gateway_admin::model::observability::{
|
||||
CurrencyCost, DesktopReleaseStatus, ProviderBillingInput,
|
||||
};
|
||||
@@ -20,8 +18,8 @@ use gateway_admin::model::provider_credentials::{
|
||||
use gateway_admin::model::{MutationActor, MutationContext, Revision};
|
||||
use gateway_admin::ports::provider::ProviderAdminErrorKind;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability as CoreAccountAvailability, AccountStateChange, CredentialRevision,
|
||||
OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation,
|
||||
CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
|
||||
ProviderAccountStore, QuotaAccessChange, QuotaEvidence, QuotaObservation, QuotaState,
|
||||
};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -515,15 +513,14 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc
|
||||
}
|
||||
}]
|
||||
});
|
||||
let observed_at = SystemTime::now();
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at,
|
||||
state: QuotaState::observed_unknown(observed_at),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -596,7 +593,7 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutating_raw_usage() {
|
||||
async fn openai_admin_keeps_confirmed_exhaustion_separate_from_raw_usage_display() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_admin_confirmed_exhaustion";
|
||||
store
|
||||
@@ -611,11 +608,12 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.await;
|
||||
let account = store.account(account_id).expect("stored account");
|
||||
let reset_at = 1_900_000_000_u64;
|
||||
let observed_at = SystemTime::now();
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota: OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
@@ -626,19 +624,17 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
),
|
||||
observed_at,
|
||||
state: QuotaState::allowed(observed_at),
|
||||
})
|
||||
.await
|
||||
.expect("persist raw quota");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::exhausted(QuotaEvidence::UsageLimitReached, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
@@ -661,7 +657,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.expect("project exhausted quota");
|
||||
|
||||
assert_eq!(exhausted.windows.len(), 1);
|
||||
assert_eq!(exhausted.windows[0].used_percent, Some(100.0));
|
||||
assert_eq!(exhausted.windows[0].used_percent, Some(86.0));
|
||||
let raw = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
@@ -669,17 +665,15 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.pop()
|
||||
.expect("raw quota");
|
||||
assert_eq!(
|
||||
raw.quota.expect("raw document").expose_to_provider()["rate_limit"]["primary_window"]["used_percent"],
|
||||
raw.quota.expose_to_provider()["rate_limit"]["primary_window"]["used_percent"],
|
||||
86
|
||||
);
|
||||
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::allowed(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("recover account");
|
||||
@@ -735,7 +729,7 @@ async fn openai_admin_provider_rejects_unprepared_mutations_before_store_commit(
|
||||
stale_record.access_token_expires_at = None;
|
||||
stale_record.next_refresh_at = None;
|
||||
stale_record.enabled = false;
|
||||
stale_record.availability = AdminAccountAvailability::Banned;
|
||||
stale_record.credential_state = CredentialState::Banned;
|
||||
let rotation_error = admin
|
||||
.prepare_rotation(PrepareCredentialRotation {
|
||||
account: stale_record,
|
||||
@@ -801,10 +795,11 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability: AdminAccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
credential_state: account.credential_state(),
|
||||
credential_observed_at: now,
|
||||
quota: account.quota(),
|
||||
last_error_reason: account.last_error_reason(),
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
|
||||
@@ -82,14 +82,6 @@ fn openai_config_restricts_upstream_base_url_to_https_or_loopback_http() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_config_keeps_the_quota_exhaustion_scheduling_switch() {
|
||||
let mut config = valid_config();
|
||||
config.quota.skip_exhausted = false;
|
||||
|
||||
assert!(!config.quota_skip_exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
let config = OpenAiConfig::default();
|
||||
@@ -104,7 +96,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
config.ws_pool.max_connecting,
|
||||
config.ws_pool.initial_event_timeout_ms,
|
||||
config.quota.refresh_interval_minutes,
|
||||
config.quota.skip_exhausted,
|
||||
config.auth.refresh_enabled,
|
||||
config.auth.oauth_client_id.as_str(),
|
||||
config.auth.oauth_token_endpoint.as_str(),
|
||||
@@ -119,7 +110,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
20_000,
|
||||
15,
|
||||
true,
|
||||
true,
|
||||
"app_EMoamEEZ73f0CkXaXp7hrann",
|
||||
"https://auth.openai.com/oauth/token",
|
||||
)
|
||||
|
||||
@@ -9,8 +9,8 @@ use chrono::{TimeZone, Utc};
|
||||
use ed25519_dalek::pkcs8::EncodePrivateKey as _;
|
||||
use ed25519_dalek::{Signature, SigningKey, Verifier as _};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, NewProviderAccount, ProviderAccount,
|
||||
ProviderAccountId, ProviderAccountStore,
|
||||
CredentialRevision, CredentialState, NewProviderAccount, ProviderAccount, ProviderAccountId,
|
||||
ProviderAccountStore, QuotaState,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::credential::{
|
||||
@@ -65,7 +65,13 @@ fn agent_account(
|
||||
Some("agent-account".to_owned()),
|
||||
Some("pro".to_owned()),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.with_refresh_schedule(false, None);
|
||||
NewProviderAccount {
|
||||
account,
|
||||
|
||||
@@ -8,9 +8,10 @@ use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy,
|
||||
AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _,
|
||||
QuotaObservation, QuotaWriteOutcome, RotationStrategy,
|
||||
AccountAttemptFeedback, AccountErrorReason, AccountFeedbackStats, AccountSelectionPolicy,
|
||||
AccountStateChange, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
|
||||
ProviderAccountStore as _, QuotaAccessChange, QuotaAccessState, QuotaEvidence,
|
||||
QuotaObservation, QuotaState, QuotaWriteOutcome, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
|
||||
@@ -105,7 +106,6 @@ fn selector_with_affinity(
|
||||
leases,
|
||||
session_affinity,
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
@@ -115,7 +115,6 @@ fn selector_with_runtime(
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
skip_exhausted: bool,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> CodexCredentialSelector {
|
||||
let profile = CodexWireProfileState::new(CodexWireProfile {
|
||||
@@ -158,10 +157,42 @@ fn selector_with_runtime(
|
||||
agent_identity,
|
||||
account_feedback,
|
||||
CodexCookiePolicy::official().expect("official cookie policy"),
|
||||
skip_exhausted,
|
||||
)
|
||||
}
|
||||
|
||||
fn persist_credential_state(
|
||||
store: &MemoryAccountStore,
|
||||
account: &ProviderAccount,
|
||||
credential_state: CredentialState,
|
||||
) {
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
credential_state,
|
||||
observed_at: SystemTime::now(),
|
||||
error_reason: credential_state.error_reason(),
|
||||
message: None,
|
||||
}))
|
||||
.expect("persist credential state");
|
||||
}
|
||||
|
||||
fn persist_quota_exhaustion(
|
||||
store: &MemoryAccountStore,
|
||||
account: &ProviderAccount,
|
||||
reset_at: Option<SystemTime>,
|
||||
) {
|
||||
block_on(store.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
state: QuotaState::exhausted(
|
||||
QuotaEvidence::UsageLimitReached,
|
||||
SystemTime::now(),
|
||||
reset_at,
|
||||
),
|
||||
}))
|
||||
.expect("persist quota exhaustion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_persists_tokens_as_plaintext_provider_json() {
|
||||
let encoded = CodexCredentialCodec::encode_new(
|
||||
@@ -485,7 +516,6 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
affinity,
|
||||
Arc::clone(&account_feedback),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
for _ in 0..4 {
|
||||
@@ -528,16 +558,7 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
|
||||
create_account(&store, "acct_first", "at-first");
|
||||
create_account(&store, "acct_second", "at-second");
|
||||
let first = store.account("acct_first").expect("first account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: first.id().clone(),
|
||||
expected_revision: first.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark first account exhausted");
|
||||
persist_quota_exhaustion(&store, &first, None);
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let key = ProviderSessionAffinityKey::try_new("quota-session").expect("affinity key");
|
||||
@@ -697,13 +718,15 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Expired
|
||||
.credential_state(),
|
||||
CredentialState::Expired
|
||||
);
|
||||
let account = store.account("acct_primary").expect("expired account");
|
||||
assert_eq!(
|
||||
store.last_error_message("acct_primary").as_deref(),
|
||||
Some("upstream_credential_expired")
|
||||
account.last_error_reason(),
|
||||
Some(AccountErrorReason::AccessTokenExpired)
|
||||
);
|
||||
assert_eq!(account.last_error_message(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -734,13 +757,13 @@ fn credential_expired_failure_keeps_expired_oauth_for_bounded_refresh_recovery()
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account retained for refresh")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
fn rate_limited_failure_records_runtime_cooldown_without_changing_persisted_facts() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
@@ -749,7 +772,6 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
let attempt = attempt(BTreeSet::new());
|
||||
@@ -774,10 +796,9 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
|
||||
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Unknown);
|
||||
assert!(
|
||||
store.quota_json("acct_primary").is_none(),
|
||||
"429 must not synthesize quota window"
|
||||
@@ -800,7 +821,7 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
|
||||
.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
reset_at: Some(SystemTime::now() + Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
)
|
||||
@@ -808,7 +829,11 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
|
||||
.expect("record usage-limit exhaustion");
|
||||
|
||||
let account = store.account("acct_primary").expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Exhausted);
|
||||
assert_eq!(
|
||||
account.quota().evidence(),
|
||||
Some(QuotaEvidence::UsageLimitReached)
|
||||
);
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
}
|
||||
|
||||
@@ -817,14 +842,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
persist_quota_exhaustion(&store, &account, None);
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
@@ -836,13 +854,13 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不写 availability,402 的 QuotaExhausted 保持不变。
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
@@ -857,7 +875,6 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
|
||||
@@ -874,8 +891,8 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
@@ -891,14 +908,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Invalid,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark account invalid");
|
||||
persist_credential_state(&store, &account, CredentialState::Invalid);
|
||||
let current = store.account("acct_primary").expect("invalid account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
@@ -911,34 +921,26 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不写 availability(限流不改变账号可用性),认证终态保持不变。
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::Invalid
|
||||
.credential_state(),
|
||||
CredentialState::Invalid
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_upstream_response_recovers_non_quota_terminal_states() {
|
||||
for stale in [
|
||||
AccountAvailability::Expired,
|
||||
AccountAvailability::Invalid,
|
||||
AccountAvailability::Banned,
|
||||
CredentialState::Expired,
|
||||
CredentialState::Invalid,
|
||||
CredentialState::Banned,
|
||||
] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: stale,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("seed stale state");
|
||||
persist_credential_state(&store, &account, stale);
|
||||
let current = store.account("acct_primary").expect("stale account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
@@ -948,13 +950,34 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("recovered account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
.credential_state(),
|
||||
CredentialState::Ready,
|
||||
"stale state {stale:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elapsed_quota_reset_remains_blocked_until_authoritative_recovery() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
persist_quota_exhaustion(
|
||||
&store,
|
||||
&account,
|
||||
Some(SystemTime::now() - Duration::from_secs(1)),
|
||||
);
|
||||
let blocked = store.account("acct_primary").expect("blocked account");
|
||||
assert!(blocked.quota().is_exhausted());
|
||||
assert_eq!(
|
||||
blocked
|
||||
.status_projection(SystemTime::now(), None)
|
||||
.status
|
||||
.as_str(),
|
||||
"quota_exhausted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -990,52 +1013,13 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
store
|
||||
.account("acct_first")
|
||||
.expect("persisted first account")
|
||||
.availability(),
|
||||
.credential_state(),
|
||||
store
|
||||
.account("acct_second")
|
||||
.expect("persisted second account")
|
||||
.availability(),
|
||||
.credential_state(),
|
||||
],
|
||||
[AccountAvailability::Ready; 2]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip_it() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("mark account exhausted");
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
let request_attempt = attempt(BTreeSet::new());
|
||||
|
||||
let lease = block_on(selector.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &request_url,
|
||||
attempt: &request_attempt,
|
||||
session_affinity_key: None,
|
||||
}))
|
||||
.expect("select quota exhausted account when skipping is disabled");
|
||||
|
||||
assert_eq!(lease.account_id().as_str(), "acct_primary");
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
[CredentialState::Ready; 2]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1054,7 +1038,6 @@ fn native_continuation_surfaces_the_original_accounts_quota_status_to_the_coordi
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
@@ -1108,14 +1091,13 @@ fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordi
|
||||
"primary_window": {"used_percent": 98}
|
||||
}
|
||||
});
|
||||
let observed_at = SystemTime::now();
|
||||
let outcome = block_on(store.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: original.id().clone(),
|
||||
expected_revision: original.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(quota.as_object().expect("quota object").clone()),
|
||||
observed_at,
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
}))
|
||||
.expect("persist quota signal");
|
||||
assert!(matches!(outcome, QuotaWriteOutcome::Updated));
|
||||
@@ -1125,7 +1107,6 @@ fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordi
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
@@ -1194,8 +1175,8 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
store
|
||||
.account(lease.account_id().as_str())
|
||||
.expect("selected account")
|
||||
.availability(),
|
||||
AccountAvailability::Invalid
|
||||
.credential_state(),
|
||||
CredentialState::Invalid
|
||||
);
|
||||
let other = if lease.account_id().as_str() == "acct_primary" {
|
||||
"acct_other"
|
||||
@@ -1203,13 +1184,16 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
"acct_primary"
|
||||
};
|
||||
assert_eq!(
|
||||
store.account(other).expect("other account").availability(),
|
||||
AccountAvailability::Ready
|
||||
store
|
||||
.account(other)
|
||||
.expect("other account")
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloudflare_challenge_does_not_change_availability() {
|
||||
fn cloudflare_challenge_does_not_change_persisted_account_facts() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
@@ -1226,7 +1210,6 @@ fn cloudflare_challenge_does_not_change_availability() {
|
||||
)
|
||||
.expect("select account");
|
||||
|
||||
// Cloudflare 挑战只走内存退避表,不写 availability(限流不改变账号可用性)。
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
@@ -1238,8 +1221,8 @@ fn cloudflare_challenge_does_not_change_availability() {
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
block_on(selector.record_success(lease.account(), None));
|
||||
}
|
||||
@@ -1279,15 +1262,15 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("affected account")
|
||||
.availability(),
|
||||
AccountAvailability::Invalid
|
||||
.credential_state(),
|
||||
CredentialState::Invalid
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_other")
|
||||
.expect("other account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1416,7 +1399,8 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("updated account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use gateway_admin::model::provider_credentials::{
|
||||
AuthorizationMutationTarget, AuthorizationOwnerBinding, PendingAuthorizationMutation,
|
||||
};
|
||||
use gateway_admin::model::{MutationActor, MutationContext};
|
||||
use gateway_core::engine::credential::AccountStatus;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::credential::token_client::{
|
||||
AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant,
|
||||
@@ -182,7 +183,13 @@ async fn first_exchange_uses_official_id_token_claim_mapping_without_signature_v
|
||||
assert_eq!(account.account.upstream_user_id(), Some("chatgpt-user"));
|
||||
assert_eq!(account.account.upstream_account_id(), Some("workspace-id"));
|
||||
assert_eq!(account.account.plan_type(), Some("pro"));
|
||||
assert!(account.account.is_schedulable(SystemTime::now()));
|
||||
assert_eq!(
|
||||
account
|
||||
.account
|
||||
.status_projection(SystemTime::now(), None)
|
||||
.status,
|
||||
AccountStatus::Normal
|
||||
);
|
||||
assert!(
|
||||
account.account.next_refresh_at().is_none(),
|
||||
"normal OAuth creation must not persist a margin-derived refresh time"
|
||||
@@ -201,7 +208,13 @@ async fn first_exchange_permits_missing_identity_claims_but_requires_a_parseable
|
||||
};
|
||||
assert_eq!(account.account.email(), Some("profile@example.com"));
|
||||
assert_eq!(account.account.upstream_user_id(), None);
|
||||
assert!(!account.account.is_schedulable(SystemTime::now()));
|
||||
assert_eq!(
|
||||
account
|
||||
.account
|
||||
.status_projection(SystemTime::now(), None)
|
||||
.status,
|
||||
AccountStatus::Error
|
||||
);
|
||||
|
||||
let error = complete("header.not-base64.signature".to_owned())
|
||||
.await
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,25 +1,15 @@
|
||||
//! 402 恢复回归。
|
||||
//!
|
||||
//! 覆盖 `QuotaExhausted -> Ready` 的证据规则:新 core primary reset 后移 + 新快照用量低于 10%。
|
||||
//! 额度刷新与凭证事实相互独立的回归测试。
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_reset() {
|
||||
async fn quota_refresh_updates_access_fact_without_recovering_credential_error() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_quota_state";
|
||||
let account_id = "acct_independent_quota_and_credential";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("seed stale expired state");
|
||||
persist_credential_state(&store, &account, CredentialState::Expired).await;
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
@@ -45,20 +35,15 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let exhausted = service
|
||||
let snapshot = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh exhausted quota");
|
||||
let current = store.account(account_id).expect("updated account");
|
||||
|
||||
assert!(exhausted.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("exhausted account")
|
||||
.availability(),
|
||||
AccountAvailability::Expired,
|
||||
"manual refresh never changes non-QuotaExhausted availability"
|
||||
);
|
||||
assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted);
|
||||
assert_eq!(current.quota().access(), QuotaAccessState::Exhausted);
|
||||
assert_eq!(current.credential_state(), CredentialState::Expired);
|
||||
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
@@ -77,111 +62,39 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let stale = service
|
||||
let snapshot = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh contradictory full quota");
|
||||
.expect("refresh allowed quota");
|
||||
let current = store.account(account_id).expect("recovered quota account");
|
||||
|
||||
// used=100 是快照级 exhaustion。
|
||||
assert!(stale.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after contradictory quota")
|
||||
.availability(),
|
||||
AccountAvailability::Expired,
|
||||
"an unchanged usage window cannot override an expired credential state"
|
||||
);
|
||||
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 99, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let unchanged_reset = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh lower usage quota");
|
||||
|
||||
assert!(!unchanged_reset.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after lower usage")
|
||||
.availability(),
|
||||
AccountAvailability::Expired,
|
||||
"a lower used_percent before the recorded reset is not quota recovery for an expired credential"
|
||||
);
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(0));
|
||||
assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed);
|
||||
assert_eq!(current.quota().access(), QuotaAccessState::Allowed);
|
||||
assert_eq!(current.credential_state(), CredentialState::Expired);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() {
|
||||
// 旧 reset 仍在未来;同一主窗口 reset 后移且用量低于 10% 时才恢复 Ready。
|
||||
async fn explicit_allowance_replaces_confirmed_exhaustion_without_usage_heuristics() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_cross_window_recovery";
|
||||
let account_id = "acct_authoritative_quota_recovery";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let old_reset = Utc::now().timestamp() + 3_600;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": old_reset,
|
||||
"limit_window_seconds": 3600
|
||||
}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed old exhausted snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
persist_quota_state(
|
||||
&store,
|
||||
&account,
|
||||
exhausted_quota(Some(SystemTime::now() + Duration::from_secs(3_600))),
|
||||
)
|
||||
.await;
|
||||
|
||||
let server = MockServer::start().await;
|
||||
let future_reset = old_reset + 3_600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 9.9, "reset_at": future_reset}
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
@@ -195,77 +108,36 @@ async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() {
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let recovered = service
|
||||
let snapshot = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh new window quota");
|
||||
.expect("refresh authoritative allowance");
|
||||
|
||||
assert!(!recovered.fact().exhausted());
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(0));
|
||||
assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after cross-window recovery")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
"an advanced reset with a nearly unused primary window must recover QuotaExhausted"
|
||||
.expect("updated account")
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Allowed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaustion() {
|
||||
async fn inconclusive_quota_refresh_preserves_confirmed_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_cross_window_ten_percent";
|
||||
let account_id = "acct_inconclusive_quota_refresh";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let old_reset = Utc::now().timestamp() + 3_600;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": old_reset,
|
||||
"limit_window_seconds": 3600
|
||||
}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed old exhausted snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
persist_quota_state(&store, &account, exhausted_quota(None)).await;
|
||||
let server = MockServer::start().await;
|
||||
let future_reset = old_reset + 3_600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 10.0, "reset_at": future_reset}
|
||||
"primary_window": {"used_percent": 12, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
@@ -279,24 +151,24 @@ async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaus
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let held = service
|
||||
let snapshot = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh non-empty new window quota");
|
||||
.expect("refresh inconclusive quota");
|
||||
|
||||
assert!(held.fact().exhausted());
|
||||
assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after ten-percent snapshot")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted,
|
||||
"a reset advance at exactly 10% must not release confirmed exhaustion"
|
||||
.expect("preserved account")
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
|
||||
async fn deactivated_workspace_quota_response_persists_credential_error_reason() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_deactivated_workspace";
|
||||
create_account(&store, account_id).await;
|
||||
@@ -304,10 +176,6 @@ async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_json(json!({
|
||||
"detail": {
|
||||
"code": "deactivated_workspace",
|
||||
@@ -326,15 +194,12 @@ async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
|
||||
);
|
||||
|
||||
assert!(service.refresh_account(account.id()).await.is_err());
|
||||
let current = store
|
||||
.account(account_id)
|
||||
.expect("account after deactivation");
|
||||
assert_eq!(current.credential_state(), CredentialState::Banned);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after deactivation")
|
||||
.availability(),
|
||||
AccountAvailability::Banned
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message(account_id).as_deref(),
|
||||
Some("deactivated_workspace")
|
||||
current.last_error_reason(),
|
||||
Some(gateway_core::engine::credential::AccountErrorReason::AccountBanned)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -7,7 +7,8 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use futures::future::join_all;
|
||||
use gateway_core::engine::credential::{
|
||||
OpaqueProviderData, ProviderAccountStore as _, QuotaObservation, QuotaWriteOutcome,
|
||||
OpaqueProviderData, ProviderAccountStore as _, QuotaAccessState, QuotaObservation, QuotaState,
|
||||
QuotaWriteOutcome,
|
||||
};
|
||||
use provider_openai::credential::{CodexQuotaWindowKind, parse_codex_quota_usage};
|
||||
use serde_json::json;
|
||||
@@ -34,11 +35,11 @@ fn parser_extracts_dynamic_windows_without_a_fixed_database_shape() {
|
||||
fact.resets_at().map(|value| value.timestamp()),
|
||||
Some(1_800_000_100)
|
||||
);
|
||||
assert!(!fact.exhausted());
|
||||
assert_eq!(fact.remaining_percent(), Some(20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
|
||||
fn parser_keeps_additional_limit_as_display_data() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {"primary_window": {"used_percent": 10}},
|
||||
"additional_rate_limits": [{
|
||||
@@ -47,7 +48,7 @@ fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
|
||||
}))
|
||||
.expect("valid quota");
|
||||
|
||||
assert!(fact.exhausted());
|
||||
assert_eq!(fact.remaining_percent(), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -61,14 +62,12 @@ fn parser_keeps_full_percent_as_display_without_provider_exhaustion_signal() {
|
||||
}))
|
||||
.expect("valid full-percent quota");
|
||||
|
||||
// 任一子窗口 used>=100 是快照级 exhaustion 事实,
|
||||
// 即使顶层 allowed=true。
|
||||
// 百分比只负责展示;额度访问结论由规范化 QuotaState 承载。
|
||||
assert_eq!(fact.remaining_percent(), Some(0));
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() {
|
||||
fn parser_projects_percent_when_access_is_denied() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
@@ -79,7 +78,6 @@ fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent()
|
||||
.expect("valid provider-confirmed quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(2));
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -94,7 +92,7 @@ fn parser_does_not_infer_exhaustion_from_unknown_credit_fields() {
|
||||
}))
|
||||
.expect("recognized credits object");
|
||||
|
||||
assert!(!fact.exhausted());
|
||||
assert_eq!(fact.remaining_percent(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -129,7 +127,6 @@ fn parser_accepts_official_null_additional_rate_limits() {
|
||||
.expect("official null additional quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(70));
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -192,11 +189,9 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io()
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::allowed(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -218,8 +213,7 @@ async fn persisted_provider_quota_projects_dynamic_windows_without_network_io()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
|
||||
// 过去数年的 1 秒窗口:O(1) 完成,不能线性循环。
|
||||
async fn expired_window_remains_the_last_observation_until_refresh() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_o1_roll").await;
|
||||
let account = store.account("acct_o1_roll").expect("account");
|
||||
@@ -240,11 +234,9 @@ async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::observed_unknown(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -254,20 +246,18 @@ async fn roll_expired_windows_is_o1_for_extremely_old_reset() {
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
let rolled = snapshot.roll_expired_windows(SystemTime::now());
|
||||
let window = &rolled.windows()[0];
|
||||
assert_eq!(window.used_percent(), Some(0.0));
|
||||
assert!(!window.limit_reached());
|
||||
// 下一 reset 必须落在 now 之后(O(1) 公式而非线性累加)。
|
||||
let window = &snapshot.windows()[0];
|
||||
assert_eq!(window.used_percent(), Some(100.0));
|
||||
assert!(
|
||||
window
|
||||
.reset_at()
|
||||
.is_some_and(|reset| SystemTime::from(reset) > SystemTime::now())
|
||||
.is_some_and(|reset| { SystemTime::from(reset) < SystemTime::now() })
|
||||
);
|
||||
assert_eq!(service.scheduling_signals(&account), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
|
||||
async fn persisted_access_fact_survives_limit_without_percent_or_reset() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_signal_only_limit").await;
|
||||
let account = store.account("acct_signal_only_limit").expect("account");
|
||||
@@ -282,11 +272,13 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::exhausted(
|
||||
gateway_core::engine::credential::QuotaEvidence::ProviderDenied,
|
||||
SystemTime::now(),
|
||||
None,
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -296,12 +288,7 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
assert!(snapshot.fact().exhausted());
|
||||
let signals = service.scheduling_signals(&account);
|
||||
assert!(
|
||||
signals.is_some_and(|signals| signals.limit_reached()),
|
||||
"limit_reached without percent/reset must still produce scheduling exclusion"
|
||||
);
|
||||
assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -334,11 +321,9 @@ async fn persisted_codex_alias_keeps_the_top_level_rate_limit_canonical() {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::observed_unknown(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -393,11 +378,9 @@ async fn persisted_quota_orders_core_window_before_additional_limit() {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::observed_unknown(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -459,11 +442,9 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(raw.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::observed_unknown(SystemTime::now()),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -490,5 +471,5 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() {
|
||||
}));
|
||||
assert_ne!(review[0].key(), review[1].key());
|
||||
assert_eq!(snapshot.windows().len(), 3);
|
||||
assert!(!snapshot.fact().exhausted());
|
||||
assert_eq!(snapshot.quota().access(), QuotaAccessState::Unknown);
|
||||
}
|
||||
|
||||
@@ -7,8 +7,9 @@ use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use chrono::Utc;
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountFeedbackStats, OpaqueProviderData, ProviderAccountId,
|
||||
ProviderAccountStore as _, QuotaObservation,
|
||||
AccountFeedbackStats, CredentialState, OpaqueProviderData, ProviderAccountId,
|
||||
ProviderAccountStore as _, QuotaAccessChange, QuotaAccessState, QuotaEvidence,
|
||||
QuotaObservation, QuotaState,
|
||||
};
|
||||
use gateway_core::engine::provider::{Provider as _, ProviderRequest};
|
||||
use gateway_core::engine::{
|
||||
@@ -166,7 +167,6 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
Arc::clone(&agent_identity),
|
||||
Arc::clone(&account_feedback),
|
||||
CodexCookiePolicy::official().expect("cookie policy"),
|
||||
true,
|
||||
));
|
||||
|
||||
let provider = CodexProvider::new(
|
||||
@@ -1414,25 +1414,26 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc
|
||||
vec!["response.created", "response.failed"]
|
||||
);
|
||||
let account = store.account(account_id).expect("rate-limited account");
|
||||
// 429 只记录限流窗口,不改变 availability(限流不改变账号可用性)。
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Unknown);
|
||||
let _ = release.send(());
|
||||
server.abort();
|
||||
let _ = server.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset_advances() {
|
||||
async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_usage_limit_request_path";
|
||||
let reset_at = 1_900_000_000;
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let observed_at = SystemTime::now();
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota: OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
@@ -1444,9 +1445,9 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
.as_object()
|
||||
.expect("stale quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
),
|
||||
observed_at,
|
||||
state: QuotaState::allowed(observed_at),
|
||||
})
|
||||
.await
|
||||
.expect("seed stale passive quota");
|
||||
@@ -1505,7 +1506,11 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
|
||||
assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted);
|
||||
let account = store.account(account_id).expect("usage-limit account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Exhausted);
|
||||
assert_eq!(
|
||||
account.quota().evidence(),
|
||||
Some(QuotaEvidence::UsageLimitReached)
|
||||
);
|
||||
let projected = store
|
||||
.get_quotas(&[account.id().clone()])
|
||||
.await
|
||||
@@ -1513,22 +1518,20 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("confirmed quota observation");
|
||||
let projected_observed_at = projected.observed_at.expect("projection observed at");
|
||||
let projected = Value::Object(
|
||||
projected
|
||||
.quota
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.expect("confirmed quota projection"),
|
||||
);
|
||||
let projected_observed_at = projected.observed_at;
|
||||
let projected = Value::Object(projected.quota.into_inner());
|
||||
assert_eq!(
|
||||
projected
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(Value::as_u64),
|
||||
Some(100)
|
||||
Some(99)
|
||||
);
|
||||
assert!(
|
||||
projected.pointer("/rate_limit/secondary_window").is_none(),
|
||||
"request failure must not manufacture a secondary quota"
|
||||
assert_eq!(
|
||||
projected
|
||||
.pointer("/rate_limit/secondary_window/used_percent")
|
||||
.and_then(Value::as_u64),
|
||||
Some(0),
|
||||
"request failure must not rewrite the raw display document"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
@@ -1549,7 +1552,7 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
.expect("read refreshed quota observation")
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|observation| observation.observed_at);
|
||||
.map(|observation| observation.observed_at);
|
||||
if observed_at.is_some_and(|observed_at| observed_at > projected_observed_at) {
|
||||
break;
|
||||
}
|
||||
@@ -1564,15 +1567,14 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
.expect("read refreshed quota")
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.map(|observation| observation.quota.into_inner())
|
||||
.map(Value::Object)
|
||||
.expect("authoritative quota projection");
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(Value::as_u64),
|
||||
Some(100)
|
||||
Some(99)
|
||||
);
|
||||
assert_eq!(
|
||||
quota
|
||||
@@ -1582,7 +1584,7 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
);
|
||||
assert!(
|
||||
quota.pointer("/rate_limit/secondary_window").is_none(),
|
||||
"background quota refresh must retain the same window projection"
|
||||
"background refresh may normalize empty display windows"
|
||||
);
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
assert_eq!(
|
||||
@@ -1596,8 +1598,9 @@ async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after stale full usage refresh")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted,
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Exhausted,
|
||||
"a contradictory full usage snapshot must not unlock a confirmed exhausted account"
|
||||
);
|
||||
}
|
||||
@@ -1976,12 +1979,11 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("test account");
|
||||
store
|
||||
.repository()
|
||||
.apply_state(
|
||||
&account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
state: QuotaState::exhausted(QuotaEvidence::UsageLimitReached, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("seed quota-exhausted state");
|
||||
store
|
||||
@@ -2033,7 +2035,7 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
.account(account_id)
|
||||
.expect("disabled test account after test");
|
||||
assert!(!account.enabled());
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Exhausted);
|
||||
assert!(!store.has_quota(account_id));
|
||||
assert_eq!(affinity.binding_count(), 0);
|
||||
}
|
||||
@@ -2051,18 +2053,19 @@ async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
});
|
||||
let observed_at = SystemTime::now();
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota: OpaqueProviderData::new(
|
||||
raw_quota
|
||||
.as_object()
|
||||
.expect("quota snapshot object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
),
|
||||
observed_at,
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
})
|
||||
.await
|
||||
.expect("seed quota-limited snapshot");
|
||||
@@ -2118,7 +2121,7 @@ async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() {
|
||||
async fn successful_response_treats_inference_success_as_authoritative_allowance() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_success_exhausted";
|
||||
create_account(&store, account_id).await;
|
||||
@@ -2164,8 +2167,9 @@ async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after successful response")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Allowed
|
||||
);
|
||||
assert!(store.has_quota(account_id));
|
||||
}
|
||||
@@ -2226,10 +2230,9 @@ async fn successful_http_sse_rate_limit_event_persists_structured_exhaustion() {
|
||||
let account = store
|
||||
.account(account_id)
|
||||
.expect("account after HTTP SSE response");
|
||||
assert_eq!(
|
||||
(account.availability(), store.has_quota(account_id),),
|
||||
(AccountAvailability::Ready, true,)
|
||||
);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Allowed);
|
||||
assert!(store.has_quota(account_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -9,10 +9,10 @@ use std::time::{Duration, SystemTime};
|
||||
use async_trait::async_trait;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountRuntimeSignals, AccountStateChange, CredentialCasOutcome,
|
||||
CredentialCasUpdate, CredentialRevision, LoadedCredential, NewProviderAccount,
|
||||
AccountErrorReason, AccountRuntimeSignals, AccountStateChange, CredentialCasOutcome,
|
||||
CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount,
|
||||
OpaqueProviderData, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
ProviderAccountUpdate, QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::{StoreError, StoreErrorKind};
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -38,7 +38,6 @@ struct StoredAccount {
|
||||
credential: gateway_core::engine::credential::PlaintextCredential,
|
||||
quota: Option<QuotaObservation>,
|
||||
state_observed_at: Option<SystemTime>,
|
||||
last_error_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -71,15 +70,6 @@ impl MemoryAccountStore {
|
||||
.map(|stored| stored.account.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn last_error_message(&self, id: &str) -> Option<String> {
|
||||
let id = ProviderAccountId::new(id).ok()?;
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("account store lock")
|
||||
.get(&id)
|
||||
.and_then(|stored| stored.last_error_message.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn quota_reads(&self) -> usize {
|
||||
self.quota_reads.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -102,8 +92,9 @@ impl MemoryAccountStore {
|
||||
.expect("account store lock")
|
||||
.get(&id)
|
||||
.and_then(|stored| stored.quota.as_ref())
|
||||
.and_then(|observation| observation.quota.as_ref())
|
||||
.map(|data| serde_json::Value::Object(data.expose_to_provider().clone()))
|
||||
.map(|observation| {
|
||||
serde_json::Value::Object(observation.quota.expose_to_provider().clone())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn fail_provider_listing(&self) {
|
||||
@@ -125,7 +116,6 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
credential: input.credential,
|
||||
quota: None,
|
||||
state_observed_at: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
@@ -224,7 +214,10 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
AccountRebuild {
|
||||
revision: next,
|
||||
enabled: stored.account.enabled(),
|
||||
availability: stored.account.availability(),
|
||||
credential_state: stored.account.credential_state(),
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: stored.account.last_error_reason(),
|
||||
last_error_message: stored.account.last_error_message().map(str::to_owned),
|
||||
access_token_expires_at,
|
||||
has_refresh_token,
|
||||
next_refresh_at,
|
||||
@@ -262,13 +255,37 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
if stored
|
||||
.quota
|
||||
.as_ref()
|
||||
.and_then(|quota| quota.observed_at)
|
||||
.zip(observation.observed_at)
|
||||
.is_some_and(|(current, incoming)| current > incoming)
|
||||
.is_some_and(|quota| quota.observed_at > observation.observed_at)
|
||||
{
|
||||
return Ok(QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
let quota = observation.state;
|
||||
stored.quota = Some(observation);
|
||||
stored.account = rebuild_account(
|
||||
&stored.account,
|
||||
AccountRebuild::preserving(&stored.account).with_quota(quota),
|
||||
);
|
||||
Ok(QuotaWriteOutcome::Updated)
|
||||
}
|
||||
|
||||
async fn apply_quota_access(
|
||||
&self,
|
||||
change: QuotaAccessChange,
|
||||
) -> Result<QuotaWriteOutcome, StoreError> {
|
||||
let mut accounts = self.accounts.lock().expect("account store lock");
|
||||
let stored = accounts
|
||||
.get_mut(&change.account_id)
|
||||
.ok_or_else(|| store_error(StoreErrorKind::InvalidData))?;
|
||||
if stored.account.revision() != change.expected_revision {
|
||||
return Ok(QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
stored.account = rebuild_account(
|
||||
&stored.account,
|
||||
AccountRebuild::preserving(&stored.account).with_quota(change.state),
|
||||
);
|
||||
if let Some(observation) = &mut stored.quota {
|
||||
observation.state = change.state;
|
||||
}
|
||||
Ok(QuotaWriteOutcome::Updated)
|
||||
}
|
||||
|
||||
@@ -291,7 +308,10 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
AccountRebuild {
|
||||
revision: stored.account.revision(),
|
||||
enabled: stored.account.enabled(),
|
||||
availability: change.availability,
|
||||
credential_state: change.credential_state,
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: change.error_reason,
|
||||
last_error_message: change.message,
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -299,11 +319,6 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
},
|
||||
);
|
||||
stored.state_observed_at = Some(change.observed_at);
|
||||
if change.availability == AccountAvailability::Ready {
|
||||
stored.last_error_message = None;
|
||||
} else if let Some(message) = change.message {
|
||||
stored.last_error_message = Some(message);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -317,7 +332,10 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
AccountRebuild {
|
||||
revision: stored.account.revision(),
|
||||
enabled: stored.account.enabled(),
|
||||
availability: stored.account.availability(),
|
||||
credential_state: stored.account.credential_state(),
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: stored.account.last_error_reason(),
|
||||
last_error_message: stored.account.last_error_message().map(str::to_owned),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -341,7 +359,10 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
AccountRebuild {
|
||||
revision: stored.account.revision(),
|
||||
enabled,
|
||||
availability: stored.account.availability(),
|
||||
credential_state: stored.account.credential_state(),
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: stored.account.last_error_reason(),
|
||||
last_error_message: stored.account.last_error_message().map(str::to_owned),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -368,13 +389,38 @@ const fn store_error(kind: StoreErrorKind) -> StoreError {
|
||||
struct AccountRebuild {
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
quota: QuotaState,
|
||||
last_error_reason: Option<AccountErrorReason>,
|
||||
last_error_message: Option<String>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
profile: Option<(String, Option<String>, Option<String>)>,
|
||||
}
|
||||
|
||||
impl AccountRebuild {
|
||||
fn preserving(account: &ProviderAccount) -> Self {
|
||||
Self {
|
||||
revision: account.revision(),
|
||||
enabled: account.enabled(),
|
||||
credential_state: account.credential_state(),
|
||||
quota: account.quota(),
|
||||
last_error_reason: account.last_error_reason(),
|
||||
last_error_message: account.last_error_message().map(str::to_owned),
|
||||
access_token_expires_at: account.access_token_expires_at(),
|
||||
has_refresh_token: account.has_refresh_token(),
|
||||
next_refresh_at: account.next_refresh_at(),
|
||||
profile: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_quota(mut self, quota: QuotaState) -> Self {
|
||||
self.quota = quota;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> ProviderAccount {
|
||||
let (name, email, plan_type) = rebuild.profile.unwrap_or_else(|| {
|
||||
(
|
||||
@@ -397,7 +443,13 @@ fn rebuild_account(current: &ProviderAccount, rebuild: AccountRebuild) -> Provid
|
||||
current.upstream_account_id().map(str::to_owned),
|
||||
plan_type,
|
||||
)
|
||||
.with_runtime_state(rebuild.enabled, rebuild.availability)
|
||||
.with_account_facts(
|
||||
rebuild.enabled,
|
||||
rebuild.credential_state,
|
||||
rebuild.quota,
|
||||
rebuild.last_error_reason,
|
||||
rebuild.last_error_message,
|
||||
)
|
||||
.with_refresh_schedule(rebuild.has_refresh_token, rebuild.next_refresh_at)
|
||||
}
|
||||
|
||||
@@ -450,7 +502,7 @@ impl ProviderLeasePort for TestLeaseCoordinator {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -660,19 +660,21 @@ impl ProviderAdmin for XaiAdminProvider {
|
||||
refresh,
|
||||
rolling_usage,
|
||||
} = request;
|
||||
self.account(&account_id).await?;
|
||||
let mut account = self.account(&account_id).await?;
|
||||
let lifecycle = self
|
||||
.repository
|
||||
.read_lifecycle(&account_id)
|
||||
.await
|
||||
.map_err(map_repository_error)?;
|
||||
let snapshot = if refresh {
|
||||
Some(
|
||||
let snapshot = Some(
|
||||
self.quota
|
||||
.refresh_account(&account_id)
|
||||
.await
|
||||
.map_err(map_quota_error)?,
|
||||
)
|
||||
);
|
||||
account = self.account(&account_id).await?;
|
||||
snapshot
|
||||
} else {
|
||||
self.quota
|
||||
.read_account(&account_id)
|
||||
@@ -681,6 +683,7 @@ impl ProviderAdmin for XaiAdminProvider {
|
||||
};
|
||||
Ok(project_quota(
|
||||
snapshot,
|
||||
account.quota().is_exhausted(),
|
||||
lifecycle.refresh_token_expires_at().copied(),
|
||||
rolling_usage,
|
||||
))
|
||||
@@ -891,8 +894,8 @@ fn prepared_create(
|
||||
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
|
||||
next_refresh_at: account.next_refresh_at().map(Into::into),
|
||||
enabled: account.enabled(),
|
||||
availability: account.availability(),
|
||||
availability_observed_at: observed_at,
|
||||
credential_state: account.credential_state(),
|
||||
credential_observed_at: observed_at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -995,7 +998,13 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
|
||||
account.upstream_account_id.clone(),
|
||||
account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(account.enabled, account.availability)
|
||||
.with_account_facts(
|
||||
account.enabled,
|
||||
account.credential_state,
|
||||
account.quota,
|
||||
account.last_error_reason,
|
||||
account.last_error_message.clone(),
|
||||
)
|
||||
.with_refresh_schedule(
|
||||
account.has_refresh_token,
|
||||
account.next_refresh_at.map(Into::into),
|
||||
@@ -1012,6 +1021,7 @@ fn account_matches_record(account: &ProviderAccount, record: &AccountRecord) ->
|
||||
|
||||
fn project_quota(
|
||||
snapshot: Option<GrokQuotaSnapshot>,
|
||||
quota_exhausted: bool,
|
||||
refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
rolling_usage: Option<gateway_admin::model::accounts::AccountUsage>,
|
||||
) -> ProviderQuota {
|
||||
@@ -1020,8 +1030,7 @@ fn project_quota(
|
||||
observed_at: None,
|
||||
refresh_token_expires_at,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
limit_reached: quota_exhausted,
|
||||
provider_data: None,
|
||||
};
|
||||
};
|
||||
@@ -1088,10 +1097,7 @@ fn project_quota(
|
||||
observed_at: Some(snapshot.observed_at()),
|
||||
refresh_token_expires_at,
|
||||
windows: vec![window],
|
||||
limit_reached: billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
rate_limited_until: None,
|
||||
limit_reached: quota_exhausted,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,8 @@ use std::time::{Duration, Instant, SystemTime};
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
|
||||
ProviderAccount, ProviderAccountId, QuotaObservation,
|
||||
AccountQuotaSignals, CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount,
|
||||
ProviderAccountId, QuotaObservation, QuotaState,
|
||||
};
|
||||
use gateway_core::engine::provider::ProviderCatalogGeneration;
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -19,7 +19,6 @@ use gateway_core::routing::ProviderKind;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use super::repository::{GrokCredentialRepository, LoadedGrokCredential};
|
||||
use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState};
|
||||
use crate::XaiWireProfileState;
|
||||
use crate::transport::catalog::{MAX_CATALOG_MODELS, valid_model_slug, validate_etag};
|
||||
use crate::{
|
||||
@@ -445,15 +444,17 @@ impl GrokCredentialQuotaService {
|
||||
let observed_at = Utc::now();
|
||||
let document = billing.into_document();
|
||||
let presentation = billing_presentation(&document)?;
|
||||
let quota = loaded
|
||||
.account
|
||||
.quota()
|
||||
.merge_observation(quota_access_observation(&presentation, observed_at.into()));
|
||||
self.repository
|
||||
.replace_quota(
|
||||
loaded.account.id().clone(),
|
||||
loaded.account.revision(),
|
||||
document,
|
||||
observed_at.into(),
|
||||
presentation
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
quota,
|
||||
)
|
||||
.await
|
||||
.map_err(map_quota_repository_error)?;
|
||||
@@ -464,32 +465,6 @@ impl GrokCredentialQuotaService {
|
||||
billing: presentation,
|
||||
};
|
||||
self.scheduling.observe(&snapshot);
|
||||
let current = self
|
||||
.repository
|
||||
.load_current(account_id)
|
||||
.await
|
||||
.map_err(map_quota_repository_error)?;
|
||||
if quota_refresh_may_update_state(&loaded.account, ¤t.account)
|
||||
&& let Some(availability) = quota_refresh_availability(
|
||||
current.account.availability(),
|
||||
authoritative_quota_exhaustion(snapshot.billing()),
|
||||
)
|
||||
{
|
||||
self.repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: current.account.id().clone(),
|
||||
expected_revision: current.account.revision(),
|
||||
availability,
|
||||
availability_reason: matches!(
|
||||
availability,
|
||||
GrokCredentialAvailability::QuotaExhausted
|
||||
)
|
||||
.then_some("quota_exhausted".to_owned()),
|
||||
observed_at,
|
||||
})
|
||||
.await
|
||||
.map_err(map_quota_repository_error)?;
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
@@ -506,10 +481,8 @@ impl GrokCredentialQuotaService {
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (Some(quota), Some(observed_at)) = (observation.quota, observation.observed_at) else {
|
||||
return Err(GrokQuotaError::InvalidData);
|
||||
};
|
||||
let document = quota.into_inner();
|
||||
let observed_at = observation.observed_at;
|
||||
let document = observation.quota.into_inner();
|
||||
let snapshot = GrokQuotaSnapshot {
|
||||
account_id: observation.account_id,
|
||||
credential_revision: observation.expected_revision,
|
||||
@@ -1150,14 +1123,10 @@ fn eligible_catalog_candidate(candidate: &LoadedGrokCredential) -> bool {
|
||||
&& candidate
|
||||
.refresh_token_expires_at
|
||||
.is_none_or(|expires_at| expires_at > Utc::now())
|
||||
&& match account.availability() {
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::QuotaExhausted => true,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => false,
|
||||
}
|
||||
&& matches!(
|
||||
account.credential_state(),
|
||||
CredentialState::Unknown | CredentialState::Ready
|
||||
)
|
||||
}
|
||||
|
||||
async fn fetch_candidate_catalog(
|
||||
@@ -1273,8 +1242,8 @@ fn billing_presentation(
|
||||
}
|
||||
|
||||
fn quota_snapshot_from_observation(observation: &QuotaObservation) -> Option<GrokQuotaSnapshot> {
|
||||
let observed_at = DateTime::<Utc>::from(observation.observed_at?);
|
||||
let document = observation.quota.as_ref()?.expose_to_provider();
|
||||
let observed_at = DateTime::<Utc>::from(observation.observed_at);
|
||||
let document = observation.quota.expose_to_provider();
|
||||
Some(GrokQuotaSnapshot {
|
||||
account_id: observation.account_id.clone(),
|
||||
credential_revision: observation.expected_revision,
|
||||
@@ -1293,29 +1262,54 @@ fn quota_projection_ttl(observed_at: DateTime<Utc>) -> Option<Duration> {
|
||||
}
|
||||
|
||||
fn quota_scheduling_signals(billing: &GrokBillingPresentation) -> Option<AccountQuotaSignals> {
|
||||
let remaining_rank = billing
|
||||
.used_percent()
|
||||
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
|
||||
.map(|used| ((100.0 - used) * 100.0).round() as u64);
|
||||
let now = SystemTime::now();
|
||||
let reset_at = billing
|
||||
let period_end = billing
|
||||
.period_end()
|
||||
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
|
||||
.map(|value| SystemTime::from(value.to_utc()));
|
||||
let observation_expired = period_end.is_some_and(|period_end| period_end <= now);
|
||||
let remaining_rank = (!observation_expired)
|
||||
.then(|| {
|
||||
billing
|
||||
.used_percent()
|
||||
.filter(|used| used.is_finite() && (0.0..=100.0).contains(used))
|
||||
.map(|used| ((100.0 - used) * 100.0).round() as u64)
|
||||
})
|
||||
.flatten();
|
||||
let reset_at = period_end.filter(|reset_at| *reset_at > now);
|
||||
(remaining_rank.is_some() || reset_at.is_some())
|
||||
.then(|| AccountQuotaSignals::new(reset_at, remaining_rank))
|
||||
}
|
||||
|
||||
/// 把 xAI billing 文档归一化为访问结论。缺少正反证据时返回 Unknown,
|
||||
/// 由 `QuotaState::merge_observation` 保留既有已确认结论。
|
||||
fn quota_access_observation(
|
||||
billing: &GrokBillingPresentation,
|
||||
observed_at: SystemTime,
|
||||
) -> QuotaState {
|
||||
if !billing.has_authoritative_quota() {
|
||||
return QuotaState::observed_unknown(observed_at);
|
||||
}
|
||||
if billing_proves_access_allowed(billing, observed_at) {
|
||||
QuotaState::allowed(observed_at)
|
||||
} else {
|
||||
QuotaState::observed_unknown(observed_at)
|
||||
}
|
||||
}
|
||||
|
||||
fn billing_proves_access_allowed(
|
||||
billing: &GrokBillingPresentation,
|
||||
observed_at: SystemTime,
|
||||
) -> bool {
|
||||
if billing
|
||||
.period_end()
|
||||
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
|
||||
.map(|value| SystemTime::from(value.to_utc()))
|
||||
.filter(|reset_at| *reset_at > now);
|
||||
(remaining_rank.is_some() || reset_at.is_some()).then(|| {
|
||||
AccountQuotaSignals::new(
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool {
|
||||
if billing
|
||||
.is_some_and(|period_end| period_end <= observed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
billing
|
||||
.prepaid_balance_cents()
|
||||
.is_some_and(|balance| balance > 0)
|
||||
|| billing
|
||||
@@ -1329,60 +1323,6 @@ fn quota_is_exhausted(billing: &GrokBillingPresentation) -> bool {
|
||||
(billing.included_used_cents(), billing.monthly_limit_cents()),
|
||||
(Some(used), Some(limit)) if limit > 0 && used < limit
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0)
|
||||
|| matches!(
|
||||
(billing.included_used_cents(), billing.monthly_limit_cents()),
|
||||
(Some(used), Some(limit)) if limit > 0 && used >= limit
|
||||
)
|
||||
|| matches!(
|
||||
(billing.on_demand_used_cents(), billing.on_demand_cap_cents()),
|
||||
(Some(used), Some(cap)) if cap > 0 && used >= cap
|
||||
)
|
||||
}
|
||||
|
||||
/// `None` means the billing document cannot prove either quota state.
|
||||
fn authoritative_quota_exhaustion(billing: &GrokBillingPresentation) -> Option<bool> {
|
||||
billing
|
||||
.has_authoritative_quota()
|
||||
.then(|| quota_is_exhausted(billing))
|
||||
}
|
||||
|
||||
fn quota_refresh_may_update_state(before: &ProviderAccount, current: &ProviderAccount) -> bool {
|
||||
current.enabled()
|
||||
&& current
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at > SystemTime::now())
|
||||
&& current.revision() == before.revision()
|
||||
&& current.availability() == before.availability()
|
||||
}
|
||||
|
||||
fn quota_refresh_availability(
|
||||
current: AccountAvailability,
|
||||
exhausted: Option<bool>,
|
||||
) -> Option<GrokCredentialAvailability> {
|
||||
match current {
|
||||
AccountAvailability::Invalid
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned => None,
|
||||
AccountAvailability::QuotaExhausted => {
|
||||
matches!(exhausted, Some(false)).then_some(GrokCredentialAvailability::Ready)
|
||||
}
|
||||
AccountAvailability::Unknown => exhausted.map(|exhausted| {
|
||||
if exhausted {
|
||||
GrokCredentialAvailability::QuotaExhausted
|
||||
} else {
|
||||
GrokCredentialAvailability::Ready
|
||||
}
|
||||
}),
|
||||
AccountAvailability::Ready => {
|
||||
matches!(exhausted, Some(true)).then_some(GrokCredentialAvailability::QuotaExhausted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn dynamic_string(
|
||||
|
||||
@@ -63,10 +63,9 @@ pub use token::{
|
||||
parse_refresh_success,
|
||||
};
|
||||
pub use types::{
|
||||
CreateGrokCredential, GrokAccountProfile, GrokCredentialAvailability, GrokCredentialRecord,
|
||||
GrokOAuthSecret, PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard,
|
||||
RotateGrokCredential, RotateManagedGrokCredential, UpdateGrokCredentialState,
|
||||
XAI_AUTHENTICATION_KIND_OAUTH,
|
||||
CreateGrokCredential, GrokAccountProfile, GrokCredentialRecord, GrokOAuthSecret,
|
||||
PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, RotateGrokCredential,
|
||||
RotateManagedGrokCredential, UpdateGrokCredentialState, XAI_AUTHENTICATION_KIND_OAUTH,
|
||||
};
|
||||
pub use verification::{
|
||||
FailClosedTokenVerifier, TokenCandidate, TokenVerificationContext, TokenVerifier,
|
||||
|
||||
@@ -9,7 +9,7 @@ use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::{StreamExt as _, stream};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, LoadedCredential, ProviderAccountId,
|
||||
AccountErrorReason, CredentialRevision, CredentialState, LoadedCredential, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
@@ -23,9 +23,8 @@ use super::repository::{
|
||||
GrokCredentialAdmin, GrokCredentialRepository, GrokCredentialRepositoryError,
|
||||
};
|
||||
use super::types::{
|
||||
GrokAccountProfile, GrokCredentialAvailability, GrokOAuthSecret,
|
||||
PreparedGrokCredentialRotation, RotateGrokCredential, RotateManagedGrokCredential,
|
||||
UpdateGrokCredentialState,
|
||||
GrokAccountProfile, GrokOAuthSecret, PreparedGrokCredentialRotation, RotateGrokCredential,
|
||||
RotateManagedGrokCredential, UpdateGrokCredentialState,
|
||||
};
|
||||
use crate::{
|
||||
DiscoveryDocument, FailureClass, GrokOAuthClient, OAuthError, RefreshTokenGrant,
|
||||
@@ -540,7 +539,8 @@ impl GrokCredentialRefreshService {
|
||||
return self
|
||||
.persist_terminal_failure(
|
||||
credential,
|
||||
GrokCredentialAvailability::Expired,
|
||||
CredentialState::Expired,
|
||||
AccountErrorReason::CredentialExpired,
|
||||
"refresh_token_expired",
|
||||
)
|
||||
.await;
|
||||
@@ -549,7 +549,8 @@ impl GrokCredentialRefreshService {
|
||||
return self
|
||||
.persist_terminal_failure(
|
||||
credential,
|
||||
GrokCredentialAvailability::Expired,
|
||||
CredentialState::Expired,
|
||||
AccountErrorReason::CredentialExpired,
|
||||
"refresh_recovery_window_exhausted",
|
||||
)
|
||||
.await;
|
||||
@@ -587,7 +588,8 @@ impl GrokCredentialRefreshService {
|
||||
Err(GrokRefreshFailure::InvalidGrant) => {
|
||||
self.persist_terminal_failure(
|
||||
credential,
|
||||
GrokCredentialAvailability::Expired,
|
||||
CredentialState::Expired,
|
||||
AccountErrorReason::CredentialExpired,
|
||||
"refresh_invalid_grant",
|
||||
)
|
||||
.await
|
||||
@@ -595,7 +597,8 @@ impl GrokCredentialRefreshService {
|
||||
Err(GrokRefreshFailure::Banned) => {
|
||||
self.persist_terminal_failure(
|
||||
credential,
|
||||
GrokCredentialAvailability::Banned,
|
||||
CredentialState::Banned,
|
||||
AccountErrorReason::AccountBanned,
|
||||
"account_banned",
|
||||
)
|
||||
.await
|
||||
@@ -717,7 +720,8 @@ impl GrokCredentialRefreshService {
|
||||
async fn persist_terminal_failure(
|
||||
&self,
|
||||
credential: DueGrokCredential,
|
||||
availability: GrokCredentialAvailability,
|
||||
credential_state: CredentialState,
|
||||
error_reason: AccountErrorReason,
|
||||
reason: &str,
|
||||
) -> Result<GrokCredentialRefreshOutcome, GrokCredentialRefreshError> {
|
||||
let account_id = credential.account_id.clone();
|
||||
@@ -737,8 +741,9 @@ impl GrokCredentialRefreshService {
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: credential.credential_revision,
|
||||
availability,
|
||||
availability_reason: Some(reason.to_owned()),
|
||||
credential_state,
|
||||
error_reason: Some(error_reason),
|
||||
error_message: Some(reason.to_owned()),
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -746,7 +751,7 @@ impl GrokCredentialRefreshService {
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
?availability,
|
||||
?credential_state,
|
||||
reason,
|
||||
access_token_expires_at = ?credential.access_token_expires_at
|
||||
.map(DateTime::<Utc>::from),
|
||||
@@ -915,14 +920,11 @@ fn account_due(
|
||||
policy: ProviderRefreshPolicy,
|
||||
now: SystemTime,
|
||||
) -> bool {
|
||||
let availability = account.availability();
|
||||
account.enabled()
|
||||
&& account.has_refresh_token()
|
||||
&& !matches!(
|
||||
availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
account.credential_state(),
|
||||
CredentialState::Expired | CredentialState::Banned | CredentialState::Invalid
|
||||
)
|
||||
&& (refresh_recovery_window_exhausted(account.access_token_expires_at(), now)
|
||||
|| (access_token_refresh_due(account.access_token_expires_at(), policy, now)
|
||||
|
||||
@@ -6,10 +6,10 @@ use std::time::SystemTime;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData,
|
||||
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
|
||||
CredentialState, LoadedCredential, NewProviderAccount, OpaqueProviderData, PlaintextCredential,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -17,8 +17,8 @@ use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::types::{
|
||||
CreateGrokCredential, GrokAccountProfile, GrokCredentialAvailability, GrokCredentialRecord,
|
||||
GrokOAuthSecret, GrokOAuthSecretWire, OwnedGrokOAuthSecretWire, PreparedGrokCredentialRotation,
|
||||
CreateGrokCredential, GrokAccountProfile, GrokCredentialRecord, GrokOAuthSecret,
|
||||
GrokOAuthSecretWire, OwnedGrokOAuthSecretWire, PreparedGrokCredentialRotation,
|
||||
RotateGrokCredential, RotateManagedGrokCredential, UpdateGrokCredentialState,
|
||||
XAI_AUTHENTICATION_KIND_OAUTH,
|
||||
};
|
||||
@@ -167,7 +167,13 @@ impl GrokCredentialAdmin {
|
||||
input.account.upstream_account_id.clone(),
|
||||
input.account.plan_type.clone(),
|
||||
)
|
||||
.with_runtime_state(input.enabled, input.initial_availability.into())
|
||||
.with_account_facts(
|
||||
input.enabled,
|
||||
input.initial_credential_state,
|
||||
QuotaState::unknown(),
|
||||
input.initial_error_reason,
|
||||
input.initial_error_message.clone(),
|
||||
)
|
||||
.with_refresh_schedule(true, None);
|
||||
Ok(NewProviderAccount {
|
||||
account,
|
||||
@@ -210,8 +216,9 @@ impl GrokCredentialAdmin {
|
||||
refresh_token_expires_at: None,
|
||||
},
|
||||
enabled: input.enabled,
|
||||
initial_availability: GrokCredentialAvailability::Ready,
|
||||
initial_availability_reason: None,
|
||||
initial_credential_state: CredentialState::Ready,
|
||||
initial_error_reason: None,
|
||||
initial_error_message: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -420,21 +427,30 @@ impl GrokCredentialRepository {
|
||||
&self,
|
||||
input: &UpdateGrokCredentialState,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
let availability = AccountAvailability::from(input.availability);
|
||||
let availability_reason = input
|
||||
.availability_reason
|
||||
let error_message = input
|
||||
.error_message
|
||||
.clone()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
|
||||
validate_reason(availability_reason.as_deref())?;
|
||||
.filter(|value| !value.trim().is_empty());
|
||||
validate_reason(error_message.as_deref())?;
|
||||
let (error_reason, error_message) = if input.credential_state == CredentialState::Ready {
|
||||
(None, None)
|
||||
} else {
|
||||
(
|
||||
input
|
||||
.error_reason
|
||||
.or_else(|| input.credential_state.error_reason()),
|
||||
error_message,
|
||||
)
|
||||
};
|
||||
self.ensure_xai_account(&input.account_id).await?;
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: input.expected_revision,
|
||||
availability,
|
||||
credential_state: input.credential_state,
|
||||
observed_at: to_system_time(input.observed_at),
|
||||
message: availability_reason,
|
||||
error_reason,
|
||||
message: error_message,
|
||||
})
|
||||
.await
|
||||
.map_err(map_store_error)
|
||||
@@ -518,16 +534,39 @@ impl GrokCredentialRepository {
|
||||
expected_revision: CredentialRevision,
|
||||
document: serde_json::Map<String, Value>,
|
||||
observed_at: SystemTime,
|
||||
limit_reached: bool,
|
||||
state: QuotaState,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id,
|
||||
expected_revision,
|
||||
quota: Some(OpaqueProviderData::new(document)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(limit_reached),
|
||||
quota: OpaqueProviderData::new(document),
|
||||
observed_at,
|
||||
state,
|
||||
})
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
match outcome {
|
||||
QuotaWriteOutcome::Updated => Ok(()),
|
||||
QuotaWriteOutcome::Conflict => {
|
||||
Err(GrokCredentialRepositoryError::StaleCredentialRevision)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_quota_access(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
expected_revision: CredentialRevision,
|
||||
state: QuotaState,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
let outcome = self
|
||||
.store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id,
|
||||
expected_revision,
|
||||
state,
|
||||
})
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
@@ -698,7 +737,7 @@ fn validate_create(input: &CreateGrokCredential) -> Result<(), GrokCredentialRep
|
||||
validate_name(&input.name)?;
|
||||
validate_profile(&input.account)?;
|
||||
validate_secret(&input.secret)?;
|
||||
validate_reason(input.initial_availability_reason.as_deref())
|
||||
validate_reason(input.initial_error_message.as_deref())
|
||||
}
|
||||
|
||||
fn validate_profile(profile: &GrokAccountProfile) -> Result<(), GrokCredentialRepositoryError> {
|
||||
@@ -786,9 +825,7 @@ fn validate_reason(reason: Option<&str>) -> Result<(), GrokCredentialRepositoryE
|
||||
if reason
|
||||
.is_some_and(|value| value.len() > MAX_REASON_BYTES || value.chars().any(char::is_control))
|
||||
{
|
||||
return Err(GrokCredentialRepositoryError::InvalidInput(
|
||||
"availability_reason",
|
||||
));
|
||||
return Err(GrokCredentialRepositoryError::InvalidInput("error_message"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6,8 +6,9 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
|
||||
AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId,
|
||||
AccountCandidate, AccountEligibilityPolicy, AccountFeedbackStats, AccountSelectionContext,
|
||||
AccountSelector, CredentialState, ProviderAccount, ProviderAccountId, QuotaAccessState,
|
||||
QuotaEvidence, QuotaState,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition,
|
||||
@@ -18,7 +19,7 @@ use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
|
||||
use super::catalog::{GrokCatalogScope, GrokCredentialCatalogCache, GrokCredentialQuotaService};
|
||||
use super::repository::{GrokCredentialRepository, GrokCredentialRepositoryError};
|
||||
use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState};
|
||||
use super::types::UpdateGrokCredentialState;
|
||||
use crate::{
|
||||
GrokCredentialFailure, GrokCredentialFeedbackFuture, GrokSessionBinding, GrokSessionSelection,
|
||||
GrokSessionSelector, GrokSessionSelectorError, GrokSessionSelectorFuture, SecretValue,
|
||||
@@ -82,7 +83,7 @@ impl GrokAccountSessionSelector {
|
||||
&self,
|
||||
request: GrokSessionSelection,
|
||||
) -> Result<SelectedGrokSession, GrokSessionSelectorError> {
|
||||
let diagnostic = request.availability() == AccountAvailabilityPolicy::BypassForDiagnostic;
|
||||
let diagnostic = request.eligibility() == AccountEligibilityPolicy::BypassForDiagnostic;
|
||||
let accounts = self
|
||||
.repository
|
||||
.list_accounts_for_provider()
|
||||
@@ -177,7 +178,7 @@ impl GrokAccountSessionSelector {
|
||||
excluded_accounts: request.excluded_accounts().clone(),
|
||||
preferred_account: request.required_account().cloned().or(affinity_account),
|
||||
round_robin_cursor: scheduling.round_robin_cursor(),
|
||||
availability: request.availability(),
|
||||
eligibility: request.eligibility(),
|
||||
};
|
||||
let mut capacity_denied = false;
|
||||
let mut retry_after = None;
|
||||
@@ -363,23 +364,18 @@ impl GrokAccountSessionSelector {
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn persist_account_failure(
|
||||
async fn persist_quota_failure(
|
||||
&self,
|
||||
session: &SelectedGrokSession,
|
||||
availability: GrokCredentialAvailability,
|
||||
reason: &'static str,
|
||||
observed_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
let update = |expected_revision| UpdateGrokCredentialState {
|
||||
account_id: session.account_id().clone(),
|
||||
expected_revision,
|
||||
availability,
|
||||
availability_reason: Some(reason.to_owned()),
|
||||
observed_at,
|
||||
};
|
||||
match self
|
||||
.repository
|
||||
.update_state(&update(session.credential_revision()))
|
||||
.update_quota_access(
|
||||
session.account_id().clone(),
|
||||
session.credential_revision(),
|
||||
QuotaState::exhausted(QuotaEvidence::UsageLimitReached, observed_at.into(), None),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => return,
|
||||
@@ -410,7 +406,11 @@ impl GrokAccountSessionSelector {
|
||||
};
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.update_state(&update(current.account.revision()))
|
||||
.update_quota_access(
|
||||
session.account_id().clone(),
|
||||
current.account.revision(),
|
||||
QuotaState::exhausted(QuotaEvidence::UsageLimitReached, observed_at.into(), None),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -450,20 +450,15 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
return;
|
||||
}
|
||||
let observed_at = Utc::now();
|
||||
let persistent = match failure {
|
||||
match failure {
|
||||
GrokCredentialFailure::Unauthorized => {
|
||||
self.record_runtime_cooldown(session, UNAUTHORIZED_COOLDOWN)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::QuotaExhausted => (
|
||||
GrokCredentialAvailability::QuotaExhausted,
|
||||
"upstream_quota_exhausted",
|
||||
),
|
||||
GrokCredentialFailure::FreeQuotaExhausted => (
|
||||
GrokCredentialAvailability::QuotaExhausted,
|
||||
"upstream_free_quota_exhausted",
|
||||
),
|
||||
GrokCredentialFailure::QuotaExhausted
|
||||
| GrokCredentialFailure::FreeQuotaExhausted => {
|
||||
self.persist_quota_failure(session, observed_at).await;
|
||||
}
|
||||
// bare 402 没有结构化 quota code,不能证明账号额度耗尽:
|
||||
// 只写短期账号级 runtime cooldown(TransientBackoff),
|
||||
// 不持久化 QuotaExhausted(结构化 QuotaExhausted/FreeQuotaExhausted 走上面)。
|
||||
@@ -472,7 +467,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
return;
|
||||
}
|
||||
// xAI 免费模型额度按该模型滚动窗口恢复:限流写 model-scoped
|
||||
// runtime cooldown(Redis 跨重启保留),不进入持久化账号状态,
|
||||
@@ -485,7 +479,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
bounded_cooldown(retry_after, MODEL_QUOTA_COOLDOWN, MAX_MODEL_COOLDOWN);
|
||||
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::ModelAccessDenied {
|
||||
upstream_model,
|
||||
@@ -498,24 +491,18 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
);
|
||||
self.record_model_runtime_cooldown(session, upstream_model, retry_after)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::RateLimited { retry_after } => {
|
||||
let retry_after = retry_after
|
||||
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::StreamInterrupted => {
|
||||
self.record_runtime_cooldown(session, STREAM_INTERRUPTION_COOLDOWN)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let (availability, reason) = persistent;
|
||||
self.persist_account_failure(session, availability, reason, observed_at)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -531,25 +518,22 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
return;
|
||||
};
|
||||
let account = ¤t.account;
|
||||
if account.revision() != session.credential_revision()
|
||||
|| !account.enabled()
|
||||
|| !matches!(
|
||||
account.availability(),
|
||||
AccountAvailability::Unknown | AccountAvailability::Expired
|
||||
)
|
||||
{
|
||||
if account.revision() != session.credential_revision() || !account.enabled() {
|
||||
return;
|
||||
}
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: GrokCredentialAvailability::Ready,
|
||||
availability_reason: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
let observed_at = Utc::now();
|
||||
if account.credential_state() != CredentialState::Ready
|
||||
&& let Err(error) = self
|
||||
.repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
credential_state: CredentialState::Ready,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
observed_at,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
@@ -557,6 +541,24 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
"xAI account state recovery after successful upstream response failed"
|
||||
);
|
||||
}
|
||||
// 已经发出的并发请求可能晚于耗尽事实成功返回;真实成功可以收敛
|
||||
// 该事实,但 selector 不会为恢复探测而放行耗尽账号。
|
||||
if account.quota().access() == QuotaAccessState::Exhausted
|
||||
&& let Err(error) = self
|
||||
.repository
|
||||
.update_quota_access(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
QuotaState::allowed(observed_at.into()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"xAI quota recovery after successful upstream response failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::fmt;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential,
|
||||
AccountErrorReason, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential,
|
||||
ProviderAccountId, ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::provider_ports::ProviderLeaseGuard;
|
||||
@@ -63,34 +63,6 @@ impl fmt::Debug for GrokAccountProfile {
|
||||
}
|
||||
}
|
||||
|
||||
/// 与 `provider_accounts.availability` 一一对应的 xAI 状态。
|
||||
///
|
||||
/// 限流(429/滚动窗口)不进入持久化状态:由 `ProviderCooldownPort`(Redis 跨重启
|
||||
/// 保留)驱动调度排除,账号状态保持 `Ready`。
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum GrokCredentialAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
Invalid,
|
||||
}
|
||||
|
||||
/// Grok 可用性语义翻入内核标准态的唯一汇入点。
|
||||
impl From<GrokCredentialAvailability> for AccountAvailability {
|
||||
fn from(value: GrokCredentialAvailability) -> Self {
|
||||
match value {
|
||||
GrokCredentialAvailability::Unknown => Self::Unknown,
|
||||
GrokCredentialAvailability::Ready => Self::Ready,
|
||||
GrokCredentialAvailability::QuotaExhausted => Self::QuotaExhausted,
|
||||
GrokCredentialAvailability::Expired => Self::Expired,
|
||||
GrokCredentialAvailability::Banned => Self::Banned,
|
||||
GrokCredentialAvailability::Invalid => Self::Invalid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建一个明文 OAuth Provider account。
|
||||
pub struct CreateGrokCredential {
|
||||
pub account_id: ProviderAccountId,
|
||||
@@ -98,8 +70,9 @@ pub struct CreateGrokCredential {
|
||||
pub secret: GrokOAuthSecret,
|
||||
pub account: GrokAccountProfile,
|
||||
pub enabled: bool,
|
||||
pub initial_availability: GrokCredentialAvailability,
|
||||
pub initial_availability_reason: Option<String>,
|
||||
pub initial_credential_state: CredentialState,
|
||||
pub initial_error_reason: Option<AccountErrorReason>,
|
||||
pub initial_error_message: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for CreateGrokCredential {
|
||||
@@ -111,7 +84,7 @@ impl fmt::Debug for CreateGrokCredential {
|
||||
.field("secret", &"[REDACTED]")
|
||||
.field("account", &self.account)
|
||||
.field("enabled", &self.enabled)
|
||||
.field("initial_availability", &self.initial_availability)
|
||||
.field("initial_credential_state", &self.initial_credential_state)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
@@ -235,8 +208,9 @@ impl fmt::Debug for RotateGrokCredential {
|
||||
pub struct UpdateGrokCredentialState {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: GrokCredentialAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub credential_state: CredentialState,
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub error_message: Option<String>,
|
||||
pub observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
|
||||
@@ -30,25 +30,25 @@ pub use credential::{
|
||||
FailClosedTokenVerifier, FailureClass, FormField, FormValue, GROK_FREE_ROLLING_WINDOW_SECONDS,
|
||||
GrokAccountExport, GrokAccountProfile, GrokAccountSessionSelector, GrokBillingPresentation,
|
||||
GrokCatalogCache, GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin,
|
||||
GrokCredentialAvailability, GrokCredentialCatalogCache, GrokCredentialCatalogError,
|
||||
GrokCredentialCatalogSeed, GrokCredentialCatalogService, GrokCredentialLifecycle,
|
||||
GrokCredentialQuotaService, GrokCredentialRecord, GrokCredentialRecovery,
|
||||
GrokCredentialRecoveryOutcome, GrokCredentialRefreshError, GrokCredentialRefreshOutcome,
|
||||
GrokCredentialRefreshService, GrokCredentialRefresher, GrokCredentialRepository,
|
||||
GrokCredentialRepositoryError, GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate,
|
||||
GrokOAuthImportDocument, GrokOAuthImportEntry, GrokOAuthImportError, GrokOAuthImportMetadata,
|
||||
GrokOAuthImportTokens, GrokOAuthRefreshClient, GrokOAuthSecret, GrokPlanCatalog,
|
||||
GrokQuotaError, GrokQuotaPeriodKind, GrokQuotaSnapshot, GrokRefreshFailure, GrokRefreshTokens,
|
||||
HttpHeader, HttpMethod, OAuthError, OAuthErrorCode, OAuthHttpRequest, OAuthHttpResponse,
|
||||
OAuthHttpTransport, OAuthOperation, OAuthPrincipal, OFFICIAL_CLIENT_ID, OFFICIAL_ISSUER,
|
||||
OFFICIAL_REDIRECT_URI, OFFICIAL_SCOPES, PendingAuthorization, Pkce,
|
||||
PreparedGrokCredentialRotation, PreparedGrokCredentialRotationGuard, ProtocolViolation,
|
||||
RedirectUriAllowlist, RefreshTokenGrant, RefreshedTokenSet, ReqwestOidcTokenVerifier,
|
||||
RotateGrokCredential, RotateManagedGrokCredential, SecretValue, TokenCandidate,
|
||||
TokenVerificationContext, TokenVerifier, TransportFailure, TransportFailureKind,
|
||||
TransportFuture, UpdateGrokCredentialState, VerificationEvidence, VerificationFailure,
|
||||
VerificationFlow, VerificationFuture, VerificationMethod, VerifiedGrokAccount,
|
||||
VerifiedTokenSet, parse_oauth_error, parse_refresh_success,
|
||||
GrokCredentialCatalogCache, GrokCredentialCatalogError, GrokCredentialCatalogSeed,
|
||||
GrokCredentialCatalogService, GrokCredentialLifecycle, GrokCredentialQuotaService,
|
||||
GrokCredentialRecord, GrokCredentialRecovery, GrokCredentialRecoveryOutcome,
|
||||
GrokCredentialRefreshError, GrokCredentialRefreshOutcome, GrokCredentialRefreshService,
|
||||
GrokCredentialRefresher, GrokCredentialRepository, GrokCredentialRepositoryError,
|
||||
GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, GrokOAuthImportDocument,
|
||||
GrokOAuthImportEntry, GrokOAuthImportError, GrokOAuthImportMetadata, GrokOAuthImportTokens,
|
||||
GrokOAuthRefreshClient, GrokOAuthSecret, GrokPlanCatalog, GrokQuotaError, GrokQuotaPeriodKind,
|
||||
GrokQuotaSnapshot, GrokRefreshFailure, GrokRefreshTokens, HttpHeader, HttpMethod, OAuthError,
|
||||
OAuthErrorCode, OAuthHttpRequest, OAuthHttpResponse, OAuthHttpTransport, OAuthOperation,
|
||||
OAuthPrincipal, OFFICIAL_CLIENT_ID, OFFICIAL_ISSUER, OFFICIAL_REDIRECT_URI, OFFICIAL_SCOPES,
|
||||
PendingAuthorization, Pkce, PreparedGrokCredentialRotation,
|
||||
PreparedGrokCredentialRotationGuard, ProtocolViolation, RedirectUriAllowlist,
|
||||
RefreshTokenGrant, RefreshedTokenSet, ReqwestOidcTokenVerifier, RotateGrokCredential,
|
||||
RotateManagedGrokCredential, SecretValue, TokenCandidate, TokenVerificationContext,
|
||||
TokenVerifier, TransportFailure, TransportFailureKind, TransportFuture,
|
||||
UpdateGrokCredentialState, VerificationEvidence, VerificationFailure, VerificationFlow,
|
||||
VerificationFuture, VerificationMethod, VerifiedGrokAccount, VerifiedTokenSet,
|
||||
parse_oauth_error, parse_refresh_success,
|
||||
};
|
||||
pub use provider::GrokBuildProvider;
|
||||
pub use transport::profile::{
|
||||
|
||||
@@ -8,8 +8,8 @@ use async_trait::async_trait;
|
||||
use futures::{StreamExt, future::BoxFuture};
|
||||
use gateway_core::engine::continuation::ContinuationBinding;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountAvailabilityPolicy, AccountFeedbackStats, ProviderAccount,
|
||||
ProviderAccountId, ProviderAccountStore,
|
||||
AccountEligibilityPolicy, AccountFeedbackStats, ProviderAccount, ProviderAccountId,
|
||||
ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::engine::provider::{
|
||||
EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration,
|
||||
@@ -392,10 +392,10 @@ async fn select_grok_session(
|
||||
context.account_selection_policy(),
|
||||
context.deadline(),
|
||||
)
|
||||
.with_availability_policy(if context.is_diagnostic_required_account() {
|
||||
AccountAvailabilityPolicy::BypassForDiagnostic
|
||||
.with_eligibility_policy(if context.is_diagnostic_required_account() {
|
||||
AccountEligibilityPolicy::BypassForDiagnostic
|
||||
} else {
|
||||
AccountAvailabilityPolicy::Enforce
|
||||
AccountEligibilityPolicy::Enforce
|
||||
})
|
||||
.with_affinity(affinity);
|
||||
let selection_deadline = remaining(context.deadline())
|
||||
@@ -1981,7 +1981,8 @@ const OAUTH_REFRESH_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const QUOTA_CATALOG_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
// grok2api 对 Free/未知额度耗尽采用 24 小时恢复探测;当前公共账号状态没有
|
||||
// 独立的 paid period-end 字段,因此 Build 账号统一使用同一保守下限。
|
||||
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = QUOTA_CATALOG_INTERVAL;
|
||||
const CLI_RELEASE_WORKER_OWNER: &str = "xai-cli-release";
|
||||
|
||||
pub(crate) fn worker_contributions(
|
||||
@@ -2175,7 +2176,8 @@ impl XaiQuotaCatalogTask {
|
||||
let due = last_periodic_refresh_at
|
||||
.get(account.id())
|
||||
.is_none_or(|last| {
|
||||
now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL
|
||||
now.saturating_duration_since(*last)
|
||||
>= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL
|
||||
});
|
||||
if due {
|
||||
last_periodic_refresh_at.insert(account.id().clone(), now);
|
||||
@@ -2191,5 +2193,7 @@ fn eligible_quota_worker_account(account: &ProviderAccount, now: SystemTime) ->
|
||||
&& account
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at > now)
|
||||
&& account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& account
|
||||
.quota()
|
||||
.exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::pin::Pin;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailabilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId,
|
||||
AccountEligibilityPolicy, AccountSelectionPolicy, CredentialRevision, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::engine::provider::ProviderResource;
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
@@ -210,7 +210,7 @@ pub struct GrokSessionSelection {
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
required_account: Option<ProviderAccountId>,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
availability: AccountAvailabilityPolicy,
|
||||
eligibility: AccountEligibilityPolicy,
|
||||
affinity: Option<GrokSessionAffinityKey>,
|
||||
deadline: SystemTime,
|
||||
}
|
||||
@@ -230,7 +230,7 @@ impl GrokSessionSelection {
|
||||
excluded_accounts,
|
||||
required_account,
|
||||
account_selection_policy,
|
||||
availability: AccountAvailabilityPolicy::Enforce,
|
||||
eligibility: AccountEligibilityPolicy::Enforce,
|
||||
affinity: None,
|
||||
deadline,
|
||||
}
|
||||
@@ -245,11 +245,11 @@ impl GrokSessionSelection {
|
||||
|
||||
/// 为固定账号的管理端诊断指定本地可用性判定策略。
|
||||
#[must_use]
|
||||
pub(crate) const fn with_availability_policy(
|
||||
pub(crate) const fn with_eligibility_policy(
|
||||
mut self,
|
||||
availability: AccountAvailabilityPolicy,
|
||||
eligibility: AccountEligibilityPolicy,
|
||||
) -> Self {
|
||||
self.availability = availability;
|
||||
self.eligibility = eligibility;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -278,8 +278,8 @@ impl GrokSessionSelection {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn availability(&self) -> AccountAvailabilityPolicy {
|
||||
self.availability
|
||||
pub const fn eligibility(&self) -> AccountEligibilityPolicy {
|
||||
self.eligibility
|
||||
}
|
||||
|
||||
/// 返回不含原始客户端会话值的账号亲和键。
|
||||
|
||||
@@ -471,10 +471,11 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
access_token_expires_at: account.access_token_expires_at().map(DateTime::<Utc>::from),
|
||||
next_refresh_at: account.next_refresh_at().map(DateTime::<Utc>::from),
|
||||
enabled: account.enabled(),
|
||||
availability: account.availability(),
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
credential_state: account.credential_state(),
|
||||
credential_observed_at: now,
|
||||
quota: account.quota(),
|
||||
last_error_reason: account.last_error_reason(),
|
||||
last_error_message: account.last_error_message().map(str::to_owned),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
@@ -503,7 +504,7 @@ impl ProviderLeasePort for TestLeases {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -5,9 +5,9 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use futures::future::{BoxFuture, join_all};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasUpdate, CredentialRevision,
|
||||
AccountStateChange, CredentialCasUpdate, CredentialRevision, CredentialState,
|
||||
OpaqueProviderData, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
QuotaObservation,
|
||||
QuotaAccessChange, QuotaEvidence, QuotaObservation, QuotaState,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderStoreError,
|
||||
@@ -200,14 +200,15 @@ async fn repository_with_accounts(
|
||||
async fn set_account_state(
|
||||
store: &MemoryProviderAccountStore,
|
||||
id: &ProviderAccountId,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
) {
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability,
|
||||
credential_state,
|
||||
error_reason: credential_state.error_reason(),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
@@ -485,12 +486,10 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() {
|
||||
repository_with_accounts(&[("quota-exhausted", "subject-quota-exhausted")]).await;
|
||||
let id = account_id("quota-exhausted");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id,
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
@@ -662,7 +661,7 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie
|
||||
.expect("read persisted quota")
|
||||
.pop()
|
||||
.expect("quota exists");
|
||||
let document = persisted.quota.expect("provider quota");
|
||||
let document = persisted.quota;
|
||||
|
||||
assert_eq!(transport.calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(snapshot.billing().used_percent(), Some(37.5));
|
||||
@@ -683,33 +682,18 @@ async fn quota_refresh_persists_dynamic_provider_document_and_projects_known_fie
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_quota_refresh_releases_existing_quota_isolation() {
|
||||
for (suffix, availability) in [
|
||||
("recover-quota", AccountAvailability::QuotaExhausted),
|
||||
("recover-ready", AccountAvailability::Ready),
|
||||
] {
|
||||
let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await;
|
||||
let id = account_id(suffix);
|
||||
set_account_state(&store, &id, availability).await;
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":25}}"#),
|
||||
)
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh recovered quota");
|
||||
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_exhaustion_preserves_quota_exhausted_state() {
|
||||
async fn billing_percent_does_not_rewrite_quota_access_fact() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("still-exhausted", "still-exhausted")]).await;
|
||||
let id = account_id("still-exhausted");
|
||||
set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await;
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
@@ -720,21 +704,75 @@ async fn authoritative_exhaustion_preserves_quota_exhausted_state() {
|
||||
.expect("refresh exhausted quota");
|
||||
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
store.account(&id).expect("account").quota().access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_billing_refresh_clears_existing_quota_exhaustion() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("recovered-quota", "recovered-quota")]).await;
|
||||
let id = account_id("recovered-quota");
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(
|
||||
br#"{"config":{"creditUsagePercent":12.5,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2099-07-22T00:00:00Z"}}}"#,
|
||||
),
|
||||
)
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh recovered quota");
|
||||
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").quota().access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Allowed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_usage_display_does_not_invent_quota_exhaustion() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("exhausted-quota", "exhausted-quota")]).await;
|
||||
let id = account_id("exhausted-quota");
|
||||
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(
|
||||
br#"{"config":{"creditUsagePercent":100,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2099-07-22T00:00:00Z"}}}"#,
|
||||
),
|
||||
)
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh exhausted quota");
|
||||
|
||||
let quota = store.account(&id).expect("account").quota();
|
||||
assert_eq!(
|
||||
quota.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Unknown
|
||||
);
|
||||
assert!(quota.reset_at().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_quota_does_not_clear_terminal_account_states() {
|
||||
for (suffix, availability) in [
|
||||
("keep-banned", AccountAvailability::Banned),
|
||||
("keep-expired", AccountAvailability::Expired),
|
||||
("keep-invalid", AccountAvailability::Invalid),
|
||||
for (suffix, credential_state) in [
|
||||
("keep-banned", CredentialState::Banned),
|
||||
("keep-expired", CredentialState::Expired),
|
||||
("keep-invalid", CredentialState::Invalid),
|
||||
] {
|
||||
let (store, repository) = repository_with_accounts(&[(suffix, suffix)]).await;
|
||||
let id = account_id(suffix);
|
||||
set_account_state(&store, &id, availability).await;
|
||||
set_account_state(&store, &id, credential_state).await;
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":10}}"#),
|
||||
@@ -744,14 +782,14 @@ async fn recovered_quota_does_not_clear_terminal_account_states() {
|
||||
.expect("refresh terminal account quota");
|
||||
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
availability
|
||||
store.account(&id).expect("account").credential_state(),
|
||||
credential_state
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_refresh_preserves_ready_availability_across_concurrent_state_write() {
|
||||
async fn quota_refresh_preserves_ready_credential_state_across_concurrent_write() {
|
||||
let (store, repository) = repository_with_accounts(&[("new-cooldown", "new-cooldown")]).await;
|
||||
let id = account_id("new-cooldown");
|
||||
let transport = Arc::new(MutatingBillingTransport {
|
||||
@@ -760,7 +798,8 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
|
||||
message: None,
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
credential_state: CredentialState::Ready,
|
||||
error_reason: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))),
|
||||
body: br#"{"config":{"creditUsagePercent":10}}"#.to_vec(),
|
||||
@@ -772,7 +811,7 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
|
||||
.expect("refresh around concurrent state write");
|
||||
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -860,12 +899,48 @@ async fn quota_projection_preserves_unknown_period_for_dynamic_duration_fallback
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_billing_window_does_not_participate_in_scheduling_rank() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("expired-billing", "expired-billing")]).await;
|
||||
let id = account_id("expired-billing");
|
||||
let account = store.account(&id).expect("account");
|
||||
let service = crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(
|
||||
br#"{"config":{"creditUsagePercent":25,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2000-01-01T00:00:00Z","end":"2000-01-08T00:00:00Z"}}}"#,
|
||||
),
|
||||
);
|
||||
|
||||
service
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh expired billing window");
|
||||
|
||||
assert_eq!(service.scheduling_signals(&account), None);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(&id)
|
||||
.expect("refreshed account")
|
||||
.quota()
|
||||
.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustion() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("free-quota", "subject-free-quota")]).await;
|
||||
let id = account_id("free-quota");
|
||||
set_account_state(&store, &id, AccountAvailability::QuotaExhausted).await;
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
let transport = QueueBillingTransport::success(
|
||||
br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"onDemandCap":{"val":0},"onDemandUsed":{"val":0},"prepaidBalance":{"val":0}}}"#,
|
||||
);
|
||||
@@ -876,8 +951,8 @@ async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustio
|
||||
|
||||
assert!(!snapshot.billing().has_authoritative_quota());
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
store.account(&id).expect("account").quota().access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
@@ -920,9 +995,9 @@ async fn quota_read_rejects_corrupt_provider_document() {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account_id("corrupt"),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
quota: Some(OpaqueProviderData::new(document)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(document),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::unknown(),
|
||||
})
|
||||
.await
|
||||
.expect("seed corrupt quota");
|
||||
@@ -975,8 +1050,7 @@ async fn failed_quota_refresh_does_not_replace_last_good_observation() {
|
||||
.expect("read quota")
|
||||
.pop()
|
||||
.expect("quota remains")
|
||||
.quota
|
||||
.expect("provider document");
|
||||
.quota;
|
||||
assert_eq!(
|
||||
persisted.expose_to_provider()["config"]["creditUsagePercent"].as_f64(),
|
||||
Some(10.0),
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::collections::VecDeque;
|
||||
use std::future::ready;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use gateway_core::engine::credential::AccountAvailability;
|
||||
use gateway_core::engine::credential::CredentialState;
|
||||
use provider_xai::{
|
||||
AuthorizationCallback, FailClosedTokenVerifier, FailureClass, FormValue, GrokCredentialAdmin,
|
||||
GrokOAuthClient, GrokOAuthConfig, GrokOAuthImportCandidate, GrokOAuthImportMetadata,
|
||||
@@ -128,7 +128,7 @@ async fn credential_import_should_require_exact_metadata_and_official_userinfo()
|
||||
enabled: true,
|
||||
})
|
||||
.expect("Provider projects verified token lifetime");
|
||||
assert_eq!(prepared.account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(prepared.account.credential_state(), CredentialState::Ready);
|
||||
assert!(prepared.account.next_refresh_at().is_none());
|
||||
assert!(
|
||||
prepared
|
||||
|
||||
@@ -2,15 +2,14 @@ use std::sync::Arc;
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasOutcome, CredentialRevision, NewProviderAccount,
|
||||
PlaintextCredential, ProviderAccount, ProviderAccountStore,
|
||||
AccountErrorReason, CredentialCasOutcome, CredentialRevision, CredentialState,
|
||||
NewProviderAccount, PlaintextCredential, ProviderAccount, ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_xai::{
|
||||
GrokCredentialAdmin, GrokCredentialAvailability, GrokCredentialRepository,
|
||||
GrokCredentialRepositoryError, GrokOAuthSecret, RotateManagedGrokCredential, SecretValue,
|
||||
UpdateGrokCredentialState,
|
||||
GrokCredentialAdmin, GrokCredentialRepository, GrokCredentialRepositoryError, GrokOAuthSecret,
|
||||
RotateManagedGrokCredential, SecretValue, UpdateGrokCredentialState,
|
||||
};
|
||||
|
||||
use crate::support::{
|
||||
@@ -219,14 +218,15 @@ async fn state_update_uses_credential_revision_fence() {
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Ready,
|
||||
availability_reason: Some("upstream_rate_limited".to_owned()),
|
||||
credential_state: CredentialState::Ready,
|
||||
error_reason: None,
|
||||
error_message: Some("upstream_rate_limited".to_owned()),
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
.expect("state update");
|
||||
let account = store.account(&input.account_id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -239,16 +239,20 @@ async fn terminal_state_without_upstream_message_uses_stable_error_reason() {
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Banned,
|
||||
availability_reason: None,
|
||||
credential_state: CredentialState::Banned,
|
||||
error_reason: Some(AccountErrorReason::AccountBanned),
|
||||
error_message: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
.expect("state update");
|
||||
|
||||
assert_eq!(
|
||||
store.last_error_message(&input.account_id).as_deref(),
|
||||
Some("account_banned")
|
||||
store
|
||||
.account(&input.account_id)
|
||||
.expect("account")
|
||||
.last_error_reason(),
|
||||
Some(AccountErrorReason::AccountBanned)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime};
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
|
||||
CredentialCasOutcome, CredentialCasUpdate, CredentialRevision, CredentialState,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -514,8 +514,8 @@ async fn unauthorized_recovery_marks_a_permanently_rejected_refresh_expired() {
|
||||
|
||||
assert_eq!(outcome, GrokCredentialRecoveryOutcome::Rejected);
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Expired
|
||||
store.account(&id).expect("account").credential_state(),
|
||||
CredentialState::Expired
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message(&id).as_deref(),
|
||||
@@ -664,8 +664,8 @@ async fn invalid_grant_marks_account_expired() {
|
||||
fixture(input, [Err(GrokRefreshFailure::InvalidGrant)], true).await;
|
||||
service.refresh_due().await.expect("refresh");
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Expired
|
||||
store.account(&id).expect("account").credential_state(),
|
||||
CredentialState::Expired
|
||||
);
|
||||
}
|
||||
|
||||
@@ -676,8 +676,8 @@ async fn banned_failure_marks_account_banned() {
|
||||
let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Banned)], true).await;
|
||||
service.refresh_due().await.expect("refresh");
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Banned
|
||||
store.account(&id).expect("account").credential_state(),
|
||||
CredentialState::Banned
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message(&id).as_deref(),
|
||||
@@ -704,8 +704,8 @@ async fn ambiguous_refresh_applies_bounded_oauth_backoff() {
|
||||
[GrokCredentialRefreshOutcome::Ambiguous { account_id }] if account_id == &id
|
||||
));
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Unknown
|
||||
store.account(&id).expect("account").credential_state(),
|
||||
CredentialState::Unknown
|
||||
);
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
@@ -736,7 +736,7 @@ async fn pre_send_transient_failure_applies_bounded_oauth_backoff() {
|
||||
let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Transient)], true).await;
|
||||
service.refresh_due().await.expect("refresh");
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Unknown);
|
||||
assert_eq!(account.credential_state(), CredentialState::Unknown);
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
assert!(
|
||||
account
|
||||
@@ -908,8 +908,11 @@ async fn recovery_window_exhaustion_marks_account_expired_without_token_exchange
|
||||
[GrokCredentialRefreshOutcome::Invalidated { account_id }] if account_id == &id
|
||||
));
|
||||
assert_eq!(
|
||||
store.account(&id).expect("expired account").availability(),
|
||||
AccountAvailability::Expired
|
||||
store
|
||||
.account(&id)
|
||||
.expect("expired account")
|
||||
.credential_state(),
|
||||
CredentialState::Expired
|
||||
);
|
||||
assert_eq!(refresher.prepare_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(refresher.responses.lock().expect("queue").len(), 1);
|
||||
|
||||
@@ -4,10 +4,10 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals,
|
||||
AccountSelectionPolicy, CredentialCasOutcome, CredentialRevision, OpaqueProviderData,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome, RotationStrategy,
|
||||
AccountAttemptFeedback, AccountFeedbackStats, AccountRuntimeSignals, AccountSelectionPolicy,
|
||||
CredentialCasOutcome, CredentialRevision, CredentialState, OpaqueProviderData,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaAccessState,
|
||||
QuotaObservation, QuotaState, QuotaWriteOutcome, RotationStrategy,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownPort, ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort,
|
||||
@@ -17,10 +17,10 @@ use gateway_core::routing::UpstreamModelId;
|
||||
use provider_xai::{
|
||||
GrokAccountSessionSelector, GrokBillingRequest, GrokBillingTransport,
|
||||
GrokBillingTransportError, GrokBillingTransportErrorKind, GrokBillingTransportFuture,
|
||||
GrokCatalogScope, GrokCredentialAdmin, GrokCredentialAvailability, GrokCredentialCatalogCache,
|
||||
GrokCredentialCatalogSeed, GrokCredentialFailure, GrokCredentialRepository, GrokPlanCatalog,
|
||||
GrokSessionSelection, GrokSessionSelector, GrokSessionSelectorError,
|
||||
RotateManagedGrokCredential, UpdateGrokCredentialState,
|
||||
GrokCatalogScope, GrokCredentialAdmin, GrokCredentialCatalogCache, GrokCredentialCatalogSeed,
|
||||
GrokCredentialFailure, GrokCredentialRepository, GrokPlanCatalog, GrokSessionSelection,
|
||||
GrokSessionSelector, GrokSessionSelectorError, RotateManagedGrokCredential,
|
||||
UpdateGrokCredentialState,
|
||||
};
|
||||
|
||||
use crate::support::{
|
||||
@@ -108,8 +108,9 @@ impl SelectorFixture {
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Ready,
|
||||
availability_reason: None,
|
||||
credential_state: CredentialState::Ready,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
@@ -131,7 +132,7 @@ impl SelectorFixture {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -241,11 +242,9 @@ impl SelectorFixture {
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
document.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
quota: OpaqueProviderData::new(document.as_object().expect("quota object").clone()),
|
||||
observed_at: SystemTime::now(),
|
||||
state: QuotaState::unknown(),
|
||||
})
|
||||
.await
|
||||
.expect("persist quota");
|
||||
@@ -305,16 +304,20 @@ async fn unauthorized_feedback_records_runtime_cooldown_without_persisting_accou
|
||||
.store
|
||||
.account(&selected)
|
||||
.expect("selected")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
let other = [account_id("feedback-a"), account_id("feedback-b")]
|
||||
.into_iter()
|
||||
.find(|id| id != &selected)
|
||||
.expect("other account");
|
||||
assert_eq!(
|
||||
fixture.store.account(&other).expect("other").availability(),
|
||||
AccountAvailability::Ready
|
||||
fixture
|
||||
.store
|
||||
.account(&other)
|
||||
.expect("other")
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
@@ -379,7 +382,7 @@ async fn account_scoped_cooldown_survives_credential_rotation() {
|
||||
));
|
||||
let account = fixture.store.account(&id).expect("rotated account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -401,7 +404,7 @@ async fn rate_limit_feedback_persists_the_grok2api_cooldown_state() {
|
||||
)
|
||||
.await;
|
||||
let account = fixture.store.account(&selected).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
@@ -440,7 +443,7 @@ async fn successful_request_clears_the_persisted_cooldown_state() {
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -469,8 +472,8 @@ async fn payment_required_feedback_writes_short_account_cooldown_without_persist
|
||||
.store
|
||||
.account(&selected)
|
||||
.expect("selected account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
@@ -508,7 +511,7 @@ async fn model_quota_feedback_writes_model_scoped_cooldown_without_blocking_othe
|
||||
.await;
|
||||
|
||||
let account = fixture.store.account(&selected).expect("selected account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
let minimum_until = SystemTime::now() + Duration::from_secs(23 * 60 * 60);
|
||||
let scope = ProviderCooldownScope::upstream_model(failed_model);
|
||||
// model-scoped cooldown:目标模型被排除,账号级无 cooldown。
|
||||
@@ -563,8 +566,8 @@ async fn model_access_feedback_reports_model_cooldown_without_blocking_the_accou
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
.credential_state(),
|
||||
CredentialState::Ready
|
||||
);
|
||||
assert!(matches!(
|
||||
fixture
|
||||
@@ -594,7 +597,7 @@ async fn interrupted_stream_feedback_persists_the_grok2api_cooldown_state() {
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
@@ -630,8 +633,9 @@ async fn quota_feedback_uses_common_quota_exhausted_state() {
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
@@ -673,13 +677,18 @@ async fn quota_feedback_should_follow_the_account_across_a_credential_rotation()
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
fixture.store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
fixture
|
||||
.store
|
||||
.account(&id)
|
||||
.expect("account")
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
|
||||
async fn ordinary_success_clears_confirmed_exhaustion() {
|
||||
let fixture = SelectorFixture::new(&["quota-success"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
@@ -690,7 +699,6 @@ async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
|
||||
.selector
|
||||
.record_failure(&session, GrokCredentialFailure::FreeQuotaExhausted)
|
||||
.await;
|
||||
|
||||
fixture.selector.record_success(&session).await;
|
||||
|
||||
assert_eq!(
|
||||
@@ -698,8 +706,9 @@ async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
.quota()
|
||||
.access(),
|
||||
QuotaAccessState::Allowed
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1061,8 +1070,8 @@ async fn bare_402_cooldown_expires_and_account_recovers_without_persisted_exhaus
|
||||
.store
|
||||
.account(&selected)
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
.credential_state(),
|
||||
CredentialState::Ready,
|
||||
"bare 402 must not persist QuotaExhausted"
|
||||
);
|
||||
let next = fixture
|
||||
|
||||
@@ -8,10 +8,10 @@ use std::time::{Duration, SystemTime};
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, PlaintextCredential, ProviderAccount,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, CredentialState, LoadedCredential, NewProviderAccount, PlaintextCredential,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
QuotaAccessChange, QuotaObservation, QuotaState, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::{StoreError, StoreErrorKind};
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -23,11 +23,10 @@ use gateway_core::provider_ports::{
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_xai::{
|
||||
CreateGrokCredential, GROK_BILLING_URL, GrokAccountProfile, GrokBillingTransport,
|
||||
GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, GrokCredentialAvailability,
|
||||
GrokCredentialCatalogCache, GrokCredentialCatalogService, GrokCredentialQuotaService,
|
||||
GrokCredentialRepository, GrokCredentialRepositoryError, GrokEndpointPolicy,
|
||||
GrokModelCatalogTransport, GrokOAuthSecret, GrokPlanCatalog, GrokReqwestTransportBuildError,
|
||||
SecretValue, XaiConfig, XaiWireProfileState,
|
||||
GrokCatalogCacheError, GrokCatalogScope, GrokCredentialAdmin, GrokCredentialCatalogCache,
|
||||
GrokCredentialCatalogService, GrokCredentialQuotaService, GrokCredentialRepository,
|
||||
GrokCredentialRepositoryError, GrokEndpointPolicy, GrokModelCatalogTransport, GrokOAuthSecret,
|
||||
GrokPlanCatalog, GrokReqwestTransportBuildError, SecretValue, XaiConfig, XaiWireProfileState,
|
||||
};
|
||||
use reqwest::Client;
|
||||
use reqwest::redirect::Policy;
|
||||
@@ -194,7 +193,6 @@ struct StoredAccount {
|
||||
account: ProviderAccount,
|
||||
credential: PlaintextCredential,
|
||||
quota: Option<QuotaObservation>,
|
||||
last_error_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -218,7 +216,7 @@ impl MemoryProviderAccountStore {
|
||||
pub fn last_error_message(&self, id: &ProviderAccountId) -> Option<String> {
|
||||
lock(&self.accounts)
|
||||
.get(id)
|
||||
.and_then(|stored| stored.last_error_message.clone())
|
||||
.and_then(|stored| stored.account.last_error_message().map(str::to_owned))
|
||||
}
|
||||
|
||||
pub fn credential(&self, id: &ProviderAccountId) -> Option<PlaintextCredential> {
|
||||
@@ -253,7 +251,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
account: account.account,
|
||||
credential: account.credential,
|
||||
quota: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
@@ -335,7 +332,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
AccountReplacement {
|
||||
revision: next,
|
||||
access_token_expires_at,
|
||||
availability: stored.account.availability(),
|
||||
credential_state: stored.account.credential_state(),
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: stored.account.last_error_reason(),
|
||||
last_error_message: stored.account.last_error_message().map(str::to_owned),
|
||||
enabled: stored.account.enabled(),
|
||||
has_refresh_token,
|
||||
next_refresh_at,
|
||||
@@ -372,7 +372,31 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
if stored.account.revision() != observation.expected_revision {
|
||||
return Ok(QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
let quota = observation.state;
|
||||
stored.quota = Some(observation);
|
||||
stored.account = rebuild_account(
|
||||
&stored.account,
|
||||
AccountReplacement::preserving(&stored.account).with_quota(quota),
|
||||
);
|
||||
Ok(QuotaWriteOutcome::Updated)
|
||||
}
|
||||
|
||||
async fn apply_quota_access(
|
||||
&self,
|
||||
change: QuotaAccessChange,
|
||||
) -> Result<QuotaWriteOutcome, StoreError> {
|
||||
let mut accounts = lock(&self.accounts);
|
||||
let stored = accounts.get_mut(&change.account_id).ok_or_else(invalid)?;
|
||||
if stored.account.revision() != change.expected_revision {
|
||||
return Ok(QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
stored.account = rebuild_account(
|
||||
&stored.account,
|
||||
AccountReplacement::preserving(&stored.account).with_quota(change.state),
|
||||
);
|
||||
if let Some(observation) = &mut stored.quota {
|
||||
observation.state = change.state;
|
||||
}
|
||||
Ok(QuotaWriteOutcome::Updated)
|
||||
}
|
||||
|
||||
@@ -387,7 +411,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
AccountReplacement {
|
||||
revision: stored.account.revision(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
availability: change.availability,
|
||||
credential_state: change.credential_state,
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: change.error_reason,
|
||||
last_error_message: change.message.clone(),
|
||||
enabled: stored.account.enabled(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -396,11 +423,6 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
plan_type: stored.account.plan_type().map(str::to_owned),
|
||||
},
|
||||
);
|
||||
if change.availability == AccountAvailability::Ready {
|
||||
stored.last_error_message = None;
|
||||
} else if let Some(message) = change.message {
|
||||
stored.last_error_message = Some(message);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -412,7 +434,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
AccountReplacement {
|
||||
revision: stored.account.revision(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
availability: stored.account.availability(),
|
||||
credential_state: stored.account.credential_state(),
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: stored.account.last_error_reason(),
|
||||
last_error_message: stored.account.last_error_message().map(str::to_owned),
|
||||
enabled: stored.account.enabled(),
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -436,7 +461,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
AccountReplacement {
|
||||
revision: stored.account.revision(),
|
||||
access_token_expires_at: stored.account.access_token_expires_at(),
|
||||
availability: stored.account.availability(),
|
||||
credential_state: stored.account.credential_state(),
|
||||
quota: stored.account.quota(),
|
||||
last_error_reason: stored.account.last_error_reason(),
|
||||
last_error_message: stored.account.last_error_message().map(str::to_owned),
|
||||
enabled,
|
||||
has_refresh_token: stored.account.has_refresh_token(),
|
||||
next_refresh_at: stored.account.next_refresh_at(),
|
||||
@@ -457,7 +485,10 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
struct AccountReplacement {
|
||||
revision: CredentialRevision,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
quota: QuotaState,
|
||||
last_error_reason: Option<AccountErrorReason>,
|
||||
last_error_message: Option<String>,
|
||||
enabled: bool,
|
||||
has_refresh_token: bool,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
@@ -466,6 +497,30 @@ struct AccountReplacement {
|
||||
plan_type: Option<String>,
|
||||
}
|
||||
|
||||
impl AccountReplacement {
|
||||
fn preserving(account: &ProviderAccount) -> Self {
|
||||
Self {
|
||||
revision: account.revision(),
|
||||
access_token_expires_at: account.access_token_expires_at(),
|
||||
credential_state: account.credential_state(),
|
||||
quota: account.quota(),
|
||||
last_error_reason: account.last_error_reason(),
|
||||
last_error_message: account.last_error_message().map(str::to_owned),
|
||||
enabled: account.enabled(),
|
||||
has_refresh_token: account.has_refresh_token(),
|
||||
next_refresh_at: account.next_refresh_at(),
|
||||
name: account.name().to_owned(),
|
||||
email: account.email().map(str::to_owned),
|
||||
plan_type: account.plan_type().map(str::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_quota(mut self, quota: QuotaState) -> Self {
|
||||
self.quota = quota;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement) -> ProviderAccount {
|
||||
ProviderAccount::new(
|
||||
previous.id().clone(),
|
||||
@@ -481,7 +536,13 @@ fn rebuild_account(previous: &ProviderAccount, replacement: AccountReplacement)
|
||||
previous.upstream_account_id().map(str::to_owned),
|
||||
replacement.plan_type,
|
||||
)
|
||||
.with_runtime_state(replacement.enabled, replacement.availability)
|
||||
.with_account_facts(
|
||||
replacement.enabled,
|
||||
replacement.credential_state,
|
||||
replacement.quota,
|
||||
replacement.last_error_reason,
|
||||
replacement.last_error_message,
|
||||
)
|
||||
.with_refresh_schedule(replacement.has_refresh_token, replacement.next_refresh_at)
|
||||
}
|
||||
|
||||
@@ -549,8 +610,9 @@ pub fn create_input(suffix: &str, subject: &str) -> CreateGrokCredential {
|
||||
},
|
||||
account: profile(subject),
|
||||
enabled: true,
|
||||
initial_availability: GrokCredentialAvailability::Unknown,
|
||||
initial_availability_reason: None,
|
||||
initial_credential_state: CredentialState::Unknown,
|
||||
initial_error_reason: None,
|
||||
initial_error_message: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql
|
||||
f4168f7e23d42cf9d37096339e931b0e0c823f7a8dbfc6580a04f096c4c3d8e3 0002_provider_accounts_nullable_upstream_user_id.sql
|
||||
4163f61183d9e90f1368c08f614040d6a1a0048de3886a77dd708ce21430b2d0 0003_backfill_openai_access_token_expiry.sql
|
||||
b41bb3be5a84b95852158d82267456f8415811a1137025581eefd1c96c0882d1 0004_normalize_account_state_facts.sql
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
-- Separate credential identity facts from quota access facts. The public account
|
||||
-- status remains derived at read time because Redis cooldowns and reset times are
|
||||
-- time-dependent.
|
||||
|
||||
alter table provider_accounts
|
||||
rename column availability to credential_state;
|
||||
|
||||
alter table provider_accounts
|
||||
rename column availability_observed_at to credential_observed_at;
|
||||
|
||||
alter table provider_accounts
|
||||
drop constraint provider_accounts_availability_ck;
|
||||
|
||||
alter table provider_accounts
|
||||
add constraint provider_accounts_credential_state_ck check (
|
||||
credential_state in ('unknown', 'ready', 'expired', 'banned', 'invalid')
|
||||
) not valid;
|
||||
|
||||
alter table provider_accounts
|
||||
add column quota_access_state text not null default 'unknown',
|
||||
add column quota_evidence text,
|
||||
add column quota_access_observed_at timestamptz,
|
||||
add column quota_reset_at timestamptz,
|
||||
add column last_error_reason text;
|
||||
|
||||
alter table provider_accounts
|
||||
drop constraint provider_accounts_quota_observation_ck;
|
||||
|
||||
-- Old quota_exhausted credential states are quota facts. Re-derive OpenAI quota
|
||||
-- access from the raw authoritative response and never trust the old percentage-
|
||||
-- derived quota_limit_reached boolean.
|
||||
update provider_accounts
|
||||
set
|
||||
quota_access_state = case
|
||||
when provider_kind = 'openai'
|
||||
and provider_quota_json #>> '{rate_limit,allowed}' = 'true'
|
||||
then 'allowed'
|
||||
when provider_kind = 'openai'
|
||||
and (
|
||||
provider_quota_json #>> '{rate_limit,allowed}' = 'false'
|
||||
or (
|
||||
coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true'
|
||||
and (
|
||||
provider_quota_json #>> '{rate_limit,limit_reached}' = 'true'
|
||||
or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true'
|
||||
)
|
||||
)
|
||||
)
|
||||
then 'exhausted'
|
||||
when provider_kind = 'xai' and credential_state = 'quota_exhausted'
|
||||
then 'exhausted'
|
||||
when provider_quota_json is not null then 'unknown'
|
||||
else 'unknown'
|
||||
end,
|
||||
quota_evidence = case
|
||||
when provider_kind = 'openai'
|
||||
and provider_quota_json #>> '{rate_limit,allowed}' = 'false'
|
||||
then 'provider_denied'
|
||||
when provider_kind = 'openai'
|
||||
and coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true'
|
||||
and (
|
||||
provider_quota_json #>> '{rate_limit,limit_reached}' = 'true'
|
||||
or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true'
|
||||
)
|
||||
then 'account_limit_reached'
|
||||
when provider_kind = 'xai' and credential_state = 'quota_exhausted'
|
||||
then 'usage_limit_reached'
|
||||
else null
|
||||
end,
|
||||
quota_access_observed_at = case
|
||||
when provider_kind = 'openai' and provider_quota_json is not null
|
||||
then quota_observed_at
|
||||
when provider_kind = 'xai' and credential_state = 'quota_exhausted'
|
||||
then credential_observed_at
|
||||
else null
|
||||
end,
|
||||
quota_reset_at = case
|
||||
when provider_kind = 'openai'
|
||||
and jsonb_typeof(provider_quota_json #> '{rate_limit,primary_window,reset_at}') = 'number'
|
||||
then to_timestamp((provider_quota_json #>> '{rate_limit,primary_window,reset_at}')::double precision)
|
||||
else null
|
||||
end,
|
||||
credential_state = case
|
||||
when credential_state = 'quota_exhausted' and upstream_user_id is not null then 'ready'
|
||||
when credential_state = 'quota_exhausted' then 'unknown'
|
||||
else credential_state
|
||||
end,
|
||||
last_error_reason = case credential_state
|
||||
when 'unknown' then 'account_unverified'
|
||||
when 'expired' then 'credential_expired'
|
||||
when 'invalid' then 'credential_invalid'
|
||||
when 'banned' then 'account_banned'
|
||||
else null
|
||||
end,
|
||||
last_error_message = case
|
||||
when credential_state in ('unknown', 'expired', 'invalid', 'banned') then last_error_message
|
||||
else null
|
||||
end;
|
||||
|
||||
alter table provider_accounts
|
||||
validate constraint provider_accounts_credential_state_ck;
|
||||
|
||||
alter table provider_accounts
|
||||
add constraint provider_accounts_quota_access_state_ck check (
|
||||
quota_access_state in ('unknown', 'allowed', 'exhausted')
|
||||
),
|
||||
add constraint provider_accounts_quota_evidence_ck check (
|
||||
quota_evidence is null
|
||||
or quota_evidence in (
|
||||
'provider_denied',
|
||||
'account_limit_reached',
|
||||
'usage_limit_reached',
|
||||
'payment_required'
|
||||
)
|
||||
),
|
||||
add constraint provider_accounts_quota_fact_ck check (
|
||||
(quota_access_state = 'unknown' and quota_evidence is null and quota_reset_at is null)
|
||||
or (
|
||||
quota_access_state = 'allowed'
|
||||
and quota_evidence is null
|
||||
and quota_access_observed_at is not null
|
||||
and quota_reset_at is null
|
||||
)
|
||||
or (
|
||||
quota_access_state = 'exhausted'
|
||||
and quota_evidence is not null
|
||||
and quota_access_observed_at is not null
|
||||
)
|
||||
),
|
||||
add constraint provider_accounts_quota_observation_ck check (
|
||||
(provider_quota_json is null) = (quota_observed_at is null)
|
||||
),
|
||||
add constraint provider_accounts_error_reason_ck check (
|
||||
last_error_reason is null
|
||||
or last_error_reason in (
|
||||
'account_unverified',
|
||||
'access_token_expired',
|
||||
'credential_expired',
|
||||
'credential_invalid',
|
||||
'account_banned'
|
||||
)
|
||||
);
|
||||
|
||||
drop index provider_accounts_availability_idx;
|
||||
create index provider_accounts_credential_state_idx
|
||||
on provider_accounts (credential_state, id);
|
||||
create index provider_accounts_quota_access_idx
|
||||
on provider_accounts (quota_access_state, quota_reset_at, id);
|
||||
|
||||
alter table provider_accounts
|
||||
drop column quota_limit_reached;
|
||||
|
||||
alter table provider_accounts
|
||||
rename constraint provider_accounts_time_ck to provider_accounts_time_ck_old;
|
||||
|
||||
alter table provider_accounts
|
||||
drop constraint provider_accounts_time_ck_old;
|
||||
|
||||
alter table provider_accounts
|
||||
add constraint provider_accounts_time_ck check (
|
||||
created_at <= updated_at
|
||||
and credential_observed_at <= updated_at
|
||||
and (quota_access_observed_at is null or quota_access_observed_at <= updated_at)
|
||||
and (quota_observed_at is null or quota_observed_at <= updated_at)
|
||||
);
|
||||
+12
-6
@@ -112,7 +112,7 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段
|
||||
账号列表支持以下稳定值:
|
||||
|
||||
- `provider`: `all`、`openai`、`xai`;
|
||||
- `status`: `active`、`expired`、`quota_exhausted`、`disabled`、`banned`;
|
||||
- `status`: `normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`;
|
||||
- `sortBy`: `email`、`status`、`planType`、`usage`、`lastUsedAt`、`expiresAt`;
|
||||
- `sortDirection`: `asc`、`desc`。
|
||||
|
||||
@@ -184,13 +184,19 @@ OAuth start 使用:
|
||||
但上游 authorization code 本身通常只能交换一次;已完成过 token exchange 时应重新创建 OAuth flow。
|
||||
- `GET /accounts/quota` 只读取最后一次落库快照;`POST /accounts/quota/refresh` 才访问上游。access token
|
||||
已过期时,额度刷新要求先走 credential 刷新或重新授权,不会拿过期 token 探测额度。
|
||||
- 成功额度观测会 revision-fenced 写入 quota,并在 `active` 与 `quota_exhausted` 等额度所属状态间同步;
|
||||
不会清除 `expired`、`banned` 等 credential/封禁终态。额度接口的 401/403 也不足以判定 refresh token
|
||||
永久失效,credential 终态只由 OAuth refresh 的明确永久错误写入。
|
||||
- 成功额度观测会 revision-fenced 写入 quota;明确 `Allowed` 投影为 `normal`,明确耗尽投影为
|
||||
`quota_exhausted`。额度观测不会清除凭据过期、无效或封禁事实;这些事实统一投影为 `error`,并由
|
||||
`errorReason` 区分。额度接口的 401/403 也不足以判定 refresh token 永久失效,credential 终态只由
|
||||
OAuth refresh 的明确永久错误写入。
|
||||
- 正常 Responses 请求会解析上游响应的 rate-limit headers,合并进同一 quota 快照并同步状态。Free、
|
||||
K12 等套餐共用该状态机;套餐只参与账号展示和按套餐隔离的模型目录 cache,不存在 K12 专属额度路径。
|
||||
- 账号页没有定时静默轮询。手工额度刷新完成后页面会重新读取该账号;请求驱动或后台任务产生的状态
|
||||
变化,需要下一次显式查询账号列表后才会显示。
|
||||
- 账号展开区的 Token 结构和模型排行使用代表性账号级额度窗口聚合,查询边界严格为
|
||||
`[resetAt - windowSeconds, resetAt)`;额度刷新若返回了更早的重置时间,会按新边界重新聚合。无法取得
|
||||
完整窗口边界或只有模型专属额度时显示无数据,不回退成历史累计。金额原值保持完整精度,USD 展示值
|
||||
小于 1 美元时最多保留四位小数,其余保留两位。
|
||||
- 账号页没有定时静默轮询。手工额度刷新只替换响应中的账号行并同步状态汇总,不触发整页 loading;若
|
||||
新状态不符合当前筛选,该行从当前页移除。请求驱动或后台任务产生的状态变化,需要下一次显式查询账号
|
||||
列表后才会显示。
|
||||
|
||||
## 6. Client Key
|
||||
|
||||
|
||||
+24
-10
@@ -116,7 +116,17 @@ Provider 独占以下职责:
|
||||
- OpenAI 是逐字节透明代理:请求 body 的未知字段和字段顺序保持不变(受控模型映射除外),上游 wire 是客户端可见的事实来源,HTTP SSE 帧以原始字节原样下发,WebSocket 上游 JSON 文本原样下发;response ID 以 opaque bytes 存储,不假设 UUID 或固定长度。canonical facts 只从同一帧旁路解析,用于观测、亲和与计费。仅网关内部帧(rate limits、metadata)被就地消费,不向客户端转发。
|
||||
- xAI 是翻译层:把 Grok wire 解码后重新合成 Responses wire 事件,不承诺字节透明;上游错误按结构化 message/code/type 透出,message 中的 UUID 账号指纹替换为占位符。
|
||||
|
||||
credential 更新使用 `credential_revision` CAS。认证永久失败、封禁、额度耗尽和带截止时间的 cooldown 是账号运行时事实。cooldown 到期后,Core 的有效调度谓词会自然允许账号重新参与选择;Redis cooldown 只是热缓存。成功调用或成功额度观测才可把账号状态重新观测为 `ready`。推理请求的账号/model cooldown 只约束数据面调度,不参与 OAuth refresh;OpenAI 与 xAI 的 OAuth refresh 请求状态不明确时只按 OAuth 自身的有界退避推进该 credential 的 `next_refresh_at`:前五次约为 5/15/45/135/300 秒,之后约每 10 分钟恢复;仍有效 AT 时不晚于其到期时刻,AT 到期后仍按该节奏恢复,至 `AT 到期 + 24h` 才终态化。自动 refresh 的瞬态失败(限流、5xx、超时、畸形响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;上游明确的永久错误(`invalid_grant`、封禁)仍立即写入终态失效。
|
||||
credential 更新使用 `credential_revision` CAS。PostgreSQL 分别保存凭据事实与额度访问事实;带截止时间的
|
||||
429 cooldown 只保存在 Redis,是唯一的临时冷却运行时事实,不进入 Provider quota 展示模型。Store 把三类
|
||||
事实交给 Core 的唯一解析器,投影为 `normal`、`quota_exhausted`、`rate_limited`、`disabled` 或 `error`;
|
||||
投影为 `rate_limited` 时同时携带有效的 `rate_limited_until`,到期重算后该时间和状态一起消失。若没有其它
|
||||
阻塞事实,账号才恢复为 `normal`。成功推理或明确的成功额度观测可以写入 `Allowed`;本地时钟与不确定响应
|
||||
不能伪造恢复。推理请求的账号/model cooldown 只约束数据面调度,不参与 OAuth refresh;OpenAI 与 xAI 的
|
||||
OAuth refresh 请求状态不明确时只按 OAuth 自身的有界退避推进该 credential 的 `next_refresh_at`:前五次
|
||||
约为 5/15/45/135/300 秒,之后约每 10 分钟恢复;仍有效 Access Token 时不晚于其到期时刻,Access Token
|
||||
到期后仍按该节奏恢复,至“到期 + 24 小时”才终态化。自动 refresh 的瞬态失败(限流、5xx、超时、畸形
|
||||
响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;上游明确的永久错误
|
||||
(`invalid_grant`、封禁)仍立即写入终态失效。
|
||||
|
||||
`refresh_token_expires_at` 不是公共 SQL 列或 Core 权威状态。xAI 可在 `provider_credentials_json` 内保存它作为 Provider 私有提示;真正失效以 refresh endpoint 返回的永久错误为准。
|
||||
|
||||
@@ -127,12 +137,15 @@ exchange 成功后执行同一补全。两条路径不执行 JWKS 验签,不
|
||||
credential revision;complete 阶段重新读取目标账号及其当前 revision,并以当前 revision 做最终 CAS。
|
||||
重新授权和手工或后台 RT refresh 只轮换目标账号的 token,保留既有账号资料与 OAuth principal。
|
||||
|
||||
credential 与 quota 是两套状态机。主动 quota refresh 会拒绝过期 access token;quota 探测的
|
||||
401/403 不具备判定 refresh token 永久失效的证据。成功主动观测和正常推理响应中的 rate-limit header
|
||||
观测都 revision-fenced 写入同一 quota,并转换 readiness、`QuotaExhausted` 或 `Cooldown` 等额度事实;
|
||||
只有明确的 deactivated workspace 才能由额度链路写入 `Banned`,且额度观测不能清除 `Invalid`、
|
||||
`Expired`、`Banned`。Free、K12 等套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider
|
||||
事实并用于隔离不同套餐的 catalog cache。
|
||||
credential 与 quota 是两组独立事实,不是两套对外状态。主动 quota refresh 会拒绝过期 Access Token;
|
||||
quota 探测的 401/403 不具备判定 Refresh Token 永久失效的证据。成功主动观测和正常推理响应中的
|
||||
rate-limit header 观测都 revision-fenced 写入同一 quota;只有明确的访问证据才写 `Allowed` 或
|
||||
`Exhausted`。只有明确的 deactivated workspace 才能由额度链路写入 `Banned` 凭据事实,且额度观测不能
|
||||
清除 `Invalid`、`Expired`、`Banned`。这些凭据事实对外统一投影为 `error` 并携带稳定原因。Free、K12 等
|
||||
套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider 事实并用于隔离不同套餐的 catalog cache。
|
||||
账号展开用量不读取 retention 范围累计值,而是选择带完整边界且可归属到账号的代表性额度窗口,并按
|
||||
`[reset_at - window_seconds, reset_at)` 聚合请求、Token、模型和成本;额度重置边界变化后,同一查询随刷新
|
||||
结果重算。模型专属窗口不会错误复用账号总用量,边界不完整时也不伪造“当前窗口”统计。
|
||||
|
||||
## 7. 控制面与内部 revision
|
||||
|
||||
@@ -156,7 +169,7 @@ Admin API 不要求客户端提交配置 revision,也不向客户端暴露配
|
||||
| `admin_audit_events` | 管理 mutation 审计 |
|
||||
| `client_api_keys` | 客户端鉴权、限额与授权范围 |
|
||||
| `runtime_settings` | 全局配置与 `config_revision` |
|
||||
| `provider_accounts` | 账号资料、Provider-owned 明文 credential JSON、revision、quota、cooldown |
|
||||
| `provider_accounts` | 账号资料、Provider-owned 明文 credential JSON、revision、凭据事实与规范化 quota 事实 |
|
||||
| `model_requests` | 请求、attempt、计费、交付与恢复事实 |
|
||||
| `ops_events` | 脱敏运行事件 |
|
||||
| `backup_settings` | S3/R2 存储、Cron 计划与保留策略单例配置 |
|
||||
@@ -184,13 +197,14 @@ heartbeat;计划时间点冲突只记录日志/指标并推进游标,不产
|
||||
|
||||
业务 schema 由 `backend/migrations/0001_initial.sql` 直接定义当前大版本的完整新库基线,其中
|
||||
response ID 使用无格式假设的 UTF-8 bytes,`model_requests.service_tier` 保存响应观测到的服务档位,
|
||||
且额度耗尽账号可以携带冷却截止时间。同一大版本内已合入的迁移永久冻结——
|
||||
账号对外五态由 PostgreSQL 凭据/额度事实与 Redis 429 冷却通过 Core 唯一解析器动态派生,
|
||||
不保存重复的 `status` 列。同一大版本内已合入的迁移永久冻结——
|
||||
`.frozen-sha256` 是冻结清单,CI 校验清单相对 PR base 只增不改,启动时 sqlx 重校验已应用
|
||||
迁移的 checksum,不一致直接拒绝启动。规则见 `backend/migrations/README.md`。
|
||||
|
||||
## 9. Redis 终态
|
||||
|
||||
Redis 保存的协调状态恰好是:客户端 admission 热状态、credential lease/fencing 与调度信号、账号 cooldown 与 availability 热缓存、catalog 缓存、Provider circuit、会话亲和与会话级账号排除、continuation pin、OAuth pending flow、worker leader lease 以及 runtime change 通知。Redis 丢失后必须从 PostgreSQL 或 Provider 事实恢复;恢复完成前需要保护的 acquire 路径 fail closed。
|
||||
Redis 保存的协调状态恰好是:客户端 admission 热状态、credential lease/fencing 与调度信号、账号 429 cooldown 与 credential state 可重建缓存、catalog 缓存、Provider circuit、会话亲和与会话级账号排除、continuation pin、OAuth pending flow、worker leader lease以及 runtime change 通知。Redis 丢失后必须从 PostgreSQL 或 Provider 事实恢复;恢复完成前需要保护的 acquire 路径 fail closed。
|
||||
|
||||
多副本部署时,跨副本协调面只有上述 Redis 状态与 PostgreSQL 事实;WebSocket 连接池、请求画像与 RuntimeSnapshot 副本都是进程内状态,其中 RuntimeSnapshot 依靠周期对账收敛,请求画像依靠各自的官方版本检查收敛。
|
||||
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
import request from '../request'
|
||||
|
||||
export type AccountStatus
|
||||
= 'normal' | 'quota_exhausted' | 'rate_limited' | 'disabled' | 'error'
|
||||
|
||||
export type AccountErrorReason
|
||||
= 'account_unverified'
|
||||
| 'access_token_expired'
|
||||
| 'credential_expired'
|
||||
| 'credential_invalid'
|
||||
| 'account_banned'
|
||||
|
||||
export interface AccountQuotaWindow {
|
||||
key: string
|
||||
group: string
|
||||
@@ -104,11 +114,9 @@ export interface Account {
|
||||
planType: string | null
|
||||
authenticationKind: string
|
||||
hasRefreshToken: boolean
|
||||
status: string
|
||||
displayStatus: string
|
||||
errorReason: string | null
|
||||
status: AccountStatus
|
||||
errorReason: AccountErrorReason | null
|
||||
errorMessage: string | null
|
||||
availability: string
|
||||
enabled: boolean
|
||||
accessTokenExpiresAt: string | null
|
||||
accessTokenExpiresAtDisplay: string | null
|
||||
|
||||
@@ -143,14 +143,11 @@ export interface DashboardAccountUsage {
|
||||
|
||||
export interface DashboardPoolSummary {
|
||||
total: number
|
||||
active: number
|
||||
rateLimited: number
|
||||
expired: number
|
||||
invalid: number
|
||||
normal: number
|
||||
quotaExhausted: number
|
||||
refreshing: number | null
|
||||
rateLimited: number
|
||||
disabled: number
|
||||
banned: number
|
||||
error: number
|
||||
}
|
||||
|
||||
export interface DashboardCapacityInfo {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import type { AccountErrorReason, AccountStatus } from '@/api'
|
||||
import { AlertTriangle, CircleCheck, Gauge, Power, Timer } from '@lucide/vue'
|
||||
import { computed } from 'vue'
|
||||
|
||||
@@ -8,8 +9,8 @@ import { errorReasonLabels, statusLabels, statusTones } from '../constants'
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
status: string
|
||||
errorReason?: string | null
|
||||
status: AccountStatus
|
||||
errorReason?: AccountErrorReason | null
|
||||
errorMessage?: string | null
|
||||
rateLimitedUntil?: string | null
|
||||
variant?: 'inline' | 'pill'
|
||||
@@ -55,12 +56,12 @@ const statusStyles = {
|
||||
},
|
||||
} as const
|
||||
|
||||
const tone = computed(() => statusTones[props.status] || 'normal')
|
||||
const tone = computed(() => statusTones[props.status])
|
||||
const statusStyle = computed(() => statusStyles[tone.value])
|
||||
const label = computed(() => statusLabels[props.status] || props.status)
|
||||
const label = computed(() => statusLabels[props.status])
|
||||
const reasonLabel = computed(() =>
|
||||
props.errorReason
|
||||
? errorReasonLabels[props.errorReason] || props.errorReason
|
||||
? errorReasonLabels[props.errorReason]
|
||||
: null,
|
||||
)
|
||||
const errorText = computed(() => props.errorMessage || null)
|
||||
@@ -102,16 +103,16 @@ const recoveryHint = computed(() => {
|
||||
return '冷却结束后,系统会自动恢复调度。'
|
||||
|
||||
switch (props.errorReason) {
|
||||
case 'expired':
|
||||
case 'token_expired':
|
||||
case 'account_unverified':
|
||||
case 'access_token_expired':
|
||||
case 'credential_expired':
|
||||
return '重新授权后,账号会重新参与调度。'
|
||||
case 'invalid':
|
||||
case 'credential_invalid':
|
||||
return '请更新或重新导入凭据,再次测试连接。'
|
||||
case 'banned':
|
||||
case 'account_banned':
|
||||
return '请确认上游账号状态,解除限制后再启用。'
|
||||
default:
|
||||
return '重新测试连接以获取最新状态。'
|
||||
}
|
||||
return '重新测试连接以获取最新状态。'
|
||||
})
|
||||
|
||||
const triggerLabel = computed(() =>
|
||||
|
||||
@@ -11,9 +11,12 @@ defineProps<{
|
||||
class="grid gap-4 rounded-lg bg-(--cp-bg-surface) p-4 shadow-(--cp-shadow-control) xl:grid-cols-[0.52fr_1.48fr]"
|
||||
>
|
||||
<div>
|
||||
<h3 class="m-0 mb-3 text-[14px] font-heavy text-(--cp-text-primary)">
|
||||
Token 结构
|
||||
</h3>
|
||||
<div class="mb-3 flex items-baseline justify-between gap-3">
|
||||
<h3 class="m-0 text-[14px] font-heavy text-(--cp-text-primary)">
|
||||
Token 结构
|
||||
</h3>
|
||||
<span class="text-[11px] font-emphasis text-(--cp-text-muted)">当前额度窗口</span>
|
||||
</div>
|
||||
<div class="grid gap-2">
|
||||
<div class="flex items-center justify-between rounded-lg bg-(--cp-success-bg) px-3 py-2">
|
||||
<span class="text-[12px] font-bold text-(--cp-success-text)">输入 Tokens</span>
|
||||
@@ -55,10 +58,11 @@ defineProps<{
|
||||
<h3 class="m-0 text-[14px] font-heavy text-(--cp-text-primary)">
|
||||
模型使用排行
|
||||
</h3>
|
||||
<span class="text-[11px] font-emphasis text-(--cp-text-muted)">当前额度窗口</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
class="grid grid-cols-[1.2fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pb-2 text-[11px] font-heavy text-(--cp-text-secondary) shadow-[inset_0_-1px_0_var(--cp-divider-subtle)]"
|
||||
class="grid grid-cols-[1.5fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pb-2 text-[11px] font-heavy text-(--cp-text-secondary) shadow-[inset_0_-1px_0_var(--cp-divider-subtle)]"
|
||||
>
|
||||
<span>模型</span>
|
||||
<span>调用</span>
|
||||
@@ -66,9 +70,9 @@ defineProps<{
|
||||
<span>输入</span>
|
||||
<span>输出</span>
|
||||
<span>缓存</span>
|
||||
<span>总TOKEN</span>
|
||||
<span>计费金额</span>
|
||||
<span>最近请求时间</span>
|
||||
<span>总计</span>
|
||||
<span>计费</span>
|
||||
<span>最近请求</span>
|
||||
</div>
|
||||
<div
|
||||
v-if="account.usage.models.length === 0"
|
||||
@@ -80,7 +84,7 @@ defineProps<{
|
||||
<div
|
||||
v-for="model in account.usage.models"
|
||||
:key="model.model"
|
||||
class="grid grid-cols-[1.2fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pt-3 text-[12px] font-emphasis text-(--cp-text-primary)"
|
||||
class="grid grid-cols-[1.5fr_0.7fr_0.8fr_1fr_1fr_1fr_1fr_1fr_1.4fr] gap-3 pt-3 text-[12px] font-emphasis text-(--cp-text-primary)"
|
||||
>
|
||||
<span class="truncate">{{ model.model }}</span>
|
||||
<span>{{ model.requestCountDisplay }}</span>
|
||||
|
||||
@@ -42,16 +42,11 @@ interface ConnectionTestEvent {
|
||||
model?: string
|
||||
payload?: ConnectionTestRequestPayload
|
||||
success?: boolean
|
||||
accountStatus?: string
|
||||
upstreamBody?: string
|
||||
upstreamStatus?: number
|
||||
error?: string
|
||||
}
|
||||
|
||||
interface AccountConnectionTestOptions {
|
||||
onAccountStatus: (accountId: string, status: string) => void
|
||||
}
|
||||
|
||||
function connectionTestErrorText(event: ConnectionTestEvent) {
|
||||
if (typeof event.upstreamBody === 'string' && event.upstreamBody.length > 0)
|
||||
return event.upstreamBody
|
||||
@@ -69,7 +64,7 @@ function connectionTestFailureLabel(event: ConnectionTestEvent) {
|
||||
: '未能向上游发起请求'
|
||||
}
|
||||
|
||||
export function useAccountConnectionTest(options: AccountConnectionTestOptions) {
|
||||
export function useAccountConnectionTest(options: { reload: () => Promise<unknown> }) {
|
||||
const showConnectionTestModal = shallowRef(false)
|
||||
const testingAccount = shallowRef<Account | null>(null)
|
||||
const connectionTestStatus = shallowRef<ConnectionTestStatus>('idle')
|
||||
@@ -232,18 +227,6 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
)
|
||||
}
|
||||
|
||||
function applyAccountStatus(status?: string) {
|
||||
const account = testingAccount.value
|
||||
if (!account || !status)
|
||||
return
|
||||
testingAccount.value = {
|
||||
...account,
|
||||
status,
|
||||
displayStatus: status,
|
||||
}
|
||||
options.onAccountStatus(account.id, status)
|
||||
}
|
||||
|
||||
function clearConnectionTestRun() {
|
||||
const run = connectionTestRun
|
||||
connectionTestRun = undefined
|
||||
@@ -283,7 +266,6 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
return
|
||||
}
|
||||
if (event.type === 'test_complete') {
|
||||
applyAccountStatus(event.accountStatus)
|
||||
if (event.success) {
|
||||
if (!connectionTestContent.value) {
|
||||
setConnectionTestLog('response', '响应完成', 'success', '上游已完成,没有返回文本内容')
|
||||
@@ -297,10 +279,10 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
finishConnectionTest('error')
|
||||
}
|
||||
clearConnectionTestRun()
|
||||
void options.reload()
|
||||
return
|
||||
}
|
||||
if (event.type === 'error') {
|
||||
applyAccountStatus(event.accountStatus)
|
||||
connectionTestError.value = connectionTestErrorText(event)
|
||||
appendConnectionTestLog(
|
||||
connectionTestFailureLabel(event),
|
||||
@@ -309,6 +291,7 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
)
|
||||
finishConnectionTest('error')
|
||||
clearConnectionTestRun()
|
||||
void options.reload()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import type { Ref } from 'vue'
|
||||
import type { getAccounts } from '@/api'
|
||||
import type { BaseTableSort } from '@/components/base/BaseTable/columns'
|
||||
|
||||
import dayjs from 'dayjs'
|
||||
import { ref, watch } from 'vue'
|
||||
import {
|
||||
@@ -9,7 +7,6 @@ import {
|
||||
disableAccount,
|
||||
enableAccount,
|
||||
exportAccounts,
|
||||
getAccountQuota,
|
||||
refreshAccount,
|
||||
refreshAccountQuota,
|
||||
} from '@/api'
|
||||
@@ -23,22 +20,11 @@ import { useAccountOnboarding } from './useAccountOnboarding'
|
||||
|
||||
type AccountRow = Awaited<ReturnType<typeof getAccounts>>['items'][number]
|
||||
|
||||
const accountStatusSortRank: Record<string, number> = {
|
||||
normal: 0,
|
||||
quota_exhausted: 1,
|
||||
rate_limited: 2,
|
||||
disabled: 3,
|
||||
error: 4,
|
||||
}
|
||||
|
||||
export function useAccountMutations(options: {
|
||||
accounts: Ref<AccountRow[]>
|
||||
accountSummary: Ref<Awaited<ReturnType<typeof getAccounts>>['summary']>
|
||||
statusQuery: Ref<string>
|
||||
sort: Ref<BaseTableSort | undefined>
|
||||
selectedIds: Ref<Set<string>>
|
||||
totalAccounts: Ref<number>
|
||||
reload: () => Promise<unknown>
|
||||
replaceAccount: (account: AccountRow) => boolean
|
||||
}) {
|
||||
const loadAccounts = options.reload
|
||||
const { downloadJson } = useDownload()
|
||||
@@ -188,13 +174,13 @@ export function useAccountMutations(options: {
|
||||
async function handleRefreshQuota(accountId: string) {
|
||||
await refreshingQuotaAccounts.run(accountId, async () => {
|
||||
try {
|
||||
await withMinimumDuration(async () => {
|
||||
await refreshAccountQuota({ accountId })
|
||||
const result = await getAccountQuota({ accountId })
|
||||
options.accounts.value = options.accounts.value.map(account =>
|
||||
account.id === accountId ? { ...account, ...result.account } : account,
|
||||
)
|
||||
})
|
||||
const result = await withMinimumDuration(() => refreshAccountQuota({ accountId }))
|
||||
const remainsVisible = options.replaceAccount(result.account)
|
||||
if (!remainsVisible) {
|
||||
const selectedIds = new Set(options.selectedIds.value)
|
||||
selectedIds.delete(accountId)
|
||||
options.selectedIds.value = selectedIds
|
||||
}
|
||||
toast.success('额度已刷新')
|
||||
}
|
||||
catch (error: unknown) {
|
||||
@@ -203,81 +189,6 @@ export function useAccountMutations(options: {
|
||||
})
|
||||
}
|
||||
|
||||
function patchAccountStatus(accountId: string, status: string) {
|
||||
const current = options.accounts.value.find(account => account.id === accountId)
|
||||
if (!current)
|
||||
return
|
||||
|
||||
if (options.statusQuery.value && options.statusQuery.value !== status) {
|
||||
options.accounts.value = options.accounts.value.filter(account => account.id !== accountId)
|
||||
options.totalAccounts.value = Math.max(0, options.totalAccounts.value - 1)
|
||||
options.selectedIds.value = new Set(
|
||||
[...options.selectedIds.value].filter(id => id !== accountId),
|
||||
)
|
||||
}
|
||||
else {
|
||||
const rows = options.accounts.value.map(account =>
|
||||
account.id === accountId
|
||||
? {
|
||||
...account,
|
||||
status,
|
||||
displayStatus: status,
|
||||
}
|
||||
: account,
|
||||
)
|
||||
options.accounts.value = sortAccountsByStatus(rows)
|
||||
}
|
||||
|
||||
if (current.status === status)
|
||||
return
|
||||
options.accountSummary.value = {
|
||||
...options.accountSummary.value,
|
||||
normal: Math.max(
|
||||
0,
|
||||
options.accountSummary.value.normal
|
||||
+ Number(status === 'normal')
|
||||
- Number(current.status === 'normal'),
|
||||
),
|
||||
quotaExhausted: Math.max(
|
||||
0,
|
||||
options.accountSummary.value.quotaExhausted
|
||||
+ Number(status === 'quota_exhausted')
|
||||
- Number(current.status === 'quota_exhausted'),
|
||||
),
|
||||
rateLimited: Math.max(
|
||||
0,
|
||||
options.accountSummary.value.rateLimited
|
||||
+ Number(status === 'rate_limited')
|
||||
- Number(current.status === 'rate_limited'),
|
||||
),
|
||||
disabled: Math.max(
|
||||
0,
|
||||
options.accountSummary.value.disabled
|
||||
+ Number(status === 'disabled')
|
||||
- Number(current.status === 'disabled'),
|
||||
),
|
||||
error: Math.max(
|
||||
0,
|
||||
options.accountSummary.value.error
|
||||
+ Number(status === 'error')
|
||||
- Number(current.status === 'error'),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
function sortAccountsByStatus(rows: AccountRow[]) {
|
||||
if (options.sort.value?.key !== 'status')
|
||||
return rows
|
||||
const direction = options.sort.value.direction === 'asc' ? 1 : -1
|
||||
return [...rows].sort((left, right) => {
|
||||
const rankDifference
|
||||
= (accountStatusSortRank[left.status] ?? 7) - (accountStatusSortRank[right.status] ?? 7)
|
||||
return rankDifference === 0
|
||||
? left.id.localeCompare(right.id) * direction
|
||||
: rankDifference * direction
|
||||
})
|
||||
}
|
||||
|
||||
async function handleToggleSchedule(account: AccountRow) {
|
||||
const action = account.enabled ? 'disable' : 'enable'
|
||||
await updatingStatusAccounts.run(account.id, async () => {
|
||||
@@ -362,7 +273,6 @@ export function useAccountMutations(options: {
|
||||
handleExportAccounts,
|
||||
handleRefresh,
|
||||
handleRefreshQuota,
|
||||
patchAccountStatus,
|
||||
handleToggleSchedule,
|
||||
scheduleActionLabel,
|
||||
}
|
||||
|
||||
@@ -7,6 +7,18 @@ import { toast } from '@/components/base/BaseToast'
|
||||
import { usePagedQuery } from '@/composables/usePagedQuery'
|
||||
import { errorMessage } from '@/utils/async'
|
||||
|
||||
type AccountRow = Awaited<ReturnType<typeof getAccounts>>['items'][number]
|
||||
type AccountSummary = Awaited<ReturnType<typeof getAccounts>>['summary']
|
||||
type AccountSummaryStatusKey = Exclude<keyof AccountSummary, 'total'>
|
||||
|
||||
const summaryKeyByStatus = {
|
||||
normal: 'normal',
|
||||
quota_exhausted: 'quotaExhausted',
|
||||
rate_limited: 'rateLimited',
|
||||
disabled: 'disabled',
|
||||
error: 'error',
|
||||
} as const satisfies Record<AccountRow['status'], AccountSummaryStatusKey>
|
||||
|
||||
export function useAccountsQuery() {
|
||||
const searchQuery = shallowRef('')
|
||||
const providerQuery = shallowRef('')
|
||||
@@ -64,6 +76,44 @@ export function useAccountsQuery() {
|
||||
void query.execute()
|
||||
}
|
||||
|
||||
function replaceAccount(updated: AccountRow) {
|
||||
const index = query.items.value.findIndex(account => account.id === updated.id)
|
||||
if (index < 0)
|
||||
return false
|
||||
|
||||
const current = query.items.value[index]
|
||||
if (!current)
|
||||
return false
|
||||
|
||||
updateAccountSummary(current.status, updated.status)
|
||||
|
||||
if (statusQuery.value && statusQuery.value !== updated.status) {
|
||||
query.items.value = query.items.value.filter(account => account.id !== updated.id)
|
||||
query.total.value = Math.max(0, query.total.value - 1)
|
||||
return false
|
||||
}
|
||||
|
||||
const accounts = [...query.items.value]
|
||||
accounts[index] = updated
|
||||
query.items.value = accounts
|
||||
return true
|
||||
}
|
||||
|
||||
function updateAccountSummary(
|
||||
previousStatus: AccountRow['status'],
|
||||
nextStatus: AccountRow['status'],
|
||||
) {
|
||||
if (previousStatus === nextStatus)
|
||||
return
|
||||
|
||||
const previousKey = summaryKeyByStatus[previousStatus]
|
||||
const nextKey = summaryKeyByStatus[nextStatus]
|
||||
const summary = { ...accountSummary.value }
|
||||
summary[previousKey] = Math.max(0, summary[previousKey] - 1)
|
||||
summary[nextKey] += 1
|
||||
accountSummary.value = summary
|
||||
}
|
||||
|
||||
watchDebounced(
|
||||
searchQuery,
|
||||
() => {
|
||||
@@ -95,6 +145,7 @@ export function useAccountsQuery() {
|
||||
sort,
|
||||
accountSummary,
|
||||
accountPagination,
|
||||
replaceAccount,
|
||||
handlePageChange,
|
||||
handlePageSizeChange,
|
||||
handleSortChange,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { getAccounts } from '@/api'
|
||||
import type { AccountErrorReason, AccountStatus, getAccounts } from '@/api'
|
||||
import type { BaseTableColumn } from '@/components/base/BaseTable/columns'
|
||||
import { clamp } from 'es-toolkit'
|
||||
import { providerDisplayName } from '@/utils/providers'
|
||||
@@ -118,7 +118,7 @@ export const accountColumns = [
|
||||
},
|
||||
] satisfies BaseTableColumn<AccountRow>[]
|
||||
|
||||
export const statusLabels: Record<string, string> = {
|
||||
export const statusLabels: Record<AccountStatus, string> = {
|
||||
normal: '正常',
|
||||
quota_exhausted: '配额耗尽',
|
||||
rate_limited: '限流中',
|
||||
@@ -126,7 +126,7 @@ export const statusLabels: Record<string, string> = {
|
||||
error: '错误',
|
||||
}
|
||||
|
||||
export const statusTones: Record<string, 'success' | 'danger' | 'warning' | 'info' | 'normal'> = {
|
||||
export const statusTones: Record<AccountStatus, 'success' | 'danger' | 'warning' | 'info' | 'normal'> = {
|
||||
normal: 'success',
|
||||
quota_exhausted: 'warning',
|
||||
rate_limited: 'warning',
|
||||
@@ -144,19 +144,19 @@ export const accountStatusFilterOptions = [
|
||||
]
|
||||
|
||||
/** `error` 分类下具体原因的展示文案(对应后端 `errorReason`)。 */
|
||||
export const errorReasonLabels: Record<string, string> = {
|
||||
expired: '账号已过期',
|
||||
token_expired: '访问令牌已过期',
|
||||
invalid: '凭据已失效',
|
||||
banned: '账号已封禁',
|
||||
unknown: '状态未知',
|
||||
export const errorReasonLabels: Record<AccountErrorReason, string> = {
|
||||
account_unverified: '账号身份尚未确认',
|
||||
access_token_expired: 'Access Token 已过期',
|
||||
credential_expired: '凭据已过期',
|
||||
credential_invalid: '凭据无效',
|
||||
account_banned: '账号不可用或已被封禁',
|
||||
}
|
||||
|
||||
/**
|
||||
* 后端已派生互斥状态(正常 / 配额耗尽 / 限流中 / 已停用 / 错误);
|
||||
* 前端只渲染,不再独立派生。
|
||||
*/
|
||||
export function derivedAccountStatus(row: AccountRow): string {
|
||||
export function derivedAccountStatus(row: AccountRow): AccountStatus {
|
||||
return row.status
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ import { accountColumns, derivedAccountStatus } from './constants'
|
||||
|
||||
const selectedIds = ref<Set<string>>(new Set())
|
||||
const {
|
||||
totalAccounts,
|
||||
loading,
|
||||
accounts,
|
||||
loadAccounts,
|
||||
@@ -37,6 +36,7 @@ const {
|
||||
sort,
|
||||
accountSummary,
|
||||
accountPagination,
|
||||
replaceAccount,
|
||||
handlePageChange,
|
||||
handlePageSizeChange,
|
||||
handleSortChange,
|
||||
@@ -67,17 +67,13 @@ const {
|
||||
handleExportAccounts,
|
||||
handleRefresh,
|
||||
handleRefreshQuota,
|
||||
patchAccountStatus,
|
||||
handleToggleSchedule,
|
||||
scheduleActionLabel,
|
||||
} = useAccountMutations({
|
||||
accounts,
|
||||
accountSummary,
|
||||
statusQuery,
|
||||
sort,
|
||||
selectedIds,
|
||||
totalAccounts,
|
||||
reload: loadAccounts,
|
||||
replaceAccount,
|
||||
})
|
||||
|
||||
const {
|
||||
@@ -99,9 +95,7 @@ const {
|
||||
openConnectionTest,
|
||||
handleRefreshConnectionTestModels,
|
||||
handleTestConnection,
|
||||
} = useAccountConnectionTest({
|
||||
onAccountStatus: patchAccountStatus,
|
||||
})
|
||||
} = useAccountConnectionTest({ reload: loadAccounts })
|
||||
|
||||
const {
|
||||
expandedAccountIds,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import type { dashboardSnapshotView, MetricTone } from '../composables/presenter'
|
||||
import { CircleCheck, RefreshCw, ShieldAlert, TriangleAlert } from '@lucide/vue'
|
||||
import { CircleCheck, CircleOff, RefreshCw, ShieldAlert, TriangleAlert } from '@lucide/vue'
|
||||
import { clamp } from 'es-toolkit'
|
||||
|
||||
import { computed } from 'vue'
|
||||
@@ -82,17 +82,19 @@ const accountStatusCounts = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p)
|
||||
return null
|
||||
const active = p.active ?? 0
|
||||
const refreshing = p.refreshing ?? 0
|
||||
const quotaLimited = (p.rateLimited ?? 0) + (p.quotaExhausted ?? 0)
|
||||
const pending = (p.expired ?? 0) + (p.invalid ?? 0) + (p.disabled ?? 0) + (p.banned ?? 0)
|
||||
const normal = p.normal ?? 0
|
||||
const quotaExhausted = p.quotaExhausted ?? 0
|
||||
const rateLimited = p.rateLimited ?? 0
|
||||
const disabled = p.disabled ?? 0
|
||||
const error = p.error ?? 0
|
||||
return {
|
||||
total: p.total,
|
||||
active,
|
||||
refreshing,
|
||||
quotaLimited,
|
||||
pending,
|
||||
unavailable: quotaLimited + pending,
|
||||
normal,
|
||||
quotaExhausted,
|
||||
rateLimited,
|
||||
disabled,
|
||||
error,
|
||||
unavailable: quotaExhausted + rateLimited + disabled + error,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -101,54 +103,58 @@ const statusRows = computed(() => {
|
||||
const counts = accountStatusCounts.value
|
||||
if (!p || !counts) {
|
||||
return [
|
||||
{ label: '活跃账号', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
|
||||
{ label: '刷新中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
|
||||
{ label: '额度受限', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
|
||||
{ label: '待处理', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
|
||||
{ label: '正常', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
|
||||
{ label: '配额耗尽', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
|
||||
{ label: '限流中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
|
||||
{ label: '已停用', description: '暂无账号池观测', value: '—', tone: 'normal', icon: CircleOff },
|
||||
{ label: '错误', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
|
||||
]
|
||||
}
|
||||
const expired = p?.expired ?? 0
|
||||
const invalid = p?.invalid ?? 0
|
||||
const disabled = p?.disabled ?? 0
|
||||
const banned = p?.banned ?? 0
|
||||
|
||||
return [
|
||||
{
|
||||
label: '活跃账号',
|
||||
description: '可直接参与调度',
|
||||
value: String(counts.active),
|
||||
label: '正常',
|
||||
description: '账号当前可用',
|
||||
value: String(counts.normal),
|
||||
tone: 'success',
|
||||
icon: CircleCheck,
|
||||
},
|
||||
{
|
||||
label: '刷新中',
|
||||
description: '令牌自动刷新中',
|
||||
value: String(counts.refreshing),
|
||||
tone: 'normal',
|
||||
icon: RefreshCw,
|
||||
},
|
||||
{
|
||||
label: '额度受限',
|
||||
description: '配额耗尽与限流中',
|
||||
value: String(counts.quotaLimited),
|
||||
label: '配额耗尽',
|
||||
description: '账号额度已耗尽',
|
||||
value: String(counts.quotaExhausted),
|
||||
tone: 'warning',
|
||||
icon: TriangleAlert,
|
||||
},
|
||||
{
|
||||
label: '待处理',
|
||||
description: `过期 ${expired} · 失效 ${invalid} · 禁用 ${disabled} · 封禁 ${banned}`,
|
||||
value: String(counts.pending),
|
||||
label: '限流中',
|
||||
description: '账号请求暂时受限',
|
||||
value: String(counts.rateLimited),
|
||||
tone: 'normal',
|
||||
icon: RefreshCw,
|
||||
},
|
||||
{
|
||||
label: '已停用',
|
||||
description: '账号已停用',
|
||||
value: String(counts.disabled),
|
||||
tone: 'normal',
|
||||
icon: CircleOff,
|
||||
},
|
||||
{
|
||||
label: '错误',
|
||||
description: '账号状态异常',
|
||||
value: String(counts.error),
|
||||
tone: 'danger',
|
||||
icon: ShieldAlert,
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
const availabilityRate = computed(() => {
|
||||
const normalRate = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p || p.total === 0)
|
||||
return '—'
|
||||
return `${((p.active / p.total) * 100).toFixed(1)}%`
|
||||
return `${((p.normal / p.total) * 100).toFixed(1)}%`
|
||||
})
|
||||
|
||||
const statusBars = computed(() => {
|
||||
@@ -156,10 +162,11 @@ const statusBars = computed(() => {
|
||||
if (!counts || counts.total === 0)
|
||||
return []
|
||||
return [
|
||||
{ pct: (counts.active / counts.total) * 100, cls: 'bg-(--cp-success)' },
|
||||
{ pct: (counts.refreshing / counts.total) * 100, cls: 'bg-(--cp-normal)' },
|
||||
{ pct: (counts.quotaLimited / counts.total) * 100, cls: 'bg-(--cp-warning)' },
|
||||
{ pct: (counts.pending / counts.total) * 100, cls: 'bg-(--cp-danger)' },
|
||||
{ pct: (counts.normal / counts.total) * 100, cls: 'bg-(--cp-success)' },
|
||||
{ pct: (counts.quotaExhausted / counts.total) * 100, cls: 'bg-(--cp-warning)' },
|
||||
{ pct: (counts.rateLimited / counts.total) * 100, cls: 'bg-(--cp-normal)' },
|
||||
{ pct: (counts.disabled / counts.total) * 100, cls: 'bg-(--cp-text-secondary)' },
|
||||
{ pct: (counts.error / counts.total) * 100, cls: 'bg-(--cp-danger)' },
|
||||
].filter(b => b.pct > 0)
|
||||
})
|
||||
</script>
|
||||
@@ -305,8 +312,8 @@ const statusBars = computed(() => {
|
||||
<div class="grid justify-items-end">
|
||||
<strong
|
||||
class="font-mono text-2xl leading-[1.05] font-heavy tabular-nums text-(--cp-success-text)"
|
||||
>{{ availabilityRate }}</strong>
|
||||
<span class="mt-0.5 text-xs leading-[1.15] font-bold text-(--cp-text-secondary)">可用率</span>
|
||||
>{{ normalRate }}</strong>
|
||||
<span class="mt-0.5 text-xs leading-[1.15] font-bold text-(--cp-text-secondary)">正常率</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -326,11 +333,11 @@ const statusBars = computed(() => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mt-6.5 grid h-65.5 w-full gap-2.5">
|
||||
<div class="mt-5 grid h-67 w-full grid-rows-5 gap-2">
|
||||
<div
|
||||
v-for="row in statusRows"
|
||||
:key="row.label"
|
||||
class="grid h-14.5 grid-cols-[28px_14px_minmax(0,1fr)_76px] items-center rounded-[14px] bg-(--cp-bg-subtle) px-3.5"
|
||||
class="grid min-h-0 grid-cols-[28px_14px_minmax(0,1fr)_76px] items-center rounded-[14px] bg-(--cp-bg-subtle) px-3.5"
|
||||
>
|
||||
<span
|
||||
class="inline-flex size-7 items-center justify-center rounded-[9px]"
|
||||
|
||||
Reference in New Issue
Block a user