feat(accounts): converge status classification and persist upstream errors

Unify account status derivation in gateway-admin via AccountStatusSignals,
removing the duplicated rules in store and use_case. AccountStatus now has
five operational classes (normal/quota_exhausted/rate_limited/disabled/error),
with AccountErrorReason carrying the concrete cause.

Add provider_accounts.last_error_message, written by record_failure with the
raw upstream error and cleared on recovery to ready. The admin error badge
shows the cause and upstream text on hover.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zyy
2026-08-03 21:23:28 +08:00
co-authored by Claude Opus 4.8
parent 00879be47d
commit 8e2edb680d
33 changed files with 521 additions and 225 deletions
@@ -13,15 +13,142 @@ use super::{PageSize, Revision, observability::TimeRange};
pub use gateway_core::engine::credential::AccountAvailability;
/// 管理页使用的归一化账号状态。
///
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountStatus {
Active,
RateLimited,
/// 正常,可调度。
Normal,
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
QuotaExhausted,
Expired,
Invalid,
/// 限流中(quota 快照级信号),自动恢复,无需干预。
RateLimited,
/// 管理员手动停用(`enabled = false`)。
Disabled,
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
Error,
}
impl AccountStatus {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Normal => "normal",
Self::QuotaExhausted => "quota_exhausted",
Self::RateLimited => "rate_limited",
Self::Disabled => "disabled",
Self::Error => "error",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"normal" => Some(Self::Normal),
"quota_exhausted" => Some(Self::QuotaExhausted),
"rate_limited" => Some(Self::RateLimited),
"disabled" => Some(Self::Disabled),
"error" => Some(Self::Error),
_ => None,
}
}
}
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountStatusSignals {
pub enabled: bool,
pub availability: AccountAvailability,
pub access_token_expired: bool,
pub quota_limit_reached: bool,
}
impl AccountStatusSignals {
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
#[must_use]
pub const fn derive(self) -> AccountStatus {
if !self.enabled {
AccountStatus::Disabled
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
AccountStatus::QuotaExhausted
} else if self.access_token_expired
|| matches!(
self.availability,
AccountAvailability::Expired
| AccountAvailability::Invalid
| AccountAvailability::Banned
| AccountAvailability::Unknown
)
{
AccountStatus::Error
} else if self.quota_limit_reached {
AccountStatus::RateLimited
} else {
AccountStatus::Normal
}
}
}
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
///
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountErrorReason {
Expired,
TokenExpired,
Invalid,
Banned,
Unknown,
}
impl AccountErrorReason {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Expired => "expired",
Self::TokenExpired => "token_expired",
Self::Invalid => "invalid",
Self::Banned => "banned",
Self::Unknown => "unknown",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"expired" => Some(Self::Expired),
"token_expired" => Some(Self::TokenExpired),
"invalid" => Some(Self::Invalid),
"banned" => Some(Self::Banned),
"unknown" => Some(Self::Unknown),
_ => None,
}
}
}
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
///
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
#[must_use]
pub fn derive_error_reason(signals: AccountStatusSignals) -> Option<AccountErrorReason> {
if !signals.enabled {
return None;
}
match signals.availability {
AccountAvailability::QuotaExhausted => None,
AccountAvailability::Ready => signals
.access_token_expired
.then_some(AccountErrorReason::TokenExpired),
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
}
}
/// 账号列表排序字段。
@@ -78,6 +205,8 @@ pub struct AccountRecord {
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub last_error_message: Option<String>,
pub quota_observed_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
@@ -170,14 +299,15 @@ pub struct AccountPage {
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
///
/// 配额耗尽与限流中是可恢复状态,分别计数;`unavailable` 只包含过期、失效、禁用和封禁账号。
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountSummary {
pub total: u64,
pub active: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub rate_limited: u64,
pub unavailable: u64,
pub disabled: u64,
pub error: u64,
}
/// 账号启停写入命令。
@@ -11,7 +11,10 @@ use gateway_core::{
use super::{
AdminError, MutationActor, MutationContext, PageSize, Revision,
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
accounts::{
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
AccountUsage,
},
};
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
@@ -688,6 +691,8 @@ impl fmt::Debug for ProviderExport {
pub struct AccountDirectoryItem {
pub account: AccountRecord,
pub status: AccountStatus,
/// `status == Error` 时的具体原因;其余状态为 `None`。
pub error_reason: Option<AccountErrorReason>,
pub usage: Option<super::accounts::AccountUsage>,
pub quota: ProviderQuota,
}
@@ -18,7 +18,8 @@ use crate::{
AdminError, MutationContext,
accounts::{
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
AccountRecord, AccountStatus, AccountUsageWindowQuery,
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
derive_error_reason,
},
observability::TimeRange,
provider_credentials::{
@@ -216,8 +217,10 @@ impl DefaultAccountsService {
std::slice::from_mut(&mut quota),
)
.await?;
let signals = account_signals(&account, quota.limit_reached, now);
Ok(AccountDirectoryItem {
status: account_status(&account, quota.limit_reached, now),
status: signals.derive(),
error_reason: derive_error_reason(signals),
usage: usage.into_iter().next(),
account,
quota,
@@ -301,11 +304,15 @@ impl AccountsService for DefaultAccountsService {
.items
.into_iter()
.zip(quotas)
.map(|(account, quota)| AccountDirectoryItem {
status: account_status(&account, quota.limit_reached, now),
usage: usage.get(&account.id).cloned(),
account,
quota,
.map(|(account, quota)| {
let signals = account_signals(&account, quota.limit_reached, now);
AccountDirectoryItem {
status: signals.derive(),
error_reason: derive_error_reason(signals),
usage: usage.get(&account.id).cloned(),
account,
quota,
}
})
.collect();
Ok(AccountDirectoryPage {
@@ -516,39 +523,29 @@ fn empty_quota() -> ProviderQuota {
}
}
/// 互斥页面状态:availability 终态优先,quota 快照级 `limit_reached` 派生
/// `rate_limited`,`Unknown` 保守映射 `invalid`(不可调度,不显示为 active)。
/// 构造状态派生所需的信号快照。
fn account_signals(
account: &AccountRecord,
quota_limit_reached: bool,
now: chrono::DateTime<Utc>,
) -> AccountStatusSignals {
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached,
}
}
/// 账号运营分类(互斥展示状态)。
fn account_status(
account: &AccountRecord,
quota_limit_reached: bool,
now: chrono::DateTime<Utc>,
) -> AccountStatus {
if !account.enabled {
AccountStatus::Disabled
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
AccountStatus::Banned
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted {
AccountStatus::QuotaExhausted
} else if matches!(
account.availability,
crate::model::accounts::AccountAvailability::Invalid
) {
AccountStatus::Invalid
} else if matches!(
account.availability,
crate::model::accounts::AccountAvailability::Expired
) || account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now)
{
AccountStatus::Expired
} else if quota_limit_reached {
AccountStatus::RateLimited
} else if account.availability == crate::model::accounts::AccountAvailability::Unknown {
AccountStatus::Invalid
} else {
AccountStatus::Active
}
account_signals(account, quota_limit_reached, now).derive()
}
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
@@ -451,10 +451,11 @@ impl AccountStore for FakeAccountStore {
total: accounts.len() as u64,
summary: AccountSummary {
total: accounts.len() as u64,
active: 1,
normal: 1,
quota_exhausted: 0,
rate_limited: 0,
unavailable: 0,
disabled: 0,
error: 0,
},
})
}
@@ -845,7 +846,7 @@ async fn connection_test_should_probe_unavailable_account() {
}
#[tokio::test]
async fn connection_test_rate_limited_probe_preserves_active_account_status() {
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events);
@@ -868,7 +869,7 @@ async fn connection_test_rate_limited_probe_preserves_active_account_status() {
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Active,
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
..
})
));
@@ -949,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
assert_eq!(
result.account.status,
gateway_admin::model::accounts::AccountStatus::Active
gateway_admin::model::accounts::AccountStatus::Normal
);
assert_eq!(
@@ -985,12 +986,12 @@ async fn accounts_list_should_return_complete_directory_semantics() {
.expect("complete account directory");
assert_eq!(page.summary.total, 1);
assert_eq!(page.summary.active, 1);
assert_eq!(page.summary.normal, 1);
let account = page.items.first().expect("account item");
assert_eq!(account.account.provider_kind.as_str(), "openai");
assert_eq!(
account.status,
gateway_admin::model::accounts::AccountStatus::Active
gateway_admin::model::accounts::AccountStatus::Normal
);
}
@@ -1089,8 +1090,8 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
}
#[tokio::test]
async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
// Unknown 不可调度,Admin 不得显示为 active。
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
// Unknown 不可调度,Admin 不得显示为 normal。
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::Unknown;
@@ -1112,7 +1113,7 @@ async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::Invalid,
gateway_admin::model::accounts::AccountStatus::Error,
);
}
@@ -1333,6 +1334,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
@@ -91,7 +91,8 @@ impl ListQuery {
let status = match self.status.as_deref().map(str::trim) {
None | Some("") => None,
Some(value) => Some(
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
DomainAccountStatus::parse(&value.to_ascii_lowercase())
.ok_or_else(|| WireValidationError::new("status"))?,
),
};
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
@@ -127,19 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
.map_err(|_| WireValidationError::new("provider"))
}
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
match value.trim().to_ascii_lowercase().as_str() {
"active" => Some(DomainAccountStatus::Active),
"rate_limited" => Some(DomainAccountStatus::RateLimited),
"expired" => Some(DomainAccountStatus::Expired),
"invalid" => Some(DomainAccountStatus::Invalid),
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
"disabled" => Some(DomainAccountStatus::Disabled),
"banned" => Some(DomainAccountStatus::Banned),
_ => None,
}
}
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
match value.trim() {
"email" => Some(AccountSortField::Email),
@@ -174,10 +162,11 @@ pub struct AccountPageData {
#[serde(rename_all = "camelCase")]
pub struct AccountSummaryView {
pub total: u64,
pub active: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub rate_limited: u64,
pub unavailable: u64,
pub disabled: u64,
pub error: u64,
}
/// 一条安全账号视图。
@@ -197,6 +186,10 @@ pub struct AccountView {
pub has_refresh_token: bool,
pub status: String,
pub display_status: String,
/// `status == "error"` 时的具体原因;其余状态为 `null`。
pub error_reason: Option<String>,
/// 最近一次失败的上游错误描述;仅错误状态存在。
pub error_message: Option<String>,
pub availability: String,
pub enabled: bool,
pub access_token_expires_at: Option<String>,
@@ -530,7 +523,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
"type": "test_complete",
"success": true,
"accountStatus": account_status_name(account_status)
"accountStatus": account_status.as_str()
}),
DomainConnectionTestEvent::Failed {
message,
@@ -548,7 +541,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
"upstreamStatus": upstream_status,
"upstreamContentType": upstream_content_type,
"upstreamBody": upstream_body,
"accountStatus": account_status_name(account_status)
"accountStatus": account_status.as_str()
}),
};
Self { data }
@@ -1336,10 +1329,11 @@ fn account_page_data(
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
summary: AccountSummaryView {
total: result.summary.total,
active: result.summary.active,
normal: result.summary.normal,
quota_exhausted: result.summary.quota_exhausted,
rate_limited: result.summary.rate_limited,
unavailable: result.summary.unavailable,
disabled: result.summary.disabled,
error: result.summary.error,
},
}
}
@@ -1370,10 +1364,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
let AccountDirectoryItem {
account,
status,
error_reason,
usage,
quota,
} = item;
let status = account_status_name(status).to_owned();
let status = status.as_str().to_owned();
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
let added_at = china_rfc3339(&account.created_at);
let updated_at = china_rfc3339(&account.updated_at);
@@ -1392,6 +1387,8 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
has_refresh_token: account.has_refresh_token,
status: status.clone(),
display_status: status,
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
error_message: account.last_error_message,
availability: account.availability.as_str().to_owned(),
enabled: account.enabled,
access_token_expires_at: expires_at,
@@ -1717,18 +1714,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
Value::Object(document.into_provider_data().into_inner())
}
fn account_status_name(status: DomainAccountStatus) -> &'static str {
match status {
DomainAccountStatus::Active => "active",
DomainAccountStatus::RateLimited => "rate_limited",
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
DomainAccountStatus::Expired => "expired",
DomainAccountStatus::Invalid => "invalid",
DomainAccountStatus::Disabled => "disabled",
DomainAccountStatus::Banned => "banned",
}
}
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
let elapsed = now.signed_duration_since(value);
if elapsed.num_seconds() < 0 {
@@ -10,7 +10,7 @@ mod query {
"pageSize": 20,
"provider": "xai",
"search": " operator ",
"status": "active",
"status": "normal",
"sortBy": "lastUsedAt",
"sortDirection": "desc"
}))
@@ -23,7 +23,7 @@ mod query {
Some(ref provider) if provider.as_str() == "xai"
));
assert_eq!(query.search.as_deref(), Some("operator"));
assert_eq!(query.status, Some(AccountStatus::Active));
assert_eq!(query.status, Some(AccountStatus::Normal));
assert_eq!(
query.sort.expect("sort").field,
AccountSortField::LastUsedAt
@@ -344,7 +344,7 @@ mod actions {
text: "OK".to_owned(),
},
DomainConnectionTestEvent::Completed {
account_status: AccountStatus::Active,
account_status: AccountStatus::Normal,
},
DomainConnectionTestEvent::Failed {
message: "upstream unavailable".to_owned(),
@@ -378,7 +378,7 @@ mod actions {
}
}),
json!({ "type": "content", "text": "OK" }),
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
json!({
"type": "error",
"error": "upstream unavailable",
@@ -607,6 +607,8 @@ pub struct AccountStateChange {
pub expected_revision: CredentialRevision,
pub availability: AccountAvailability,
pub observed_at: SystemTime,
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
pub message: Option<String>,
}
/// `provider_accounts` 的数据库中立端口。
@@ -247,6 +247,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.map_err(core_store_error)?,
availability: change.availability,
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
message: change.message,
})
.await
.map_err(core_store_error)?;
@@ -2,6 +2,8 @@
use super::*;
use gateway_admin::model::accounts::AccountStatusSignals;
pub(crate) struct AdminAccountListItem {
pub(crate) account: ProviderAccountSummary,
pub(crate) status: AdminAccountStatus,
@@ -59,27 +61,16 @@ pub(crate) fn admin_account_status(
account: &ProviderAccountSummary,
now: DateTime<Utc>,
) -> AdminAccountStatus {
if !account.enabled {
AdminAccountStatus::Disabled
} else if account.availability == AccountAvailability::Banned {
AdminAccountStatus::Banned
} else if account.availability == AccountAvailability::QuotaExhausted {
AdminAccountStatus::QuotaExhausted
} else if account.availability == AccountAvailability::Invalid {
AdminAccountStatus::Invalid
} else if account.availability == AccountAvailability::Expired
|| account
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now)
{
AdminAccountStatus::Expired
} else if account.quota_limit_reached {
AdminAccountStatus::RateLimited
} else if account.availability == AccountAvailability::Unknown {
AdminAccountStatus::Invalid
} else {
AdminAccountStatus::Active
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached: account.quota_limit_reached,
}
.derive()
}
pub(crate) fn admin_account_summary(
@@ -88,26 +79,25 @@ pub(crate) fn admin_account_summary(
) -> AccountSummary {
let mut summary = AccountSummary {
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
active: 0,
normal: 0,
quota_exhausted: 0,
rate_limited: 0,
unavailable: 0,
disabled: 0,
error: 0,
};
for account in accounts {
match admin_account_status(account, now) {
AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1),
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
AdminAccountStatus::QuotaExhausted => {
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
}
AdminAccountStatus::RateLimited => {
summary.rate_limited = summary.rate_limited.saturating_add(1);
}
AdminAccountStatus::Expired
| AdminAccountStatus::Invalid
| AdminAccountStatus::Disabled
| AdminAccountStatus::Banned => {
summary.unavailable = summary.unavailable.saturating_add(1);
AdminAccountStatus::Disabled => {
summary.disabled = summary.disabled.saturating_add(1);
}
AdminAccountStatus::Error => summary.error = summary.error.saturating_add(1),
}
}
summary
@@ -124,9 +114,7 @@ pub(crate) fn sort_admin_account_items(
items.sort_by(|left, right| {
let ordering = match sort.field {
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
AdminAccountSortField::Status => {
admin_account_status_name(left.status).cmp(admin_account_status_name(right.status))
}
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
AdminAccountSortField::Usage => left
.usage
@@ -156,18 +144,6 @@ pub(crate) fn sort_admin_account_items(
});
}
pub(crate) const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str {
match status {
AdminAccountStatus::Active => "active",
AdminAccountStatus::RateLimited => "rate_limited",
AdminAccountStatus::Expired => "expired",
AdminAccountStatus::Invalid => "invalid",
AdminAccountStatus::QuotaExhausted => "quota_exhausted",
AdminAccountStatus::Disabled => "disabled",
AdminAccountStatus::Banned => "banned",
}
}
pub(crate) fn admin_account_record(
summary: ProviderAccountSummary,
) -> AdminStoreResult<AccountRecord> {
@@ -193,6 +169,7 @@ pub(crate) fn admin_account_record(
enabled: summary.enabled,
availability: summary.availability,
availability_observed_at: summary.availability_observed_at,
last_error_message: summary.last_error_message,
quota_observed_at: summary.quota_observed_at,
created_at: summary.created_at,
updated_at: summary.updated_at,
@@ -95,7 +95,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, availability,
availability_observed_at,
quota_observed_at, quota_limit_reached, created_at, updated_at
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
from provider_accounts
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
order by provider_kind, name, id",
@@ -214,6 +214,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
when enabled then $4
else availability_observed_at
end,
last_error_message = case
when enabled then
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
else last_error_message
end,
updated_at = case when enabled then greatest(now(), $4) else updated_at end
where id = $1 and credential_revision = $2
and (availability_observed_at is null or availability_observed_at <= $4)",
@@ -222,6 +227,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(to_i64(update.expected_revision.get())?)
.bind(update.availability.as_str())
.bind(update.availability_observed_at)
.bind(update.message.as_deref())
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("apply provider account state"))?;
@@ -90,6 +90,8 @@ pub struct ProviderAccountSummary {
pub quota_observed_at: Option<DateTime<Utc>>,
/// 快照级限流事实(Provider 物化)。
pub quota_limit_reached: bool,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub last_error_message: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -310,6 +312,8 @@ pub struct ProviderAccountStateUpdate {
pub expected_revision: Revision,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
pub message: Option<String>,
}
impl ProviderAccountStateUpdate {
@@ -335,14 +339,14 @@ impl ProviderAccountObservation {
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
provider_quota_json, quota_limit_reached,
provider_quota_json, quota_limit_reached, last_error_message,
availability_observed_at, quota_observed_at, created_at, updated_at
from provider_accounts where id = $1";
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
provider_quota_json, quota_limit_reached,
provider_quota_json, quota_limit_reached, last_error_message,
availability_observed_at, quota_observed_at, created_at, updated_at
from provider_accounts
where id = any($1::text[]) and provider_kind = $2
@@ -445,6 +449,7 @@ pub(crate) fn account_summary_from_row(
availability_observed_at: get(&row, "availability_observed_at")?,
quota_observed_at: get(&row, "quota_observed_at")?,
quota_limit_reached: get(&row, "quota_limit_reached")?,
last_error_message: get(&row, "last_error_message")?,
created_at: get(&row, "created_at")?,
updated_at: get(&row, "updated_at")?,
})
@@ -160,6 +160,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::QuotaExhausted,
@@ -170,6 +171,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
assert!(
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::Ready,
@@ -200,6 +202,61 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
database.close().await;
}
#[tokio::test]
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let account_id = ProviderAccountId::new("acct_state_message").expect("account ID");
repository
.insert_provider_account(account(account_id.as_str(), "user-state-message"))
.await
.expect("insert state message fixture");
let revision = CredentialRevision::new(1).expect("revision");
repository
.apply_state_change(AccountStateChange {
message: Some("upstream returned 402 payment required".to_owned()),
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
})
.await
.expect("persist error message");
let stored: Option<String> =
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("read error message");
assert_eq!(
stored.as_deref(),
Some("upstream returned 402 payment required")
);
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::Ready,
observed_at: SystemTime::now(),
})
.await
.expect("recover account");
let cleared: Option<String> =
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("read cleared error message");
assert_eq!(cleared, None);
database.close().await;
}
#[tokio::test]
async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_usage() {
let Some(database) = TestDatabase::create("provider_account_terminal_list").await else {
@@ -322,16 +379,18 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
assert_eq!(usage_page.config_revision.get(), 1);
assert_eq!(usage_page.total, 6);
assert_eq!(usage_page.summary.total, 6);
assert_eq!(usage_page.summary.active, 1);
assert_eq!(usage_page.summary.normal, 1);
assert_eq!(usage_page.summary.quota_exhausted, 1);
assert_eq!(usage_page.summary.rate_limited, 1);
assert_eq!(usage_page.summary.unavailable, 3);
assert_eq!(usage_page.summary.disabled, 1);
assert_eq!(usage_page.summary.error, 2);
assert_eq!(
usage_page.summary.total,
usage_page.summary.active
usage_page.summary.normal
+ usage_page.summary.quota_exhausted
+ usage_page.summary.rate_limited
+ usage_page.summary.unavailable
+ usage_page.summary.disabled
+ usage_page.summary.error
);
assert_eq!(
usage_page
@@ -371,7 +430,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
page_size: PageSize::new(10).expect("page size"),
provider_kind: Some(ProviderKind::new("openai").expect("Provider kind")),
search: Some("ALPHA@EXAMPLE".to_owned()),
status: Some(AccountStatus::Active),
status: Some(AccountStatus::Normal),
sort: None,
})
.await
@@ -381,18 +440,20 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
assert_eq!(filtered.items[0].id, "acct_alpha");
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
let banned = store
let error_accounts = store
.list_accounts(AccountListQuery {
page: 1,
page_size: PageSize::new(10).expect("page size"),
provider_kind: None,
search: None,
status: Some(AccountStatus::Banned),
status: Some(AccountStatus::Error),
sort: None,
})
.await
.expect("filter banned accounts");
assert_eq!(banned.items[0].id, "acct_beta");
.expect("filter error accounts");
assert_eq!(error_accounts.items.len(), 2);
assert_eq!(error_accounts.items[0].id, "acct_beta");
assert_eq!(error_accounts.items[1].id, "acct_invalid");
database.close().await;
}
@@ -1210,6 +1271,7 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::QuotaExhausted,
@@ -1248,6 +1310,7 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
let observed_at = SystemTime::now();
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Ready,
@@ -1297,6 +1360,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Ready,
@@ -234,6 +234,18 @@ impl CodexCredentialRepository {
account: &ProviderAccount,
availability: AccountAvailability,
observed_at: SystemTime,
) -> Result<(), CredentialRepositoryError> {
self.apply_state_with_message(account, availability, observed_at, None)
.await
}
/// 带上游错误描述的状态写入;`message` 为 `Some` 时持久化最近错误,恢复 `Ready` 时清空。
pub async fn apply_state_with_message(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
observed_at: SystemTime,
message: Option<String>,
) -> Result<(), CredentialRepositoryError> {
if !account.enabled() {
return Ok(());
@@ -244,6 +256,7 @@ impl CodexCredentialRepository {
expected_revision: account.revision(),
availability,
observed_at,
message,
})
.await?;
Ok(())
@@ -685,26 +685,32 @@ impl CodexCredentialSelector {
&self,
account: &ProviderAccount,
failure: CodexAccountFailure,
message: Option<String>,
) -> Result<(), CredentialSelectionError> {
let now = SystemTime::now();
match failure {
CodexAccountFailure::CredentialExpired => {
self.apply_state(account, AccountAvailability::Expired, now)
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
.await
}
CodexAccountFailure::IdentityVerificationRequired => {
self.apply_state(account, AccountAvailability::Invalid, now)
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
.await
}
CodexAccountFailure::Banned => {
self.apply_state(account, AccountAvailability::Banned, now)
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
.await
}
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
CodexAccountFailure::QuotaExhausted
| CodexAccountFailure::UsageLimitExhausted { .. } => {
self.apply_state(account, AccountAvailability::QuotaExhausted, now)
.await
self.apply_state_with_message(
account,
AccountAvailability::QuotaExhausted,
now,
message,
)
.await
}
// 429:临时限流只写 quota,availability 不改变。
CodexAccountFailure::RateLimited { retry_after } => {
@@ -727,8 +733,13 @@ impl CodexCredentialSelector {
CodexAccountFailure::CloudflarePathBlocked => {
let blocked = self.record_cloudflare_path_block(account.id(), now);
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
self.apply_state(account, AccountAvailability::Invalid, now)
.await?;
self.apply_state_with_message(
account,
AccountAvailability::Invalid,
now,
message,
)
.await?;
}
self.apply_cookie_recovery(account, CookieRecovery::Clear)
.await?;
@@ -737,14 +748,15 @@ impl CodexCredentialSelector {
}
}
async fn apply_state(
async fn apply_state_with_message(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
observed_at: SystemTime,
message: Option<String>,
) -> Result<(), CredentialSelectionError> {
self.repository
.apply_state(account, availability, observed_at)
.apply_state_with_message(account, availability, observed_at, message)
.await?;
Ok(())
}
@@ -1697,6 +1697,8 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
struct MappedProviderFailure {
error: ProviderError,
account_failure: Option<CodexAccountFailure>,
/// 原始上游错误描述,随 availability 状态一并持久化。
error_message: Option<String>,
cyber_policy_failure: bool,
set_cookie_headers: Vec<String>,
rate_limit_headers: Vec<(String, String)>,
@@ -1709,6 +1711,7 @@ impl MappedProviderFailure {
Self {
error,
account_failure: None,
error_message: None,
cyber_policy_failure: false,
set_cookie_headers: Vec::new(),
rate_limit_headers: Vec::new(),
@@ -1882,7 +1885,7 @@ async fn apply_failure(
.await;
if let Err(error) = context
.selector
.record_failure(account, account_failure)
.record_failure(account, account_failure, failure.error_message.clone())
.await
{
tracing::warn!(
@@ -2162,6 +2165,7 @@ fn map_upstream_failure(
MappedProviderFailure {
error,
account_failure: account_failure(category, failure.retry_after_seconds),
error_message: failure.client_message,
cyber_policy_failure,
set_cookie_headers: failure.set_cookie_headers,
rate_limit_headers: failure.rate_limit_headers,
@@ -544,6 +544,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
.expect("persist raw quota");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: CoreAccountAvailability::QuotaExhausted,
@@ -584,6 +585,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: CoreAccountAvailability::Ready,
@@ -711,6 +713,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
enabled: account.enabled(),
availability: AdminAccountAvailability::Ready,
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
@@ -507,6 +507,7 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
let first = store.account("acct_first").expect("first account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: first.id().clone(),
expected_revision: first.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -663,8 +664,12 @@ fn credential_expired_failure_marks_unified_account_expired() {
}),
)
.expect("select account");
block_on(selector.record_failure(lease.account(), CodexAccountFailure::CredentialExpired))
.expect("record credential expiry");
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CredentialExpired,
None,
))
.expect("record credential expiry");
assert_eq!(
store
.account("acct_primary")
@@ -697,6 +702,7 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -743,6 +749,7 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
CodexAccountFailure::UsageLimitExhausted {
retry_after: Some(Duration::from_secs(30)),
},
None,
)
.await
.expect("record usage-limit exhaustion");
@@ -758,6 +765,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -771,6 +779,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -796,6 +805,7 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -816,6 +826,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Invalid,
@@ -830,6 +841,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -854,6 +866,7 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: stale,
@@ -892,12 +905,14 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
),
selector.record_failure(
&second,
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
),
);
first_result.expect("record first account failure");
@@ -925,7 +940,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
create_account(&store, "acct_primary", "at-primary");
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
let account = store.account("acct_primary").expect("account");
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted))
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
.expect("mark account exhausted");
let selector = selector_with_runtime(
@@ -979,6 +994,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::IdentityVerificationRequired,
None,
))
.expect("record identity verification failure");
@@ -1022,6 +1038,7 @@ fn cloudflare_challenge_does_not_change_availability() {
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CloudflareChallenge { retry_after: None },
None,
))
.expect("record challenge");
@@ -1058,9 +1075,11 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
.expect("select account");
for _ in 0..3 {
block_on(
selector.record_failure(lease.account(), CodexAccountFailure::CloudflarePathBlocked),
)
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CloudflarePathBlocked,
None,
))
.expect("record path block");
}
@@ -1114,6 +1133,7 @@ fn cloudflare_challenge_expires_provider_owned_cookies_at_cooldown_boundary() {
block_on(selector.record_failure(
second.account(),
CodexAccountFailure::CloudflareChallenge { retry_after: None },
None,
))
.expect("record challenge");
@@ -1157,8 +1177,12 @@ fn cloudflare_path_block_deletes_provider_owned_cookies() {
session_affinity_key: None,
}))
.expect("select revised account");
block_on(selector.record_failure(second.account(), CodexAccountFailure::CloudflarePathBlocked))
.expect("record path block");
block_on(selector.record_failure(
second.account(),
CodexAccountFailure::CloudflarePathBlocked,
None,
))
.expect("record path block");
let account = store.account("acct_primary").expect("account");
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
@@ -1194,7 +1218,7 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
assert_eq!(outcome.credential_revision, Some(current.revision().get()));
assert_ne!(current.revision(), lease.account().revision());
block_on(selector.record_failure(&current, CodexAccountFailure::QuotaExhausted))
block_on(selector.record_failure(&current, CodexAccountFailure::QuotaExhausted, None))
.expect("record failure with current revision");
assert_eq!(
store
@@ -235,6 +235,7 @@ async fn manual_quota_refresh_preserves_disabled_account_state() {
let account = store.account(account_id).expect("created disabled account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Expired,
@@ -288,6 +289,7 @@ async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential(
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -494,6 +496,7 @@ async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks(
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Ready,
@@ -560,6 +563,7 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -599,6 +603,7 @@ async fn periodic_quota_synchronization_respects_retry_after_on_service_unavaila
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -639,6 +644,7 @@ async fn periodic_quota_synchronization_skips_ready_accounts_without_quota_reach
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Ready,
@@ -671,6 +677,7 @@ async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() {
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -696,6 +703,7 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -761,6 +769,7 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
.expect("seed post-failure quota snapshot");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -851,6 +860,7 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -900,6 +910,7 @@ async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() {
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -923,6 +934,7 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -977,6 +989,7 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -12,6 +12,7 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Expired,
@@ -161,6 +162,7 @@ async fn expired_old_reset_with_fresh_allowed_snapshot_recovers_ready() {
.expect("seed old exhausted snapshot");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -436,6 +436,7 @@ impl GrokCredentialRepository {
expected_revision: input.expected_revision,
availability: input.availability.into(),
observed_at: to_system_time(input.observed_at),
message: input.availability_reason.clone(),
})
.await
.map_err(map_store_error)
@@ -473,6 +473,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
enabled: account.enabled(),
availability: account.availability(),
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
@@ -204,6 +204,7 @@ async fn set_account_state(
) {
store
.apply_state_change(AccountStateChange {
message: None,
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability,
@@ -485,6 +486,7 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() {
let id = account_id("quota-exhausted");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: id,
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: AccountAvailability::QuotaExhausted,
@@ -755,6 +757,7 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
let transport = Arc::new(MutatingBillingTransport {
store: Arc::clone(&store),
mutation: Mutex::new(Some(BillingMutation::State(AccountStateChange {
message: None,
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: AccountAvailability::Ready,
+1 -1
View File
@@ -1 +1 @@
609d455f410f7be362bcdce841630728aaf2681f5cd6b2a6012529e5f8b8b1f6 0001_initial.sql
5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql
+1
View File
@@ -133,6 +133,7 @@ create table provider_accounts (
availability text not null default 'unknown',
provider_quota_json jsonb,
quota_limit_reached boolean not null default false,
last_error_message text,
availability_observed_at timestamptz not null,
quota_observed_at timestamptz,
created_at timestamptz not null,
+5 -2
View File
@@ -104,6 +104,8 @@ export interface Account {
hasRefreshToken: boolean
status: string
displayStatus: string
errorReason: string | null
errorMessage: string | null
availability: string
enabled: boolean
accessTokenExpiresAt: string | null
@@ -127,10 +129,11 @@ export interface AccountPageMeta {
export interface AccountSummary {
total: number
active: number
normal: number
quotaExhausted: number
rateLimited: number
unavailable: number
disabled: number
error: number
}
export interface AccountListResponse {
@@ -57,7 +57,12 @@ function connectionLogClass(tone: string) {
class="flex items-center justify-between gap-4 rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3"
>
<AccountIdentityCell :account="account" size="lg" show-plan />
<AccountStatusBadge :status="account.status" variant="pill" />
<AccountStatusBadge
:status="account.status"
:error-reason="account.errorReason"
:error-message="account.errorMessage"
variant="pill"
/>
</section>
<section class="rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3">
@@ -20,7 +20,7 @@ const overviewItems = computed(() => [
},
{
label: '正常账号',
value: formatCount(props.summary.active),
value: formatCount(props.summary.normal),
caption: '可参与调度',
tone: 'success',
icon: ShieldCheck,
@@ -33,9 +33,9 @@ const overviewItems = computed(() => [
icon: Gauge,
},
{
label: '异常账号',
value: formatCount(props.summary.unavailable),
caption: '过期 / 失效 / 禁用 / 封禁',
label: '待处理',
value: formatCount((props.summary.disabled ?? 0) + (props.summary.error ?? 0)),
caption: '已停用 / 错误',
tone: 'danger',
icon: AlertTriangle,
},
@@ -1,20 +1,36 @@
<script setup lang="ts">
import { computed } from 'vue'
import { statusLabels, statusTones } from '../constants'
import BasePopover from '@/components/base/BasePopover.vue'
import { errorReasonLabels, statusLabels, statusTones } from '../constants'
const props = withDefaults(
defineProps<{
status: string
/** `status === 'error'` 时的具体原因(对应后端 `errorReason`)。 */
errorReason?: string | null
/** 最近一次失败的上游错误原文(对应后端 `errorMessage`),作为辅助证据展示。 */
errorMessage?: string | null
variant?: 'inline' | 'pill'
}>(),
{
errorReason: null,
errorMessage: null,
variant: 'inline',
},
)
const tone = computed(() => statusTones[props.status])
const label = computed(() => statusLabels[props.status] || props.status)
/** 分类原因(受控枚举映射文案);`status === 'error'` 时有值。 */
const reasonLabel = computed(() => {
if (props.errorReason)
return errorReasonLabels[props.errorReason] || props.errorReason
return null
})
/** 上游错误原文(辅助证据);有则显、无则隐。 */
const errorText = computed(() => props.errorMessage || null)
const hasDetail = computed(() => Boolean(reasonLabel.value) || Boolean(errorText.value))
const textClass = computed(() => {
if (tone.value === 'success') {
@@ -50,19 +66,35 @@ const dotClass = computed(() => {
</script>
<template>
<span
v-if="variant === 'pill'"
class="inline-flex h-7 shrink-0 items-center rounded-full px-2.5 text-[12px] font-heavy"
:class="textClass"
<BasePopover
:disabled="!hasDetail"
trigger="hover"
placement="top-start"
:width="240"
panel-class="p-2"
>
{{ label }}
</span>
<span
v-else
class="inline-flex min-w-16 items-center gap-1.5 text-[12px] leading-none font-emphasis"
:class="textClass"
>
<span class="size-1.5 rounded-full" :class="dotClass" />
<span>{{ label }}</span>
</span>
<template #trigger>
<span
v-if="variant === 'pill'"
class="inline-flex h-7 shrink-0 items-center rounded-full px-2.5 text-[12px] font-heavy"
:class="textClass"
>
{{ label }}
</span>
<span
v-else
class="inline-flex min-w-16 items-center gap-1.5 text-[12px] leading-none font-emphasis"
:class="textClass"
>
<span class="size-1.5 rounded-full" :class="dotClass" />
<span>{{ label }}</span>
</span>
</template>
<p v-if="reasonLabel" class="m-0 text-[12px] font-medium leading-relaxed text-(--cp-text-primary)">
{{ reasonLabel }}
</p>
<p v-if="errorText" class="m-0 mt-1 text-[12px] leading-relaxed text-(--cp-text-secondary)">
{{ errorText }}
</p>
</BasePopover>
</template>
@@ -26,7 +26,7 @@ const emit = defineEmits<{
const canRefreshToken = computed(
() =>
props.account.hasRefreshToken
&& (props.account.status === 'active' || props.account.status === 'quota_exhausted'),
&& (props.account.status === 'normal' || props.account.status === 'quota_exhausted'),
)
const canReauthorize = computed(() => props.account.provider === 'openai')
</script>
@@ -24,13 +24,11 @@ import { useAccountOnboarding } from './useAccountOnboarding'
type AccountRow = Awaited<ReturnType<typeof getAccounts>>['items'][number]
const accountStatusSortRank: Record<string, number> = {
active: 0,
normal: 0,
quota_exhausted: 1,
rate_limited: 2,
expired: 3,
invalid: 4,
disabled: 5,
banned: 6,
disabled: 3,
error: 4,
}
export function useAccountMutations(options: {
@@ -234,11 +232,11 @@ export function useAccountMutations(options: {
return
options.accountSummary.value = {
...options.accountSummary.value,
active: Math.max(
normal: Math.max(
0,
options.accountSummary.value.active
+ Number(status === 'active')
- Number(current.status === 'active'),
options.accountSummary.value.normal
+ Number(status === 'normal')
- Number(current.status === 'normal'),
),
quotaExhausted: Math.max(
0,
@@ -252,22 +250,21 @@ export function useAccountMutations(options: {
+ Number(status === 'rate_limited')
- Number(current.status === 'rate_limited'),
),
unavailable: Math.max(
disabled: Math.max(
0,
options.accountSummary.value.unavailable
+ Number(isUnavailableAccountStatus(status))
- Number(isUnavailableAccountStatus(current.status)),
options.accountSummary.value.disabled
+ Number(status === 'disabled')
- Number(current.status === 'disabled'),
),
error: Math.max(
0,
options.accountSummary.value.error
+ Number(status === 'error')
- Number(current.status === 'error'),
),
}
}
function isUnavailableAccountStatus(status: string) {
return status === 'expired'
|| status === 'invalid'
|| status === 'disabled'
|| status === 'banned'
}
function sortAccountsByStatus(rows: AccountRow[]) {
if (options.sort.value?.key !== 'status')
return rows
@@ -14,10 +14,11 @@ export function useAccountsQuery() {
const sort = shallowRef<BaseTableSort>()
const accountSummary = shallowRef({
total: 0,
active: 0,
normal: 0,
quotaExhausted: 0,
rateLimited: 0,
unavailable: 0,
disabled: 0,
error: 0,
})
const query = usePagedQuery({
+18 -15
View File
@@ -123,38 +123,41 @@ export const accountColumns = [
] satisfies BaseTableColumn<AccountRow>[]
export const statusLabels: Record<string, string> = {
active: '正常',
expired: '已过期',
invalid: '失效',
disabled: '已禁用',
banned: '已封禁',
normal: '正常',
quota_exhausted: '配额耗尽',
rate_limited: '限流中',
disabled: '已停用',
error: '错误',
}
export const statusTones: Record<string, 'success' | 'danger' | 'warning' | 'info' | 'normal'> = {
active: 'success',
expired: 'warning',
invalid: 'warning',
disabled: 'normal',
banned: 'danger',
normal: 'success',
quota_exhausted: 'warning',
rate_limited: 'warning',
disabled: 'normal',
error: 'danger',
}
export const accountStatusFilterOptions = [
{ label: '全部状态', value: '' },
{ label: statusLabels.active, value: 'active' },
{ label: statusLabels.normal, value: 'normal' },
{ label: statusLabels.quota_exhausted, value: 'quota_exhausted' },
{ label: statusLabels.rate_limited, value: 'rate_limited' },
{ label: statusLabels.expired, value: 'expired' },
{ label: statusLabels.invalid, value: 'invalid' },
{ label: statusLabels.disabled, value: 'disabled' },
{ label: statusLabels.banned, value: 'banned' },
{ label: statusLabels.error, value: 'error' },
]
/** `error` 分类下具体原因的展示文案(对应后端 `errorReason`)。 */
export const errorReasonLabels: Record<string, string> = {
expired: '账号已过期',
token_expired: '访问令牌已过期',
invalid: '凭据已失效',
banned: '账号已封禁',
unknown: '状态未知',
}
/**
* 后端已派生互斥状态(含 `rate_limited`,由快照级 quota.limitReached 产生);
* 后端已派生互斥状态(正常 / 配额耗尽 / 限流中 / 已停用 / 错误);
* 前端只渲染,不再独立派生。
*/
export function derivedAccountStatus(row: AccountRow): string {
+5 -1
View File
@@ -202,7 +202,11 @@ const {
</template>
<template #status="{ row }">
<AccountStatusBadge :status="derivedAccountStatus(row)" />
<AccountStatusBadge
:status="derivedAccountStatus(row)"
:error-reason="row.errorReason"
:error-message="row.errorMessage"
/>
</template>
<template #planType="{ row }">