Compare commits

...
13 Commits
Author SHA1 Message Date
ZyyandCodex 19c35ba901 chore: bump version to 3.1.6
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 15:12:29 +08:00
ZyyandCodex 3c06ae97e8 fix: reconcile quota, backup, and session behavior
- preserve confirmed quota exhaustion until its recorded reset advances
- normalize account quota presentation and subagent session affinity
- repair S3-compatible backup handling and related admin UI details

Co-authored-by: Codex <noreply@openai.com>
2026-08-04 15:12:15 +08:00
ZyyandCodex f34966e4c7 chore: bump version to 3.1.5
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 08:54:00 +08:00
ZyyandCodex e9ebf45640 fix(accounts): reconcile quota status and dashboard metrics
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 08:53:29 +08:00
ZyyandCodex c00e5bd243 chore: bump version to 3.1.4
Co-authored-by: Codex <noreply@openai.com>
2026-08-04 01:10:51 +08:00
ZyyandCodex ab5f18a437 fix(accounts): update admin list test for cooldown-driven rate_limited
quota_limit_reached without Redis cooldown maps to quota_exhausted
now that rate_limited comes only from 429 cooldown. Adjust summary
assertions: quota_exhausted 1→2, rate_limited 1→0.

Co-authored-by: Codex <noreply@openai.com>
2026-08-04 01:10:40 +08:00
ZyyandClaude Fable 5 330b222359 feat(accounts): separate rate-limited status into Redis cooldown
429 临时限流从 quota 快照分离为 Redis 冷却(ProviderCooldownPort):
- apply_rate_limit_429 只写 Redis(put_if_later,不缩短已有冷却),不再合成 quota 窗口,额度重置时间不被 429 污染
- 调度选择时排除冷却中的账号,到期自动恢复;诊断请求绕过
- 状态派生分离:rate_limited_until 存在 → 限流中;quota_limit_reached → 配额耗尽
- store 层(账号列表/筛选 + dashboard poolSummary)注入 cooldown 端口,精确区分限流中与配额耗尽,"正常"筛选不再混入限流中
- wire 透传 rate_limited_until,前端状态弹窗改为状态卡片(icon 块 + 状态名 + 原因/到期明细),BasePopover 加 hover delay
- 更新 429 相关测试断言,新增配额耗尽映射测试

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-04 00:39:38 +08:00
ZyyandCodex d81cb2d4f0 chore: bump version to 3.1.3
Co-authored-by: Codex <noreply@openai.com>
2026-08-03 21:29:16 +08:00
ZyyandClaude Opus 4.8 8e2edb680d feat(accounts): converge status classification and persist upstream errors
Unify account status derivation in gateway-admin via AccountStatusSignals,
removing the duplicated rules in store and use_case. AccountStatus now has
five operational classes (normal/quota_exhausted/rate_limited/disabled/error),
with AccountErrorReason carrying the concrete cause.

Add provider_accounts.last_error_message, written by record_failure with the
raw upstream error and cleared on recovery to ready. The admin error badge
shows the cause and upstream text on hover.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-08-03 21:23:28 +08:00
ZyyandCodex 00879be47d fix(accounts): separate rate-limited account status
Co-authored-by: Codex <noreply@openai.com>
2026-08-03 17:01:23 +08:00
ZyyandCodex 34656a3f07 refactor(ui): standardize card spacing and layouts
Co-authored-by: Codex <noreply@openai.com>
2026-08-03 17:01:23 +08:00
ZyyandCodex 27cf037309 fix(ui): center button loading spinner
Co-authored-by: Codex <noreply@openai.com>
2026-08-03 17:01:23 +08:00
ZyyandCodex 2198fa69ab refactor(migrations): consolidate schema into initial baseline
Co-authored-by: Codex <noreply@openai.com>
2026-08-03 17:01:23 +08:00
99 changed files with 3079 additions and 904 deletions
+26 -10
View File
@@ -220,7 +220,8 @@ impl BackupTask {
sha256: Some(artifact.sha256.clone()),
..Default::default()
};
self.repository
let Some(uploading_record) = self
.repository
.transition_status(
&record.id,
BackupStatus::Dumping,
@@ -229,9 +230,12 @@ impl BackupTask {
now,
)
.await
.map_err(repo_error)?;
.map_err(repo_error)?
else {
return Ok(());
};
self.upload_and_verify(
record,
&uploading_record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
@@ -394,7 +398,8 @@ impl BackupTask {
sha256: Some(artifact.sha256.clone()),
..Default::default()
};
self.repository
let Some(uploading_record) = self
.repository
.transition_status(
&record.id,
BackupStatus::Dumping,
@@ -403,10 +408,13 @@ impl BackupTask {
now,
)
.await
.map_err(repo_error)?;
.map_err(repo_error)?
else {
return Ok(());
};
self.upload_and_verify(
record,
&uploading_record,
&artifact.path,
artifact.size_bytes,
&artifact.sha256,
@@ -448,8 +456,12 @@ impl BackupTask {
.await
{
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(record, error.code(), "上传到对象存储失败")
.await?;
self.fail_task(
record,
error.code(),
format!("上传到对象存储失败:{}", error.message()),
)
.await?;
return Ok(());
}
@@ -486,8 +498,12 @@ impl BackupTask {
}
Err(error) => {
let _ = store.delete_object(&config, &record.object_key).await;
self.fail_task(record, error.code(), "远端对象校验失败")
.await?;
self.fail_task(
record,
error.code(),
format!("远端对象校验失败:{}", error.message()),
)
.await?;
}
}
Ok(())
@@ -1,6 +1,6 @@
//! 多 Provider 账号目录与连接测试的公共事实。
use std::pin::Pin;
use std::{pin::Pin, time::SystemTime};
use chrono::{DateTime, Utc};
use futures::Stream;
@@ -13,15 +13,146 @@ use super::{PageSize, Revision, observability::TimeRange};
pub use gateway_core::engine::credential::AccountAvailability;
/// 管理页使用的归一化账号状态。
///
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountStatus {
Active,
RateLimited,
/// 正常,可调度。
Normal,
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
QuotaExhausted,
Expired,
Invalid,
/// 限流中(quota 快照级信号),自动恢复,无需干预。
RateLimited,
/// 管理员手动停用(`enabled = false`)。
Disabled,
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
Error,
}
impl AccountStatus {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Normal => "normal",
Self::QuotaExhausted => "quota_exhausted",
Self::RateLimited => "rate_limited",
Self::Disabled => "disabled",
Self::Error => "error",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"normal" => Some(Self::Normal),
"quota_exhausted" => Some(Self::QuotaExhausted),
"rate_limited" => Some(Self::RateLimited),
"disabled" => Some(Self::Disabled),
"error" => Some(Self::Error),
_ => None,
}
}
}
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusSignals {
pub enabled: bool,
pub availability: AccountAvailability,
pub access_token_expired: bool,
pub quota_limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
pub rate_limited_until: Option<SystemTime>,
}
impl AccountStatusSignals {
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
#[must_use]
pub const fn derive(self) -> AccountStatus {
if !self.enabled {
AccountStatus::Disabled
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
AccountStatus::QuotaExhausted
} else if self.access_token_expired
|| matches!(
self.availability,
AccountAvailability::Expired
| AccountAvailability::Invalid
| AccountAvailability::Banned
| AccountAvailability::Unknown
)
{
AccountStatus::Error
} else if self.rate_limited_until.is_some() {
AccountStatus::RateLimited
} else if self.quota_limit_reached {
AccountStatus::QuotaExhausted
} else {
AccountStatus::Normal
}
}
}
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
///
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AccountErrorReason {
Expired,
TokenExpired,
Invalid,
Banned,
Unknown,
}
impl AccountErrorReason {
/// 返回稳定 wire 值。
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Expired => "expired",
Self::TokenExpired => "token_expired",
Self::Invalid => "invalid",
Self::Banned => "banned",
Self::Unknown => "unknown",
}
}
/// 解析稳定 wire 值。
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"expired" => Some(Self::Expired),
"token_expired" => Some(Self::TokenExpired),
"invalid" => Some(Self::Invalid),
"banned" => Some(Self::Banned),
"unknown" => Some(Self::Unknown),
_ => None,
}
}
}
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
///
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
#[must_use]
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
if !signals.enabled {
return None;
}
match signals.availability {
AccountAvailability::QuotaExhausted => None,
AccountAvailability::Ready => signals
.access_token_expired
.then_some(AccountErrorReason::TokenExpired),
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
}
}
/// 账号列表排序字段。
@@ -78,6 +209,8 @@ pub struct AccountRecord {
pub enabled: bool,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub last_error_message: Option<String>,
pub quota_observed_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
@@ -170,13 +303,15 @@ pub struct AccountPage {
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
///
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountSummary {
pub total: u64,
pub active: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub unavailable: u64,
pub rate_limited: u64,
pub disabled: u64,
pub error: u64,
}
/// 账号启停写入命令。
@@ -11,7 +11,10 @@ use gateway_core::{
use super::{
AdminError, MutationActor, MutationContext, PageSize, Revision,
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
accounts::{
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
AccountUsage,
},
};
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
@@ -592,6 +595,9 @@ pub struct ProviderQuota {
pub windows: Vec<ProviderQuotaWindow>,
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
pub limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
pub rate_limited_until: Option<DateTime<Utc>>,
pub provider_data: Option<ProviderDocument>,
}
@@ -688,6 +694,8 @@ impl fmt::Debug for ProviderExport {
pub struct AccountDirectoryItem {
pub account: AccountRecord,
pub status: AccountStatus,
/// `status == Error` 时的具体原因;其余状态为 `None`。
pub error_reason: Option<AccountErrorReason>,
pub usage: Option<super::accounts::AccountUsage>,
pub quota: ProviderQuota,
}
@@ -1,6 +1,6 @@
//! 统一账号目录与跨 Provider 动态分派。
use std::{collections::BTreeMap, sync::Arc};
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
use async_trait::async_trait;
use chrono::{Duration, Utc};
@@ -18,7 +18,8 @@ use crate::{
AdminError, MutationContext,
accounts::{
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
AccountRecord, AccountStatus, AccountUsageWindowQuery,
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
derive_error_reason,
},
observability::TimeRange,
provider_credentials::{
@@ -216,8 +217,11 @@ impl DefaultAccountsService {
std::slice::from_mut(&mut quota),
)
.await?;
let signals = account_signals(&account, &quota, now);
let error_reason = derive_error_reason(&signals);
Ok(AccountDirectoryItem {
status: account_status(&account, quota.limit_reached, now),
status: signals.derive(),
error_reason,
usage: usage.into_iter().next(),
account,
quota,
@@ -301,11 +305,16 @@ impl AccountsService for DefaultAccountsService {
.items
.into_iter()
.zip(quotas)
.map(|(account, quota)| AccountDirectoryItem {
status: account_status(&account, quota.limit_reached, now),
usage: usage.get(&account.id).cloned(),
account,
quota,
.map(|(account, quota)| {
let signals = account_signals(&account, &quota, now);
let error_reason = derive_error_reason(&signals);
AccountDirectoryItem {
status: signals.derive(),
error_reason,
usage: usage.get(&account.id).cloned(),
account,
quota,
}
})
.collect();
Ok(AccountDirectoryPage {
@@ -512,43 +521,39 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
/// 互斥页面状态:availability 终态优先,quota 快照级 `limit_reached` 派生
/// `rate_limited`,`Unknown` 保守映射 `invalid`(不可调度,不显示为 active)。
/// 构造状态派生所需的信号快照。
fn account_signals(
account: &AccountRecord,
quota: &ProviderQuota,
now: chrono::DateTime<Utc>,
) -> AccountStatusSignals {
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached: quota.limit_reached,
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
}
}
/// 账号运营分类(互斥展示状态)。
fn account_status(
account: &AccountRecord,
quota_limit_reached: bool,
now: chrono::DateTime<Utc>,
) -> AccountStatus {
if !account.enabled {
AccountStatus::Disabled
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
AccountStatus::Banned
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted {
AccountStatus::QuotaExhausted
} else if matches!(
account.availability,
crate::model::accounts::AccountAvailability::Invalid
) {
AccountStatus::Invalid
} else if matches!(
account.availability,
crate::model::accounts::AccountAvailability::Expired
) || account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now)
{
AccountStatus::Expired
} else if quota_limit_reached {
AccountStatus::RateLimited
} else if account.availability == crate::model::accounts::AccountAvailability::Unknown {
AccountStatus::Invalid
} else {
AccountStatus::Active
}
let quota = ProviderQuota {
limit_reached: quota_limit_reached,
..empty_quota()
};
account_signals(account, &quota, now).derive()
}
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
@@ -13,9 +13,10 @@ use gateway_admin::{
MutationActor, MutationContext, Revision,
auth::AdminAuditEvent,
backup::{
BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind,
ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
UpdateBackupStorageCommand,
},
},
ports::{
@@ -497,15 +498,23 @@ impl DatabaseDumpPort for FakeDumpPort {
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
pub(crate) struct FakeObjectStore {
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
upload_error: Option<BackupError>,
}
impl FakeObjectStore {
pub(crate) fn new() -> Self {
Self {
objects: Mutex::new(std::collections::HashMap::new()),
upload_error: None,
}
}
/// 让上传返回指定的脱敏 S3 错误。
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
self.upload_error = Some(BackupError::new(code, message.into()));
self
}
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
self.objects.lock().expect("objects").get(key).cloned()
}
@@ -530,6 +539,9 @@ impl BackupObjectStorePort for FakeObjectStore {
_config: &BackupStorageConfig,
request: UploadObjectRequest,
) -> Result<(), gateway_admin::model::backup::BackupError> {
if let Some(error) = &self.upload_error {
return Err(error.clone());
}
let content = std::fs::read(&request.source).map_err(|_| {
gateway_admin::model::backup::BackupError::new(
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
@@ -6,9 +6,9 @@ use chrono::{Duration, Utc};
use gateway_admin::{
backup::task::BackupTask,
model::backup::{
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand,
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
},
ports::backup::BackupRepository,
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
};
use gateway_core::engine::CancellationToken;
@@ -89,6 +89,78 @@ async fn backup_task_marks_failed_when_dump_port_fails() {
assert!(object_store.object(&seed.object_key).is_none());
}
#[tokio::test]
async fn backup_task_marks_failed_with_the_upload_error() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store =
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
let seed = BackupRecordSeed {
id: backup_id("upload-failure"),
trigger_kind: BackupTriggerKind::Manual,
scheduled_at: None,
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
expires_at: None,
};
repository
.insert_backup_record(seed.clone())
.await
.expect("insert queued");
let task = BackupTask::new(repository.clone(), dump, object_store);
task.run_cycle(&CancellationToken::new())
.await
.expect("run cycle");
let records = repository.all_records();
let failed = &records[0];
assert_eq!(failed.status, BackupStatus::Failed);
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
assert_eq!(
failed.error_message.as_deref(),
Some("上传到对象存储失败:对象存储拒绝上传")
);
}
#[tokio::test]
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
let dump = Arc::new(FakeDumpPort::new());
let object_store =
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
let seed = BackupRecordSeed {
id: backup_id("recovered-upload-failure"),
trigger_kind: BackupTriggerKind::Manual,
scheduled_at: None,
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
expires_at: None,
};
repository
.insert_backup_record(seed.clone())
.await
.expect("insert queued");
let cancellation = CancellationToken::new();
dump.dump(DumpRequest {
backup_id: seed.id.clone(),
cancellation: cancellation.clone(),
})
.await
.expect("stage dump");
repository.force_status(&seed.id, BackupStatus::Dumping);
let task = BackupTask::new(repository.clone(), dump, object_store);
task.run_cycle(&cancellation).await.expect("run cycle");
let records = repository.all_records();
let failed = &records[0];
assert_eq!(failed.status, BackupStatus::Failed);
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
assert_eq!(
failed.error_message.as_deref(),
Some("上传到对象存储失败:对象存储拒绝上传")
);
}
#[tokio::test]
async fn retention_cleans_expired_scheduled_backups() {
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
@@ -128,6 +128,7 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
quota_window("shortTerm", Some(14_400), Some(45.0)),
],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
};
@@ -451,9 +451,11 @@ impl AccountStore for FakeAccountStore {
total: accounts.len() as u64,
summary: AccountSummary {
total: accounts.len() as u64,
active: 1,
normal: 1,
quota_exhausted: 0,
unavailable: 0,
rate_limited: 0,
disabled: 0,
error: 0,
},
})
}
@@ -844,7 +846,7 @@ async fn connection_test_should_probe_unavailable_account() {
}
#[tokio::test]
async fn connection_test_rate_limited_probe_preserves_active_account_status() {
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
let events = events();
let provider = FakeProviderAdmin::new("xai", events.clone());
let store = FakeAccountStore::new("xai", events);
@@ -867,7 +869,7 @@ async fn connection_test_rate_limited_probe_preserves_active_account_status() {
assert!(matches!(
events.last(),
Some(AccountConnectionTestEvent::Failed {
account_status: gateway_admin::model::accounts::AccountStatus::Active,
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
..
})
));
@@ -948,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
assert_eq!(
result.account.status,
gateway_admin::model::accounts::AccountStatus::Active
gateway_admin::model::accounts::AccountStatus::Normal
);
assert_eq!(
@@ -984,12 +986,12 @@ async fn accounts_list_should_return_complete_directory_semantics() {
.expect("complete account directory");
assert_eq!(page.summary.total, 1);
assert_eq!(page.summary.active, 1);
assert_eq!(page.summary.normal, 1);
let account = page.items.first().expect("account item");
assert_eq!(account.account.provider_kind.as_str(), "openai");
assert_eq!(
account.status,
gateway_admin::model::accounts::AccountStatus::Active
gateway_admin::model::accounts::AccountStatus::Normal
);
}
@@ -1013,6 +1015,7 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
provider_data: None,
}],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let failing = FakeProviderAdmin::new("xai", events.clone());
@@ -1088,8 +1091,8 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
}
#[tokio::test]
async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
// Unknown 不可调度,Admin 不得显示为 active。
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
// Unknown 不可调度,Admin 不得显示为 normal。
let provider = FakeProviderAdmin::new("openai", events());
let mut account = account_record("openai");
account.availability = AccountAvailability::Unknown;
@@ -1111,13 +1114,14 @@ async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::Invalid,
gateway_admin::model::accounts::AccountStatus::Error,
);
}
#[tokio::test]
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
// ready + 快照级 quota.limitReached → rate_limited。
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
@@ -1135,6 +1139,7 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
provider_data: None,
}],
limit_reached: true,
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1159,6 +1164,51 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
);
}
#[tokio::test]
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
observed_at: Some(Utc::now()),
refresh_token_expires_at: None,
windows: vec![ProviderQuotaWindow {
key: "primary".to_owned(),
group: "shortTerm".to_owned(),
label: "5小时限额".to_owned(),
source: None,
window_seconds: Some(5 * 60 * 60),
used_percent: Some(100.0),
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
limit_reached: true,
local_usage: None,
provider_data: None,
}],
limit_reached: true,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
let page = accounts_service(provider, store)
.await
.accounts()
.list(AccountListQuery {
page: 1,
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
provider_kind: None,
search: None,
status: None,
sort: None,
})
.await
.expect("quota exhausted account directory");
assert_eq!(
page.items.first().expect("account item").status,
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
);
}
#[tokio::test]
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
let provider = FakeProviderAdmin::new("openai", events());
@@ -1179,6 +1229,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
provider_data: None,
}],
limit_reached: false,
rate_limited_until: None,
provider_data: None,
});
let store = FakeAccountStore::new("openai", events());
@@ -1287,6 +1338,7 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
@@ -1332,6 +1384,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
enabled: true,
availability: AccountAvailability::Ready,
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
@@ -91,7 +91,8 @@ impl ListQuery {
let status = match self.status.as_deref().map(str::trim) {
None | Some("") => None,
Some(value) => Some(
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
DomainAccountStatus::parse(&value.to_ascii_lowercase())
.ok_or_else(|| WireValidationError::new("status"))?,
),
};
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
@@ -127,17 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
.map_err(|_| WireValidationError::new("provider"))
}
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
match value.trim().to_ascii_lowercase().as_str() {
"active" => Some(DomainAccountStatus::Active),
"expired" => Some(DomainAccountStatus::Expired),
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
"disabled" => Some(DomainAccountStatus::Disabled),
"banned" => Some(DomainAccountStatus::Banned),
_ => None,
}
}
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
match value.trim() {
"email" => Some(AccountSortField::Email),
@@ -172,9 +162,11 @@ pub struct AccountPageData {
#[serde(rename_all = "camelCase")]
pub struct AccountSummaryView {
pub total: u64,
pub active: u64,
pub normal: u64,
pub quota_exhausted: u64,
pub unavailable: u64,
pub rate_limited: u64,
pub disabled: u64,
pub error: u64,
}
/// 一条安全账号视图。
@@ -194,6 +186,10 @@ pub struct AccountView {
pub has_refresh_token: bool,
pub status: String,
pub display_status: String,
/// `status == "error"` 时的具体原因;其余状态为 `null`。
pub error_reason: Option<String>,
/// 最近一次失败的上游错误描述;仅错误状态存在。
pub error_message: Option<String>,
pub availability: String,
pub enabled: bool,
pub access_token_expires_at: Option<String>,
@@ -214,6 +210,8 @@ pub struct AccountView {
pub struct AccountQuotaView {
pub refreshed_at_display: String,
pub limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
pub rate_limited_until: Option<String>,
pub windows: Vec<AccountQuotaWindowView>,
}
@@ -527,7 +525,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
"type": "test_complete",
"success": true,
"accountStatus": account_status_name(account_status)
"accountStatus": account_status.as_str()
}),
DomainConnectionTestEvent::Failed {
message,
@@ -545,7 +543,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
"upstreamStatus": upstream_status,
"upstreamContentType": upstream_content_type,
"upstreamBody": upstream_body,
"accountStatus": account_status_name(account_status)
"accountStatus": account_status.as_str()
}),
};
Self { data }
@@ -1333,9 +1331,11 @@ fn account_page_data(
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
summary: AccountSummaryView {
total: result.summary.total,
active: result.summary.active,
normal: result.summary.normal,
quota_exhausted: result.summary.quota_exhausted,
unavailable: result.summary.unavailable,
rate_limited: result.summary.rate_limited,
disabled: result.summary.disabled,
error: result.summary.error,
},
}
}
@@ -1366,10 +1366,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
let AccountDirectoryItem {
account,
status,
error_reason,
usage,
quota,
} = item;
let status = account_status_name(status).to_owned();
let status = status.as_str().to_owned();
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
let added_at = china_rfc3339(&account.created_at);
let updated_at = china_rfc3339(&account.updated_at);
@@ -1388,6 +1389,8 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
has_refresh_token: account.has_refresh_token,
status: status.clone(),
display_status: status,
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
error_message: account.last_error_message,
availability: account.availability.as_str().to_owned(),
enabled: account.enabled,
access_token_expires_at: expires_at,
@@ -1417,10 +1420,12 @@ fn account_quota_view(
.observed_at
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
let windows = quota.windows.into_iter().map(quota_window_view).collect();
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
(
AccountQuotaView {
refreshed_at_display,
limit_reached: quota.limit_reached,
rate_limited_until,
windows,
},
refresh_token_expires_at,
@@ -1713,18 +1718,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
Value::Object(document.into_provider_data().into_inner())
}
fn account_status_name(status: DomainAccountStatus) -> &'static str {
match status {
DomainAccountStatus::Active => "active",
DomainAccountStatus::RateLimited => "rate_limited",
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
DomainAccountStatus::Expired => "expired",
DomainAccountStatus::Invalid => "invalid",
DomainAccountStatus::Disabled => "disabled",
DomainAccountStatus::Banned => "banned",
}
}
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
let elapsed = now.signed_duration_since(value);
if elapsed.num_seconds() < 0 {
@@ -10,7 +10,7 @@ mod query {
"pageSize": 20,
"provider": "xai",
"search": " operator ",
"status": "active",
"status": "normal",
"sortBy": "lastUsedAt",
"sortDirection": "desc"
}))
@@ -23,7 +23,7 @@ mod query {
Some(ref provider) if provider.as_str() == "xai"
));
assert_eq!(query.search.as_deref(), Some("operator"));
assert_eq!(query.status, Some(AccountStatus::Active));
assert_eq!(query.status, Some(AccountStatus::Normal));
assert_eq!(
query.sort.expect("sort").field,
AccountSortField::LastUsedAt
@@ -34,6 +34,19 @@ mod query {
);
}
#[test]
fn account_query_should_parse_rate_limited_status() {
let query: ListQuery = serde_json::from_value(json!({
"status": "rate_limited"
}))
.expect("deserialize account query");
assert_eq!(
query.validate().expect("validate account query").status,
Some(AccountStatus::RateLimited)
);
}
#[test]
fn account_query_should_reject_unbounded_page_size() {
let query: ListQuery =
@@ -331,7 +344,7 @@ mod actions {
text: "OK".to_owned(),
},
DomainConnectionTestEvent::Completed {
account_status: AccountStatus::Active,
account_status: AccountStatus::Normal,
},
DomainConnectionTestEvent::Failed {
message: "upstream unavailable".to_owned(),
@@ -365,7 +378,7 @@ mod actions {
}
}),
json!({ "type": "content", "text": "OK" }),
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
json!({
"type": "error",
"error": "upstream unavailable",
@@ -660,6 +660,9 @@ where
ProviderBoundary::Result(result) => match *result {
Ok(stream) => stream,
Err(error) => {
if self.prepare_unavailable_native_continuation_replay(&error) {
return Ok(Some(PullOutcome::AttemptDiscarded));
}
if matches!(
error.kind(),
ProviderErrorKind::AccountCapacityUnavailable
@@ -1096,6 +1099,41 @@ where
true
}
/// 原生续写的原账号在本地调度阶段已不可用时,改用完整输入重放。
///
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
/// 对应 Provider 的可携带会话状态时,才可安全移除 native handle 并换号。
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
if self.account_selection.required_account().is_some()
|| self.continuation_attempt != ContinuationAttempt::Native
|| !matches!(
error.kind(),
ProviderErrorKind::NoEligibleAccount
| ProviderErrorKind::AccountCapacityUnavailable
)
{
return false;
}
let Some(pin) = self
.continuation
.as_ref()
.and_then(ContinuationBinding::pinned)
else {
return false;
};
if self
.operation
.provider_session_state(pin.provider().as_str())
.is_none()
{
return false;
}
self.continuation_attempt = ContinuationAttempt::ReplayAny;
self.excluded_accounts.insert(pin.account().clone());
true
}
fn reset_uncommitted_observations(&mut self) {
self.usage = Usage::new();
self.cost = CostEstimate::unavailable();
@@ -607,6 +607,8 @@ pub struct AccountStateChange {
pub expected_revision: CredentialRevision,
pub availability: AccountAvailability,
pub observed_at: SystemTime,
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
pub message: Option<String>,
}
/// `provider_accounts` 的数据库中立端口。
@@ -513,17 +513,27 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
}
fn plan(operation: &Operation) -> RoutingPlan {
plan_with_policy(
operation,
AccountSelectionPolicy::new(
RotationStrategy::Smart,
NonZeroU32::new(2).expect("account concurrency"),
Duration::from_millis(50),
),
)
}
fn plan_with_policy(
operation: &Operation,
account_selection_policy: AccountSelectionPolicy,
) -> RoutingPlan {
let provider = ProviderKind::new("openai").expect("provider");
let public_model = PublicModelId::new("gpt-5").expect("public model");
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
.with_upstream_feature_validation();
RuntimeSnapshot::new(
ConfigRevision::new(1).expect("config revision"),
AccountSelectionPolicy::new(
RotationStrategy::Smart,
NonZeroU32::new(2).expect("account concurrency"),
Duration::from_millis(50),
),
account_selection_policy,
vec![provider.clone()],
vec![ProviderModel::new(
provider.clone(),
@@ -1514,6 +1524,70 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
);
}
#[test]
fn unavailable_native_continuation_replays_with_the_configured_account_policy() {
let Operation::Generate(generate) = generate_operation() else {
panic!("generate operation");
};
let operation = Operation::Generate(generate.with_provider_session_state(
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
));
let policy = AccountSelectionPolicy::new(
RotationStrategy::RoundRobin,
NonZeroU32::new(2).expect("account concurrency"),
Duration::from_millis(50),
);
let route_plan = plan_with_policy(&operation, policy);
let (coordinator, store, provider) = coordinator(vec![
Script::Error(ProviderError::new(
ProviderErrorKind::NoEligibleAccount,
UpstreamSendState::NotSent,
)),
Script::Stream {
account_id: "acct_two",
items: complete_stream(None),
},
]);
let original = ProviderAccountId::new("acct_one").expect("account");
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-response"),
PreviousResponseId::new("upstream-response"),
ProviderKind::new("openai").expect("provider"),
original.clone(),
);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
Some(ContinuationBinding::Pinned(continuation)),
CancellationToken::new(),
))
.expect("start execution");
block_on(session.collect_uncommitted()).expect("portable replay succeeds");
block_on(session.commit_downstream(Some(200))).expect("commit response");
let contexts = provider.contexts.lock().expect("contexts lock");
assert_eq!(contexts.len(), 2);
assert_eq!(
contexts[0].continuation_attempt(),
ContinuationAttempt::Native
);
assert_eq!(
contexts[1].continuation_attempt(),
ContinuationAttempt::ReplayAny
);
assert_eq!(
contexts[1].account_selection_policy().strategy(),
RotationStrategy::RoundRobin
);
assert!(contexts[1].excluded_accounts().contains(&original));
let state = store.state.lock().expect("store lock");
assert_eq!(state.attempts.len(), 1);
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
}
#[test]
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
let operation = operation(RetrySafety::Idempotent);
+6 -1
View File
@@ -1,6 +1,7 @@
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
use gateway_core::engine::credential::ProviderAccountStore;
use gateway_core::provider_ports::ProviderCooldownPort;
use super::*;
@@ -85,7 +86,10 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
)?);
let admin_ports = AdminStorePorts::new(
Arc::new(postgres::PgAdminAccountStore::new(pool.clone())),
Arc::new(postgres::PgAdminAccountStore::new(
pool.clone(),
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
)),
Arc::new(AdminAuthStoreAdapter {
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
@@ -98,6 +102,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
Arc::new(postgres::PgAdminObservabilityStore::new(
pool.clone(),
Some(credential_leases.clone()),
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
)),
Arc::new(AdminSettingsStoreAdapter {
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
@@ -1,5 +1,10 @@
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
use std::sync::Arc;
use std::time::SystemTime;
use gateway_core::provider_ports::ProviderCooldownPort;
use super::*;
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
@@ -7,12 +12,82 @@ use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepositor
#[derive(Clone)]
pub struct PgObservabilityRepository {
pool: PgPool,
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
}
impl PgObservabilityRepository {
#[must_use]
pub const fn new(pool: PgPool) -> Self {
Self { pool }
pub const fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
Self { pool, cooldowns }
}
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
async fn provider_account_metrics_with_cooldowns(
&self,
observed_at: DateTime<Utc>,
) -> StoreResult<ProviderAccountMetrics> {
let base = provider_account_metrics(&self.pool, observed_at).await?;
let Some(cooldowns) = &self.cooldowns else {
return Ok(base);
};
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
if candidates.is_empty() {
return Ok(base);
}
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
let mut active = base.active;
let mut rate_limited = base.rate_limited;
let mut quota_exhausted = base.quota_exhausted;
let now = SystemTime::now();
for (account_id, revision, quota_reached) in candidates {
let Ok(account_id) =
gateway_core::engine::credential::ProviderAccountId::new(account_id)
else {
continue;
};
let cooling = cooldowns
.read(&account_id)
.await
.ok()
.flatten()
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
.filter(|cooldown| cooldown.until() > now)
.is_some();
if !cooling {
continue;
}
if quota_reached {
if quota_exhausted == 0 {
continue;
}
quota_exhausted -= 1;
} else {
if active == 0 {
continue;
}
active -= 1;
}
rate_limited += 1;
}
Ok(ProviderAccountMetrics {
total: base.total,
enabled: base.enabled,
unavailable: rate_limited
+ quota_exhausted
+ base.expired
+ base.invalid
+ base.disabled
+ base.banned,
active,
rate_limited,
expired: base.expired,
invalid: base.invalid,
quota_exhausted,
disabled: base.disabled,
banned: base.banned,
})
}
}
@@ -28,9 +103,13 @@ pub struct PgAdminObservabilityStore {
impl PgAdminObservabilityStore {
#[must_use]
pub fn new(pool: PgPool, runtime_signals: Option<RedisCredentialLeaseRepository>) -> Self {
pub fn new(
pool: PgPool,
runtime_signals: Option<RedisCredentialLeaseRepository>,
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
) -> Self {
Self {
repository: PgObservabilityRepository::new(pool),
repository: PgObservabilityRepository::new(pool, cooldowns),
runtime_signals,
}
}
@@ -62,11 +141,13 @@ impl ObservabilityRepository for PgObservabilityRepository {
};
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
let (requests, attempts, provider_accounts) = futures::try_join!(
let (requests, attempts) = futures::try_join!(
request_metrics(&self.pool, range, &filter),
attempt_metrics(&self.pool, range, &filter),
provider_account_metrics(&self.pool, range.end),
)?;
let provider_accounts = self
.provider_account_metrics_with_cooldowns(range.end)
.await?;
let (trend, account_usage, recent_requests) = futures::try_join!(
request_metric_series(&self.pool, range, &filter),
provider_account_usage(&self.pool, account_usage_query),
@@ -357,8 +357,8 @@ pub(crate) async fn provider_account_metrics(
when availability = 'expired'
or (access_token_expires_at is not null and access_token_expires_at <= $1)
then 'expired'
when quota_limit_reached then 'rate_limited'
when availability = 'unknown' then 'invalid'
when quota_limit_reached then 'quota_exhausted'
else 'active'
end as status
from provider_accounts
@@ -425,6 +425,35 @@ pub(crate) async fn active_provider_account_ids(
.collect()
}
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
pub(crate) async fn schedulable_metric_candidates(
pool: &PgPool,
observed_at: DateTime<Utc>,
) -> StoreResult<Vec<(String, i64, bool)>> {
sqlx::query(
"select id, credential_revision, quota_limit_reached
from provider_accounts
where enabled
and availability = 'ready'
and (access_token_expires_at is null or access_token_expires_at > $1)
order by id",
)
.bind(observed_at)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
.into_iter()
.map(|row| {
Ok((
get(&row, "id")?,
get(&row, "credential_revision")?,
get(&row, "quota_limit_reached")?,
))
})
.collect::<StoreResult<Vec<_>>>()
}
pub(crate) async fn provider_observations(
pool: &PgPool,
range: ObservabilityRange,
@@ -1,5 +1,11 @@
//! `gateway-admin` 账号端口的 PostgreSQL adapter。
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::SystemTime;
use gateway_core::provider_ports::ProviderCooldownPort;
use super::*;
/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。
@@ -11,19 +17,49 @@ pub struct PgAdminAccountStore {
accounts: PgProviderAccountRepository,
observability: PgObservabilityRepository,
control_plane: PgControlPlaneRepository,
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
}
impl PgAdminAccountStore {
#[must_use]
pub fn new(pool: PgPool) -> Self {
pub fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
Self {
pool: pool.clone(),
accounts: PgProviderAccountRepository::new(pool.clone()),
observability: PgObservabilityRepository::new(pool.clone()),
observability: PgObservabilityRepository::new(pool.clone(), cooldowns.clone()),
control_plane: PgControlPlaneRepository::new(pool),
cooldowns,
}
}
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
async fn load_rate_limited_until(
&self,
accounts: &[ProviderAccountSummary],
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
let Some(cooldowns) = &self.cooldowns else {
return Ok(BTreeMap::new());
};
let mut map = BTreeMap::new();
for account in accounts {
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
continue;
};
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
continue;
};
if cooldown.credential_revision().get() != account.credential_revision.get() {
continue;
}
let until = cooldown.until();
if until > SystemTime::now() {
map.insert(account.id.clone(), until);
}
}
Ok(map)
}
async fn usage_observations(
&self,
range: ObservabilityRange,
@@ -242,11 +278,17 @@ impl AccountStore for PgAdminAccountStore {
)
.map_err(|error| admin_store_error(ENTITY, error))?;
let now = Utc::now();
let summary = admin_account_summary(&accounts, now);
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
let rate_limited_until = self
.load_rate_limited_until(&accounts)
.await
.unwrap_or_default();
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
let mut items = accounts
.into_iter()
.filter_map(|account| {
let status = admin_account_status(&account, now);
let until = rate_limited_until.get(&account.id).copied();
let status = admin_account_status(&account, now, until);
account_matches_admin_query(&account, status, &query).then_some(
AdminAccountListItem {
account,
@@ -247,6 +247,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.map_err(core_store_error)?,
availability: change.availability,
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
message: change.message,
})
.await
.map_err(core_store_error)?;
@@ -2,6 +2,8 @@
use super::*;
use gateway_admin::model::accounts::AccountStatusSignals;
pub(crate) struct AdminAccountListItem {
pub(crate) account: ProviderAccountSummary,
pub(crate) status: AdminAccountStatus,
@@ -58,52 +60,48 @@ pub(crate) fn account_matches_admin_query(
pub(crate) fn admin_account_status(
account: &ProviderAccountSummary,
now: DateTime<Utc>,
rate_limited_until: Option<SystemTime>,
) -> AdminAccountStatus {
if !account.enabled {
AdminAccountStatus::Disabled
} else if account.availability == AccountAvailability::Banned {
AdminAccountStatus::Banned
} else if account.availability == AccountAvailability::QuotaExhausted {
AdminAccountStatus::QuotaExhausted
} else if account.availability == AccountAvailability::Expired
|| account.availability == AccountAvailability::Invalid
|| account
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
AccountStatusSignals {
enabled: account.enabled,
availability: account.availability,
access_token_expired: account
.access_token_expires_at
.is_some_and(|expires_at| expires_at <= now)
{
AdminAccountStatus::Expired
} else if account.quota_limit_reached {
AdminAccountStatus::RateLimited
} else if account.availability == AccountAvailability::Unknown {
AdminAccountStatus::Invalid
} else {
AdminAccountStatus::Active
.is_some_and(|expires_at| expires_at <= now),
quota_limit_reached: account.quota_limit_reached,
rate_limited_until,
}
.derive()
}
pub(crate) fn admin_account_summary(
accounts: &[ProviderAccountSummary],
now: DateTime<Utc>,
rate_limited_until: &BTreeMap<String, SystemTime>,
) -> AccountSummary {
let mut summary = AccountSummary {
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
active: 0,
normal: 0,
quota_exhausted: 0,
unavailable: 0,
rate_limited: 0,
disabled: 0,
error: 0,
};
for account in accounts {
match admin_account_status(account, now) {
AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1),
let until = rate_limited_until.get(&account.id).copied();
match admin_account_status(account, now, until) {
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
AdminAccountStatus::QuotaExhausted => {
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
}
AdminAccountStatus::RateLimited
| AdminAccountStatus::Expired
| AdminAccountStatus::Invalid
| AdminAccountStatus::Disabled
| AdminAccountStatus::Banned => {
summary.unavailable = summary.unavailable.saturating_add(1);
AdminAccountStatus::RateLimited => {
summary.rate_limited = summary.rate_limited.saturating_add(1);
}
AdminAccountStatus::Disabled => {
summary.disabled = summary.disabled.saturating_add(1);
}
AdminAccountStatus::Error => summary.error = summary.error.saturating_add(1),
}
}
summary
@@ -120,9 +118,7 @@ pub(crate) fn sort_admin_account_items(
items.sort_by(|left, right| {
let ordering = match sort.field {
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
AdminAccountSortField::Status => {
admin_account_status_name(left.status).cmp(admin_account_status_name(right.status))
}
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
AdminAccountSortField::Usage => left
.usage
@@ -152,18 +148,6 @@ pub(crate) fn sort_admin_account_items(
});
}
pub(crate) const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str {
match status {
AdminAccountStatus::Active => "active",
AdminAccountStatus::RateLimited => "rate_limited",
AdminAccountStatus::Expired => "expired",
AdminAccountStatus::Invalid => "invalid",
AdminAccountStatus::QuotaExhausted => "quota_exhausted",
AdminAccountStatus::Disabled => "disabled",
AdminAccountStatus::Banned => "banned",
}
}
pub(crate) fn admin_account_record(
summary: ProviderAccountSummary,
) -> AdminStoreResult<AccountRecord> {
@@ -189,6 +173,7 @@ pub(crate) fn admin_account_record(
enabled: summary.enabled,
availability: summary.availability,
availability_observed_at: summary.availability_observed_at,
last_error_message: summary.last_error_message,
quota_observed_at: summary.quota_observed_at,
created_at: summary.created_at,
updated_at: summary.updated_at,
@@ -4,6 +4,7 @@ use std::{
collections::{BTreeMap, BTreeSet},
fmt,
str::FromStr,
time::SystemTime,
};
use async_trait::async_trait;
@@ -95,7 +95,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, availability,
availability_observed_at,
quota_observed_at, quota_limit_reached, created_at, updated_at
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
from provider_accounts
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
order by provider_kind, name, id",
@@ -214,6 +214,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
when enabled then $4
else availability_observed_at
end,
last_error_message = case
when enabled then
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
else last_error_message
end,
updated_at = case when enabled then greatest(now(), $4) else updated_at end
where id = $1 and credential_revision = $2
and (availability_observed_at is null or availability_observed_at <= $4)",
@@ -222,6 +227,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(to_i64(update.expected_revision.get())?)
.bind(update.availability.as_str())
.bind(update.availability_observed_at)
.bind(update.message.as_deref())
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("apply provider account state"))?;
@@ -90,6 +90,8 @@ pub struct ProviderAccountSummary {
pub quota_observed_at: Option<DateTime<Utc>>,
/// 快照级限流事实(Provider 物化)。
pub quota_limit_reached: bool,
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
pub last_error_message: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -310,6 +312,8 @@ pub struct ProviderAccountStateUpdate {
pub expected_revision: Revision,
pub availability: AccountAvailability,
pub availability_observed_at: DateTime<Utc>,
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
pub message: Option<String>,
}
impl ProviderAccountStateUpdate {
@@ -335,14 +339,14 @@ impl ProviderAccountObservation {
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
provider_quota_json, quota_limit_reached,
provider_quota_json, quota_limit_reached, last_error_message,
availability_observed_at, quota_observed_at, created_at, updated_at
from provider_accounts where id = $1";
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
provider_quota_json, quota_limit_reached,
provider_quota_json, quota_limit_reached, last_error_message,
availability_observed_at, quota_observed_at, created_at, updated_at
from provider_accounts
where id = any($1::text[]) and provider_kind = $2
@@ -445,6 +449,7 @@ pub(crate) fn account_summary_from_row(
availability_observed_at: get(&row, "availability_observed_at")?,
quota_observed_at: get(&row, "quota_observed_at")?,
quota_limit_reached: get(&row, "quota_limit_reached")?,
last_error_message: get(&row, "last_error_message")?,
created_at: get(&row, "created_at")?,
updated_at: get(&row, "updated_at")?,
})
@@ -156,7 +156,7 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
.expect("drop migration test database");
admin.close().await;
assert_eq!((first_table_count, migration_count), (10, 4));
assert_eq!((first_table_count, migration_count), (10, 1));
assert_eq!(response_id_types, ["bytea", "bytea"]);
assert!(!raw_response_id_index_exists);
}
@@ -1,10 +1,23 @@
use std::num::NonZeroU32;
use std::{
collections::BTreeMap,
num::NonZeroU32,
sync::Arc,
time::{Duration, SystemTime},
};
use chrono::{TimeDelta, Utc};
use futures::future::BoxFuture;
use gateway_admin::{
model::{PageSize, observability as admin_observability},
ports::store::ObservabilityStore as AdminObservabilityStore,
};
use gateway_core::{
engine::credential::{CredentialRevision, ProviderAccountId},
provider_ports::{
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
ProviderStoreError,
},
};
use gateway_store::postgres::{
DiagnosticDimension, ObservabilityPageNumber, ObservabilityPageSize, ObservabilityRange,
ObservabilityRepository, OpsErrorFilter, OpsErrorQuery, PgAdminObservabilityStore,
@@ -83,7 +96,7 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
.expect("persist opaque response IDs");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let records = repository
.list_usage_records(UsageRecordQuery {
@@ -147,7 +160,7 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
.execute(&database.pool)
.await
.expect("seed account metric states");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("dashboard range");
@@ -179,6 +192,184 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
database.close().await;
}
#[tokio::test]
async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligible_accounts() {
let Some(database) = TestDatabase::create("dashboard_account_metrics_cooldowns").await else {
return;
};
let now = Utc::now();
sqlx::query(
"insert into provider_accounts (
id, provider_kind, name, upstream_user_id, authentication_kind,
provider_credentials_json, credential_revision, has_refresh_token,
access_token_expires_at, enabled, availability, quota_limit_reached,
availability_observed_at, created_at, updated_at
) values
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
$1, $1, $1),
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
$1, $1, $1),
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
$1, $1, $1),
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
$1, $1, $1),
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
$1, $1, $1),
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
$1, $1, $1),
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
$1, $1, $1),
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
$1, $1, $1),
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
$1, $1, $1)",
)
.bind(now)
.execute(&database.pool)
.await
.expect("seed cooldown account metric states");
let cooldowns = StaticCooldowns::new([
test_cooldown("acct_metrics_active_cooling", 2),
test_cooldown("acct_metrics_cooling", 3),
test_cooldown("acct_metrics_expired", 5),
test_cooldown("acct_metrics_unknown", 6),
test_cooldown("acct_metrics_stale_cooldown", 3),
]);
let repository =
PgObservabilityRepository::new(database.pool.clone(), Some(Arc::new(cooldowns)));
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("dashboard range");
let metrics = repository
.dashboard_summary(range)
.await
.expect("dashboard summary")
.provider_accounts;
assert_eq!(
(
metrics.total,
metrics.enabled,
metrics.active,
metrics.rate_limited,
metrics.quota_exhausted,
metrics.expired,
metrics.invalid,
metrics.disabled,
metrics.banned,
metrics.unavailable,
),
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
);
assert_eq!(
metrics.total,
metrics.active
+ metrics.rate_limited
+ metrics.quota_exhausted
+ metrics.expired
+ metrics.invalid
+ metrics.disabled
+ metrics.banned
);
assert_eq!(
metrics.unavailable,
metrics.rate_limited
+ metrics.quota_exhausted
+ metrics.expired
+ metrics.invalid
+ metrics.disabled
+ metrics.banned
);
database.close().await;
}
struct StaticCooldowns {
cooldowns: BTreeMap<ProviderAccountId, ProviderCooldown>,
}
impl StaticCooldowns {
fn new(cooldowns: impl IntoIterator<Item = ProviderCooldown>) -> Self {
Self {
cooldowns: cooldowns
.into_iter()
.map(|cooldown| (cooldown.account_id().clone(), cooldown))
.collect(),
}
}
}
impl ProviderCooldownPort for StaticCooldowns {
fn put_if_later(
&self,
_cooldown: ProviderCooldown,
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn read<'a>(
&'a self,
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
Box::pin(async move { Ok(self.cooldowns.get(account_id).cloned()) })
}
fn clear<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_through_revision: CredentialRevision,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn put_scoped_if_later(
&self,
_cooldown: ProviderScopedCooldown,
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn read_scoped<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_scope: &'a ProviderCooldownScope,
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
Box::pin(async { Ok(None) })
}
fn clear_scoped<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_scope: &'a ProviderCooldownScope,
_through_revision: CredentialRevision,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn clear_all<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
}
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
ProviderCooldown::new(
ProviderAccountId::new(account_id).expect("test account ID"),
CredentialRevision::new(revision).expect("test revision"),
SystemTime::now() + Duration::from_secs(60),
)
}
#[tokio::test]
async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
let Some(database) = TestDatabase::create("calculated_usage_billing_facts").await else {
@@ -193,7 +384,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
.expect("seed calculated billing facts");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let facts = repository
.usage_calculated_billing_facts(range, UsageRecordFilter::default())
@@ -207,7 +398,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
assert_eq!(facts[0].total.amount.as_str(), "1.25");
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
let facts = store
.usage_calculated_billing_facts(
admin_observability::TimeRange::new(range.start, range.end)
@@ -235,7 +426,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
let range =
admin_observability::TimeRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("admin observability range");
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
let dashboard = store
.dashboard_summary(range)
@@ -524,7 +715,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
.expect("seed observability facts");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let dashboard = repository
.dashboard_summary(range)
@@ -160,6 +160,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::QuotaExhausted,
@@ -170,6 +171,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
assert!(
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::Ready,
@@ -200,6 +202,61 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
database.close().await;
}
#[tokio::test]
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let account_id = ProviderAccountId::new("acct_state_message").expect("account ID");
repository
.insert_provider_account(account(account_id.as_str(), "user-state-message"))
.await
.expect("insert state message fixture");
let revision = CredentialRevision::new(1).expect("revision");
repository
.apply_state_change(AccountStateChange {
message: Some("upstream returned 402 payment required".to_owned()),
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::QuotaExhausted,
observed_at: SystemTime::now(),
})
.await
.expect("persist error message");
let stored: Option<String> =
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("read error message");
assert_eq!(
stored.as_deref(),
Some("upstream returned 402 payment required")
);
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: revision,
availability: AccountAvailability::Ready,
observed_at: SystemTime::now(),
})
.await
.expect("recover account");
let cleared: Option<String> =
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
.bind(account_id.as_str())
.fetch_one(&database.pool)
.await
.expect("read cleared error message");
assert_eq!(cleared, None);
database.close().await;
}
#[tokio::test]
async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_usage() {
let Some(database) = TestDatabase::create("provider_account_terminal_list").await else {
@@ -231,6 +288,21 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
.await
.expect("insert account list fixture");
}
repository
.compare_and_swap_quota(QuotaObservation {
account_id: ProviderAccountId::new("acct_charlie").expect("account id"),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
quota: Some(OpaqueProviderData::new(
json!({"rate_limit": {"limit_reached": true}})
.as_object()
.expect("quota object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: Some(true),
})
.await
.expect("seed rate-limited account");
seed_model_request(
&database.pool,
@@ -289,7 +361,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
.await
.expect("seed charlie usage");
let store = PgAdminAccountStore::new(database.pool.clone());
let store = PgAdminAccountStore::new(database.pool.clone(), None);
let usage_page = store
.list_accounts(AccountListQuery {
page: 1,
@@ -307,14 +379,18 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
assert_eq!(usage_page.config_revision.get(), 1);
assert_eq!(usage_page.total, 6);
assert_eq!(usage_page.summary.total, 6);
assert_eq!(usage_page.summary.active, 2);
assert_eq!(usage_page.summary.quota_exhausted, 1);
assert_eq!(usage_page.summary.unavailable, 3);
assert_eq!(usage_page.summary.normal, 1);
assert_eq!(usage_page.summary.quota_exhausted, 2);
assert_eq!(usage_page.summary.rate_limited, 0);
assert_eq!(usage_page.summary.disabled, 1);
assert_eq!(usage_page.summary.error, 2);
assert_eq!(
usage_page.summary.total,
usage_page.summary.active
usage_page.summary.normal
+ usage_page.summary.quota_exhausted
+ usage_page.summary.unavailable
+ usage_page.summary.rate_limited
+ usage_page.summary.disabled
+ usage_page.summary.error
);
assert_eq!(
usage_page
@@ -354,7 +430,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
page_size: PageSize::new(10).expect("page size"),
provider_kind: Some(ProviderKind::new("openai").expect("Provider kind")),
search: Some("ALPHA@EXAMPLE".to_owned()),
status: Some(AccountStatus::Active),
status: Some(AccountStatus::Normal),
sort: None,
})
.await
@@ -364,18 +440,20 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
assert_eq!(filtered.items[0].id, "acct_alpha");
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
let banned = store
let error_accounts = store
.list_accounts(AccountListQuery {
page: 1,
page_size: PageSize::new(10).expect("page size"),
provider_kind: None,
search: None,
status: Some(AccountStatus::Banned),
status: Some(AccountStatus::Error),
sort: None,
})
.await
.expect("filter banned accounts");
assert_eq!(banned.items[0].id, "acct_beta");
.expect("filter error accounts");
assert_eq!(error_accounts.items.len(), 2);
assert_eq!(error_accounts.items[0].id, "acct_beta");
assert_eq!(error_accounts.items[1].id, "acct_invalid");
database.close().await;
}
@@ -403,7 +481,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
.expect("insert xAI credential fixture");
}
let store = PgAdminAccountStore::new(database.pool.clone());
let store = PgAdminAccountStore::new(database.pool.clone(), None);
let provider = ProviderKind::new("xai").expect("xAI Provider kind");
let complete = store
.list_credentials(
@@ -484,7 +562,7 @@ async fn terminal_admin_usage_chunks_large_selections_and_preserves_exact_costs(
let mut account_ids = vec!["acct_usage_exact".to_owned()];
account_ids.extend((0..200).map(|index| format!("missing_account_{index}")));
let usage = PgAdminAccountStore::new(database.pool.clone())
let usage = PgAdminAccountStore::new(database.pool.clone(), None)
.load_account_usage(
TimeRange {
start: now - TimeDelta::hours(1),
@@ -576,7 +654,7 @@ async fn terminal_admin_quota_window_usage_prefers_provider_total_and_falls_back
.await
.expect("separate provider total from fallback components");
let mut usage = PgAdminAccountStore::new(database.pool.clone())
let mut usage = PgAdminAccountStore::new(database.pool.clone(), None)
.load_account_usage_by_windows(&[
AccountUsageWindowQuery {
account_id: "acct_quota_window".to_owned(),
@@ -616,7 +694,7 @@ async fn terminal_admin_mutations_keep_revision_account_and_audit_atomic() {
.insert_provider_account(account("acct_terminal_mutation", "user-terminal-mutation"))
.await
.expect("insert mutation account");
let store = PgAdminAccountStore::new(database.pool.clone());
let store = PgAdminAccountStore::new(database.pool.clone(), None);
let context = MutationContext {
actor: MutationActor::System,
request_id: "request_terminal_mutation".to_owned(),
@@ -689,7 +767,7 @@ async fn terminal_admin_delete_removes_enabled_accounts_in_one_transaction() {
.expect("insert enabled account");
}
let revision = PgAdminAccountStore::new(database.pool.clone())
let revision = PgAdminAccountStore::new(database.pool.clone(), None)
.delete_accounts(
DeleteAccounts {
account_ids: vec![
@@ -1193,6 +1271,7 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::QuotaExhausted,
@@ -1231,6 +1310,7 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
let observed_at = SystemTime::now();
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Ready,
@@ -1280,6 +1360,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
repository
.apply_state_change(AccountStateChange {
message: None,
account_id: account_id.clone(),
expected_revision: CredentialRevision::new(1).expect("credential revision"),
availability: AccountAvailability::Ready,
@@ -39,7 +39,7 @@ async fn request_snapshots_should_survive_account_deletion() {
.await
.expect("delete provider account");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let page = repository
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
.await
@@ -154,7 +154,7 @@ async fn attempts_should_keep_their_own_account_snapshots() {
.await
.expect("delete provider accounts");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let detail = repository
.usage_record_detail("req_snap_ab")
.await
@@ -276,7 +276,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
.await
.expect("delete provider accounts");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let errors = repository
.list_ops_errors(OpsErrorQuery {
range: range_around(started_at),
@@ -376,7 +376,7 @@ async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
.await
.expect("delete provider accounts");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let diagnostics = repository
.usage_diagnostics(
range_around(started_at),
@@ -445,7 +445,7 @@ async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
.expect("insert legacy request without live account");
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let diagnostics = repository
.usage_diagnostics(
range_around(started_at),
@@ -511,7 +511,7 @@ async fn failure_diagnostics_should_only_include_errored_requests() {
.await
.expect("mark request failed");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let diagnostics = repository
.usage_diagnostics(
range_around(started_at),
@@ -566,7 +566,7 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
.await
.expect("set request b latency");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let diagnostics = repository
.usage_diagnostics(
range_around(started_at),
@@ -633,7 +633,7 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() {
.await
.expect("rename provider account");
let repository = PgObservabilityRepository::new(database.pool.clone());
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
let page = repository
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
.await
+63 -20
View File
@@ -50,7 +50,7 @@ use crate::credential::{
CodexCredentialAdminService, CodexCredentialCatalogError, CodexCredentialCatalogService,
CodexCredentialQuotaError, CodexCredentialQuotaService, CodexOAuthAdmin, CodexOAuthAdminError,
CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore, CodexOAuthPendingStoreError,
CodexPendingAuthorization, CodexQuotaWindowKind, CodexQuotaWindowRole,
CodexPendingAuthorization, CodexQuotaWindow, CodexQuotaWindowKind, CodexQuotaWindowRole,
CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential, ExportManagedCodexCredential,
RotateManagedCodexCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
refresh_time,
@@ -475,7 +475,14 @@ impl ProviderAdmin for OpenAiAdminProvider {
.await
.map_err(map_quota_error)?
};
Ok(project_quota(snapshot, &account))
let mut quota = project_quota(snapshot, &account);
quota.rate_limited_until = self
.quota
.rate_limited_until(account.id(), account.revision())
.await
.map_err(map_quota_error)?
.map(DateTime::<Utc>::from);
Ok(quota)
}
async fn models(
@@ -740,6 +747,7 @@ fn empty_quota() -> ProviderQuota {
refresh_token_expires_at: None,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
}
}
@@ -774,6 +782,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
let windows: Vec<ProviderQuotaWindow> = snapshot
.windows()
.iter()
.filter(|window| should_project_quota_window(window))
.map(|window| {
let mut data = Map::new();
data.insert(
@@ -790,6 +799,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
label: codex_quota_window_label(
window.limit_name(),
window.kind(),
window.role(),
window.source(),
window.window_seconds(),
),
@@ -811,12 +821,24 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
refresh_token_expires_at: None,
windows,
limit_reached,
rate_limited_until: None,
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
provider_data,
))),
}
}
/// `secondary_window` 有时只是上游的空占位:没有时长、重置时间,也没有用量。
/// 它不能被可靠翻译成 5 小时或周额度,因此不应占用账号面板;带有实际事实的
/// 次级窗口(时长、重置、非零用量或触顶)仍完整保留。
fn should_project_quota_window(window: &CodexQuotaWindow) -> bool {
window.role() != CodexQuotaWindowRole::Secondary
|| window.window_seconds().is_some()
|| window.reset_at().is_some()
|| window.used_percent().is_some_and(|used| used > 0.0)
|| window.limit_reached()
}
fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool {
windows.iter().any(|window| window.limit_reached)
}
@@ -882,45 +904,66 @@ const fn quota_role(role: CodexQuotaWindowRole) -> &'static str {
}
}
/// 窗口展示名,优先使用上游 `limit_name`(原文,`_` 转空格);
/// 无 limit_name 时按窗口时长猜(±5% 容差,汉化输出),猜不中兜底「额度」。
/// 按窗口时长显示汉化额度名;非核心桶补上上游桶名称作为前缀。
fn codex_quota_window_label(
limit_name: Option<&str>,
kind: CodexQuotaWindowKind,
role: CodexQuotaWindowRole,
source: &str,
window_seconds: Option<u64>,
) -> String {
if let Some(limit_name) = limit_name {
return limit_name.replace('_', " ");
}
let base = match kind {
CodexQuotaWindowKind::Monthly => "月限额".to_owned(),
CodexQuotaWindowKind::Weekly => "周限额".to_owned(),
CodexQuotaWindowKind::Monthly => "月额度".to_owned(),
CodexQuotaWindowKind::Weekly => "周额度".to_owned(),
CodexQuotaWindowKind::ShortTerm => {
if window_seconds.is_some_and(|seconds| seconds > 86_400) {
"周限额".to_owned()
"周额度".to_owned()
} else {
"5小时限额".to_owned()
"5小时额度".to_owned()
}
}
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds),
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds, role),
};
if matches!(source, "core" | "codex" | "code_review" | "spend_control") {
return base;
if let Some(prefix) = codex_quota_window_prefix(limit_name, source) {
return format!("{prefix} · {base}");
}
format!("{source} · {base}")
base
}
fn custom_quota_window_label(window_seconds: Option<u64>) -> String {
fn codex_quota_window_prefix(limit_name: Option<&str>, source: &str) -> Option<String> {
let source_identifier = quota_identifier(source);
if matches!(source_identifier.as_str(), "core" | "codex") {
return None;
}
let label = limit_name.unwrap_or(source);
match quota_identifier(label).as_str() {
"premium" => Some("Premium".to_owned()),
"code_review" => Some("代码审查".to_owned()),
"spend_control" => Some("消费控制".to_owned()),
_ => Some(label.replace('_', " ")),
}
}
fn quota_identifier(value: &str) -> String {
value.trim().to_ascii_lowercase().replace(['-', ' '], "_")
}
fn custom_quota_window_label(window_seconds: Option<u64>, role: CodexQuotaWindowRole) -> String {
let Some(seconds) = window_seconds.filter(|seconds| *seconds > 0) else {
return "额度".to_owned();
// 与官方客户端一致:没有时长时不臆测为 5 小时或周额度;保留
// primary/secondary 语义,让用户知道这是上游未标明时长的独立窗口。
return match role {
CodexQuotaWindowRole::Primary => "主额度".to_owned(),
CodexQuotaWindowRole::Secondary => "次级额度".to_owned(),
CodexQuotaWindowRole::Monthly => "月额度".to_owned(),
};
};
if seconds % 86_400 == 0 {
format!("{}日限额", seconds / 86_400)
format!("{}日额度", seconds / 86_400)
} else if seconds % 3_600 == 0 {
format!("{}小时限额", seconds / 3_600)
format!("{}小时额度", seconds / 3_600)
} else {
format!("{}分钟限额", seconds.div_ceil(60))
format!("{}分钟额度", seconds.div_ceil(60))
}
}
@@ -10,19 +10,33 @@ use crate::transport::request::derive_conversation_anchor;
pub(crate) fn derive_codex_session_affinity_key(
request: &CodexResponsesRequest,
) -> Option<ProviderSessionAffinityKey> {
if let Some(conversation_id) = request
let session_key = if let Some(conversation_id) = request
.local_conversation_id
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
if let Ok(key) = ProviderSessionAffinityKey::try_new(conversation_id.to_owned()) {
return Some(key);
Some(key)
} else {
opaque_affinity_key("local-conversation", conversation_id)
}
return opaque_affinity_key("local-conversation", conversation_id);
}
let (domain, value) = derive_conversation_anchor(request)?;
opaque_affinity_key(domain, &value)
} else {
let (domain, value) = derive_conversation_anchor(request)?;
opaque_affinity_key(domain, &value)
};
let session_discriminator = request.subagent_kind();
session_key.and_then(|session_key| match session_discriminator.as_deref() {
None => Some(session_key),
Some(discriminator) => {
// 仅扩展原会话锚点的键空间;后续查找、绑定和调度仍使用同一套亲和逻辑。
opaque_affinity_key(
"subagent-session",
&format!("{discriminator}\0{}", session_key.expose_to_store()),
)
}
})
}
fn opaque_affinity_key(domain: &str, value: &str) -> Option<ProviderSessionAffinityKey> {
@@ -20,12 +20,13 @@ use gateway_core::engine::credential::{
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, QuotaWriteOutcome,
};
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
use gateway_protocol::openai::events::{
ParsedRateLimits, RateLimitDetails, RateLimitWindow, parse_rate_limit_headers,
};
use reqwest::Client;
use secrecy::ExposeSecret;
use serde_json::{Map, Number, Value};
use serde_json::{Map, Value};
use thiserror::Error;
use tokio::sync::Mutex;
use uuid::Uuid;
@@ -35,7 +36,7 @@ use crate::transport::{CodexBackendClient, CodexClientError, CodexRequestContext
use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeCredential};
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
use recovery::{QuotaRecoveryEvidence, quota_state_transition, quota_success_state};
use recovery::{quota_state_transition, quota_success_state};
use snapshot::{
parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation,
scheduling_signals_from_snapshot,
@@ -120,6 +121,7 @@ pub struct CodexCredentialQuotaService {
http: Client,
base_url: String,
agent_identity: Arc<CodexAgentIdentityTaskService>,
cooldowns: Arc<dyn ProviderCooldownPort>,
scheduling: CodexQuotaSchedulingProjection,
}
@@ -379,6 +381,7 @@ impl CodexCredentialQuotaService {
http: Client,
base_url: String,
agent_identity: Arc<CodexAgentIdentityTaskService>,
cooldowns: Arc<dyn ProviderCooldownPort>,
) -> Self {
Self {
store: Arc::clone(repository.store()),
@@ -387,11 +390,12 @@ impl CodexCredentialQuotaService {
http,
base_url,
agent_identity,
cooldowns,
scheduling: CodexQuotaSchedulingProjection::default(),
}
}
/// 写入新快照前读取旧确认 reset(402 恢复用「旧 reset 已到期」判定)。
/// 写入新快照前读取已确认耗尽窗口的 reset,用于判断窗口是否真的推进。
async fn previous_confirmed_reset(&self, account: &ProviderAccount) -> Option<SystemTime> {
let observations = self
.store
@@ -406,6 +410,52 @@ impl CodexCredentialQuotaService {
snapshot.fact().resets_at().map(SystemTime::from)
}
/// 将已确认的额度拒绝投影为 100%,使展示、调度和恢复共用同一窗口事实。
///
/// `reset_at` 来自官方 `usage_limit_reached` 错误体;缺失时保留当前快照的 reset。
pub(crate) async fn record_confirmed_exhaustion(
&self,
account: &ProviderAccount,
reset_at: Option<i64>,
) -> Result<(), CodexCredentialQuotaError> {
let existing = self
.store
.get_quotas(std::slice::from_ref(account.id()))
.await?
.into_iter()
.find(|observation| {
observation.account_id == *account.id()
&& observation.expected_revision == account.revision()
})
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.unwrap_or_default();
let quota = confirmed_exhaustion_projection(existing, reset_at);
let observed_at = SystemTime::now();
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(quota.clone())),
observed_at: Some(observed_at),
limit_reached: Some(true),
})
.await?;
if outcome == QuotaWriteOutcome::Conflict {
return Err(CodexCredentialQuotaError::RevisionConflict);
}
if let Ok(snapshot) = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(quota),
) {
self.scheduling.observe(&snapshot);
}
Ok(())
}
/// 批量预热请求级额度投影;持久层或 Provider JSON 异常只退化为未知额度。
pub async fn prepare_scheduling(&self, accounts: &[ProviderAccount]) {
if self.scheduling.hydration_targets(accounts).is_empty() {
@@ -535,18 +585,42 @@ impl CodexCredentialQuotaService {
observed_at: SystemTime,
summary: &mut CodexQuotaSyncSummary,
) -> Result<(), CodexCredentialQuotaError> {
let snapshot = parse_account_quota_snapshot(
let refreshed_snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
value,
)?;
let fact = snapshot.fact();
let refreshed_fact = refreshed_snapshot.fact();
let previous_reset_at = self.previous_confirmed_reset(account).await;
let now = SystemTime::now();
let object = value
.as_object()
.cloned()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
let object = if account.availability() == AccountAvailability::QuotaExhausted
&& quota_success_state(
account.availability(),
refreshed_fact,
previous_reset_at,
now,
)
.is_none()
{
confirmed_exhaustion_projection(
object,
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
)
} else {
object
};
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(object.clone()),
)?;
let fact = snapshot.fact();
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
@@ -577,10 +651,9 @@ impl CodexCredentialQuotaService {
}
if let Some(availability) = quota_success_state(
current.availability(),
fact,
refreshed_fact,
previous_reset_at,
SystemTime::now(),
QuotaRecoveryEvidence::RefreshedSnapshot,
now,
) {
let _ = self
.repository
@@ -599,6 +672,15 @@ impl CodexCredentialQuotaService {
let Some(rate_limits) = parse_rate_limit_headers(headers) else {
return Ok(false);
};
let has_quota_facts = rate_limits
.limits
.values()
.any(|details| passive_rate_limit_snapshot(details).is_some());
let previous_reset_at = if account.availability() == AccountAvailability::QuotaExhausted {
self.previous_confirmed_reset(account).await
} else {
None
};
let existing = self
.store
.get_quotas(std::slice::from_ref(account.id()))
@@ -607,12 +689,48 @@ impl CodexCredentialQuotaService {
.find(|observation| {
observation.account_id == *account.id()
&& observation.expected_revision == account.revision()
})
});
let existing_observed_at = existing
.as_ref()
.and_then(|observation| observation.observed_at);
let existing = existing
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.unwrap_or_default();
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
// 也不能借旧快照重新推导 quota state。
if !has_quota_facts {
let Some(observed_at) = existing_observed_at else {
return Ok(false);
};
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(merge_passive_quota(
existing,
&rate_limits,
))),
observed_at: Some(observed_at),
limit_reached: None,
})
.await?;
return Ok(outcome != QuotaWriteOutcome::Conflict);
}
let quota = merge_passive_quota(existing, &rate_limits);
let observed_at = SystemTime::now();
let refreshed_fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
let quota = if account.availability() == AccountAvailability::QuotaExhausted {
// 被动成功响应不能解除确认额度耗尽;继续展示 100%,直到权威 usage 快照
// 证明旧 reset 已跨过且新窗口已经推进。
confirmed_exhaustion_projection(
quota,
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
)
} else {
quota
};
let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
@@ -627,123 +745,68 @@ impl CodexCredentialQuotaService {
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(quota)),
observed_at: Some(observed_at),
limit_reached: Some(true),
limit_reached: Some(fact.exhausted()),
})
.await?;
if outcome == QuotaWriteOutcome::Conflict {
return Ok(false);
}
// 可用性写回必须基于最新账号状态:revision fence 只覆盖凭据轮换,
// 不覆盖其他路径的仅状态变化(Cooldown/QuotaExhausted 转换)。
let Some(current) = self.store.get_account(account.id()).await? else {
return Ok(false);
};
if current.revision() != account.revision() {
return Ok(false);
}
if let Ok(snapshot) = snapshot {
self.scheduling.observe(&snapshot);
}
if let Some(availability) = quota_success_state(
current.availability(),
fact,
None,
observed_at,
QuotaRecoveryEvidence::SuccessfulResponse,
) {
self.repository
.apply_state(&current, availability, observed_at)
.await?;
}
Ok(true)
}
/// 真实 429 的单一事实入口:合成/更新 quota 主窗口(`limit_reached=true`、
/// `used_percent=100`、`reset_at=retry-after`)。availability 不改变,
/// 窗口 reset 到期后由投影自动滚动恢复(429 只记录限流窗口)。
/// 已有更晚的 `reset_at` 不会被缩短(never-shrink)。
/// 真实 429 的单一事实入口:写入 Redis 临时限流冷却(`until = now + retry_after`)。
/// availability 与 quota 主窗口(额度重置时间)都不改变——临时限流是独立维度,
/// 到期由 Redis key 过期自动解除,不污染配额耗尽状态。
/// 已有更晚的冷却不会被缩短(put_if_later)。
pub async fn apply_rate_limit_429(
&self,
account: &ProviderAccount,
retry_after: Option<Duration>,
observed_at: SystemTime,
) -> Result<(), CodexCredentialQuotaError> {
let mut object = self
.store
.get_quotas(std::slice::from_ref(account.id()))
.await?
.into_iter()
.find(|observation| {
observation.account_id == *account.id()
&& observation.expected_revision == account.revision()
})
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.unwrap_or_default();
let new_reset_at = observed_at
let until = observed_at
.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN))
.and_then(|reset| reset.duration_since(UNIX_EPOCH).ok())
.map(|duration| duration.as_secs() as i64);
let existing_reset_at = object
.get("rate_limit")
.and_then(Value::as_object)
.and_then(|rate_limit| rate_limit.get("primary_window"))
.and_then(Value::as_object)
.and_then(|window| window.get("reset_at"))
.and_then(Value::as_i64);
let reset_at = match (existing_reset_at, new_reset_at) {
(Some(existing), Some(next)) if existing > next => Some(existing),
(_, next) => next,
};
let mut rate_limit = object
.get("rate_limit")
.cloned()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
let mut primary = rate_limit
.get("primary_window")
.cloned()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
let used_percent = primary
.get("used_percent")
.and_then(Value::as_f64)
.unwrap_or(0.0)
.max(100.0);
primary.insert(
"used_percent".to_owned(),
Number::from_f64(used_percent)
.map(Value::Number)
.unwrap_or_else(|| Value::Number(Number::from(100))),
);
if let Some(reset_at) = reset_at {
primary.insert("reset_at".to_owned(), Value::from(reset_at));
}
rate_limit.insert("primary_window".to_owned(), Value::Object(primary));
object.insert("rate_limit".to_owned(), Value::Object(rate_limit));
if self
.store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(object)),
observed_at: Some(observed_at),
limit_reached: Some(true),
})
.await?
== QuotaWriteOutcome::Conflict
{
return Ok(());
}
// 429 合成的限流窗口要立即反映到调度信号(reset 到期由投影滚动解除)。
if let Ok(Some(snapshot)) = self.read_snapshot_for(account).await {
self.scheduling.observe(&snapshot);
}
.unwrap_or(observed_at);
self.cooldowns
.put_if_later(ProviderCooldown::new(
account.id().clone(),
account.revision(),
until,
))
.await
.map_err(|error| CodexCredentialQuotaError::Store {
detail: error.to_string(),
})?;
Ok(())
}
/// 读取账号当前是否处于临时限流(429)冷却,及到期时间。
pub async fn rate_limited_until(
&self,
account_id: &ProviderAccountId,
credential_revision: CredentialRevision,
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
CodexCredentialQuotaError::Store {
detail: error.to_string(),
}
})?
else {
return Ok(None);
};
if cooldown.credential_revision() != credential_revision {
return Ok(None);
}
let until = cooldown.until();
if until <= SystemTime::now() {
return Ok(None);
}
Ok(Some(until))
}
/// 读取单账号最后一次落库的 Provider quota,并由 Codex 域解析展示窗口。
pub async fn read_account(
&self,
@@ -855,17 +918,41 @@ impl CodexCredentialQuotaService {
});
}
};
let snapshot = parse_account_quota_snapshot(
let refreshed_snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&value,
)?;
let refreshed_fact = refreshed_snapshot.fact();
let previous_reset_at = self.previous_confirmed_reset(&account).await;
let now = SystemTime::now();
let object = value
.as_object()
.cloned()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
let object = if account.availability() == AccountAvailability::QuotaExhausted
&& quota_success_state(
account.availability(),
refreshed_fact,
previous_reset_at,
now,
)
.is_none()
{
confirmed_exhaustion_projection(
object,
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
)
} else {
object
};
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
account.revision(),
observed_at,
&Value::Object(object.clone()),
)?;
if self
.store
.compare_and_swap_quota(QuotaObservation {
@@ -891,10 +978,9 @@ impl CodexCredentialQuotaService {
self.scheduling.observe(&snapshot);
if let Some(availability) = quota_success_state(
current.availability(),
snapshot.fact(),
refreshed_fact,
previous_reset_at,
SystemTime::now(),
QuotaRecoveryEvidence::RefreshedSnapshot,
now,
) {
self.repository
.apply_state(&current, availability, observed_at)
@@ -1095,6 +1181,66 @@ fn merge_passive_quota(
quota
}
/// 用已确认的上游额度拒绝覆盖 core 限额投影。
///
/// 上游 usage 快照可能在结算期间短暂滞后为 98/99% 或 `allowed=true`;保留其他
/// 原始 metadata,但把 core 窗口固定为 100%,直到权威快照证明进入下一窗口。
fn confirmed_exhaustion_projection(
mut quota: Map<String, Value>,
reset_at: Option<i64>,
) -> Map<String, Value> {
let mut rate_limit = quota
.remove("rate_limit")
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
for field in ["primary_window", "secondary_window"] {
let existed = rate_limit.contains_key(field);
let mut window = rate_limit
.remove(field)
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
if field == "primary_window" || existed {
window.insert("used_percent".to_owned(), Value::from(100_u64));
if field == "primary_window"
&& let Some(reset_at) = reset_at
{
window.insert("reset_at".to_owned(), Value::from(reset_at));
}
rate_limit.insert(field.to_owned(), Value::Object(window));
}
}
quota.insert("rate_limit".to_owned(), Value::Object(rate_limit));
quota
}
/// 选择应继续锁定的 reset:新鲜快照仍确认耗尽时可把锁推进到更晚窗口;
/// 否则保留已确认失败的旧窗口,避免 98/99% 快照反向解锁。
fn confirmed_exhaustion_reset_at(
previous_reset_at: Option<SystemTime>,
refreshed_fact: CodexQuotaFact,
) -> Option<i64> {
let refreshed_reset_at = refreshed_fact.resets_at().map(SystemTime::from);
let reset_at = match (previous_reset_at, refreshed_reset_at) {
(Some(previous), Some(refreshed)) if refreshed_fact.exhausted() && refreshed > previous => {
refreshed
}
(Some(previous), _) => previous,
(None, refreshed) => refreshed?,
};
system_time_to_unix_seconds(reset_at)
}
fn system_time_to_unix_seconds(value: SystemTime) -> Option<i64> {
value
.duration_since(UNIX_EPOCH)
.ok()?
.as_secs()
.try_into()
.ok()
}
fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option<Map<String, Value>> {
// 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。
let mut snapshot = Map::new();
@@ -1,7 +1,7 @@
//! 402 恢复证据与 availability 转换。
//! 确认额度耗尽后的恢复证据与 availability 转换。
//!
//! `QuotaExhausted -> Ready` 只接受同一 credential revision 的真实成功响应、
//! 新鲜快照显式 `allowed=true` 且无 exhaustion、或已到期旧 reset 配合未耗尽新快照。
//! `QuotaExhausted -> Ready` 只能由同一 credential revision 的权威 usage 快照证明:
//! 已确认的旧窗口 reset 已到期,且新快照的 reset 已推进到下一窗口。
use std::time::SystemTime;
@@ -12,47 +12,24 @@ use serde_json::Value;
use super::snapshot::CodexQuotaFact;
use crate::transport::CodexClientError;
/// 402 恢复可接受的证据强度。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum QuotaRecoveryEvidence {
/// 一次真实成功推理响应(最强证据)。
SuccessfulResponse,
/// 周期 worker 刷新出的新快照(需旧 reset 已到期佐证)。
RefreshedSnapshot,
}
/// 只处理 `QuotaExhausted` 的恢复——429/触顶不再进入该状态,由 quota 数据
/// 驱动调度排除(限流不改变账号可用性)。`Invalid/Expired/Banned` 等终态由 selector 的
/// 成功响应恢复路径处理。
/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独
/// 覆盖一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。
///
/// `previous_reset_at` 是写入新快照前读取的旧确认 reset;`RefreshedSnapshot`
/// 恢复要求它已到期(窗口真实滚动后新快照的 reset 指向下一个未来周期,
/// 不能用它判定旧耗尽窗口是否已结束)。
/// `previous_reset_at` 是写入新快照前读取的已确认耗尽窗口。恢复必须同时满足:
/// 旧窗口已经到期,且权威快照的 reset 指向比旧窗口更晚的新周期。
pub(crate) fn quota_success_state(
current: AccountAvailability,
fact: CodexQuotaFact,
previous_reset_at: Option<SystemTime>,
now: SystemTime,
recovery_evidence: QuotaRecoveryEvidence,
) -> Option<AccountAvailability> {
if current != AccountAvailability::QuotaExhausted || fact.exhausted() {
return None;
}
// 新鲜快照显式 `allowed=true` 且无 exhaustion,直接恢复。
if fact.explicit_allowed() {
return Some(AccountAvailability::Ready);
}
// 一次真实成功响应是最强证据;周期刷新需要旧确认 reset 已到期
// (used_percent 是可滞后、会取整的观测值,其回落不能覆盖仍未到期的旧 reset)。
match recovery_evidence {
QuotaRecoveryEvidence::SuccessfulResponse => Some(AccountAvailability::Ready),
QuotaRecoveryEvidence::RefreshedSnapshot
if previous_reset_at.is_some_and(|reset_at| reset_at <= now) =>
{
Some(AccountAvailability::Ready)
}
QuotaRecoveryEvidence::RefreshedSnapshot => None,
}
let previous_reset_at = previous_reset_at?;
let refreshed_reset_at = fact.resets_at().map(SystemTime::from)?;
(previous_reset_at <= now && refreshed_reset_at > previous_reset_at)
.then_some(AccountAvailability::Ready)
}
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
@@ -18,8 +18,6 @@ pub struct CodexQuotaFact {
remaining_percent: Option<u8>,
resets_at: Option<DateTime<Utc>>,
exhausted: bool,
/// 上游显式 `allowed=true`(恢复证据:新鲜快照明确放行)。
explicit_allowed: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -41,7 +39,7 @@ pub enum CodexQuotaWindowRole {
pub struct CodexQuotaWindow {
key: String,
source: String,
/// 上游提供的限流名称(`metered_feature`/`limit_name`/`limitId`);显示优先用它。
/// 上游提供的限流桶名称(`metered_feature`/`limit_name`/`limitId`);用于非核心桶的展示前缀。
limit_name: Option<String>,
kind: CodexQuotaWindowKind,
role: CodexQuotaWindowRole,
@@ -189,12 +187,6 @@ impl CodexQuotaFact {
pub const fn exhausted(&self) -> bool {
self.exhausted
}
/// 上游是否显式 `allowed=true`(新鲜快照的恢复证据)。
#[must_use]
pub const fn explicit_allowed(&self) -> bool {
self.explicit_allowed
}
}
pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCredentialQuotaError> {
@@ -220,7 +212,6 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCre
remaining_percent: aggregate.remaining_percent,
resets_at: aggregate.resets_at,
exhausted: aggregate.exhausted,
explicit_allowed: aggregate.explicit_allowed,
})
}
@@ -319,6 +310,7 @@ pub(crate) fn parse_account_quota_snapshot(
let mut windows = Vec::new();
for limit in canonical_rate_limits(object)? {
parse_rate_limit_windows(
&limit.key,
&limit.source,
limit.limit_name.as_deref(),
limit.rate_limit,
@@ -335,8 +327,9 @@ pub(crate) fn parse_account_quota_snapshot(
})
}
/// 一个规范化后的限流桶:source 是调度/展示分组键,limit_name 是上游显示名。
/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。
struct CanonicalRateLimit<'a> {
key: String,
source: String,
limit_name: Option<String>,
rate_limit: &'a Value,
@@ -346,9 +339,15 @@ fn canonical_rate_limits(
object: &Map<String, Value>,
) -> Result<Vec<CanonicalRateLimit<'_>>, CodexCredentialQuotaError> {
let mut limits = Vec::new();
let has_primary_codex_rate_limit = object
.get("rate_limit")
.is_some_and(|value| !value.is_null());
if let Some(rate_limit) = object.get("rate_limit") {
// 官方 Codex 将顶层 `rate_limit` 固定识别为 `codex`,而不把
// `active_limit` 投影成另一个展示桶。后者只是当前活动套餐的元数据。
limits.push(CanonicalRateLimit {
source: "core".to_owned(),
key: "core".to_owned(),
source: "codex".to_owned(),
limit_name: None,
rate_limit,
});
@@ -358,6 +357,7 @@ fn canonical_rate_limits(
.filter(|value| !value.is_null())
{
limits.push(CanonicalRateLimit {
key: "code-review".to_owned(),
source: "code_review".to_owned(),
limit_name: Some("code_review".to_owned()),
rate_limit,
@@ -379,23 +379,29 @@ fn canonical_rate_limits(
.as_object()
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
let (source, limit_name) = canonical_rate_limit_source(item, index);
// 部分上游响应会在 `additional_rate_limits` 中重复默认的 `codex`
// 桶。它没有独立的 limit id,不能在账号面板再次展示。
if has_primary_codex_rate_limit && is_codex_limit_id(&source) {
continue;
}
if let Some(rate_limit) = item.get("rate_limit") {
// additional 里的 codex 桶覆盖顶层 rate_limit(同源替换)。
if let Some(existing) = limits.iter_mut().find(|existing| existing.source == source) {
existing.rate_limit = rate_limit;
existing.limit_name = limit_name;
} else {
limits.push(CanonicalRateLimit {
source,
limit_name,
rate_limit,
});
}
// 非 Codex additional 桶保留为独立快照;额外用索引生成窗口键,
// 避免同名 additional 产生重复前端 key。
limits.push(CanonicalRateLimit {
key: format!("additional-{index}-{source}"),
source,
limit_name,
rate_limit,
});
}
}
Ok(limits)
}
fn is_codex_limit_id(value: &str) -> bool {
value.trim().eq_ignore_ascii_case("codex")
}
fn canonical_rate_limit_source(
item: &Map<String, Value>,
index: usize,
@@ -417,14 +423,11 @@ fn canonical_rate_limit_source(
!value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control)
})
.map(str::to_owned);
if source.trim().to_ascii_lowercase().replace(['-', ' '], "_") == "codex" {
("core".to_owned(), limit_name)
} else {
(source, limit_name)
}
(source, limit_name)
}
fn parse_rate_limit_windows(
key_source: &str,
source: &str,
limit_name: Option<&str>,
value: &Value,
@@ -482,7 +485,7 @@ fn parse_rate_limit_windows(
|| used_percent.is_some_and(|used| used >= 100.0);
let kind = quota_window_kind(window_seconds);
output.push(CodexQuotaWindow {
key: format!("{}-{}", quota_key(source), quota_role_name(role, kind)),
key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)),
source: source.to_owned(),
limit_name: limit_name.map(str::to_owned),
kind,
@@ -559,7 +562,6 @@ const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind)
struct QuotaAggregate {
recognized: bool,
exhausted: bool,
explicit_allowed: bool,
remaining_percent: Option<u8>,
resets_at: Option<DateTime<Utc>>,
}
@@ -572,7 +574,6 @@ impl QuotaAggregate {
self.recognized = true;
self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false);
self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
self.explicit_allowed |= optional_bool(object, "allowed")?.is_some_and(|allowed| allowed);
for key in ["primary_window", "secondary_window"] {
if let Some(window) = object.get(key) {
self.observe_window(window)?;
@@ -234,6 +234,18 @@ impl CodexCredentialRepository {
account: &ProviderAccount,
availability: AccountAvailability,
observed_at: SystemTime,
) -> Result<(), CredentialRepositoryError> {
self.apply_state_with_message(account, availability, observed_at, None)
.await
}
/// 带上游错误描述的状态写入;`message` 为 `Some` 时持久化最近错误,恢复 `Ready` 时清空。
pub async fn apply_state_with_message(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
observed_at: SystemTime,
message: Option<String>,
) -> Result<(), CredentialRepositoryError> {
if !account.enabled() {
return Ok(());
@@ -244,6 +256,7 @@ impl CodexCredentialRepository {
expected_revision: account.revision(),
availability,
observed_at,
message,
})
.await?;
Ok(())
@@ -294,6 +294,24 @@ impl CodexCredentialSelector {
if !diagnostic {
self.quota.prepare_scheduling(&accounts).await;
}
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
let accounts = if diagnostic {
accounts
} else {
let mut retained = Vec::with_capacity(accounts.len());
for account in accounts {
let cooling = self
.quota
.rate_limited_until(account.id(), account.revision())
.await
.unwrap_or(None);
if cooling.is_none() {
retained.push(account);
}
}
retained
};
let account_ids = accounts
.iter()
.map(|account| account.id().clone())
@@ -327,46 +345,6 @@ impl CodexCredentialSelector {
AccountCandidate { account, signals }
})
.collect::<Vec<_>>();
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
let scheduling_candidates = if self.skip_exhausted {
candidates
.iter()
.filter(|candidate| !candidate.signals.quota_limit_reached)
.cloned()
.collect::<Vec<_>>()
} else {
candidates
.iter()
.cloned()
.map(|mut candidate| {
candidate.account = quota_exhausted_scheduling_projection(&candidate.account);
candidate
})
.collect::<Vec<_>>()
};
let scheduling_candidates = scheduling_candidates.as_slice();
let mut affinity = if diagnostic {
AffinitySelection::default()
} else {
self.resolve_session_affinity(
request.session_affinity_key,
&candidates,
SystemTime::now(),
)
.await
};
let cyber_policy_scope = self
.prepare_cyber_policy_scope(cyber_policy_session_key)
.await;
let mut excluded = request.attempt.excluded_accounts().clone();
if let Some(state) = cyber_policy_scope
.as_ref()
.and_then(|scope| scope.state.as_ref())
{
excluded.extend(state.excluded_accounts().iter().cloned());
}
let continuation_account = match request.attempt.continuation_attempt() {
ContinuationAttempt::Native => request
.attempt
@@ -388,7 +366,74 @@ impl CodexCredentialSelector {
{
return Err(CredentialSelectionError::NoEligibleCredential);
}
let pinned_account = required_account.or(continuation_account);
let pinned_account = required_account.or_else(|| continuation_account.clone());
let preserve_native_continuation_state = matches!(
request.attempt.continuation_attempt(),
ContinuationAttempt::Native
) && continuation_account.is_some();
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
let scheduling_candidates = if diagnostic {
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
candidates.clone()
} else if self.skip_exhausted {
candidates
.iter()
.filter(|candidate| !candidate.signals.quota_limit_reached)
.cloned()
.collect::<Vec<_>>()
} else {
candidates
.iter()
.cloned()
.map(|mut candidate| {
// 已命中 previous_response_id 的 native continuation 先检查原账号。
// 即使常规调度允许探测额度耗尽账号,也必须保留原账号真实状态,
// 由 Coordinator 据此切换到可携带重放,而非伪装为 Ready。
if !preserve_native_continuation_state
|| pinned_account.as_ref() != Some(candidate.account.id())
{
candidate.account =
quota_exhausted_scheduling_projection(&candidate.account);
}
candidate
})
.collect::<Vec<_>>()
};
let scheduling_candidates = scheduling_candidates.as_slice();
let mut affinity = if diagnostic {
AffinitySelection::default()
} else {
self.resolve_session_affinity(
request.session_affinity_key,
&candidates,
SystemTime::now(),
)
.await
};
let cyber_policy_scope = self
.prepare_cyber_policy_scope(cyber_policy_session_key)
.await;
let mut excluded = request.attempt.excluded_accounts().clone();
if !diagnostic
&& preserve_native_continuation_state
&& let Some(pinned) = pinned_account.as_ref()
&& candidates.iter().any(|candidate| {
candidate.account.id() == pinned && candidate.signals.quota_limit_reached
})
{
// `skip_exhausted = false` 只允许普通新请求探测可疑账号;已绑定的
// native response handle 先将真实触顶状态交给 Coordinator,随后才可
// 通过 ReplayAny 按当前调度策略换号。
excluded.insert(pinned.clone());
}
if let Some(state) = cyber_policy_scope
.as_ref()
.and_then(|scope| scope.state.as_ref())
{
excluded.extend(state.excluded_accounts().iter().cloned());
}
if let Some(required) = pinned_account.as_ref() {
for candidate in &candidates {
if candidate.account.id() != required {
@@ -685,26 +730,32 @@ impl CodexCredentialSelector {
&self,
account: &ProviderAccount,
failure: CodexAccountFailure,
message: Option<String>,
) -> Result<(), CredentialSelectionError> {
let now = SystemTime::now();
match failure {
CodexAccountFailure::CredentialExpired => {
self.apply_state(account, AccountAvailability::Expired, now)
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
.await
}
CodexAccountFailure::IdentityVerificationRequired => {
self.apply_state(account, AccountAvailability::Invalid, now)
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
.await
}
CodexAccountFailure::Banned => {
self.apply_state(account, AccountAvailability::Banned, now)
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
.await
}
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
CodexAccountFailure::QuotaExhausted
| CodexAccountFailure::UsageLimitExhausted { .. } => {
self.apply_state(account, AccountAvailability::QuotaExhausted, now)
.await
self.apply_state_with_message(
account,
AccountAvailability::QuotaExhausted,
now,
message,
)
.await
}
// 429:临时限流只写 quota,availability 不改变。
CodexAccountFailure::RateLimited { retry_after } => {
@@ -727,8 +778,13 @@ impl CodexCredentialSelector {
CodexAccountFailure::CloudflarePathBlocked => {
let blocked = self.record_cloudflare_path_block(account.id(), now);
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
self.apply_state(account, AccountAvailability::Invalid, now)
.await?;
self.apply_state_with_message(
account,
AccountAvailability::Invalid,
now,
message,
)
.await?;
}
self.apply_cookie_recovery(account, CookieRecovery::Clear)
.await?;
@@ -737,14 +793,15 @@ impl CodexCredentialSelector {
}
}
async fn apply_state(
async fn apply_state_with_message(
&self,
account: &ProviderAccount,
availability: AccountAvailability,
observed_at: SystemTime,
message: Option<String>,
) -> Result<(), CredentialSelectionError> {
self.repository
.apply_state(account, availability, observed_at)
.apply_state_with_message(account, availability, observed_at, message)
.await?;
Ok(())
}
@@ -111,6 +111,7 @@ pub async fn initialize(
http.clone(),
config.base_url().to_owned(),
Arc::clone(&agent_identity),
ports.cooldowns(),
));
let quota_skip_exhausted = config.quota_skip_exhausted();
let selector = Arc::new(CodexCredentialSelector::new(
@@ -1697,9 +1697,13 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
struct MappedProviderFailure {
error: ProviderError,
account_failure: Option<CodexAccountFailure>,
/// 原始上游错误描述,随 availability 状态一并持久化。
error_message: Option<String>,
cyber_policy_failure: bool,
set_cookie_headers: Vec<String>,
rate_limit_headers: Vec<(String, String)>,
/// 结构化 usage-limit 错误给出的窗口 reset(Unix 秒)。
confirmed_exhaustion_reset_at: Option<i64>,
observation: Option<ProviderResponseObservation>,
capture_response_cookies: bool,
}
@@ -1709,9 +1713,11 @@ impl MappedProviderFailure {
Self {
error,
account_failure: None,
error_message: None,
cyber_policy_failure: false,
set_cookie_headers: Vec::new(),
rate_limit_headers: Vec::new(),
confirmed_exhaustion_reset_at: None,
observation: None,
capture_response_cookies: false,
}
@@ -1869,6 +1875,18 @@ async fn apply_failure(
failure.account_failure,
Some(CodexAccountFailure::QuotaExhausted | CodexAccountFailure::UsageLimitExhausted { .. })
);
if needs_authoritative_quota_refresh
&& let Err(error) = context
.quota
.record_confirmed_exhaustion(account, failure.confirmed_exhaustion_reset_at)
.await
{
tracing::warn!(
account_id = %account.id(),
error = %error,
"Failed to project confirmed OpenAI quota exhaustion"
);
}
if failure.cyber_policy_failure {
context
.selector
@@ -1882,7 +1900,7 @@ async fn apply_failure(
.await;
if let Err(error) = context
.selector
.record_failure(account, account_failure)
.record_failure(account, account_failure, failure.error_message.clone())
.await
{
tracing::warn!(
@@ -2162,9 +2180,11 @@ fn map_upstream_failure(
MappedProviderFailure {
error,
account_failure: account_failure(category, failure.retry_after_seconds),
error_message: failure.client_message,
cyber_policy_failure,
set_cookie_headers: failure.set_cookie_headers,
rate_limit_headers: failure.rate_limit_headers,
confirmed_exhaustion_reset_at: failure.usage_limit_resets_at,
observation,
capture_response_cookies: !matches!(
category,
@@ -87,8 +87,13 @@ impl CodexBackendClient {
let headers = self.request_headers_for_http_response(upstream_request, context)?;
let headers_started_at = Instant::now();
// 与官方 Codex app-server 一致:/v1/responses 请求体默认 zstd 压缩。
// Codex 上游只交付 SSE;即使下游请求 `stream: false`,也要上游流式执行,
// 再由 API 层收集 canonical events 并返回完整 JSON。不能把下游的传输偏好
// 直接透传给 Codex,否则上游会以 400 拒绝非流式请求。
let mut upstream_body = upstream_request.body().clone();
upstream_body.insert("stream".to_owned(), serde_json::Value::Bool(true));
let body =
serde_json::to_vec(&upstream_request).map_err(CodexClientError::RequestBodyEncode)?;
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
.map_err(CodexClientError::RequestCompression)?;
let response = self
@@ -3,6 +3,7 @@
use super::client::CodexClientVisibleUpstreamResponse;
use super::protocol::responses::ResponsesSseFailure;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use chrono::{DateTime, Utc};
use reqwest::{StatusCode, header::HeaderMap};
use serde_json::Value;
@@ -113,6 +114,8 @@ pub struct CodexUpstreamFailure {
pub(crate) client_error_type: Option<String>,
pub(crate) client_response: Option<Box<CodexClientVisibleUpstreamResponse>>,
pub(crate) identity_error_code: Option<String>,
/// 官方 `usage_limit_reached` 错误体携带的窗口重置时间(Unix 秒)。
pub(crate) usage_limit_resets_at: Option<i64>,
pub(crate) retry_after_seconds: Option<u64>,
pub(crate) request_id: Option<String>,
pub(crate) set_cookie_headers: Vec<String>,
@@ -139,6 +142,7 @@ impl CodexUpstreamFailure {
) -> Self {
let fields = ParsedUpstreamError::from_body(body);
let category = classify_upstream_failure(
UpstreamFailureSource::HttpResponse,
Some(status),
body,
&fields,
@@ -153,6 +157,9 @@ impl CodexUpstreamFailure {
client_error_type: fields.error_type,
client_response: client_response.cloned().map(Box::new),
identity_error_code: diagnostics.identity_error_code.clone(),
usage_limit_resets_at: (category == CodexFailureCategory::UsageLimitExhausted)
.then_some(fields.resets_at)
.flatten(),
retry_after_seconds,
request_id: diagnostics.request_id.clone(),
set_cookie_headers: set_cookie_headers.to_vec(),
@@ -174,11 +181,13 @@ impl CodexUpstreamFailure {
error_type: failure.upstream_type.clone(),
message: failure.message.clone(),
client_message: Some(failure.message.clone()),
resets_at: None,
};
let status = failure
.explicit_status_code
.and_then(|code| StatusCode::from_u16(code).ok());
let category = classify_upstream_failure(
UpstreamFailureSource::SseFailure,
status,
"",
&fields,
@@ -193,6 +202,7 @@ impl CodexUpstreamFailure {
client_error_type: fields.error_type,
client_response: None,
identity_error_code: diagnostics.identity_error_code.clone(),
usage_limit_resets_at: None,
retry_after_seconds: failure.retry_after_seconds,
request_id: diagnostics.request_id.clone(),
set_cookie_headers: set_cookie_headers.to_vec(),
@@ -246,6 +256,7 @@ struct ParsedUpstreamError {
error_type: Option<String>,
message: String,
client_message: Option<String>,
resets_at: Option<i64>,
}
impl ParsedUpstreamError {
@@ -256,6 +267,7 @@ impl ParsedUpstreamError {
error_type: None,
message: body.to_owned(),
client_message: None,
resets_at: None,
};
};
let error = value
@@ -282,15 +294,26 @@ impl ParsedUpstreamError {
.clone()
.or_else(|| error.as_str().and_then(non_empty_owned))
.unwrap_or_else(|| body.to_owned());
let resets_at = error
.get("resets_at")
.and_then(Value::as_i64)
.filter(|seconds| DateTime::<Utc>::from_timestamp(*seconds, 0).is_some());
Self {
code,
error_type,
message,
client_message,
resets_at,
}
}
}
#[derive(Clone, Copy)]
enum UpstreamFailureSource {
HttpResponse,
SseFailure,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CodexUpstreamDiagnostics {
pub status_code: Option<u16>,
@@ -363,6 +386,7 @@ fn decode_identity_error_code(encoded: &str) -> Option<String> {
}
fn classify_upstream_failure(
source: UpstreamFailureSource,
status: Option<StatusCode>,
body: &str,
fields: &ParsedUpstreamError,
@@ -376,6 +400,19 @@ fn classify_upstream_failure(
let message = fields.message.to_ascii_lowercase();
let body = body.to_ascii_lowercase();
// 与官方 Codex HTTP/WS 路径一致:429 只有结构化
// `error.type=usage_limit_reached` 才能确认额度窗口耗尽;其余 429 都是临时限流。
// SSE `response.failed` 的结构化字段形态不同,保留其 code/type 语义单独分类。
if matches!(source, UpstreamFailureSource::HttpResponse)
&& status == Some(StatusCode::TOO_MANY_REQUESTS)
{
return if error_type == "usage_limit_reached" {
CodexFailureCategory::UsageLimitExhausted
} else {
CodexFailureCategory::RateLimited
};
}
if [code.as_str(), message.as_str(), body.as_str()]
.into_iter()
.any(is_model_unsupported)
@@ -606,6 +606,26 @@ impl CodexResponsesRequest {
)
}
/// 返回会话亲和所用的子代理区分值。
///
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;网关的
/// `/responses/review` 等入口则通过 `client_metadata.x-openai-subagent`
/// 传递给上游。该可选值仅参与会话亲和键派生。
pub fn subagent_kind(&self) -> Option<String> {
self.semantics()
.subagent_kind
.filter(|value| !value.trim().is_empty())
.or_else(|| {
self.client_metadata()
.and_then(Value::as_object)
.and_then(|metadata| metadata.get("x-openai-subagent"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_owned)
})
}
/// 设置 / 合并 client metadata。
pub fn set_client_metadata(&mut self, client_metadata: Option<Value>) {
match client_metadata {
+47 -5
View File
@@ -428,7 +428,8 @@ async fn openai_admin_provider_projects_cached_quota_models_and_canonical_export
}
#[tokio::test]
async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_quota() {
async fn openai_admin_provider_projects_official_codex_quota_and_independent_buckets_with_chinese_labels()
{
let store = Arc::new(MemoryAccountStore::default());
store
.seed_oauth_credential(ImportCodexOAuthCredential {
@@ -444,11 +445,15 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
.account("acct_admin_canonical_quota")
.expect("stored account");
let raw = json!({
"active_limit": "premium",
"rate_limit": {
"primary_window": {
"used_percent": 91,
"reset_at": 1_900_000_000,
"limit_window_seconds": 2_592_000
},
"secondary_window": {
"used_percent": 88
}
},
"additional_rate_limits": [{
@@ -459,6 +464,19 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
"used_percent": 2,
"reset_at": 1_900_000_000,
"limit_window_seconds": 2_592_000
},
"secondary_window": {
"used_percent": 0
}
}
}, {
"limit_name": "code_review",
"metered_feature": "code_review",
"rate_limit": {
"primary_window": {
"used_percent": 12,
"reset_at": 1_900_000_000,
"limit_window_seconds": 604_800
}
}
}]
@@ -499,10 +517,31 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
.collect::<Vec<_>>();
assert_eq!(monthly.len(), 1);
// limit_name 优先显示,不再猜成「月限额」。
assert_eq!(monthly[0].label, "custom codex label");
assert_eq!(monthly[0].source.as_deref(), Some("core"));
assert_eq!(monthly[0].used_percent, Some(2.0));
assert!(monthly.iter().any(|window| {
window.label == "月额度"
&& window.source.as_deref() == Some("codex")
&& window.used_percent == Some(91.0)
}));
assert!(
!quota
.windows
.iter()
.any(|window| window.key.starts_with("additional-0-codex")),
"the additional codex alias should not become a second display bucket"
);
let secondary = quota
.windows
.iter()
.find(|window| window.key == "core-secondary")
.expect("core secondary quota");
assert_eq!(secondary.label, "次级额度");
assert_eq!(secondary.used_percent, Some(88.0));
let review = quota
.windows
.iter()
.find(|window| window.source.as_deref() == Some("code_review"))
.expect("code review quota");
assert_eq!(review.label, "代码审查 · 周额度");
}
#[tokio::test]
@@ -544,6 +583,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
.expect("persist raw quota");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: CoreAccountAvailability::QuotaExhausted,
@@ -584,6 +624,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: CoreAccountAvailability::Ready,
@@ -711,6 +752,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
enabled: account.enabled(),
availability: AdminAccountAvailability::Ready,
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
@@ -4,15 +4,21 @@ use std::sync::Arc;
use std::time::{Duration, SystemTime};
use futures::executor::block_on;
use gateway_core::engine::continuation::{
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
};
use gateway_core::engine::credential::{
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy,
AccountStateChange, ProviderAccountId, ProviderAccountStore as _, RotationStrategy,
AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _,
QuotaObservation, QuotaWriteOutcome, RotationStrategy,
};
use gateway_core::engine::{
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
};
use gateway_core::policy::ClientApiKeyId;
use gateway_core::provider_ports::{ProviderSessionAffinityKey, ProviderSessionAffinityPort};
use gateway_core::provider_ports::{
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
};
use gateway_core::routing::ProviderKind;
use provider_openai::OFFICIAL_CODEX_BASE_URL;
use provider_openai::credential::{
@@ -22,11 +28,12 @@ use provider_openai::credential::{
};
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
use secrecy::ExposeSecret;
use serde_json::json;
use url::Url;
use crate::support::{
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
account_policy, agent_identity_service, catalog_cache, profile, secret,
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
TestLeaseCoordinator, account_policy, agent_identity_service, catalog_cache, profile, secret,
};
fn create_account(store: &Arc<MemoryAccountStore>, id: &str, token: &str) {
@@ -99,6 +106,7 @@ fn selector_with_affinity(
session_affinity,
Arc::new(AccountFeedbackStats::default()),
true,
Arc::new(MemoryCooldownPort::new()),
)
}
@@ -108,6 +116,7 @@ fn selector_with_runtime(
session_affinity: Arc<MemorySessionAffinity>,
account_feedback: Arc<AccountFeedbackStats>,
skip_exhausted: bool,
cooldowns: Arc<dyn ProviderCooldownPort>,
) -> CodexCredentialSelector {
let profile = CodexWireProfileState::new(CodexWireProfile {
originator: "codex_cli_rs".to_owned(),
@@ -136,6 +145,7 @@ fn selector_with_runtime(
http,
OFFICIAL_CODEX_BASE_URL.to_owned(),
Arc::clone(&agent_identity),
cooldowns,
));
CodexCredentialSelector::new(
ProviderKind::new("openai").expect("provider"),
@@ -464,6 +474,7 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
affinity,
Arc::clone(&account_feedback),
true,
Arc::new(MemoryCooldownPort::new()),
);
for _ in 0..4 {
account_feedback.report(
@@ -507,6 +518,7 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
let first = store.account("acct_first").expect("first account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: first.id().clone(),
expected_revision: first.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -663,8 +675,12 @@ fn credential_expired_failure_marks_unified_account_expired() {
}),
)
.expect("select account");
block_on(selector.record_failure(lease.account(), CodexAccountFailure::CredentialExpired))
.expect("record credential expiry");
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CredentialExpired,
None,
))
.expect("record credential expiry");
assert_eq!(
store
.account("acct_primary")
@@ -678,7 +694,15 @@ fn credential_expired_failure_marks_unified_account_expired() {
fn rate_limited_failure_records_quota_window_without_changing_availability() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
let cooldowns = Arc::new(MemoryCooldownPort::new());
let selector = selector_with_runtime(
&store,
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
true,
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
);
let attempt = attempt(BTreeSet::new());
let lease =
block_on(
@@ -697,36 +721,22 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
// 429 不改变 availability(限流不改变账号可用性),只把限流窗口合成进 quota JSON。
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
let account = store.account("acct_primary").expect("account");
assert_eq!(account.availability(), AccountAvailability::Ready);
let quota = store.quota_json("acct_primary").expect("quota");
assert_eq!(
quota
.pointer("/rate_limit/limit_reached")
.and_then(serde_json::Value::as_bool),
Some(true)
);
assert_eq!(
quota
.pointer("/rate_limit/primary_window/used_percent")
.and_then(serde_json::Value::as_f64),
Some(100.0)
);
assert!(
quota
.pointer("/rate_limit/primary_window/reset_at")
.and_then(serde_json::Value::as_i64)
.is_some_and(|reset_at| {
reset_at
> SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock")
.as_secs() as i64
})
store.quota_json("acct_primary").is_none(),
"429 must not synthesize quota window"
);
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
assert!(
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
"429 must record a Redis cooldown expiring in the future"
);
}
@@ -743,6 +753,7 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
CodexAccountFailure::UsageLimitExhausted {
retry_after: Some(Duration::from_secs(30)),
},
None,
)
.await
.expect("record usage-limit exhaustion");
@@ -758,6 +769,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -771,6 +783,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -789,13 +802,22 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
let cooldowns = Arc::new(MemoryCooldownPort::new());
let selector = selector_with_runtime(
&store,
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
true,
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
);
block_on(selector.record_failure(
&account,
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -806,8 +828,13 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
.availability(),
AccountAvailability::Ready
);
// 429 合成限流窗口需要读写 quota JSON。
assert!(store.quota_reads() >= 1);
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
assert_eq!(store.quota_reads(), 0);
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
assert!(
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
"429 must record a Redis cooldown"
);
}
#[test]
@@ -816,6 +843,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Invalid,
@@ -830,6 +858,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
))
.expect("record rate-limit failure");
@@ -854,6 +883,7 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
create_account(&store, "acct_primary", "at-primary");
let account = store.account("acct_primary").expect("account");
block_on(store.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: stale,
@@ -892,12 +922,14 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
),
selector.record_failure(
&second,
CodexAccountFailure::RateLimited {
retry_after: Some(Duration::from_secs(30)),
},
None,
),
);
first_result.expect("record first account failure");
@@ -925,7 +957,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
create_account(&store, "acct_primary", "at-primary");
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
let account = store.account("acct_primary").expect("account");
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted))
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
.expect("mark account exhausted");
let selector = selector_with_runtime(
@@ -934,6 +966,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
false,
Arc::new(MemoryCooldownPort::new()),
);
let request_url =
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
@@ -957,6 +990,131 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
);
}
#[test]
fn native_continuation_surfaces_the_original_accounts_quota_status_to_the_coordinator() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_original", "at-original");
create_account(&store, "acct_fallback", "at-fallback");
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
let original = store.account("acct_original").expect("original account");
block_on(strict_selector.record_failure(&original, CodexAccountFailure::QuotaExhausted, None))
.expect("mark original account exhausted");
let selector = selector_with_runtime(
&store,
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
false,
Arc::new(MemoryCooldownPort::new()),
);
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-response"),
PreviousResponseId::new("upstream-response"),
ProviderKind::new("openai").expect("provider"),
original.id().clone(),
);
let attempt = AttemptContext::new(
RequestAttemptContext::new(
ModelRequestId::new("req_native_continuation").expect("request id"),
ClientApiKeyId::new("key_codex_contract").expect("client key id"),
),
NonZeroU32::new(1).expect("attempt"),
SystemTime::now() + Duration::from_secs(30),
account_policy(),
AccountAttemptContext::new(BTreeSet::new(), None, None),
Some(ContinuationBinding::Pinned(continuation)),
CancellationToken::new(),
);
let error =
block_on(
selector.select(&SelectCodexCredential {
upstream_model: "gpt-5.4",
request_url: &Url::parse("https://chatgpt.com/backend-api/codex/responses")
.expect("request URL"),
attempt: &attempt,
session_affinity_key: None,
}),
)
.expect_err("the selector must surface the unavailable native account");
assert!(matches!(
error,
CredentialSelectionError::NoEligibleCredential
));
}
#[test]
fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordinator() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_original_signal", "at-original-signal");
create_account(&store, "acct_fallback_signal", "at-fallback-signal");
let original = store
.account("acct_original_signal")
.expect("original account");
let quota = json!({
"rate_limit": {
"allowed": false,
"limit_reached": true,
"primary_window": {"used_percent": 98}
}
});
let outcome = block_on(store.compare_and_swap_quota(QuotaObservation {
account_id: original.id().clone(),
expected_revision: original.revision(),
quota: Some(OpaqueProviderData::new(
quota.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
}))
.expect("persist quota signal");
assert!(matches!(outcome, QuotaWriteOutcome::Updated));
let selector = selector_with_runtime(
&store,
Arc::new(TestLeaseCoordinator::default()),
Arc::new(MemorySessionAffinity::default()),
Arc::new(AccountFeedbackStats::default()),
false,
Arc::new(MemoryCooldownPort::new()),
);
let continuation = NativeContinuationPin::new(
PreviousResponseId::new("previous-response"),
PreviousResponseId::new("upstream-response"),
ProviderKind::new("openai").expect("provider"),
original.id().clone(),
);
let attempt = AttemptContext::new(
RequestAttemptContext::new(
ModelRequestId::new("req_native_continuation_signal").expect("request id"),
ClientApiKeyId::new("key_codex_contract").expect("client key id"),
),
NonZeroU32::new(1).expect("attempt"),
SystemTime::now() + Duration::from_secs(30),
account_policy(),
AccountAttemptContext::new(BTreeSet::new(), None, None),
Some(ContinuationBinding::Pinned(continuation)),
CancellationToken::new(),
);
let error =
block_on(
selector.select(&SelectCodexCredential {
upstream_model: "gpt-5.4",
request_url: &Url::parse("https://chatgpt.com/backend-api/codex/responses")
.expect("request URL"),
attempt: &attempt,
session_affinity_key: None,
}),
)
.expect_err("the selector must surface the quota-limited native account");
assert!(matches!(
error,
CredentialSelectionError::NoEligibleCredential
));
}
#[test]
fn identity_verification_failure_isolates_only_selected_account() {
let store = Arc::new(MemoryAccountStore::default());
@@ -979,6 +1137,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::IdentityVerificationRequired,
None,
))
.expect("record identity verification failure");
@@ -1022,6 +1181,7 @@ fn cloudflare_challenge_does_not_change_availability() {
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CloudflareChallenge { retry_after: None },
None,
))
.expect("record challenge");
@@ -1058,9 +1218,11 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
.expect("select account");
for _ in 0..3 {
block_on(
selector.record_failure(lease.account(), CodexAccountFailure::CloudflarePathBlocked),
)
block_on(selector.record_failure(
lease.account(),
CodexAccountFailure::CloudflarePathBlocked,
None,
))
.expect("record path block");
}
@@ -1114,6 +1276,7 @@ fn cloudflare_challenge_expires_provider_owned_cookies_at_cooldown_boundary() {
block_on(selector.record_failure(
second.account(),
CodexAccountFailure::CloudflareChallenge { retry_after: None },
None,
))
.expect("record challenge");
@@ -1157,8 +1320,12 @@ fn cloudflare_path_block_deletes_provider_owned_cookies() {
session_affinity_key: None,
}))
.expect("select revised account");
block_on(selector.record_failure(second.account(), CodexAccountFailure::CloudflarePathBlocked))
.expect("record path block");
block_on(selector.record_failure(
second.account(),
CodexAccountFailure::CloudflarePathBlocked,
None,
))
.expect("record path block");
let account = store.account("acct_primary").expect("account");
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
@@ -1194,7 +1361,7 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
assert_eq!(outcome.credential_revision, Some(current.revision().get()));
assert_ne!(current.revision(), lease.account().revision());
block_on(selector.record_failure(&current, CodexAccountFailure::QuotaExhausted))
block_on(selector.record_failure(&current, CodexAccountFailure::QuotaExhausted, None))
.expect("record failure with current revision");
assert_eq!(
store
@@ -71,6 +71,7 @@ fn quota_service_with_base_url(
http,
base_url,
crate::support::agent_identity_service(store),
Arc::new(crate::support::MemoryCooldownPort::new()),
)
}
@@ -193,6 +194,40 @@ async fn passive_full_percent_with_explicit_allowance_keeps_account_ready() {
);
}
#[tokio::test]
async fn passive_headers_should_materialize_false_when_quota_is_not_exhausted() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_passive_quota_available").await;
let account = store
.account("acct_passive_quota_available")
.expect("account");
let service = quota_service(&store);
let headers = vec![
("x-codex-active-limit".to_owned(), "codex".to_owned()),
("x-codex-primary-used-percent".to_owned(), "31".to_owned()),
("x-codex-allowed".to_owned(), "true".to_owned()),
("x-codex-limit-reached".to_owned(), "false".to_owned()),
];
assert!(
service
.synchronize_passive_headers(&account, &headers)
.await
.expect("passive quota")
);
let observations = store
.get_quotas(std::slice::from_ref(account.id()))
.await
.expect("read quota observation");
assert_eq!(
observations
.first()
.and_then(|observation| observation.limit_reached),
Some(false)
);
}
#[tokio::test]
async fn manual_quota_refresh_preserves_disabled_account_state() {
let store = Arc::new(MemoryAccountStore::default());
@@ -201,6 +236,7 @@ async fn manual_quota_refresh_preserves_disabled_account_state() {
let account = store.account(account_id).expect("created disabled account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Expired,
@@ -254,6 +290,7 @@ async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential(
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -397,6 +434,130 @@ async fn passive_rate_limit_headers_replace_stale_secondary_window() {
assert_eq!(monthly[0].used_percent(), Some(92.0));
}
#[tokio::test]
async fn passive_active_limit_keeps_the_top_level_codex_bucket_canonical() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_passive_active_limit").await;
let account = store.account("acct_passive_active_limit").expect("account");
let baseline = json!({
"additional_rate_limits": [{
"metered_feature": "codex",
"rate_limit": {
"allowed": true,
"primary_window": {
"used_percent": 20,
"reset_at": 1_900_000_000,
"limit_window_seconds": 604_800
}
}
}]
});
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
baseline.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
})
.await
.expect("persist baseline quota");
let service = quota_service(&store);
let headers = vec![
("x-codex-active-limit".to_owned(), "premium".to_owned()),
("x-premium-primary-used-percent".to_owned(), "55".to_owned()),
(
"x-premium-primary-window-minutes".to_owned(),
"10080".to_owned(),
),
(
"x-premium-primary-reset-at".to_owned(),
"1900000000".to_owned(),
),
];
assert!(
service
.synchronize_passive_headers(&account, &headers)
.await
.expect("passive quota")
);
let snapshot = service
.read_account(account.id())
.await
.expect("read quota")
.expect("quota snapshot");
assert_eq!(snapshot.windows().len(), 1);
assert!(
snapshot
.windows()
.iter()
.any(|window| { window.source() == "codex" && window.used_percent() == Some(55.0) })
);
}
#[tokio::test]
async fn passive_metadata_headers_preserve_quota_observation_time() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_passive_metadata").await;
let account = store.account("acct_passive_metadata").expect("account");
let baseline = json!({
"rate_limit": {
"allowed": true,
"primary_window": {
"used_percent": 20,
"reset_at": 1_900_000_000,
"limit_window_seconds": 604_800
}
}
});
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
baseline.as_object().expect("quota object").clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: None,
})
.await
.expect("persist baseline quota");
let before = store
.get_quotas(std::slice::from_ref(account.id()))
.await
.expect("read baseline quota");
let service = quota_service(&store);
let headers = vec![
("x-codex-plan-type".to_owned(), "free".to_owned()),
("x-codex-credits-has-credits".to_owned(), "true".to_owned()),
("x-codex-credits-unlimited".to_owned(), "false".to_owned()),
];
assert!(
service
.synchronize_passive_headers(&account, &headers)
.await
.expect("metadata-only passive response")
);
let after = store
.get_quotas(std::slice::from_ref(account.id()))
.await
.expect("read updated quota metadata");
assert_eq!(after[0].observed_at, before[0].observed_at);
assert_eq!(after[0].limit_reached, before[0].limit_reached);
assert_eq!(
store
.quota_json("acct_passive_metadata")
.and_then(|quota| quota.get("plan_type").cloned()),
Some(json!("free"))
);
}
#[tokio::test]
async fn synchronize_without_accounts_is_a_noop_before_network_io() {
let store = Arc::new(MemoryAccountStore::default());
@@ -460,6 +621,7 @@ async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks(
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Ready,
@@ -526,6 +688,7 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -565,6 +728,7 @@ async fn periodic_quota_synchronization_respects_retry_after_on_service_unavaila
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -605,6 +769,7 @@ async fn periodic_quota_synchronization_skips_ready_accounts_without_quota_reach
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Ready,
@@ -637,6 +802,7 @@ async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() {
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -662,6 +828,7 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -727,6 +894,7 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
.expect("seed post-failure quota snapshot");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -798,7 +966,7 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
.await
.expect("lower percent quota recheck");
assert_eq!(lower_percent.updated, 1);
assert_eq!(lower_percent.exhausted, 1);
assert_eq!(
store
.account(account_id)
@@ -807,6 +975,12 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
AccountAvailability::QuotaExhausted,
"a provider percentage change is not sufficient to release a confirmed exhaustion"
);
let snapshot = next_cycle
.read_account(account.id())
.await
.expect("read held exhaustion projection")
.expect("held exhaustion projection");
assert_eq!(snapshot.fact().remaining_percent(), Some(0));
}
#[tokio::test]
@@ -815,8 +989,31 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
let account_id = "acct_periodic_reset_elapsed";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("created account");
let elapsed_reset = Utc::now().timestamp() - 60;
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
json!({
"rate_limit": {
"allowed": false,
"limit_reached": true,
"primary_window": {"used_percent": 100, "reset_at": elapsed_reset}
}
})
.as_object()
.expect("recorded exhausted quota")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: Some(true),
})
.await
.expect("record elapsed exhausted reset");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -825,14 +1022,14 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
.await
.expect("mark account exhausted with an elapsed reset");
let server = MockServer::start().await;
let elapsed_reset = Utc::now().timestamp() - 1;
let next_reset = Utc::now().timestamp() + 3600;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"rate_limit": {
"allowed": true,
"limit_reached": false,
"primary_window": {"used_percent": 1, "reset_at": elapsed_reset}
"primary_window": {"used_percent": 1, "reset_at": next_reset}
}
})))
.expect(1)
@@ -866,6 +1063,7 @@ async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() {
.expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -889,6 +1087,7 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -943,6 +1142,7 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -951,7 +1151,33 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
.await
.expect("mark account exhausted");
}
let good = store
.account("acct_quota_batch_good")
.expect("good account");
let elapsed_reset = Utc::now().timestamp() - 60;
store
.compare_and_swap_quota(QuotaObservation {
account_id: good.id().clone(),
expected_revision: good.revision(),
quota: Some(OpaqueProviderData::new(
json!({
"rate_limit": {
"allowed": false,
"limit_reached": true,
"primary_window": {"used_percent": 100, "reset_at": elapsed_reset}
}
})
.as_object()
.expect("recorded exhausted quota")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: Some(true),
})
.await
.expect("seed good account old reset");
let server = MockServer::start().await;
let next_reset = Utc::now().timestamp() + 3600;
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.and(header("authorization", "Bearer token-acct_quota_batch_bad"))
@@ -970,7 +1196,7 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"rate_limit": {
"allowed": true,
"primary_window": {"used_percent": 25.0, "reset_at": 1_900_000_000}
"primary_window": {"used_percent": 25.0, "reset_at": next_reset}
}
})))
.mount(&server)
@@ -12,6 +12,7 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
let account = store.account(account_id).expect("created account");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::Expired,
@@ -161,6 +162,7 @@ async fn expired_old_reset_with_fresh_allowed_snapshot_recovers_ready() {
.expect("seed old exhausted snapshot");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: account.id().clone(),
expected_revision: account.revision(),
availability: AccountAvailability::QuotaExhausted,
@@ -305,7 +305,7 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
}
#[tokio::test]
async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
async fn persisted_codex_alias_keeps_the_top_level_rate_limit_canonical() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_canonical_codex_limit").await;
let account = store
@@ -323,7 +323,7 @@ async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
"metered_feature": "codex",
"rate_limit": {
"primary_window": {
"used_percent": 2,
"used_percent": 98,
"reset_at": 1_900_000_000,
"limit_window_seconds": 2_592_000
}
@@ -355,10 +355,13 @@ async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
.filter(|window| window.kind() == CodexQuotaWindowKind::Monthly)
.collect::<Vec<_>>();
assert_eq!(snapshot.fact().remaining_percent(), Some(98));
assert_eq!(snapshot.fact().remaining_percent(), Some(9));
assert_eq!(monthly.len(), 1);
assert_eq!(monthly[0].source(), "core");
assert_eq!(monthly[0].used_percent(), Some(2.0));
assert!(
monthly
.iter()
.any(|window| { window.source() == "codex" && window.used_percent() == Some(91.0) })
);
}
#[tokio::test]
@@ -419,16 +422,21 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() {
.expect("read quota")
.expect("quota snapshot");
// spend_control 不再生成窗口(只作 exhaustion 信号),
// 顶层 code_review_rate_limit 与 additional 的 code_review 桶同源替换后保留一个。
// spend_control 不再生成窗口(只作 exhaustion 信号);顶层和 additional
// 的 code_review 桶分别保留,避免一个快照覆盖另一个。
let review = snapshot
.windows()
.iter()
.filter(|window| window.source() == "code_review")
.collect::<Vec<_>>();
assert_eq!(review.len(), 1);
assert_eq!(review[0].limit_name(), Some("code review"));
assert_eq!(review[0].used_percent(), Some(55.0));
assert_eq!(snapshot.windows().len(), 2);
assert_eq!(review.len(), 2);
assert!(review.iter().any(|window| {
window.limit_name() == Some("code_review") && window.used_percent() == Some(80.0)
}));
assert!(review.iter().any(|window| {
window.limit_name() == Some("code review") && window.used_percent() == Some(55.0)
}));
assert_ne!(review[0].key(), review[1].key());
assert_eq!(snapshot.windows().len(), 3);
assert!(!snapshot.fact().exhausted());
}
@@ -43,8 +43,9 @@ use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use crate::support::{
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
account_policy, agent_identity_service_with_pool, catalog_cache, profile, secret,
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
TestLeaseCoordinator, account_policy, agent_identity_service_with_pool, catalog_cache, profile,
secret,
};
use crate::transport::accept_codex_test_websocket;
@@ -115,6 +116,21 @@ fn provider_with_affinity_and_base_url_and_leases(
base_url: String,
leases: Arc<TestLeaseCoordinator>,
) -> CodexProvider {
provider_and_quota_with_affinity_and_base_url_and_leases(
store,
session_affinity,
base_url,
leases,
)
.0
}
fn provider_and_quota_with_affinity_and_base_url_and_leases(
store: &Arc<MemoryAccountStore>,
session_affinity: Arc<MemorySessionAffinity>,
base_url: String,
leases: Arc<TestLeaseCoordinator>,
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
let profile = wire_profile();
let http = reqwest::Client::builder()
.no_proxy()
@@ -136,6 +152,7 @@ fn provider_with_affinity_and_base_url_and_leases(
http.clone(),
base_url.clone(),
Arc::clone(&agent_identity),
Arc::new(MemoryCooldownPort::new()),
));
let account_feedback = Arc::new(AccountFeedbackStats::default());
let selector = Arc::new(CodexCredentialSelector::new(
@@ -152,10 +169,10 @@ fn provider_with_affinity_and_base_url_and_leases(
true,
));
CodexProvider::new(
let provider = CodexProvider::new(
selector,
catalog,
quota,
Arc::clone(&quota),
agent_identity,
account_feedback,
http,
@@ -163,7 +180,8 @@ fn provider_with_affinity_and_base_url_and_leases(
base_url,
websocket_pool,
)
.expect("official OpenAI provider")
.expect("official OpenAI provider");
(provider, quota)
}
async fn create_account(store: &Arc<MemoryAccountStore>, id: &str) {
@@ -1053,6 +1071,66 @@ async fn prompt_cache_key_should_become_an_opaque_session_affinity_lookup_key()
assert!(keys[0].bytes().all(|byte| byte.is_ascii_hexdigit()));
}
#[tokio::test]
async fn subagent_requests_should_use_an_affinity_key_isolated_from_the_root_session() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_subagent").await;
let affinity = Arc::new(MemorySessionAffinity::default());
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("content-type", "text/event-stream")
.set_body_string(CAPTURE_COMPLETED_SSE),
)
.mount(&server)
.await;
let provider = provider_with_affinity_and_base_url(&store, Arc::clone(&affinity), server.uri());
for (request_id, subagent_kind) in [
("req_root_affinity", None),
("req_subagent_affinity_first", Some("review")),
("req_subagent_affinity_second", Some("review")),
] {
let mut body = Map::from_iter([
("model".to_owned(), json!("gpt-5.4")),
("input".to_owned(), json!("new task")),
("prompt_cache_key".to_owned(), json!("root-session-key")),
]);
if let Some(subagent_kind) = subagent_kind {
body.insert(
"client_metadata".to_owned(),
json!({"x-openai-subagent": subagent_kind}),
);
}
let generation = GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object("openai", body)
.expect("OpenAI payload")
.with_context(Map::from_iter([("use_websocket".to_owned(), json!(false))])),
);
let mut stream = provider
.execute(
planned_request("openai", Operation::Generate(generation)),
context(request_id, CancellationToken::new()),
)
.await
.expect("prepare subagent provider stream");
while let Some(event) = stream.next().await {
event.expect("subagent response");
}
drop(stream);
}
let keys = affinity.lookup_keys();
assert_eq!(keys.len(), 3);
assert_ne!(keys[0], keys[1], "subagent must not inherit the root key");
assert_eq!(
keys[1], keys[2],
"the same subagent should retain its own key"
);
}
#[tokio::test]
async fn explicit_session_id_should_override_turn_specific_prompt_cache_keys_for_affinity() {
let store = Arc::new(MemoryAccountStore::default());
@@ -1271,9 +1349,10 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc
}
#[tokio::test]
async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_quota_snapshot() {
async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset_advances() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_usage_limit_request_path";
let reset_at = 1_900_000_000;
create_account(&store, account_id).await;
let account = store.account(account_id).expect("created account");
store
@@ -1285,7 +1364,7 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
"rate_limit": {
"allowed": true,
"limit_reached": false,
"primary_window": {"used_percent": 99, "reset_at": 1_900_000_000}
"primary_window": {"used_percent": 99, "reset_at": reset_at}
}
})
.as_object()
@@ -1301,8 +1380,8 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
Mock::given(method("GET"))
.and(path("/api/codex/usage"))
.and(header("authorization", format!("Bearer at-{account_id}")))
// 生产环境会短暂返回互相矛盾的快照:数字已到 100%,但布尔位仍声称可用。
// 后台同步应覆盖展示数据,不能据此撤销真实请求已经确认的额度耗尽。
// 生产环境会短暂返回互相矛盾的快照:98/99% 且布尔位仍声称可用。
// 后台同步不能覆盖真实请求已确认的额度耗尽。
.respond_with(
ResponseTemplate::new(200)
// 验证后台 usage 同步不能把原始的额度错误响应拖到查询完成之后。
@@ -1311,7 +1390,7 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
"rate_limit": {
"allowed": true,
"limit_reached": false,
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
"primary_window": {"used_percent": 99, "reset_at": reset_at}
}
})),
)
@@ -1320,16 +1399,13 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
.await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("content-type", "text/event-stream")
.set_body_string(concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_usage_limit_confirm\",\"model\":\"gpt-5.4\",\"status\":\"in_progress\"}}\n\n",
"event: response.failed\n",
"data: {\"type\":\"response.failed\",\"status_code\":429,\"retry_after_seconds\":86400,\"response\":{\"id\":\"resp_usage_limit_confirm\",\"status\":\"failed\",\"error\":{\"code\":\"usage_limit_reached\",\"message\":\"usage limit reached\"}}}\n\n"
)),
)
.respond_with(ResponseTemplate::new(429).set_body_json(json!({
"error": {
"type": "usage_limit_reached",
"message": "usage limit reached",
"resets_at": reset_at
}
})))
.mount(&server)
.await;
@@ -1355,6 +1431,24 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted);
let account = store.account(account_id).expect("usage-limit account");
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
let projected = store
.get_quotas(&[account.id().clone()])
.await
.expect("read immediate quota projection")
.into_iter()
.next()
.expect("confirmed quota observation");
let projected_observed_at = projected.observed_at.expect("projection observed at");
let projected = projected
.quota
.map(OpaqueProviderData::into_inner)
.expect("confirmed quota projection");
assert_eq!(
Value::Object(projected)
.pointer("/rate_limit/primary_window/used_percent")
.and_then(Value::as_u64),
Some(100)
);
tokio::time::sleep(Duration::from_millis(200)).await;
let requests = server.received_requests().await.expect("received requests");
assert_eq!(
@@ -1366,35 +1460,45 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
"usage refresh must wait for the upstream quota settlement delay"
);
let quota = timeout(Duration::from_secs(5), async {
timeout(Duration::from_secs(5), async {
loop {
let quota = store
let observed_at = store
.get_quotas(&[account.id().clone()])
.await
.expect("read refreshed quota")
.expect("read refreshed quota observation")
.into_iter()
.next()
.and_then(|observation| observation.quota)
.expect("authoritative quota snapshot");
let quota = Value::Object(quota.into_inner());
if quota
.pointer("/rate_limit/primary_window/used_percent")
.and_then(Value::as_u64)
== Some(100)
{
break quota;
.and_then(|observation| observation.observed_at);
if observed_at.is_some_and(|observed_at| observed_at > projected_observed_at) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("background usage refresh must replace the stale quota snapshot");
.expect("background usage refresh must run");
let quota = store
.get_quotas(&[account.id().clone()])
.await
.expect("read refreshed quota")
.into_iter()
.next()
.and_then(|observation| observation.quota)
.map(OpaqueProviderData::into_inner)
.map(Value::Object)
.expect("authoritative quota projection");
assert_eq!(
quota
.pointer("/rate_limit/primary_window/used_percent")
.and_then(Value::as_u64),
Some(100)
);
assert_eq!(
quota
.pointer("/rate_limit/primary_window/reset_at")
.and_then(Value::as_i64),
Some(reset_at)
);
let requests = server.received_requests().await.expect("received requests");
assert_eq!(
requests
@@ -1849,6 +1953,85 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
assert_eq!(affinity.binding_count(), 0);
}
#[tokio::test]
async fn quota_limited_account_diagnostic_uses_upstream() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_quota_limited_diagnostic";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("test account");
let raw_quota = json!({
"rate_limit": {
"allowed": false,
"limit_reached": true,
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
}
});
store
.compare_and_swap_quota(QuotaObservation {
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: Some(OpaqueProviderData::new(
raw_quota
.as_object()
.expect("quota snapshot object")
.clone(),
)),
observed_at: Some(SystemTime::now()),
limit_reached: Some(true),
})
.await
.expect("seed quota-limited snapshot");
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("content-type", "text/event-stream")
.set_body_string(concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\"}}\n\n",
"event: response.completed\n",
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
)),
)
.expect(1)
.mount(&server)
.await;
let (provider, quota) = provider_and_quota_with_affinity_and_base_url_and_leases(
&store,
Arc::new(MemorySessionAffinity::default()),
server.uri(),
Arc::new(TestLeaseCoordinator::default()),
);
quota
.prepare_scheduling(std::slice::from_ref(&account))
.await;
let mut stream = provider
.execute(
planned_request("openai", http_generate_operation()),
diagnostic_context("req_quota_limited_diagnostic", account_id),
)
.await
.expect("quota-limited diagnostic should prepare a fixed-account stream");
let mut completed = false;
while let Some(event) = stream.next().await {
let event = event.expect("quota-limited diagnostic upstream response");
completed |= event
.canonical_facts()
.iter()
.any(|event| matches!(event, GatewayEvent::Completed(_)));
}
assert!(completed);
let requests = server
.received_requests()
.await
.expect("captured quota-limited diagnostic request");
assert_eq!(requests.len(), 1);
}
#[tokio::test]
async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() {
let store = Arc::new(MemoryAccountStore::default());
@@ -16,9 +16,10 @@ use gateway_core::engine::credential::{
};
use gateway_core::error::{StoreError, StoreErrorKind};
use gateway_core::provider_ports::{
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderLeaseAcquisition, ProviderLeasePort,
ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderSessionAffinityKey,
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort,
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest,
ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey,
ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions,
ProviderStoreError,
};
@@ -771,3 +772,90 @@ pub(crate) fn codex_account(id: &str) -> ProviderAccount {
.with_runtime_state(true, AccountAvailability::Ready)
.with_refresh_schedule(true, Some(SystemTime::now() + Duration::from_secs(2_700)))
}
/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector
/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。
#[derive(Clone, Default)]
pub(crate) struct MemoryCooldownPort {
pub(crate) cooldowns: Arc<Mutex<BTreeMap<ProviderAccountId, ProviderCooldown>>>,
}
impl MemoryCooldownPort {
#[must_use]
pub(crate) fn new() -> Self {
Self::default()
}
}
impl ProviderCooldownPort for MemoryCooldownPort {
fn put_if_later(
&self,
cooldown: ProviderCooldown,
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
Box::pin(async move {
let mut cooldowns = self.cooldowns.lock().expect("cooldown lock");
let should_write = cooldowns.get(cooldown.account_id()).is_none_or(|current| {
current.credential_revision() < cooldown.credential_revision()
|| (current.credential_revision() == cooldown.credential_revision()
&& current.until() < cooldown.until())
});
if should_write {
cooldowns.insert(cooldown.account_id().clone(), cooldown);
}
Ok(should_write)
})
}
fn read<'a>(
&'a self,
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
Box::pin(async move {
Ok(self
.cooldowns
.lock()
.expect("cooldown lock")
.get(account_id)
.cloned())
})
}
fn clear<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_through_revision: CredentialRevision,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn put_scoped_if_later(
&self,
_cooldown: ProviderScopedCooldown,
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn read_scoped<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_scope: &'a ProviderCooldownScope,
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
Box::pin(async { Ok(None) })
}
fn clear_scoped<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_scope: &'a ProviderCooldownScope,
_through_revision: CredentialRevision,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn clear_all<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
}
@@ -146,13 +146,13 @@ fn structured_expiry_code_classifies_without_an_auth_status() {
}
#[test]
fn structured_quota_signal_takes_precedence_over_http_429() {
fn http_429_without_usage_limit_type_remains_a_temporary_rate_limit() {
assert_eq!(
classify(
429,
r#"{"error":{"code":"insufficient_quota","message":"rate limit reached"}}"#
),
CodexFailureCategory::QuotaExhausted
CodexFailureCategory::RateLimited
);
}
@@ -168,13 +168,24 @@ fn structured_usage_limit_type_is_resettable_quota_exhaustion() {
}
#[test]
fn usage_limit_message_without_a_type_is_resettable_quota_exhaustion() {
fn http_429_usage_limit_message_without_a_type_remains_a_temporary_rate_limit() {
assert_eq!(
classify(
429,
r#"{"error":{"message":"The usage limit has been reached"}}"#
),
CodexFailureCategory::UsageLimitExhausted
CodexFailureCategory::RateLimited
);
}
#[test]
fn http_429_usage_limit_code_without_a_type_remains_a_temporary_rate_limit() {
assert_eq!(
classify(
429,
r#"{"error":{"code":"usage_limit_reached","message":"The usage limit has been reached"}}"#
),
CodexFailureCategory::RateLimited
);
}
@@ -789,6 +789,70 @@ async fn backend_http_should_zstd_compress_codex_responses_request_body() {
assert_eq!(json["instructions"], "compress me");
}
#[tokio::test]
async fn backend_http_should_force_upstream_sse_for_non_streaming_client_request() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind non-streaming HTTP server");
let address = listener
.local_addr()
.expect("non-streaming HTTP server address");
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("accept HTTP client");
let request = read_http_request_with_body(&mut stream).await;
write_completed_sse_response(&mut stream).await;
request
});
let mut request = CodexResponsesRequest::from_body(
json!({
"model": "gpt-test",
"instructions": "collect this response",
"input": [],
"stream": false
})
.as_object()
.expect("request body")
.clone(),
);
request.force_http_sse = true;
assert!(
!request.stream(),
"client-facing preference remains non-streaming"
);
let client = CodexBackendClient::new(
reqwest::Client::builder()
.no_proxy()
.build()
.expect("HTTP client"),
format!("http://{address}"),
test_wire_profile(),
);
client
.create_response(
&request,
request_context("req_non_streaming_client", Some("acct-non-streaming")),
)
.await
.expect("upstream SSE response");
let raw = server.await.expect("non-streaming HTTP server task");
let separator = raw
.windows(4)
.position(|window| window == b"\r\n\r\n")
.expect("HTTP head/body separator");
let compressed_body = &raw[separator + 4..];
let decompressed = zstd::stream::decode_all(std::io::Cursor::new(compressed_body))
.expect("zstd body should decode");
let json: serde_json::Value =
serde_json::from_slice(&decompressed).expect("decompressed body should be valid JSON");
assert_eq!(json["stream"], true);
assert!(
!request.stream(),
"upstream normalization must not mutate local metadata"
);
}
#[tokio::test]
async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_latest_codex_core_wire_profile()
{
@@ -401,6 +401,41 @@ fn observability_semantics_should_reuse_codex_turn_metadata() {
assert!(semantics.compact);
}
#[test]
fn encoder_should_extract_subagent_kind_from_wire_or_turn_metadata() {
let metadata_request = request(Map::from_iter([
("model".to_owned(), json!("client-model")),
("input".to_owned(), json!("prompt")),
(
"client_metadata".to_owned(),
json!({"x-openai-subagent": "review"}),
),
]));
let metadata_encoded =
encode_generate_request(&metadata_request, "gpt-test").expect("encode metadata request");
assert_eq!(metadata_encoded.subagent_kind().as_deref(), Some("review"));
let turn_metadata_payload = ProtocolPayload::json_object(
"openai",
Map::from_iter([
("model".to_owned(), json!("client-model")),
("input".to_owned(), json!("prompt")),
]),
)
.expect("OpenAI payload")
.with_context(Map::from_iter([(
"turn_metadata".to_owned(),
Value::String(r#"{"subagent_kind":"worker"}"#.to_owned()),
)]));
let turn_metadata_request = GenerateRequest::from_protocol_payload(turn_metadata_payload);
let turn_metadata_encoded = encode_generate_request(&turn_metadata_request, "gpt-test")
.expect("encode turn metadata request");
assert_eq!(
turn_metadata_encoded.subagent_kind().as_deref(),
Some("worker")
);
}
#[test]
fn observability_semantics_should_use_the_transparent_openai_payload() {
let payload = ProtocolPayload::json_object(
@@ -1048,6 +1048,7 @@ fn project_quota(
refresh_token_expires_at,
windows: Vec::new(),
limit_reached: false,
rate_limited_until: None,
provider_data: None,
};
};
@@ -1115,6 +1116,7 @@ fn project_quota(
limit_reached: billing
.used_percent()
.is_some_and(|used| used.is_finite() && used >= 100.0),
rate_limited_until: None,
provider_data: None,
}
}
@@ -436,6 +436,7 @@ impl GrokCredentialRepository {
expected_revision: input.expected_revision,
availability: input.availability.into(),
observed_at: to_system_time(input.observed_at),
message: input.availability_reason.clone(),
})
.await
.map_err(map_store_error)
@@ -473,6 +473,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
enabled: account.enabled(),
availability: account.availability(),
availability_observed_at: now,
last_error_message: None,
quota_observed_at: None,
created_at: now,
updated_at: now,
@@ -204,6 +204,7 @@ async fn set_account_state(
) {
store
.apply_state_change(AccountStateChange {
message: None,
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability,
@@ -485,6 +486,7 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() {
let id = account_id("quota-exhausted");
store
.apply_state_change(AccountStateChange {
message: None,
account_id: id,
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: AccountAvailability::QuotaExhausted,
@@ -755,6 +757,7 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
let transport = Arc::new(MutatingBillingTransport {
store: Arc::clone(&store),
mutation: Mutex::new(Some(BillingMutation::State(AccountStateChange {
message: None,
account_id: id.clone(),
expected_revision: CredentialRevision::new(1).expect("revision"),
availability: AccountAvailability::Ready,
+1 -4
View File
@@ -1,4 +1 @@
54be07512951a561391b6e611355a1bfc119d82363253fb9bbdb225c9e3e39bb 0001_initial.sql
f480ac4cb70f6963eee71c0b8a67777ebb492f85501c90ebe7544bee0b2c0525 0002_remove_account_cooldown.sql
5cf5b5387c66b4d437d79c002cec46cb85196789060fa87c39e97a382487cac9 0004_fix_quota_limit_reached_null.sql
a3f236d558e7598f69277520458c801ee43130f7704ca25455df468cf8899ff2 0003_materialize_quota_limit_reached.sql
5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql
+4 -18
View File
@@ -131,9 +131,9 @@ create table provider_accounts (
next_refresh_at timestamptz,
enabled boolean not null default true,
availability text not null default 'unknown',
availability_reason text,
cooldown_until timestamptz,
provider_quota_json jsonb,
quota_limit_reached boolean not null default false,
last_error_message text,
availability_observed_at timestamptz not null,
quota_observed_at timestamptz,
created_at timestamptz not null,
@@ -163,23 +163,12 @@ create table provider_accounts (
availability in (
'unknown',
'ready',
'cooldown',
'quota_exhausted',
'expired',
'banned',
'invalid'
)
),
constraint provider_accounts_cooldown_ck check (
(
availability != 'cooldown'
or cooldown_until is not null
)
and (
cooldown_until is null
or availability in ('cooldown', 'quota_exhausted')
)
),
constraint provider_accounts_time_ck check (
created_at <= updated_at
and availability_observed_at <= updated_at
@@ -196,7 +185,7 @@ create unique index provider_accounts_upstream_identity_uq
create index provider_accounts_runtime_idx
on provider_accounts (provider_kind, enabled, id);
create index provider_accounts_availability_idx
on provider_accounts (availability, cooldown_until, id);
on provider_accounts (availability, id);
create index provider_accounts_access_expiry_idx
on provider_accounts (access_token_expires_at, id);
create index provider_accounts_refresh_due_idx
@@ -235,8 +224,6 @@ create table model_requests (
outcome text not null default 'running',
client_status_code integer,
upstream_status_code integer,
-- Provider response IDs are opaque JSON strings. Bytes preserve embedded NUL
-- and avoid imposing PostgreSQL text-index limits on protocol handles.
client_response_id bytea,
upstream_request_id text,
upstream_response_id bytea,
@@ -438,6 +425,7 @@ create index model_requests_running_deadline_idx
create index model_requests_retention_idx
on model_requests (completed_at)
where outcome <> 'running';
-- 运行期警告与错误事件。
create table ops_events (
id text primary key,
@@ -691,11 +679,9 @@ create unique index backup_records_active_uq
create unique index backup_records_object_key_uq
on backup_records (object_key)
where object_key is not null;
-- 同时服务 retentionDays 与 retentionCount。
create index backup_records_scheduled_completed_idx
on backup_records (completed_at desc, id desc)
where trigger_kind = 'scheduled' and status = 'completed';
-- 可清理记录(expires_at 已到期且非活跃)的扫描索引。
create index backup_records_retention_idx
on backup_records (expires_at)
where expires_at is not null and status in ('completed', 'failed');
@@ -1,43 +0,0 @@
-- 移除账号级 cooldown 状态模型(照 v2 语义:429 限流不进 availability,由 quota 数据驱动)。
--
-- 存量迁移:
-- - availability='cooldown' 的账号:429/Cloudflare 等临时冷却 → 'ready'
-- (限流事实留在 provider_quota_json,由投影滚动兜底);
-- 402 语义的 usage_limit_exhausted 且未到期 → 'quota_exhausted'。
-- 1. 迁移存量 cooldown 账号。
update provider_accounts
set availability = case
when availability_reason = 'usage_limit_exhausted'
and cooldown_until is not null
and cooldown_until > now()
then 'quota_exhausted'
else 'ready'
end
where availability = 'cooldown';
-- 2. 删除依赖 cooldown_until 的约束与索引。
alter table provider_accounts drop constraint provider_accounts_cooldown_ck;
drop index provider_accounts_availability_idx;
-- 3. availability CHECK 去掉 'cooldown'。
alter table provider_accounts drop constraint provider_accounts_availability_ck;
alter table provider_accounts
add constraint provider_accounts_availability_ck check (
availability in (
'unknown',
'ready',
'quota_exhausted',
'expired',
'banned',
'invalid'
)
);
-- 4. 删除列。
alter table provider_accounts drop column availability_reason;
alter table provider_accounts drop column cooldown_until;
-- 5. 重建可用性索引(不含 cooldown_until)。
create index provider_accounts_availability_idx
on provider_accounts (availability, id);
@@ -1,20 +0,0 @@
-- 物化快照级限流事实:Dashboard 复用 Provider 解析结果,不再 SQL 解析 raw quota JSON。
--
-- `quota_limit_reached` 由 Provider 在写 quota 时计算(快照级:顶层或任一窗口触顶,
-- 已过 reset 的窗口不计数),Admin/Dashboard 直接读该列。
alter table provider_accounts
add column quota_limit_reached boolean not null default false;
-- 存量回填:从现有 raw JSON 推导(仅 primary_window 的 limit_reached + reset 未过期)。
update provider_accounts
set quota_limit_reached = coalesce(
provider_quota_json is not null
and provider_quota_json->'rate_limit'->>'limit_reached' = 'true'
and (
provider_quota_json->'rate_limit'->'primary_window'->>'reset_at' is null
or (provider_quota_json->'rate_limit'->'primary_window'->>'reset_at')::bigint > extract(epoch from now())::bigint
),
false
)
where provider_quota_json is not null;
@@ -1,8 +0,0 @@
-- 修正 0003 存量回填:raw JSON 缺 rate_limit 键时表达式为 NULL,
-- 赋 NULL 违反 not-null 约束。0004 把仍为 NULL 的行兜底为 false
-- (0003 已应用的环境,add column 用 default false 填充,通常无 NULL;
-- 本迁移保证任何环境下该列都非 NULL)。
update provider_accounts
set quota_limit_reached = coalesce(quota_limit_reached, false)
where quota_limit_reached is null;
+8 -2
View File
@@ -15,6 +15,8 @@ export interface AccountQuotaWindow {
export interface AccountQuota {
refreshedAtDisplay: string
limitReached: boolean
// 429 临时限流(Redis 冷却)到期时间;非限流中为 null。
rateLimitedUntil: string | null
windows: AccountQuotaWindow[]
}
@@ -104,6 +106,8 @@ export interface Account {
hasRefreshToken: boolean
status: string
displayStatus: string
errorReason: string | null
errorMessage: string | null
availability: string
enabled: boolean
accessTokenExpiresAt: string | null
@@ -127,9 +131,11 @@ export interface AccountPageMeta {
export interface AccountSummary {
total: number
active: number
normal: number
quotaExhausted: number
unavailable: number
rateLimited: number
disabled: number
error: number
}
export interface AccountListResponse {
+6 -4
View File
@@ -108,11 +108,13 @@ const labelClasses = computed(() => [
:title="title || (iconOnly ? label : undefined)"
:aria-busy="loading"
>
<LoaderCircle
<span
v-if="loading"
class="shrink-0 animate-spin origin-center transform-view will-change-transform"
:size="loadingIconSize[size]"
/>
class="inline-flex shrink-0 animate-spin origin-center will-change-transform"
aria-hidden="true"
>
<LoaderCircle :size="loadingIconSize[size]" />
</span>
<span v-if="$slots.icon && !loading" class="inline-flex shrink-0">
<slot name="icon" />
</span>
+26 -20
View File
@@ -1,14 +1,15 @@
<script setup lang="ts">
import { computed } from 'vue'
type CardVariant = 'default' | 'dashboard' | 'elevated'
type CardPadding = 'default' | 'compact'
type CardLayout = 'content' | 'flush'
type HeaderCollapseAt = 'sm' | 'lg' | 'none'
const props = withDefaults(
defineProps<{
as?: keyof HTMLElementTagNameMap
padded?: boolean
variant?: CardVariant
padding?: CardPadding
layout?: CardLayout
title?: string
description?: string
headerCollapseAt?: HeaderCollapseAt
@@ -17,8 +18,8 @@ const props = withDefaults(
}>(),
{
as: 'section',
padded: true,
variant: 'default',
padding: 'default',
layout: 'content',
title: undefined,
description: undefined,
headerCollapseAt: 'sm',
@@ -36,16 +37,9 @@ const slots = defineSlots<{
default?: () => unknown
}>()
const paddingClasses: Record<CardVariant, string> = {
default: 'px-5 py-3',
dashboard: 'px-7 py-5.5',
elevated: 'px-5 py-3',
}
const shadowClasses: Record<CardVariant, string> = {
default: 'shadow-(--cp-shadow-card)',
dashboard: 'shadow-(--cp-shadow-card)',
elevated: 'shadow-(--cp-shadow-popover)',
const paddingClasses: Record<CardPadding, string> = {
default: 'p-5.5',
compact: 'p-4',
}
const hasManagedHeader = computed(
@@ -53,6 +47,12 @@ const hasManagedHeader = computed(
!!props.title || !!props.description || !!slots.actions || !!slots.title || !!slots.description,
)
const outerPaddingClass = computed(() => {
if (props.layout === 'flush')
return undefined
return paddingClasses[props.padding]
})
const managedHeaderLayoutClasses = computed(() => {
if (props.headerCollapseAt === 'none') {
return 'flex items-start justify-between gap-3'
@@ -69,14 +69,17 @@ const managedHeaderLayoutClasses = computed(() => {
<template>
<component
:is="props.as"
class="[--cp-input-current-bg:var(--cp-input-soft-bg)] [--cp-input-current-bg-hover:var(--cp-input-soft-bg-hover)] overflow-hidden bg-(--cp-bg-surface) rounded-(--cp-card-radius)"
class="[--cp-input-current-bg:var(--cp-input-soft-bg)] [--cp-input-current-bg-hover:var(--cp-input-soft-bg-hover)] overflow-hidden rounded-(--cp-card-radius) bg-(--cp-bg-surface) shadow-(--cp-shadow-card)"
:class="[
shadowClasses[props.variant],
props.padded ? paddingClasses[props.variant] : undefined,
outerPaddingClass,
]"
>
<template v-if="$slots.header || hasManagedHeader || $slots.body">
<header v-if="$slots.header || hasManagedHeader" class="shrink-0" :class="props.headerClass">
<header
v-if="$slots.header || hasManagedHeader"
class="shrink-0"
:class="props.headerClass"
>
<slot name="header">
<div :class="managedHeaderLayoutClasses">
<div class="min-w-0 pt-0.5">
@@ -105,7 +108,10 @@ const managedHeaderLayoutClasses = computed(() => {
</slot>
</header>
<div v-if="$slots.body || $slots.default" :class="props.bodyClass">
<div
v-if="$slots.body || $slots.default"
:class="props.bodyClass"
>
<slot name="body">
<slot />
</slot>
@@ -1,12 +1,20 @@
<script setup lang="ts">
defineProps<{
const props = defineProps<{
title: string
description?: string
actionsInline?: boolean
}>()
</script>
<template>
<header class="grid min-h-17 shrink-0 grid-cols-[minmax(0,1fr)_auto] items-start gap-x-4 gap-y-3">
<header
class="grid min-h-17 shrink-0 items-start"
:class="
props.actionsInline
? 'grid-cols-[minmax(0,1fr)_auto] gap-x-4'
: 'grid-cols-1 gap-y-3 sm:grid-cols-[minmax(0,1fr)_auto] sm:gap-x-4'
"
>
<div class="min-w-0">
<h1 class="m-0 text-[34px] leading-[1.15] font-extrabold text-(--cp-text-primary)">
{{ title }}
@@ -23,6 +31,7 @@ defineProps<{
<div
v-if="$slots.actions"
class="mt-0.5 flex shrink-0 items-center justify-end gap-2"
:class="props.actionsInline ? undefined : 'max-sm:w-full'"
>
<slot name="actions" />
</div>
+38 -6
View File
@@ -6,7 +6,7 @@ import { computed, nextTick, onBeforeUnmount, ref, shallowRef, useAttrs, watch }
type PopoverPlacement
= 'top' | 'top-start' | 'top-end' | 'right' | 'bottom' | 'bottom-start' | 'bottom-end' | 'left'
type PopoverTrigger = 'click' | 'hover'
type PopoverTrigger = 'click' | 'hover' | 'hover-click'
interface PopoverPoint {
left: number
@@ -29,6 +29,8 @@ const props = withDefaults(
offset?: number
width?: number | string
disabled?: boolean
/** 包含 hover 的触发方式打开弹窗的延迟(毫秒);click 触发不生效。 */
hoverDelay?: number
panelClass?: string
triggerClass?: string
anchorElement?: HTMLElement | null
@@ -39,6 +41,7 @@ const props = withDefaults(
trigger: 'click',
offset: 6,
disabled: false,
hoverDelay: 400,
panelClass: '',
triggerClass: '',
anchorElement: null,
@@ -55,6 +58,7 @@ const popoverRef = ref<HTMLElement | null>(null)
const popoverStyle = ref<CSSProperties>({})
const popoverArrowStyle = ref<CSSProperties>({})
const hoverCloseTimer = shallowRef<number>()
const hoverOpenTimer = shallowRef<number>()
const viewportTarget = computed(() => (open.value && typeof window !== 'undefined' ? window : null))
const popoverClasses = computed(() => [
@@ -260,6 +264,7 @@ async function openPopover() {
if (props.disabled || open.value)
return
clearHoverOpenTimer()
clearHoverCloseTimer()
open.value = true
await nextTick()
@@ -272,7 +277,7 @@ function closePopover() {
}
function togglePopover() {
if (props.trigger !== 'click') {
if (!supportsClickTrigger()) {
return
}
@@ -284,6 +289,14 @@ function togglePopover() {
void openPopover()
}
function supportsClickTrigger() {
return props.trigger === 'click' || props.trigger === 'hover-click'
}
function supportsHoverTrigger() {
return props.trigger === 'hover' || props.trigger === 'hover-click'
}
function clearHoverCloseTimer() {
if (hoverCloseTimer.value === undefined) {
return
@@ -293,20 +306,36 @@ function clearHoverCloseTimer() {
hoverCloseTimer.value = undefined
}
function clearHoverOpenTimer() {
if (hoverOpenTimer.value === undefined) {
return
}
window.clearTimeout(hoverOpenTimer.value)
hoverOpenTimer.value = undefined
}
function handleHoverEnter() {
if (props.trigger !== 'hover') {
if (!supportsHoverTrigger()) {
return
}
clearHoverCloseTimer()
void openPopover()
// 延迟打开:鼠标稳定悬停一小段时间再展示,避免扫过表格时弹窗频繁闪烁。
if (hoverOpenTimer.value === undefined) {
hoverOpenTimer.value = window.setTimeout(() => {
hoverOpenTimer.value = undefined
void openPopover()
}, props.hoverDelay)
}
}
function handleHoverLeave() {
if (props.trigger !== 'hover') {
if (!supportsHoverTrigger()) {
return
}
clearHoverOpenTimer()
clearHoverCloseTimer()
hoverCloseTimer.value = window.setTimeout(closePopover, 90)
}
@@ -341,7 +370,10 @@ useEventListener(viewportTarget, 'keydown', (event) => {
})
useEventListener(viewportTarget, 'resize', updatePopoverPositionThrottled)
useEventListener(viewportTarget, 'scroll', updatePopoverPositionThrottled, { capture: true })
onBeforeUnmount(clearHoverCloseTimer)
onBeforeUnmount(() => {
clearHoverOpenTimer()
clearHoverCloseTimer()
})
</script>
<template>
@@ -57,7 +57,12 @@ function connectionLogClass(tone: string) {
class="flex items-center justify-between gap-4 rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3"
>
<AccountIdentityCell :account="account" size="lg" show-plan />
<AccountStatusBadge :status="account.status" variant="pill" />
<AccountStatusBadge
:status="account.status"
:error-reason="account.errorReason"
:error-message="account.errorMessage"
variant="pill"
/>
</section>
<section class="rounded-(--cp-card-radius) bg-(--cp-bg-subtle) px-4 py-3">
@@ -54,10 +54,13 @@ const provider = defineModel<string>('provider', { required: true })
/>
</div>
<div class="flex shrink-0 self-end items-center justify-end gap-2 md:ml-auto">
<div
class="grid w-full grid-cols-2 gap-2 md:flex md:w-auto md:shrink-0 md:self-end md:items-center md:justify-end md:ml-auto"
>
<BaseButton
v-if="selectedCount > 0"
variant="danger"
class="w-full whitespace-nowrap md:w-auto"
:disabled="batchDeleting"
@click="emit('deleteSelected')"
>
@@ -67,13 +70,19 @@ const provider = defineModel<string>('provider', { required: true })
<BaseButton
v-if="selectedCount > 0"
variant="default"
class="w-full whitespace-nowrap md:w-auto"
:loading="exportingAccounts"
@click="emit('exportSelected')"
>
<Download class="size-4" />
导出选中 ({{ selectedCount }})
</BaseButton>
<BaseButton variant="primary" @click="emit('create')">
<BaseButton
variant="primary"
class="whitespace-nowrap md:w-auto"
:class="selectedCount > 0 ? 'col-span-2 w-full' : 'col-span-2 justify-self-end'"
@click="emit('create')"
>
<Upload class="size-4" />
导入账号
</BaseButton>
@@ -20,22 +20,22 @@ const overviewItems = computed(() => [
},
{
label: '正常账号',
value: formatCount(props.summary.active),
value: formatCount(props.summary.normal),
caption: '可参与调度',
tone: 'success',
icon: ShieldCheck,
},
{
label: '配额耗尽',
value: formatCount(props.summary.quotaExhausted),
caption: '等待配额恢复',
label: '额度受限',
value: formatCount((props.summary.quotaExhausted ?? 0) + (props.summary.rateLimited ?? 0)),
caption: '等待额度恢复',
tone: 'warning',
icon: Gauge,
},
{
label: '异常账号',
value: formatCount(props.summary.unavailable),
caption: '过期 / 禁用 / 封禁',
label: '待处理',
value: formatCount((props.summary.disabled ?? 0) + (props.summary.error ?? 0)),
caption: '已停用 / 错误',
tone: 'danger',
icon: AlertTriangle,
},
@@ -65,16 +65,15 @@ function overviewIconClass(tone: string) {
v-for="item in overviewItems"
:key="item.label"
as="article"
:padded="false"
class="h-24"
padding="compact"
>
<div class="flex h-full items-stretch justify-between gap-3 px-5 py-3">
<div class="flex min-w-0 flex-col justify-between">
<div class="flex items-stretch justify-between gap-3">
<div class="flex min-w-0 flex-col">
<p class="m-0 text-[12px] leading-none font-heavy text-(--cp-text-secondary)">
{{ item.label }}
</p>
<strong
class="block font-mono text-[26px] leading-none font-extrabold text-(--cp-text-primary)"
class="my-2 block font-mono text-[26px] leading-none font-extrabold text-(--cp-text-primary)"
>
{{ item.value }}
</strong>
@@ -1,68 +1,233 @@
<script setup lang="ts">
import { AlertTriangle, CircleCheck, Gauge, Power, Timer } from '@lucide/vue'
import { computed } from 'vue'
import { statusLabels, statusTones } from '../constants'
import BasePopover from '@/components/base/BasePopover.vue'
import BaseScrollbar from '@/components/base/BaseScrollbar.vue'
import { errorReasonLabels, statusLabels, statusTones } from '../constants'
const props = withDefaults(
defineProps<{
status: string
errorReason?: string | null
errorMessage?: string | null
rateLimitedUntil?: string | null
variant?: 'inline' | 'pill'
}>(),
{
errorReason: null,
errorMessage: null,
rateLimitedUntil: null,
variant: 'inline',
},
)
const tone = computed(() => statusTones[props.status])
const label = computed(() => statusLabels[props.status] || props.status)
const statusStyles = {
success: {
text: 'text-(--cp-success-text)',
dot: 'bg-(--cp-success)',
badge: 'border-(--cp-success-border) bg-(--cp-success-bg) text-(--cp-success-text)',
icon: 'bg-(--cp-success-bg) text-(--cp-success-text) shadow-[inset_0_0_0_1px_var(--cp-success-border)]',
},
danger: {
text: 'text-(--cp-danger-text)',
dot: 'bg-(--cp-danger)',
badge: 'border-(--cp-danger-border) bg-(--cp-danger-bg) text-(--cp-danger-text)',
icon: 'bg-(--cp-danger-bg) text-(--cp-danger-text) shadow-[inset_0_0_0_1px_var(--cp-danger-border)]',
},
warning: {
text: 'text-(--cp-warning-text)',
dot: 'bg-(--cp-warning)',
badge: 'border-(--cp-warning-border) bg-(--cp-warning-bg) text-(--cp-warning-text)',
icon: 'bg-(--cp-warning-bg) text-(--cp-warning-text) shadow-[inset_0_0_0_1px_var(--cp-warning-border)]',
},
info: {
text: 'text-(--cp-info-text)',
dot: 'bg-(--cp-info)',
badge: 'border-(--cp-info-border) bg-(--cp-info-bg) text-(--cp-info-text)',
icon: 'bg-(--cp-info-bg) text-(--cp-info-text) shadow-[inset_0_0_0_1px_var(--cp-info-border)]',
},
normal: {
text: 'text-(--cp-text-secondary)',
dot: 'bg-(--cp-text-muted)',
badge: 'border-(--cp-divider-subtle) bg-(--cp-bg-subtle) text-(--cp-text-secondary)',
icon: 'bg-(--cp-bg-subtle) text-(--cp-text-secondary)',
},
} as const
const textClass = computed(() => {
if (tone.value === 'success') {
return 'text-(--cp-success-text)'
}
if (tone.value === 'danger') {
return 'text-(--cp-danger-text)'
}
if (tone.value === 'warning') {
return 'text-(--cp-warning-text)'
}
if (tone.value === 'info') {
return 'text-(--cp-info-text)'
}
return 'text-(--cp-text-secondary)'
const tone = computed(() => statusTones[props.status] || 'normal')
const statusStyle = computed(() => statusStyles[tone.value])
const label = computed(() => statusLabels[props.status] || props.status)
const reasonLabel = computed(() =>
props.errorReason
? errorReasonLabels[props.errorReason] || props.errorReason
: null,
)
const errorText = computed(() => props.errorMessage || null)
const detailTitle = computed(() => reasonLabel.value || label.value)
const isRateLimited = computed(() =>
props.status === 'rate_limited' && Boolean(props.rateLimitedUntil),
)
const rateLimitedRelative = computed(() => {
const rateLimitedUntil = props.rateLimitedUntil
if (!rateLimitedUntil)
return null
const until = new Date(rateLimitedUntil.replace(' ', 'T')).getTime()
const diffMinutes = Math.round((until - Date.now()) / 60_000)
if (Number.isNaN(diffMinutes) || diffMinutes < 1)
return null
if (diffMinutes < 60)
return `剩余 ${diffMinutes} 分钟`
const hours = Math.floor(diffMinutes / 60)
const mins = diffMinutes % 60
if (mins === 0)
return `剩余 ${hours} 小时`
return `剩余 ${hours} 小时 ${mins} 分`
})
const hasDetail = computed(() =>
props.status === 'error' || isRateLimited.value,
)
const detailDescription = computed(() => {
if (props.status === 'rate_limited')
return '上游暂时限制了请求频率,系统正在冷却该账号。'
if (props.status === 'error')
return '该账号暂不参与调度,处理凭据后可重新测试连接。'
return '该账号当前状态需要关注。'
})
const dotClass = computed(() => {
if (tone.value === 'success') {
return 'bg-(--cp-success)'
const recoveryHint = computed(() => {
if (props.status === 'rate_limited')
return '冷却结束后,系统会自动恢复调度。'
switch (props.errorReason) {
case 'expired':
case 'token_expired':
return '重新授权后,账号会重新参与调度。'
case 'invalid':
return '请更新或重新导入凭据,再次测试连接。'
case 'banned':
return '请确认上游账号状态,解除限制后再启用。'
default:
return '重新测试连接以获取最新状态。'
}
if (tone.value === 'danger') {
return 'bg-(--cp-danger)'
})
const triggerLabel = computed(() =>
`${detailTitle.value}。${detailDescription.value} 点击或聚焦查看详情。`,
)
const cardIcon = computed(() => {
switch (props.status) {
case 'error': return AlertTriangle
case 'rate_limited': return Timer
case 'quota_exhausted': return Gauge
case 'disabled': return Power
default: return CircleCheck
}
if (tone.value === 'warning') {
return 'bg-(--cp-warning)'
}
if (tone.value === 'info') {
return 'bg-(--cp-info)'
}
return 'bg-(--cp-text-muted)'
})
</script>
<template>
<span
v-if="variant === 'pill'"
class="inline-flex h-7 shrink-0 items-center rounded-full px-2.5 text-[12px] font-heavy"
:class="textClass"
<BasePopover
:disabled="!hasDetail"
trigger="hover-click"
placement="top-start"
width="352px"
panel-class="!p-0 text-(--cp-text-primary)"
>
{{ label }}
</span>
<span
v-else
class="inline-flex min-w-16 items-center gap-1.5 text-[12px] leading-none font-emphasis"
:class="textClass"
>
<span class="size-1.5 rounded-full" :class="dotClass" />
<span>{{ label }}</span>
</span>
<template #trigger="{ open }">
<component
:is="hasDetail ? 'button' : 'span'"
:type="hasDetail ? 'button' : undefined"
:aria-label="hasDetail ? triggerLabel : undefined"
:aria-expanded="hasDetail ? open : undefined"
:aria-haspopup="hasDetail ? 'dialog' : undefined"
class="group inline-flex min-w-0 cursor-default border-0 bg-transparent p-0 text-left outline-none"
>
<span
v-if="variant === 'pill'"
class="inline-flex h-7 shrink-0 items-center rounded-full px-2.5 text-[12px] leading-none font-heavy transition-colors duration-150 motion-reduce:transition-none"
:class="statusStyle.text"
>
{{ label }}
</span>
<span
v-else
class="inline-flex min-w-16 items-center gap-1.5 rounded-md px-1.5 py-1 text-[12px] leading-none font-emphasis transition-colors duration-150 motion-reduce:transition-none"
:class="statusStyle.text"
>
<span aria-hidden="true" class="size-1.5 rounded-full" :class="statusStyle.dot" />
<span>{{ label }}</span>
</span>
</component>
</template>
<section class="overflow-hidden rounded-(--cp-popover-radius)">
<header class="flex items-start gap-3 border-b border-(--cp-divider-subtle) bg-(--cp-bg-subtle) px-4 py-3">
<span
aria-hidden="true"
class="inline-flex size-9 shrink-0 items-center justify-center rounded-(--cp-icon-button-radius)"
:class="statusStyle.icon"
>
<component :is="cardIcon" class="size-4.5" />
</span>
<div class="min-w-0 flex-1">
<p class="m-0 text-[11px] leading-none font-heavy tracking-[0.08em] text-(--cp-text-tertiary)">
账号状态
</p>
<h3 class="mt-1.5 mb-0 text-[14px] leading-5 font-heavy text-(--cp-text-primary) text-balance">
{{ detailTitle }}
</h3>
</div>
<span
class="inline-flex h-5 shrink-0 items-center rounded-full border px-2 text-[10px] leading-none font-heavy"
:class="statusStyle.badge"
>
{{ label }}
</span>
</header>
<div class="grid gap-3 px-4 py-3 text-[12px] leading-5">
<p class="m-0 text-pretty font-emphasis text-(--cp-text-secondary)">
{{ detailDescription }}
</p>
<div
v-if="isRateLimited"
class="flex items-center justify-between gap-3 rounded-(--cp-input-radius-base) bg-(--cp-bg-subtle) px-3 py-2"
>
<span class="font-heavy text-(--cp-text-tertiary)">预计恢复</span>
<span class="font-mono font-emphasis tabular-nums text-(--cp-text-primary)">
{{ rateLimitedRelative ?? props.rateLimitedUntil }}
</span>
</div>
<div class="rounded-(--cp-input-radius-base) border border-(--cp-divider-subtle) bg-(--cp-bg-surface) px-3 py-2.5">
<p class="m-0 text-[11px] leading-none font-heavy text-(--cp-text-tertiary)">
建议操作
</p>
<p class="mt-1.5 mb-0 text-pretty font-emphasis text-(--cp-text-primary)">
{{ recoveryHint }}
</p>
</div>
<div
v-if="errorText"
class="overflow-hidden rounded-(--cp-input-radius-base) border border-(--cp-divider-subtle) bg-(--cp-bg-subtle)"
>
<div class="flex items-center justify-between gap-3 border-b border-(--cp-divider-subtle) px-3 py-2">
<span class="text-[11px] leading-none font-heavy text-(--cp-text-tertiary)">上游原始反馈</span>
<span class="shrink-0 text-[10px] leading-none font-emphasis text-(--cp-text-muted)">仅供排查</span>
</div>
<BaseScrollbar max-height="124px" view-class="px-3 py-2 pr-2">
<pre class="m-0 whitespace-pre-wrap wrap-break-word font-mono text-[11px] leading-[1.55] font-emphasis text-(--cp-text-secondary)">{{ errorText }}</pre>
</BaseScrollbar>
</div>
</div>
</section>
</BasePopover>
</template>
@@ -26,7 +26,7 @@ const emit = defineEmits<{
const canRefreshToken = computed(
() =>
props.account.hasRefreshToken
&& (props.account.status === 'active' || props.account.status === 'quota_exhausted'),
&& (props.account.status === 'normal' || props.account.status === 'quota_exhausted'),
)
const canReauthorize = computed(() => props.account.provider === 'openai')
</script>
@@ -60,6 +60,15 @@ function connectionTestErrorText(event: ConnectionTestEvent) {
return event.error || '测试连接失败'
}
function connectionTestFailureLabel(event: ConnectionTestEvent) {
const hasUpstreamResponse
= (typeof event.upstreamBody === 'string' && event.upstreamBody.length > 0)
|| Number.isInteger(event.upstreamStatus)
return hasUpstreamResponse
? '上游返回错误'
: '未能向上游发起请求'
}
export function useAccountConnectionTest(options: AccountConnectionTestOptions) {
const showConnectionTestModal = shallowRef(false)
const testingAccount = shallowRef<Account | null>(null)
@@ -293,7 +302,11 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
if (event.type === 'error') {
applyAccountStatus(event.accountStatus)
connectionTestError.value = connectionTestErrorText(event)
appendConnectionTestLog('上游返回错误', 'danger', connectionTestError.value)
appendConnectionTestLog(
connectionTestFailureLabel(event),
'danger',
connectionTestError.value,
)
finishConnectionTest('error')
clearConnectionTestRun()
}
@@ -24,11 +24,11 @@ import { useAccountOnboarding } from './useAccountOnboarding'
type AccountRow = Awaited<ReturnType<typeof getAccounts>>['items'][number]
const accountStatusSortRank: Record<string, number> = {
active: 0,
normal: 0,
quota_exhausted: 1,
expired: 2,
rate_limited: 2,
disabled: 3,
banned: 4,
error: 4,
}
export function useAccountMutations(options: {
@@ -232,11 +232,11 @@ export function useAccountMutations(options: {
return
options.accountSummary.value = {
...options.accountSummary.value,
active: Math.max(
normal: Math.max(
0,
options.accountSummary.value.active
+ Number(status === 'active')
- Number(current.status === 'active'),
options.accountSummary.value.normal
+ Number(status === 'normal')
- Number(current.status === 'normal'),
),
quotaExhausted: Math.max(
0,
@@ -244,26 +244,34 @@ export function useAccountMutations(options: {
+ Number(status === 'quota_exhausted')
- Number(current.status === 'quota_exhausted'),
),
unavailable: Math.max(
rateLimited: Math.max(
0,
options.accountSummary.value.unavailable
+ Number(isUnavailableAccountStatus(status))
- Number(isUnavailableAccountStatus(current.status)),
options.accountSummary.value.rateLimited
+ Number(status === 'rate_limited')
- Number(current.status === 'rate_limited'),
),
disabled: Math.max(
0,
options.accountSummary.value.disabled
+ Number(status === 'disabled')
- Number(current.status === 'disabled'),
),
error: Math.max(
0,
options.accountSummary.value.error
+ Number(status === 'error')
- Number(current.status === 'error'),
),
}
}
function isUnavailableAccountStatus(status: string) {
return status === 'expired' || status === 'disabled' || status === 'banned'
}
function sortAccountsByStatus(rows: AccountRow[]) {
if (options.sort.value?.key !== 'status')
return rows
const direction = options.sort.value.direction === 'asc' ? 1 : -1
return [...rows].sort((left, right) => {
const rankDifference
= (accountStatusSortRank[left.status] ?? 5) - (accountStatusSortRank[right.status] ?? 5)
= (accountStatusSortRank[left.status] ?? 7) - (accountStatusSortRank[right.status] ?? 7)
return rankDifference === 0
? left.id.localeCompare(right.id) * direction
: rankDifference * direction
@@ -14,9 +14,11 @@ export function useAccountsQuery() {
const sort = shallowRef<BaseTableSort>()
const accountSummary = shallowRef({
total: 0,
active: 0,
normal: 0,
quotaExhausted: 0,
unavailable: 0,
rateLimited: 0,
disabled: 0,
error: 0,
})
const query = usePagedQuery({
+39 -26
View File
@@ -19,14 +19,9 @@ export interface AccountLocalUsage {
requestBuckets?: AccountRequestBucket[]
}
const summaryGroupOrder = new Map([
const quotaGroupOrder = new Map([
['shortTerm', 0],
['monthly', 1],
])
const panelGroupOrder = new Map([
['monthly', 0],
['shortTerm', 1],
['other', 2],
])
@@ -123,34 +118,41 @@ export const accountColumns = [
] satisfies BaseTableColumn<AccountRow>[]
export const statusLabels: Record<string, string> = {
active: '正常',
expired: '已过期',
disabled: '已禁用',
banned: '已封禁',
normal: '正常',
quota_exhausted: '配额耗尽',
rate_limited: '限流中',
disabled: '已停用',
error: '错误',
}
export const statusTones: Record<string, 'success' | 'danger' | 'warning' | 'info' | 'normal'> = {
active: 'success',
expired: 'warning',
disabled: 'normal',
banned: 'danger',
normal: 'success',
quota_exhausted: 'warning',
rate_limited: 'warning',
disabled: 'normal',
error: 'danger',
}
export const accountStatusFilterOptions = [
{ label: '全部状态', value: '' },
{ label: statusLabels.active, value: 'active' },
{ label: statusLabels.normal, value: 'normal' },
{ label: statusLabels.quota_exhausted, value: 'quota_exhausted' },
{ label: statusLabels.expired, value: 'expired' },
{ label: statusLabels.rate_limited, value: 'rate_limited' },
{ label: statusLabels.disabled, value: 'disabled' },
{ label: statusLabels.banned, value: 'banned' },
{ label: statusLabels.error, value: 'error' },
]
/** `error` 分类下具体原因的展示文案(对应后端 `errorReason`)。 */
export const errorReasonLabels: Record<string, string> = {
expired: '账号已过期',
token_expired: '访问令牌已过期',
invalid: '凭据已失效',
banned: '账号已封禁',
unknown: '状态未知',
}
/**
* 后端已派生互斥状态(含 `rate_limited`,由快照级 quota.limitReached 产生);
* 后端已派生互斥状态(正常 / 配额耗尽 / 限流中 / 已停用 / 错误);
* 前端只渲染,不再独立派生。
*/
export function derivedAccountStatus(row: AccountRow): string {
@@ -159,15 +161,13 @@ export function derivedAccountStatus(row: AccountRow): string {
export function visibleSummaryQuotaWindows(windows: AccountQuotaWindow[]) {
const known = [...windows]
.filter(window => summaryGroupOrder.has(window.group))
.sort((left, right) => groupOrder(left, summaryGroupOrder) - groupOrder(right, summaryGroupOrder))
return known.length > 0 ? known : windows
.filter(window => window.group !== 'other')
.sort(compareQuotaWindows)
return known.length > 0 ? known : [...windows].sort(compareQuotaWindows)
}
export function orderedPanelQuotaWindows(windows: AccountQuotaWindow[]) {
return [...windows].sort(
(left, right) => groupOrder(left, panelGroupOrder) - groupOrder(right, panelGroupOrder),
)
return [...windows].sort(compareQuotaWindows)
}
function quotaWindowPercent(window: AccountQuotaWindow) {
@@ -202,8 +202,21 @@ export function quotaWindowPercentTextClass(window: AccountQuotaWindow) {
return 'text-(--cp-success-text)'
}
function groupOrder(window: AccountQuotaWindow, order: Map<string, number>) {
return order.get(window.group) ?? order.size
function compareQuotaWindows(left: AccountQuotaWindow, right: AccountQuotaWindow) {
const groupDifference = groupOrder(left) - groupOrder(right)
if (groupDifference !== 0)
return groupDifference
const secondsDifference = (left.windowSeconds ?? Number.MAX_SAFE_INTEGER)
- (right.windowSeconds ?? Number.MAX_SAFE_INTEGER)
if (secondsDifference !== 0)
return secondsDifference
return left.key.localeCompare(right.key)
}
function groupOrder(window: AccountQuotaWindow) {
return quotaGroupOrder.get(window.group) ?? quotaGroupOrder.size
}
function accountProviderLabel(value?: string | null) {
+8 -5
View File
@@ -126,10 +126,8 @@ const {
<AccountOverviewCards :summary="accountSummary" />
<BaseCard
:padded="false"
class="mt-4 flex h-[calc(100dvh-248px)] min-h-125 flex-col"
header-class="px-4 pt-4 pb-2 md:px-5"
body-class="flex min-h-0 flex-1 px-4 pb-3 md:px-5"
class="mt-4 flex h-[calc(100dvh-250px)] min-h-125 flex-col"
body-class="mt-3 flex min-h-0 flex-1"
>
<template #header>
<AccountFilters
@@ -204,7 +202,12 @@ const {
</template>
<template #status="{ row }">
<AccountStatusBadge :status="derivedAccountStatus(row)" />
<AccountStatusBadge
:status="derivedAccountStatus(row)"
:error-reason="row.errorReason"
:error-message="row.errorMessage"
:rate-limited-until="row.quota.rateLimitedUntil"
/>
</template>
<template #planType="{ row }">
@@ -19,11 +19,11 @@ const search = defineModel<string>('search', { required: true })
<template>
<div
class="flex w-full flex-col gap-3 md:flex-row md:flex-wrap md:items-center"
class="flex w-full flex-wrap items-center gap-3"
role="group"
aria-label="API Key 筛选与操作"
>
<div class="min-w-0 w-full md:w-96 md:flex-none">
<div class="min-w-0 flex-1 md:w-96 md:flex-none">
<BaseInput v-model="search" placeholder="搜索名称、标签或 ID" class="w-full">
<template #prefix>
<Search class="size-4.5 text-(--cp-text-tertiary)" />
@@ -31,7 +31,7 @@ const search = defineModel<string>('search', { required: true })
</BaseInput>
</div>
<div class="flex shrink-0 self-end items-center justify-end gap-2 md:ml-auto">
<div class="flex shrink-0 items-center justify-end gap-2 md:ml-auto">
<BaseButton
v-if="selectedCount > 0"
variant="danger"
+1 -3
View File
@@ -88,10 +88,8 @@ const {
/>
<BaseCard
:padded="false"
class="mt-5 flex h-[calc(100dvh-136px)] min-h-125 flex-col"
header-class="px-5 pt-4"
body-class="flex min-h-0 flex-1 px-5 py-3"
body-class="mt-3 flex min-h-0 flex-1"
>
<template #header>
<ApiKeyFilters
@@ -78,58 +78,66 @@ function toneValueClass(tone: string) {
return metricToneValueClasses[tone as MetricTone]
}
const accountStatusCounts = computed(() => {
const p = props.pool
if (!p)
return null
const active = p.active ?? 0
const refreshing = p.refreshing ?? 0
const quotaLimited = (p.rateLimited ?? 0) + (p.quotaExhausted ?? 0)
const pending = (p.expired ?? 0) + (p.invalid ?? 0) + (p.disabled ?? 0) + (p.banned ?? 0)
return {
total: p.total,
active,
refreshing,
quotaLimited,
pending,
unavailable: quotaLimited + pending,
}
})
const statusRows = computed(() => {
const p = props.pool
if (!p) {
const counts = accountStatusCounts.value
if (!p || !counts) {
return [
{ label: '活跃账号', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
{ label: '刷新中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
{ label: '额度受限', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
{ label: '不可用', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
{ label: '待处理', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
]
}
const active = p?.active ?? 0
const refreshing = p?.refreshing ?? 0
const quota = p?.quotaExhausted ?? 0
const expired = p?.expired ?? 0
const invalid = p?.invalid ?? 0
const disabled = p?.disabled ?? 0
const banned = p?.banned ?? 0
const unavailable = expired + invalid + disabled + banned
return [
{
label: '活跃账号',
description: '可直接参与调度',
value: String(active),
value: String(counts.active),
tone: 'success',
icon: CircleCheck,
},
{
label: '刷新中',
description: '令牌自动刷新中',
value: String(refreshing),
value: String(counts.refreshing),
tone: 'normal',
icon: RefreshCw,
},
{
label: '限流中',
description: '任一额度窗口已触顶',
value: String(p?.rateLimited ?? 0),
label: '额度受限',
description: '配额耗尽与限流中',
value: String(counts.quotaLimited),
tone: 'warning',
icon: TriangleAlert,
},
{
label: '配额耗尽',
description: '已确认 402 耗尽',
value: String(quota),
tone: 'warning',
icon: TriangleAlert,
},
{
label: '不可用',
label: '待处理',
description: `过期 ${expired} · 失效 ${invalid} · 禁用 ${disabled} · 封禁 ${banned}`,
value: String(unavailable),
value: String(counts.pending),
tone: 'danger',
icon: ShieldAlert,
},
@@ -144,28 +152,22 @@ const availabilityRate = computed(() => {
})
const statusBars = computed(() => {
const p = props.pool
if (!p || p.total === 0)
const counts = accountStatusCounts.value
if (!counts || counts.total === 0)
return []
const active = (p.active / p.total) * 100
const refreshing = ((p.refreshing ?? 0) / p.total) * 100
const rateLimited = ((p.rateLimited ?? 0) / p.total) * 100
const quota = (p.quotaExhausted / p.total) * 100
const unavailable = ((p.expired + (p.invalid ?? 0) + p.disabled + p.banned) / p.total) * 100
return [
{ pct: active, cls: 'bg-(--cp-success)' },
{ pct: refreshing, cls: 'bg-(--cp-normal)' },
{ pct: rateLimited, cls: 'bg-(--cp-warning)' },
{ pct: quota, cls: 'bg-(--cp-warning)' },
{ pct: unavailable, cls: 'bg-(--cp-danger)' },
{ pct: (counts.active / counts.total) * 100, cls: 'bg-(--cp-success)' },
{ pct: (counts.refreshing / counts.total) * 100, cls: 'bg-(--cp-normal)' },
{ pct: (counts.quotaLimited / counts.total) * 100, cls: 'bg-(--cp-warning)' },
{ pct: (counts.pending / counts.total) * 100, cls: 'bg-(--cp-danger)' },
].filter(b => b.pct > 0)
})
</script>
<template>
<BaseCard as="article" :padded="false" class="w-full xl:h-112.5">
<BaseCard as="article" class="w-full xl:h-112.5">
<div
class="grid px-4 pt-5 pb-6 lg:px-7 lg:pt-6 xl:grid-cols-[minmax(0,0.9fr)_minmax(0,1.28fr)_minmax(280px,0.9fr)] xl:gap-7 xl:pb-0"
class="grid xl:grid-cols-[minmax(0,0.9fr)_minmax(0,1.28fr)_minmax(280px,0.9fr)] xl:gap-7"
>
<section class="min-w-0 w-full pb-6 xl:h-100.5 xl:pb-0">
<h2 class="m-0 text-xl leading-[1.15] font-heavy text-(--cp-text-primary)">
@@ -311,7 +313,7 @@ const statusBars = computed(() => {
<div class="mt-5.5 h-10.5 w-full">
<div class="flex h-4 items-center justify-between">
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-text-secondary)">状态分布</span>
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-danger-text)">不可用 {{ statusRows[3]?.value || '0' }}</span>
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-danger-text)">不可用 {{ accountStatusCounts?.unavailable ?? '—' }}</span>
</div>
<div class="mt-2.5 flex h-3 w-full overflow-hidden rounded-full bg-(--cp-bg-muted)">
<i
@@ -69,7 +69,7 @@ const sparklineOption = computed<EChartsOption | null>(() => {
</script>
<template>
<BaseCard as="article" :padded="false" class="relative h-38.5 w-full px-6 pt-5">
<BaseCard as="article" padding="compact" class="relative h-38.5 w-full">
<div class="flex items-start gap-3">
<BaseMotionIcon
aria-hidden="true"
@@ -207,7 +207,6 @@ onBeforeUnmount(() => {
<template>
<BaseCard
as="article"
variant="dashboard"
:title="timeline.title"
:description="timeline.description"
header-collapse-at="lg"
@@ -39,7 +39,7 @@ const {
</script>
<template>
<BaseCard as="article" variant="dashboard" title="使用趋势" class="min-h-95 w-full">
<BaseCard as="article" title="使用趋势" class="min-h-95 w-full">
<template #actions>
<BaseSegmented
v-model="activeKind"
@@ -17,7 +17,6 @@ const dashboardUsageRecordColumns = usageRecordColumns.filter(column => column.k
<template>
<BaseCard
as="article"
variant="dashboard"
title="使用记录"
description="最近 10 条成功请求"
class="h-117 w-full"
@@ -171,7 +171,6 @@ function providerLabel(provider: string) {
<template>
<BaseCard
as="article"
variant="dashboard"
title="上游请求身份"
body-class="flex min-h-0 flex-1 flex-col"
class="flex min-h-95 w-full flex-col"
+1 -1
View File
@@ -35,7 +35,7 @@ const {
<template>
<div class="w-full">
<BasePageHeader title="系统概览">
<BasePageHeader title="系统概览" actions-inline>
<template #description>
<span>当日统计</span>
<DashboardHeartbeat :updated-at="lastRefreshedAt" />
@@ -48,8 +48,7 @@ function togglePasswordVisible(): void {
<template>
<BaseCard
as="form"
:padded="false"
variant="elevated"
layout="flush"
class="login-form relative grid min-h-129.25 w-[min(440px,100%)] gap-3 rounded-lg px-7.5 pt-6.5 pb-6 max-[560px]:min-h-auto max-[560px]:gap-3 max-[560px]:p-5.5"
@submit.prevent="emit('submit')"
>
@@ -26,11 +26,9 @@ const emit = defineEmits<{
<template>
<BaseCard
:padded="false"
title="管理员 API Key"
description="用于外部系统集成,具有管理员权限"
header-class="px-5 pt-4"
body-class="px-5 py-5"
body-class="mt-5"
>
<template #actions>
<div class="flex flex-wrap items-center gap-2">
@@ -23,11 +23,9 @@ const emit = defineEmits<{
<template>
<BaseCard
:padded="false"
title="模型映射"
description="配置请求模型与上游模型的映射关系"
header-class="px-5 pt-4"
body-class="px-5 py-5"
body-class="mt-5"
>
<div class="grid gap-4">
<div class="flex flex-wrap items-center gap-3">
@@ -14,11 +14,9 @@ const model = defineModel<RotationStrategy | ''>({ required: true })
<template>
<BaseCard
:padded="false"
title="调度策略"
description="决定每次请求如何调度账号池"
header-class="px-5 pt-4"
body-class="px-5 py-5"
body-class="mt-5"
>
<div class="grid max-w-6xl gap-3 lg:grid-cols-4">
<button
@@ -14,11 +14,9 @@ const requestIntervalMs = defineModel<string>('requestIntervalMs', { required: t
<template>
<BaseCard
:padded="false"
title="运行参数"
description="请求节奏、账号并发和 Token 刷新"
header-class="px-5 pt-4"
body-class="px-5 py-5"
body-class="mt-5"
>
<BaseForm :columns="2" class="max-w-6xl">
<BaseFormItem
@@ -90,11 +90,9 @@ function canDelete(record: BackupRecord): boolean {
<template>
<BaseCard
:padded="false"
title="备份记录"
description="创建手动备份和管理已有备份记录"
header-class="px-5 pt-4"
body-class="px-5 py-3"
body-class="mt-3"
>
<template #actions>
<div class="flex flex-wrap items-center gap-2">
@@ -135,7 +133,7 @@ function canDelete(record: BackupRecord): boolean {
@page-size-change="emit('pageSizeChange', $event)"
>
<template #id="{ row }">
<span class="text-[12px] text-(--cp-text-secondary)" :title="row.id">{{ shortId(row.id) }}</span>
<span class="text-(--cp-text-secondary)" :title="row.id">{{ shortId(row.id) }}</span>
</template>
<template #status="{ row }">
@@ -143,27 +141,27 @@ function canDelete(record: BackupRecord): boolean {
</template>
<template #fileName="{ row }">
<span class="text-[12px] text-(--cp-text-secondary)" :title="row.objectKey">{{ formatFileName(row) }}</span>
<span class="text-(--cp-text-secondary)" :title="row.objectKey">{{ formatFileName(row) }}</span>
</template>
<template #sizeBytes="{ row }">
<span class="text-[12px] text-(--cp-text-secondary)">{{ formatSize(row.sizeBytes) }}</span>
<span class="text-(--cp-text-secondary)">{{ formatSize(row.sizeBytes) }}</span>
</template>
<template #expiresAt="{ row }">
<span class="text-[12px] text-(--cp-text-secondary)">
<span class="text-(--cp-text-secondary)">
{{ row.expiresAt ? formatDateTime(row.expiresAt) : '—' }}
</span>
</template>
<template #trigger="{ row }">
<span class="text-[12px] text-(--cp-text-secondary)">
<span class="text-(--cp-text-secondary)">
{{ triggerLabel(row.triggerKind) }}
</span>
</template>
<template #startedAt="{ row }">
<span class="text-[12px] text-(--cp-text-secondary)">
<span class="text-(--cp-text-secondary)">
{{ row.startedAt ? formatDateTime(row.startedAt) : '—' }}
</span>
</template>
@@ -40,11 +40,9 @@ const TIMEZONE_OPTIONS = [
<template>
<BaseCard
:padded="false"
title="定时备份"
description="配置自动定时备份"
header-class="px-5 pt-4"
body-class="px-5 py-5"
body-class="mt-5"
>
<template #actions>
<BaseButton variant="primary" :loading="saving" :disabled="loading" @click="emit('save')">
@@ -51,10 +51,8 @@ function toggleSecretVisible(): void {
<template>
<BaseCard
:padded="false"
title="S3 存储配置"
header-class="px-5 pt-4"
body-class="px-5 py-5"
body-class="mt-5"
>
<template #description>
<span class="text-(--cp-text-secondary)">
+14 -17
View File
@@ -71,24 +71,9 @@ watch(
<template>
<div class="w-full">
<BasePageHeader title="系统设置" description="管理运行参数、调度策略、模型映射与备份配置">
<template #actions>
<BaseButton
v-if="section === 'runtime'"
variant="primary"
:loading="saving"
:disabled="loading"
@click="saveSettings"
>
<template #icon>
<Save class="size-4" />
</template>
{{ saving ? '保存中...' : '保存' }}
</BaseButton>
</template>
</BasePageHeader>
<BasePageHeader title="系统设置" description="管理运行参数、调度策略、模型映射与备份配置" />
<div class="mt-4">
<div class="mt-4 flex flex-wrap items-center justify-between gap-3">
<BaseSegmented
:model-value="section"
class="bg-(--cp-input-soft-bg)!"
@@ -99,6 +84,18 @@ watch(
aria-label="设置分区"
@update:model-value="switchSection"
/>
<BaseButton
v-if="section === 'runtime'"
variant="primary"
:loading="saving"
:disabled="loading"
@click="saveSettings"
>
<template #icon>
<Save class="size-4" />
</template>
{{ saving ? '保存中...' : '保存' }}
</BaseButton>
</div>
<div v-show="section === 'runtime'" class="mt-5 grid w-full gap-5">
@@ -239,12 +239,10 @@ function formatTooltip(params: unknown) {
<template>
<BaseCard
as="article"
:padded="false"
title="成本效率"
description="预估费用、Token 与缓存收益"
header-collapse-at="none"
header-class="px-5 pt-4"
body-class="px-5 pt-3 pb-4"
body-class="mt-3"
class="h-full min-h-90"
>
<template #actions>
@@ -114,12 +114,10 @@ function diagnosticNameDisplay(name: string) {
<template>
<BaseCard
as="article"
:padded="false"
title="热点诊断"
:description="`按${selectedDimensionLabel}定位错误、慢请求与费用热点`"
header-collapse-at="lg"
header-class="px-5 pt-4"
body-class="min-h-0 min-w-0 px-5 pt-3"
body-class="mt-3 min-h-0 min-w-0"
class="grid h-105 min-h-105 max-h-105 min-w-0 w-full grid-rows-[auto_minmax(0,1fr)] lg:h-90 lg:min-h-90 lg:max-h-90"
>
<template #actions>
@@ -146,11 +146,9 @@ function formatTooltip(params: unknown) {
<template>
<BaseCard
as="article"
:padded="false"
title="请求健康"
:description="`按${granularityText}观察请求量、成功率与失败时段`"
header-class="px-5 pt-4"
body-class="px-5 pt-3 pb-4"
body-class="mt-3"
class="h-full min-h-90"
>
<template #body>
@@ -167,12 +167,10 @@ function formatTooltip(params: unknown) {
<template>
<BaseCard
as="article"
:padded="false"
title="响应速度"
description="总耗时或首字的分位耗时,越低越快"
header-collapse-at="none"
header-class="px-5 pt-4"
body-class="px-5 pt-3 pb-4"
body-class="mt-3"
class="h-full min-h-90"
>
<template #actions>
@@ -1,6 +1,7 @@
<script setup lang="ts">
import type { EChartsOption } from 'echarts'
import type { UsageViewModel } from '@/api'
import type { BaseTableColumn } from '@/components/base/BaseTable/columns'
import { computed } from 'vue'
import BaseButton from '@/components/base/BaseButton.vue'
@@ -114,6 +115,7 @@ const attemptColumns = [
label: '序号',
width: '56px',
align: 'center' as const,
fixed: false as const,
},
{ key: 'outcome', label: '结果', width: '76px' },
{ key: 'provider', label: '平台', width: '72px' },
@@ -127,7 +129,7 @@ const attemptColumns = [
width: '84px',
align: 'right' as const,
},
]
] satisfies BaseTableColumn[]
const attemptRows = computed(() =>
(props.record?.attempts ?? []).map(attempt => ({
@@ -414,7 +416,7 @@ const tokenDonutOption = computed<EChartsOption>(() => {
</span>
</div>
<BaseTable
class="mt-3 min-w-0"
class="mt-3 min-w-0 font-sans"
:columns="attemptColumns"
:rows="attemptRows"
compact
@@ -3,6 +3,7 @@ import type { getUsageRecordSummary } from '@/api'
import { Activity, Database, FileText, Timer } from '@lucide/vue'
import { computed } from 'vue'
import BaseCard from '@/components/base/BaseCard.vue'
import BaseMotionIcon from '@/components/base/BaseMotionIcon.vue'
const props = defineProps<{
@@ -54,10 +55,12 @@ const items = computed(() => [
class="mt-5 grid shrink-0 grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-4"
aria-label="使用概览"
>
<article
<BaseCard
v-for="item in items"
:key="item.key"
class="grid min-h-23 grid-cols-[36px_minmax(0,1fr)] items-stretch gap-3 rounded-(--cp-card-radius) bg-(--cp-bg-surface) px-4 py-3 shadow-(--cp-shadow-card)"
as="article"
padding="compact"
class="grid min-h-23 grid-cols-[36px_minmax(0,1fr)] items-stretch gap-3"
>
<BaseMotionIcon
aria-hidden="true"
@@ -79,6 +82,6 @@ const items = computed(() => [
{{ item.detail }}
</span>
</div>
</article>
</BaseCard>
</section>
</template>
+1
View File
@@ -5,6 +5,7 @@ export const usageRecordColumns = [
key: 'accountEmail',
label: '账号',
width: '280px',
fixed: false as const,
ellipsis: true,
},
{
+1 -3
View File
@@ -79,10 +79,8 @@ watch(timeRange, () => {
/>
<BaseCard
:padded="false"
class="mt-5 flex flex-col"
header-class="px-5 pt-4"
body-class="flex min-h-0 flex-col px-5 pt-3 pb-4"
body-class="mt-3 flex min-h-0 flex-col"
>
<template #header>
<div class="flex flex-wrap items-center justify-between gap-3">
+1 -1
View File
@@ -1 +1 @@
version: "3.1.2"
version: "3.1.6"