mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-09 08:06:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f34966e4c7 | ||
|
|
e9ebf45640 | ||
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 |
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -58,12 +58,14 @@ impl AccountStatus {
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
@@ -84,8 +86,10 @@ impl AccountStatusSignals {
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.quota_limit_reached {
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
@@ -135,7 +139,7 @@ impl AccountErrorReason {
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -595,6 +595,9 @@ pub struct ProviderQuota {
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -217,10 +217,11 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, quota.limit_reached, now);
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
Ok(AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason: derive_error_reason(signals),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
quota,
|
||||
@@ -305,10 +306,11 @@ impl AccountsService for DefaultAccountsService {
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, quota.limit_reached, now);
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason: derive_error_reason(signals),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
@@ -519,6 +521,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -526,7 +529,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
@@ -535,7 +538,8 @@ fn account_signals(
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached,
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -545,7 +549,11 @@ fn account_status(
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
account_signals(account, quota_limit_reached, now).derive()
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
|
||||
@@ -128,6 +128,7 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
|
||||
@@ -1015,6 +1015,7 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
@@ -1119,7 +1120,8 @@ async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → rate_limited。
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
@@ -1137,6 +1139,7 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1161,6 +1164,51 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -1181,6 +1229,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1289,6 +1338,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,6 +210,8 @@ pub struct AccountView {
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -1418,10 +1420,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
|
||||
|
||||
use gateway_core::engine::credential::ProviderAccountStore;
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -85,7 +86,10 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
)?);
|
||||
|
||||
let admin_ports = AdminStorePorts::new(
|
||||
Arc::new(postgres::PgAdminAccountStore::new(pool.clone())),
|
||||
Arc::new(postgres::PgAdminAccountStore::new(
|
||||
pool.clone(),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminAuthStoreAdapter {
|
||||
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
|
||||
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
|
||||
@@ -98,6 +102,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
Arc::new(postgres::PgAdminObservabilityStore::new(
|
||||
pool.clone(),
|
||||
Some(credential_leases.clone()),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminSettingsStoreAdapter {
|
||||
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
@@ -7,12 +12,82 @@ use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepositor
|
||||
#[derive(Clone)]
|
||||
pub struct PgObservabilityRepository {
|
||||
pool: PgPool,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgObservabilityRepository {
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
pub const fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
|
||||
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let base = provider_account_metrics(&self.pool, observed_at).await?;
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
|
||||
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
|
||||
let mut active = base.active;
|
||||
let mut rate_limited = base.rate_limited;
|
||||
let mut quota_exhausted = base.quota_exhausted;
|
||||
let now = SystemTime::now();
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let cooling = cooldowns
|
||||
.read(&account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > now)
|
||||
.is_some();
|
||||
if !cooling {
|
||||
continue;
|
||||
}
|
||||
if quota_reached {
|
||||
if quota_exhausted == 0 {
|
||||
continue;
|
||||
}
|
||||
quota_exhausted -= 1;
|
||||
} else {
|
||||
if active == 0 {
|
||||
continue;
|
||||
}
|
||||
active -= 1;
|
||||
}
|
||||
rate_limited += 1;
|
||||
}
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
unavailable: rate_limited
|
||||
+ quota_exhausted
|
||||
+ base.expired
|
||||
+ base.invalid
|
||||
+ base.disabled
|
||||
+ base.banned,
|
||||
active,
|
||||
rate_limited,
|
||||
expired: base.expired,
|
||||
invalid: base.invalid,
|
||||
quota_exhausted,
|
||||
disabled: base.disabled,
|
||||
banned: base.banned,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,9 +103,13 @@ pub struct PgAdminObservabilityStore {
|
||||
|
||||
impl PgAdminObservabilityStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool, runtime_signals: Option<RedisCredentialLeaseRepository>) -> Self {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
runtime_signals: Option<RedisCredentialLeaseRepository>,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository: PgObservabilityRepository::new(pool),
|
||||
repository: PgObservabilityRepository::new(pool, cooldowns),
|
||||
runtime_signals,
|
||||
}
|
||||
}
|
||||
@@ -62,11 +141,13 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
};
|
||||
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
|
||||
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts, provider_accounts) = futures::try_join!(
|
||||
let (requests, attempts) = futures::try_join!(
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
provider_account_metrics(&self.pool, range.end),
|
||||
)?;
|
||||
let provider_accounts = self
|
||||
.provider_account_metrics_with_cooldowns(range.end)
|
||||
.await?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
|
||||
@@ -357,8 +357,8 @@ pub(crate) async fn provider_account_metrics(
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when quota_limit_reached then 'rate_limited'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
when quota_limit_reached then 'quota_exhausted'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
@@ -425,6 +425,35 @@ pub(crate) async fn active_provider_account_ids(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
|
||||
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(&row, "id")?,
|
||||
get(&row, "credential_revision")?,
|
||||
get(&row, "quota_limit_reached")?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
//! `gateway-admin` 账号端口的 PostgreSQL adapter。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。
|
||||
@@ -11,19 +17,49 @@ pub struct PgAdminAccountStore {
|
||||
accounts: PgProviderAccountRepository,
|
||||
observability: PgObservabilityRepository,
|
||||
control_plane: PgControlPlaneRepository,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgAdminAccountStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
pub fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self {
|
||||
pool: pool.clone(),
|
||||
accounts: PgProviderAccountRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone(), cooldowns.clone()),
|
||||
control_plane: PgControlPlaneRepository::new(pool),
|
||||
cooldowns,
|
||||
}
|
||||
}
|
||||
|
||||
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
|
||||
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
|
||||
async fn load_rate_limited_until(
|
||||
&self,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
let mut map = BTreeMap::new();
|
||||
for account in accounts {
|
||||
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
|
||||
continue;
|
||||
};
|
||||
if cooldown.credential_revision().get() != account.credential_revision.get() {
|
||||
continue;
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until > SystemTime::now() {
|
||||
map.insert(account.id.clone(), until);
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
async fn usage_observations(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
@@ -242,11 +278,17 @@ impl AccountStore for PgAdminAccountStore {
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let now = Utc::now();
|
||||
let summary = admin_account_summary(&accounts, now);
|
||||
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
|
||||
let rate_limited_until = self
|
||||
.load_rate_limited_until(&accounts)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
|
||||
let mut items = accounts
|
||||
.into_iter()
|
||||
.filter_map(|account| {
|
||||
let status = admin_account_status(&account, now);
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
let status = admin_account_status(&account, now, until);
|
||||
account_matches_admin_query(&account, status, &query).then_some(
|
||||
AdminAccountListItem {
|
||||
account,
|
||||
|
||||
@@ -60,6 +60,7 @@ pub(crate) fn account_matches_admin_query(
|
||||
pub(crate) fn admin_account_status(
|
||||
account: &ProviderAccountSummary,
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AdminAccountStatus {
|
||||
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
|
||||
AccountStatusSignals {
|
||||
@@ -69,6 +70,7 @@ pub(crate) fn admin_account_status(
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: account.quota_limit_reached,
|
||||
rate_limited_until,
|
||||
}
|
||||
.derive()
|
||||
}
|
||||
@@ -76,6 +78,7 @@ pub(crate) fn admin_account_status(
|
||||
pub(crate) fn admin_account_summary(
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: &BTreeMap<String, SystemTime>,
|
||||
) -> AccountSummary {
|
||||
let mut summary = AccountSummary {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
@@ -86,7 +89,8 @@ pub(crate) fn admin_account_summary(
|
||||
error: 0,
|
||||
};
|
||||
for account in accounts {
|
||||
match admin_account_status(account, now) {
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
match admin_account_status(account, now, until) {
|
||||
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
|
||||
AdminAccountStatus::QuotaExhausted => {
|
||||
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt,
|
||||
str::FromStr,
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -1,10 +1,23 @@
|
||||
use std::num::NonZeroU32;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
num::NonZeroU32,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::{
|
||||
model::{PageSize, observability as admin_observability},
|
||||
ports::store::ObservabilityStore as AdminObservabilityStore,
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError,
|
||||
},
|
||||
};
|
||||
use gateway_store::postgres::{
|
||||
DiagnosticDimension, ObservabilityPageNumber, ObservabilityPageSize, ObservabilityRange,
|
||||
ObservabilityRepository, OpsErrorFilter, OpsErrorQuery, PgAdminObservabilityStore,
|
||||
@@ -83,7 +96,7 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
.expect("persist opaque response IDs");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let records = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
@@ -147,7 +160,7 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed account metric states");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
@@ -179,6 +192,184 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligible_accounts() {
|
||||
let Some(database) = TestDatabase::create("dashboard_account_metrics_cooldowns").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability, quota_limit_reached,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
|
||||
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
|
||||
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
|
||||
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
|
||||
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed cooldown account metric states");
|
||||
let cooldowns = StaticCooldowns::new([
|
||||
test_cooldown("acct_metrics_active_cooling", 2),
|
||||
test_cooldown("acct_metrics_cooling", 3),
|
||||
test_cooldown("acct_metrics_expired", 5),
|
||||
test_cooldown("acct_metrics_unknown", 6),
|
||||
test_cooldown("acct_metrics_stale_cooldown", 3),
|
||||
]);
|
||||
let repository =
|
||||
PgObservabilityRepository::new(database.pool.clone(), Some(Arc::new(cooldowns)));
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
let metrics = repository
|
||||
.dashboard_summary(range)
|
||||
.await
|
||||
.expect("dashboard summary")
|
||||
.provider_accounts;
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
metrics.total,
|
||||
metrics.enabled,
|
||||
metrics.active,
|
||||
metrics.rate_limited,
|
||||
metrics.quota_exhausted,
|
||||
metrics.expired,
|
||||
metrics.invalid,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
metrics.unavailable,
|
||||
),
|
||||
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.total,
|
||||
metrics.active
|
||||
+ metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.unavailable,
|
||||
metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
struct StaticCooldowns {
|
||||
cooldowns: BTreeMap<ProviderAccountId, ProviderCooldown>,
|
||||
}
|
||||
|
||||
impl StaticCooldowns {
|
||||
fn new(cooldowns: impl IntoIterator<Item = ProviderCooldown>) -> Self {
|
||||
Self {
|
||||
cooldowns: cooldowns
|
||||
.into_iter()
|
||||
.map(|cooldown| (cooldown.account_id().clone(), cooldown))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for StaticCooldowns {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(self.cooldowns.get(account_id).cloned()) })
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
|
||||
ProviderCooldown::new(
|
||||
ProviderAccountId::new(account_id).expect("test account ID"),
|
||||
CredentialRevision::new(revision).expect("test revision"),
|
||||
SystemTime::now() + Duration::from_secs(60),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
let Some(database) = TestDatabase::create("calculated_usage_billing_facts").await else {
|
||||
@@ -193,7 +384,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
.expect("seed calculated billing facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let facts = repository
|
||||
.usage_calculated_billing_facts(range, UsageRecordFilter::default())
|
||||
@@ -207,7 +398,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(facts[0].total.amount.as_str(), "1.25");
|
||||
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
let facts = store
|
||||
.usage_calculated_billing_facts(
|
||||
admin_observability::TimeRange::new(range.start, range.end)
|
||||
@@ -235,7 +426,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
let range =
|
||||
admin_observability::TimeRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("admin observability range");
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
|
||||
let dashboard = store
|
||||
.dashboard_summary(range)
|
||||
@@ -524,7 +715,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.expect("seed observability facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let dashboard = repository
|
||||
.dashboard_summary(range)
|
||||
|
||||
@@ -361,7 +361,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("seed charlie usage");
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let usage_page = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
@@ -380,8 +380,8 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(usage_page.total, 6);
|
||||
assert_eq!(usage_page.summary.total, 6);
|
||||
assert_eq!(usage_page.summary.normal, 1);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 1);
|
||||
assert_eq!(usage_page.summary.rate_limited, 1);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 2);
|
||||
assert_eq!(usage_page.summary.rate_limited, 0);
|
||||
assert_eq!(usage_page.summary.disabled, 1);
|
||||
assert_eq!(usage_page.summary.error, 2);
|
||||
assert_eq!(
|
||||
@@ -481,7 +481,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.expect("insert xAI credential fixture");
|
||||
}
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let provider = ProviderKind::new("xai").expect("xAI Provider kind");
|
||||
let complete = store
|
||||
.list_credentials(
|
||||
@@ -562,7 +562,7 @@ async fn terminal_admin_usage_chunks_large_selections_and_preserves_exact_costs(
|
||||
let mut account_ids = vec!["acct_usage_exact".to_owned()];
|
||||
account_ids.extend((0..200).map(|index| format!("missing_account_{index}")));
|
||||
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage(
|
||||
TimeRange {
|
||||
start: now - TimeDelta::hours(1),
|
||||
@@ -654,7 +654,7 @@ async fn terminal_admin_quota_window_usage_prefers_provider_total_and_falls_back
|
||||
.await
|
||||
.expect("separate provider total from fallback components");
|
||||
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage_by_windows(&[
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_quota_window".to_owned(),
|
||||
@@ -694,7 +694,7 @@ async fn terminal_admin_mutations_keep_revision_account_and_audit_atomic() {
|
||||
.insert_provider_account(account("acct_terminal_mutation", "user-terminal-mutation"))
|
||||
.await
|
||||
.expect("insert mutation account");
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let context = MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "request_terminal_mutation".to_owned(),
|
||||
@@ -767,7 +767,7 @@ async fn terminal_admin_delete_removes_enabled_accounts_in_one_transaction() {
|
||||
.expect("insert enabled account");
|
||||
}
|
||||
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone())
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
account_ids: vec![
|
||||
|
||||
@@ -39,7 +39,7 @@ async fn request_snapshots_should_survive_account_deletion() {
|
||||
.await
|
||||
.expect("delete provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
@@ -154,7 +154,7 @@ async fn attempts_should_keep_their_own_account_snapshots() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let detail = repository
|
||||
.usage_record_detail("req_snap_ab")
|
||||
.await
|
||||
@@ -276,7 +276,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let errors = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range: range_around(started_at),
|
||||
@@ -376,7 +376,7 @@ async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -445,7 +445,7 @@ async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
|
||||
.expect("insert legacy request without live account");
|
||||
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -511,7 +511,7 @@ async fn failure_diagnostics_should_only_include_errored_requests() {
|
||||
.await
|
||||
.expect("mark request failed");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -566,7 +566,7 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
|
||||
.await
|
||||
.expect("set request b latency");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -633,7 +633,7 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
||||
.await
|
||||
.expect("rename provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
|
||||
@@ -475,7 +475,14 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
};
|
||||
Ok(project_quota(snapshot, &account))
|
||||
let mut quota = project_quota(snapshot, &account);
|
||||
quota.rate_limited_until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
.map(DateTime::<Utc>::from);
|
||||
Ok(quota)
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -740,6 +747,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -811,6 +819,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
refresh_token_expires_at: None,
|
||||
windows,
|
||||
limit_reached,
|
||||
rate_limited_until: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
provider_data,
|
||||
))),
|
||||
|
||||
@@ -13,19 +13,20 @@ pub use snapshot::{
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
|
||||
use gateway_protocol::openai::events::{
|
||||
ParsedRateLimits, RateLimitDetails, RateLimitWindow, parse_rate_limit_headers,
|
||||
};
|
||||
use reqwest::Client;
|
||||
use secrecy::ExposeSecret;
|
||||
use serde_json::{Map, Number, Value};
|
||||
use serde_json::{Map, Value};
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
@@ -120,6 +121,7 @@ pub struct CodexCredentialQuotaService {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
agent_identity: Arc<CodexAgentIdentityTaskService>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
scheduling: CodexQuotaSchedulingProjection,
|
||||
}
|
||||
|
||||
@@ -379,6 +381,7 @@ impl CodexCredentialQuotaService {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
agent_identity: Arc<CodexAgentIdentityTaskService>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store: Arc::clone(repository.store()),
|
||||
@@ -387,6 +390,7 @@ impl CodexCredentialQuotaService {
|
||||
http,
|
||||
base_url,
|
||||
agent_identity,
|
||||
cooldowns,
|
||||
scheduling: CodexQuotaSchedulingProjection::default(),
|
||||
}
|
||||
}
|
||||
@@ -599,6 +603,10 @@ impl CodexCredentialQuotaService {
|
||||
let Some(rate_limits) = parse_rate_limit_headers(headers) else {
|
||||
return Ok(false);
|
||||
};
|
||||
let has_quota_facts = rate_limits
|
||||
.limits
|
||||
.values()
|
||||
.any(|details| passive_rate_limit_snapshot(details).is_some());
|
||||
let existing = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
@@ -607,10 +615,35 @@ impl CodexCredentialQuotaService {
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
});
|
||||
let existing_observed_at = existing
|
||||
.as_ref()
|
||||
.and_then(|observation| observation.observed_at);
|
||||
let existing = existing
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
|
||||
// 也不能借旧快照重新推导 quota state。
|
||||
if !has_quota_facts {
|
||||
let Some(observed_at) = existing_observed_at else {
|
||||
return Ok(false);
|
||||
};
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(merge_passive_quota(
|
||||
existing,
|
||||
&rate_limits,
|
||||
))),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await?;
|
||||
return Ok(outcome != QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
let quota = merge_passive_quota(existing, &rate_limits);
|
||||
let observed_at = SystemTime::now();
|
||||
let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
|
||||
@@ -658,92 +691,56 @@ impl CodexCredentialQuotaService {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// 真实 429 的单一事实入口:合成/更新 quota 主窗口(`limit_reached=true`、
|
||||
/// `used_percent=100`、`reset_at=retry-after`)。availability 不改变,
|
||||
/// 窗口 reset 到期后由投影自动滚动恢复(429 只记录限流窗口)。
|
||||
/// 已有更晚的 `reset_at` 不会被缩短(never-shrink)。
|
||||
/// 真实 429 的单一事实入口:写入 Redis 临时限流冷却(`until = now + retry_after`)。
|
||||
/// availability 与 quota 主窗口(额度重置时间)都不改变——临时限流是独立维度,
|
||||
/// 到期由 Redis key 过期自动解除,不污染配额耗尽状态。
|
||||
/// 已有更晚的冷却不会被缩短(put_if_later)。
|
||||
pub async fn apply_rate_limit_429(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
retry_after: Option<Duration>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let mut object = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
let new_reset_at = observed_at
|
||||
let until = observed_at
|
||||
.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN))
|
||||
.and_then(|reset| reset.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|duration| duration.as_secs() as i64);
|
||||
let existing_reset_at = object
|
||||
.get("rate_limit")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|rate_limit| rate_limit.get("primary_window"))
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|window| window.get("reset_at"))
|
||||
.and_then(Value::as_i64);
|
||||
let reset_at = match (existing_reset_at, new_reset_at) {
|
||||
(Some(existing), Some(next)) if existing > next => Some(existing),
|
||||
(_, next) => next,
|
||||
};
|
||||
let mut rate_limit = object
|
||||
.get("rate_limit")
|
||||
.cloned()
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
|
||||
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
|
||||
let mut primary = rate_limit
|
||||
.get("primary_window")
|
||||
.cloned()
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
let used_percent = primary
|
||||
.get("used_percent")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(0.0)
|
||||
.max(100.0);
|
||||
primary.insert(
|
||||
"used_percent".to_owned(),
|
||||
Number::from_f64(used_percent)
|
||||
.map(Value::Number)
|
||||
.unwrap_or_else(|| Value::Number(Number::from(100))),
|
||||
);
|
||||
if let Some(reset_at) = reset_at {
|
||||
primary.insert("reset_at".to_owned(), Value::from(reset_at));
|
||||
}
|
||||
rate_limit.insert("primary_window".to_owned(), Value::Object(primary));
|
||||
object.insert("rate_limit".to_owned(), Value::Object(rate_limit));
|
||||
if self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(object)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await?
|
||||
== QuotaWriteOutcome::Conflict
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
// 429 合成的限流窗口要立即反映到调度信号(reset 到期由投影滚动解除)。
|
||||
if let Ok(Some(snapshot)) = self.read_snapshot_for(account).await {
|
||||
self.scheduling.observe(&snapshot);
|
||||
}
|
||||
.unwrap_or(observed_at);
|
||||
self.cooldowns
|
||||
.put_if_later(ProviderCooldown::new(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
until,
|
||||
))
|
||||
.await
|
||||
.map_err(|error| CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 读取账号当前是否处于临时限流(429)冷却,及到期时间。
|
||||
pub async fn rate_limited_until(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
|
||||
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
|
||||
CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
}
|
||||
})?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if cooldown.credential_revision() != credential_revision {
|
||||
return Ok(None);
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until <= SystemTime::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(until))
|
||||
}
|
||||
|
||||
/// 读取单账号最后一次落库的 Provider quota,并由 Codex 域解析展示窗口。
|
||||
pub async fn read_account(
|
||||
&self,
|
||||
@@ -1056,6 +1053,14 @@ fn merge_passive_quota(
|
||||
.remove("additional_rate_limits")
|
||||
.and_then(|value| value.as_array().cloned())
|
||||
.unwrap_or_default();
|
||||
let replaces_core_limit = active_limit
|
||||
.and_then(|limit_id| rate_limits.limits.get(limit_id))
|
||||
.is_some_and(|details| passive_rate_limit_snapshot(details).is_some());
|
||||
if replaces_core_limit {
|
||||
// `codex` additional 与顶层 `rate_limit` 归为同一个 core 源。保留旧
|
||||
// additional 会在解析时反向覆盖当前 active limit,导致页面显示旧百分比。
|
||||
additional.retain(|item| !is_codex_core_alias(item));
|
||||
}
|
||||
for (limit_id, details) in &rate_limits.limits {
|
||||
let Some(rate_limit) = passive_rate_limit_snapshot(details) else {
|
||||
continue;
|
||||
@@ -1095,6 +1100,16 @@ fn merge_passive_quota(
|
||||
quota
|
||||
}
|
||||
|
||||
fn is_codex_core_alias(value: &Value) -> bool {
|
||||
value
|
||||
.get("metered_feature")
|
||||
.or_else(|| value.get("limit_name"))
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|source| {
|
||||
source.trim().to_ascii_lowercase().replace(['-', ' '], "_") == "codex"
|
||||
})
|
||||
}
|
||||
|
||||
fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option<Map<String, Value>> {
|
||||
// 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。
|
||||
let mut snapshot = Map::new();
|
||||
|
||||
@@ -294,6 +294,24 @@ impl CodexCredentialSelector {
|
||||
if !diagnostic {
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
}
|
||||
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
|
||||
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
|
||||
let accounts = if diagnostic {
|
||||
accounts
|
||||
} else {
|
||||
let mut retained = Vec::with_capacity(accounts.len());
|
||||
for account in accounts {
|
||||
let cooling = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
if cooling.is_none() {
|
||||
retained.push(account);
|
||||
}
|
||||
}
|
||||
retained
|
||||
};
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
@@ -330,7 +348,10 @@ impl CodexCredentialSelector {
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if self.skip_exhausted {
|
||||
let scheduling_candidates = if diagnostic {
|
||||
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
|
||||
candidates.clone()
|
||||
} else if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
|
||||
@@ -111,6 +111,7 @@ pub async fn initialize(
|
||||
http.clone(),
|
||||
config.base_url().to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
ports.cooldowns(),
|
||||
));
|
||||
let quota_skip_exhausted = config.quota_skip_exhausted();
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
|
||||
@@ -12,7 +12,9 @@ use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{ProviderSessionAffinityKey, ProviderSessionAffinityPort};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::OFFICIAL_CODEX_BASE_URL;
|
||||
use provider_openai::credential::{
|
||||
@@ -25,8 +27,8 @@ use secrecy::ExposeSecret;
|
||||
use url::Url;
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
};
|
||||
|
||||
fn create_account(store: &Arc<MemoryAccountStore>, id: &str, token: &str) {
|
||||
@@ -99,6 +101,7 @@ fn selector_with_affinity(
|
||||
session_affinity,
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -108,6 +111,7 @@ fn selector_with_runtime(
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
skip_exhausted: bool,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> CodexCredentialSelector {
|
||||
let profile = CodexWireProfileState::new(CodexWireProfile {
|
||||
originator: "codex_cli_rs".to_owned(),
|
||||
@@ -136,6 +140,7 @@ fn selector_with_runtime(
|
||||
http,
|
||||
OFFICIAL_CODEX_BASE_URL.to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
cooldowns,
|
||||
));
|
||||
CodexCredentialSelector::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
@@ -464,6 +469,7 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
|
||||
affinity,
|
||||
Arc::clone(&account_feedback),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
for _ in 0..4 {
|
||||
account_feedback.report(
|
||||
@@ -683,7 +689,15 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
let attempt = attempt(BTreeSet::new());
|
||||
let lease =
|
||||
block_on(
|
||||
@@ -706,33 +720,18 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不改变 availability(限流不改变账号可用性),只把限流窗口合成进 quota JSON。
|
||||
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
|
||||
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
let quota = store.quota_json("acct_primary").expect("quota");
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/limit_reached")
|
||||
.and_then(serde_json::Value::as_bool),
|
||||
Some(true)
|
||||
);
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(serde_json::Value::as_f64),
|
||||
Some(100.0)
|
||||
);
|
||||
assert!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/reset_at")
|
||||
.and_then(serde_json::Value::as_i64)
|
||||
.is_some_and(|reset_at| {
|
||||
reset_at
|
||||
> SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("clock")
|
||||
.as_secs() as i64
|
||||
})
|
||||
store.quota_json("acct_primary").is_none(),
|
||||
"429 must not synthesize quota window"
|
||||
);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown expiring in the future"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -798,7 +797,15 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
@@ -816,8 +823,13 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
// 429 合成限流窗口需要读写 quota JSON。
|
||||
assert!(store.quota_reads() >= 1);
|
||||
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -949,6 +961,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
|
||||
@@ -71,6 +71,7 @@ fn quota_service_with_base_url(
|
||||
http,
|
||||
base_url,
|
||||
crate::support::agent_identity_service(store),
|
||||
Arc::new(crate::support::MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -433,6 +434,125 @@ async fn passive_rate_limit_headers_replace_stale_secondary_window() {
|
||||
assert_eq!(monthly[0].used_percent(), Some(92.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_active_limit_replaces_stale_codex_alias() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_active_limit").await;
|
||||
let account = store.account("acct_passive_active_limit").expect("account");
|
||||
let baseline = json!({
|
||||
"additional_rate_limits": [{
|
||||
"metered_feature": "codex",
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 20,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-active-limit".to_owned(), "premium".to_owned()),
|
||||
("x-premium-primary-used-percent".to_owned(), "55".to_owned()),
|
||||
(
|
||||
"x-premium-primary-window-minutes".to_owned(),
|
||||
"10080".to_owned(),
|
||||
),
|
||||
(
|
||||
"x-premium-primary-reset-at".to_owned(),
|
||||
"1900000000".to_owned(),
|
||||
),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("passive quota")
|
||||
);
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(55.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_metadata_headers_preserve_quota_observation_time() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_metadata").await;
|
||||
let account = store.account("acct_passive_metadata").expect("account");
|
||||
let baseline = json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 20,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
let before = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read baseline quota");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-plan-type".to_owned(), "free".to_owned()),
|
||||
("x-codex-credits-has-credits".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-unlimited".to_owned(), "false".to_owned()),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("metadata-only passive response")
|
||||
);
|
||||
let after = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read updated quota metadata");
|
||||
|
||||
assert_eq!(after[0].observed_at, before[0].observed_at);
|
||||
assert_eq!(after[0].limit_reached, before[0].limit_reached);
|
||||
assert_eq!(
|
||||
store
|
||||
.quota_json("acct_passive_metadata")
|
||||
.and_then(|quota| quota.get("plan_type").cloned()),
|
||||
Some(json!("free"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn synchronize_without_accounts_is_a_noop_before_network_io() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -43,8 +43,9 @@ use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service_with_pool, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service_with_pool, catalog_cache, profile,
|
||||
secret,
|
||||
};
|
||||
use crate::transport::accept_codex_test_websocket;
|
||||
|
||||
@@ -115,6 +116,21 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
) -> CodexProvider {
|
||||
provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store,
|
||||
session_affinity,
|
||||
base_url,
|
||||
leases,
|
||||
)
|
||||
.0
|
||||
}
|
||||
|
||||
fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -136,6 +152,7 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
http.clone(),
|
||||
base_url.clone(),
|
||||
Arc::clone(&agent_identity),
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
));
|
||||
let account_feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
@@ -152,10 +169,10 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
true,
|
||||
));
|
||||
|
||||
CodexProvider::new(
|
||||
let provider = CodexProvider::new(
|
||||
selector,
|
||||
catalog,
|
||||
quota,
|
||||
Arc::clone("a),
|
||||
agent_identity,
|
||||
account_feedback,
|
||||
http,
|
||||
@@ -163,7 +180,8 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
base_url,
|
||||
websocket_pool,
|
||||
)
|
||||
.expect("official OpenAI provider")
|
||||
.expect("official OpenAI provider");
|
||||
(provider, quota)
|
||||
}
|
||||
|
||||
async fn create_account(store: &Arc<MemoryAccountStore>, id: &str) {
|
||||
@@ -1849,6 +1867,85 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
assert_eq!(affinity.binding_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_quota_limited_diagnostic";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("test account");
|
||||
let raw_quota = json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw_quota
|
||||
.as_object()
|
||||
.expect("quota snapshot object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed quota-limited snapshot");
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
|
||||
)),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (provider, quota) = provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
&store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
);
|
||||
quota
|
||||
.prepare_scheduling(std::slice::from_ref(&account))
|
||||
.await;
|
||||
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
diagnostic_context("req_quota_limited_diagnostic", account_id),
|
||||
)
|
||||
.await
|
||||
.expect("quota-limited diagnostic should prepare a fixed-account stream");
|
||||
let mut completed = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("quota-limited diagnostic upstream response");
|
||||
completed |= event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
}
|
||||
|
||||
assert!(completed);
|
||||
let requests = server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("captured quota-limited diagnostic request");
|
||||
assert_eq!(requests.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -16,9 +16,10 @@ use gateway_core::engine::credential::{
|
||||
};
|
||||
use gateway_core::error::{StoreError, StoreErrorKind};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderLeaseAcquisition, ProviderLeasePort,
|
||||
ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
|
||||
ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderSessionAffinityKey,
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort,
|
||||
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest,
|
||||
ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey,
|
||||
ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
ProviderStoreError,
|
||||
};
|
||||
@@ -771,3 +772,90 @@ pub(crate) fn codex_account(id: &str) -> ProviderAccount {
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(true, Some(SystemTime::now() + Duration::from_secs(2_700)))
|
||||
}
|
||||
|
||||
/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector
|
||||
/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct MemoryCooldownPort {
|
||||
pub(crate) cooldowns: Arc<Mutex<BTreeMap<ProviderAccountId, ProviderCooldown>>>,
|
||||
}
|
||||
|
||||
impl MemoryCooldownPort {
|
||||
#[must_use]
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut cooldowns = self.cooldowns.lock().expect("cooldown lock");
|
||||
let should_write = cooldowns.get(cooldown.account_id()).is_none_or(|current| {
|
||||
current.credential_revision() < cooldown.credential_revision()
|
||||
|| (current.credential_revision() == cooldown.credential_revision()
|
||||
&& current.until() < cooldown.until())
|
||||
});
|
||||
if should_write {
|
||||
cooldowns.insert(cooldown.account_id().clone(), cooldown);
|
||||
}
|
||||
Ok(should_write)
|
||||
})
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
Ok(self
|
||||
.cooldowns
|
||||
.lock()
|
||||
.expect("cooldown lock")
|
||||
.get(account_id)
|
||||
.cloned())
|
||||
})
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1048,6 +1048,7 @@ fn project_quota(
|
||||
refresh_token_expires_at,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
};
|
||||
@@ -1115,6 +1116,7 @@ fn project_quota(
|
||||
limit_reached: billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ export interface AccountQuotaWindow {
|
||||
export interface AccountQuota {
|
||||
refreshedAtDisplay: string
|
||||
limitReached: boolean
|
||||
// 429 临时限流(Redis 冷却)到期时间;非限流中为 null。
|
||||
rateLimitedUntil: string | null
|
||||
windows: AccountQuotaWindow[]
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ const props = withDefaults(
|
||||
offset?: number
|
||||
width?: number | string
|
||||
disabled?: boolean
|
||||
/** hover 触发时打开弹窗的延迟(毫秒);click 触发不生效。 */
|
||||
hoverDelay?: number
|
||||
panelClass?: string
|
||||
triggerClass?: string
|
||||
anchorElement?: HTMLElement | null
|
||||
@@ -39,6 +41,7 @@ const props = withDefaults(
|
||||
trigger: 'click',
|
||||
offset: 6,
|
||||
disabled: false,
|
||||
hoverDelay: 400,
|
||||
panelClass: '',
|
||||
triggerClass: '',
|
||||
anchorElement: null,
|
||||
@@ -55,6 +58,7 @@ const popoverRef = ref<HTMLElement | null>(null)
|
||||
const popoverStyle = ref<CSSProperties>({})
|
||||
const popoverArrowStyle = ref<CSSProperties>({})
|
||||
const hoverCloseTimer = shallowRef<number>()
|
||||
const hoverOpenTimer = shallowRef<number>()
|
||||
const viewportTarget = computed(() => (open.value && typeof window !== 'undefined' ? window : null))
|
||||
|
||||
const popoverClasses = computed(() => [
|
||||
@@ -260,6 +264,7 @@ async function openPopover() {
|
||||
if (props.disabled || open.value)
|
||||
return
|
||||
|
||||
clearHoverOpenTimer()
|
||||
clearHoverCloseTimer()
|
||||
open.value = true
|
||||
await nextTick()
|
||||
@@ -293,13 +298,28 @@ function clearHoverCloseTimer() {
|
||||
hoverCloseTimer.value = undefined
|
||||
}
|
||||
|
||||
function clearHoverOpenTimer() {
|
||||
if (hoverOpenTimer.value === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
window.clearTimeout(hoverOpenTimer.value)
|
||||
hoverOpenTimer.value = undefined
|
||||
}
|
||||
|
||||
function handleHoverEnter() {
|
||||
if (props.trigger !== 'hover') {
|
||||
return
|
||||
}
|
||||
|
||||
clearHoverCloseTimer()
|
||||
void openPopover()
|
||||
// 延迟打开:鼠标稳定悬停一小段时间再展示,避免扫过表格时弹窗频繁闪烁。
|
||||
if (hoverOpenTimer.value === undefined) {
|
||||
hoverOpenTimer.value = window.setTimeout(() => {
|
||||
hoverOpenTimer.value = undefined
|
||||
void openPopover()
|
||||
}, props.hoverDelay)
|
||||
}
|
||||
}
|
||||
|
||||
function handleHoverLeave() {
|
||||
@@ -307,6 +327,7 @@ function handleHoverLeave() {
|
||||
return
|
||||
}
|
||||
|
||||
clearHoverOpenTimer()
|
||||
clearHoverCloseTimer()
|
||||
hoverCloseTimer.value = window.setTimeout(closePopover, 90)
|
||||
}
|
||||
@@ -341,7 +362,10 @@ useEventListener(viewportTarget, 'keydown', (event) => {
|
||||
})
|
||||
useEventListener(viewportTarget, 'resize', updatePopoverPositionThrottled)
|
||||
useEventListener(viewportTarget, 'scroll', updatePopoverPositionThrottled, { capture: true })
|
||||
onBeforeUnmount(clearHoverCloseTimer)
|
||||
onBeforeUnmount(() => {
|
||||
clearHoverOpenTimer()
|
||||
clearHoverCloseTimer()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { AlertTriangle, CircleCheck, Gauge, Power, Timer } from '@lucide/vue'
|
||||
import { computed } from 'vue'
|
||||
|
||||
import BasePopover from '@/components/base/BasePopover.vue'
|
||||
@@ -11,11 +12,14 @@ const props = withDefaults(
|
||||
errorReason?: string | null
|
||||
/** 最近一次失败的上游错误原文(对应后端 `errorMessage`),作为辅助证据展示。 */
|
||||
errorMessage?: string | null
|
||||
// 429 临时限流到期时间(对应后端 `quota.rateLimitedUntil`);`status === 'rate_limited'` 时有值。
|
||||
rateLimitedUntil?: string | null
|
||||
variant?: 'inline' | 'pill'
|
||||
}>(),
|
||||
{
|
||||
errorReason: null,
|
||||
errorMessage: null,
|
||||
rateLimitedUntil: null,
|
||||
variant: 'inline',
|
||||
},
|
||||
)
|
||||
@@ -30,7 +34,35 @@ const reasonLabel = computed(() => {
|
||||
})
|
||||
/** 上游错误原文(辅助证据);有则显、无则隐。 */
|
||||
const errorText = computed(() => props.errorMessage || null)
|
||||
const hasDetail = computed(() => Boolean(reasonLabel.value) || Boolean(errorText.value))
|
||||
// 429 临时限流中且带到期时间。
|
||||
const hasRateLimit = computed(() =>
|
||||
props.status === 'rate_limited' && Boolean(props.rateLimitedUntil),
|
||||
)
|
||||
/** 限流到期相对时间:如「剩余 12 分钟」。到点返回 null。 */
|
||||
const rateLimitedRelative = computed(() => {
|
||||
if (!hasRateLimit.value)
|
||||
return null
|
||||
const until = new Date(props.rateLimitedUntil!.replace(' ', 'T')).getTime()
|
||||
const diffMinutes = Math.round((until - Date.now()) / 60_000)
|
||||
if (Number.isNaN(diffMinutes) || diffMinutes < 1)
|
||||
return null
|
||||
if (diffMinutes < 60)
|
||||
return `剩余 ${diffMinutes} 分钟`
|
||||
const hours = Math.floor(diffMinutes / 60)
|
||||
const mins = diffMinutes % 60
|
||||
if (mins === 0)
|
||||
return `剩余 ${hours} 小时`
|
||||
return `剩余 ${hours} 小时 ${mins} 分`
|
||||
})
|
||||
// `error` 状态始终显示弹窗(即使 reason/message 为空也展示状态卡片),
|
||||
// `rate_limited` 需要带到期时间才显示;其余状态不弹窗。
|
||||
const hasDetail = computed(() =>
|
||||
props.status === 'error' || hasRateLimit.value,
|
||||
)
|
||||
/** 弹窗明细区是否渲染(错误原因 / 限流到期 / 错误反馈至少一项)。 */
|
||||
const hasDetailRows = computed(() =>
|
||||
hasRateLimit.value || Boolean(reasonLabel.value) || Boolean(errorText.value),
|
||||
)
|
||||
|
||||
const textClass = computed(() => {
|
||||
if (tone.value === 'success') {
|
||||
@@ -63,6 +95,32 @@ const dotClass = computed(() => {
|
||||
}
|
||||
return 'bg-(--cp-text-muted)'
|
||||
})
|
||||
|
||||
/** 状态卡片左侧图标块:状态色 soft-bg + 语义图标。 */
|
||||
const cardIcon = computed(() => {
|
||||
switch (props.status) {
|
||||
case 'error': return AlertTriangle
|
||||
case 'rate_limited': return Timer
|
||||
case 'quota_exhausted': return Gauge
|
||||
case 'disabled': return Power
|
||||
default: return CircleCheck
|
||||
}
|
||||
})
|
||||
const iconBlockClass = computed(() => {
|
||||
if (tone.value === 'success') {
|
||||
return 'bg-(--cp-success-bg) text-(--cp-success-text)'
|
||||
}
|
||||
if (tone.value === 'danger') {
|
||||
return 'bg-(--cp-danger-bg) text-(--cp-danger-text)'
|
||||
}
|
||||
if (tone.value === 'warning') {
|
||||
return 'bg-(--cp-warning-bg) text-(--cp-warning-text)'
|
||||
}
|
||||
if (tone.value === 'info') {
|
||||
return 'bg-(--cp-info-bg) text-(--cp-info-text)'
|
||||
}
|
||||
return 'bg-(--cp-bg-subtle) text-(--cp-text-secondary)'
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -71,7 +129,7 @@ const dotClass = computed(() => {
|
||||
trigger="hover"
|
||||
placement="top-start"
|
||||
:width="240"
|
||||
panel-class="p-2"
|
||||
panel-class="!p-3"
|
||||
>
|
||||
<template #trigger>
|
||||
<span
|
||||
@@ -90,11 +148,46 @@ const dotClass = computed(() => {
|
||||
<span>{{ label }}</span>
|
||||
</span>
|
||||
</template>
|
||||
<p v-if="reasonLabel" class="m-0 text-[12px] font-medium leading-relaxed text-(--cp-text-primary)">
|
||||
{{ reasonLabel }}
|
||||
</p>
|
||||
<p v-if="errorText" class="m-0 mt-1 text-[12px] leading-relaxed text-(--cp-text-secondary)">
|
||||
{{ errorText }}
|
||||
</p>
|
||||
|
||||
<div
|
||||
class="flex gap-3 text-[12px] leading-none"
|
||||
:class="hasDetailRows ? 'items-start' : 'items-center'"
|
||||
>
|
||||
<!-- 状态图标块:soft 状态色背景圆角块 -->
|
||||
<span
|
||||
class="inline-flex size-8 shrink-0 items-center justify-center rounded-(--cp-icon-button-radius)"
|
||||
:class="iconBlockClass"
|
||||
>
|
||||
<component :is="cardIcon" class="size-4" />
|
||||
</span>
|
||||
|
||||
<div class="min-w-0 flex-1">
|
||||
<!-- 状态标题行:只显示状态名 -->
|
||||
<div class="font-heavy text-(--cp-text-primary)">
|
||||
{{ label }}
|
||||
</div>
|
||||
|
||||
<div v-if="hasDetailRows" class="mt-2 border-t border-(--cp-divider-subtle) pt-2">
|
||||
<!-- 错误:具体原因独立成行 -->
|
||||
<div v-if="reasonLabel" class="leading-4 text-(--cp-text-secondary)">
|
||||
{{ reasonLabel }}
|
||||
</div>
|
||||
<!-- 限流中:显示相对到期时间 -->
|
||||
<div v-if="hasRateLimit" class="flex items-baseline gap-2">
|
||||
<span class="w-20 shrink-0 font-semibold text-(--cp-text-tertiary)">限流到期</span>
|
||||
<span class="font-medium text-(--cp-text-primary)">
|
||||
{{ rateLimitedRelative ?? props.rateLimitedUntil }}
|
||||
</span>
|
||||
</div>
|
||||
<!-- 错误:显示上游反馈原文 -->
|
||||
<div v-if="errorText" class="mt-1 flex items-start gap-2">
|
||||
<span class="w-14 shrink-0 pt-px font-semibold leading-3.5 text-(--cp-text-tertiary)">上游反馈</span>
|
||||
<span class="min-w-0 flex-1 wrap-break-word leading-3.5 text-(--cp-text-secondary)">
|
||||
{{ errorText }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</BasePopover>
|
||||
</template>
|
||||
|
||||
@@ -60,6 +60,15 @@ function connectionTestErrorText(event: ConnectionTestEvent) {
|
||||
return event.error || '测试连接失败'
|
||||
}
|
||||
|
||||
function connectionTestFailureLabel(event: ConnectionTestEvent) {
|
||||
const hasUpstreamResponse
|
||||
= (typeof event.upstreamBody === 'string' && event.upstreamBody.length > 0)
|
||||
|| Number.isInteger(event.upstreamStatus)
|
||||
return hasUpstreamResponse
|
||||
? '上游返回错误'
|
||||
: '未能向上游发起请求'
|
||||
}
|
||||
|
||||
export function useAccountConnectionTest(options: AccountConnectionTestOptions) {
|
||||
const showConnectionTestModal = shallowRef(false)
|
||||
const testingAccount = shallowRef<Account | null>(null)
|
||||
@@ -293,7 +302,11 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
if (event.type === 'error') {
|
||||
applyAccountStatus(event.accountStatus)
|
||||
connectionTestError.value = connectionTestErrorText(event)
|
||||
appendConnectionTestLog('上游返回错误', 'danger', connectionTestError.value)
|
||||
appendConnectionTestLog(
|
||||
connectionTestFailureLabel(event),
|
||||
'danger',
|
||||
connectionTestError.value,
|
||||
)
|
||||
finishConnectionTest('error')
|
||||
clearConnectionTestRun()
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ const {
|
||||
:status="derivedAccountStatus(row)"
|
||||
:error-reason="row.errorReason"
|
||||
:error-message="row.errorMessage"
|
||||
:rate-limited-until="row.quota.rateLimitedUntil"
|
||||
/>
|
||||
</template>
|
||||
|
||||
|
||||
@@ -78,53 +78,66 @@ function toneValueClass(tone: string) {
|
||||
return metricToneValueClasses[tone as MetricTone]
|
||||
}
|
||||
|
||||
const accountStatusCounts = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p)
|
||||
return null
|
||||
const active = p.active ?? 0
|
||||
const refreshing = p.refreshing ?? 0
|
||||
const quotaLimited = (p.rateLimited ?? 0) + (p.quotaExhausted ?? 0)
|
||||
const pending = (p.expired ?? 0) + (p.invalid ?? 0) + (p.disabled ?? 0) + (p.banned ?? 0)
|
||||
return {
|
||||
total: p.total,
|
||||
active,
|
||||
refreshing,
|
||||
quotaLimited,
|
||||
pending,
|
||||
unavailable: quotaLimited + pending,
|
||||
}
|
||||
})
|
||||
|
||||
const statusRows = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p) {
|
||||
const counts = accountStatusCounts.value
|
||||
if (!p || !counts) {
|
||||
return [
|
||||
{ label: '活跃账号', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
|
||||
{ label: '刷新中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
|
||||
{ label: '额度受限', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
|
||||
{ label: '不可用', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
|
||||
{ label: '待处理', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
|
||||
]
|
||||
}
|
||||
const active = p?.active ?? 0
|
||||
const refreshing = p?.refreshing ?? 0
|
||||
const rateLimited = p?.rateLimited ?? 0
|
||||
const quotaExhausted = p?.quotaExhausted ?? 0
|
||||
const quotaLimited = rateLimited + quotaExhausted
|
||||
const expired = p?.expired ?? 0
|
||||
const invalid = p?.invalid ?? 0
|
||||
const disabled = p?.disabled ?? 0
|
||||
const banned = p?.banned ?? 0
|
||||
const unavailable = expired + invalid + disabled + banned
|
||||
|
||||
return [
|
||||
{
|
||||
label: '活跃账号',
|
||||
description: '可直接参与调度',
|
||||
value: String(active),
|
||||
value: String(counts.active),
|
||||
tone: 'success',
|
||||
icon: CircleCheck,
|
||||
},
|
||||
{
|
||||
label: '刷新中',
|
||||
description: '令牌自动刷新中',
|
||||
value: String(refreshing),
|
||||
value: String(counts.refreshing),
|
||||
tone: 'normal',
|
||||
icon: RefreshCw,
|
||||
},
|
||||
{
|
||||
label: '额度受限',
|
||||
description: `限流中 ${rateLimited} · 配额耗尽 ${quotaExhausted}`,
|
||||
value: String(quotaLimited),
|
||||
description: '配额耗尽与限流中',
|
||||
value: String(counts.quotaLimited),
|
||||
tone: 'warning',
|
||||
icon: TriangleAlert,
|
||||
},
|
||||
{
|
||||
label: '不可用',
|
||||
label: '待处理',
|
||||
description: `过期 ${expired} · 失效 ${invalid} · 禁用 ${disabled} · 封禁 ${banned}`,
|
||||
value: String(unavailable),
|
||||
value: String(counts.pending),
|
||||
tone: 'danger',
|
||||
icon: ShieldAlert,
|
||||
},
|
||||
@@ -139,18 +152,14 @@ const availabilityRate = computed(() => {
|
||||
})
|
||||
|
||||
const statusBars = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p || p.total === 0)
|
||||
const counts = accountStatusCounts.value
|
||||
if (!counts || counts.total === 0)
|
||||
return []
|
||||
const active = (p.active / p.total) * 100
|
||||
const refreshing = ((p.refreshing ?? 0) / p.total) * 100
|
||||
const quotaLimited = (((p.rateLimited ?? 0) + p.quotaExhausted) / p.total) * 100
|
||||
const unavailable = ((p.expired + (p.invalid ?? 0) + p.disabled + p.banned) / p.total) * 100
|
||||
return [
|
||||
{ pct: active, cls: 'bg-(--cp-success)' },
|
||||
{ pct: refreshing, cls: 'bg-(--cp-normal)' },
|
||||
{ pct: quotaLimited, cls: 'bg-(--cp-warning)' },
|
||||
{ pct: unavailable, cls: 'bg-(--cp-danger)' },
|
||||
{ pct: (counts.active / counts.total) * 100, cls: 'bg-(--cp-success)' },
|
||||
{ pct: (counts.refreshing / counts.total) * 100, cls: 'bg-(--cp-normal)' },
|
||||
{ pct: (counts.quotaLimited / counts.total) * 100, cls: 'bg-(--cp-warning)' },
|
||||
{ pct: (counts.pending / counts.total) * 100, cls: 'bg-(--cp-danger)' },
|
||||
].filter(b => b.pct > 0)
|
||||
})
|
||||
</script>
|
||||
@@ -304,7 +313,7 @@ const statusBars = computed(() => {
|
||||
<div class="mt-5.5 h-10.5 w-full">
|
||||
<div class="flex h-4 items-center justify-between">
|
||||
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-text-secondary)">状态分布</span>
|
||||
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-danger-text)">不可用 {{ statusRows[3]?.value || '0' }}</span>
|
||||
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-danger-text)">不可用 {{ accountStatusCounts?.unavailable ?? '—' }}</span>
|
||||
</div>
|
||||
<div class="mt-2.5 flex h-3 w-full overflow-hidden rounded-full bg-(--cp-bg-muted)">
|
||||
<i
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.1.3"
|
||||
version: "3.1.5"
|
||||
|
||||
Reference in New Issue
Block a user