mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-09 08:06:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f34966e4c7 | ||
|
|
e9ebf45640 |
@@ -21,9 +21,8 @@ impl PgObservabilityRepository {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,用 Redis 429 冷却对每个非终态账号重新归类——
|
||||
/// 冷却中 → `rate_limited`(限流中);否则按 `quota_limit_reached` 归
|
||||
/// `quota_exhausted`(配额耗尽)或 `active`。冷却不可用时不修正(限流中计入配额耗尽)。
|
||||
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
|
||||
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
@@ -32,13 +31,16 @@ impl PgObservabilityRepository {
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool).await?;
|
||||
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
let mut active = 0_u64;
|
||||
let mut rate_limited = 0_u64;
|
||||
let mut quota_exhausted = 0_u64;
|
||||
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
|
||||
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
|
||||
let mut active = base.active;
|
||||
let mut rate_limited = base.rate_limited;
|
||||
let mut quota_exhausted = base.quota_exhausted;
|
||||
let now = SystemTime::now();
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
@@ -51,17 +53,24 @@ impl PgObservabilityRepository {
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > SystemTime::now())
|
||||
.filter(|cooldown| cooldown.until() > now)
|
||||
.is_some();
|
||||
if cooling {
|
||||
rate_limited += 1;
|
||||
} else if quota_reached {
|
||||
quota_exhausted += 1;
|
||||
} else {
|
||||
active += 1;
|
||||
if !cooling {
|
||||
continue;
|
||||
}
|
||||
if quota_reached {
|
||||
if quota_exhausted == 0 {
|
||||
continue;
|
||||
}
|
||||
quota_exhausted -= 1;
|
||||
} else {
|
||||
if active == 0 {
|
||||
continue;
|
||||
}
|
||||
active -= 1;
|
||||
}
|
||||
rate_limited += 1;
|
||||
}
|
||||
// 终态计数(expired/invalid/disabled/banned)保持 SQL 原值。
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
|
||||
@@ -357,8 +357,8 @@ pub(crate) async fn provider_account_metrics(
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when quota_limit_reached then 'rate_limited'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
when quota_limit_reached then 'quota_exhausted'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
@@ -425,19 +425,21 @@ pub(crate) async fn active_provider_account_ids(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询非终态候选账号:id + credential revision + 是否 quota 触顶。
|
||||
/// 仅含 ready/unknown 等可被 Redis 冷却影响的账号,终态(禁用/封禁/过期/无效)由
|
||||
/// [`provider_account_metrics`] 直接计数。
|
||||
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
|
||||
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability in ('ready', 'unknown')
|
||||
and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
|
||||
@@ -1,10 +1,23 @@
|
||||
use std::num::NonZeroU32;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
num::NonZeroU32,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::{
|
||||
model::{PageSize, observability as admin_observability},
|
||||
ports::store::ObservabilityStore as AdminObservabilityStore,
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError,
|
||||
},
|
||||
};
|
||||
use gateway_store::postgres::{
|
||||
DiagnosticDimension, ObservabilityPageNumber, ObservabilityPageSize, ObservabilityRange,
|
||||
ObservabilityRepository, OpsErrorFilter, OpsErrorQuery, PgAdminObservabilityStore,
|
||||
@@ -179,6 +192,184 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligible_accounts() {
|
||||
let Some(database) = TestDatabase::create("dashboard_account_metrics_cooldowns").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability, quota_limit_reached,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
|
||||
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
|
||||
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
|
||||
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
|
||||
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed cooldown account metric states");
|
||||
let cooldowns = StaticCooldowns::new([
|
||||
test_cooldown("acct_metrics_active_cooling", 2),
|
||||
test_cooldown("acct_metrics_cooling", 3),
|
||||
test_cooldown("acct_metrics_expired", 5),
|
||||
test_cooldown("acct_metrics_unknown", 6),
|
||||
test_cooldown("acct_metrics_stale_cooldown", 3),
|
||||
]);
|
||||
let repository =
|
||||
PgObservabilityRepository::new(database.pool.clone(), Some(Arc::new(cooldowns)));
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
let metrics = repository
|
||||
.dashboard_summary(range)
|
||||
.await
|
||||
.expect("dashboard summary")
|
||||
.provider_accounts;
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
metrics.total,
|
||||
metrics.enabled,
|
||||
metrics.active,
|
||||
metrics.rate_limited,
|
||||
metrics.quota_exhausted,
|
||||
metrics.expired,
|
||||
metrics.invalid,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
metrics.unavailable,
|
||||
),
|
||||
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.total,
|
||||
metrics.active
|
||||
+ metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.unavailable,
|
||||
metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
struct StaticCooldowns {
|
||||
cooldowns: BTreeMap<ProviderAccountId, ProviderCooldown>,
|
||||
}
|
||||
|
||||
impl StaticCooldowns {
|
||||
fn new(cooldowns: impl IntoIterator<Item = ProviderCooldown>) -> Self {
|
||||
Self {
|
||||
cooldowns: cooldowns
|
||||
.into_iter()
|
||||
.map(|cooldown| (cooldown.account_id().clone(), cooldown))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for StaticCooldowns {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(self.cooldowns.get(account_id).cloned()) })
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
|
||||
ProviderCooldown::new(
|
||||
ProviderAccountId::new(account_id).expect("test account ID"),
|
||||
CredentialRevision::new(revision).expect("test revision"),
|
||||
SystemTime::now() + Duration::from_secs(60),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
let Some(database) = TestDatabase::create("calculated_usage_billing_facts").await else {
|
||||
|
||||
@@ -603,6 +603,10 @@ impl CodexCredentialQuotaService {
|
||||
let Some(rate_limits) = parse_rate_limit_headers(headers) else {
|
||||
return Ok(false);
|
||||
};
|
||||
let has_quota_facts = rate_limits
|
||||
.limits
|
||||
.values()
|
||||
.any(|details| passive_rate_limit_snapshot(details).is_some());
|
||||
let existing = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
@@ -611,10 +615,35 @@ impl CodexCredentialQuotaService {
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
});
|
||||
let existing_observed_at = existing
|
||||
.as_ref()
|
||||
.and_then(|observation| observation.observed_at);
|
||||
let existing = existing
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
|
||||
// 也不能借旧快照重新推导 quota state。
|
||||
if !has_quota_facts {
|
||||
let Some(observed_at) = existing_observed_at else {
|
||||
return Ok(false);
|
||||
};
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(merge_passive_quota(
|
||||
existing,
|
||||
&rate_limits,
|
||||
))),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await?;
|
||||
return Ok(outcome != QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
let quota = merge_passive_quota(existing, &rate_limits);
|
||||
let observed_at = SystemTime::now();
|
||||
let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
|
||||
@@ -1024,6 +1053,14 @@ fn merge_passive_quota(
|
||||
.remove("additional_rate_limits")
|
||||
.and_then(|value| value.as_array().cloned())
|
||||
.unwrap_or_default();
|
||||
let replaces_core_limit = active_limit
|
||||
.and_then(|limit_id| rate_limits.limits.get(limit_id))
|
||||
.is_some_and(|details| passive_rate_limit_snapshot(details).is_some());
|
||||
if replaces_core_limit {
|
||||
// `codex` additional 与顶层 `rate_limit` 归为同一个 core 源。保留旧
|
||||
// additional 会在解析时反向覆盖当前 active limit,导致页面显示旧百分比。
|
||||
additional.retain(|item| !is_codex_core_alias(item));
|
||||
}
|
||||
for (limit_id, details) in &rate_limits.limits {
|
||||
let Some(rate_limit) = passive_rate_limit_snapshot(details) else {
|
||||
continue;
|
||||
@@ -1063,6 +1100,16 @@ fn merge_passive_quota(
|
||||
quota
|
||||
}
|
||||
|
||||
fn is_codex_core_alias(value: &Value) -> bool {
|
||||
value
|
||||
.get("metered_feature")
|
||||
.or_else(|| value.get("limit_name"))
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|source| {
|
||||
source.trim().to_ascii_lowercase().replace(['-', ' '], "_") == "codex"
|
||||
})
|
||||
}
|
||||
|
||||
fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option<Map<String, Value>> {
|
||||
// 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。
|
||||
let mut snapshot = Map::new();
|
||||
|
||||
@@ -348,7 +348,10 @@ impl CodexCredentialSelector {
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if self.skip_exhausted {
|
||||
let scheduling_candidates = if diagnostic {
|
||||
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
|
||||
candidates.clone()
|
||||
} else if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
|
||||
@@ -434,6 +434,125 @@ async fn passive_rate_limit_headers_replace_stale_secondary_window() {
|
||||
assert_eq!(monthly[0].used_percent(), Some(92.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_active_limit_replaces_stale_codex_alias() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_active_limit").await;
|
||||
let account = store.account("acct_passive_active_limit").expect("account");
|
||||
let baseline = json!({
|
||||
"additional_rate_limits": [{
|
||||
"metered_feature": "codex",
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 20,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-active-limit".to_owned(), "premium".to_owned()),
|
||||
("x-premium-primary-used-percent".to_owned(), "55".to_owned()),
|
||||
(
|
||||
"x-premium-primary-window-minutes".to_owned(),
|
||||
"10080".to_owned(),
|
||||
),
|
||||
(
|
||||
"x-premium-primary-reset-at".to_owned(),
|
||||
"1900000000".to_owned(),
|
||||
),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("passive quota")
|
||||
);
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(55.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_metadata_headers_preserve_quota_observation_time() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_metadata").await;
|
||||
let account = store.account("acct_passive_metadata").expect("account");
|
||||
let baseline = json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 20,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
let before = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read baseline quota");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-plan-type".to_owned(), "free".to_owned()),
|
||||
("x-codex-credits-has-credits".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-unlimited".to_owned(), "false".to_owned()),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("metadata-only passive response")
|
||||
);
|
||||
let after = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read updated quota metadata");
|
||||
|
||||
assert_eq!(after[0].observed_at, before[0].observed_at);
|
||||
assert_eq!(after[0].limit_reached, before[0].limit_reached);
|
||||
assert_eq!(
|
||||
store
|
||||
.quota_json("acct_passive_metadata")
|
||||
.and_then(|quota| quota.get("plan_type").cloned()),
|
||||
Some(json!("free"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn synchronize_without_accounts_is_a_noop_before_network_io() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -116,6 +116,21 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
) -> CodexProvider {
|
||||
provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store,
|
||||
session_affinity,
|
||||
base_url,
|
||||
leases,
|
||||
)
|
||||
.0
|
||||
}
|
||||
|
||||
fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -154,10 +169,10 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
true,
|
||||
));
|
||||
|
||||
CodexProvider::new(
|
||||
let provider = CodexProvider::new(
|
||||
selector,
|
||||
catalog,
|
||||
quota,
|
||||
Arc::clone("a),
|
||||
agent_identity,
|
||||
account_feedback,
|
||||
http,
|
||||
@@ -165,7 +180,8 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
base_url,
|
||||
websocket_pool,
|
||||
)
|
||||
.expect("official OpenAI provider")
|
||||
.expect("official OpenAI provider");
|
||||
(provider, quota)
|
||||
}
|
||||
|
||||
async fn create_account(store: &Arc<MemoryAccountStore>, id: &str) {
|
||||
@@ -1851,6 +1867,85 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
assert_eq!(affinity.binding_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_quota_limited_diagnostic";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("test account");
|
||||
let raw_quota = json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw_quota
|
||||
.as_object()
|
||||
.expect("quota snapshot object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed quota-limited snapshot");
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
|
||||
)),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (provider, quota) = provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
&store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
);
|
||||
quota
|
||||
.prepare_scheduling(std::slice::from_ref(&account))
|
||||
.await;
|
||||
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
diagnostic_context("req_quota_limited_diagnostic", account_id),
|
||||
)
|
||||
.await
|
||||
.expect("quota-limited diagnostic should prepare a fixed-account stream");
|
||||
let mut completed = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("quota-limited diagnostic upstream response");
|
||||
completed |= event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
}
|
||||
|
||||
assert!(completed);
|
||||
let requests = server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("captured quota-limited diagnostic request");
|
||||
assert_eq!(requests.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -60,6 +60,15 @@ function connectionTestErrorText(event: ConnectionTestEvent) {
|
||||
return event.error || '测试连接失败'
|
||||
}
|
||||
|
||||
function connectionTestFailureLabel(event: ConnectionTestEvent) {
|
||||
const hasUpstreamResponse
|
||||
= (typeof event.upstreamBody === 'string' && event.upstreamBody.length > 0)
|
||||
|| Number.isInteger(event.upstreamStatus)
|
||||
return hasUpstreamResponse
|
||||
? '上游返回错误'
|
||||
: '未能向上游发起请求'
|
||||
}
|
||||
|
||||
export function useAccountConnectionTest(options: AccountConnectionTestOptions) {
|
||||
const showConnectionTestModal = shallowRef(false)
|
||||
const testingAccount = shallowRef<Account | null>(null)
|
||||
@@ -293,7 +302,11 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
if (event.type === 'error') {
|
||||
applyAccountStatus(event.accountStatus)
|
||||
connectionTestError.value = connectionTestErrorText(event)
|
||||
appendConnectionTestLog('上游返回错误', 'danger', connectionTestError.value)
|
||||
appendConnectionTestLog(
|
||||
connectionTestFailureLabel(event),
|
||||
'danger',
|
||||
connectionTestError.value,
|
||||
)
|
||||
finishConnectionTest('error')
|
||||
clearConnectionTestRun()
|
||||
}
|
||||
|
||||
@@ -78,53 +78,66 @@ function toneValueClass(tone: string) {
|
||||
return metricToneValueClasses[tone as MetricTone]
|
||||
}
|
||||
|
||||
const accountStatusCounts = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p)
|
||||
return null
|
||||
const active = p.active ?? 0
|
||||
const refreshing = p.refreshing ?? 0
|
||||
const quotaLimited = (p.rateLimited ?? 0) + (p.quotaExhausted ?? 0)
|
||||
const pending = (p.expired ?? 0) + (p.invalid ?? 0) + (p.disabled ?? 0) + (p.banned ?? 0)
|
||||
return {
|
||||
total: p.total,
|
||||
active,
|
||||
refreshing,
|
||||
quotaLimited,
|
||||
pending,
|
||||
unavailable: quotaLimited + pending,
|
||||
}
|
||||
})
|
||||
|
||||
const statusRows = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p) {
|
||||
const counts = accountStatusCounts.value
|
||||
if (!p || !counts) {
|
||||
return [
|
||||
{ label: '活跃账号', description: '暂无账号池观测', value: '—', tone: 'success', icon: CircleCheck },
|
||||
{ label: '刷新中', description: '暂无账号池观测', value: '—', tone: 'normal', icon: RefreshCw },
|
||||
{ label: '额度受限', description: '暂无账号池观测', value: '—', tone: 'warning', icon: TriangleAlert },
|
||||
{ label: '不可用', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
|
||||
{ label: '待处理', description: '暂无账号池观测', value: '—', tone: 'danger', icon: ShieldAlert },
|
||||
]
|
||||
}
|
||||
const active = p?.active ?? 0
|
||||
const refreshing = p?.refreshing ?? 0
|
||||
const rateLimited = p?.rateLimited ?? 0
|
||||
const quotaExhausted = p?.quotaExhausted ?? 0
|
||||
const quotaLimited = rateLimited + quotaExhausted
|
||||
const expired = p?.expired ?? 0
|
||||
const invalid = p?.invalid ?? 0
|
||||
const disabled = p?.disabled ?? 0
|
||||
const banned = p?.banned ?? 0
|
||||
const unavailable = expired + invalid + disabled + banned
|
||||
|
||||
return [
|
||||
{
|
||||
label: '活跃账号',
|
||||
description: '可直接参与调度',
|
||||
value: String(active),
|
||||
value: String(counts.active),
|
||||
tone: 'success',
|
||||
icon: CircleCheck,
|
||||
},
|
||||
{
|
||||
label: '刷新中',
|
||||
description: '令牌自动刷新中',
|
||||
value: String(refreshing),
|
||||
value: String(counts.refreshing),
|
||||
tone: 'normal',
|
||||
icon: RefreshCw,
|
||||
},
|
||||
{
|
||||
label: '额度受限',
|
||||
description: '配额耗尽与限流中',
|
||||
value: String(quotaLimited),
|
||||
value: String(counts.quotaLimited),
|
||||
tone: 'warning',
|
||||
icon: TriangleAlert,
|
||||
},
|
||||
{
|
||||
label: '不可用',
|
||||
label: '待处理',
|
||||
description: `过期 ${expired} · 失效 ${invalid} · 禁用 ${disabled} · 封禁 ${banned}`,
|
||||
value: String(unavailable),
|
||||
value: String(counts.pending),
|
||||
tone: 'danger',
|
||||
icon: ShieldAlert,
|
||||
},
|
||||
@@ -139,18 +152,14 @@ const availabilityRate = computed(() => {
|
||||
})
|
||||
|
||||
const statusBars = computed(() => {
|
||||
const p = props.pool
|
||||
if (!p || p.total === 0)
|
||||
const counts = accountStatusCounts.value
|
||||
if (!counts || counts.total === 0)
|
||||
return []
|
||||
const active = (p.active / p.total) * 100
|
||||
const refreshing = ((p.refreshing ?? 0) / p.total) * 100
|
||||
const quotaLimited = (((p.rateLimited ?? 0) + p.quotaExhausted) / p.total) * 100
|
||||
const unavailable = ((p.expired + (p.invalid ?? 0) + p.disabled + p.banned) / p.total) * 100
|
||||
return [
|
||||
{ pct: active, cls: 'bg-(--cp-success)' },
|
||||
{ pct: refreshing, cls: 'bg-(--cp-normal)' },
|
||||
{ pct: quotaLimited, cls: 'bg-(--cp-warning)' },
|
||||
{ pct: unavailable, cls: 'bg-(--cp-danger)' },
|
||||
{ pct: (counts.active / counts.total) * 100, cls: 'bg-(--cp-success)' },
|
||||
{ pct: (counts.refreshing / counts.total) * 100, cls: 'bg-(--cp-normal)' },
|
||||
{ pct: (counts.quotaLimited / counts.total) * 100, cls: 'bg-(--cp-warning)' },
|
||||
{ pct: (counts.pending / counts.total) * 100, cls: 'bg-(--cp-danger)' },
|
||||
].filter(b => b.pct > 0)
|
||||
})
|
||||
</script>
|
||||
@@ -304,7 +313,7 @@ const statusBars = computed(() => {
|
||||
<div class="mt-5.5 h-10.5 w-full">
|
||||
<div class="flex h-4 items-center justify-between">
|
||||
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-text-secondary)">状态分布</span>
|
||||
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-danger-text)">不可用 {{ statusRows[3]?.value || '0' }}</span>
|
||||
<span class="text-xs leading-[1.15] font-emphasis text-(--cp-danger-text)">不可用 {{ accountStatusCounts?.unavailable ?? '—' }}</span>
|
||||
</div>
|
||||
<div class="mt-2.5 flex h-3 w-full overflow-hidden rounded-full bg-(--cp-bg-muted)">
|
||||
<i
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.1.4"
|
||||
version: "3.1.5"
|
||||
|
||||
Reference in New Issue
Block a user