feat(accounts): add opt-in capacity cooldown and recovery (#126)

Add opt-in account capacity cooldown, recovery probes, and atomic adaptive concurrency updates. Include overload and slow_down in Smart scheduling feedback, expose rate-limit reasons, and keep probe results fenced by cooldown generation.

Exclude diagnostic probes and local connection failures from capacity evidence. Default auto-freeze to disabled and preserve explicitly saved settings with a forward migration.

Closes #120

Co-authored-by: zhangwentian <2han9wen71an@gmail.com>
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
2han9wen71an
2026-09-17 03:23:38 +08:00
committed by GitHub
co-authored by Zyy Codex
parent ae2da31380
commit 499b43043e
72 changed files with 3599 additions and 242 deletions
+3
View File
@@ -83,6 +83,9 @@ API Key 持有者可在同一登录页切换登录身份,进入 `/key-usage`
2. 按需建立账号分组,再创建客户端密钥并选择可用分组。**不选分组表示可使用全部账号**。
3. 打开密钥的「使用密钥」,复制客户端配置。
账号自动冻结默认关闭,可在「系统设置 → 运行参数」中启用。启用后,账号频繁命中上游容量错误时
会暂停调度并显示为限流中;若开启恢复前探测,冷却到期后需探测成功才恢复。
## 客户端接入
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入。
@@ -0,0 +1,304 @@
//! 容量熔断冻结的恢复编排:到期探测、失败顺延与自适应并发下调。
//!
//! 冻结事实保存在可丢失的 Redis 冷却中,本服务只组合 Admin 端口:
//! 观测(`AccountRuntimeStore`)、探测(`AccountsService`)与写回
//! (原子降低并发 / 按冻结代次结束探测)。旧观测不会覆盖管理员操作。
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use chrono::{DateTime, Utc};
use futures::StreamExt as _;
use gateway_core::account::{AccountConcurrencyLimit, ProviderAccountId};
use gateway_core::routing::UpstreamModelId;
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
use tracing::warn;
use crate::model::accounts::{
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
};
use crate::model::{MutationActor, MutationContext};
use crate::ports::store::{AccountRuntimeStore, AccountStore, SettingsStore};
use crate::use_case::accounts::AccountsService;
/// 自适应并发下调保留系数:下调到观测在途峰值的 80%。
const ADAPTIVE_CONCURRENCY_FACTOR: f64 = 0.8;
/// 自适应并发下限:过低的并发让账号几乎不可用,宁可保持冻结。
const ADAPTIVE_CONCURRENCY_FLOOR: u32 = 2;
/// worker 周期;冷却观测与探测都依赖该节奏。
pub const FREEZE_RECOVERY_INTERVAL: Duration = Duration::from_secs(30);
pub const FREEZE_RECOVERY_WORKER_OWNER: &str = "account-freeze-recovery";
pub const WORKER_INITIAL_BACKOFF: Duration = Duration::from_secs(1);
pub const WORKER_MAXIMUM_BACKOFF: Duration = Duration::from_secs(60);
pub const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60);
pub const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60);
/// 系统审计上下文的固定请求 ID;账号并发更新审计由此溯源到自动冻结编排。
const SYSTEM_REQUEST_ID: &str = "account-freeze-recovery";
/// 冻结恢复所需的管理端口组合。
#[derive(Clone)]
pub struct FreezeRecoveryDeps {
pub accounts: Arc<dyn AccountsService>,
pub store: Arc<dyn AccountStore>,
pub runtime: Arc<dyn AccountRuntimeStore>,
pub settings: Arc<dyn SettingsStore>,
}
pub struct FreezeRecoveryTask {
deps: FreezeRecoveryDeps,
}
impl FreezeRecoveryTask {
pub fn new(deps: FreezeRecoveryDeps) -> Self {
Self { deps }
}
fn system_context() -> MutationContext {
MutationContext {
actor: MutationActor::System,
request_id: SYSTEM_REQUEST_ID.to_owned(),
}
}
/// 读取冻结策略;读取或校验失败跳过本轮,不能把未知配置当作允许解冻。
async fn freeze_policy(&self) -> Option<gateway_core::provider_ports::ProviderFreezePolicy> {
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
settings.account_auto_freeze_enabled,
settings.account_auto_freeze_threshold,
settings.account_auto_freeze_window_seconds,
settings.account_auto_freeze_duration_seconds,
settings.account_auto_freeze_probe_enabled,
settings.account_auto_freeze_probe_model,
settings.account_auto_freeze_adaptive_concurrency,
)
.ok()
}
async fn active_freezes(&self) -> BTreeMap<String, AccountFreeze> {
self.deps.runtime.active_freezes().await.unwrap_or_default()
}
async fn run_cycle_inner(&self) {
let Some(policy) = self.freeze_policy().await else {
return;
};
let freezes = self.active_freezes().await;
if freezes.is_empty() {
return;
}
if policy.enabled() && policy.adaptive_concurrency() {
self.adapt_concurrency_limits(&freezes, &policy).await;
}
self.recover_due_freezes(&freezes, &policy).await;
}
/// 对冻结中的账号执行自适应并发下调:目标为观测在途峰值的 80%(下限 2),
/// 只降不升;未观测到峰值证据的账号保持现状。
async fn adapt_concurrency_limits(
&self,
freezes: &BTreeMap<String, AccountFreeze>,
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
) {
let account_ids = freezes.keys().cloned().collect::<Vec<_>>();
let Ok(peaks) = self.deps.runtime.capacity_peaks(&account_ids).await else {
return;
};
for (account_id, peak) in peaks {
let Some(target) = adaptive_target(peak) else {
continue;
};
let Ok(account) = ProviderAccountId::new(account_id.clone()) else {
continue;
};
let Some(limit) = AccountConcurrencyLimit::new(target) else {
continue;
};
match self
.deps
.accounts
.lower_concurrency_limit(&Self::system_context(), account, limit)
.await
{
Ok(Some(_)) => {
tracing::info!(
account_id,
adapted_limit = target,
freeze_seconds = policy.freeze_duration().as_secs(),
"容量熔断:按观测在途峰值下调账号并发上限",
);
}
Ok(None) => {}
Err(error) => {
warn!(account_id, error = %error, "自适应并发下调失败");
}
}
}
}
async fn load_account(&self, account_id: &str) -> Option<Option<AccountPageItem>> {
let account_id = ProviderAccountId::new(account_id.to_owned()).ok()?;
self.deps
.store
.load_account(account_id.as_str(), AccountRuntimeSnapshot::default())
.await
.ok()
}
/// 到达探测时间后才恢复;关闭自动冻结或探测时,已有冻结仍等待原冷却结束。
async fn recover_due_freezes(
&self,
freezes: &BTreeMap<String, AccountFreeze>,
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
) {
for (account_id, freeze) in freezes {
if freeze.until > Utc::now() {
continue;
}
if policy.enabled() && policy.probe_enabled() && freeze.requires_probe {
self.probe_and_recover(account_id, freeze, policy).await;
} else if let Err(error) = self
.deps
.runtime
.finish_freeze(account_id, freeze, None)
.await
{
warn!(account_id, error = %error, "解除到期冻结失败");
}
}
}
async fn probe_and_recover(
&self,
account_id: &str,
freeze: &AccountFreeze,
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
) {
let Some(Some(item)) = self.load_account(account_id).await else {
return;
};
if !item.account.enabled {
// 停用账号没有自动恢复意义;解冻交给管理员手动恢复。
return;
}
let Ok(account) = ProviderAccountId::new(account_id.to_owned()) else {
return;
};
let Some(model) = self.resolve_probe_model(&account, policy).await else {
// 没有可用探测模型时按失败处理,顺延冻结等待下一轮。
self.postpone(&account, freeze, policy).await;
return;
};
let probe_succeeded = match self
.deps
.accounts
.test_connection(account.clone(), model)
.await
{
Ok(events) => drain_probe(events).await,
Err(error) => {
tracing::debug!(account_id, error = %error, "冻结恢复探测发起失败");
false
}
};
if probe_succeeded {
match self
.deps
.runtime
.finish_freeze(account_id, freeze, None)
.await
{
Ok(true) => tracing::info!(account_id, "冻结恢复探测成功:账号已解冻"),
Ok(false) => {}
Err(error) => warn!(account_id, error = %error, "解除冻结失败"),
}
} else {
self.postpone(&account, freeze, policy).await;
}
}
/// 探测失败只顺延本次仍存在的冻结,旧探测不能重新冻结已手动恢复的账号。
async fn postpone(
&self,
account: &ProviderAccountId,
freeze: &AccountFreeze,
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
) {
let Some(until) = SystemTime::now().checked_add(policy.freeze_duration()) else {
return;
};
let until = DateTime::<Utc>::from(until);
match self
.deps
.runtime
.finish_freeze(account.as_str(), freeze, Some(until))
.await
{
Ok(true) => {
tracing::info!(
account_id = account.as_str(),
postpone_seconds = policy.freeze_duration().as_secs(),
"冻结恢复探测失败:账号冻结顺延",
);
}
Ok(false) => {}
Err(error) => {
warn!(
account_id = account.as_str(),
error = %error,
"冻结顺延失败"
);
}
}
}
async fn resolve_probe_model(
&self,
account: &ProviderAccountId,
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
) -> Option<UpstreamModelId> {
if let Some(model) = policy.probe_model() {
return UpstreamModelId::new(model.to_owned()).ok();
}
let models = self.deps.accounts.models(account, false).await.ok()?;
models.models.into_iter().next().map(|model| model.id)
}
}
impl ScheduledTask for FreezeRecoveryTask {
fn run_cycle(
&self,
context: WorkerCycleContext,
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
Box::pin(async move {
if context.cancellation().is_cancelled() {
return Ok(());
}
self.run_cycle_inner().await;
Ok(())
})
}
}
/// 自适应目标并发:观测峰值 × 0.8 向下取整,下限 2;无证据(峰值为 0)不调整。
fn adaptive_target(peak_in_flight: u32) -> Option<u32> {
if peak_in_flight == 0 {
return None;
}
let scaled = (f64::from(peak_in_flight) * ADAPTIVE_CONCURRENCY_FACTOR).floor() as u32;
Some(scaled.max(ADAPTIVE_CONCURRENCY_FLOOR))
}
/// 消耗连接测试事件流并返回是否以 `Completed` 终止。
async fn drain_probe(mut events: crate::model::accounts::AccountConnectionTestEventStream) -> bool {
let mut completed = false;
while let Some(event) = events.next().await {
match event {
AccountConnectionTestEvent::Completed => completed = true,
AccountConnectionTestEvent::Failed { .. } => return false,
_ => {}
}
}
completed
}
+49 -3
View File
@@ -10,14 +10,15 @@ use gateway_core::{
routing::ProviderKind,
runtime::SnapshotControl,
task::{
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
WorkerRunnable,
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
WorkerRegistration, WorkerRunnable, WorkerSchedule,
},
};
use secrecy::{ExposeSecret as _, SecretString};
use serde::Deserialize;
pub mod backup;
pub mod freeze_recovery;
pub mod model;
pub mod ports;
mod use_case;
@@ -210,6 +211,12 @@ impl AdminServices {
self.key_usage.as_ref()
}
/// 取得账号服务的共享句柄;后台编排(冻结恢复 worker)需要持有 Arc。
#[must_use]
pub fn accounts_handle(&self) -> Arc<dyn AccountsService> {
Arc::clone(&self.accounts)
}
#[must_use]
pub fn proxies(&self) -> &dyn ProxiesService {
self.proxies.as_ref()
@@ -392,7 +399,7 @@ pub async fn initialize(
registry.clone(),
)),
auth,
accounts,
accounts: accounts.clone(),
account_groups: Arc::new(DefaultAccountGroupService::new(
store.account_groups(),
store.account_runtime(),
@@ -419,6 +426,13 @@ pub async fn initialize(
import_tasks,
backups,
};
let freeze_recovery =
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
store: store.accounts(),
runtime: store.account_runtime(),
settings: store.settings(),
});
let mut worker_contributions = backup_worker_contribution(backup_task)?;
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
@@ -433,6 +447,7 @@ pub async fn initialize(
)
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
worker_contributions.push(WorkerContribution::Registration(registration));
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
Ok(AdminBundle {
services,
worker_contributions,
@@ -458,6 +473,37 @@ fn backup_worker_contribution(
Ok(vec![WorkerContribution::Registration(registration)])
}
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定。
fn freeze_recovery_worker_contribution(
task: freeze_recovery::FreezeRecoveryTask,
) -> Result<Vec<WorkerContribution>, AdminError> {
let id = WorkerId::try_new(
WorkerKind::AccountFreezeRecovery,
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
)
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
let schedule = WorkerSchedule::try_new(
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
freeze_recovery::WORKER_INITIAL_BACKOFF,
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
freeze_recovery::WORKER_LEASE_TTL,
freeze_recovery::WORKER_LEASE_RENEWAL,
)
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
let registration = WorkerRegistration::try_new(
id,
WorkerRunnable::Scheduled {
schedule,
lease: Some(lease),
task: Box::new(task),
},
)
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
Ok(vec![WorkerContribution::Registration(registration)])
}
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
ProviderKind::new(value).map_err(|_| AdminError::internal("内置 Provider 类型不合法"))
}
@@ -69,11 +69,20 @@ pub struct AccountListQuery {
/// Admin query service 从运行态存储取得的当前账号冷却快照。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AccountRuntimeSnapshot {
pub rate_limited_until: BTreeMap<String, DateTime<Utc>>,
pub cooldown: BTreeMap<String, gateway_core::account::AccountCooldown>,
/// `None` 表示实时 lease 存储不可用;`Some` 中未出现的账号当前使用量为零。
pub in_flight: Option<BTreeMap<String, u64>>,
}
/// 恢复任务读取的冻结快照;generation 将异步结果绑定到本次冻结。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountFreeze {
pub credential_revision: Revision,
pub until: DateTime<Utc>,
pub generation: String,
pub requires_probe: bool,
}
/// Optional account membership filter.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AccountGroupFilter {
@@ -32,6 +32,13 @@ pub struct RuntimeSettings {
pub usage_retention_days: u32,
pub ops_event_retention_days: u32,
pub audit_retention_days: u32,
pub account_auto_freeze_enabled: bool,
pub account_auto_freeze_threshold: u32,
pub account_auto_freeze_window_seconds: u64,
pub account_auto_freeze_duration_seconds: u64,
pub account_auto_freeze_probe_enabled: bool,
pub account_auto_freeze_probe_model: Option<String>,
pub account_auto_freeze_adaptive_concurrency: bool,
pub updated_at: DateTime<Utc>,
}
@@ -52,6 +59,13 @@ pub struct ReplaceRuntimeSettings {
pub usage_retention_days: u32,
pub ops_event_retention_days: u32,
pub audit_retention_days: u32,
pub account_auto_freeze_enabled: bool,
pub account_auto_freeze_threshold: u32,
pub account_auto_freeze_window_seconds: u64,
pub account_auto_freeze_duration_seconds: u64,
pub account_auto_freeze_probe_enabled: bool,
pub account_auto_freeze_probe_model: Option<String>,
pub account_auto_freeze_adaptive_concurrency: bool,
}
/// 明文管理员 API Key;按产品约束明文落库,但禁止 Debug 泄漏。
@@ -2,7 +2,7 @@
//!
//! 端口按业务资源拆分,方法使用领域模型,不暴露连接池、事务或 Redis client。
use std::{net::IpAddr, sync::Arc, time::Duration};
use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
@@ -159,6 +159,14 @@ pub trait AccountStore: Send + Sync {
context: &MutationContext,
) -> AdminStoreResult<AccountUpdateResult>;
/// 在事务内按最新启用状态、账号上限和全局默认值判断,只降低并发上限。
async fn lower_concurrency_limit(
&self,
account_id: &gateway_core::account::ProviderAccountId,
limit: gateway_core::account::AccountConcurrencyLimit,
context: &MutationContext,
) -> AdminStoreResult<Option<AccountUpdateResult>>;
async fn recover_account(
&self,
account_id: &gateway_core::account::ProviderAccountId,
@@ -193,6 +201,25 @@ pub trait AccountRuntimeStore: Send + Sync {
&self,
account_ids: &[String],
) -> AdminStoreResult<AccountRuntimeSnapshot>;
/// 容量熔断自动冻结中的账号与其冻结截止时间;429 临时限流不包含在内。
async fn active_freezes(
&self,
) -> AdminStoreResult<BTreeMap<String, crate::model::accounts::AccountFreeze>>;
/// 读取容量失败窗口内观测到的在途并发峰值(自适应并发下调的证据)。
async fn capacity_peaks(
&self,
account_ids: &[String],
) -> AdminStoreResult<BTreeMap<String, u32>>;
/// 仅当冻结快照仍匹配时解除或顺延;旧探测不得覆盖手动恢复或新一轮冻结。
async fn finish_freeze(
&self,
account_id: &str,
expected: &crate::model::accounts::AccountFreeze,
postpone_until: Option<DateTime<Utc>>,
) -> AdminStoreResult<bool>;
}
/// 控制面凭据、统一会话、登录限流与管理员安全审计。
@@ -192,11 +192,7 @@ fn project_group_runtime(
let mut accounts_by_group = BTreeMap::<&str, Vec<&str>>::new();
for member in members {
let mut status = member.status.clone();
status.rate_limited_until = runtime
.rate_limited_until
.get(&member.account_id)
.copied()
.map(Into::into);
status.cooldown = runtime.cooldown.get(&member.account_id).copied();
status_by_account
.entry(member.account_id.as_str())
.or_insert_with(|| resolve_account_status(&status, now).status);
@@ -73,6 +73,13 @@ pub trait AccountsService: Send + Sync {
command: UpdateAccount,
) -> Result<AccountUpdateResult, AdminError>;
async fn lower_concurrency_limit(
&self,
context: &MutationContext,
account_id: ProviderAccountId,
limit: gateway_core::account::AccountConcurrencyLimit,
) -> Result<Option<AccountUpdateResult>, AdminError>;
async fn batch_update(
&self,
context: &MutationContext,
@@ -582,6 +589,27 @@ impl AccountsService for DefaultAccountsService {
Ok(result)
}
async fn lower_concurrency_limit(
&self,
context: &MutationContext,
account_id: ProviderAccountId,
limit: gateway_core::account::AccountConcurrencyLimit,
) -> Result<Option<AccountUpdateResult>, AdminError> {
let (_, provider) = self.provider_for_account(&account_id).await?;
let result = self
.accounts
.lower_concurrency_limit(&account_id, limit, context)
.await
.map_err(|error| map_store_error(error, "provider account"))?;
if let Some(result) = &result {
provider
.account_facts_changed(std::slice::from_ref(&account_id))
.await;
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
}
Ok(result)
}
async fn batch_update(
&self,
context: &MutationContext,
@@ -125,7 +125,11 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
&& command.audit_retention_days > 0
&& valid_client_version(command.min_codex_desktop_version.as_deref())
&& valid_client_version(command.min_codex_cli_version.as_deref())
&& i64::try_from(command.request_interval_ms).is_ok();
&& valid_probe_model(command.account_auto_freeze_probe_model.as_deref())
&& i64::try_from(command.request_interval_ms).is_ok()
&& (2..=1_000).contains(&command.account_auto_freeze_threshold)
&& (60..=3_600).contains(&command.account_auto_freeze_window_seconds)
&& (300..=604_800).contains(&command.account_auto_freeze_duration_seconds);
if valid {
Ok(())
} else {
@@ -136,3 +140,12 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
fn valid_client_version(value: Option<&str>) -> bool {
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
}
fn valid_probe_model(value: Option<&str>) -> bool {
value.is_none_or(|value| {
!value.is_empty()
&& value.len() <= 128
&& value == value.trim()
&& !value.bytes().any(|byte| byte.is_ascii_control())
})
}
@@ -5,7 +5,7 @@ use std::{
};
use async_trait::async_trait;
use chrono::{Duration, Utc};
use chrono::{DateTime, Duration, Utc};
use gateway_admin::{
model::{
MutationContext, PageSize, Revision,
@@ -168,13 +168,35 @@ impl AccountRuntimeStore for FakeRuntimeStore {
) -> AdminStoreResult<AccountRuntimeSnapshot> {
*self.requested_accounts.lock().expect("requested accounts") = account_ids.to_vec();
Ok(AccountRuntimeSnapshot {
rate_limited_until: BTreeMap::from([(
cooldown: BTreeMap::from([(
"acct_limited".to_owned(),
Utc::now() + Duration::minutes(5),
std::time::SystemTime::from(Utc::now() + Duration::minutes(5)).into(),
)]),
in_flight: Some(BTreeMap::from([("acct_available".to_owned(), 2)])),
})
}
async fn active_freezes(
&self,
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
Ok(BTreeMap::new())
}
async fn capacity_peaks(
&self,
_account_ids: &[String],
) -> AdminStoreResult<BTreeMap<String, u32>> {
Ok(BTreeMap::new())
}
async fn finish_freeze(
&self,
_account_id: &str,
_expected: &gateway_admin::model::accounts::AccountFreeze,
_postpone_until: Option<DateTime<Utc>>,
) -> AdminStoreResult<bool> {
Ok(false)
}
}
fn group_record() -> AccountGroupRecord {
@@ -215,7 +237,7 @@ fn member(account_id: &str, total_slots: u64) -> AccountGroupMemberFact {
credential_state: CredentialState::Ready,
access_token_expires_at: None,
quota: QuotaState::default(),
rate_limited_until: None,
cooldown: None,
last_error_reason: None,
last_error_message: None,
},
@@ -544,6 +544,8 @@ pub(super) struct FakeAccountStore {
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
quota_forecast_history: Mutex<QuotaForecastHistory>,
update_commands: Mutex<Vec<UpdateAccount>>,
pub(super) lowered_limits: Mutex<Vec<(String, u32)>>,
}
impl FakeAccountStore {
@@ -551,7 +553,7 @@ impl FakeAccountStore {
Self::with_account(account_record(kind), events)
}
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
pub(super) fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
Arc::new(Self {
events,
accounts: Mutex::new(vec![account]),
@@ -562,9 +564,18 @@ impl FakeAccountStore {
quota_window_usage: Mutex::new(Vec::new()),
quota_window_queries: Mutex::new(Vec::new()),
quota_forecast_history: Mutex::new(QuotaForecastHistory::default()),
update_commands: Mutex::new(Vec::new()),
lowered_limits: Mutex::new(Vec::new()),
})
}
pub(super) fn update_commands(&self) -> Vec<UpdateAccount> {
self.update_commands
.lock()
.expect("update commands")
.clone()
}
pub(super) fn fail_next_commit(&self) {
*self.fail_commit.lock().expect("store failure") = true;
}
@@ -630,7 +641,7 @@ impl FakeAccountStore {
credential_state: account.credential_state,
access_token_expires_at: account.access_token_expires_at.map(Into::into),
quota: account.quota,
rate_limited_until: None,
cooldown: None,
last_error_reason: account.last_error_reason,
last_error_message: account.last_error_message.clone(),
};
@@ -852,12 +863,45 @@ impl AccountStore for FakeAccountStore {
self.record("store.update_account");
self.record_context(context);
self.require_commit()?;
self.update_commands
.lock()
.expect("update commands")
.push(command.clone());
Ok(AccountUpdateResult {
config_revision: revision(2),
account_id: ProviderAccountId::new(command.account_id).expect("account ID"),
})
}
async fn lower_concurrency_limit(
&self,
account_id: &ProviderAccountId,
limit: gateway_core::account::AccountConcurrencyLimit,
_: &MutationContext,
) -> AdminStoreResult<Option<AccountUpdateResult>> {
let mut accounts = self.accounts.lock().expect("accounts");
let Some(account) = accounts
.iter_mut()
.find(|account| account.id == account_id.as_str())
else {
return Ok(None);
};
if !account.enabled
|| account.concurrency_limit.map_or(5, |value| value.get()) <= limit.get()
{
return Ok(None);
}
account.concurrency_limit = Some(limit);
self.lowered_limits
.lock()
.expect("lowered limits")
.push((account.id.clone(), limit.get()));
Ok(Some(AccountUpdateResult {
config_revision: revision(2),
account_id: account_id.clone(),
}))
}
async fn recover_account(
&self,
account_id: &ProviderAccountId,
@@ -944,6 +988,13 @@ impl SettingsStore for StaticSettingsStore {
usage_retention_days: 30,
ops_event_retention_days: 30,
audit_retention_days: 30,
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
updated_at: Utc::now(),
})
}
@@ -0,0 +1,425 @@
//! 容量熔断冻结恢复编排回归:到期探测、失败顺延与自适应并发下调。
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use chrono::{DateTime, Duration as TimeDelta, Utc};
use gateway_admin::freeze_recovery::{FreezeRecoveryDeps, FreezeRecoveryTask};
use gateway_admin::model::MutationContext;
use gateway_admin::model::accounts::{AccountFreeze, AccountRuntimeSnapshot};
use gateway_admin::model::settings::{
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RuntimeSettings,
};
use gateway_admin::ports::store::{
AccountRuntimeStore, AccountStore, AdminStoreError, AdminStoreResult, SettingsStore,
};
use gateway_core::engine::probe::{
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
};
use gateway_core::lifecycle::CancellationToken;
use gateway_core::task::{ScheduledTask as _, WorkerCycleContext, WorkerId, WorkerKind};
use super::AdminHarness;
use super::accounts::{FakeAccountStore, FakeProviderAdmin, account_record, events, revision};
fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> RuntimeSettings {
RuntimeSettings {
config_revision: revision(1),
model_mappings: Default::default(),
refresh_margin_seconds: 300,
refresh_concurrency: 2,
max_concurrent_per_account: 5,
request_interval_ms: 0,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart,
min_codex_desktop_version: None,
min_codex_cli_version: None,
usage_retention_days: 31,
ops_event_retention_days: 30,
audit_retention_days: 90,
account_auto_freeze_enabled: enabled,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: probe_enabled,
account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()),
account_auto_freeze_adaptive_concurrency: adaptive,
updated_at: Utc::now(),
}
}
struct FreezeSettingsStore {
settings: RuntimeSettings,
}
#[async_trait]
impl SettingsStore for FreezeSettingsStore {
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
Ok(self.settings.clone())
}
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
Ok(false)
}
async fn replace_runtime_settings(
&self,
_: ReplaceRuntimeSettings,
_: &MutationContext,
) -> AdminStoreResult<RuntimeSettings> {
Err(store_unavailable())
}
async fn replace_admin_api_key(
&self,
_: AdminApiKey,
_: &MutationContext,
) -> AdminStoreResult<AdminApiKeyMutation> {
Err(store_unavailable())
}
async fn delete_admin_api_key(
&self,
_: &MutationContext,
) -> AdminStoreResult<AdminApiKeyMutation> {
Err(store_unavailable())
}
}
fn store_unavailable() -> AdminStoreError {
AdminStoreError::new(
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
"freeze recovery test",
"unused",
)
}
/// 记录解冻/顺延调用的运行态 fake;冻结与峰值证据由测试预置。
struct FreezeRuntimeStore {
freezes: BTreeMap<String, AccountFreeze>,
peaks: BTreeMap<String, u32>,
cleared: Mutex<Vec<String>>,
extended: Mutex<Vec<(String, DateTime<Utc>)>>,
}
impl FreezeRuntimeStore {
fn new(freezes: BTreeMap<String, AccountFreeze>, peaks: BTreeMap<String, u32>) -> Arc<Self> {
Arc::new(Self {
freezes,
peaks,
cleared: Mutex::new(Vec::new()),
extended: Mutex::new(Vec::new()),
})
}
fn cleared(&self) -> Vec<String> {
self.cleared.lock().expect("cleared lock").clone()
}
fn extended(&self) -> Vec<(String, DateTime<Utc>)> {
self.extended.lock().expect("extended lock").clone()
}
}
#[async_trait]
impl AccountRuntimeStore for FreezeRuntimeStore {
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
Ok(AccountRuntimeSnapshot::default())
}
async fn account_runtime(
&self,
_account_ids: &[String],
) -> AdminStoreResult<AccountRuntimeSnapshot> {
Ok(AccountRuntimeSnapshot::default())
}
async fn active_freezes(&self) -> AdminStoreResult<BTreeMap<String, AccountFreeze>> {
Ok(self.freezes.clone())
}
async fn capacity_peaks(
&self,
account_ids: &[String],
) -> AdminStoreResult<BTreeMap<String, u32>> {
Ok(account_ids
.iter()
.filter_map(|id| self.peaks.get(id).map(|peak| (id.clone(), *peak)))
.collect())
}
async fn finish_freeze(
&self,
account_id: &str,
expected: &AccountFreeze,
postpone_until: Option<DateTime<Utc>>,
) -> AdminStoreResult<bool> {
assert_eq!(self.freezes.get(account_id), Some(expected));
if let Some(until) = postpone_until {
self.extended
.lock()
.expect("extended")
.push((account_id.to_owned(), until));
} else {
self.cleared
.lock()
.expect("cleared")
.push(account_id.to_owned());
}
Ok(true)
}
}
struct SuccessfulProbe;
impl AccountProbe for SuccessfulProbe {
fn probe(
&self,
_: AccountProbeRequest,
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
Box::pin(async {
Ok(AccountProbeResult {
text: vec!["OK".to_owned()],
})
})
}
}
struct FailingProbe;
impl AccountProbe for FailingProbe {
fn probe(
&self,
_: AccountProbeRequest,
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
Box::pin(async {
Err(AccountProbeError::new(
gateway_core::error::GatewayError::new(
gateway_core::error::GatewayErrorKind::RateLimited,
"still overloaded",
),
gateway_core::engine::probe::AccountProbeErrorSource::Upstream,
Some(gateway_core::upstream::UpstreamSendState::NotSent),
None,
))
})
}
}
async fn recovery_task(
settings: RuntimeSettings,
runtime: Arc<FreezeRuntimeStore>,
probe: Arc<dyn AccountProbe>,
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
recovery_task_with_store(settings, runtime, probe).await
}
async fn recovery_task_with_store(
settings: RuntimeSettings,
runtime: Arc<FreezeRuntimeStore>,
probe: Arc<dyn AccountProbe>,
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
let store = FakeAccountStore::new("openai", events());
let services = AdminHarness::new()
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
.settings(Arc::new(FreezeSettingsStore {
settings: settings.clone(),
}))
.provider(FakeProviderAdmin::new("openai", events()))
.probe(probe)
.build()
.await;
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
accounts: services.accounts_handle(),
store: store.clone(),
runtime,
settings: Arc::new(FreezeSettingsStore {
settings: settings.clone(),
}),
});
(task, store)
}
async fn run_cycle(task: &FreezeRecoveryTask) {
let worker = WorkerId::try_new(WorkerKind::AccountFreezeRecovery, "test").expect("worker id");
let context = WorkerCycleContext::new(worker, None, CancellationToken::new());
task.run_cycle(context)
.await
.expect("freeze recovery cycle");
}
fn freeze_until(until: DateTime<Utc>) -> BTreeMap<String, AccountFreeze> {
BTreeMap::from([(
"acct_test".to_owned(),
AccountFreeze {
credential_revision: revision(1),
until,
generation: "freeze-generation".to_owned(),
requires_probe: true,
},
)])
}
fn freeze_due() -> BTreeMap<String, AccountFreeze> {
freeze_until(Utc::now() - TimeDelta::seconds(1))
}
#[tokio::test]
async fn probe_success_clears_freeze() {
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
let (task, _store) = recovery_task(
runtime_settings(true, true, false),
Arc::clone(&runtime),
Arc::new(SuccessfulProbe),
)
.await;
run_cycle(&task).await;
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
assert!(runtime.extended().is_empty());
}
#[tokio::test]
async fn probe_failure_postpones_freeze() {
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
let (task, _store) = recovery_task(
runtime_settings(true, true, false),
Arc::clone(&runtime),
Arc::new(FailingProbe),
)
.await;
run_cycle(&task).await;
assert!(runtime.cleared().is_empty());
let extended = runtime.extended();
assert_eq!(extended.len(), 1);
let (account_id, until) = &extended[0];
assert_eq!(account_id, "acct_test");
let remaining = until.timestamp_millis() - Utc::now().timestamp_millis();
assert!(
remaining > 7_000_000 && remaining <= 7_200_000,
"postponed freeze should be about 2 hours, got {remaining}ms"
);
}
#[tokio::test]
async fn disabled_policy_releases_due_freeze_without_probing() {
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
let (task, _store) = recovery_task(
runtime_settings(false, true, false),
Arc::clone(&runtime),
Arc::new(SuccessfulProbe),
)
.await;
run_cycle(&task).await;
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
assert!(runtime.extended().is_empty());
}
#[tokio::test]
async fn adaptive_concurrency_lowers_limit_to_observed_peak() {
let runtime = FreezeRuntimeStore::new(
freeze_until(Utc::now() + TimeDelta::hours(1)),
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
);
let (task, store) = recovery_task(
runtime_settings(true, false, true),
Arc::clone(&runtime),
Arc::new(SuccessfulProbe),
)
.await;
run_cycle(&task).await;
// 峰值 4 × 0.8 = 3(不低于下限 2),低于全局默认 5,应下调到 3。
assert_eq!(
*store.lowered_limits.lock().expect("lowered limits"),
vec![("acct_test".to_owned(), 3)]
);
assert!(store.update_commands().is_empty());
assert!(runtime.extended().is_empty());
assert!(runtime.cleared().is_empty());
}
#[tokio::test]
async fn adaptive_concurrency_never_raises_limit() {
let runtime = FreezeRuntimeStore::new(
freeze_until(Utc::now() + TimeDelta::hours(1)),
BTreeMap::from([("acct_test".to_owned(), 40_u32)]),
);
let (task, store) = recovery_task(
runtime_settings(true, false, true),
Arc::clone(&runtime),
Arc::new(SuccessfulProbe),
)
.await;
run_cycle(&task).await;
// 峰值 40 的目标 32 高于全局默认 5;只降不升,不下发任何更新。
assert!(store.update_commands().is_empty());
}
/// 到期时间还早(超过探测提前量)的冻结不发起探测。
#[tokio::test]
async fn probe_skips_freezes_far_from_expiry() {
let runtime = FreezeRuntimeStore::new(
freeze_until(Utc::now() + TimeDelta::hours(1)),
BTreeMap::new(),
);
let (task, _store) = recovery_task(
runtime_settings(true, true, false),
Arc::clone(&runtime),
Arc::new(SuccessfulProbe),
)
.await;
run_cycle(&task).await;
assert!(runtime.cleared().is_empty());
assert!(runtime.extended().is_empty());
}
/// 停用账号不参与探测恢复;并发下调同样跳过。
#[tokio::test]
async fn disabled_accounts_are_skipped() {
let runtime = FreezeRuntimeStore::new(
freeze_due(),
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
);
let mut record = account_record("openai");
record.enabled = false;
let store = FakeAccountStore::with_account(record, events());
let services = AdminHarness::new()
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
.settings(Arc::new(FreezeSettingsStore {
settings: runtime_settings(true, true, true),
}))
.provider(FakeProviderAdmin::new("openai", events()))
.probe(Arc::new(SuccessfulProbe))
.build()
.await;
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
accounts: services.accounts_handle(),
store: store.clone(),
runtime: Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>,
settings: Arc::new(FreezeSettingsStore {
settings: runtime_settings(true, true, true),
}),
});
run_cycle(&task).await;
assert!(runtime.cleared().is_empty());
assert!(runtime.extended().is_empty());
assert!(store.update_commands().is_empty());
}
@@ -4,6 +4,7 @@ mod auth;
mod auth_key;
mod backup;
mod client_keys;
mod freeze_recovery;
mod import_tasks;
mod observability;
mod openai;
@@ -82,6 +83,7 @@ use gateway_core::{
routing::{ConfigRevision, ProviderKind},
runtime::SnapshotControl,
};
use std::collections::BTreeMap;
pub(super) struct AdminHarness {
default_password: String,
@@ -488,6 +490,15 @@ impl AccountStore for UnavailableStore {
Err(unavailable("account enabled"))
}
async fn lower_concurrency_limit(
&self,
_: &gateway_core::account::ProviderAccountId,
_: gateway_core::account::AccountConcurrencyLimit,
_: &MutationContext,
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
Ok(None)
}
async fn recover_account(
&self,
_: &ProviderAccountId,
@@ -530,6 +541,28 @@ impl AccountRuntimeStore for UnavailableStore {
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
Ok(AccountRuntimeSnapshot::default())
}
async fn active_freezes(
&self,
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
Ok(BTreeMap::new())
}
async fn capacity_peaks(
&self,
_account_ids: &[String],
) -> AdminStoreResult<BTreeMap<String, u32>> {
Ok(BTreeMap::new())
}
async fn finish_freeze(
&self,
_account_id: &str,
_expected: &gateway_admin::model::accounts::AccountFreeze,
_postpone_until: Option<DateTime<Utc>>,
) -> AdminStoreResult<bool> {
Ok(false)
}
}
#[async_trait]
@@ -769,6 +769,13 @@ impl SettingsStore for FixtureSettingsStore {
ops_event_retention_days: 30,
audit_retention_days: 30,
updated_at: Utc::now(),
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
})
}
@@ -75,6 +75,13 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
usage_retention_days: 31,
ops_event_retention_days: 30,
audit_retention_days: 30,
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
},
)
.await
@@ -66,7 +66,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
quota,
} = item;
let status = projection.status.as_str().to_owned();
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
let cooldown = projection.cooldown;
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
let added_at = china_rfc3339(&account.created_at);
let updated_at = china_rfc3339(&account.updated_at);
@@ -75,7 +75,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
if account.authentication_kind == "api_key" {
usage.window_label_display = "通用额度".to_owned();
}
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
let (quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now);
AccountView {
id: account.id.clone(),
name: account.name,
@@ -189,7 +189,7 @@ fn forecast_usd_display(value: Option<f64>) -> String {
pub(super) fn account_quota_view(
mut quota: ProviderQuota,
rate_limited_until: Option<DateTime<Utc>>,
cooldown: Option<gateway_core::account::AccountCooldown>,
now: DateTime<Utc>,
) -> (AccountQuotaView, Option<String>) {
quota.apply_limit_reached_display();
@@ -200,12 +200,22 @@ pub(super) fn account_quota_view(
.observed_at
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
let windows = quota.windows.into_iter().map(quota_window_view).collect();
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
let rate_limited_until = cooldown.map(|value| china_datetime(&value.until.into()));
let rate_limit_reason = cooldown.map(|value| {
if value.kind.is_capacity_freeze() {
"capacity_freeze".to_owned()
} else {
"upstream_rate_limit".to_owned()
}
});
let recovery_probe_required = cooldown.is_some_and(|value| value.kind.requires_probe());
(
AccountQuotaView {
refreshed_at_display,
limit_reached: quota.limit_reached,
rate_limited_until,
rate_limit_reason,
recovery_probe_required,
windows,
},
refresh_token_expires_at,
@@ -361,8 +361,10 @@ pub struct AccountGroupRefView {
pub struct AccountQuotaView {
pub refreshed_at_display: String,
pub limit_reached: bool,
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
/// 冷却到期或可开始恢复探测的时间;非限流中为 `null`。
pub rate_limited_until: Option<String>,
pub rate_limit_reason: Option<String>,
pub recovery_probe_required: bool,
pub windows: Vec<AccountQuotaWindowView>,
}
@@ -48,6 +48,13 @@ pub struct RuntimeSettingsView {
pub usage_retention_days: u64,
pub ops_event_retention_days: u64,
pub audit_retention_days: u64,
pub account_auto_freeze_enabled: bool,
pub account_auto_freeze_threshold: u64,
pub account_auto_freeze_window_seconds: u64,
pub account_auto_freeze_duration_seconds: u64,
pub account_auto_freeze_probe_enabled: bool,
pub account_auto_freeze_probe_model: Option<String>,
pub account_auto_freeze_adaptive_concurrency: bool,
pub updated_at: DateTime<Utc>,
}
@@ -69,6 +76,13 @@ pub struct UpdateRuntimeSettingsRequest {
pub usage_retention_days: u64,
pub ops_event_retention_days: u64,
pub audit_retention_days: u64,
pub account_auto_freeze_enabled: bool,
pub account_auto_freeze_threshold: u64,
pub account_auto_freeze_window_seconds: u64,
pub account_auto_freeze_duration_seconds: u64,
pub account_auto_freeze_probe_enabled: bool,
pub account_auto_freeze_probe_model: Option<String>,
pub account_auto_freeze_adaptive_concurrency: bool,
}
impl UpdateRuntimeSettingsRequest {
@@ -110,6 +124,35 @@ impl UpdateRuntimeSettingsRequest {
self.min_codex_cli_version.as_deref(),
"minCodexCliVersion",
)?;
for (value, field) in [
(
self.account_auto_freeze_threshold,
"accountAutoFreezeThreshold",
),
(
self.account_auto_freeze_window_seconds,
"accountAutoFreezeWindowSeconds",
),
(
self.account_auto_freeze_duration_seconds,
"accountAutoFreezeDurationSeconds",
),
] {
require_positive_i64(value, field)?;
}
if !(2..=1_000).contains(&self.account_auto_freeze_threshold) {
return Err(WireValidationError::new("accountAutoFreezeThreshold"));
}
if !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) {
return Err(WireValidationError::new("accountAutoFreezeWindowSeconds"));
}
if !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) {
return Err(WireValidationError::new("accountAutoFreezeDurationSeconds"));
}
validate_optional_probe_model(
self.account_auto_freeze_probe_model.as_deref(),
"accountAutoFreezeProbeModel",
)?;
Ok(())
}
@@ -136,6 +179,14 @@ impl UpdateRuntimeSettingsRequest {
.map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?,
audit_retention_days: u32::try_from(self.audit_retention_days)
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
account_auto_freeze_enabled: self.account_auto_freeze_enabled,
account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold)
.map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?,
account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds,
account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds,
account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled,
account_auto_freeze_probe_model: self.account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency,
})
}
}
@@ -157,6 +208,14 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
usage_retention_days: u64::from(settings.usage_retention_days),
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
audit_retention_days: u64::from(settings.audit_retention_days),
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold),
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency: settings
.account_auto_freeze_adaptive_concurrency,
updated_at: settings.updated_at,
}
}
@@ -452,6 +511,22 @@ fn validate_optional_client_version(
Ok(())
}
/// 探测模型为可选自由文本:非空、去首尾空白后不变、无控制字符且不超过 128 字节。
fn validate_optional_probe_model(
value: Option<&str>,
field: &'static str,
) -> Result<(), WireValidationError> {
if value.is_some_and(|value| {
value.is_empty()
|| value.len() > 128
|| value != value.trim()
|| value.bytes().any(|byte| byte.is_ascii_control())
}) {
return Err(WireValidationError::new(field));
}
Ok(())
}
fn map_wire_error(error: WireValidationError) -> AdminError {
let message = match error.field() {
"settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(),
@@ -459,6 +534,11 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
"settingsUsageRetentionOverflow" => "usageRetentionDays 不合法".to_owned(),
"settingsOpsRetentionOverflow" => "opsEventRetentionDays 不合法".to_owned(),
"settingsAuditRetentionOverflow" => "auditRetentionDays 不合法".to_owned(),
"settingsFreezeThresholdOverflow" => "accountAutoFreezeThreshold 不合法".to_owned(),
"accountAutoFreezeThreshold" => "账号自动冻结阈值应为 2~1000 的整数".to_owned(),
"accountAutoFreezeWindowSeconds" => "账号自动冻结统计窗口应为 60~3600 秒".to_owned(),
"accountAutoFreezeDurationSeconds" => "账号自动冻结时长应为 300~604800 秒".to_owned(),
"accountAutoFreezeProbeModel" => "探测模型格式不合法".to_owned(),
"minCodexDesktopVersion" => "Codex Desktop 最低版本格式不合法".to_owned(),
"minCodexCliVersion" => "Codex CLI 最低版本格式不合法".to_owned(),
field => format!("{field} 字段不合法"),
@@ -463,7 +463,7 @@ mod provider {
credential_state: CredentialState::Ready,
access_token_expires_at: None,
quota: QuotaState::unknown(),
rate_limited_until: None,
cooldown: None,
last_error_reason: None,
last_error_message: None,
};
+49 -4
View File
@@ -417,6 +417,13 @@ impl SettingsStore for MemorySettingsStore {
usage_retention_days: command.usage_retention_days,
ops_event_retention_days: command.ops_event_retention_days,
audit_retention_days: command.audit_retention_days,
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
updated_at: Utc::now(),
};
*settings = updated.clone();
@@ -587,7 +594,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
credential_state: CredentialState::Ready,
access_token_expires_at: None,
quota: QuotaState::default(),
rate_limited_until: None,
cooldown: None,
last_error_reason: None,
last_error_message: None,
},
@@ -601,7 +608,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
credential_state: CredentialState::Ready,
access_token_expires_at: None,
quota: QuotaState::default(),
rate_limited_until: None,
cooldown: None,
last_error_reason: None,
last_error_message: None,
},
@@ -896,6 +903,15 @@ impl AccountStore for UnusedStore {
Err(unavailable("account enabled"))
}
async fn lower_concurrency_limit(
&self,
_: &gateway_core::account::ProviderAccountId,
_: gateway_core::account::AccountConcurrencyLimit,
_: &MutationContext,
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
Ok(None)
}
async fn recover_account(
&self,
_: &ProviderAccountId,
@@ -933,17 +949,39 @@ impl AccountStore for UnusedStore {
impl AccountRuntimeStore for UnusedStore {
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
Ok(AccountRuntimeSnapshot {
rate_limited_until: BTreeMap::new(),
cooldown: BTreeMap::new(),
in_flight: Some(BTreeMap::new()),
})
}
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
Ok(AccountRuntimeSnapshot {
rate_limited_until: BTreeMap::new(),
cooldown: BTreeMap::new(),
in_flight: Some(BTreeMap::new()),
})
}
async fn active_freezes(
&self,
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
Ok(BTreeMap::new())
}
async fn capacity_peaks(
&self,
_account_ids: &[String],
) -> AdminStoreResult<BTreeMap<String, u32>> {
Ok(BTreeMap::new())
}
async fn finish_freeze(
&self,
_account_id: &str,
_expected: &gateway_admin::model::accounts::AccountFreeze,
_postpone_until: Option<DateTime<Utc>>,
) -> AdminStoreResult<bool> {
Ok(false)
}
}
#[async_trait]
@@ -1245,6 +1283,13 @@ fn test_runtime_settings() -> RuntimeSettings {
usage_retention_days: 31,
ops_event_retention_days: 30,
audit_retention_days: 90,
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
updated_at: Utc::now(),
}
}
@@ -53,7 +53,14 @@ fn update_body() -> Value {
"minCodexCliVersion": "0.40.0",
"usageRetentionDays": 32,
"opsEventRetentionDays": 31,
"auditRetentionDays": 91
"auditRetentionDays": 91,
"accountAutoFreezeEnabled": true,
"accountAutoFreezeThreshold": 12,
"accountAutoFreezeWindowSeconds": 600,
"accountAutoFreezeDurationSeconds": 7200,
"accountAutoFreezeProbeEnabled": true,
"accountAutoFreezeProbeModel": null,
"accountAutoFreezeAdaptiveConcurrency": true
})
}
@@ -116,6 +123,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
usage_retention_days: 32,
ops_event_retention_days: 31,
audit_retention_days: 91,
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
updated_at: Utc
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
.single()
@@ -143,6 +157,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
"usageRetentionDays": 32,
"opsEventRetentionDays": 31,
"auditRetentionDays": 91,
"accountAutoFreezeEnabled": true,
"accountAutoFreezeThreshold": 12,
"accountAutoFreezeWindowSeconds": 600,
"accountAutoFreezeDurationSeconds": 7200,
"accountAutoFreezeProbeEnabled": true,
"accountAutoFreezeProbeModel": null,
"accountAutoFreezeAdaptiveConcurrency": true,
"updatedAt": "2026-08-02T10:30:00Z"
})
);
@@ -195,6 +216,13 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
updated_at: chrono::Utc::now(),
};
@@ -583,6 +583,72 @@ impl AccountErrorReason {
}
}
/// 账号冷却的原因与恢复方式。
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum AccountCooldownKind {
/// 上游 429 临时限流;默认类别,兼容升级前未标记的存量 key。
#[default]
RateLimit,
/// 容量类错误高频触发后由熔断策略写入的自动冻结。
CapacityFreeze,
/// 到期后仍阻止调度,必须由恢复探测成功或管理员恢复解除。
CapacityFreezeProbe,
}
impl AccountCooldownKind {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::RateLimit => "rate_limit",
Self::CapacityFreeze => "capacity_freeze",
Self::CapacityFreezeProbe => "capacity_freeze_probe",
}
}
#[must_use]
pub const fn is_capacity_freeze(self) -> bool {
matches!(self, Self::CapacityFreeze | Self::CapacityFreezeProbe)
}
#[must_use]
pub const fn requires_probe(self) -> bool {
matches!(self, Self::CapacityFreezeProbe)
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"rate_limit" => Some(Self::RateLimit),
"capacity_freeze" => Some(Self::CapacityFreeze),
"capacity_freeze_probe" => Some(Self::CapacityFreezeProbe),
_ => None,
}
}
}
/// 账号级冷却投影;探测冻结的 until 是下次探测时间,不是自动放行时间。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AccountCooldown {
pub until: SystemTime,
pub kind: AccountCooldownKind,
}
impl AccountCooldown {
#[must_use]
pub fn is_active(self, now: SystemTime) -> bool {
self.kind.requires_probe() || self.until > now
}
}
impl From<SystemTime> for AccountCooldown {
fn from(until: SystemTime) -> Self {
Self {
until,
kind: AccountCooldownKind::RateLimit,
}
}
}
/// 唯一状态解析器的完整输入事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountStatusFacts {
@@ -590,7 +656,7 @@ pub struct AccountStatusFacts {
pub credential_state: CredentialState,
pub access_token_expires_at: Option<SystemTime>,
pub quota: QuotaState,
pub rate_limited_until: Option<SystemTime>,
pub cooldown: Option<AccountCooldown>,
pub last_error_reason: Option<AccountErrorReason>,
pub last_error_message: Option<String>,
}
@@ -601,8 +667,8 @@ pub struct AccountStatusProjection {
pub status: AccountStatus,
pub error_reason: Option<AccountErrorReason>,
pub error_message: Option<String>,
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。
pub rate_limited_until: Option<SystemTime>,
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却事实。
pub cooldown: Option<AccountCooldown>,
}
/// 从独立事实派生唯一、互斥的对外状态。
@@ -625,21 +691,21 @@ pub fn resolve_account_status(
status: AccountStatus::Error,
error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)),
error_message: facts.last_error_message.clone(),
rate_limited_until: None,
cooldown: None,
};
}
if facts.quota.is_exhausted() {
return status_projection(AccountStatus::QuotaExhausted);
}
if facts
.rate_limited_until
.is_some_and(|rate_limited_until| rate_limited_until > now)
.cooldown
.is_some_and(|cooldown| cooldown.is_active(now))
{
return AccountStatusProjection {
status: AccountStatus::RateLimited,
error_reason: None,
error_message: None,
rate_limited_until: facts.rate_limited_until,
cooldown: facts.cooldown,
};
}
status_projection(AccountStatus::Normal)
@@ -650,7 +716,7 @@ const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
status,
error_reason: None,
error_message: None,
rate_limited_until: None,
cooldown: None,
}
}
@@ -905,7 +971,7 @@ impl ProviderAccount {
pub fn status_projection(
&self,
now: SystemTime,
rate_limited_until: Option<SystemTime>,
cooldown: Option<AccountCooldown>,
) -> AccountStatusProjection {
resolve_account_status(
&AccountStatusFacts {
@@ -913,7 +979,7 @@ impl ProviderAccount {
credential_state: self.credential_state,
access_token_expires_at: self.access_token_expires_at,
quota: self.quota,
rate_limited_until,
cooldown,
last_error_reason: self.last_error_reason,
last_error_message: self.last_error_message.clone(),
},
@@ -106,7 +106,7 @@ pub struct AccountRuntimeSignals {
pub quota_reset_at: Option<SystemTime>,
/// Provider 归一化的剩余额度基点:0 耗尽,10_000 全部可用。
pub quota_remaining_rank: Option<u64>,
pub rate_limited_until: Option<SystemTime>,
pub cooldown: Option<super::AccountCooldown>,
pub failure_rate_basis_points: Option<u16>,
pub first_output_latency_ms: Option<u64>,
}
@@ -316,8 +316,8 @@ impl AccountRuntimeSignals {
}
#[must_use]
pub const fn with_rate_limit(mut self, rate_limited_until: Option<SystemTime>) -> Self {
self.rate_limited_until = rate_limited_until;
pub const fn with_rate_limit(mut self, cooldown: Option<super::AccountCooldown>) -> Self {
self.cooldown = cooldown;
self
}
@@ -654,7 +654,7 @@ impl AccountSelector {
if !context.eligibility.bypasses_local_eligibility() {
let status = candidate
.account
.status_projection(context.now, candidate.signals.rate_limited_until)
.status_projection(context.now, candidate.signals.cooldown)
.status;
if status != AccountStatus::Normal {
return Some(AccountSchedulingBlocker::LocalAvailability);
@@ -181,6 +181,16 @@ pub trait ProviderLeasePort: Send + Sync {
&self,
request: ProviderLeaseRequest,
) -> BoxFuture<'_, Result<ProviderLeaseAcquisition, ProviderStoreError>>;
/// 读取指定账号当前的在途请求数;只用于容量熔断的峰值证据,
/// 支持租约信号的存储实现覆盖,否则视为不可观测(空映射)。
fn account_in_flight<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
) -> BoxFuture<'a, Result<BTreeMap<ProviderAccountId, u32>, ProviderStoreError>> {
let _ = account_ids;
Box::pin(async move { Ok(BTreeMap::new()) })
}
}
/// Provider 从原始会话锚点派生的不可逆亲和键。
@@ -596,12 +606,16 @@ pub trait ProviderCredentialStatePort: Send + Sync {
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
}
/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。
/// 账号级 cooldown 的来源类别;调度状态统一按 `rate_limited` 处理。
pub use crate::account::AccountCooldownKind as ProviderCooldownKind;
/// 可丢失的账号冷却事实,不进入持久状态;探测冻结到期后仍需确认恢复。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderCooldown {
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
until: SystemTime,
kind: ProviderCooldownKind,
}
impl ProviderCooldown {
@@ -610,11 +624,27 @@ impl ProviderCooldown {
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
until: SystemTime,
) -> Self {
Self::new_with_kind(
account_id,
credential_revision,
until,
ProviderCooldownKind::RateLimit,
)
}
#[must_use]
pub const fn new_with_kind(
account_id: ProviderAccountId,
credential_revision: CredentialRevision,
until: SystemTime,
kind: ProviderCooldownKind,
) -> Self {
Self {
account_id,
credential_revision,
until,
kind,
}
}
@@ -632,6 +662,19 @@ impl ProviderCooldown {
pub const fn until(&self) -> SystemTime {
self.until
}
#[must_use]
pub const fn scheduling_state(&self) -> crate::account::AccountCooldown {
crate::account::AccountCooldown {
until: self.until,
kind: self.kind,
}
}
#[must_use]
pub const fn kind(&self) -> ProviderCooldownKind {
self.kind
}
}
/// 可丢失 cooldown 的细粒度作用域。
@@ -749,6 +792,29 @@ pub trait ProviderCooldownPort: Send + Sync {
&'a self,
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
/// 记录一次容量类失败并返回滑动窗口内的累计次数,同时把观测到的账号
/// 在途并发并入窗口峰值(`in_flight` 为 0 表示本次未观测,跳过峰值更新)。
/// 只服务容量熔断触发器;调用频率受失败频率约束,不需要批量接口。
fn record_capacity_failure<'a>(
&'a self,
account_id: &'a ProviderAccountId,
window: Duration,
in_flight: u32,
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
/// 普通请求成功后原子清除临时限流及失败证据;必须保留任何容量冻结。
fn clear_after_success<'a>(
&'a self,
account_id: &'a ProviderAccountId,
through_revision: CredentialRevision,
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
/// 读取窗口内观测到的在途并发峰值;无证据时返回 `None`。
fn capacity_peak_in_flight<'a>(
&'a self,
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -864,6 +930,118 @@ pub trait ProviderRuntimePolicyPort: Send + Sync {
fn load_refresh_policy(
&self,
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>>;
/// 读取账号容量熔断策略;默认关闭,只有实现运行时设置的存储需要覆盖。
fn load_freeze_policy(
&self,
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
Box::pin(async move { Ok(ProviderFreezePolicy::disabled()) })
}
}
/// 账号容量熔断(自动冻结)策略;来源于 `runtime_settings`,
/// 由 Provider 触发路径与恢复 worker 共享同一份配置事实。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderFreezePolicy {
enabled: bool,
threshold: u32,
window: Duration,
freeze_duration: Duration,
probe_enabled: bool,
probe_model: Option<String>,
adaptive_concurrency: bool,
}
impl ProviderFreezePolicy {
/// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致;
/// store 层写入前已校验,这里兜底防御越界配置。
pub fn try_new(
enabled: bool,
threshold: u32,
window_seconds: u64,
freeze_duration_seconds: u64,
probe_enabled: bool,
probe_model: Option<String>,
adaptive_concurrency: bool,
) -> Result<Self, ProviderStoreError> {
if !(2..=1_000).contains(&threshold)
|| !(60..=3_600).contains(&window_seconds)
|| !(300..=604_800).contains(&freeze_duration_seconds)
|| probe_model.as_deref().is_some_and(|model| {
model.is_empty()
|| model.len() > 128
|| model != model.trim()
|| model.bytes().any(|byte| byte.is_ascii_control())
})
{
return Err(ProviderStoreError::new(
ProviderStoreErrorKind::InvalidData,
"validate freeze policy",
));
}
Ok(Self {
enabled,
threshold,
window: Duration::from_secs(window_seconds),
freeze_duration: Duration::from_secs(freeze_duration_seconds),
probe_enabled,
probe_model,
adaptive_concurrency,
})
}
/// 功能关闭时的全零策略;触发路径与 worker 都以此短路。
#[must_use]
pub const fn disabled() -> Self {
Self {
enabled: false,
threshold: 2,
window: Duration::from_secs(60),
freeze_duration: Duration::from_secs(300),
probe_enabled: false,
probe_model: None,
adaptive_concurrency: false,
}
}
#[must_use]
pub const fn enabled(&self) -> bool {
self.enabled
}
/// 窗口内触发冻结的请求级失败次数阈值。
#[must_use]
pub const fn threshold(&self) -> u32 {
self.threshold
}
/// 失败计数滑动窗口;每次失败都会顺延窗口。
#[must_use]
pub const fn window(&self) -> Duration {
self.window
}
/// 冻结时长;探测失败后的顺延也使用该值。
#[must_use]
pub const fn freeze_duration(&self) -> Duration {
self.freeze_duration
}
#[must_use]
pub const fn probe_enabled(&self) -> bool {
self.probe_enabled
}
/// 探测模型;`None` 表示由 worker 选择账号可用的第一个模型。
#[must_use]
pub fn probe_model(&self) -> Option<&str> {
self.probe_model.as_deref()
}
#[must_use]
pub const fn adaptive_concurrency(&self) -> bool {
self.adaptive_concurrency
}
}
/// OAuth pending flow 的原始绑定只在 Provider 与 Store 边界内短暂存在。
+4 -1
View File
@@ -23,10 +23,11 @@ pub enum WorkerKind {
OpsFlush,
Backup,
AccountImport,
AccountFreezeRecovery,
}
impl WorkerKind {
pub const ALL: [Self; 9] = [
pub const ALL: [Self; 10] = [
Self::OAuthRefresh,
Self::QuotaCatalogHealth,
Self::RuntimeSnapshotReconciliation,
@@ -36,6 +37,7 @@ impl WorkerKind {
Self::OpsFlush,
Self::Backup,
Self::AccountImport,
Self::AccountFreezeRecovery,
];
#[must_use]
@@ -50,6 +52,7 @@ impl WorkerKind {
Self::OpsFlush => "ops_flush",
Self::Backup => "backup",
Self::AccountImport => "account_import",
Self::AccountFreezeRecovery => "account_freeze_recovery",
}
}
}
@@ -49,7 +49,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: remaining,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -222,7 +222,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -240,7 +240,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -313,13 +313,13 @@ fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
let active_until = now + Duration::from_secs(60);
let account = account("acct_rate_limited");
let active = account.status_projection(now, Some(active_until));
let active = account.status_projection(now, Some(active_until.into()));
assert_eq!(active.status, AccountStatus::RateLimited);
assert_eq!(active.rate_limited_until, Some(active_until));
assert_eq!(active.cooldown, Some(active_until.into()));
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
let elapsed = account.status_projection(now, Some((now - Duration::from_secs(1)).into()));
assert_eq!(elapsed.status, AccountStatus::Normal);
assert_eq!(elapsed.rate_limited_until, None);
assert_eq!(elapsed.cooldown, None);
}
#[test]
@@ -492,8 +492,8 @@ fn highest_priority_affinity_should_still_obey_existing_scheduling_constraints()
);
}
AccountSchedulingBlocker::LocalAvailability => {
candidates[0].signals.rate_limited_until =
Some(selection.now + Duration::from_secs(60))
candidates[0].signals.cooldown =
Some((selection.now + Duration::from_secs(60)).into())
}
_ => unreachable!("test cases"),
}
@@ -643,7 +643,7 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
last_started_at: Some(last_started_at),
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
}
@@ -1062,3 +1062,32 @@ fn elapsed_quota_reset_does_not_fabricate_recovery() {
.expect("healthy candidate available");
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
}
#[test]
fn due_probe_freeze_blocks_scheduling_and_remains_visible_as_rate_limited() {
use gateway_core::account::{AccountCooldown, AccountCooldownKind};
let mut candidates = [
weighted_candidate("acct_frozen", 100, 0),
weighted_candidate("acct_ready", 1, 0),
];
let selection = context(RotationStrategy::Smart);
candidates[0].signals.cooldown = Some(AccountCooldown {
until: selection.now - Duration::from_secs(120),
kind: AccountCooldownKind::CapacityFreezeProbe,
});
let projection = candidates[0]
.account
.status_projection(selection.now, candidates[0].signals.cooldown);
assert_eq!(projection.status, AccountStatus::RateLimited);
assert!(
projection
.cooldown
.expect("freeze details")
.kind
.requires_probe()
);
let selected = AccountSelector
.select(&candidates, &selection)
.expect("available account");
assert_eq!(selected.candidate().account.id().as_str(), "acct_ready");
}
@@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: Some(7),
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: Some(125),
first_output_latency_ms: Some(250),
},
@@ -17,7 +17,7 @@ use gateway_core::task::{
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
use tokio::sync::Notify;
const ACTIVE_KINDS: [WorkerKind; 8] = [
const ACTIVE_KINDS: [WorkerKind; 9] = [
WorkerKind::OAuthRefresh,
WorkerKind::QuotaCatalogHealth,
WorkerKind::RuntimeSnapshotReconciliation,
@@ -26,6 +26,7 @@ const ACTIVE_KINDS: [WorkerKind; 8] = [
WorkerKind::Retention,
WorkerKind::Backup,
WorkerKind::AccountImport,
WorkerKind::AccountFreezeRecovery,
];
#[derive(Clone)]
@@ -54,6 +54,14 @@ impl SettingsStore for AdminSettingsStoreAdapter {
usage_retention_days: command.usage_retention_days,
ops_event_retention_days: command.ops_event_retention_days,
audit_retention_days: command.audit_retention_days,
account_auto_freeze_enabled: command.account_auto_freeze_enabled,
account_auto_freeze_threshold: command.account_auto_freeze_threshold,
account_auto_freeze_window_seconds: command.account_auto_freeze_window_seconds,
account_auto_freeze_duration_seconds: command.account_auto_freeze_duration_seconds,
account_auto_freeze_probe_enabled: command.account_auto_freeze_probe_enabled,
account_auto_freeze_probe_model: command.account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency: command
.account_auto_freeze_adaptive_concurrency,
},
audit: mutation_audit(
context,
@@ -73,6 +81,7 @@ impl SettingsStore for AdminSettingsStoreAdapter {
"min_codex_desktop_version".to_owned(),
"min_codex_cli_version".to_owned(),
"retention".to_owned(),
"account_auto_freeze".to_owned(),
],
),
};
@@ -181,6 +190,13 @@ pub(crate) fn admin_runtime_settings(
usage_retention_days: settings.usage_retention_days,
ops_event_retention_days: settings.ops_event_retention_days,
audit_retention_days: settings.audit_retention_days,
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
account_auto_freeze_threshold: settings.account_auto_freeze_threshold,
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency: settings.account_auto_freeze_adaptive_concurrency,
updated_at: settings.updated_at,
})
}
@@ -208,7 +208,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
credential_state: account.credential_state,
access_token_expires_at: account.access_token_expires_at.map(Into::into),
quota: account.quota,
rate_limited_until: None,
cooldown: None,
last_error_reason: account.last_error_reason,
last_error_message: account.last_error_message,
},
@@ -9,7 +9,7 @@ use super::*;
use crate::postgres::{
PgProviderAccountRepository, ProviderAccountRepository, ProviderAccountSummary,
account_status_projection, load_rate_limited_until,
account_status_projection, load_cooldown,
};
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
@@ -49,19 +49,15 @@ impl PgObservabilityRepository {
)
.await?;
let now = observed_at.into();
let rate_limited_until =
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await;
let cooldown = load_cooldown(self.cooldowns.as_deref(), &accounts, now).await;
let mut metrics = ProviderAccountMetrics {
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
..ProviderAccountMetrics::default()
};
let mut normal_accounts = Vec::new();
for account in &accounts {
let projection = account_status_projection(
account,
now,
rate_limited_until.get(&account.id).copied(),
);
let projection =
account_status_projection(account, now, cooldown.get(&account.id).copied());
match projection.status {
gateway_core::account::AccountStatus::Normal => {
metrics.normal = metrics.normal.saturating_add(1);
@@ -407,15 +407,11 @@ impl AccountStore for PgAdminAccountStore {
));
}
let now = Utc::now();
let rate_limited_until = runtime
.rate_limited_until
.into_iter()
.map(|(account_id, until)| (account_id, until.into()))
.collect::<BTreeMap<_, _>>();
let cooldown = runtime.cooldown;
let now_system_time = std::time::SystemTime::from(now);
let active_rate_limited_ids = rate_limited_until
let active_rate_limited_ids = cooldown
.iter()
.filter(|(_, until)| **until > now_system_time)
.filter(|(_, cooldown)| cooldown.is_active(now_system_time))
.map(|(account_id, _)| account_id.clone())
.collect::<Vec<_>>();
let page =
@@ -434,7 +430,7 @@ impl AccountStore for PgAdminAccountStore {
let projection = account_status_projection(
&summary,
now.into(),
rate_limited_until.get(&account_id).copied(),
cooldown.get(&account_id).copied(),
);
let mut account = admin_account_record(summary)?;
account.groups = groups_by_account.remove(&account_id).unwrap_or_default();
@@ -466,12 +462,8 @@ impl AccountStore for PgAdminAccountStore {
return Ok(None);
};
let now = Utc::now();
let rate_limited_until = runtime
.rate_limited_until
.get(account_id)
.copied()
.map(Into::into);
let projection = account_status_projection(&record.summary, now.into(), rate_limited_until);
let cooldown = runtime.cooldown.get(account_id).copied();
let projection = account_status_projection(&record.summary, now.into(), cooldown);
let account_id = record.summary.id.clone();
let mut groups = self
.account_groups_by_account(std::slice::from_ref(&account_id))
@@ -745,6 +737,69 @@ impl AccountStore for PgAdminAccountStore {
})
}
async fn lower_concurrency_limit(
&self,
account_id: &CoreProviderAccountId,
limit: AccountConcurrencyLimit,
context: &MutationContext,
) -> AdminStoreResult<Option<AccountUpdateResult>> {
let mut transaction = self.pool.begin().await.map_err(|_| {
admin_store_error(ENTITY, postgres_unavailable("begin concurrency reduction"))
})?;
let result =
async {
// 与管理写入采用相同锁顺序;锁住默认值后再检查账号最新设置,避免把旧快照写回。
let default_limit: i64 = sqlx::query_scalar(
"select max_concurrent_per_account from runtime_settings where id = 1 for update"
).fetch_one(&mut *transaction).await
.map_err(|_| postgres_unavailable("lock default concurrency"))?;
let changed = sqlx::query_scalar::<_, String>(
"update provider_accounts set concurrency_limit = $2, updated_at = now()
where id = $1 and enabled = true and coalesce(concurrency_limit, $3) > $2
returning id",
)
.bind(account_id.as_str())
.bind(i64::from(limit.get()))
.bind(default_limit)
.fetch_optional(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("lower account concurrency"))?;
if changed.is_none() {
return Ok(None);
}
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
append_admin_audit_event_in_transaction(
&mut transaction,
mutation_audit(
context,
"adapt_concurrency",
"provider_account",
account_id.as_str(),
vec!["concurrency_limit".to_owned()],
),
revision,
)
.await?;
Ok(Some(revision))
}
.await;
let revision = super::repository::finish_admin_transaction(
transaction,
result,
"lower account concurrency",
)
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
revision
.map(|revision| {
Ok(AccountUpdateResult {
config_revision: admin_revision(revision)?,
account_id: account_id.clone(),
})
})
.transpose()
}
async fn recover_account(
&self,
account_id: &CoreProviderAccountId,
@@ -16,7 +16,7 @@ use super::ProviderAccountSummary;
pub(crate) fn account_status_projection(
account: &ProviderAccountSummary,
now: SystemTime,
rate_limited_until: Option<SystemTime>,
cooldown: Option<gateway_core::account::AccountCooldown>,
) -> AccountStatusProjection {
resolve_account_status(
&AccountStatusFacts {
@@ -24,7 +24,7 @@ pub(crate) fn account_status_projection(
credential_state: account.credential_state,
access_token_expires_at: account.access_token_expires_at.map(Into::into),
quota: account.quota,
rate_limited_until,
cooldown,
last_error_reason: account.last_error_reason,
last_error_message: account.last_error_message.clone(),
},
@@ -36,18 +36,21 @@ pub(crate) fn account_status_projection(
///
/// 冷却属于可丢失的账号级运行时事实:端口不可用、单条读取失败或脏账号 ID
/// 都视为没有冷却,不污染持久账号状态。凭据轮换不会解除上游对同一账号的限流。
pub(crate) async fn load_rate_limited_until(
pub(crate) async fn load_cooldown(
cooldowns: Option<&dyn ProviderCooldownPort>,
accounts: &[ProviderAccountSummary],
now: SystemTime,
) -> BTreeMap<String, SystemTime> {
) -> BTreeMap<String, gateway_core::account::AccountCooldown> {
let Some(cooldowns) = cooldowns else {
return BTreeMap::new();
};
let reads = accounts.iter().map(|account| async move {
let account_id = ProviderAccountId::new(account.id.clone()).ok()?;
let cooldown = cooldowns.read(&account_id).await.ok().flatten()?;
(cooldown.until() > now).then(|| (account.id.clone(), cooldown.until()))
cooldown
.scheduling_state()
.is_active(now)
.then(|| (account.id.clone(), cooldown.scheduling_state()))
});
join_all(reads).await.into_iter().flatten().collect()
}
@@ -11,7 +11,8 @@ use sqlx::{PgPool, Postgres, Transaction};
use gateway_core::account::RotationStrategy;
use gateway_core::policy::CodexClientVersion;
use gateway_core::provider_ports::{
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
ProviderFreezePolicy, ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError,
ProviderStoreErrorKind,
};
use crate::{Revision, StoreError, StoreResult, postgres_unavailable};
@@ -34,6 +35,13 @@ pub struct RuntimeSettings {
pub usage_retention_days: u32,
pub ops_event_retention_days: u32,
pub audit_retention_days: u32,
pub account_auto_freeze_enabled: bool,
pub account_auto_freeze_threshold: u32,
pub account_auto_freeze_window_seconds: u64,
pub account_auto_freeze_duration_seconds: u64,
pub account_auto_freeze_probe_enabled: bool,
pub account_auto_freeze_probe_model: Option<String>,
pub account_auto_freeze_adaptive_concurrency: bool,
pub updated_at: DateTime<Utc>,
}
@@ -60,6 +68,34 @@ impl fmt::Debug for RuntimeSettings {
.field("usage_retention_days", &self.usage_retention_days)
.field("ops_event_retention_days", &self.ops_event_retention_days)
.field("audit_retention_days", &self.audit_retention_days)
.field(
"account_auto_freeze_enabled",
&self.account_auto_freeze_enabled,
)
.field(
"account_auto_freeze_threshold",
&self.account_auto_freeze_threshold,
)
.field(
"account_auto_freeze_window_seconds",
&self.account_auto_freeze_window_seconds,
)
.field(
"account_auto_freeze_duration_seconds",
&self.account_auto_freeze_duration_seconds,
)
.field(
"account_auto_freeze_probe_enabled",
&self.account_auto_freeze_probe_enabled,
)
.field(
"account_auto_freeze_probe_model",
&self.account_auto_freeze_probe_model,
)
.field(
"account_auto_freeze_adaptive_concurrency",
&self.account_auto_freeze_adaptive_concurrency,
)
.field("updated_at", &self.updated_at)
.finish()
}
@@ -82,6 +118,13 @@ pub struct RuntimeSettingsUpdate {
pub usage_retention_days: u32,
pub ops_event_retention_days: u32,
pub audit_retention_days: u32,
pub account_auto_freeze_enabled: bool,
pub account_auto_freeze_threshold: u32,
pub account_auto_freeze_window_seconds: u64,
pub account_auto_freeze_duration_seconds: u64,
pub account_auto_freeze_probe_enabled: bool,
pub account_auto_freeze_probe_model: Option<String>,
pub account_auto_freeze_adaptive_concurrency: bool,
}
impl fmt::Debug for RuntimeSettingsUpdate {
@@ -109,9 +152,13 @@ impl RuntimeSettingsUpdate {
|| self.usage_retention_days < 31
|| self.ops_event_retention_days == 0
|| self.audit_retention_days == 0
|| !(2..=1_000).contains(&self.account_auto_freeze_threshold)
|| !(60..=3_600).contains(&self.account_auto_freeze_window_seconds)
|| !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds)
|| !valid_model_mappings(&self.model_mappings)
|| !valid_client_version(self.min_codex_desktop_version.as_deref())
|| !valid_client_version(self.min_codex_cli_version.as_deref())
|| !valid_probe_model(self.account_auto_freeze_probe_model.as_deref())
|| RotationStrategy::parse(&self.rotation_strategy).is_none()
{
return Err(StoreError::InvalidData {
@@ -173,7 +220,11 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
refresh_concurrency, max_concurrent_per_account, request_interval_ms,
rotation_strategy, model_mappings_json, usage_retention_days, ops_event_retention_days,
audit_retention_days, min_codex_desktop_version,
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds,
account_auto_freeze_enabled, account_auto_freeze_threshold,
account_auto_freeze_window_seconds, account_auto_freeze_duration_seconds,
account_auto_freeze_probe_enabled, account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency
from runtime_settings where id = 1",
)
.fetch_optional(pool)
@@ -202,6 +253,25 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
)
})
}
fn load_freeze_policy(
&self,
) -> futures::future::BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
Box::pin(async move {
let settings = RuntimeSettingsRepository::load_runtime_settings(self)
.await
.map_err(|_| provider_unavailable("load freeze policy"))?;
ProviderFreezePolicy::try_new(
settings.account_auto_freeze_enabled,
settings.account_auto_freeze_threshold,
settings.account_auto_freeze_window_seconds,
settings.account_auto_freeze_duration_seconds,
settings.account_auto_freeze_probe_enabled,
settings.account_auto_freeze_probe_model,
settings.account_auto_freeze_adaptive_concurrency,
)
})
}
}
pub(crate) async fn load_runtime_settings_in_transaction(
@@ -212,7 +282,11 @@ pub(crate) async fn load_runtime_settings_in_transaction(
refresh_concurrency, max_concurrent_per_account, request_interval_ms,
rotation_strategy, model_mappings_json, usage_retention_days, ops_event_retention_days,
audit_retention_days, min_codex_desktop_version,
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds,
account_auto_freeze_enabled, account_auto_freeze_threshold,
account_auto_freeze_window_seconds, account_auto_freeze_duration_seconds,
account_auto_freeze_probe_enabled, account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency
from runtime_settings where id = 1",
)
.fetch_optional(&mut **transaction)
@@ -250,6 +324,13 @@ pub(crate) async fn update_runtime_settings_in_transaction(
max_waiting_per_key = $13,
max_waiting_per_account = $14,
concurrency_wait_timeout_seconds = $15,
account_auto_freeze_enabled = $16,
account_auto_freeze_threshold = $17,
account_auto_freeze_window_seconds = $18,
account_auto_freeze_duration_seconds = $19,
account_auto_freeze_probe_enabled = $20,
account_auto_freeze_probe_model = $21,
account_auto_freeze_adaptive_concurrency = $22,
updated_at = now()
where id = 1
returning config_revision",
@@ -269,6 +350,16 @@ pub(crate) async fn update_runtime_settings_in_transaction(
.bind(i64::from(update.max_waiting_per_key))
.bind(i64::from(update.max_waiting_per_account))
.bind(i64::from(update.concurrency_wait_timeout_seconds))
.bind(update.account_auto_freeze_enabled)
.bind(i64::from(update.account_auto_freeze_threshold))
.bind(i64::try_from(update.account_auto_freeze_window_seconds).map_err(|_| invalid_numeric())?)
.bind(
i64::try_from(update.account_auto_freeze_duration_seconds)
.map_err(|_| invalid_numeric())?,
)
.bind(update.account_auto_freeze_probe_enabled)
.bind(update.account_auto_freeze_probe_model.as_deref())
.bind(update.account_auto_freeze_adaptive_concurrency)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("update runtime settings in transaction"))?
@@ -335,6 +426,13 @@ struct RuntimeSettingsRow {
max_waiting_per_key: i64,
max_waiting_per_account: i64,
concurrency_wait_timeout_seconds: i64,
account_auto_freeze_enabled: bool,
account_auto_freeze_threshold: i64,
account_auto_freeze_window_seconds: i64,
account_auto_freeze_duration_seconds: i64,
account_auto_freeze_probe_enabled: bool,
account_auto_freeze_probe_model: Option<String>,
account_auto_freeze_adaptive_concurrency: bool,
}
fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSettings> {
@@ -356,6 +454,13 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSett
max_waiting_per_key: to_u32(row.max_waiting_per_key)?,
max_waiting_per_account: to_u32(row.max_waiting_per_account)?,
concurrency_wait_timeout_seconds: to_u32(row.concurrency_wait_timeout_seconds)?,
account_auto_freeze_enabled: row.account_auto_freeze_enabled,
account_auto_freeze_threshold: to_u32(row.account_auto_freeze_threshold)?,
account_auto_freeze_window_seconds: to_u64(row.account_auto_freeze_window_seconds)?,
account_auto_freeze_duration_seconds: to_u64(row.account_auto_freeze_duration_seconds)?,
account_auto_freeze_probe_enabled: row.account_auto_freeze_probe_enabled,
account_auto_freeze_probe_model: row.account_auto_freeze_probe_model,
account_auto_freeze_adaptive_concurrency: row.account_auto_freeze_adaptive_concurrency,
})
}
@@ -398,3 +503,12 @@ fn valid_model_name(value: &str, max_len: usize) -> bool {
fn valid_client_version(value: Option<&str>) -> bool {
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
}
fn valid_probe_model(value: Option<&str>) -> bool {
value.is_none_or(|value| {
!value.is_empty()
&& value.len() <= 128
&& value == value.trim()
&& !value.bytes().any(|byte| byte.is_ascii_control())
})
}
@@ -3,6 +3,7 @@
use std::collections::BTreeMap;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use futures::future::join_all;
use gateway_admin::{
model::accounts::AccountRuntimeSnapshot,
@@ -49,15 +50,23 @@ impl AccountRuntimeStore for RedisAdminAccountRuntimeStore {
.read_credential_cooldown(account_id)
.await
.map(|cooldown| {
cooldown.map(|cooldown| (account_id.clone(), cooldown.cooldown_until))
cooldown.map(|cooldown| {
(
account_id.clone(),
gateway_core::account::AccountCooldown {
until: cooldown.cooldown_until.into(),
kind: cooldown.kind,
},
)
})
})
});
let mut rate_limited_until = BTreeMap::new();
let mut cooldown = BTreeMap::new();
for result in join_all(reads).await {
if let Some((account_id, until)) =
result.map_err(|error| crate::admin_store_error("account runtime", error))?
{
rate_limited_until.insert(account_id, until);
cooldown.insert(account_id, until);
}
}
let in_flight = self
@@ -72,8 +81,50 @@ impl AccountRuntimeStore for RedisAdminAccountRuntimeStore {
.collect()
});
Ok(AccountRuntimeSnapshot {
rate_limited_until,
cooldown,
in_flight,
})
}
async fn active_freezes(
&self,
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
self.cooldowns
.active_freezes()
.await
.map_err(|error| crate::admin_store_error("account runtime", error))
}
async fn capacity_peaks(
&self,
account_ids: &[String],
) -> AdminStoreResult<BTreeMap<String, u32>> {
let reads = account_ids.iter().map(|account_id| async move {
self.cooldowns
.read_capacity_peak(account_id)
.await
.map(|peak| peak.map(|value| (account_id.clone(), value)))
});
let mut peaks = BTreeMap::new();
for result in join_all(reads).await {
if let Some((account_id, peak)) =
result.map_err(|error| crate::admin_store_error("account runtime", error))?
{
peaks.insert(account_id, peak);
}
}
Ok(peaks)
}
async fn finish_freeze(
&self,
account_id: &str,
expected: &gateway_admin::model::accounts::AccountFreeze,
postpone_until: Option<DateTime<Utc>>,
) -> AdminStoreResult<bool> {
self.cooldowns
.finish_freeze(account_id, expected, postpone_until)
.await
.map_err(|error| crate::admin_store_error("account runtime", error))
}
}
@@ -6,11 +6,13 @@ use gateway_admin::model::accounts::AccountRuntimeSnapshot;
use gateway_core::{
account::{CredentialRevision, ProviderAccountId},
provider_ports::{
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
ProviderStoreError, ProviderStoreErrorKind,
ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope,
ProviderScopedCooldown, ProviderStoreError, ProviderStoreErrorKind,
},
};
use redis::{Script, aio::ConnectionManager};
use std::collections::BTreeMap;
use std::time::Duration;
use crate::{Revision, StoreError, StoreResult, redis_unavailable, require_nonempty};
@@ -22,17 +24,21 @@ local incoming = tonumber(ARGV[1])
local incoming_until = tonumber(ARGV[2])
if current > incoming then return 0 end
local current_until = tonumber(redis.call('HGET', KEYS[1], 'until_ms') or '0')
if current == incoming and current_until >= incoming_until then return 0 end
local current_kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit'
if current == incoming then
if current_kind ~= 'rate_limit' and ARGV[4] == 'rate_limit' then return 0 end
if current_kind == ARGV[4] and current_until >= incoming_until then return 0 end
incoming_until = math.max(current_until, incoming_until)
end
local clock = redis.call('TIME')
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
if incoming_until <= now_ms then
redis.call('DEL', KEYS[1])
if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[3]) end
return 0
if incoming_until <= now_ms and ARGV[4] ~= 'capacity_freeze_probe' then return 0 end
redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'until_ms', incoming_until, 'kind', ARGV[4], 'generation', ARGV[5])
if ARGV[4] == 'capacity_freeze_probe' then
redis.call('PERSIST', KEYS[1])
else
redis.call('PEXPIRE', KEYS[1], incoming_until - now_ms + 60000)
end
redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'until_ms', ARGV[2])
local ttl = incoming_until - now_ms + 60000
redis.call('PEXPIRE', KEYS[1], ttl)
if #KEYS > 1 then redis.call('ZADD', KEYS[2], incoming_until, ARGV[3]) end
return 1
"#;
@@ -40,19 +46,15 @@ return 1
const READ_SCRIPT: &str = r#"
local revision = redis.call('HGET', KEYS[1], 'revision')
local until_ms = redis.call('HGET', KEYS[1], 'until_ms')
if revision == false or until_ms == false then
redis.call('DEL', KEYS[1])
if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[1]) end
return {0, '0', '0'}
end
local kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit'
local clock = redis.call('TIME')
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
if tonumber(until_ms) <= now_ms then
if revision == false or until_ms == false or (kind ~= 'capacity_freeze_probe' and tonumber(until_ms) <= now_ms) then
redis.call('DEL', KEYS[1])
if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[1]) end
return {0, '0', '0'}
return {0, '0', '0', 'rate_limit', ''}
end
return {1, revision, until_ms}
return {1, revision, until_ms, kind, redis.call('HGET', KEYS[1], 'generation') or ''}
"#;
const INVALIDATE_SCRIPT: &str = r#"
@@ -63,11 +65,56 @@ if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[2]) end
return 1
"#;
const ACTIVE_COOLDOWNS_SCRIPT: &str = r#"
local clock = redis.call('TIME')
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms)
return redis.call('ZRANGEBYSCORE', KEYS[1], '(' .. now_ms, '+inf', 'WITHSCORES')
// 普通推理成功只能清除临时限流和未形成冻结的证据,判断与删除必须原子执行。
const SUCCESS_SCRIPT: &str = r#"
local current = tonumber(redis.call('HGET', KEYS[1], 'revision') or '0')
local kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit'
if current > tonumber(ARGV[1]) or kind ~= 'rate_limit' then return 0 end
redis.call('DEL', KEYS[1], KEYS[3], KEYS[4])
redis.call('ZREM', KEYS[2], ARGV[2])
return 1
"#;
// 探测结果只能修改读到的这一代冻结;删除后重建同 revision 的冻结也不匹配。
const FINISH_FREEZE_SCRIPT: &str = r#"
if redis.call('HGET', KEYS[1], 'generation') ~= ARGV[2]
or redis.call('HGET', KEYS[1], 'revision') ~= ARGV[1] then return 0 end
local kind = redis.call('HGET', KEYS[1], 'kind')
if kind ~= 'capacity_freeze' and kind ~= 'capacity_freeze_probe' then return 0 end
if ARGV[4] == '' then
redis.call('DEL', KEYS[1], KEYS[3], KEYS[4])
redis.call('ZREM', KEYS[2], ARGV[3])
else
local until_ms = math.max(tonumber(redis.call('HGET', KEYS[1], 'until_ms')), tonumber(ARGV[4]))
redis.call('HSET', KEYS[1], 'until_ms', until_ms, 'generation', ARGV[5])
redis.call('ZADD', KEYS[2], until_ms, ARGV[3])
if kind == 'capacity_freeze_probe' then
redis.call('PERSIST', KEYS[1])
else
local clock = redis.call('TIME')
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
redis.call('PEXPIRE', KEYS[1], math.max(1, until_ms - now_ms + 60000))
end
end
return 1
"#;
// 每次容量失败都顺延窗口 TTL(与刷新退避计数同语义),并把本次观测到的
// 在途并发并入峰值证据;峰值与计数共享同一窗口生命周期。
const RECORD_CAPACITY_FAILURE_SCRIPT: &str = r#"
local count = redis.call('INCR', KEYS[1])
local ttl_ms = tonumber(ARGV[1])
redis.call('PEXPIRE', KEYS[1], ttl_ms)
local in_flight = tonumber(ARGV[2])
if in_flight > 0 then
local peak = tonumber(redis.call('GET', KEYS[2]) or '0')
if in_flight > peak then
redis.call('SET', KEYS[2], in_flight, 'PX', ttl_ms)
else
redis.call('PEXPIRE', KEYS[2], ttl_ms)
end
end
return count
"#;
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -75,6 +122,7 @@ pub struct CredentialCooldown {
pub provider_account_id: String,
pub credential_revision: Revision,
pub cooldown_until: DateTime<Utc>,
pub kind: ProviderCooldownKind,
}
#[async_trait]
@@ -116,6 +164,22 @@ impl RedisCredentialCooldownRepository {
format!("{}:account:active-cooldowns", self.namespace)
}
fn capacity_failures_key(&self, provider_account_id: &str) -> StoreResult<String> {
let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?;
Ok(format!(
"{}:account:{fingerprint}:capacity-failures",
self.namespace
))
}
fn capacity_peak_key(&self, provider_account_id: &str) -> StoreResult<String> {
let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?;
Ok(format!(
"{}:account:{fingerprint}:capacity-peak-inflight",
self.namespace
))
}
fn scoped_key(
&self,
provider_account_id: &str,
@@ -135,6 +199,7 @@ impl RedisCredentialCooldownRepository {
key: String,
credential_revision: Revision,
cooldown_until: DateTime<Utc>,
kind: ProviderCooldownKind,
index_member: Option<&str>,
) -> StoreResult<bool> {
let until_ms = cooldown_until.timestamp_millis();
@@ -149,6 +214,8 @@ impl RedisCredentialCooldownRepository {
.arg(credential_revision.get())
.arg(until_ms)
.arg(index_member)
.arg(kind.as_str())
.arg(uuid::Uuid::new_v4().to_string())
.invoke_async::<i64>(&mut connection)
.await
} else {
@@ -157,6 +224,8 @@ impl RedisCredentialCooldownRepository {
.arg(credential_revision.get())
.arg(until_ms)
.arg("")
.arg(kind.as_str())
.arg(uuid::Uuid::new_v4().to_string())
.invoke_async::<i64>(&mut connection)
.await
}
@@ -168,7 +237,7 @@ impl RedisCredentialCooldownRepository {
&self,
key: String,
index_member: Option<&str>,
) -> StoreResult<Option<(Revision, DateTime<Utc>)>> {
) -> StoreResult<Option<(Revision, DateTime<Utc>, ProviderCooldownKind, String)>> {
let mut connection = self.connection.clone();
let result = if let Some(index_member) = index_member {
Script::new(READ_SCRIPT)
@@ -184,7 +253,7 @@ impl RedisCredentialCooldownRepository {
.invoke_async(&mut connection)
.await
};
let (present, revision, until_ms): (i64, String, String) =
let (present, revision, until_ms, kind, generation): (i64, String, String, String, String) =
result.map_err(|_| redis_unavailable("read credential cooldown"))?;
if present == 0 {
return Ok(None);
@@ -197,7 +266,13 @@ impl RedisCredentialCooldownRepository {
.map_err(|_| invalid("cached cooldown expiry is invalid"))?;
let cooldown_until = DateTime::from_timestamp_millis(until_ms)
.ok_or_else(|| invalid("cached cooldown expiry is invalid"))?;
Ok(Some((Revision::new(revision)?, cooldown_until)))
let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit);
Ok(Some((
Revision::new(revision)?,
cooldown_until,
kind,
generation,
)))
}
async fn invalidate_at_key(
@@ -227,30 +302,107 @@ impl RedisCredentialCooldownRepository {
Ok(removed == 1)
}
pub(crate) async fn active_cooldowns(&self) -> StoreResult<AccountRuntimeSnapshot> {
async fn indexed_accounts(&self) -> StoreResult<Vec<String>> {
let mut connection = self.connection.clone();
let values = Script::new(ACTIVE_COOLDOWNS_SCRIPT)
.key(self.active_index_key())
.invoke_async::<Vec<String>>(&mut connection)
redis::cmd("ZRANGE")
.arg(self.active_index_key())
.arg(0)
.arg(-1)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("list active credential cooldowns"))?;
if values.len() % 2 != 0 {
return Err(invalid("active cooldown index is invalid"));
}
let mut rate_limited_until = std::collections::BTreeMap::new();
for pair in values.chunks_exact(2) {
let until_ms = pair[1]
.parse::<i64>()
.map_err(|_| invalid("active cooldown expiry is invalid"))?;
let until = DateTime::from_timestamp_millis(until_ms)
.ok_or_else(|| invalid("active cooldown expiry is invalid"))?;
rate_limited_until.insert(pair[0].clone(), until);
.map_err(|_| redis_unavailable("list indexed cooldowns"))
}
pub(crate) async fn active_cooldowns(&self) -> StoreResult<AccountRuntimeSnapshot> {
let mut cooldown = BTreeMap::new();
for account_id in self.indexed_accounts().await? {
if let Some((_, until, kind, _)) = self
.read_at_key(self.key(&account_id)?, Some(&account_id))
.await?
{
cooldown.insert(
account_id,
gateway_core::account::AccountCooldown {
until: until.into(),
kind,
},
);
}
}
Ok(AccountRuntimeSnapshot {
rate_limited_until,
cooldown,
in_flight: None,
})
}
/// 包含已到探测时间但尚未确认恢复的冻结;到期不能从 worker 工作集中移除。
pub(crate) async fn active_freezes(
&self,
) -> StoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
let mut freezes = BTreeMap::new();
for account_id in self.indexed_accounts().await? {
if let Some((revision, until, kind, generation)) = self
.read_at_key(self.key(&account_id)?, Some(&account_id))
.await?
&& kind.is_capacity_freeze()
{
freezes.insert(
account_id,
gateway_admin::model::accounts::AccountFreeze {
credential_revision: gateway_admin::model::Revision::new(revision.get())
.map_err(|_| invalid("freeze revision"))?,
until,
generation,
requires_probe: kind.requires_probe(),
},
);
}
}
Ok(freezes)
}
pub(crate) async fn finish_freeze(
&self,
account_id: &str,
expected: &gateway_admin::model::accounts::AccountFreeze,
postpone_until: Option<DateTime<Utc>>,
) -> StoreResult<bool> {
let mut connection = self.connection.clone();
let changed: i64 = Script::new(FINISH_FREEZE_SCRIPT)
.key(self.key(account_id)?)
.key(self.active_index_key())
.key(self.capacity_failures_key(account_id)?)
.key(self.capacity_peak_key(account_id)?)
.arg(expected.credential_revision.get())
.arg(&expected.generation)
.arg(account_id)
.arg(
postpone_until
.map(|until| until.timestamp_millis().to_string())
.unwrap_or_default(),
)
.arg(uuid::Uuid::new_v4().to_string())
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("finish capacity freeze"))?;
Ok(changed == 1)
}
/// 读取窗口内观测到的在途并发峰值;key 随窗口 TTL 过期,无需额外清理。
pub(crate) async fn read_capacity_peak(
&self,
provider_account_id: &str,
) -> StoreResult<Option<u32>> {
let mut connection = self.connection.clone();
let peak: Option<i64> = redis::cmd("GET")
.arg(self.capacity_peak_key(provider_account_id)?)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("read capacity peak in-flight"))?;
peak.map(u32::try_from)
.transpose()
.map_err(|_| invalid("capacity peak in-flight is invalid"))
}
}
#[async_trait]
@@ -265,6 +417,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
self.key(&cooldown.provider_account_id)?,
cooldown.credential_revision,
cooldown.cooldown_until,
cooldown.kind,
Some(&cooldown.provider_account_id),
)
.await
@@ -282,11 +435,14 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
self.read_at_key(self.key(provider_account_id)?, Some(provider_account_id))
.await
.map(|value| {
value.map(|(credential_revision, cooldown_until)| CredentialCooldown {
provider_account_id: provider_account_id.to_owned(),
credential_revision,
cooldown_until,
})
value.map(
|(credential_revision, cooldown_until, kind, _)| CredentialCooldown {
provider_account_id: provider_account_id.to_owned(),
credential_revision,
cooldown_until,
kind,
},
)
})
}
@@ -317,8 +473,11 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
// 账号删除:清除该账号的 account key 与全部 model-scoped key。
// 用 SCAN 精确匹配命名空间内该账号前缀,避免 KEYS 阻塞。
let mut connection = self.connection.clone();
let account_key = self.key(provider_account_id)?;
let mut keys = vec![account_key.clone()];
let mut keys = vec![
self.key(provider_account_id)?,
self.capacity_failures_key(provider_account_id)?,
self.capacity_peak_key(provider_account_id)?,
];
let pattern = format!(
"{}:account:{}:cooldown:*",
self.namespace,
@@ -341,21 +500,21 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
break;
}
}
if keys.is_empty() {
return Ok(false);
}
let removed: i64 = redis::cmd("DEL")
.arg(keys)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("delete account cooldown keys"))?;
let index_removed: i64 = redis::cmd("ZREM")
.arg(self.active_index_key())
.arg(provider_account_id)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("remove active account cooldown"))?;
Ok(removed > 0 || index_removed > 0)
// 删除与索引移除同属一个原子边界;否则新冻结可能在两步之间写入后丢失索引。
let removed: i64 = Script::new(
r#"
local removed = 0
for i = 2, #KEYS do removed = removed + redis.call('DEL', KEYS[i]) end
return removed + redis.call('ZREM', KEYS[1], ARGV[1])
"#,
)
.key(self.active_index_key())
.key(keys)
.arg(provider_account_id)
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("delete account cooldowns and index"))?;
Ok(removed > 0)
}
}
@@ -370,6 +529,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
credential_revision: Revision::new(cooldown.credential_revision().get())
.map_err(|_| provider_invalid("encode credential cooldown"))?,
cooldown_until: cooldown.until().into(),
kind: cooldown.kind(),
};
CredentialCooldownRepository::cache_credential_cooldown(self, &record)
.await
@@ -390,10 +550,11 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
.map_err(|_| provider_invalid("decode credential cooldown"))?;
let revision = CredentialRevision::new(record.credential_revision.get())
.map_err(|_| provider_invalid("decode credential cooldown"))?;
Ok(ProviderCooldown::new(
Ok(ProviderCooldown::new_with_kind(
account_id,
revision,
record.cooldown_until.into(),
record.kind,
))
})
.transpose()
@@ -430,6 +591,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
.map_err(|_| provider_invalid("encode scoped credential cooldown"))?,
revision,
cooldown.until().into(),
ProviderCooldownKind::RateLimit,
None,
)
.await
@@ -451,7 +613,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
)
.await
.map_err(|_| provider_unavailable("read scoped credential cooldown"))?
.map(|(revision, until)| {
.map(|(revision, until, _, _)| {
Ok(ProviderScopedCooldown::new(
account_id.clone(),
CredentialRevision::new(revision.get())
@@ -494,6 +656,75 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
.map_err(|_| provider_unavailable("clear all credential cooldowns"))
})
}
fn record_capacity_failure<'a>(
&'a self,
account_id: &'a ProviderAccountId,
window: Duration,
in_flight: u32,
) -> futures::future::BoxFuture<'a, Result<u32, ProviderStoreError>> {
Box::pin(async move {
let mut connection = self.connection.clone();
let window_ms = u64::try_from(window.as_millis())
.map_err(|_| provider_invalid("encode capacity failure window"))?;
let count: i64 = Script::new(RECORD_CAPACITY_FAILURE_SCRIPT)
.key(
self.capacity_failures_key(account_id.as_str())
.map_err(|_| provider_invalid("encode capacity failure key"))?,
)
.key(
self.capacity_peak_key(account_id.as_str())
.map_err(|_| provider_invalid("encode capacity peak key"))?,
)
.arg(i64::try_from(window_ms).unwrap_or(i64::MAX))
.arg(i64::from(in_flight))
.invoke_async(&mut connection)
.await
.map_err(|_| provider_unavailable("record capacity failure"))?;
u32::try_from(count).map_err(|_| provider_invalid("decode capacity failure count"))
})
}
fn clear_after_success<'a>(
&'a self,
account_id: &'a ProviderAccountId,
through_revision: CredentialRevision,
) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> {
Box::pin(async move {
let mut connection = self.connection.clone();
let _: i64 = Script::new(SUCCESS_SCRIPT)
.key(
self.key(account_id.as_str())
.map_err(|_| provider_invalid("cooldown key"))?,
)
.key(self.active_index_key())
.key(
self.capacity_failures_key(account_id.as_str())
.map_err(|_| provider_invalid("capacity count key"))?,
)
.key(
self.capacity_peak_key(account_id.as_str())
.map_err(|_| provider_invalid("capacity peak key"))?,
)
.arg(through_revision.get())
.arg(account_id.as_str())
.invoke_async(&mut connection)
.await
.map_err(|_| provider_unavailable("clear cooldown after success"))?;
Ok(())
})
}
fn capacity_peak_in_flight<'a>(
&'a self,
account_id: &'a ProviderAccountId,
) -> futures::future::BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
Box::pin(async move {
self.read_capacity_peak(account_id.as_str())
.await
.map_err(|_| provider_unavailable("read capacity peak in-flight"))
})
}
}
fn invalid(message: &str) -> StoreError {
@@ -524,7 +524,7 @@ impl RedisProviderLeaseCoordinator {
last_started_at: signal.last_started_at.map(Into::into),
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -645,6 +645,36 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator {
}
})
}
fn account_in_flight<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
) -> futures::future::BoxFuture<
'a,
Result<std::collections::BTreeMap<ProviderAccountId, u32>, ProviderStoreError>,
> {
Box::pin(async move {
if account_ids.is_empty() {
return Ok(std::collections::BTreeMap::new());
}
let ids = account_ids
.iter()
.map(|account_id| account_id.as_str().to_owned())
.collect::<Vec<_>>();
let signals = self
.repository
.credential_runtime_signals(&ids)
.await
.map_err(|_| provider_unavailable("load account in-flight signals"))?;
Ok(signals
.into_iter()
.filter_map(|signal| {
let account_id = ProviderAccountId::new(signal.resource_id).ok()?;
Some((account_id, signal.in_flight))
})
.collect())
})
}
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
@@ -855,6 +855,30 @@ impl ProviderCooldownPort for StaticCooldowns {
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn record_capacity_failure<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_window: Duration,
_in_flight: u32,
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
Box::pin(async { Ok(0) })
}
fn clear_after_success<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_through_revision: gateway_core::account::CredentialRevision,
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
Box::pin(async { Ok(()) })
}
fn capacity_peak_in_flight<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
Box::pin(async { Ok(None) })
}
}
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
@@ -852,9 +852,9 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
sort: None,
},
AccountRuntimeSnapshot {
rate_limited_until: BTreeMap::from([(
cooldown: BTreeMap::from([(
"acct_alpha".to_owned(),
now + TimeDelta::minutes(5),
SystemTime::from(now + TimeDelta::minutes(5)).into(),
)]),
in_flight: None,
},
@@ -3387,3 +3387,83 @@ async fn model_access_only_batch_update_preserves_other_settings_and_survives_re
assert_eq!(loaded.summary.model_access, AccountModelAccess::all());
database.close().await;
}
#[tokio::test]
async fn adaptive_concurrency_uses_latest_locked_settings_without_overwriting_admin_fields() {
let Some(database) = TestDatabase::create("adaptive_concurrency").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let store = admin_account_store(&database.pool);
let id = ProviderAccountId::new("acct_adaptive").expect("id");
repository
.insert_provider_account(account(id.as_str(), "adaptive-user"))
.await
.expect("account");
let group = "grp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, 'kept group', '#112233FF', now(), now())")
.bind(group).execute(&database.pool).await.expect("group");
sqlx::query("insert into account_group_accounts (account_group_id, provider_account_id, created_at) values ($1, $2, now())")
.bind(group).bind(id.as_str()).execute(&database.pool).await.expect("membership");
for (enabled, account_limit, default_limit, expected_limit, changes) in [
(false, Some(8_i64), 10_i64, Some(8_i64), false),
(true, Some(2), 10, Some(2), false),
(true, None, 2, None, false),
(true, None, 10, Some(3), true),
] {
let mut admin = database.pool.begin().await.expect("admin transaction");
sqlx::query("update runtime_settings set max_concurrent_per_account = $1 where id = 1")
.bind(default_limit)
.execute(&mut *admin)
.await
.expect("latest default");
let worker_store = store.clone();
let worker_id = id.clone();
let worker = tokio::spawn(async move {
worker_store
.lower_concurrency_limit(
&worker_id,
gateway_core::account::AccountConcurrencyLimit::new(3).expect("limit"),
&MutationContext {
actor: MutationActor::System,
request_id: "adaptive-worker".to_owned(),
},
)
.await
.expect("atomic adaptation")
});
// 管理员持有设置锁时提交新的账号事实,worker 必须依据提交后的值判断。
sqlx::query("update provider_accounts set enabled = $2, concurrency_limit = $3, weight = 7, notes = 'administrator edit' where id = $1")
.bind(id.as_str()).bind(enabled).bind(account_limit).execute(&mut *admin).await.expect("concurrent admin edit");
admin.commit().await.expect("commit admin edit");
let result = worker.await.expect("worker");
assert_eq!(result.is_some(), changes);
let actual = repository
.load_provider_account(id.as_str())
.await
.expect("read account")
.expect("account");
assert_eq!(actual.summary.enabled, enabled);
assert_eq!(
actual
.summary
.concurrency_limit
.map(|value| i64::from(value.get())),
expected_limit
);
assert_eq!(actual.summary.weight.get(), 7);
assert_eq!(actual.summary.notes.as_deref(), Some("administrator edit"));
assert_eq!(
account_group_ids(&database.pool, id.as_str()).await,
vec![group.to_owned()]
);
}
let audited_fields: Vec<Vec<String>> = sqlx::query_scalar(
"select changed_fields from admin_audit_events where action = 'adapt_concurrency'",
)
.fetch_all(&database.pool)
.await
.expect("audit");
assert_eq!(audited_fields, vec![vec!["concurrency_limit".to_owned()]]);
database.close().await;
}
@@ -27,6 +27,13 @@ fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettingsUpdate {
usage_retention_days: 31,
ops_event_retention_days: 30,
audit_retention_days: 90,
account_auto_freeze_enabled: true,
account_auto_freeze_threshold: 12,
account_auto_freeze_window_seconds: 600,
account_auto_freeze_duration_seconds: 7_200,
account_auto_freeze_probe_enabled: true,
account_auto_freeze_probe_model: None,
account_auto_freeze_adaptive_concurrency: true,
}
}
@@ -46,6 +53,30 @@ fn runtime_settings_reject_invalid_model_mapping() {
assert!(settings.validate().is_err());
}
#[test]
fn runtime_settings_reject_out_of_range_auto_freeze() {
for update in [
RuntimeSettingsUpdate {
account_auto_freeze_threshold: 1,
..settings_with_margin(3_600)
},
RuntimeSettingsUpdate {
account_auto_freeze_window_seconds: 59,
..settings_with_margin(3_600)
},
RuntimeSettingsUpdate {
account_auto_freeze_duration_seconds: 299,
..settings_with_margin(3_600)
},
RuntimeSettingsUpdate {
account_auto_freeze_probe_model: Some(" pad ".to_owned()),
..settings_with_margin(3_600)
},
] {
assert!(update.validate().is_err());
}
}
#[test]
fn runtime_settings_reject_non_semver_client_min() {
let settings = RuntimeSettingsUpdate {
@@ -238,3 +269,52 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
assert!(snapshot.config_revision > before.config_revision);
database.close().await;
}
#[tokio::test]
async fn auto_freeze_defaults_off_and_migration_preserves_explicit_opt_in() {
let Some(database) = TestDatabase::create("freeze_opt_in").await else {
return;
};
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
assert!(
!repository
.load_runtime_settings()
.await
.expect("default settings")
.account_auto_freeze_enabled
);
sqlx::query("update runtime_settings set account_auto_freeze_enabled = true where id = 1")
.execute(&database.pool)
.await
.expect("old default");
let migration = include_str!("../../../../migrations/0011_account_auto_freeze_opt_in.sql");
sqlx::raw_sql(migration)
.execute(&database.pool)
.await
.expect("migrate untouched default");
assert!(
!repository
.load_runtime_settings()
.await
.expect("settings")
.account_auto_freeze_enabled
);
sqlx::query("update runtime_settings set account_auto_freeze_enabled = true where id = 1")
.execute(&database.pool)
.await
.expect("explicit opt-in");
sqlx::query("insert into admin_audit_events (id, actor_kind, actor_ref, action, entity_kind, entity_ref, changed_fields, created_at) values ('freeze-opt-in', 'system', 'test', 'settings.replace', 'runtime_settings', '1', array['account_auto_freeze'], now())")
.execute(&database.pool).await.expect("saved settings audit");
sqlx::raw_sql(migration)
.execute(&database.pool)
.await
.expect("preserve opt-in");
assert!(
repository
.load_runtime_settings()
.await
.expect("settings")
.account_auto_freeze_enabled
);
database.close().await;
}
@@ -3,7 +3,9 @@ use std::time::{Duration as StdDuration, SystemTime};
use chrono::{DateTime, Duration, Utc};
use gateway_core::{
account::{CredentialRevision, ProviderAccountId},
provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown},
provider_ports::{
ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
},
};
use gateway_store::{
Revision,
@@ -18,6 +20,7 @@ fn credential_cooldown_is_revision_fenced() {
provider_account_id: "account-1".to_owned(),
credential_revision: Revision::new(2).expect("positive revision"),
cooldown_until: Utc::now() + Duration::seconds(30),
kind: ProviderCooldownKind::RateLimit,
};
assert_eq!(cooldown.credential_revision.get(), 2);
}
@@ -159,6 +162,7 @@ async fn credential_cooldown_read_removes_expired_grace_key() {
provider_account_id: "acct_cooldown_expiry".to_owned(),
credential_revision: Revision::new(1).expect("positive revision"),
cooldown_until: millisecond_precision(cooldown_until),
kind: ProviderCooldownKind::RateLimit,
};
repository
.cache_credential_cooldown(&cooldown)
@@ -284,6 +288,7 @@ fn cooldown(account_id: &str, revision: u64, seconds: i64) -> CredentialCooldown
provider_account_id: account_id.to_owned(),
credential_revision: Revision::new(revision).expect("positive revision"),
cooldown_until: millisecond_precision(Utc::now() + Duration::seconds(seconds)),
kind: ProviderCooldownKind::RateLimit,
}
}
@@ -311,3 +316,233 @@ async fn namespace_keys(connection: &mut ConnectionManager, namespace: &str) ->
.await
.expect("list isolated cooldown keys")
}
fn runtime_store(
repository: &RedisCredentialCooldownRepository,
connection: ConnectionManager,
namespace: &str,
) -> gateway_store::redis::RedisAdminAccountRuntimeStore {
gateway_store::redis::RedisAdminAccountRuntimeStore::new(
repository.clone(),
gateway_store::redis::RedisCredentialLeaseRepository::new(connection, namespace)
.expect("lease repository"),
)
}
#[tokio::test]
async fn capacity_freeze_survives_due_time_and_runtime_restart_until_probe_succeeds() {
use gateway_admin::ports::store::AccountRuntimeStore as _;
let Some((repository, mut connection, namespace)) = repository().await else {
return;
};
let frozen = CredentialCooldown {
kind: ProviderCooldownKind::CapacityFreezeProbe,
cooldown_until: millisecond_precision(Utc::now() - Duration::seconds(90)),
..cooldown("acct_probe_due", 1, 30)
};
repository
.cache_credential_cooldown(&frozen)
.await
.expect("write due freeze");
let runtime = runtime_store(&repository, connection.clone(), &namespace);
let snapshot = runtime.active_rate_limits().await.expect("status snapshot");
assert!(snapshot.cooldown[&frozen.provider_account_id].is_active(SystemTime::now()));
assert!(
snapshot.cooldown[&frozen.provider_account_id]
.kind
.requires_probe()
);
let key = namespace_keys(&mut connection, &namespace)
.await
.into_iter()
.find(|key| key.ends_with(":cooldown"))
.expect("freeze key");
let ttl: i64 = redis::cmd("PTTL")
.arg(key)
.query_async(&mut connection)
.await
.expect("TTL");
assert_eq!(
ttl, -1,
"awaiting probe must not expire while worker is unavailable"
);
drop(runtime);
let restarted = runtime_store(&repository, connection, &namespace);
let freezes = restarted
.active_freezes()
.await
.expect("due freezes after restart");
assert_eq!(freezes.len(), 1);
assert!(
restarted
.finish_freeze(
&frozen.provider_account_id,
&freezes[&frozen.provider_account_id],
None
)
.await
.expect("probe completion")
);
assert!(
repository
.read_credential_cooldown(&frozen.provider_account_id)
.await
.expect("read")
.is_none()
);
}
#[tokio::test]
async fn stale_probe_cannot_recreate_manual_recovery_or_clear_or_extend_new_freeze() {
use gateway_admin::ports::store::AccountRuntimeStore as _;
let Some((repository, connection, namespace)) = repository().await else {
return;
};
let frozen = CredentialCooldown {
kind: ProviderCooldownKind::CapacityFreezeProbe,
..cooldown("acct_probe_generation", 1, 30)
};
let runtime = runtime_store(&repository, connection, &namespace);
repository
.cache_credential_cooldown(&frozen)
.await
.expect("freeze");
let original =
runtime.active_freezes().await.expect("snapshot")[&frozen.provider_account_id].clone();
repository
.delete_account_cooldowns(&frozen.provider_account_id)
.await
.expect("manual recovery");
let postponed = Utc::now() + Duration::hours(2);
assert!(
!runtime
.finish_freeze(&frozen.provider_account_id, &original, Some(postponed))
.await
.expect("stale failure")
);
assert!(
repository
.read_credential_cooldown(&frozen.provider_account_id)
.await
.expect("read")
.is_none()
);
// 同凭据版本、相同截止时间的新冻结仍必须视为不同代次。
repository
.cache_credential_cooldown(&frozen)
.await
.expect("new freeze");
assert!(
!runtime
.finish_freeze(&frozen.provider_account_id, &original, None)
.await
.expect("stale success")
);
assert!(
!runtime
.finish_freeze(&frozen.provider_account_id, &original, Some(postponed))
.await
.expect("stale failure")
);
let current =
runtime.active_freezes().await.expect("new snapshot")[&frozen.provider_account_id].clone();
assert_ne!(current.generation, original.generation);
assert!(
runtime
.finish_freeze(&frozen.provider_account_id, &current, Some(postponed))
.await
.expect("postpone current freeze")
);
assert!(
!runtime
.finish_freeze(&frozen.provider_account_id, &current, None)
.await
.expect("result from previous probe cycle")
);
let extended = runtime.active_freezes().await.expect("extended snapshot")
[&frozen.provider_account_id]
.clone();
assert!(extended.until >= current.until);
assert!(
runtime
.finish_freeze(&frozen.provider_account_id, &extended, None)
.await
.expect("recover current cycle")
);
}
#[tokio::test]
async fn ordinary_success_and_later_429_preserve_capacity_freeze_and_peak_evidence() {
let Some((repository, _connection, _namespace)) = repository().await else {
return;
};
let id = ProviderAccountId::new("acct_late_success").expect("account id");
let revision = CredentialRevision::new(1).expect("revision");
repository
.record_capacity_failure(&id, StdDuration::from_secs(600), 8)
.await
.expect("failure evidence");
let frozen = CredentialCooldown {
kind: ProviderCooldownKind::CapacityFreezeProbe,
..cooldown(id.as_str(), 1, 30)
};
repository
.cache_credential_cooldown(&frozen)
.await
.expect("freeze");
repository
.clear_after_success(&id, revision)
.await
.expect("late success");
assert_eq!(
repository.capacity_peak_in_flight(&id).await.expect("peak"),
Some(8)
);
assert!(
!repository
.cache_credential_cooldown(&cooldown(id.as_str(), 1, 60))
.await
.expect("429 must not downgrade freeze")
);
assert_eq!(
repository
.read_credential_cooldown(id.as_str())
.await
.expect("read"),
Some(frozen)
);
}
#[tokio::test]
async fn ordinary_success_still_clears_rate_limit_and_unfrozen_failure_evidence() {
let Some((repository, _connection, _namespace)) = repository().await else {
return;
};
let id = ProviderAccountId::new("acct_success_429").expect("id");
repository
.record_capacity_failure(&id, StdDuration::from_secs(600), 3)
.await
.expect("failure evidence");
repository
.cache_credential_cooldown(&cooldown(id.as_str(), 1, 30))
.await
.expect("429");
repository
.clear_after_success(&id, CredentialRevision::new(1).expect("revision"))
.await
.expect("success");
assert!(
repository
.read_credential_cooldown(id.as_str())
.await
.expect("read")
.is_none()
);
assert!(
repository
.capacity_peak_in_flight(&id)
.await
.expect("peak")
.is_none()
);
}
@@ -24,7 +24,10 @@ use gateway_core::account::{
QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaObservationTouch,
QuotaState, QuotaWriteOutcome,
};
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
use gateway_core::provider_ports::{
ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderFreezePolicy,
ProviderLeasePort, ProviderRuntimePolicyPort,
};
use gateway_protocol::openai::events::{
ParsedRateLimits, RateLimitDetails, RateLimitWindow, parse_rate_limit_headers,
};
@@ -184,10 +187,17 @@ pub struct CodexCredentialQuotaService {
http: Client,
base_url: String,
cooldowns: Arc<dyn ProviderCooldownPort>,
leases: Arc<dyn ProviderLeasePort>,
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
/// 冻结策略的短 TTL 缓存:失败路径热读,避免每个容量错误都查询设置。
freeze_policy_cache: Mutex<Option<(ProviderFreezePolicy, Instant)>>,
scheduling: CodexQuotaSchedulingProjection,
reset_consume_locks: Mutex<HashMap<ProviderAccountId, Arc<Mutex<()>>>>,
}
/// 冻结策略缓存活跃期;过期后下一次容量错误重新读取运行时设置。
const FREEZE_POLICY_CACHE_TTL: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum QuotaRefreshAuthority {
ObserveAccess,
@@ -481,6 +491,8 @@ impl CodexCredentialQuotaService {
http: Client,
base_url: String,
cooldowns: Arc<dyn ProviderCooldownPort>,
leases: Arc<dyn ProviderLeasePort>,
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
) -> Self {
Self {
store: Arc::clone(repository.store()),
@@ -489,11 +501,89 @@ impl CodexCredentialQuotaService {
http,
base_url,
cooldowns,
leases,
runtime_policy,
freeze_policy_cache: Mutex::new(None),
scheduling: CodexQuotaSchedulingProjection::default(),
reset_consume_locks: Mutex::new(HashMap::new()),
}
}
/// 读取容量熔断策略(带短 TTL 缓存);读取失败退化为关闭,熔断不得放大失败。
async fn freeze_policy(&self) -> ProviderFreezePolicy {
{
let cache = self.freeze_policy_cache.lock().await;
if let Some((policy, loaded_at)) = cache.as_ref()
&& loaded_at.elapsed() < FREEZE_POLICY_CACHE_TTL
{
return policy.clone();
}
}
let loaded = self.runtime_policy.load_freeze_policy().await.ok();
let mut cache = self.freeze_policy_cache.lock().await;
if let Some(policy) = loaded {
*cache = Some((policy.clone(), Instant::now()));
return policy;
}
cache
.as_ref()
.filter(|(_, loaded_at)| loaded_at.elapsed() < FREEZE_POLICY_CACHE_TTL)
.map_or_else(ProviderFreezePolicy::disabled, |(policy, _)| policy.clone())
}
/// 容量熔断入口:滑动窗口内累计容量类失败,达到阈值即写入带
/// `CapacityFreeze` 类别的账号级冷却。调度侧立即屏蔽该账号,恢复由
/// freeze-recovery worker 处理;本路径只依赖 Redis 可丢失事实。
pub async fn apply_capacity_failure(&self, account: &ProviderAccount, observed_at: SystemTime) {
let policy = self.freeze_policy().await;
if !policy.enabled() {
return;
}
let in_flight = self.current_in_flight(account.id()).await;
let Ok(count) = self
.cooldowns
.record_capacity_failure(account.id(), policy.window(), in_flight)
.await
else {
return;
};
if count < policy.threshold() {
return;
}
let Some(until) = observed_at.checked_add(policy.freeze_duration()) else {
return;
};
let cooldown = ProviderCooldown::new_with_kind(
account.id().clone(),
account.revision(),
until,
if policy.probe_enabled() {
ProviderCooldownKind::CapacityFreezeProbe
} else {
ProviderCooldownKind::CapacityFreeze
},
);
if self.cooldowns.put_if_later(cooldown).await.is_ok() {
tracing::warn!(
account_id = account.id().as_str(),
threshold = policy.threshold(),
window_seconds = policy.window().as_secs(),
freeze_seconds = policy.freeze_duration().as_secs(),
peak_in_flight = in_flight,
"账号容量熔断触发:冻结该账号一段时间",
);
}
}
async fn current_in_flight(&self, account_id: &ProviderAccountId) -> u32 {
self.leases
.account_in_flight(std::slice::from_ref(account_id))
.await
.ok()
.and_then(|signals| signals.get(account_id).copied())
.unwrap_or(0)
}
/// 查询当前账号由 Codex Desktop 暴露的主动额度重置卡。
pub async fn list_reset_credits(
&self,
@@ -607,7 +697,7 @@ impl CodexCredentialQuotaService {
}
/// 成功推理是额度可访问的权威证据,同时解除账号级 429 冷却。
pub(crate) async fn record_successful_inference(
pub async fn record_successful_inference(
&self,
account: &ProviderAccount,
observed_at: SystemTime,
@@ -628,7 +718,7 @@ impl CodexCredentialQuotaService {
}
}
self.cooldowns
.clear(account.id(), account.revision())
.clear_after_success(account.id(), account.revision())
.await
.map_err(|error| CodexCredentialQuotaError::Store {
detail: error.to_string(),
@@ -975,11 +1065,11 @@ impl CodexCredentialQuotaService {
Ok(())
}
/// 读取账号当前是否处于临时限流(429)冷却,及到期时间。
pub async fn rate_limited_until(
/// 读取有效的账号冷却事实;等待恢复探测的冻结到期后仍有效。
pub async fn cooldown(
&self,
account_id: &ProviderAccountId,
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
) -> Result<Option<gateway_core::account::AccountCooldown>, CodexCredentialQuotaError> {
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
CodexCredentialQuotaError::Store {
detail: error.to_string(),
@@ -988,11 +1078,8 @@ impl CodexCredentialQuotaService {
else {
return Ok(None);
};
let until = cooldown.until();
if until <= SystemTime::now() {
return Ok(None);
}
Ok(Some(until))
let state = cooldown.scheduling_state();
Ok(state.is_active(SystemTime::now()).then_some(state))
}
/// 读取单账号最后一次落库的 Provider quota,并由 Codex 域解析展示窗口。
@@ -451,11 +451,7 @@ impl CodexCredentialSelector {
let mut rate_limits = HashMap::with_capacity(accounts.len());
if !diagnostic {
for account in &accounts {
let until = self
.quota
.rate_limited_until(account.id())
.await
.unwrap_or(None);
let until = self.quota.cooldown(account.id()).await.unwrap_or(None);
rate_limits.insert(account.id().clone(), until);
}
}
@@ -487,7 +483,7 @@ impl CodexCredentialSelector {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
})
@@ -1330,7 +1326,7 @@ fn affinity_selection_for_bound_account(
};
match candidate
.account
.status_projection(now, candidate.signals.rate_limited_until)
.status_projection(now, candidate.signals.cooldown)
.status
{
AccountStatus::Normal => AffinitySelection::preferred(account_id),
@@ -1349,7 +1345,7 @@ fn affinity_unavailable_reason(
) -> AffinityEscapeReason {
match candidate
.account
.status_projection(now, candidate.signals.rate_limited_until)
.status_projection(now, candidate.signals.cooldown)
.status
{
AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
@@ -112,6 +112,8 @@ pub async fn initialize(
http.clone(),
config.base_url().to_owned(),
ports.cooldowns(),
Arc::clone(&leases),
Arc::clone(&runtime_policy),
));
let profile_statistics = Arc::new(CodexCredentialProfileService::new(
repository.clone(),
@@ -403,6 +403,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
response_origin: &request.response_origin,
cyber_policy_scope: None,
allows_account_state_mutation,
allows_capacity_feedback: !request.context.is_diagnostic_required_account(),
};
let active_account = request.lease.account().clone();
let cookie_header = build_cookie_header(request.lease.cookies())?;
@@ -577,6 +578,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
response_origin: &response_origin,
cyber_policy_scope: cyber_policy_scope.as_ref(),
allows_account_state_mutation,
allows_capacity_feedback: !context.is_diagnostic_required_account(),
};
let mut active_account = lease.account().clone();
let cookie_header = build_cookie_header(lease.cookies())?;
@@ -1,13 +1,15 @@
//! OpenAI 上游失败分类、恢复决策与稳定错误投影。
use super::*;
use crate::transport::diagnostics::is_capacity_error;
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集。
///
/// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type。
/// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集。
/// 容量拒绝影响短期调度健康度,但不证明账号凭据或额度失效。
const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[
"server_is_overloaded",
"slow_down",
"rate_limit_exceeded",
"rate_limit_error",
"server_error",
@@ -32,17 +34,6 @@ fn openai_account_score_failure_reason(error: &ProviderError) -> Option<&str> {
/// 返回该 OpenAI 失败是否属于 Smart 账号计分闭集。
#[doc(hidden)]
pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
let client_error = error.client_visible_upstream_error();
if is_capacity_error(
error
.upstream_code()
.map(OpaqueUpstreamValue::as_str)
.or_else(|| client_error.and_then(ClientVisibleUpstreamError::code)),
client_error.and_then(ClientVisibleUpstreamError::error_type),
client_error.map(ClientVisibleUpstreamError::message),
) {
return false;
}
openai_account_score_failure_reason(error).is_some_and(is_openai_account_score_failure_reason)
}
@@ -53,6 +44,7 @@ pub(super) struct MappedProviderFailure {
/// 原始上游错误描述,仅在凭据错误状态下持久化。
pub(super) error_message: Option<String>,
pub(super) cyber_policy_failure: bool,
pub(super) upstream_capacity_failure: bool,
pub(super) set_cookie_headers: Vec<String>,
pub(super) rate_limit_headers: Vec<(String, String)>,
pub(super) observation: Option<ProviderResponseObservation>,
@@ -67,6 +59,7 @@ impl MappedProviderFailure {
account_failure: None,
error_message: None,
cyber_policy_failure: false,
upstream_capacity_failure: false,
set_cookie_headers: Vec::new(),
rate_limit_headers: Vec::new(),
observation: None,
@@ -211,6 +204,7 @@ pub(super) struct OpenAiFailureContext<'a> {
pub(super) response_origin: &'a Url,
pub(super) cyber_policy_scope: Option<&'a CodexCyberPolicyScope>,
pub(super) allows_account_state_mutation: bool,
pub(super) allows_capacity_feedback: bool,
}
#[derive(Clone, Copy)]
@@ -410,6 +404,13 @@ pub(super) async fn apply_failure(
"Failed to persist OpenAI response cookies"
);
}
// 探测只验证恢复,不能把探测自身的单请求并发当作业务负载证据。
if context.allows_capacity_feedback && failure.upstream_capacity_failure {
context
.quota
.apply_capacity_failure(account, SystemTime::now())
.await;
}
}
pub(super) fn schedule_authoritative_quota_refresh_after_failure(
@@ -1269,6 +1270,12 @@ pub(super) fn map_upstream_failure(
),
error_message: failure.client_message,
cyber_policy_failure,
// 仅上游明确返回的容量/服务不可用计数;本地连接保护也会映射为
// Unavailable,不能用通用 ProviderErrorKind 推断容量证据。
upstream_capacity_failure: matches!(
category,
CodexFailureCategory::CapacityUnavailable | CodexFailureCategory::Unavailable
),
set_cookie_headers: failure.set_cookie_headers,
rate_limit_headers: failure.rate_limit_headers,
observation,
@@ -1591,6 +1591,30 @@ impl ProviderCooldownPort for TestCooldown {
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn record_capacity_failure<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_window: Duration,
_in_flight: u32,
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
Box::pin(async { Ok(0) })
}
fn clear_after_success<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_through_revision: gateway_core::account::CredentialRevision,
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
Box::pin(async { Ok(()) })
}
fn capacity_peak_in_flight<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
Box::pin(async { Ok(None) })
}
}
struct TestRuntimePolicy;
@@ -20,7 +20,8 @@ use gateway_core::engine::{
use gateway_core::lifecycle::CancellationToken;
use gateway_core::policy::ClientApiKeyId;
use gateway_core::provider_ports::{
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
ProviderCooldownPort, ProviderLeasePort, ProviderSessionAffinityKey,
ProviderSessionAffinityPort,
};
use gateway_core::routing::{
ClientRoutingScope, FrozenAccountScope, ProviderKind, RuntimeAccount, RuntimeAccountDirectory,
@@ -178,6 +179,8 @@ fn selector_with_runtime(
http,
OFFICIAL_CODEX_BASE_URL.to_owned(),
cooldowns,
Arc::clone(&leases) as Arc<dyn ProviderLeasePort>,
crate::support::runtime_policy(),
));
CodexCredentialSelector::new(
ProviderKind::new("openai").expect("provider"),
@@ -0,0 +1,195 @@
//! 容量熔断触发:高频容量类失败按滑动窗口计数冻结账号,成功调用清空证据。
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use gateway_core::provider_ports::{
ProviderCooldownKind, ProviderCooldownPort as _, ProviderFreezePolicy,
};
use provider_openai::credential::CodexCredentialQuotaService;
use super::{MemoryAccountStore, create_account, wire_profile};
use crate::support::{MemoryCooldownPort, StaticFreezePolicy, TestLeaseCoordinator};
fn freeze_policy(threshold: u32) -> ProviderFreezePolicy {
ProviderFreezePolicy::try_new(true, threshold, 600, 7_200, true, None, true)
.expect("valid freeze policy")
}
async fn freeze_service(
store: &Arc<MemoryAccountStore>,
policy: ProviderFreezePolicy,
) -> (Arc<CodexCredentialQuotaService>, Arc<MemoryCooldownPort>) {
let cooldowns = Arc::new(MemoryCooldownPort::new());
let quota = Arc::new(CodexCredentialQuotaService::new(
store.repository(),
wire_profile(),
reqwest::Client::builder()
.no_proxy()
.build()
.expect("client"),
provider_openai::OFFICIAL_CODEX_BASE_URL.to_owned(),
Arc::clone(&cooldowns) as Arc<dyn gateway_core::provider_ports::ProviderCooldownPort>,
Arc::new(TestLeaseCoordinator::default()),
StaticFreezePolicy::policy_port(policy),
));
(quota, cooldowns)
}
#[tokio::test]
async fn capacity_failures_below_threshold_do_not_freeze() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_capacity_below").await;
let account = store
.account("acct_capacity_below")
.expect("created account");
let (quota, cooldowns) = freeze_service(&store, freeze_policy(3)).await;
for _ in 0..2 {
quota
.apply_capacity_failure(&account, SystemTime::now())
.await;
}
assert_eq!(
cooldowns
.capacity_evidence(account.id())
.map(|(count, _)| count),
Some(2)
);
assert!(
quota
.cooldown(account.id())
.await
.expect("cooldown read")
.is_none()
);
}
#[tokio::test]
async fn capacity_failures_reaching_threshold_write_capacity_freeze_cooldown() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_capacity_freeze").await;
let account = store
.account("acct_capacity_freeze")
.expect("created account");
let (quota, cooldowns) = freeze_service(&store, freeze_policy(3)).await;
for _ in 0..3 {
quota
.apply_capacity_failure(&account, SystemTime::now())
.await;
}
let until = quota
.cooldown(account.id())
.await
.expect("cooldown read")
.expect("freeze cooldown written");
let remaining = until
.until
.duration_since(SystemTime::now())
.expect("freeze in the future");
assert!(
remaining > Duration::from_secs(7_000) && remaining <= Duration::from_secs(7_200),
"freeze should last about 2 hours, got {remaining:?}"
);
let cooldown = cooldowns.read(account.id()).await.expect("read cooldown");
assert_eq!(
cooldown.expect("cooldown").kind(),
ProviderCooldownKind::CapacityFreezeProbe
);
}
#[tokio::test]
async fn disabled_freeze_policy_records_nothing() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_capacity_disabled").await;
let account = store
.account("acct_capacity_disabled")
.expect("created account");
let (quota, cooldowns) = freeze_service(&store, ProviderFreezePolicy::disabled()).await;
for _ in 0..5 {
quota
.apply_capacity_failure(&account, SystemTime::now())
.await;
}
assert!(cooldowns.capacity_evidence(account.id()).is_none());
assert!(
quota
.cooldown(account.id())
.await
.expect("cooldown read")
.is_none()
);
}
#[tokio::test]
async fn successful_inference_clears_capacity_evidence() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_capacity_success").await;
let account = store
.account("acct_capacity_success")
.expect("created account");
let (quota, cooldowns) = freeze_service(&store, freeze_policy(100)).await;
for _ in 0..3 {
quota
.apply_capacity_failure(&account, SystemTime::now())
.await;
}
assert!(cooldowns.capacity_evidence(account.id()).is_some());
quota
.record_successful_inference(&account, SystemTime::now())
.await
.expect("record success");
assert!(cooldowns.capacity_evidence(account.id()).is_none());
}
/// 阈值与窗口边界使用与迁移 check 约束一致的构造校验。
#[test]
fn freeze_policy_rejects_out_of_range_configuration() {
for (threshold, window, duration) in [(1, 600, 7_200), (12, 59, 7_200), (12, 600, 299)] {
assert!(
ProviderFreezePolicy::try_new(true, threshold, window, duration, true, None, true)
.is_err()
);
}
assert!(
ProviderFreezePolicy::try_new(true, 12, 600, 7_200, true, Some(" ".to_owned()), true)
.is_err()
);
assert!(ProviderFreezePolicy::try_new(true, 12, 600, 7_200, true, None, true).is_ok());
}
#[tokio::test]
async fn in_flight_success_does_not_release_a_new_capacity_freeze() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_freeze_late_success").await;
let account = store.account("acct_freeze_late_success").expect("account");
let (quota, cooldowns) = freeze_service(&store, freeze_policy(2)).await;
for _ in 0..2 {
quota
.apply_capacity_failure(&account, SystemTime::now())
.await;
}
let before = cooldowns.read(account.id()).await.expect("freeze");
quota
.record_successful_inference(&account, SystemTime::now())
.await
.expect("late success");
assert_eq!(
cooldowns.read(account.id()).await.expect("freeze remains"),
before
);
assert_eq!(
cooldowns
.capacity_evidence(account.id())
.map(|(count, _)| count),
Some(2)
);
}
@@ -1,5 +1,6 @@
//! OpenAI 额度事实边界与展示快照回归。
mod capacity_freeze;
mod recovery;
mod refresh_timing;
mod scheduling;
@@ -68,6 +69,8 @@ fn quota_service_with_base_url(
http,
base_url,
Arc::new(crate::support::MemoryCooldownPort::new()),
Arc::new(crate::support::TestLeaseCoordinator::default()),
crate::support::runtime_policy(),
)
}
@@ -84,7 +84,7 @@ async fn old_weekly_quota_should_outweigh_one_in_flight_title_request() {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
}
@@ -32,6 +32,7 @@ use gateway_core::operation::{
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
};
use gateway_core::policy::ClientApiKeyId;
use gateway_core::provider_ports::ProviderLeasePort;
use gateway_core::routing::{
ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ModelServiceTier,
ProviderKind, ProviderModel, PublicModelId, RoutingContext, RuntimeAccount,
@@ -227,6 +228,31 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
leases: Arc<TestLeaseCoordinator>,
stream_max_retries: u32,
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
let (provider, quota, _) = provider_and_quota_with_runtime_ports(
store,
session_affinity,
base_url,
leases,
stream_max_retries,
Arc::new(MemoryCooldownPort::new()),
crate::support::runtime_policy(),
);
(provider, quota)
}
fn provider_and_quota_with_runtime_ports(
store: &Arc<MemoryAccountStore>,
session_affinity: Arc<MemorySessionAffinity>,
base_url: String,
leases: Arc<TestLeaseCoordinator>,
stream_max_retries: u32,
cooldowns: Arc<MemoryCooldownPort>,
policy: Arc<dyn gateway_core::provider_ports::ProviderRuntimePolicyPort>,
) -> (
CodexProvider,
Arc<CodexCredentialQuotaService>,
Arc<CodexWebSocketPool>,
) {
let profile = wire_profile();
let http = reqwest::Client::builder()
.no_proxy()
@@ -245,7 +271,9 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
profile.clone(),
http.clone(),
base_url.clone(),
Arc::new(MemoryCooldownPort::new()),
cooldowns,
Arc::clone(&leases) as Arc<dyn ProviderLeasePort>,
policy,
));
let account_feedback = Arc::new(AccountFeedbackStats::default());
let selector = Arc::new(CodexCredentialSelector::new(
@@ -268,11 +296,11 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
http,
profile,
base_url,
websocket_pool,
Arc::clone(&websocket_pool),
stream_max_retries,
)
.expect("official OpenAI provider");
(provider, quota)
(provider, quota, websocket_pool)
}
async fn create_account(store: &Arc<MemoryAccountStore>, id: &str) {
@@ -5476,7 +5504,11 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
create_account(&store, account_id).await;
let server = MockServer::start().await;
let body = json!({"error": {
"code": if status == 503 { Some("server_is_overloaded") } else { None },
"code": match status {
429 => Some("slow_down"),
503 => Some("server_is_overloaded"),
_ => None,
},
"type": "server_error",
"message": "Selected model is at capacity. Please try a different model.",
"extension": {"preserved": true}
@@ -5518,7 +5550,7 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
max_retries, initial_delay, max_delay,
}) if max_retries.get() == 3 && initial_delay == Duration::from_secs(8) && max_delay == Duration::from_secs(8))
);
assert!(!provider_openai::openai_failure_affects_account_score(
assert!(provider_openai::openai_failure_affects_account_score(
&error
));
assert_eq!(
@@ -5540,6 +5572,196 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
}
}
fn capacity_websocket_operation() -> Operation {
let payload = ProtocolPayload::json_object(
"openai",
Map::from_iter([
("model".to_owned(), json!("gpt-5.4")),
("input".to_owned(), json!("hello")),
("store".to_owned(), json!(false)),
("session_id".to_owned(), json!("capacity-feedback-session")),
]),
)
.expect("WebSocket payload")
.with_context(Map::from_iter([(
"downstream_websocket_connection_id".to_owned(),
json!("ws_capacity_feedback"),
)]));
Operation::Generate(
GenerateRequest::from_protocol_payload(payload).with_provider_session_state(
ProviderSessionState::new(
"openai",
Map::from_iter([
("account_id".to_owned(), json!("acct_provider_contract")),
(
"conversation_id".to_owned(),
json!("capacity-feedback-session"),
),
("continuation_scope".to_owned(), json!("persisted")),
]),
)
.expect("WebSocket session scope"),
),
)
}
fn provider_with_capacity_tracking(
store: &Arc<MemoryAccountStore>,
base_url: String,
cooldowns: Arc<MemoryCooldownPort>,
) -> (CodexProvider, Arc<CodexWebSocketPool>) {
let (provider, _, pool) = provider_and_quota_with_runtime_ports(
store,
Arc::new(MemorySessionAffinity::default()),
base_url,
Arc::new(TestLeaseCoordinator::default()),
0,
cooldowns,
crate::support::StaticFreezePolicy::policy_port(
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
true, 12, 600, 7_200, true, None, true,
)
.expect("freeze policy"),
),
);
(provider, pool)
}
#[tokio::test]
async fn capacity_feedback_counts_business_rejections_but_excludes_diagnostic_probes() {
use gateway_core::provider_ports::ProviderCooldownPort as _;
for websocket in [false, true] {
for (status, code) in [
(429, "slow_down"),
(503, "server_is_overloaded"),
(500, "server_error"),
] {
for diagnostic in [false, true] {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_provider_contract";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("account");
let cooldowns = Arc::new(MemoryCooldownPort::new());
let server = MockServer::start().await;
Mock::given(method(if websocket { "GET" } else { "POST" }))
.and(path("/codex/responses"))
.respond_with(
ResponseTemplate::new(status).set_body_json(json!({"error": {
"code": code,
"message": "Upstream temporarily unavailable"
}})),
)
.expect(2)
.mount(&server)
.await;
let (provider, _) =
provider_with_capacity_tracking(&store, server.uri(), Arc::clone(&cooldowns));
// 同时验证窗口证据已过期与尚未过期:探测不能重建峰值,也不能改写原计数。
for existing_evidence in [None, Some((4, 20))] {
if let Some((count, peak)) = existing_evidence {
for _ in 0..count {
cooldowns
.record_capacity_failure(
account.id(),
Duration::from_secs(600),
peak,
)
.await
.expect("seed normal-request evidence");
}
}
let before = cooldowns.capacity_evidence(account.id());
let attempt = if diagnostic {
diagnostic_context("req_capacity_feedback_probe", account_id)
} else {
context("req_capacity_feedback_business", CancellationToken::new())
};
let operation = if websocket {
capacity_websocket_operation()
} else {
http_generate_operation()
};
let mut stream = provider
.execute(planned_request("openai", operation), attempt)
.await
.expect("prepare upstream attempt");
let error = loop {
match stream.next().await {
Some(Ok(_)) => {}
Some(Err(error)) => break error,
None => panic!("expected upstream rejection"),
}
};
assert_eq!(
error.upstream_status(),
Some(status),
"unexpected upstream error: {error:?}"
);
let after = cooldowns.capacity_evidence(account.id());
if diagnostic {
assert_eq!(after, before, "probe must preserve capacity count and peak");
} else {
assert_eq!(
after.map(|(count, _)| count),
Some(before.map_or(1, |(count, _)| count + 1))
);
}
}
}
}
}
}
#[tokio::test]
async fn local_websocket_connection_cancellation_does_not_supply_capacity_evidence() {
let store = Arc::new(MemoryAccountStore::default());
let account_id = "acct_provider_contract";
create_account(&store, account_id).await;
let account = store.account(account_id).expect("account");
let cooldowns = Arc::new(MemoryCooldownPort::new());
let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener");
let base_url = format!("http://{}", listener.local_addr().expect("address"));
let (provider, pool) =
provider_with_capacity_tracking(&store, base_url, Arc::clone(&cooldowns));
let operation = capacity_websocket_operation();
let mut stream = provider
.execute(
planned_request("openai", operation),
context("req_capacity_local", CancellationToken::new()),
)
.await
.expect("prepare WebSocket attempt");
let attempt = tokio::spawn(async move {
loop {
match stream.next().await {
Some(Ok(_)) => {}
Some(Err(error)) => break error,
None => panic!("cancelled opening must fail"),
}
}
});
let (mut opening, _) = timeout(Duration::from_secs(5), listener.accept())
.await
.expect("opening deadline")
.expect("opening");
read_http_request(&mut opening).await;
// 复现账号更新驱逐正在建连的连接,未收到任何上游容量拒绝。
pool.evict_account(account_id).await;
let error = timeout(Duration::from_secs(5), attempt)
.await
.expect("cancelled attempt deadline")
.expect("attempt task");
assert_eq!(error.kind(), ProviderErrorKind::Unavailable);
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
assert_eq!(
error.diagnostic().and_then(|diagnostic| diagnostic.code()),
Some("shared_connect_failed")
);
assert_eq!(cooldowns.capacity_evidence(account.id()), None);
pool.shutdown().await;
}
#[tokio::test]
async fn websocket_usage_limit_rejection_preserves_quota_state_for_account_rotation() {
let store = Arc::new(MemoryAccountStore::default());
@@ -5578,7 +5800,12 @@ async fn websocket_usage_limit_rejection_preserves_quota_state_for_account_rotat
#[tokio::test]
async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_transports() {
for use_websocket in [false, true] {
for (use_websocket, code) in [
(false, "server_is_overloaded"),
(true, "server_is_overloaded"),
(false, "slow_down"),
(true, "slow_down"),
] {
for semantic_output in [false, true] {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_provider_contract").await;
@@ -5588,7 +5815,7 @@ async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_t
if semantic_output {
events.push(json!({"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello"}));
}
let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": "server_is_overloaded", "message": "Selected model is at capacity. Please try a different model."}}});
let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": code, "message": "Selected model is at capacity. Please try a different model."}}});
events.push(original.clone());
let (base_url, _http_server, websocket_server) = if use_websocket {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener");
@@ -5653,7 +5880,7 @@ async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_t
assert_eq!(error.replay_is_safe(), !semantic_output);
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
assert_eq!(error.pre_delivery_retry().is_some(), !semantic_output);
assert!(!provider_openai::openai_failure_affects_account_score(
assert!(provider_openai::openai_failure_affects_account_score(
&error
));
assert_eq!(
@@ -1,4 +1,6 @@
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::Duration;
use futures::{StreamExt, stream};
use gateway_core::account::{AccountFeedbackStats, ProviderAccountId};
@@ -45,7 +47,7 @@ fn deliver_error_with_openai_feedback(
}
#[test]
fn account_score_failure_filter_should_exclude_request_scoped_capacity_errors() {
fn account_score_failure_filter_should_accept_capacity_errors_with_scored_reasons() {
for code in ["server_is_overloaded", "slow_down", "server_error"] {
let error = sent_error(ProviderErrorKind::Unavailable, Some(code))
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
@@ -53,13 +55,15 @@ fn account_score_failure_filter_should_exclude_request_scoped_capacity_errors()
Some("server_error".to_owned()),
Some("server_error".to_owned()),
));
assert!(!openai_failure_affects_account_score(&error));
assert!(openai_failure_affects_account_score(&error));
}
}
#[test]
fn account_score_failure_filter_should_accept_the_closed_reason_list() {
for code in [
"server_is_overloaded",
"slow_down",
"rate_limit_exceeded",
"rate_limit_error",
"server_error",
@@ -82,14 +86,20 @@ fn account_score_failure_filter_should_normalize_case_and_whitespace() {
#[test]
fn account_score_failure_filter_should_use_a_structured_type_when_code_is_absent() {
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"upstream unavailable",
None,
Some("SERVICE_UNAVAILABLE_ERROR".to_owned()),
));
for error_type in [
"SERVICE_UNAVAILABLE_ERROR",
"SERVER_IS_OVERLOADED",
"SLOW_DOWN",
] {
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"upstream unavailable",
None,
Some(error_type.to_owned()),
));
assert!(openai_failure_affects_account_score(&error));
assert!(openai_failure_affects_account_score(&error));
}
}
#[test]
@@ -185,17 +195,27 @@ fn openai_feedback_should_not_amplify_repeated_client_configuration_errors() {
}
#[test]
fn openai_feedback_should_not_penalize_an_account_for_server_overload() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("provider");
let account = ProviderAccountId::new("acct_overloaded").expect("account");
fn openai_feedback_should_penalize_capacity_rejections_even_with_transient_retry() {
for code in ["server_is_overloaded", "slow_down"] {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("provider");
let account = ProviderAccountId::new("acct_overloaded").expect("account");
let error = sent_error(ProviderErrorKind::UpstreamCapacityUnavailable, Some(code))
.with_replay_safe()
.with_transient_retry(
NonZeroU32::new(3).expect("retry count"),
Duration::from_millis(500),
Duration::from_secs(8),
);
deliver_error_with_openai_feedback(
&feedback,
&provider,
&account,
sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded")),
);
deliver_error_with_openai_feedback(&feedback, &provider, &account, error);
assert_eq!(feedback.scheduling_signals(&provider, &account).0, None);
assert!(
feedback
.scheduling_signals(&provider, &account)
.0
.is_some_and(|failure_rate| failure_rate > 0),
"capacity rejection did not affect the account score: {code}"
);
}
}
@@ -20,11 +20,11 @@ use gateway_core::error::{StoreError, StoreErrorKind};
use gateway_core::policy::ClientApiKeyId;
use gateway_core::provider_ports::{
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort,
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest,
ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey,
ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions,
ProviderStoreError,
ProviderCooldownScope, ProviderFreezePolicy, ProviderLeaseAcquisition, ProviderLeasePort,
ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderScopedCooldown,
ProviderSessionAffinityKey, ProviderSessionAffinityPort, ProviderSessionExclusionPort,
ProviderSessionExclusions, ProviderStoreError,
};
use gateway_core::routing::ProviderKind;
use provider_openai::credential::{
@@ -615,7 +615,7 @@ impl ProviderLeasePort for TestLeaseCoordinator {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -956,6 +956,36 @@ pub(crate) fn runtime_policy() -> Arc<dyn ProviderRuntimePolicyPort> {
Arc::new(StaticRuntimePolicy)
}
/// 冻结策略可配置的运行时策略 fake;容量熔断触发测试用。
pub(crate) struct StaticFreezePolicy(ProviderFreezePolicy);
impl StaticFreezePolicy {
/// 构造带固定冻结策略的端口句柄;命名沿用测试构造器语义。
#[must_use]
pub(crate) fn policy_port(policy: ProviderFreezePolicy) -> Arc<dyn ProviderRuntimePolicyPort> {
Arc::new(Self(policy))
}
}
impl ProviderRuntimePolicyPort for StaticFreezePolicy {
fn load_refresh_policy(
&self,
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>> {
Box::pin(async {
ProviderRefreshPolicy::try_new(
Duration::from_secs(60 * 60),
NonZeroU32::new(2).expect("positive concurrency"),
)
})
}
fn load_freeze_policy(
&self,
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
Box::pin(async { Ok(self.0.clone()) })
}
}
pub(crate) fn secret(access_token: &str) -> CodexOAuthSecret {
CodexOAuthSecret {
access_token: SecretString::from(access_token.to_owned()),
@@ -972,11 +1002,13 @@ pub(crate) fn account_policy() -> gateway_core::account::AccountSelectionPolicy
)
}
/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector
/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。
/// 内存 `ProviderCooldownPort`:实现 `read`/`put_if_later` 与容量失败计数
/// 证据(openai selector 429 冷却与容量熔断路径用),scope 变体测试不涉及,
/// 返回占位。
#[derive(Clone, Default)]
pub(crate) struct MemoryCooldownPort {
pub(crate) cooldowns: Arc<Mutex<BTreeMap<ProviderAccountId, ProviderCooldown>>>,
capacity_failures: Arc<Mutex<BTreeMap<ProviderAccountId, (u32, u32)>>>,
}
impl MemoryCooldownPort {
@@ -984,6 +1016,16 @@ impl MemoryCooldownPort {
pub(crate) fn new() -> Self {
Self::default()
}
/// 测试断言用:窗口内累计失败次数与在途峰值。
#[must_use]
pub(crate) fn capacity_evidence(&self, account_id: &ProviderAccountId) -> Option<(u32, u32)> {
self.capacity_failures
.lock()
.expect("capacity failures lock")
.get(account_id)
.copied()
}
}
impl ProviderCooldownPort for MemoryCooldownPort {
@@ -1057,4 +1099,57 @@ impl ProviderCooldownPort for MemoryCooldownPort {
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn record_capacity_failure<'a>(
&'a self,
account_id: &'a ProviderAccountId,
_window: Duration,
in_flight: u32,
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
Box::pin(async move {
let mut failures = self
.capacity_failures
.lock()
.expect("capacity failures lock");
let entry = failures.entry(account_id.clone()).or_insert((0, 0));
entry.0 = entry.0.saturating_add(1);
entry.1 = entry.1.max(in_flight);
Ok(entry.0)
})
}
fn clear_after_success<'a>(
&'a self,
account_id: &'a ProviderAccountId,
_through_revision: gateway_core::account::CredentialRevision,
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
Box::pin(async move {
let mut cooldowns = self.cooldowns.lock().expect("cooldown lock");
if cooldowns.get(account_id).is_some_and(|value| {
value.kind().is_capacity_freeze() || value.credential_revision() > _through_revision
}) {
return Ok(());
}
cooldowns.remove(account_id);
self.capacity_failures
.lock()
.expect("capacity failures lock")
.remove(account_id);
Ok(())
})
}
fn capacity_peak_in_flight<'a>(
&'a self,
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
Box::pin(async move {
Ok(self
.capacity_failures
.lock()
.expect("capacity failures lock")
.get(account_id)
.map(|(_, peak)| *peak))
})
}
}
+25 -1
View File
@@ -666,7 +666,7 @@ impl ProviderLeasePort for TestLeases {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -862,6 +862,30 @@ impl ProviderCooldownPort for TestCooldown {
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
Box::pin(async { Ok(false) })
}
fn record_capacity_failure<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_window: Duration,
_in_flight: u32,
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
Box::pin(async { Ok(0) })
}
fn clear_after_success<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_through_revision: gateway_core::account::CredentialRevision,
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
Box::pin(async { Ok(()) })
}
fn capacity_peak_in_flight<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
Box::pin(async { Ok(None) })
}
}
struct TestRuntimePolicy;
@@ -138,7 +138,7 @@ impl SelectorFixture {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -1065,7 +1065,7 @@ impl ProviderLeasePort for DiagnosticLeasePort {
last_started_at: None,
quota_reset_at: None,
quota_remaining_rank: None,
rate_limited_until: None,
cooldown: None,
failure_rate_basis_points: None,
first_output_latency_ms: None,
},
@@ -972,6 +972,30 @@ impl ProviderCooldownPort for MemoryCooldownPort {
Ok(account_removed || scoped_removed)
})
}
fn record_capacity_failure<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_window: Duration,
_in_flight: u32,
) -> futures::future::BoxFuture<'a, Result<u32, ProviderStoreError>> {
Box::pin(async { Ok(0) })
}
fn clear_after_success<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
_through_revision: gateway_core::account::CredentialRevision,
) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> {
Box::pin(async { Ok(()) })
}
fn capacity_peak_in_flight<'a>(
&'a self,
_account_id: &'a ProviderAccountId,
) -> futures::future::BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
Box::pin(async { Ok(None) })
}
}
impl ProviderRuntimePolicyPort for StaticRuntimePolicy {
+2
View File
@@ -7,3 +7,5 @@ b7d3df7b48d835a00a3b74cbefc958a039d38fea5529cdad04bfbc703dca72aa 0001_initial.s
18e8c1b71c896a9d576d9e4ef9701506f654f55383af908898195b9eb2980fde 0007_account_notes.sql
36b720c9d3e6bf0bdb799b9b6a89b81638c642ac866992a880bd82357012a208 0008_concurrency_queues.sql
161df9ae43da34d51b340bbf95d71c61a8c1a2aa142382f693a9d127c834630d 0009_account_model_access.sql
dd4aa34e9bcf1698bb9688421964bbf410314d8e905448b8229460b972cff2f6 0010_account_auto_freeze.sql
9bd261b7e94872126754eb7f749c56187264a47c567b0bef5c394d6776b4f463 0011_account_auto_freeze_opt_in.sql
@@ -0,0 +1,22 @@
-- 账号容量熔断:统计窗口内高频出现容量类上游错误时自动冻结账号一段时间;
-- 恢复前可选执行一次探测调用验证上游已恢复;冻结期间按观测到的在途并发峰值
-- 下调账号并发上限,让账号带着更安全的并发配置自愈。默认启用,已有实例可关闭。
alter table runtime_settings
add column account_auto_freeze_enabled boolean not null default true,
add column account_auto_freeze_threshold bigint not null default 12
check (account_auto_freeze_threshold between 2 and 1000),
add column account_auto_freeze_window_seconds bigint not null default 600
check (account_auto_freeze_window_seconds between 60 and 3600),
add column account_auto_freeze_duration_seconds bigint not null default 7200
check (account_auto_freeze_duration_seconds between 300 and 604800),
add column account_auto_freeze_probe_enabled boolean not null default true,
add column account_auto_freeze_probe_model text
check (
account_auto_freeze_probe_model is null
or (
octet_length(account_auto_freeze_probe_model) between 1 and 128
and account_auto_freeze_probe_model = btrim(account_auto_freeze_probe_model)
and account_auto_freeze_probe_model !~ '[[:cntrl:]]'
)
),
add column account_auto_freeze_adaptive_concurrency boolean not null default true;
@@ -0,0 +1,12 @@
-- 自动冻结默认关闭;保留已经通过设置页显式保存的冻结配置。
alter table runtime_settings
alter column account_auto_freeze_enabled set default false;
update runtime_settings
set account_auto_freeze_enabled = false
where account_auto_freeze_enabled = true
and not exists (
select 1 from admin_audit_events
where action = 'settings.replace'
and changed_fields @> array['account_auto_freeze']::text[]
);
+27 -1
View File
@@ -253,7 +253,10 @@ Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
OpenAI 明确返回 `server_is_overloaded`、`slow_down` 或模型容量不足错误时,代理在允许安全重放且
尚未交付输出的前提下,先做最多 3 次同账号指数退避,再通过现有调度换号。默认间隔从 500ms 开始,
上游 `Retry-After` 参与退避计算,单次等待不超过 8 秒;重试同时受请求总尝试次数和截止时间约束。
容量不足不扣 Smart 账号健康分,不触发 Provider 全局熔断,也不作为账号额度耗尽写入冷却状态。
`server_is_overloaded`、`slow_down` 与既有可计分的结构化错误按已发送的失败尝试计入 Smart 账号
健康分;容量分类不再豁免计分。失败率沿用账号级平滑与时间衰减,影响后续普通选路,已有可用账号的
会话亲和仍优先。容量不足不触发 Provider 全局熔断,也不作为账号额度耗尽;启用账号自动冻结时,
达到容量失败阈值会另外写入临时冷却。
最终交付的上游错误仍按上述透明边界保留原始状态码、错误码和正文。
明确额度耗尽继续走现有账号隔离与安全换号流程,
包括 WebSocket 握手返回的 429;不会因其长 `Retry-After` 而转入同账号传输恢复等待。
@@ -367,6 +370,11 @@ Token 明细、费用明细、用时/首字与状态。Token 和费用复用现
- `sortBy`: `email`、`status`、`planType`、`usage`、`lastUsedAt`、`expiresAt`;
- `sortDirection`: `asc`、`desc`。
账号限流详情在 `quota` 中返回:`rateLimitReason` 为 `upstream_rate_limit`(上游临时限流)、
`capacity_freeze`(容量错误触发自动冻结)或 `null`。`recoveryProbeRequired` 表示解除冻结是否需要成功探测;
此时 `rateLimitedUntil` 是最早探测时间,到期后仍保持 `rate_limited`,直到探测成功或手动恢复。
未要求探测时,该字段表示冷却结束时间。所有此类情况统一显示“限流中”,仅详情原因和恢复条件不同。
账号列表和详情返回 `notes`(无备注时为 `null`)。编辑时省略或 `null` 保留原备注;字符串最多 500 个 Unicode
字符,允许换行和制表符,保存时去除首尾空白,空字符串清空备注。备注独立于上游身份,导入时未显式提供备注、
重新授权、凭据刷新及批量调度更新均保留已有备注。
@@ -910,6 +918,13 @@ minCodexCliVersion
usageRetentionDays
opsEventRetentionDays
auditRetentionDays
accountAutoFreezeEnabled
accountAutoFreezeThreshold
accountAutoFreezeWindowSeconds
accountAutoFreezeDurationSeconds
accountAutoFreezeProbeEnabled
accountAutoFreezeProbeModel
accountAutoFreezeAdaptiveConcurrency
```
`maxWaitingPerKey` 与 `maxWaitingPerAccount` 是全局统一的排队容量,取值 0~1,000,默认 0(关闭);
@@ -922,6 +937,17 @@ auditRetentionDays
`rotationStrategy` 可取 `smart`、`quota_reset_priority`、`round_robin`、`sticky`。
两个 `minCodex*Version` 字段为 `string | null`,只设置最低版本,不存在最大版本字段。
账号自动冻结(`accountAutoFreezeEnabled`)默认关闭。启用后,在统计窗口内按尝试累计容量类上游错误(`server_is_overloaded`
等与 5xx 不可用),达到阈值后把该账号冻结为带恢复倒计时的 `rate_limited` 状态。`accountAutoFreezeThreshold`
取值 2~1,000(默认 12,按普通请求的 attempt 计数,含请求内同账号重试,不含诊断探测与本地连接保护错误);`accountAutoFreezeWindowSeconds`
取值 60~3,600(默认 600,随每次失败滑动顺延);`accountAutoFreezeDurationSeconds` 取值 300~604,800
(默认 7,200,即 2 小时,探测失败后按该时长顺延)。`accountAutoFreezeProbeEnabled` 开启时恢复 worker
在到期后执行真实探测调用,成功才解冻;探测过程中和服务重启后继续阻止普通请求。关闭自动冻结或探测后,
已有冻结等待当前冷却结束再恢复调度。`accountAutoFreezeProbeModel` 为 `string | null`,留空时自动
选择账号可用的第一个模型。`accountAutoFreezeAdaptiveConcurrency` 开启时冻结期间把账号并发上限下调到
观测在途峰值的 80%(下限 2,只降不升)。这会持久修改账号并发设置;跟随全局默认的账号也会设为独立上限,
解冻后不自动恢复,管理员可手动改回。
Windows 离线包接口固定解析 Microsoft Store Product ID `9PLM9XGG6VKS` 的 Retail 包,不接受调用方提供
产品 ID、上游地址、ring 或文件名。后端只返回通过包名、架构、Microsoft CDN host/path、scheme 和失效
时间校验的 `x64` / `arm64` MSIX 直链,不代理安装包字节。Store 内容通道返回 HTTP/80 临时地址时保留
+15
View File
@@ -422,6 +422,10 @@ PostgreSQL 周期对账才是正确性基础。
`normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`。只有明确上游证据才能恢复或终态化账号,
本地时钟和不确定响应不能伪造事实。
容量冻结与普通上游限流共享账号冷却投影,但恢复条件不同:需要探测的冻结在截止时间后仍阻止调度,
普通推理成功不能解除。Admin 恢复任务按冻结代次原子提交探测结果,避免旧结果覆盖手动恢复或新冻结;
自动降低并发通过现有管理事务与快照发布链路,仅按数据库最新设置更新并发上限。
PostgreSQL schema 由迁移目录按编号管理。已应用迁移按字节冻结,后续 schema 变化只能新增编号迁移,
详见 [迁移规则](../backend/migrations/README.md)。
@@ -497,8 +501,19 @@ Worker 由各 Bundle 贡献、由 Host 统一监督:
- Store:过期请求恢复、历史保留和 PostgreSQL/Redis 观测队列;
- Core:`runtime` owner 的 RuntimeSnapshot 周期对账和 Redis change 订阅;
- Admin:S3/R2 备份 daemon,负责调度、执行、删除收敛与保留清理;
以及账号冻结恢复 worker(容量熔断的自适应并发下调与到期探测解冻);
- Provider:credential refresh、quota/catalog 健康和官方版本/etag 检查。
账号容量熔断默认关闭。启用后,仅普通请求收到的容量类上游错误(`server_is_overloaded` 等与
5xx 不可用)按滑动窗口计数,并把当时观测到的在途并发并入峰值证据;本地连接保护与诊断探测
不参与容量计数或峰值采样。这些错误不证明凭据或配额失效,不进入账号失败状态。达到阈值后写入
账号级 Redis 容量冷却,调度立即跳过该账号;管理端沿用限流状态,通过原因区分容量冻结与上游限流。
冻结与计数由 Redis 保存,不改变 PostgreSQL 账号状态。需要探测的冻结在到期后仍阻止调度,恢复
worker 复用连接测试探针执行真实上游调用,并按冻结代次处理结果:成功清除,失败按配置时长顺延,
过期探测结果不得覆盖更新的冻结或管理员操作。关闭自动冻结或探测后,已有冻结在冷却到期后恢复。
自适应并发下调按观测峰值的 80%(下限 2)原子更新最新有效账号上限,只降不升,不覆盖其他账号
设置,审计标注为系统变更;该上限持久保存,恢复调度后不自动调高。
周期任务的 Redis lease 只保证单周期互斥,不构成多副本 leader 选举。备份 daemon 依赖单副本部署边界,
自更新也只替换处理请求的当前进程,因此整个应用必须保持单副本。
+2
View File
@@ -33,6 +33,8 @@ export interface AccountQuota {
limitReached: boolean
// 429 临时限流(Redis 冷却)到期时间;非限流中为 null。
rateLimitedUntil: string | null
rateLimitReason: 'upstream_rate_limit' | 'capacity_freeze' | null
recoveryProbeRequired: boolean
windows: AccountQuotaWindow[]
}
+7
View File
@@ -18,6 +18,13 @@ export interface RuntimeSettings {
usageRetentionDays: number
opsEventRetentionDays: number
auditRetentionDays: number
accountAutoFreezeEnabled: boolean
accountAutoFreezeThreshold: number
accountAutoFreezeWindowSeconds: number
accountAutoFreezeDurationSeconds: number
accountAutoFreezeProbeEnabled: boolean
accountAutoFreezeProbeModel: string | null
accountAutoFreezeAdaptiveConcurrency: boolean
updatedAt: string
}
@@ -62,6 +62,9 @@ function connectionLogClass(tone: string) {
:status="account.status"
:error-reason="account.errorReason"
:error-message="account.errorMessage"
:rate-limited-until="account.quota.rateLimitedUntil"
:rate-limit-reason="account.quota.rateLimitReason"
:recovery-probe-required="account.quota.recoveryProbeRequired"
:next-refresh-at="account.nextRefreshAt"
variant="pill"
/>
@@ -13,6 +13,8 @@ const props = withDefaults(
errorReason?: AccountErrorReason | null
errorMessage?: string | null
rateLimitedUntil?: string | null
rateLimitReason?: 'upstream_rate_limit' | 'capacity_freeze' | null
recoveryProbeRequired?: boolean
nextRefreshAt?: string | null
variant?: 'inline' | 'pill'
}>(),
@@ -20,6 +22,8 @@ const props = withDefaults(
errorReason: null,
errorMessage: null,
rateLimitedUntil: null,
rateLimitReason: null,
recoveryProbeRequired: false,
nextRefreshAt: null,
variant: 'inline',
},
@@ -32,6 +36,8 @@ const presentation = computed(() =>
errorReason: props.errorReason,
errorMessage: props.errorMessage,
rateLimitedUntil: props.rateLimitedUntil,
rateLimitReason: props.rateLimitReason,
recoveryProbeRequired: props.recoveryProbeRequired,
nextRefreshAt: props.nextRefreshAt,
now: now.value.getTime(),
}),
@@ -102,7 +108,7 @@ const presentation = computed(() =>
v-if="presentation.rateLimitRecovery"
class="flex items-center justify-between gap-3 rounded-cp bg-cp-fill-quaternary px-3 py-2"
>
<span class="font-heavy text-cp-text-tertiary">预计恢复</span>
<span class="font-heavy text-cp-text-tertiary">{{ presentation.recoveryTimeLabel }}</span>
<span class="font-mono font-emphasis tabular-nums text-cp-text">
{{ presentation.rateLimitRecovery }}
</span>
@@ -37,6 +37,7 @@ export interface AccountStatusPresentation {
errorText: string | null
nextRefreshDisplay: string | null
rateLimitRecovery: string | null
recoveryTimeLabel: string
triggerLabel: string
}
@@ -45,6 +46,8 @@ export interface AccountStatusPresentationInput {
errorReason: AccountErrorReason | null
errorMessage: string | null
rateLimitedUntil: string | null
rateLimitReason: 'upstream_rate_limit' | 'capacity_freeze' | null
recoveryProbeRequired: boolean
nextRefreshAt: string | null
now: number
}
@@ -146,19 +149,32 @@ export function resolveAccountStatusPresentation(
&& nextRefreshTimestamp !== null
&& nextRefreshTimestamp > input.now
const mode: AccountStatusDisplayMode = isBackoff ? 'refresh_backoff' : input.status
const definition = displayDefinitions[mode]
const isFreeze = mode === 'rate_limited' && input.rateLimitReason === 'capacity_freeze'
const waitsForProbe = isFreeze && input.recoveryProbeRequired
const definition = isFreeze
? {
...displayDefinitions.rate_limited,
description: '容量类请求失败累计达到阈值,系统已暂停该账号的调度。',
recoveryHint: waitsForProbe
? '冷却结束后进行恢复探测,成功后恢复调度;也可手动恢复账号。'
: '冷却结束后自动恢复调度,也可手动恢复账号。',
}
: displayDefinitions[mode]
const nextRefreshDisplay = isBackoff ? formatDateTime(nextRefreshTimestamp) : null
const reasonLabel = input.errorReason ? errorReasonLabels[input.errorReason] : null
const title = definition.title ?? reasonLabel ?? definition.label
const rateLimitRecovery = input.status === 'rate_limited'
? remainingTime(input.rateLimitedUntil, input.now)
const until = input.rateLimitedUntil ? parseTimestamp(input.rateLimitedUntil) : null
const rateLimitRecovery = mode === 'rate_limited'
? waitsForProbe && until !== null && until <= input.now
? '等待探测成功'
: remainingTime(input.rateLimitedUntil, input.now)
: null
const recoveryHint = mode !== 'refresh_backoff'
&& mode !== 'rate_limited'
&& input.errorReason
? errorRecoveryHints[input.errorReason]
: definition.recoveryHint
const hasDetail = input.status === 'error' || rateLimitRecovery !== null || isBackoff
const hasDetail = input.status === 'error' || mode === 'rate_limited' || isBackoff
const retry = nextRefreshDisplay ? ` 下次尝试:${nextRefreshDisplay}。` : ''
const triggerLabel = `${title}。${definition.description}${retry} 点击或聚焦查看详情。`
@@ -174,6 +190,7 @@ export function resolveAccountStatusPresentation(
errorText: input.errorMessage || null,
nextRefreshDisplay,
rateLimitRecovery,
recoveryTimeLabel: waitsForProbe ? '恢复探测' : '预计恢复',
triggerLabel,
}
}
+2
View File
@@ -291,6 +291,8 @@ const {
:error-reason="row.errorReason"
:error-message="row.errorMessage"
:rate-limited-until="row.quota.rateLimitedUntil"
:rate-limit-reason="row.quota.rateLimitReason"
:recovery-probe-required="row.quota.recoveryProbeRequired"
:next-refresh-at="row.nextRefreshAt"
/>
</template>
@@ -0,0 +1,135 @@
<script setup lang="ts">
import { Activity, Gauge, Snowflake, Timer } from '@lucide/vue'
import BaseCard from '@/components/base/BaseCard.vue'
import BaseCheckbox from '@/components/base/BaseCheckbox.vue'
import BaseFormItem from '@/components/base/BaseForm/FormItem.vue'
import BaseForm from '@/components/base/BaseForm/index.vue'
import BaseInput from '@/components/base/BaseInput.vue'
import BaseSwitch from '@/components/base/BaseSwitch.vue'
const enabled = defineModel<boolean>('enabled', { required: true })
const threshold = defineModel<string>('threshold', { required: true })
const windowSeconds = defineModel<string>('windowSeconds', { required: true })
const durationSeconds = defineModel<string>('durationSeconds', { required: true })
const probeEnabled = defineModel<boolean>('probeEnabled', { required: true })
const probeModel = defineModel<string>('probeModel', { required: true })
const adaptiveConcurrency = defineModel<boolean>('adaptiveConcurrency', { required: true })
</script>
<template>
<BaseCard
title="账号自动冻结"
description="容量类错误高频出现时暂停账号调度,在账号管理中显示为限流中"
>
<BaseForm class="max-w-6xl sm:grid-cols-2">
<BaseSwitch
v-model="enabled"
class="col-span-full justify-self-start"
label="启用自动冻结"
show-label
/>
<BaseFormItem
label="触发阈值"
description="窗口内累计失败尝试达到此次数后触发"
>
<BaseInput
v-model="threshold"
:disabled="!enabled"
aria-label="触发阈值"
type="number"
min="2"
max="1000"
step="1"
>
<template #prefix>
<Activity class="size-4" />
</template>
<template #suffix>
<span class="text-cp-sm">次</span>
</template>
</BaseInput>
</BaseFormItem>
<BaseFormItem
label="统计窗口"
description="每次失败后,计数有效期向后顺延"
>
<BaseInput
v-model="windowSeconds"
:disabled="!enabled"
aria-label="统计窗口秒数"
type="number"
min="60"
max="3600"
step="1"
>
<template #prefix>
<Timer class="size-4" />
</template>
<template #suffix>
<span class="text-cp-sm">秒</span>
</template>
</BaseInput>
</BaseFormItem>
<BaseFormItem
label="冷却时长"
description="探测失败时,按此时长延后恢复"
>
<BaseInput
v-model="durationSeconds"
:disabled="!enabled"
aria-label="冷却时长秒数"
type="number"
min="300"
max="604800"
step="1"
>
<template #prefix>
<Snowflake class="size-4" />
</template>
<template #suffix>
<span class="text-cp-sm">秒</span>
</template>
</BaseInput>
</BaseFormItem>
<BaseFormItem
label="恢复探测模型"
description="启用后需探测成功才恢复,模型留空时选择首个可用模型"
>
<template #extra>
<BaseCheckbox
v-model="probeEnabled"
:disabled="!enabled"
label="恢复前探测"
show-label
/>
</template>
<BaseInput
v-model="probeModel"
:disabled="!enabled || !probeEnabled"
aria-label="恢复探测模型"
placeholder="留空自动选择"
>
<template #prefix>
<Gauge class="size-4" />
</template>
</BaseInput>
</BaseFormItem>
<div class="col-span-full flex flex-wrap items-center gap-x-3 gap-y-2">
<BaseCheckbox
v-model="adaptiveConcurrency"
:disabled="!enabled"
label="自适应并发下调"
show-label
/>
<p class="m-0 text-cp-sm leading-5 text-cp-text-secondary">
修改账号并发上限,恢复后不自动调高
</p>
</div>
</BaseForm>
</BaseCard>
</template>
@@ -30,9 +30,17 @@ export function useSettingsForm() {
usageRetentionDays: 31,
opsEventRetentionDays: 30,
auditRetentionDays: 90,
accountAutoFreezeEnabled: false,
accountAutoFreezeThreshold: null as number | null,
accountAutoFreezeWindowSeconds: null as number | null,
accountAutoFreezeDurationSeconds: null as number | null,
accountAutoFreezeProbeEnabled: true,
accountAutoFreezeProbeModel: '',
accountAutoFreezeAdaptiveConcurrency: true,
})
function numericModel(key: 'refreshMarginSeconds' | 'refreshConcurrency' | 'maxConcurrentPerAccount' | 'requestIntervalMs' | 'maxWaitingPerKey' | 'maxWaitingPerAccount' | 'concurrencyWaitTimeoutSeconds') {
function numericModel(key: 'refreshMarginSeconds' | 'refreshConcurrency' | 'maxConcurrentPerAccount' | 'requestIntervalMs' | 'maxWaitingPerKey' | 'maxWaitingPerAccount' | 'concurrencyWaitTimeoutSeconds' | 'accountAutoFreezeThreshold' | 'accountAutoFreezeWindowSeconds' | 'accountAutoFreezeDurationSeconds') {
return computed({
get: () => (form[key] === null ? '' : String(form[key])),
set: (value: string) => {
@@ -53,6 +61,9 @@ export function useSettingsForm() {
const maxWaitingPerKeyValue = numericModel('maxWaitingPerKey')
const maxWaitingPerAccountValue = numericModel('maxWaitingPerAccount')
const concurrencyWaitTimeoutSecondsValue = numericModel('concurrencyWaitTimeoutSeconds')
const accountAutoFreezeThresholdValue = numericModel('accountAutoFreezeThreshold')
const accountAutoFreezeWindowSecondsValue = numericModel('accountAutoFreezeWindowSeconds')
const accountAutoFreezeDurationSecondsValue = numericModel('accountAutoFreezeDurationSeconds')
const minCodexDesktopVersionError = computed(() => versionError(form.minCodexDesktopVersion))
const minCodexCliVersionError = computed(() => versionError(form.minCodexCliVersion))
@@ -77,6 +88,13 @@ export function useSettingsForm() {
form.usageRetentionDays = data.usageRetentionDays
form.opsEventRetentionDays = data.opsEventRetentionDays
form.auditRetentionDays = data.auditRetentionDays
form.accountAutoFreezeEnabled = data.accountAutoFreezeEnabled
form.accountAutoFreezeThreshold = data.accountAutoFreezeThreshold
form.accountAutoFreezeWindowSeconds = data.accountAutoFreezeWindowSeconds
form.accountAutoFreezeDurationSeconds = data.accountAutoFreezeDurationSeconds
form.accountAutoFreezeProbeEnabled = data.accountAutoFreezeProbeEnabled
form.accountAutoFreezeProbeModel = data.accountAutoFreezeProbeModel ?? ''
form.accountAutoFreezeAdaptiveConcurrency = data.accountAutoFreezeAdaptiveConcurrency
mappings.value = Object.entries(data.modelMappings || {}).map(([requestedModel, upstreamModel]) => ({
requestedModel,
upstreamModel: String(upstreamModel),
@@ -132,7 +150,7 @@ export function useSettingsForm() {
async function saveSettings() {
if (saving.value || loading.value)
return
const { refreshMarginSeconds, refreshConcurrency, maxConcurrentPerAccount, requestIntervalMs, rotationStrategy, maxWaitingPerKey, maxWaitingPerAccount, concurrencyWaitTimeoutSeconds } = form
const { refreshMarginSeconds, refreshConcurrency, maxConcurrentPerAccount, requestIntervalMs, rotationStrategy, maxWaitingPerKey, maxWaitingPerAccount, concurrencyWaitTimeoutSeconds, accountAutoFreezeThreshold, accountAutoFreezeWindowSeconds, accountAutoFreezeDurationSeconds } = form
if (refreshMarginSeconds === null || refreshConcurrency === null || maxConcurrentPerAccount === null || requestIntervalMs === null || !rotationStrategy || maxWaitingPerKey === null || maxWaitingPerAccount === null || concurrencyWaitTimeoutSeconds === null) {
toast.warning('请完整填写运行参数和调度策略')
return
@@ -146,6 +164,21 @@ export function useSettingsForm() {
toast.warning('请修正客户端最低版本格式')
return
}
if (accountAutoFreezeThreshold === null || accountAutoFreezeWindowSeconds === null || accountAutoFreezeDurationSeconds === null) {
toast.warning('请完整填写账号自动冻结参数')
return
}
if (!Number.isInteger(accountAutoFreezeThreshold) || accountAutoFreezeThreshold < 2 || accountAutoFreezeThreshold > 1000
|| !Number.isInteger(accountAutoFreezeWindowSeconds) || accountAutoFreezeWindowSeconds < 60 || accountAutoFreezeWindowSeconds > 3600
|| !Number.isInteger(accountAutoFreezeDurationSeconds) || accountAutoFreezeDurationSeconds < 300 || accountAutoFreezeDurationSeconds > 604800) {
toast.warning('自动冻结阈值应为 2~1000,统计窗口为 60~3600 秒,冻结时长为 300~604800 秒')
return
}
const probeModel = form.accountAutoFreezeProbeModel.trim()
if (probeModel && (probeModel.length > 128 || probeModel !== probeModel.trim())) {
toast.warning('探测模型名称不能超过 128 个字符')
return
}
await saveAction.run(async () => {
const result = await updateSettings({
modelMappings: mappingPayload(),
@@ -162,6 +195,13 @@ export function useSettingsForm() {
usageRetentionDays: form.usageRetentionDays,
opsEventRetentionDays: form.opsEventRetentionDays,
auditRetentionDays: form.auditRetentionDays,
accountAutoFreezeEnabled: form.accountAutoFreezeEnabled,
accountAutoFreezeThreshold,
accountAutoFreezeWindowSeconds,
accountAutoFreezeDurationSeconds,
accountAutoFreezeProbeEnabled: form.accountAutoFreezeProbeEnabled,
accountAutoFreezeProbeModel: probeModel || null,
accountAutoFreezeAdaptiveConcurrency: form.accountAutoFreezeAdaptiveConcurrency,
})
applySettings(result)
toast.success('设置已保存')
@@ -189,6 +229,9 @@ export function useSettingsForm() {
maxWaitingPerKeyValue,
maxWaitingPerAccountValue,
concurrencyWaitTimeoutSecondsValue,
accountAutoFreezeThresholdValue,
accountAutoFreezeWindowSecondsValue,
accountAutoFreezeDurationSecondsValue,
minCodexDesktopVersionError,
minCodexCliVersionError,
saveSettings,
+14
View File
@@ -8,6 +8,7 @@ import BaseConfirmModal from '@/components/base/BaseConfirmModal.vue'
import BasePageHeader from '@/components/base/BasePageHeader.vue'
import BaseSegmented from '@/components/base/BaseSegmented.vue'
import AccountAutoFreezeCard from './components/AccountAutoFreezeCard.vue'
import AdminApiKeyCard from './components/AdminApiKeyCard.vue'
import SettingsBackupSection from './components/backup/SettingsBackupSection.vue'
import ClientVersionSettings from './components/client-version/index.vue'
@@ -52,6 +53,9 @@ const {
maxWaitingPerKeyValue,
maxWaitingPerAccountValue,
concurrencyWaitTimeoutSecondsValue,
accountAutoFreezeThresholdValue,
accountAutoFreezeWindowSecondsValue,
accountAutoFreezeDurationSecondsValue,
minCodexDesktopVersionError,
minCodexCliVersionError,
@@ -136,6 +140,16 @@ watch(
v-model:concurrency-wait-timeout-seconds="concurrencyWaitTimeoutSecondsValue"
/>
<AccountAutoFreezeCard
v-model:enabled="form.accountAutoFreezeEnabled"
v-model:threshold="accountAutoFreezeThresholdValue"
v-model:window-seconds="accountAutoFreezeWindowSecondsValue"
v-model:duration-seconds="accountAutoFreezeDurationSecondsValue"
v-model:probe-enabled="form.accountAutoFreezeProbeEnabled"
v-model:probe-model="form.accountAutoFreezeProbeModel"
v-model:adaptive-concurrency="form.accountAutoFreezeAdaptiveConcurrency"
/>
<ClientVersionSettings
v-model:min-codex-desktop-version="form.minCodexDesktopVersion"
v-model:min-codex-cli-version="form.minCodexCliVersion"