diff --git a/README.md b/README.md index ec186f1de..040c1cafc 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,9 @@ API Key 持有者可在同一登录页切换登录身份,进入 `/key-usage` 2. 按需建立账号分组,再创建客户端密钥并选择可用分组。**不选分组表示可使用全部账号**。 3. 打开密钥的「使用密钥」,复制客户端配置。 +账号自动冻结默认关闭,可在「系统设置 → 运行参数」中启用。启用后,账号频繁命中上游容量错误时 +会暂停调度并显示为限流中;若开启恢复前探测,冷却到期后需探测成功才恢复。 + ## 客户端接入 **Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入。 diff --git a/backend/crates/gateway-admin/src/freeze_recovery.rs b/backend/crates/gateway-admin/src/freeze_recovery.rs new file mode 100644 index 000000000..b3488b99d --- /dev/null +++ b/backend/crates/gateway-admin/src/freeze_recovery.rs @@ -0,0 +1,304 @@ +//! 容量熔断冻结的恢复编排:到期探测、失败顺延与自适应并发下调。 +//! +//! 冻结事实保存在可丢失的 Redis 冷却中,本服务只组合 Admin 端口: +//! 观测(`AccountRuntimeStore`)、探测(`AccountsService`)与写回 +//! (原子降低并发 / 按冻结代次结束探测)。旧观测不会覆盖管理员操作。 + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use chrono::{DateTime, Utc}; +use futures::StreamExt as _; +use gateway_core::account::{AccountConcurrencyLimit, ProviderAccountId}; +use gateway_core::routing::UpstreamModelId; +use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError}; +use tracing::warn; + +use crate::model::accounts::{ + AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot, +}; +use crate::model::{MutationActor, MutationContext}; +use crate::ports::store::{AccountRuntimeStore, AccountStore, SettingsStore}; +use crate::use_case::accounts::AccountsService; + +/// 自适应并发下调保留系数:下调到观测在途峰值的 80%。 +const ADAPTIVE_CONCURRENCY_FACTOR: f64 = 0.8; +/// 自适应并发下限:过低的并发让账号几乎不可用,宁可保持冻结。 +const ADAPTIVE_CONCURRENCY_FLOOR: u32 = 2; +/// worker 周期;冷却观测与探测都依赖该节奏。 +pub const FREEZE_RECOVERY_INTERVAL: Duration = Duration::from_secs(30); +pub const FREEZE_RECOVERY_WORKER_OWNER: &str = "account-freeze-recovery"; +pub const WORKER_INITIAL_BACKOFF: Duration = Duration::from_secs(1); +pub const WORKER_MAXIMUM_BACKOFF: Duration = Duration::from_secs(60); +pub const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60); +pub const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60); +/// 系统审计上下文的固定请求 ID;账号并发更新审计由此溯源到自动冻结编排。 +const SYSTEM_REQUEST_ID: &str = "account-freeze-recovery"; + +/// 冻结恢复所需的管理端口组合。 +#[derive(Clone)] +pub struct FreezeRecoveryDeps { + pub accounts: Arc, + pub store: Arc, + pub runtime: Arc, + pub settings: Arc, +} + +pub struct FreezeRecoveryTask { + deps: FreezeRecoveryDeps, +} + +impl FreezeRecoveryTask { + pub fn new(deps: FreezeRecoveryDeps) -> Self { + Self { deps } + } + + fn system_context() -> MutationContext { + MutationContext { + actor: MutationActor::System, + request_id: SYSTEM_REQUEST_ID.to_owned(), + } + } + + /// 读取冻结策略;读取或校验失败跳过本轮,不能把未知配置当作允许解冻。 + async fn freeze_policy(&self) -> Option { + let settings = self.deps.settings.load_runtime_settings().await.ok()?; + gateway_core::provider_ports::ProviderFreezePolicy::try_new( + settings.account_auto_freeze_enabled, + settings.account_auto_freeze_threshold, + settings.account_auto_freeze_window_seconds, + settings.account_auto_freeze_duration_seconds, + settings.account_auto_freeze_probe_enabled, + settings.account_auto_freeze_probe_model, + settings.account_auto_freeze_adaptive_concurrency, + ) + .ok() + } + + async fn active_freezes(&self) -> BTreeMap { + self.deps.runtime.active_freezes().await.unwrap_or_default() + } + + async fn run_cycle_inner(&self) { + let Some(policy) = self.freeze_policy().await else { + return; + }; + let freezes = self.active_freezes().await; + if freezes.is_empty() { + return; + } + if policy.enabled() && policy.adaptive_concurrency() { + self.adapt_concurrency_limits(&freezes, &policy).await; + } + self.recover_due_freezes(&freezes, &policy).await; + } + + /// 对冻结中的账号执行自适应并发下调:目标为观测在途峰值的 80%(下限 2), + /// 只降不升;未观测到峰值证据的账号保持现状。 + async fn adapt_concurrency_limits( + &self, + freezes: &BTreeMap, + policy: &gateway_core::provider_ports::ProviderFreezePolicy, + ) { + let account_ids = freezes.keys().cloned().collect::>(); + let Ok(peaks) = self.deps.runtime.capacity_peaks(&account_ids).await else { + return; + }; + for (account_id, peak) in peaks { + let Some(target) = adaptive_target(peak) else { + continue; + }; + let Ok(account) = ProviderAccountId::new(account_id.clone()) else { + continue; + }; + let Some(limit) = AccountConcurrencyLimit::new(target) else { + continue; + }; + match self + .deps + .accounts + .lower_concurrency_limit(&Self::system_context(), account, limit) + .await + { + Ok(Some(_)) => { + tracing::info!( + account_id, + adapted_limit = target, + freeze_seconds = policy.freeze_duration().as_secs(), + "容量熔断:按观测在途峰值下调账号并发上限", + ); + } + Ok(None) => {} + Err(error) => { + warn!(account_id, error = %error, "自适应并发下调失败"); + } + } + } + } + + async fn load_account(&self, account_id: &str) -> Option> { + let account_id = ProviderAccountId::new(account_id.to_owned()).ok()?; + self.deps + .store + .load_account(account_id.as_str(), AccountRuntimeSnapshot::default()) + .await + .ok() + } + + /// 到达探测时间后才恢复;关闭自动冻结或探测时,已有冻结仍等待原冷却结束。 + async fn recover_due_freezes( + &self, + freezes: &BTreeMap, + policy: &gateway_core::provider_ports::ProviderFreezePolicy, + ) { + for (account_id, freeze) in freezes { + if freeze.until > Utc::now() { + continue; + } + if policy.enabled() && policy.probe_enabled() && freeze.requires_probe { + self.probe_and_recover(account_id, freeze, policy).await; + } else if let Err(error) = self + .deps + .runtime + .finish_freeze(account_id, freeze, None) + .await + { + warn!(account_id, error = %error, "解除到期冻结失败"); + } + } + } + + async fn probe_and_recover( + &self, + account_id: &str, + freeze: &AccountFreeze, + policy: &gateway_core::provider_ports::ProviderFreezePolicy, + ) { + let Some(Some(item)) = self.load_account(account_id).await else { + return; + }; + if !item.account.enabled { + // 停用账号没有自动恢复意义;解冻交给管理员手动恢复。 + return; + } + let Ok(account) = ProviderAccountId::new(account_id.to_owned()) else { + return; + }; + let Some(model) = self.resolve_probe_model(&account, policy).await else { + // 没有可用探测模型时按失败处理,顺延冻结等待下一轮。 + self.postpone(&account, freeze, policy).await; + return; + }; + let probe_succeeded = match self + .deps + .accounts + .test_connection(account.clone(), model) + .await + { + Ok(events) => drain_probe(events).await, + Err(error) => { + tracing::debug!(account_id, error = %error, "冻结恢复探测发起失败"); + false + } + }; + if probe_succeeded { + match self + .deps + .runtime + .finish_freeze(account_id, freeze, None) + .await + { + Ok(true) => tracing::info!(account_id, "冻结恢复探测成功:账号已解冻"), + Ok(false) => {} + Err(error) => warn!(account_id, error = %error, "解除冻结失败"), + } + } else { + self.postpone(&account, freeze, policy).await; + } + } + + /// 探测失败只顺延本次仍存在的冻结,旧探测不能重新冻结已手动恢复的账号。 + async fn postpone( + &self, + account: &ProviderAccountId, + freeze: &AccountFreeze, + policy: &gateway_core::provider_ports::ProviderFreezePolicy, + ) { + let Some(until) = SystemTime::now().checked_add(policy.freeze_duration()) else { + return; + }; + let until = DateTime::::from(until); + match self + .deps + .runtime + .finish_freeze(account.as_str(), freeze, Some(until)) + .await + { + Ok(true) => { + tracing::info!( + account_id = account.as_str(), + postpone_seconds = policy.freeze_duration().as_secs(), + "冻结恢复探测失败:账号冻结顺延", + ); + } + Ok(false) => {} + Err(error) => { + warn!( + account_id = account.as_str(), + error = %error, + "冻结顺延失败" + ); + } + } + } + + async fn resolve_probe_model( + &self, + account: &ProviderAccountId, + policy: &gateway_core::provider_ports::ProviderFreezePolicy, + ) -> Option { + if let Some(model) = policy.probe_model() { + return UpstreamModelId::new(model.to_owned()).ok(); + } + let models = self.deps.accounts.models(account, false).await.ok()?; + models.models.into_iter().next().map(|model| model.id) + } +} + +impl ScheduledTask for FreezeRecoveryTask { + fn run_cycle( + &self, + context: WorkerCycleContext, + ) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> { + Box::pin(async move { + if context.cancellation().is_cancelled() { + return Ok(()); + } + self.run_cycle_inner().await; + Ok(()) + }) + } +} + +/// 自适应目标并发:观测峰值 × 0.8 向下取整,下限 2;无证据(峰值为 0)不调整。 +fn adaptive_target(peak_in_flight: u32) -> Option { + if peak_in_flight == 0 { + return None; + } + let scaled = (f64::from(peak_in_flight) * ADAPTIVE_CONCURRENCY_FACTOR).floor() as u32; + Some(scaled.max(ADAPTIVE_CONCURRENCY_FLOOR)) +} + +/// 消耗连接测试事件流并返回是否以 `Completed` 终止。 +async fn drain_probe(mut events: crate::model::accounts::AccountConnectionTestEventStream) -> bool { + let mut completed = false; + while let Some(event) = events.next().await { + match event { + AccountConnectionTestEvent::Completed => completed = true, + AccountConnectionTestEvent::Failed { .. } => return false, + _ => {} + } + } + completed +} diff --git a/backend/crates/gateway-admin/src/lib.rs b/backend/crates/gateway-admin/src/lib.rs index e78d3052b..30a18f77d 100644 --- a/backend/crates/gateway-admin/src/lib.rs +++ b/backend/crates/gateway-admin/src/lib.rs @@ -10,14 +10,15 @@ use gateway_core::{ routing::ProviderKind, runtime::SnapshotControl, task::{ - DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration, - WorkerRunnable, + DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest, + WorkerRegistration, WorkerRunnable, WorkerSchedule, }, }; use secrecy::{ExposeSecret as _, SecretString}; use serde::Deserialize; pub mod backup; +pub mod freeze_recovery; pub mod model; pub mod ports; mod use_case; @@ -210,6 +211,12 @@ impl AdminServices { self.key_usage.as_ref() } + /// 取得账号服务的共享句柄;后台编排(冻结恢复 worker)需要持有 Arc。 + #[must_use] + pub fn accounts_handle(&self) -> Arc { + Arc::clone(&self.accounts) + } + #[must_use] pub fn proxies(&self) -> &dyn ProxiesService { self.proxies.as_ref() @@ -392,7 +399,7 @@ pub async fn initialize( registry.clone(), )), auth, - accounts, + accounts: accounts.clone(), account_groups: Arc::new(DefaultAccountGroupService::new( store.account_groups(), store.account_runtime(), @@ -419,6 +426,13 @@ pub async fn initialize( import_tasks, backups, }; + let freeze_recovery = + freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps { + accounts: Arc::clone(&accounts) as Arc, + store: store.accounts(), + runtime: store.account_runtime(), + settings: store.settings(), + }); let mut worker_contributions = backup_worker_contribution(backup_task)?; let id = WorkerId::try_new(WorkerKind::AccountImport, "admin") .map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?; @@ -433,6 +447,7 @@ pub async fn initialize( ) .map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?; worker_contributions.push(WorkerContribution::Registration(registration)); + worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?); Ok(AdminBundle { services, worker_contributions, @@ -458,6 +473,37 @@ fn backup_worker_contribution( Ok(vec![WorkerContribution::Registration(registration)]) } +/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定。 +fn freeze_recovery_worker_contribution( + task: freeze_recovery::FreezeRecoveryTask, +) -> Result, AdminError> { + let id = WorkerId::try_new( + WorkerKind::AccountFreezeRecovery, + freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER, + ) + .map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?; + let schedule = WorkerSchedule::try_new( + freeze_recovery::FREEZE_RECOVERY_INTERVAL, + freeze_recovery::WORKER_INITIAL_BACKOFF, + freeze_recovery::WORKER_MAXIMUM_BACKOFF, + freeze_recovery::WORKER_LEASE_TTL, + freeze_recovery::WORKER_LEASE_RENEWAL, + ) + .map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?; + let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL) + .map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?; + let registration = WorkerRegistration::try_new( + id, + WorkerRunnable::Scheduled { + schedule, + lease: Some(lease), + task: Box::new(task), + }, + ) + .map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?; + Ok(vec![WorkerContribution::Registration(registration)]) +} + fn provider_kind(value: &'static str) -> Result { ProviderKind::new(value).map_err(|_| AdminError::internal("内置 Provider 类型不合法")) } diff --git a/backend/crates/gateway-admin/src/model/accounts.rs b/backend/crates/gateway-admin/src/model/accounts.rs index 751a05612..29abee64d 100644 --- a/backend/crates/gateway-admin/src/model/accounts.rs +++ b/backend/crates/gateway-admin/src/model/accounts.rs @@ -69,11 +69,20 @@ pub struct AccountListQuery { /// Admin query service 从运行态存储取得的当前账号冷却快照。 #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct AccountRuntimeSnapshot { - pub rate_limited_until: BTreeMap>, + pub cooldown: BTreeMap, /// `None` 表示实时 lease 存储不可用;`Some` 中未出现的账号当前使用量为零。 pub in_flight: Option>, } +/// 恢复任务读取的冻结快照;generation 将异步结果绑定到本次冻结。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountFreeze { + pub credential_revision: Revision, + pub until: DateTime, + pub generation: String, + pub requires_probe: bool, +} + /// Optional account membership filter. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccountGroupFilter { diff --git a/backend/crates/gateway-admin/src/model/settings.rs b/backend/crates/gateway-admin/src/model/settings.rs index da628b80e..031224dcc 100644 --- a/backend/crates/gateway-admin/src/model/settings.rs +++ b/backend/crates/gateway-admin/src/model/settings.rs @@ -32,6 +32,13 @@ pub struct RuntimeSettings { pub usage_retention_days: u32, pub ops_event_retention_days: u32, pub audit_retention_days: u32, + pub account_auto_freeze_enabled: bool, + pub account_auto_freeze_threshold: u32, + pub account_auto_freeze_window_seconds: u64, + pub account_auto_freeze_duration_seconds: u64, + pub account_auto_freeze_probe_enabled: bool, + pub account_auto_freeze_probe_model: Option, + pub account_auto_freeze_adaptive_concurrency: bool, pub updated_at: DateTime, } @@ -52,6 +59,13 @@ pub struct ReplaceRuntimeSettings { pub usage_retention_days: u32, pub ops_event_retention_days: u32, pub audit_retention_days: u32, + pub account_auto_freeze_enabled: bool, + pub account_auto_freeze_threshold: u32, + pub account_auto_freeze_window_seconds: u64, + pub account_auto_freeze_duration_seconds: u64, + pub account_auto_freeze_probe_enabled: bool, + pub account_auto_freeze_probe_model: Option, + pub account_auto_freeze_adaptive_concurrency: bool, } /// 明文管理员 API Key;按产品约束明文落库,但禁止 Debug 泄漏。 diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index 394978dec..4c1147ec7 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -2,7 +2,7 @@ //! //! 端口按业务资源拆分,方法使用领域模型,不暴露连接池、事务或 Redis client。 -use std::{net::IpAddr, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration}; use async_trait::async_trait; use chrono::{DateTime, Utc}; @@ -159,6 +159,14 @@ pub trait AccountStore: Send + Sync { context: &MutationContext, ) -> AdminStoreResult; + /// 在事务内按最新启用状态、账号上限和全局默认值判断,只降低并发上限。 + async fn lower_concurrency_limit( + &self, + account_id: &gateway_core::account::ProviderAccountId, + limit: gateway_core::account::AccountConcurrencyLimit, + context: &MutationContext, + ) -> AdminStoreResult>; + async fn recover_account( &self, account_id: &gateway_core::account::ProviderAccountId, @@ -193,6 +201,25 @@ pub trait AccountRuntimeStore: Send + Sync { &self, account_ids: &[String], ) -> AdminStoreResult; + + /// 容量熔断自动冻结中的账号与其冻结截止时间;429 临时限流不包含在内。 + async fn active_freezes( + &self, + ) -> AdminStoreResult>; + + /// 读取容量失败窗口内观测到的在途并发峰值(自适应并发下调的证据)。 + async fn capacity_peaks( + &self, + account_ids: &[String], + ) -> AdminStoreResult>; + + /// 仅当冻结快照仍匹配时解除或顺延;旧探测不得覆盖手动恢复或新一轮冻结。 + async fn finish_freeze( + &self, + account_id: &str, + expected: &crate::model::accounts::AccountFreeze, + postpone_until: Option>, + ) -> AdminStoreResult; } /// 控制面凭据、统一会话、登录限流与管理员安全审计。 diff --git a/backend/crates/gateway-admin/src/use_case/account_groups.rs b/backend/crates/gateway-admin/src/use_case/account_groups.rs index f6321877a..a278e0640 100644 --- a/backend/crates/gateway-admin/src/use_case/account_groups.rs +++ b/backend/crates/gateway-admin/src/use_case/account_groups.rs @@ -192,11 +192,7 @@ fn project_group_runtime( let mut accounts_by_group = BTreeMap::<&str, Vec<&str>>::new(); for member in members { let mut status = member.status.clone(); - status.rate_limited_until = runtime - .rate_limited_until - .get(&member.account_id) - .copied() - .map(Into::into); + status.cooldown = runtime.cooldown.get(&member.account_id).copied(); status_by_account .entry(member.account_id.as_str()) .or_insert_with(|| resolve_account_status(&status, now).status); diff --git a/backend/crates/gateway-admin/src/use_case/accounts.rs b/backend/crates/gateway-admin/src/use_case/accounts.rs index 785fc3ebd..cf7d9bfc6 100644 --- a/backend/crates/gateway-admin/src/use_case/accounts.rs +++ b/backend/crates/gateway-admin/src/use_case/accounts.rs @@ -73,6 +73,13 @@ pub trait AccountsService: Send + Sync { command: UpdateAccount, ) -> Result; + async fn lower_concurrency_limit( + &self, + context: &MutationContext, + account_id: ProviderAccountId, + limit: gateway_core::account::AccountConcurrencyLimit, + ) -> Result, AdminError>; + async fn batch_update( &self, context: &MutationContext, @@ -582,6 +589,27 @@ impl AccountsService for DefaultAccountsService { Ok(result) } + async fn lower_concurrency_limit( + &self, + context: &MutationContext, + account_id: ProviderAccountId, + limit: gateway_core::account::AccountConcurrencyLimit, + ) -> Result, AdminError> { + let (_, provider) = self.provider_for_account(&account_id).await?; + let result = self + .accounts + .lower_concurrency_limit(&account_id, limit, context) + .await + .map_err(|error| map_store_error(error, "provider account"))?; + if let Some(result) = &result { + provider + .account_facts_changed(std::slice::from_ref(&account_id)) + .await; + publish_committed(self.snapshot.as_ref(), result.config_revision).await?; + } + Ok(result) + } + async fn batch_update( &self, context: &MutationContext, diff --git a/backend/crates/gateway-admin/src/use_case/settings.rs b/backend/crates/gateway-admin/src/use_case/settings.rs index 330dc1038..60cfb4280 100644 --- a/backend/crates/gateway-admin/src/use_case/settings.rs +++ b/backend/crates/gateway-admin/src/use_case/settings.rs @@ -125,7 +125,11 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError> && command.audit_retention_days > 0 && valid_client_version(command.min_codex_desktop_version.as_deref()) && valid_client_version(command.min_codex_cli_version.as_deref()) - && i64::try_from(command.request_interval_ms).is_ok(); + && valid_probe_model(command.account_auto_freeze_probe_model.as_deref()) + && i64::try_from(command.request_interval_ms).is_ok() + && (2..=1_000).contains(&command.account_auto_freeze_threshold) + && (60..=3_600).contains(&command.account_auto_freeze_window_seconds) + && (300..=604_800).contains(&command.account_auto_freeze_duration_seconds); if valid { Ok(()) } else { @@ -136,3 +140,12 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError> fn valid_client_version(value: Option<&str>) -> bool { value.is_none_or(|value| CodexClientVersion::parse(value).is_ok()) } + +fn valid_probe_model(value: Option<&str>) -> bool { + value.is_none_or(|value| { + !value.is_empty() + && value.len() <= 128 + && value == value.trim() + && !value.bytes().any(|byte| byte.is_ascii_control()) + }) +} diff --git a/backend/crates/gateway-admin/tests/use_case/account_groups.rs b/backend/crates/gateway-admin/tests/use_case/account_groups.rs index 1bcecb1a2..8ba996ec8 100644 --- a/backend/crates/gateway-admin/tests/use_case/account_groups.rs +++ b/backend/crates/gateway-admin/tests/use_case/account_groups.rs @@ -5,7 +5,7 @@ use std::{ }; use async_trait::async_trait; -use chrono::{Duration, Utc}; +use chrono::{DateTime, Duration, Utc}; use gateway_admin::{ model::{ MutationContext, PageSize, Revision, @@ -168,13 +168,35 @@ impl AccountRuntimeStore for FakeRuntimeStore { ) -> AdminStoreResult { *self.requested_accounts.lock().expect("requested accounts") = account_ids.to_vec(); Ok(AccountRuntimeSnapshot { - rate_limited_until: BTreeMap::from([( + cooldown: BTreeMap::from([( "acct_limited".to_owned(), - Utc::now() + Duration::minutes(5), + std::time::SystemTime::from(Utc::now() + Duration::minutes(5)).into(), )]), in_flight: Some(BTreeMap::from([("acct_available".to_owned(), 2)])), }) } + + async fn active_freezes( + &self, + ) -> AdminStoreResult> { + Ok(BTreeMap::new()) + } + + async fn capacity_peaks( + &self, + _account_ids: &[String], + ) -> AdminStoreResult> { + Ok(BTreeMap::new()) + } + + async fn finish_freeze( + &self, + _account_id: &str, + _expected: &gateway_admin::model::accounts::AccountFreeze, + _postpone_until: Option>, + ) -> AdminStoreResult { + Ok(false) + } } fn group_record() -> AccountGroupRecord { @@ -215,7 +237,7 @@ fn member(account_id: &str, total_slots: u64) -> AccountGroupMemberFact { credential_state: CredentialState::Ready, access_token_expires_at: None, quota: QuotaState::default(), - rate_limited_until: None, + cooldown: None, last_error_reason: None, last_error_message: None, }, diff --git a/backend/crates/gateway-admin/tests/use_case/accounts.rs b/backend/crates/gateway-admin/tests/use_case/accounts.rs index f36c15d10..74b9fa897 100644 --- a/backend/crates/gateway-admin/tests/use_case/accounts.rs +++ b/backend/crates/gateway-admin/tests/use_case/accounts.rs @@ -544,6 +544,8 @@ pub(super) struct FakeAccountStore { quota_window_usage: Mutex>, quota_window_queries: Mutex>, quota_forecast_history: Mutex, + update_commands: Mutex>, + pub(super) lowered_limits: Mutex>, } impl FakeAccountStore { @@ -551,7 +553,7 @@ impl FakeAccountStore { Self::with_account(account_record(kind), events) } - fn with_account(account: AccountRecord, events: EventLog) -> Arc { + pub(super) fn with_account(account: AccountRecord, events: EventLog) -> Arc { Arc::new(Self { events, accounts: Mutex::new(vec![account]), @@ -562,9 +564,18 @@ impl FakeAccountStore { quota_window_usage: Mutex::new(Vec::new()), quota_window_queries: Mutex::new(Vec::new()), quota_forecast_history: Mutex::new(QuotaForecastHistory::default()), + update_commands: Mutex::new(Vec::new()), + lowered_limits: Mutex::new(Vec::new()), }) } + pub(super) fn update_commands(&self) -> Vec { + self.update_commands + .lock() + .expect("update commands") + .clone() + } + pub(super) fn fail_next_commit(&self) { *self.fail_commit.lock().expect("store failure") = true; } @@ -630,7 +641,7 @@ impl FakeAccountStore { credential_state: account.credential_state, access_token_expires_at: account.access_token_expires_at.map(Into::into), quota: account.quota, - rate_limited_until: None, + cooldown: None, last_error_reason: account.last_error_reason, last_error_message: account.last_error_message.clone(), }; @@ -852,12 +863,45 @@ impl AccountStore for FakeAccountStore { self.record("store.update_account"); self.record_context(context); self.require_commit()?; + self.update_commands + .lock() + .expect("update commands") + .push(command.clone()); Ok(AccountUpdateResult { config_revision: revision(2), account_id: ProviderAccountId::new(command.account_id).expect("account ID"), }) } + async fn lower_concurrency_limit( + &self, + account_id: &ProviderAccountId, + limit: gateway_core::account::AccountConcurrencyLimit, + _: &MutationContext, + ) -> AdminStoreResult> { + let mut accounts = self.accounts.lock().expect("accounts"); + let Some(account) = accounts + .iter_mut() + .find(|account| account.id == account_id.as_str()) + else { + return Ok(None); + }; + if !account.enabled + || account.concurrency_limit.map_or(5, |value| value.get()) <= limit.get() + { + return Ok(None); + } + account.concurrency_limit = Some(limit); + self.lowered_limits + .lock() + .expect("lowered limits") + .push((account.id.clone(), limit.get())); + Ok(Some(AccountUpdateResult { + config_revision: revision(2), + account_id: account_id.clone(), + })) + } + async fn recover_account( &self, account_id: &ProviderAccountId, @@ -944,6 +988,13 @@ impl SettingsStore for StaticSettingsStore { usage_retention_days: 30, ops_event_retention_days: 30, audit_retention_days: 30, + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, updated_at: Utc::now(), }) } diff --git a/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs b/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs new file mode 100644 index 000000000..9dd2ce6a8 --- /dev/null +++ b/backend/crates/gateway-admin/tests/use_case/freeze_recovery.rs @@ -0,0 +1,425 @@ +//! 容量熔断冻结恢复编排回归:到期探测、失败顺延与自适应并发下调。 + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use chrono::{DateTime, Duration as TimeDelta, Utc}; +use gateway_admin::freeze_recovery::{FreezeRecoveryDeps, FreezeRecoveryTask}; +use gateway_admin::model::MutationContext; +use gateway_admin::model::accounts::{AccountFreeze, AccountRuntimeSnapshot}; +use gateway_admin::model::settings::{ + AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RuntimeSettings, +}; +use gateway_admin::ports::store::{ + AccountRuntimeStore, AccountStore, AdminStoreError, AdminStoreResult, SettingsStore, +}; +use gateway_core::engine::probe::{ + AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult, +}; +use gateway_core::lifecycle::CancellationToken; +use gateway_core::task::{ScheduledTask as _, WorkerCycleContext, WorkerId, WorkerKind}; + +use super::AdminHarness; +use super::accounts::{FakeAccountStore, FakeProviderAdmin, account_record, events, revision}; + +fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> RuntimeSettings { + RuntimeSettings { + config_revision: revision(1), + model_mappings: Default::default(), + refresh_margin_seconds: 300, + refresh_concurrency: 2, + max_concurrent_per_account: 5, + request_interval_ms: 0, + max_waiting_per_key: 0, + max_waiting_per_account: 0, + concurrency_wait_timeout_seconds: 30, + rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart, + min_codex_desktop_version: None, + min_codex_cli_version: None, + usage_retention_days: 31, + ops_event_retention_days: 30, + audit_retention_days: 90, + account_auto_freeze_enabled: enabled, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: probe_enabled, + account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()), + account_auto_freeze_adaptive_concurrency: adaptive, + updated_at: Utc::now(), + } +} + +struct FreezeSettingsStore { + settings: RuntimeSettings, +} + +#[async_trait] +impl SettingsStore for FreezeSettingsStore { + async fn load_runtime_settings(&self) -> AdminStoreResult { + Ok(self.settings.clone()) + } + + async fn admin_api_key_exists(&self) -> AdminStoreResult { + Ok(false) + } + + async fn replace_runtime_settings( + &self, + _: ReplaceRuntimeSettings, + _: &MutationContext, + ) -> AdminStoreResult { + Err(store_unavailable()) + } + + async fn replace_admin_api_key( + &self, + _: AdminApiKey, + _: &MutationContext, + ) -> AdminStoreResult { + Err(store_unavailable()) + } + + async fn delete_admin_api_key( + &self, + _: &MutationContext, + ) -> AdminStoreResult { + Err(store_unavailable()) + } +} + +fn store_unavailable() -> AdminStoreError { + AdminStoreError::new( + gateway_admin::ports::store::AdminStoreErrorKind::Unavailable, + "freeze recovery test", + "unused", + ) +} + +/// 记录解冻/顺延调用的运行态 fake;冻结与峰值证据由测试预置。 +struct FreezeRuntimeStore { + freezes: BTreeMap, + peaks: BTreeMap, + cleared: Mutex>, + extended: Mutex)>>, +} + +impl FreezeRuntimeStore { + fn new(freezes: BTreeMap, peaks: BTreeMap) -> Arc { + Arc::new(Self { + freezes, + peaks, + cleared: Mutex::new(Vec::new()), + extended: Mutex::new(Vec::new()), + }) + } + + fn cleared(&self) -> Vec { + self.cleared.lock().expect("cleared lock").clone() + } + + fn extended(&self) -> Vec<(String, DateTime)> { + self.extended.lock().expect("extended lock").clone() + } +} + +#[async_trait] +impl AccountRuntimeStore for FreezeRuntimeStore { + async fn active_rate_limits(&self) -> AdminStoreResult { + Ok(AccountRuntimeSnapshot::default()) + } + + async fn account_runtime( + &self, + _account_ids: &[String], + ) -> AdminStoreResult { + Ok(AccountRuntimeSnapshot::default()) + } + + async fn active_freezes(&self) -> AdminStoreResult> { + Ok(self.freezes.clone()) + } + + async fn capacity_peaks( + &self, + account_ids: &[String], + ) -> AdminStoreResult> { + Ok(account_ids + .iter() + .filter_map(|id| self.peaks.get(id).map(|peak| (id.clone(), *peak))) + .collect()) + } + + async fn finish_freeze( + &self, + account_id: &str, + expected: &AccountFreeze, + postpone_until: Option>, + ) -> AdminStoreResult { + assert_eq!(self.freezes.get(account_id), Some(expected)); + if let Some(until) = postpone_until { + self.extended + .lock() + .expect("extended") + .push((account_id.to_owned(), until)); + } else { + self.cleared + .lock() + .expect("cleared") + .push(account_id.to_owned()); + } + Ok(true) + } +} + +struct SuccessfulProbe; + +impl AccountProbe for SuccessfulProbe { + fn probe( + &self, + _: AccountProbeRequest, + ) -> futures::future::BoxFuture<'_, Result> { + Box::pin(async { + Ok(AccountProbeResult { + text: vec!["OK".to_owned()], + }) + }) + } +} + +struct FailingProbe; + +impl AccountProbe for FailingProbe { + fn probe( + &self, + _: AccountProbeRequest, + ) -> futures::future::BoxFuture<'_, Result> { + Box::pin(async { + Err(AccountProbeError::new( + gateway_core::error::GatewayError::new( + gateway_core::error::GatewayErrorKind::RateLimited, + "still overloaded", + ), + gateway_core::engine::probe::AccountProbeErrorSource::Upstream, + Some(gateway_core::upstream::UpstreamSendState::NotSent), + None, + )) + }) + } +} + +async fn recovery_task( + settings: RuntimeSettings, + runtime: Arc, + probe: Arc, +) -> (FreezeRecoveryTask, Arc) { + recovery_task_with_store(settings, runtime, probe).await +} + +async fn recovery_task_with_store( + settings: RuntimeSettings, + runtime: Arc, + probe: Arc, +) -> (FreezeRecoveryTask, Arc) { + let store = FakeAccountStore::new("openai", events()); + let services = AdminHarness::new() + .accounts(Arc::clone(&store) as Arc) + .account_runtime(Arc::clone(&runtime) as Arc) + .settings(Arc::new(FreezeSettingsStore { + settings: settings.clone(), + })) + .provider(FakeProviderAdmin::new("openai", events())) + .probe(probe) + .build() + .await; + let task = FreezeRecoveryTask::new(FreezeRecoveryDeps { + accounts: services.accounts_handle(), + store: store.clone(), + runtime, + settings: Arc::new(FreezeSettingsStore { + settings: settings.clone(), + }), + }); + (task, store) +} + +async fn run_cycle(task: &FreezeRecoveryTask) { + let worker = WorkerId::try_new(WorkerKind::AccountFreezeRecovery, "test").expect("worker id"); + let context = WorkerCycleContext::new(worker, None, CancellationToken::new()); + task.run_cycle(context) + .await + .expect("freeze recovery cycle"); +} + +fn freeze_until(until: DateTime) -> BTreeMap { + BTreeMap::from([( + "acct_test".to_owned(), + AccountFreeze { + credential_revision: revision(1), + until, + generation: "freeze-generation".to_owned(), + requires_probe: true, + }, + )]) +} + +fn freeze_due() -> BTreeMap { + freeze_until(Utc::now() - TimeDelta::seconds(1)) +} + +#[tokio::test] +async fn probe_success_clears_freeze() { + let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new()); + let (task, _store) = recovery_task( + runtime_settings(true, true, false), + Arc::clone(&runtime), + Arc::new(SuccessfulProbe), + ) + .await; + + run_cycle(&task).await; + + assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]); + assert!(runtime.extended().is_empty()); +} + +#[tokio::test] +async fn probe_failure_postpones_freeze() { + let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new()); + let (task, _store) = recovery_task( + runtime_settings(true, true, false), + Arc::clone(&runtime), + Arc::new(FailingProbe), + ) + .await; + + run_cycle(&task).await; + + assert!(runtime.cleared().is_empty()); + let extended = runtime.extended(); + assert_eq!(extended.len(), 1); + let (account_id, until) = &extended[0]; + assert_eq!(account_id, "acct_test"); + let remaining = until.timestamp_millis() - Utc::now().timestamp_millis(); + assert!( + remaining > 7_000_000 && remaining <= 7_200_000, + "postponed freeze should be about 2 hours, got {remaining}ms" + ); +} + +#[tokio::test] +async fn disabled_policy_releases_due_freeze_without_probing() { + let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new()); + let (task, _store) = recovery_task( + runtime_settings(false, true, false), + Arc::clone(&runtime), + Arc::new(SuccessfulProbe), + ) + .await; + + run_cycle(&task).await; + + assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]); + assert!(runtime.extended().is_empty()); +} + +#[tokio::test] +async fn adaptive_concurrency_lowers_limit_to_observed_peak() { + let runtime = FreezeRuntimeStore::new( + freeze_until(Utc::now() + TimeDelta::hours(1)), + BTreeMap::from([("acct_test".to_owned(), 4_u32)]), + ); + let (task, store) = recovery_task( + runtime_settings(true, false, true), + Arc::clone(&runtime), + Arc::new(SuccessfulProbe), + ) + .await; + + run_cycle(&task).await; + + // 峰值 4 × 0.8 = 3(不低于下限 2),低于全局默认 5,应下调到 3。 + assert_eq!( + *store.lowered_limits.lock().expect("lowered limits"), + vec![("acct_test".to_owned(), 3)] + ); + assert!(store.update_commands().is_empty()); + assert!(runtime.extended().is_empty()); + assert!(runtime.cleared().is_empty()); +} + +#[tokio::test] +async fn adaptive_concurrency_never_raises_limit() { + let runtime = FreezeRuntimeStore::new( + freeze_until(Utc::now() + TimeDelta::hours(1)), + BTreeMap::from([("acct_test".to_owned(), 40_u32)]), + ); + let (task, store) = recovery_task( + runtime_settings(true, false, true), + Arc::clone(&runtime), + Arc::new(SuccessfulProbe), + ) + .await; + + run_cycle(&task).await; + + // 峰值 40 的目标 32 高于全局默认 5;只降不升,不下发任何更新。 + assert!(store.update_commands().is_empty()); +} + +/// 到期时间还早(超过探测提前量)的冻结不发起探测。 +#[tokio::test] +async fn probe_skips_freezes_far_from_expiry() { + let runtime = FreezeRuntimeStore::new( + freeze_until(Utc::now() + TimeDelta::hours(1)), + BTreeMap::new(), + ); + let (task, _store) = recovery_task( + runtime_settings(true, true, false), + Arc::clone(&runtime), + Arc::new(SuccessfulProbe), + ) + .await; + + run_cycle(&task).await; + + assert!(runtime.cleared().is_empty()); + assert!(runtime.extended().is_empty()); +} + +/// 停用账号不参与探测恢复;并发下调同样跳过。 +#[tokio::test] +async fn disabled_accounts_are_skipped() { + let runtime = FreezeRuntimeStore::new( + freeze_due(), + BTreeMap::from([("acct_test".to_owned(), 4_u32)]), + ); + let mut record = account_record("openai"); + record.enabled = false; + let store = FakeAccountStore::with_account(record, events()); + let services = AdminHarness::new() + .accounts(Arc::clone(&store) as Arc) + .account_runtime(Arc::clone(&runtime) as Arc) + .settings(Arc::new(FreezeSettingsStore { + settings: runtime_settings(true, true, true), + })) + .provider(FakeProviderAdmin::new("openai", events())) + .probe(Arc::new(SuccessfulProbe)) + .build() + .await; + let task = FreezeRecoveryTask::new(FreezeRecoveryDeps { + accounts: services.accounts_handle(), + store: store.clone(), + runtime: Arc::clone(&runtime) as Arc, + settings: Arc::new(FreezeSettingsStore { + settings: runtime_settings(true, true, true), + }), + }); + + run_cycle(&task).await; + + assert!(runtime.cleared().is_empty()); + assert!(runtime.extended().is_empty()); + assert!(store.update_commands().is_empty()); +} diff --git a/backend/crates/gateway-admin/tests/use_case/mod.rs b/backend/crates/gateway-admin/tests/use_case/mod.rs index 4c726f9db..f23e98188 100644 --- a/backend/crates/gateway-admin/tests/use_case/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/mod.rs @@ -4,6 +4,7 @@ mod auth; mod auth_key; mod backup; mod client_keys; +mod freeze_recovery; mod import_tasks; mod observability; mod openai; @@ -82,6 +83,7 @@ use gateway_core::{ routing::{ConfigRevision, ProviderKind}, runtime::SnapshotControl, }; +use std::collections::BTreeMap; pub(super) struct AdminHarness { default_password: String, @@ -488,6 +490,15 @@ impl AccountStore for UnavailableStore { Err(unavailable("account enabled")) } + async fn lower_concurrency_limit( + &self, + _: &gateway_core::account::ProviderAccountId, + _: gateway_core::account::AccountConcurrencyLimit, + _: &MutationContext, + ) -> AdminStoreResult> { + Ok(None) + } + async fn recover_account( &self, _: &ProviderAccountId, @@ -530,6 +541,28 @@ impl AccountRuntimeStore for UnavailableStore { async fn account_runtime(&self, _: &[String]) -> AdminStoreResult { Ok(AccountRuntimeSnapshot::default()) } + + async fn active_freezes( + &self, + ) -> AdminStoreResult> { + Ok(BTreeMap::new()) + } + + async fn capacity_peaks( + &self, + _account_ids: &[String], + ) -> AdminStoreResult> { + Ok(BTreeMap::new()) + } + + async fn finish_freeze( + &self, + _account_id: &str, + _expected: &gateway_admin::model::accounts::AccountFreeze, + _postpone_until: Option>, + ) -> AdminStoreResult { + Ok(false) + } } #[async_trait] diff --git a/backend/crates/gateway-admin/tests/use_case/observability.rs b/backend/crates/gateway-admin/tests/use_case/observability.rs index f460d7c28..87dd2bd6f 100644 --- a/backend/crates/gateway-admin/tests/use_case/observability.rs +++ b/backend/crates/gateway-admin/tests/use_case/observability.rs @@ -769,6 +769,13 @@ impl SettingsStore for FixtureSettingsStore { ops_event_retention_days: 30, audit_retention_days: 30, updated_at: Utc::now(), + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, }) } diff --git a/backend/crates/gateway-admin/tests/use_case/settings.rs b/backend/crates/gateway-admin/tests/use_case/settings.rs index 31a1b0b69..7f67c2abb 100644 --- a/backend/crates/gateway-admin/tests/use_case/settings.rs +++ b/backend/crates/gateway-admin/tests/use_case/settings.rs @@ -75,6 +75,13 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() { usage_retention_days: 31, ops_event_retention_days: 30, audit_retention_days: 30, + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, }, ) .await diff --git a/backend/crates/gateway-api/src/admin/accounts/presenter.rs b/backend/crates/gateway-api/src/admin/accounts/presenter.rs index 9bc6bb252..2f2980f1b 100644 --- a/backend/crates/gateway-api/src/admin/accounts/presenter.rs +++ b/backend/crates/gateway-api/src/admin/accounts/presenter.rs @@ -66,7 +66,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime) -> Ac quota, } = item; let status = projection.status.as_str().to_owned(); - let rate_limited_until = projection.rate_limited_until.map(DateTime::::from); + let cooldown = projection.cooldown; let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339); let added_at = china_rfc3339(&account.created_at); let updated_at = china_rfc3339(&account.updated_at); @@ -75,7 +75,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime) -> Ac if account.authentication_kind == "api_key" { usage.window_label_display = "通用额度".to_owned(); } - let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now); + let (quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now); AccountView { id: account.id.clone(), name: account.name, @@ -189,7 +189,7 @@ fn forecast_usd_display(value: Option) -> String { pub(super) fn account_quota_view( mut quota: ProviderQuota, - rate_limited_until: Option>, + cooldown: Option, now: DateTime, ) -> (AccountQuotaView, Option) { quota.apply_limit_reached_display(); @@ -200,12 +200,22 @@ pub(super) fn account_quota_view( .observed_at .map_or_else(|| "—".to_owned(), |value| relative_time(value, now)); let windows = quota.windows.into_iter().map(quota_window_view).collect(); - let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until)); + let rate_limited_until = cooldown.map(|value| china_datetime(&value.until.into())); + let rate_limit_reason = cooldown.map(|value| { + if value.kind.is_capacity_freeze() { + "capacity_freeze".to_owned() + } else { + "upstream_rate_limit".to_owned() + } + }); + let recovery_probe_required = cooldown.is_some_and(|value| value.kind.requires_probe()); ( AccountQuotaView { refreshed_at_display, limit_reached: quota.limit_reached, rate_limited_until, + rate_limit_reason, + recovery_probe_required, windows, }, refresh_token_expires_at, diff --git a/backend/crates/gateway-api/src/admin/accounts/wire.rs b/backend/crates/gateway-api/src/admin/accounts/wire.rs index e90ca705b..cee3b5eab 100644 --- a/backend/crates/gateway-api/src/admin/accounts/wire.rs +++ b/backend/crates/gateway-api/src/admin/accounts/wire.rs @@ -361,8 +361,10 @@ pub struct AccountGroupRefView { pub struct AccountQuotaView { pub refreshed_at_display: String, pub limit_reached: bool, - /// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。 + /// 冷却到期或可开始恢复探测的时间;非限流中为 `null`。 pub rate_limited_until: Option, + pub rate_limit_reason: Option, + pub recovery_probe_required: bool, pub windows: Vec, } diff --git a/backend/crates/gateway-api/src/admin/settings.rs b/backend/crates/gateway-api/src/admin/settings.rs index 34046b692..7547a8774 100644 --- a/backend/crates/gateway-api/src/admin/settings.rs +++ b/backend/crates/gateway-api/src/admin/settings.rs @@ -48,6 +48,13 @@ pub struct RuntimeSettingsView { pub usage_retention_days: u64, pub ops_event_retention_days: u64, pub audit_retention_days: u64, + pub account_auto_freeze_enabled: bool, + pub account_auto_freeze_threshold: u64, + pub account_auto_freeze_window_seconds: u64, + pub account_auto_freeze_duration_seconds: u64, + pub account_auto_freeze_probe_enabled: bool, + pub account_auto_freeze_probe_model: Option, + pub account_auto_freeze_adaptive_concurrency: bool, pub updated_at: DateTime, } @@ -69,6 +76,13 @@ pub struct UpdateRuntimeSettingsRequest { pub usage_retention_days: u64, pub ops_event_retention_days: u64, pub audit_retention_days: u64, + pub account_auto_freeze_enabled: bool, + pub account_auto_freeze_threshold: u64, + pub account_auto_freeze_window_seconds: u64, + pub account_auto_freeze_duration_seconds: u64, + pub account_auto_freeze_probe_enabled: bool, + pub account_auto_freeze_probe_model: Option, + pub account_auto_freeze_adaptive_concurrency: bool, } impl UpdateRuntimeSettingsRequest { @@ -110,6 +124,35 @@ impl UpdateRuntimeSettingsRequest { self.min_codex_cli_version.as_deref(), "minCodexCliVersion", )?; + for (value, field) in [ + ( + self.account_auto_freeze_threshold, + "accountAutoFreezeThreshold", + ), + ( + self.account_auto_freeze_window_seconds, + "accountAutoFreezeWindowSeconds", + ), + ( + self.account_auto_freeze_duration_seconds, + "accountAutoFreezeDurationSeconds", + ), + ] { + require_positive_i64(value, field)?; + } + if !(2..=1_000).contains(&self.account_auto_freeze_threshold) { + return Err(WireValidationError::new("accountAutoFreezeThreshold")); + } + if !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) { + return Err(WireValidationError::new("accountAutoFreezeWindowSeconds")); + } + if !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) { + return Err(WireValidationError::new("accountAutoFreezeDurationSeconds")); + } + validate_optional_probe_model( + self.account_auto_freeze_probe_model.as_deref(), + "accountAutoFreezeProbeModel", + )?; Ok(()) } @@ -136,6 +179,14 @@ impl UpdateRuntimeSettingsRequest { .map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?, audit_retention_days: u32::try_from(self.audit_retention_days) .map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?, + account_auto_freeze_enabled: self.account_auto_freeze_enabled, + account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold) + .map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?, + account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds, + account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds, + account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled, + account_auto_freeze_probe_model: self.account_auto_freeze_probe_model, + account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency, }) } } @@ -157,6 +208,14 @@ impl From for RuntimeSettingsView { usage_retention_days: u64::from(settings.usage_retention_days), ops_event_retention_days: u64::from(settings.ops_event_retention_days), audit_retention_days: u64::from(settings.audit_retention_days), + account_auto_freeze_enabled: settings.account_auto_freeze_enabled, + account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold), + account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds, + account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds, + account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled, + account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model, + account_auto_freeze_adaptive_concurrency: settings + .account_auto_freeze_adaptive_concurrency, updated_at: settings.updated_at, } } @@ -452,6 +511,22 @@ fn validate_optional_client_version( Ok(()) } +/// 探测模型为可选自由文本:非空、去首尾空白后不变、无控制字符且不超过 128 字节。 +fn validate_optional_probe_model( + value: Option<&str>, + field: &'static str, +) -> Result<(), WireValidationError> { + if value.is_some_and(|value| { + value.is_empty() + || value.len() > 128 + || value != value.trim() + || value.bytes().any(|byte| byte.is_ascii_control()) + }) { + return Err(WireValidationError::new(field)); + } + Ok(()) +} + fn map_wire_error(error: WireValidationError) -> AdminError { let message = match error.field() { "settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(), @@ -459,6 +534,11 @@ fn map_wire_error(error: WireValidationError) -> AdminError { "settingsUsageRetentionOverflow" => "usageRetentionDays 不合法".to_owned(), "settingsOpsRetentionOverflow" => "opsEventRetentionDays 不合法".to_owned(), "settingsAuditRetentionOverflow" => "auditRetentionDays 不合法".to_owned(), + "settingsFreezeThresholdOverflow" => "accountAutoFreezeThreshold 不合法".to_owned(), + "accountAutoFreezeThreshold" => "账号自动冻结阈值应为 2~1000 的整数".to_owned(), + "accountAutoFreezeWindowSeconds" => "账号自动冻结统计窗口应为 60~3600 秒".to_owned(), + "accountAutoFreezeDurationSeconds" => "账号自动冻结时长应为 300~604800 秒".to_owned(), + "accountAutoFreezeProbeModel" => "探测模型格式不合法".to_owned(), "minCodexDesktopVersion" => "Codex Desktop 最低版本格式不合法".to_owned(), "minCodexCliVersion" => "Codex CLI 最低版本格式不合法".to_owned(), field => format!("{field} 字段不合法"), diff --git a/backend/crates/gateway-api/tests/admin/errors.rs b/backend/crates/gateway-api/tests/admin/errors.rs index b84c653e3..b4ecc9932 100644 --- a/backend/crates/gateway-api/tests/admin/errors.rs +++ b/backend/crates/gateway-api/tests/admin/errors.rs @@ -463,7 +463,7 @@ mod provider { credential_state: CredentialState::Ready, access_token_expires_at: None, quota: QuotaState::unknown(), - rate_limited_until: None, + cooldown: None, last_error_reason: None, last_error_message: None, }; diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index cd8079eb6..0f2e9b0cf 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -417,6 +417,13 @@ impl SettingsStore for MemorySettingsStore { usage_retention_days: command.usage_retention_days, ops_event_retention_days: command.ops_event_retention_days, audit_retention_days: command.audit_retention_days, + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, updated_at: Utc::now(), }; *settings = updated.clone(); @@ -587,7 +594,7 @@ impl AccountGroupStore for MemoryAccountGroupStore { credential_state: CredentialState::Ready, access_token_expires_at: None, quota: QuotaState::default(), - rate_limited_until: None, + cooldown: None, last_error_reason: None, last_error_message: None, }, @@ -601,7 +608,7 @@ impl AccountGroupStore for MemoryAccountGroupStore { credential_state: CredentialState::Ready, access_token_expires_at: None, quota: QuotaState::default(), - rate_limited_until: None, + cooldown: None, last_error_reason: None, last_error_message: None, }, @@ -896,6 +903,15 @@ impl AccountStore for UnusedStore { Err(unavailable("account enabled")) } + async fn lower_concurrency_limit( + &self, + _: &gateway_core::account::ProviderAccountId, + _: gateway_core::account::AccountConcurrencyLimit, + _: &MutationContext, + ) -> AdminStoreResult> { + Ok(None) + } + async fn recover_account( &self, _: &ProviderAccountId, @@ -933,17 +949,39 @@ impl AccountStore for UnusedStore { impl AccountRuntimeStore for UnusedStore { async fn active_rate_limits(&self) -> AdminStoreResult { Ok(AccountRuntimeSnapshot { - rate_limited_until: BTreeMap::new(), + cooldown: BTreeMap::new(), in_flight: Some(BTreeMap::new()), }) } async fn account_runtime(&self, _: &[String]) -> AdminStoreResult { Ok(AccountRuntimeSnapshot { - rate_limited_until: BTreeMap::new(), + cooldown: BTreeMap::new(), in_flight: Some(BTreeMap::new()), }) } + + async fn active_freezes( + &self, + ) -> AdminStoreResult> { + Ok(BTreeMap::new()) + } + + async fn capacity_peaks( + &self, + _account_ids: &[String], + ) -> AdminStoreResult> { + Ok(BTreeMap::new()) + } + + async fn finish_freeze( + &self, + _account_id: &str, + _expected: &gateway_admin::model::accounts::AccountFreeze, + _postpone_until: Option>, + ) -> AdminStoreResult { + Ok(false) + } } #[async_trait] @@ -1245,6 +1283,13 @@ fn test_runtime_settings() -> RuntimeSettings { usage_retention_days: 31, ops_event_retention_days: 30, audit_retention_days: 90, + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, updated_at: Utc::now(), } } diff --git a/backend/crates/gateway-api/tests/admin/settings.rs b/backend/crates/gateway-api/tests/admin/settings.rs index 222d38a14..139342e75 100644 --- a/backend/crates/gateway-api/tests/admin/settings.rs +++ b/backend/crates/gateway-api/tests/admin/settings.rs @@ -53,7 +53,14 @@ fn update_body() -> Value { "minCodexCliVersion": "0.40.0", "usageRetentionDays": 32, "opsEventRetentionDays": 31, - "auditRetentionDays": 91 + "auditRetentionDays": 91, + "accountAutoFreezeEnabled": true, + "accountAutoFreezeThreshold": 12, + "accountAutoFreezeWindowSeconds": 600, + "accountAutoFreezeDurationSeconds": 7200, + "accountAutoFreezeProbeEnabled": true, + "accountAutoFreezeProbeModel": null, + "accountAutoFreezeAdaptiveConcurrency": true }) } @@ -116,6 +123,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() { usage_retention_days: 32, ops_event_retention_days: 31, audit_retention_days: 91, + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, updated_at: Utc .with_ymd_and_hms(2026, 8, 2, 10, 30, 0) .single() @@ -143,6 +157,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() { "usageRetentionDays": 32, "opsEventRetentionDays": 31, "auditRetentionDays": 91, + "accountAutoFreezeEnabled": true, + "accountAutoFreezeThreshold": 12, + "accountAutoFreezeWindowSeconds": 600, + "accountAutoFreezeDurationSeconds": 7200, + "accountAutoFreezeProbeEnabled": true, + "accountAutoFreezeProbeModel": null, + "accountAutoFreezeAdaptiveConcurrency": true, "updatedAt": "2026-08-02T10:30:00Z" }) ); @@ -195,6 +216,13 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() { usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"), ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"), audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"), + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, updated_at: chrono::Utc::now(), }; diff --git a/backend/crates/gateway-core/src/account/model.rs b/backend/crates/gateway-core/src/account/model.rs index dcedeceb3..aa4c60d96 100644 --- a/backend/crates/gateway-core/src/account/model.rs +++ b/backend/crates/gateway-core/src/account/model.rs @@ -583,6 +583,72 @@ impl AccountErrorReason { } } +/// 账号冷却的原因与恢复方式。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AccountCooldownKind { + /// 上游 429 临时限流;默认类别,兼容升级前未标记的存量 key。 + #[default] + RateLimit, + /// 容量类错误高频触发后由熔断策略写入的自动冻结。 + CapacityFreeze, + /// 到期后仍阻止调度,必须由恢复探测成功或管理员恢复解除。 + CapacityFreezeProbe, +} + +impl AccountCooldownKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::RateLimit => "rate_limit", + Self::CapacityFreeze => "capacity_freeze", + Self::CapacityFreezeProbe => "capacity_freeze_probe", + } + } + + #[must_use] + pub const fn is_capacity_freeze(self) -> bool { + matches!(self, Self::CapacityFreeze | Self::CapacityFreezeProbe) + } + + #[must_use] + pub const fn requires_probe(self) -> bool { + matches!(self, Self::CapacityFreezeProbe) + } + + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "rate_limit" => Some(Self::RateLimit), + "capacity_freeze" => Some(Self::CapacityFreeze), + "capacity_freeze_probe" => Some(Self::CapacityFreezeProbe), + _ => None, + } + } +} + +/// 账号级冷却投影;探测冻结的 until 是下次探测时间,不是自动放行时间。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AccountCooldown { + pub until: SystemTime, + pub kind: AccountCooldownKind, +} + +impl AccountCooldown { + #[must_use] + pub fn is_active(self, now: SystemTime) -> bool { + self.kind.requires_probe() || self.until > now + } +} + +impl From for AccountCooldown { + fn from(until: SystemTime) -> Self { + Self { + until, + kind: AccountCooldownKind::RateLimit, + } + } +} + /// 唯一状态解析器的完整输入事实。 #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStatusFacts { @@ -590,7 +656,7 @@ pub struct AccountStatusFacts { pub credential_state: CredentialState, pub access_token_expires_at: Option, pub quota: QuotaState, - pub rate_limited_until: Option, + pub cooldown: Option, pub last_error_reason: Option, pub last_error_message: Option, } @@ -601,8 +667,8 @@ pub struct AccountStatusProjection { pub status: AccountStatus, pub error_reason: Option, pub error_message: Option, - /// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。 - pub rate_limited_until: Option, + /// 仅 `rate_limited` 状态携带仍有效的运行时冷却事实。 + pub cooldown: Option, } /// 从独立事实派生唯一、互斥的对外状态。 @@ -625,21 +691,21 @@ pub fn resolve_account_status( status: AccountStatus::Error, error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)), error_message: facts.last_error_message.clone(), - rate_limited_until: None, + cooldown: None, }; } if facts.quota.is_exhausted() { return status_projection(AccountStatus::QuotaExhausted); } if facts - .rate_limited_until - .is_some_and(|rate_limited_until| rate_limited_until > now) + .cooldown + .is_some_and(|cooldown| cooldown.is_active(now)) { return AccountStatusProjection { status: AccountStatus::RateLimited, error_reason: None, error_message: None, - rate_limited_until: facts.rate_limited_until, + cooldown: facts.cooldown, }; } status_projection(AccountStatus::Normal) @@ -650,7 +716,7 @@ const fn status_projection(status: AccountStatus) -> AccountStatusProjection { status, error_reason: None, error_message: None, - rate_limited_until: None, + cooldown: None, } } @@ -905,7 +971,7 @@ impl ProviderAccount { pub fn status_projection( &self, now: SystemTime, - rate_limited_until: Option, + cooldown: Option, ) -> AccountStatusProjection { resolve_account_status( &AccountStatusFacts { @@ -913,7 +979,7 @@ impl ProviderAccount { credential_state: self.credential_state, access_token_expires_at: self.access_token_expires_at, quota: self.quota, - rate_limited_until, + cooldown, last_error_reason: self.last_error_reason, last_error_message: self.last_error_message.clone(), }, diff --git a/backend/crates/gateway-core/src/account/selection.rs b/backend/crates/gateway-core/src/account/selection.rs index 12c926801..77a13a09e 100644 --- a/backend/crates/gateway-core/src/account/selection.rs +++ b/backend/crates/gateway-core/src/account/selection.rs @@ -106,7 +106,7 @@ pub struct AccountRuntimeSignals { pub quota_reset_at: Option, /// Provider 归一化的剩余额度基点:0 耗尽,10_000 全部可用。 pub quota_remaining_rank: Option, - pub rate_limited_until: Option, + pub cooldown: Option, pub failure_rate_basis_points: Option, pub first_output_latency_ms: Option, } @@ -316,8 +316,8 @@ impl AccountRuntimeSignals { } #[must_use] - pub const fn with_rate_limit(mut self, rate_limited_until: Option) -> Self { - self.rate_limited_until = rate_limited_until; + pub const fn with_rate_limit(mut self, cooldown: Option) -> Self { + self.cooldown = cooldown; self } @@ -654,7 +654,7 @@ impl AccountSelector { if !context.eligibility.bypasses_local_eligibility() { let status = candidate .account - .status_projection(context.now, candidate.signals.rate_limited_until) + .status_projection(context.now, candidate.signals.cooldown) .status; if status != AccountStatus::Normal { return Some(AccountSchedulingBlocker::LocalAvailability); diff --git a/backend/crates/gateway-core/src/provider_ports/mod.rs b/backend/crates/gateway-core/src/provider_ports/mod.rs index ed621a7f7..56e6c1482 100644 --- a/backend/crates/gateway-core/src/provider_ports/mod.rs +++ b/backend/crates/gateway-core/src/provider_ports/mod.rs @@ -181,6 +181,16 @@ pub trait ProviderLeasePort: Send + Sync { &self, request: ProviderLeaseRequest, ) -> BoxFuture<'_, Result>; + + /// 读取指定账号当前的在途请求数;只用于容量熔断的峰值证据, + /// 支持租约信号的存储实现覆盖,否则视为不可观测(空映射)。 + fn account_in_flight<'a>( + &'a self, + account_ids: &'a [ProviderAccountId], + ) -> BoxFuture<'a, Result, ProviderStoreError>> { + let _ = account_ids; + Box::pin(async move { Ok(BTreeMap::new()) }) + } } /// Provider 从原始会话锚点派生的不可逆亲和键。 @@ -596,12 +606,16 @@ pub trait ProviderCredentialStatePort: Send + Sync { ) -> BoxFuture<'a, Result<(), ProviderStoreError>>; } -/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。 +/// 账号级 cooldown 的来源类别;调度状态统一按 `rate_limited` 处理。 +pub use crate::account::AccountCooldownKind as ProviderCooldownKind; + +/// 可丢失的账号冷却事实,不进入持久状态;探测冻结到期后仍需确认恢复。 #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProviderCooldown { account_id: ProviderAccountId, credential_revision: CredentialRevision, until: SystemTime, + kind: ProviderCooldownKind, } impl ProviderCooldown { @@ -610,11 +624,27 @@ impl ProviderCooldown { account_id: ProviderAccountId, credential_revision: CredentialRevision, until: SystemTime, + ) -> Self { + Self::new_with_kind( + account_id, + credential_revision, + until, + ProviderCooldownKind::RateLimit, + ) + } + + #[must_use] + pub const fn new_with_kind( + account_id: ProviderAccountId, + credential_revision: CredentialRevision, + until: SystemTime, + kind: ProviderCooldownKind, ) -> Self { Self { account_id, credential_revision, until, + kind, } } @@ -632,6 +662,19 @@ impl ProviderCooldown { pub const fn until(&self) -> SystemTime { self.until } + + #[must_use] + pub const fn scheduling_state(&self) -> crate::account::AccountCooldown { + crate::account::AccountCooldown { + until: self.until, + kind: self.kind, + } + } + + #[must_use] + pub const fn kind(&self) -> ProviderCooldownKind { + self.kind + } } /// 可丢失 cooldown 的细粒度作用域。 @@ -749,6 +792,29 @@ pub trait ProviderCooldownPort: Send + Sync { &'a self, account_id: &'a ProviderAccountId, ) -> BoxFuture<'a, Result>; + + /// 记录一次容量类失败并返回滑动窗口内的累计次数,同时把观测到的账号 + /// 在途并发并入窗口峰值(`in_flight` 为 0 表示本次未观测,跳过峰值更新)。 + /// 只服务容量熔断触发器;调用频率受失败频率约束,不需要批量接口。 + fn record_capacity_failure<'a>( + &'a self, + account_id: &'a ProviderAccountId, + window: Duration, + in_flight: u32, + ) -> BoxFuture<'a, Result>; + + /// 普通请求成功后原子清除临时限流及失败证据;必须保留任何容量冻结。 + fn clear_after_success<'a>( + &'a self, + account_id: &'a ProviderAccountId, + through_revision: CredentialRevision, + ) -> BoxFuture<'a, Result<(), ProviderStoreError>>; + + /// 读取窗口内观测到的在途并发峰值;无证据时返回 `None`。 + fn capacity_peak_in_flight<'a>( + &'a self, + account_id: &'a ProviderAccountId, + ) -> BoxFuture<'a, Result, ProviderStoreError>>; } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -864,6 +930,118 @@ pub trait ProviderRuntimePolicyPort: Send + Sync { fn load_refresh_policy( &self, ) -> BoxFuture<'_, Result>; + + /// 读取账号容量熔断策略;默认关闭,只有实现运行时设置的存储需要覆盖。 + fn load_freeze_policy( + &self, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { Ok(ProviderFreezePolicy::disabled()) }) + } +} + +/// 账号容量熔断(自动冻结)策略;来源于 `runtime_settings`, +/// 由 Provider 触发路径与恢复 worker 共享同一份配置事实。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderFreezePolicy { + enabled: bool, + threshold: u32, + window: Duration, + freeze_duration: Duration, + probe_enabled: bool, + probe_model: Option, + adaptive_concurrency: bool, +} + +impl ProviderFreezePolicy { + /// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致; + /// store 层写入前已校验,这里兜底防御越界配置。 + pub fn try_new( + enabled: bool, + threshold: u32, + window_seconds: u64, + freeze_duration_seconds: u64, + probe_enabled: bool, + probe_model: Option, + adaptive_concurrency: bool, + ) -> Result { + if !(2..=1_000).contains(&threshold) + || !(60..=3_600).contains(&window_seconds) + || !(300..=604_800).contains(&freeze_duration_seconds) + || probe_model.as_deref().is_some_and(|model| { + model.is_empty() + || model.len() > 128 + || model != model.trim() + || model.bytes().any(|byte| byte.is_ascii_control()) + }) + { + return Err(ProviderStoreError::new( + ProviderStoreErrorKind::InvalidData, + "validate freeze policy", + )); + } + Ok(Self { + enabled, + threshold, + window: Duration::from_secs(window_seconds), + freeze_duration: Duration::from_secs(freeze_duration_seconds), + probe_enabled, + probe_model, + adaptive_concurrency, + }) + } + + /// 功能关闭时的全零策略;触发路径与 worker 都以此短路。 + #[must_use] + pub const fn disabled() -> Self { + Self { + enabled: false, + threshold: 2, + window: Duration::from_secs(60), + freeze_duration: Duration::from_secs(300), + probe_enabled: false, + probe_model: None, + adaptive_concurrency: false, + } + } + + #[must_use] + pub const fn enabled(&self) -> bool { + self.enabled + } + + /// 窗口内触发冻结的请求级失败次数阈值。 + #[must_use] + pub const fn threshold(&self) -> u32 { + self.threshold + } + + /// 失败计数滑动窗口;每次失败都会顺延窗口。 + #[must_use] + pub const fn window(&self) -> Duration { + self.window + } + + /// 冻结时长;探测失败后的顺延也使用该值。 + #[must_use] + pub const fn freeze_duration(&self) -> Duration { + self.freeze_duration + } + + #[must_use] + pub const fn probe_enabled(&self) -> bool { + self.probe_enabled + } + + /// 探测模型;`None` 表示由 worker 选择账号可用的第一个模型。 + #[must_use] + pub fn probe_model(&self) -> Option<&str> { + self.probe_model.as_deref() + } + + #[must_use] + pub const fn adaptive_concurrency(&self) -> bool { + self.adaptive_concurrency + } } /// OAuth pending flow 的原始绑定只在 Provider 与 Store 边界内短暂存在。 diff --git a/backend/crates/gateway-core/src/task/mod.rs b/backend/crates/gateway-core/src/task/mod.rs index 166fee18c..613bfbbae 100644 --- a/backend/crates/gateway-core/src/task/mod.rs +++ b/backend/crates/gateway-core/src/task/mod.rs @@ -23,10 +23,11 @@ pub enum WorkerKind { OpsFlush, Backup, AccountImport, + AccountFreezeRecovery, } impl WorkerKind { - pub const ALL: [Self; 9] = [ + pub const ALL: [Self; 10] = [ Self::OAuthRefresh, Self::QuotaCatalogHealth, Self::RuntimeSnapshotReconciliation, @@ -36,6 +37,7 @@ impl WorkerKind { Self::OpsFlush, Self::Backup, Self::AccountImport, + Self::AccountFreezeRecovery, ]; #[must_use] @@ -50,6 +52,7 @@ impl WorkerKind { Self::OpsFlush => "ops_flush", Self::Backup => "backup", Self::AccountImport => "account_import", + Self::AccountFreezeRecovery => "account_freeze_recovery", } } } diff --git a/backend/crates/gateway-core/tests/account/mod.rs b/backend/crates/gateway-core/tests/account/mod.rs index c92bece3c..0c4bb707d 100644 --- a/backend/crates/gateway-core/tests/account/mod.rs +++ b/backend/crates/gateway-core/tests/account/mod.rs @@ -49,7 +49,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option) -> AccountCandida last_started_at: None, quota_reset_at: None, quota_remaining_rank: remaining, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -222,7 +222,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -240,7 +240,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -313,13 +313,13 @@ fn rate_limited_projection_carries_only_the_active_cooldown_deadline() { let active_until = now + Duration::from_secs(60); let account = account("acct_rate_limited"); - let active = account.status_projection(now, Some(active_until)); + let active = account.status_projection(now, Some(active_until.into())); assert_eq!(active.status, AccountStatus::RateLimited); - assert_eq!(active.rate_limited_until, Some(active_until)); + assert_eq!(active.cooldown, Some(active_until.into())); - let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1))); + let elapsed = account.status_projection(now, Some((now - Duration::from_secs(1)).into())); assert_eq!(elapsed.status, AccountStatus::Normal); - assert_eq!(elapsed.rate_limited_until, None); + assert_eq!(elapsed.cooldown, None); } #[test] @@ -492,8 +492,8 @@ fn highest_priority_affinity_should_still_obey_existing_scheduling_constraints() ); } AccountSchedulingBlocker::LocalAvailability => { - candidates[0].signals.rate_limited_until = - Some(selection.now + Duration::from_secs(60)) + candidates[0].signals.cooldown = + Some((selection.now + Duration::from_secs(60)).into()) } _ => unreachable!("test cases"), } @@ -643,7 +643,7 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() { last_started_at: Some(last_started_at), quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, } @@ -1062,3 +1062,32 @@ fn elapsed_quota_reset_does_not_fabricate_recovery() { .expect("healthy candidate available"); assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy"); } + +#[test] +fn due_probe_freeze_blocks_scheduling_and_remains_visible_as_rate_limited() { + use gateway_core::account::{AccountCooldown, AccountCooldownKind}; + let mut candidates = [ + weighted_candidate("acct_frozen", 100, 0), + weighted_candidate("acct_ready", 1, 0), + ]; + let selection = context(RotationStrategy::Smart); + candidates[0].signals.cooldown = Some(AccountCooldown { + until: selection.now - Duration::from_secs(120), + kind: AccountCooldownKind::CapacityFreezeProbe, + }); + let projection = candidates[0] + .account + .status_projection(selection.now, candidates[0].signals.cooldown); + assert_eq!(projection.status, AccountStatus::RateLimited); + assert!( + projection + .cooldown + .expect("freeze details") + .kind + .requires_probe() + ); + let selected = AccountSelector + .select(&candidates, &selection) + .expect("available account"); + assert_eq!(selected.candidate().account.id().as_str(), "acct_ready"); +} diff --git a/backend/crates/gateway-core/tests/provider_ports/mod.rs b/backend/crates/gateway-core/tests/provider_ports/mod.rs index 30950a01b..511cf357d 100644 --- a/backend/crates/gateway-core/tests/provider_ports/mod.rs +++ b/backend/crates/gateway-core/tests/provider_ports/mod.rs @@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() { last_started_at: None, quota_reset_at: None, quota_remaining_rank: Some(7), - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: Some(125), first_output_latency_ms: Some(250), }, diff --git a/backend/crates/gateway-host/tests/workers/mod.rs b/backend/crates/gateway-host/tests/workers/mod.rs index 1a9f87ddd..5af943534 100644 --- a/backend/crates/gateway-host/tests/workers/mod.rs +++ b/backend/crates/gateway-host/tests/workers/mod.rs @@ -17,7 +17,7 @@ use gateway_core::task::{ use gateway_host::workers::{WorkerStartError, WorkerSupervisor}; use tokio::sync::Notify; -const ACTIVE_KINDS: [WorkerKind; 8] = [ +const ACTIVE_KINDS: [WorkerKind; 9] = [ WorkerKind::OAuthRefresh, WorkerKind::QuotaCatalogHealth, WorkerKind::RuntimeSnapshotReconciliation, @@ -26,6 +26,7 @@ const ACTIVE_KINDS: [WorkerKind; 8] = [ WorkerKind::Retention, WorkerKind::Backup, WorkerKind::AccountImport, + WorkerKind::AccountFreezeRecovery, ]; #[derive(Clone)] diff --git a/backend/crates/gateway-store/src/admin_adapter.rs b/backend/crates/gateway-store/src/admin_adapter.rs index f08a3000c..82a0f6de6 100644 --- a/backend/crates/gateway-store/src/admin_adapter.rs +++ b/backend/crates/gateway-store/src/admin_adapter.rs @@ -54,6 +54,14 @@ impl SettingsStore for AdminSettingsStoreAdapter { usage_retention_days: command.usage_retention_days, ops_event_retention_days: command.ops_event_retention_days, audit_retention_days: command.audit_retention_days, + account_auto_freeze_enabled: command.account_auto_freeze_enabled, + account_auto_freeze_threshold: command.account_auto_freeze_threshold, + account_auto_freeze_window_seconds: command.account_auto_freeze_window_seconds, + account_auto_freeze_duration_seconds: command.account_auto_freeze_duration_seconds, + account_auto_freeze_probe_enabled: command.account_auto_freeze_probe_enabled, + account_auto_freeze_probe_model: command.account_auto_freeze_probe_model, + account_auto_freeze_adaptive_concurrency: command + .account_auto_freeze_adaptive_concurrency, }, audit: mutation_audit( context, @@ -73,6 +81,7 @@ impl SettingsStore for AdminSettingsStoreAdapter { "min_codex_desktop_version".to_owned(), "min_codex_cli_version".to_owned(), "retention".to_owned(), + "account_auto_freeze".to_owned(), ], ), }; @@ -181,6 +190,13 @@ pub(crate) fn admin_runtime_settings( usage_retention_days: settings.usage_retention_days, ops_event_retention_days: settings.ops_event_retention_days, audit_retention_days: settings.audit_retention_days, + account_auto_freeze_enabled: settings.account_auto_freeze_enabled, + account_auto_freeze_threshold: settings.account_auto_freeze_threshold, + account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds, + account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds, + account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled, + account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model, + account_auto_freeze_adaptive_concurrency: settings.account_auto_freeze_adaptive_concurrency, updated_at: settings.updated_at, }) } diff --git a/backend/crates/gateway-store/src/postgres/account_groups.rs b/backend/crates/gateway-store/src/postgres/account_groups.rs index b27896aff..e31a3df4f 100644 --- a/backend/crates/gateway-store/src/postgres/account_groups.rs +++ b/backend/crates/gateway-store/src/postgres/account_groups.rs @@ -208,7 +208,7 @@ impl AccountGroupStore for PgAccountGroupRepository { credential_state: account.credential_state, access_token_expires_at: account.access_token_expires_at.map(Into::into), quota: account.quota, - rate_limited_until: None, + cooldown: None, last_error_reason: account.last_error_reason, last_error_message: account.last_error_message, }, diff --git a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs index fd532616c..21f7c1421 100644 --- a/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/observability/admin_adapter.rs @@ -9,7 +9,7 @@ use super::*; use crate::postgres::{ PgProviderAccountRepository, ProviderAccountRepository, ProviderAccountSummary, - account_status_projection, load_rate_limited_until, + account_status_projection, load_cooldown, }; use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository}; @@ -49,19 +49,15 @@ impl PgObservabilityRepository { ) .await?; let now = observed_at.into(); - let rate_limited_until = - load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await; + let cooldown = load_cooldown(self.cooldowns.as_deref(), &accounts, now).await; let mut metrics = ProviderAccountMetrics { total: u64::try_from(accounts.len()).unwrap_or(u64::MAX), ..ProviderAccountMetrics::default() }; let mut normal_accounts = Vec::new(); for account in &accounts { - let projection = account_status_projection( - account, - now, - rate_limited_until.get(&account.id).copied(), - ); + let projection = + account_status_projection(account, now, cooldown.get(&account.id).copied()); match projection.status { gateway_core::account::AccountStatus::Normal => { metrics.normal = metrics.normal.saturating_add(1); diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs index ce46b26e7..c20fdd77c 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -407,15 +407,11 @@ impl AccountStore for PgAdminAccountStore { )); } let now = Utc::now(); - let rate_limited_until = runtime - .rate_limited_until - .into_iter() - .map(|(account_id, until)| (account_id, until.into())) - .collect::>(); + let cooldown = runtime.cooldown; let now_system_time = std::time::SystemTime::from(now); - let active_rate_limited_ids = rate_limited_until + let active_rate_limited_ids = cooldown .iter() - .filter(|(_, until)| **until > now_system_time) + .filter(|(_, cooldown)| cooldown.is_active(now_system_time)) .map(|(account_id, _)| account_id.clone()) .collect::>(); let page = @@ -434,7 +430,7 @@ impl AccountStore for PgAdminAccountStore { let projection = account_status_projection( &summary, now.into(), - rate_limited_until.get(&account_id).copied(), + cooldown.get(&account_id).copied(), ); let mut account = admin_account_record(summary)?; account.groups = groups_by_account.remove(&account_id).unwrap_or_default(); @@ -466,12 +462,8 @@ impl AccountStore for PgAdminAccountStore { return Ok(None); }; let now = Utc::now(); - let rate_limited_until = runtime - .rate_limited_until - .get(account_id) - .copied() - .map(Into::into); - let projection = account_status_projection(&record.summary, now.into(), rate_limited_until); + let cooldown = runtime.cooldown.get(account_id).copied(); + let projection = account_status_projection(&record.summary, now.into(), cooldown); let account_id = record.summary.id.clone(); let mut groups = self .account_groups_by_account(std::slice::from_ref(&account_id)) @@ -745,6 +737,69 @@ impl AccountStore for PgAdminAccountStore { }) } + async fn lower_concurrency_limit( + &self, + account_id: &CoreProviderAccountId, + limit: AccountConcurrencyLimit, + context: &MutationContext, + ) -> AdminStoreResult> { + let mut transaction = self.pool.begin().await.map_err(|_| { + admin_store_error(ENTITY, postgres_unavailable("begin concurrency reduction")) + })?; + let result = + async { + // 与管理写入采用相同锁顺序;锁住默认值后再检查账号最新设置,避免把旧快照写回。 + let default_limit: i64 = sqlx::query_scalar( + "select max_concurrent_per_account from runtime_settings where id = 1 for update" + ).fetch_one(&mut *transaction).await + .map_err(|_| postgres_unavailable("lock default concurrency"))?; + let changed = sqlx::query_scalar::<_, String>( + "update provider_accounts set concurrency_limit = $2, updated_at = now() + where id = $1 and enabled = true and coalesce(concurrency_limit, $3) > $2 + returning id", + ) + .bind(account_id.as_str()) + .bind(i64::from(limit.get())) + .bind(default_limit) + .fetch_optional(&mut *transaction) + .await + .map_err(|_| postgres_unavailable("lower account concurrency"))?; + if changed.is_none() { + return Ok(None); + } + let revision = bump_config_revision_in_transaction(&mut transaction).await?; + append_admin_audit_event_in_transaction( + &mut transaction, + mutation_audit( + context, + "adapt_concurrency", + "provider_account", + account_id.as_str(), + vec!["concurrency_limit".to_owned()], + ), + revision, + ) + .await?; + Ok(Some(revision)) + } + .await; + let revision = super::repository::finish_admin_transaction( + transaction, + result, + "lower account concurrency", + ) + .await + .map_err(|error| admin_store_error(ENTITY, error))?; + revision + .map(|revision| { + Ok(AccountUpdateResult { + config_revision: admin_revision(revision)?, + account_id: account_id.clone(), + }) + }) + .transpose() + } + async fn recover_account( &self, account_id: &CoreProviderAccountId, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs index c328a246b..318b64f3a 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/runtime.rs @@ -16,7 +16,7 @@ use super::ProviderAccountSummary; pub(crate) fn account_status_projection( account: &ProviderAccountSummary, now: SystemTime, - rate_limited_until: Option, + cooldown: Option, ) -> AccountStatusProjection { resolve_account_status( &AccountStatusFacts { @@ -24,7 +24,7 @@ pub(crate) fn account_status_projection( credential_state: account.credential_state, access_token_expires_at: account.access_token_expires_at.map(Into::into), quota: account.quota, - rate_limited_until, + cooldown, last_error_reason: account.last_error_reason, last_error_message: account.last_error_message.clone(), }, @@ -36,18 +36,21 @@ pub(crate) fn account_status_projection( /// /// 冷却属于可丢失的账号级运行时事实:端口不可用、单条读取失败或脏账号 ID /// 都视为没有冷却,不污染持久账号状态。凭据轮换不会解除上游对同一账号的限流。 -pub(crate) async fn load_rate_limited_until( +pub(crate) async fn load_cooldown( cooldowns: Option<&dyn ProviderCooldownPort>, accounts: &[ProviderAccountSummary], now: SystemTime, -) -> BTreeMap { +) -> BTreeMap { let Some(cooldowns) = cooldowns else { return BTreeMap::new(); }; let reads = accounts.iter().map(|account| async move { let account_id = ProviderAccountId::new(account.id.clone()).ok()?; let cooldown = cooldowns.read(&account_id).await.ok().flatten()?; - (cooldown.until() > now).then(|| (account.id.clone(), cooldown.until())) + cooldown + .scheduling_state() + .is_active(now) + .then(|| (account.id.clone(), cooldown.scheduling_state())) }); join_all(reads).await.into_iter().flatten().collect() } diff --git a/backend/crates/gateway-store/src/postgres/runtime_settings.rs b/backend/crates/gateway-store/src/postgres/runtime_settings.rs index ddb821652..df0f9a40a 100644 --- a/backend/crates/gateway-store/src/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/src/postgres/runtime_settings.rs @@ -11,7 +11,8 @@ use sqlx::{PgPool, Postgres, Transaction}; use gateway_core::account::RotationStrategy; use gateway_core::policy::CodexClientVersion; use gateway_core::provider_ports::{ - ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind, + ProviderFreezePolicy, ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, + ProviderStoreErrorKind, }; use crate::{Revision, StoreError, StoreResult, postgres_unavailable}; @@ -34,6 +35,13 @@ pub struct RuntimeSettings { pub usage_retention_days: u32, pub ops_event_retention_days: u32, pub audit_retention_days: u32, + pub account_auto_freeze_enabled: bool, + pub account_auto_freeze_threshold: u32, + pub account_auto_freeze_window_seconds: u64, + pub account_auto_freeze_duration_seconds: u64, + pub account_auto_freeze_probe_enabled: bool, + pub account_auto_freeze_probe_model: Option, + pub account_auto_freeze_adaptive_concurrency: bool, pub updated_at: DateTime, } @@ -60,6 +68,34 @@ impl fmt::Debug for RuntimeSettings { .field("usage_retention_days", &self.usage_retention_days) .field("ops_event_retention_days", &self.ops_event_retention_days) .field("audit_retention_days", &self.audit_retention_days) + .field( + "account_auto_freeze_enabled", + &self.account_auto_freeze_enabled, + ) + .field( + "account_auto_freeze_threshold", + &self.account_auto_freeze_threshold, + ) + .field( + "account_auto_freeze_window_seconds", + &self.account_auto_freeze_window_seconds, + ) + .field( + "account_auto_freeze_duration_seconds", + &self.account_auto_freeze_duration_seconds, + ) + .field( + "account_auto_freeze_probe_enabled", + &self.account_auto_freeze_probe_enabled, + ) + .field( + "account_auto_freeze_probe_model", + &self.account_auto_freeze_probe_model, + ) + .field( + "account_auto_freeze_adaptive_concurrency", + &self.account_auto_freeze_adaptive_concurrency, + ) .field("updated_at", &self.updated_at) .finish() } @@ -82,6 +118,13 @@ pub struct RuntimeSettingsUpdate { pub usage_retention_days: u32, pub ops_event_retention_days: u32, pub audit_retention_days: u32, + pub account_auto_freeze_enabled: bool, + pub account_auto_freeze_threshold: u32, + pub account_auto_freeze_window_seconds: u64, + pub account_auto_freeze_duration_seconds: u64, + pub account_auto_freeze_probe_enabled: bool, + pub account_auto_freeze_probe_model: Option, + pub account_auto_freeze_adaptive_concurrency: bool, } impl fmt::Debug for RuntimeSettingsUpdate { @@ -109,9 +152,13 @@ impl RuntimeSettingsUpdate { || self.usage_retention_days < 31 || self.ops_event_retention_days == 0 || self.audit_retention_days == 0 + || !(2..=1_000).contains(&self.account_auto_freeze_threshold) + || !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) + || !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) || !valid_model_mappings(&self.model_mappings) || !valid_client_version(self.min_codex_desktop_version.as_deref()) || !valid_client_version(self.min_codex_cli_version.as_deref()) + || !valid_probe_model(self.account_auto_freeze_probe_model.as_deref()) || RotationStrategy::parse(&self.rotation_strategy).is_none() { return Err(StoreError::InvalidData { @@ -173,7 +220,11 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul refresh_concurrency, max_concurrent_per_account, request_interval_ms, rotation_strategy, model_mappings_json, usage_retention_days, ops_event_retention_days, audit_retention_days, min_codex_desktop_version, - min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds + min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, + account_auto_freeze_enabled, account_auto_freeze_threshold, + account_auto_freeze_window_seconds, account_auto_freeze_duration_seconds, + account_auto_freeze_probe_enabled, account_auto_freeze_probe_model, + account_auto_freeze_adaptive_concurrency from runtime_settings where id = 1", ) .fetch_optional(pool) @@ -202,6 +253,25 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { ) }) } + + fn load_freeze_policy( + &self, + ) -> futures::future::BoxFuture<'_, Result> { + Box::pin(async move { + let settings = RuntimeSettingsRepository::load_runtime_settings(self) + .await + .map_err(|_| provider_unavailable("load freeze policy"))?; + ProviderFreezePolicy::try_new( + settings.account_auto_freeze_enabled, + settings.account_auto_freeze_threshold, + settings.account_auto_freeze_window_seconds, + settings.account_auto_freeze_duration_seconds, + settings.account_auto_freeze_probe_enabled, + settings.account_auto_freeze_probe_model, + settings.account_auto_freeze_adaptive_concurrency, + ) + }) + } } pub(crate) async fn load_runtime_settings_in_transaction( @@ -212,7 +282,11 @@ pub(crate) async fn load_runtime_settings_in_transaction( refresh_concurrency, max_concurrent_per_account, request_interval_ms, rotation_strategy, model_mappings_json, usage_retention_days, ops_event_retention_days, audit_retention_days, min_codex_desktop_version, - min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds + min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, + account_auto_freeze_enabled, account_auto_freeze_threshold, + account_auto_freeze_window_seconds, account_auto_freeze_duration_seconds, + account_auto_freeze_probe_enabled, account_auto_freeze_probe_model, + account_auto_freeze_adaptive_concurrency from runtime_settings where id = 1", ) .fetch_optional(&mut **transaction) @@ -250,6 +324,13 @@ pub(crate) async fn update_runtime_settings_in_transaction( max_waiting_per_key = $13, max_waiting_per_account = $14, concurrency_wait_timeout_seconds = $15, + account_auto_freeze_enabled = $16, + account_auto_freeze_threshold = $17, + account_auto_freeze_window_seconds = $18, + account_auto_freeze_duration_seconds = $19, + account_auto_freeze_probe_enabled = $20, + account_auto_freeze_probe_model = $21, + account_auto_freeze_adaptive_concurrency = $22, updated_at = now() where id = 1 returning config_revision", @@ -269,6 +350,16 @@ pub(crate) async fn update_runtime_settings_in_transaction( .bind(i64::from(update.max_waiting_per_key)) .bind(i64::from(update.max_waiting_per_account)) .bind(i64::from(update.concurrency_wait_timeout_seconds)) + .bind(update.account_auto_freeze_enabled) + .bind(i64::from(update.account_auto_freeze_threshold)) + .bind(i64::try_from(update.account_auto_freeze_window_seconds).map_err(|_| invalid_numeric())?) + .bind( + i64::try_from(update.account_auto_freeze_duration_seconds) + .map_err(|_| invalid_numeric())?, + ) + .bind(update.account_auto_freeze_probe_enabled) + .bind(update.account_auto_freeze_probe_model.as_deref()) + .bind(update.account_auto_freeze_adaptive_concurrency) .fetch_optional(&mut **transaction) .await .map_err(|_| postgres_unavailable("update runtime settings in transaction"))? @@ -335,6 +426,13 @@ struct RuntimeSettingsRow { max_waiting_per_key: i64, max_waiting_per_account: i64, concurrency_wait_timeout_seconds: i64, + account_auto_freeze_enabled: bool, + account_auto_freeze_threshold: i64, + account_auto_freeze_window_seconds: i64, + account_auto_freeze_duration_seconds: i64, + account_auto_freeze_probe_enabled: bool, + account_auto_freeze_probe_model: Option, + account_auto_freeze_adaptive_concurrency: bool, } fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult { @@ -356,6 +454,13 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult bool { fn valid_client_version(value: Option<&str>) -> bool { value.is_none_or(|value| CodexClientVersion::parse(value).is_ok()) } + +fn valid_probe_model(value: Option<&str>) -> bool { + value.is_none_or(|value| { + !value.is_empty() + && value.len() <= 128 + && value == value.trim() + && !value.bytes().any(|byte| byte.is_ascii_control()) + }) +} diff --git a/backend/crates/gateway-store/src/redis/admin_account_runtime.rs b/backend/crates/gateway-store/src/redis/admin_account_runtime.rs index 1d236259f..5014698db 100644 --- a/backend/crates/gateway-store/src/redis/admin_account_runtime.rs +++ b/backend/crates/gateway-store/src/redis/admin_account_runtime.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use async_trait::async_trait; +use chrono::{DateTime, Utc}; use futures::future::join_all; use gateway_admin::{ model::accounts::AccountRuntimeSnapshot, @@ -49,15 +50,23 @@ impl AccountRuntimeStore for RedisAdminAccountRuntimeStore { .read_credential_cooldown(account_id) .await .map(|cooldown| { - cooldown.map(|cooldown| (account_id.clone(), cooldown.cooldown_until)) + cooldown.map(|cooldown| { + ( + account_id.clone(), + gateway_core::account::AccountCooldown { + until: cooldown.cooldown_until.into(), + kind: cooldown.kind, + }, + ) + }) }) }); - let mut rate_limited_until = BTreeMap::new(); + let mut cooldown = BTreeMap::new(); for result in join_all(reads).await { if let Some((account_id, until)) = result.map_err(|error| crate::admin_store_error("account runtime", error))? { - rate_limited_until.insert(account_id, until); + cooldown.insert(account_id, until); } } let in_flight = self @@ -72,8 +81,50 @@ impl AccountRuntimeStore for RedisAdminAccountRuntimeStore { .collect() }); Ok(AccountRuntimeSnapshot { - rate_limited_until, + cooldown, in_flight, }) } + + async fn active_freezes( + &self, + ) -> AdminStoreResult> { + self.cooldowns + .active_freezes() + .await + .map_err(|error| crate::admin_store_error("account runtime", error)) + } + + async fn capacity_peaks( + &self, + account_ids: &[String], + ) -> AdminStoreResult> { + let reads = account_ids.iter().map(|account_id| async move { + self.cooldowns + .read_capacity_peak(account_id) + .await + .map(|peak| peak.map(|value| (account_id.clone(), value))) + }); + let mut peaks = BTreeMap::new(); + for result in join_all(reads).await { + if let Some((account_id, peak)) = + result.map_err(|error| crate::admin_store_error("account runtime", error))? + { + peaks.insert(account_id, peak); + } + } + Ok(peaks) + } + + async fn finish_freeze( + &self, + account_id: &str, + expected: &gateway_admin::model::accounts::AccountFreeze, + postpone_until: Option>, + ) -> AdminStoreResult { + self.cooldowns + .finish_freeze(account_id, expected, postpone_until) + .await + .map_err(|error| crate::admin_store_error("account runtime", error)) + } } diff --git a/backend/crates/gateway-store/src/redis/credential_cooldown.rs b/backend/crates/gateway-store/src/redis/credential_cooldown.rs index 78e80ab43..354d0a307 100644 --- a/backend/crates/gateway-store/src/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/src/redis/credential_cooldown.rs @@ -6,11 +6,13 @@ use gateway_admin::model::accounts::AccountRuntimeSnapshot; use gateway_core::{ account::{CredentialRevision, ProviderAccountId}, provider_ports::{ - ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown, - ProviderStoreError, ProviderStoreErrorKind, + ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope, + ProviderScopedCooldown, ProviderStoreError, ProviderStoreErrorKind, }, }; use redis::{Script, aio::ConnectionManager}; +use std::collections::BTreeMap; +use std::time::Duration; use crate::{Revision, StoreError, StoreResult, redis_unavailable, require_nonempty}; @@ -22,17 +24,21 @@ local incoming = tonumber(ARGV[1]) local incoming_until = tonumber(ARGV[2]) if current > incoming then return 0 end local current_until = tonumber(redis.call('HGET', KEYS[1], 'until_ms') or '0') -if current == incoming and current_until >= incoming_until then return 0 end +local current_kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit' +if current == incoming then + if current_kind ~= 'rate_limit' and ARGV[4] == 'rate_limit' then return 0 end + if current_kind == ARGV[4] and current_until >= incoming_until then return 0 end + incoming_until = math.max(current_until, incoming_until) +end local clock = redis.call('TIME') local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) -if incoming_until <= now_ms then - redis.call('DEL', KEYS[1]) - if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[3]) end - return 0 +if incoming_until <= now_ms and ARGV[4] ~= 'capacity_freeze_probe' then return 0 end +redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'until_ms', incoming_until, 'kind', ARGV[4], 'generation', ARGV[5]) +if ARGV[4] == 'capacity_freeze_probe' then + redis.call('PERSIST', KEYS[1]) +else + redis.call('PEXPIRE', KEYS[1], incoming_until - now_ms + 60000) end -redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'until_ms', ARGV[2]) -local ttl = incoming_until - now_ms + 60000 -redis.call('PEXPIRE', KEYS[1], ttl) if #KEYS > 1 then redis.call('ZADD', KEYS[2], incoming_until, ARGV[3]) end return 1 "#; @@ -40,19 +46,15 @@ return 1 const READ_SCRIPT: &str = r#" local revision = redis.call('HGET', KEYS[1], 'revision') local until_ms = redis.call('HGET', KEYS[1], 'until_ms') -if revision == false or until_ms == false then - redis.call('DEL', KEYS[1]) - if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[1]) end - return {0, '0', '0'} -end +local kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit' local clock = redis.call('TIME') local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) -if tonumber(until_ms) <= now_ms then +if revision == false or until_ms == false or (kind ~= 'capacity_freeze_probe' and tonumber(until_ms) <= now_ms) then redis.call('DEL', KEYS[1]) if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[1]) end - return {0, '0', '0'} + return {0, '0', '0', 'rate_limit', ''} end -return {1, revision, until_ms} +return {1, revision, until_ms, kind, redis.call('HGET', KEYS[1], 'generation') or ''} "#; const INVALIDATE_SCRIPT: &str = r#" @@ -63,11 +65,56 @@ if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[2]) end return 1 "#; -const ACTIVE_COOLDOWNS_SCRIPT: &str = r#" -local clock = redis.call('TIME') -local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) -redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms) -return redis.call('ZRANGEBYSCORE', KEYS[1], '(' .. now_ms, '+inf', 'WITHSCORES') +// 普通推理成功只能清除临时限流和未形成冻结的证据,判断与删除必须原子执行。 +const SUCCESS_SCRIPT: &str = r#" +local current = tonumber(redis.call('HGET', KEYS[1], 'revision') or '0') +local kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit' +if current > tonumber(ARGV[1]) or kind ~= 'rate_limit' then return 0 end +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4]) +redis.call('ZREM', KEYS[2], ARGV[2]) +return 1 +"#; + +// 探测结果只能修改读到的这一代冻结;删除后重建同 revision 的冻结也不匹配。 +const FINISH_FREEZE_SCRIPT: &str = r#" +if redis.call('HGET', KEYS[1], 'generation') ~= ARGV[2] + or redis.call('HGET', KEYS[1], 'revision') ~= ARGV[1] then return 0 end +local kind = redis.call('HGET', KEYS[1], 'kind') +if kind ~= 'capacity_freeze' and kind ~= 'capacity_freeze_probe' then return 0 end +if ARGV[4] == '' then + redis.call('DEL', KEYS[1], KEYS[3], KEYS[4]) + redis.call('ZREM', KEYS[2], ARGV[3]) +else + local until_ms = math.max(tonumber(redis.call('HGET', KEYS[1], 'until_ms')), tonumber(ARGV[4])) + redis.call('HSET', KEYS[1], 'until_ms', until_ms, 'generation', ARGV[5]) + redis.call('ZADD', KEYS[2], until_ms, ARGV[3]) + if kind == 'capacity_freeze_probe' then + redis.call('PERSIST', KEYS[1]) + else + local clock = redis.call('TIME') + local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) + redis.call('PEXPIRE', KEYS[1], math.max(1, until_ms - now_ms + 60000)) + end +end +return 1 +"#; + +// 每次容量失败都顺延窗口 TTL(与刷新退避计数同语义),并把本次观测到的 +// 在途并发并入峰值证据;峰值与计数共享同一窗口生命周期。 +const RECORD_CAPACITY_FAILURE_SCRIPT: &str = r#" +local count = redis.call('INCR', KEYS[1]) +local ttl_ms = tonumber(ARGV[1]) +redis.call('PEXPIRE', KEYS[1], ttl_ms) +local in_flight = tonumber(ARGV[2]) +if in_flight > 0 then + local peak = tonumber(redis.call('GET', KEYS[2]) or '0') + if in_flight > peak then + redis.call('SET', KEYS[2], in_flight, 'PX', ttl_ms) + else + redis.call('PEXPIRE', KEYS[2], ttl_ms) + end +end +return count "#; #[derive(Debug, Clone, PartialEq, Eq)] @@ -75,6 +122,7 @@ pub struct CredentialCooldown { pub provider_account_id: String, pub credential_revision: Revision, pub cooldown_until: DateTime, + pub kind: ProviderCooldownKind, } #[async_trait] @@ -116,6 +164,22 @@ impl RedisCredentialCooldownRepository { format!("{}:account:active-cooldowns", self.namespace) } + fn capacity_failures_key(&self, provider_account_id: &str) -> StoreResult { + let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?; + Ok(format!( + "{}:account:{fingerprint}:capacity-failures", + self.namespace + )) + } + + fn capacity_peak_key(&self, provider_account_id: &str) -> StoreResult { + let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?; + Ok(format!( + "{}:account:{fingerprint}:capacity-peak-inflight", + self.namespace + )) + } + fn scoped_key( &self, provider_account_id: &str, @@ -135,6 +199,7 @@ impl RedisCredentialCooldownRepository { key: String, credential_revision: Revision, cooldown_until: DateTime, + kind: ProviderCooldownKind, index_member: Option<&str>, ) -> StoreResult { let until_ms = cooldown_until.timestamp_millis(); @@ -149,6 +214,8 @@ impl RedisCredentialCooldownRepository { .arg(credential_revision.get()) .arg(until_ms) .arg(index_member) + .arg(kind.as_str()) + .arg(uuid::Uuid::new_v4().to_string()) .invoke_async::(&mut connection) .await } else { @@ -157,6 +224,8 @@ impl RedisCredentialCooldownRepository { .arg(credential_revision.get()) .arg(until_ms) .arg("") + .arg(kind.as_str()) + .arg(uuid::Uuid::new_v4().to_string()) .invoke_async::(&mut connection) .await } @@ -168,7 +237,7 @@ impl RedisCredentialCooldownRepository { &self, key: String, index_member: Option<&str>, - ) -> StoreResult)>> { + ) -> StoreResult, ProviderCooldownKind, String)>> { let mut connection = self.connection.clone(); let result = if let Some(index_member) = index_member { Script::new(READ_SCRIPT) @@ -184,7 +253,7 @@ impl RedisCredentialCooldownRepository { .invoke_async(&mut connection) .await }; - let (present, revision, until_ms): (i64, String, String) = + let (present, revision, until_ms, kind, generation): (i64, String, String, String, String) = result.map_err(|_| redis_unavailable("read credential cooldown"))?; if present == 0 { return Ok(None); @@ -197,7 +266,13 @@ impl RedisCredentialCooldownRepository { .map_err(|_| invalid("cached cooldown expiry is invalid"))?; let cooldown_until = DateTime::from_timestamp_millis(until_ms) .ok_or_else(|| invalid("cached cooldown expiry is invalid"))?; - Ok(Some((Revision::new(revision)?, cooldown_until))) + let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit); + Ok(Some(( + Revision::new(revision)?, + cooldown_until, + kind, + generation, + ))) } async fn invalidate_at_key( @@ -227,30 +302,107 @@ impl RedisCredentialCooldownRepository { Ok(removed == 1) } - pub(crate) async fn active_cooldowns(&self) -> StoreResult { + async fn indexed_accounts(&self) -> StoreResult> { let mut connection = self.connection.clone(); - let values = Script::new(ACTIVE_COOLDOWNS_SCRIPT) - .key(self.active_index_key()) - .invoke_async::>(&mut connection) + redis::cmd("ZRANGE") + .arg(self.active_index_key()) + .arg(0) + .arg(-1) + .query_async(&mut connection) .await - .map_err(|_| redis_unavailable("list active credential cooldowns"))?; - if values.len() % 2 != 0 { - return Err(invalid("active cooldown index is invalid")); - } - let mut rate_limited_until = std::collections::BTreeMap::new(); - for pair in values.chunks_exact(2) { - let until_ms = pair[1] - .parse::() - .map_err(|_| invalid("active cooldown expiry is invalid"))?; - let until = DateTime::from_timestamp_millis(until_ms) - .ok_or_else(|| invalid("active cooldown expiry is invalid"))?; - rate_limited_until.insert(pair[0].clone(), until); + .map_err(|_| redis_unavailable("list indexed cooldowns")) + } + + pub(crate) async fn active_cooldowns(&self) -> StoreResult { + let mut cooldown = BTreeMap::new(); + for account_id in self.indexed_accounts().await? { + if let Some((_, until, kind, _)) = self + .read_at_key(self.key(&account_id)?, Some(&account_id)) + .await? + { + cooldown.insert( + account_id, + gateway_core::account::AccountCooldown { + until: until.into(), + kind, + }, + ); + } } Ok(AccountRuntimeSnapshot { - rate_limited_until, + cooldown, in_flight: None, }) } + + /// 包含已到探测时间但尚未确认恢复的冻结;到期不能从 worker 工作集中移除。 + pub(crate) async fn active_freezes( + &self, + ) -> StoreResult> { + let mut freezes = BTreeMap::new(); + for account_id in self.indexed_accounts().await? { + if let Some((revision, until, kind, generation)) = self + .read_at_key(self.key(&account_id)?, Some(&account_id)) + .await? + && kind.is_capacity_freeze() + { + freezes.insert( + account_id, + gateway_admin::model::accounts::AccountFreeze { + credential_revision: gateway_admin::model::Revision::new(revision.get()) + .map_err(|_| invalid("freeze revision"))?, + until, + generation, + requires_probe: kind.requires_probe(), + }, + ); + } + } + Ok(freezes) + } + + pub(crate) async fn finish_freeze( + &self, + account_id: &str, + expected: &gateway_admin::model::accounts::AccountFreeze, + postpone_until: Option>, + ) -> StoreResult { + let mut connection = self.connection.clone(); + let changed: i64 = Script::new(FINISH_FREEZE_SCRIPT) + .key(self.key(account_id)?) + .key(self.active_index_key()) + .key(self.capacity_failures_key(account_id)?) + .key(self.capacity_peak_key(account_id)?) + .arg(expected.credential_revision.get()) + .arg(&expected.generation) + .arg(account_id) + .arg( + postpone_until + .map(|until| until.timestamp_millis().to_string()) + .unwrap_or_default(), + ) + .arg(uuid::Uuid::new_v4().to_string()) + .invoke_async(&mut connection) + .await + .map_err(|_| redis_unavailable("finish capacity freeze"))?; + Ok(changed == 1) + } + + /// 读取窗口内观测到的在途并发峰值;key 随窗口 TTL 过期,无需额外清理。 + pub(crate) async fn read_capacity_peak( + &self, + provider_account_id: &str, + ) -> StoreResult> { + let mut connection = self.connection.clone(); + let peak: Option = redis::cmd("GET") + .arg(self.capacity_peak_key(provider_account_id)?) + .query_async(&mut connection) + .await + .map_err(|_| redis_unavailable("read capacity peak in-flight"))?; + peak.map(u32::try_from) + .transpose() + .map_err(|_| invalid("capacity peak in-flight is invalid")) + } } #[async_trait] @@ -265,6 +417,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository { self.key(&cooldown.provider_account_id)?, cooldown.credential_revision, cooldown.cooldown_until, + cooldown.kind, Some(&cooldown.provider_account_id), ) .await @@ -282,11 +435,14 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository { self.read_at_key(self.key(provider_account_id)?, Some(provider_account_id)) .await .map(|value| { - value.map(|(credential_revision, cooldown_until)| CredentialCooldown { - provider_account_id: provider_account_id.to_owned(), - credential_revision, - cooldown_until, - }) + value.map( + |(credential_revision, cooldown_until, kind, _)| CredentialCooldown { + provider_account_id: provider_account_id.to_owned(), + credential_revision, + cooldown_until, + kind, + }, + ) }) } @@ -317,8 +473,11 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository { // 账号删除:清除该账号的 account key 与全部 model-scoped key。 // 用 SCAN 精确匹配命名空间内该账号前缀,避免 KEYS 阻塞。 let mut connection = self.connection.clone(); - let account_key = self.key(provider_account_id)?; - let mut keys = vec![account_key.clone()]; + let mut keys = vec![ + self.key(provider_account_id)?, + self.capacity_failures_key(provider_account_id)?, + self.capacity_peak_key(provider_account_id)?, + ]; let pattern = format!( "{}:account:{}:cooldown:*", self.namespace, @@ -341,21 +500,21 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository { break; } } - if keys.is_empty() { - return Ok(false); - } - let removed: i64 = redis::cmd("DEL") - .arg(keys) - .query_async(&mut connection) - .await - .map_err(|_| redis_unavailable("delete account cooldown keys"))?; - let index_removed: i64 = redis::cmd("ZREM") - .arg(self.active_index_key()) - .arg(provider_account_id) - .query_async(&mut connection) - .await - .map_err(|_| redis_unavailable("remove active account cooldown"))?; - Ok(removed > 0 || index_removed > 0) + // 删除与索引移除同属一个原子边界;否则新冻结可能在两步之间写入后丢失索引。 + let removed: i64 = Script::new( + r#" + local removed = 0 + for i = 2, #KEYS do removed = removed + redis.call('DEL', KEYS[i]) end + return removed + redis.call('ZREM', KEYS[1], ARGV[1]) + "#, + ) + .key(self.active_index_key()) + .key(keys) + .arg(provider_account_id) + .invoke_async(&mut connection) + .await + .map_err(|_| redis_unavailable("delete account cooldowns and index"))?; + Ok(removed > 0) } } @@ -370,6 +529,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { credential_revision: Revision::new(cooldown.credential_revision().get()) .map_err(|_| provider_invalid("encode credential cooldown"))?, cooldown_until: cooldown.until().into(), + kind: cooldown.kind(), }; CredentialCooldownRepository::cache_credential_cooldown(self, &record) .await @@ -390,10 +550,11 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { .map_err(|_| provider_invalid("decode credential cooldown"))?; let revision = CredentialRevision::new(record.credential_revision.get()) .map_err(|_| provider_invalid("decode credential cooldown"))?; - Ok(ProviderCooldown::new( + Ok(ProviderCooldown::new_with_kind( account_id, revision, record.cooldown_until.into(), + record.kind, )) }) .transpose() @@ -430,6 +591,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { .map_err(|_| provider_invalid("encode scoped credential cooldown"))?, revision, cooldown.until().into(), + ProviderCooldownKind::RateLimit, None, ) .await @@ -451,7 +613,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { ) .await .map_err(|_| provider_unavailable("read scoped credential cooldown"))? - .map(|(revision, until)| { + .map(|(revision, until, _, _)| { Ok(ProviderScopedCooldown::new( account_id.clone(), CredentialRevision::new(revision.get()) @@ -494,6 +656,75 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { .map_err(|_| provider_unavailable("clear all credential cooldowns")) }) } + + fn record_capacity_failure<'a>( + &'a self, + account_id: &'a ProviderAccountId, + window: Duration, + in_flight: u32, + ) -> futures::future::BoxFuture<'a, Result> { + Box::pin(async move { + let mut connection = self.connection.clone(); + let window_ms = u64::try_from(window.as_millis()) + .map_err(|_| provider_invalid("encode capacity failure window"))?; + let count: i64 = Script::new(RECORD_CAPACITY_FAILURE_SCRIPT) + .key( + self.capacity_failures_key(account_id.as_str()) + .map_err(|_| provider_invalid("encode capacity failure key"))?, + ) + .key( + self.capacity_peak_key(account_id.as_str()) + .map_err(|_| provider_invalid("encode capacity peak key"))?, + ) + .arg(i64::try_from(window_ms).unwrap_or(i64::MAX)) + .arg(i64::from(in_flight)) + .invoke_async(&mut connection) + .await + .map_err(|_| provider_unavailable("record capacity failure"))?; + u32::try_from(count).map_err(|_| provider_invalid("decode capacity failure count")) + }) + } + + fn clear_after_success<'a>( + &'a self, + account_id: &'a ProviderAccountId, + through_revision: CredentialRevision, + ) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> { + Box::pin(async move { + let mut connection = self.connection.clone(); + let _: i64 = Script::new(SUCCESS_SCRIPT) + .key( + self.key(account_id.as_str()) + .map_err(|_| provider_invalid("cooldown key"))?, + ) + .key(self.active_index_key()) + .key( + self.capacity_failures_key(account_id.as_str()) + .map_err(|_| provider_invalid("capacity count key"))?, + ) + .key( + self.capacity_peak_key(account_id.as_str()) + .map_err(|_| provider_invalid("capacity peak key"))?, + ) + .arg(through_revision.get()) + .arg(account_id.as_str()) + .invoke_async(&mut connection) + .await + .map_err(|_| provider_unavailable("clear cooldown after success"))?; + Ok(()) + }) + } + + fn capacity_peak_in_flight<'a>( + &'a self, + account_id: &'a ProviderAccountId, + ) -> futures::future::BoxFuture<'a, Result, ProviderStoreError>> { + Box::pin(async move { + self.read_capacity_peak(account_id.as_str()) + .await + .map_err(|_| provider_unavailable("read capacity peak in-flight")) + }) + } } fn invalid(message: &str) -> StoreError { diff --git a/backend/crates/gateway-store/src/redis/credential_leases.rs b/backend/crates/gateway-store/src/redis/credential_leases.rs index da192142b..ed9fa8e0f 100644 --- a/backend/crates/gateway-store/src/redis/credential_leases.rs +++ b/backend/crates/gateway-store/src/redis/credential_leases.rs @@ -524,7 +524,7 @@ impl RedisProviderLeaseCoordinator { last_started_at: signal.last_started_at.map(Into::into), quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -645,6 +645,36 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator { } }) } + + fn account_in_flight<'a>( + &'a self, + account_ids: &'a [ProviderAccountId], + ) -> futures::future::BoxFuture< + 'a, + Result, ProviderStoreError>, + > { + Box::pin(async move { + if account_ids.is_empty() { + return Ok(std::collections::BTreeMap::new()); + } + let ids = account_ids + .iter() + .map(|account_id| account_id.as_str().to_owned()) + .collect::>(); + let signals = self + .repository + .credential_runtime_signals(&ids) + .await + .map_err(|_| provider_unavailable("load account in-flight signals"))?; + Ok(signals + .into_iter() + .filter_map(|signal| { + let account_id = ProviderAccountId::new(signal.resource_id).ok()?; + Some((account_id, signal.in_flight)) + }) + .collect()) + }) + } } fn provider_unavailable(operation: &'static str) -> ProviderStoreError { diff --git a/backend/crates/gateway-store/tests/postgres/observability/mod.rs b/backend/crates/gateway-store/tests/postgres/observability/mod.rs index 53a1d6e6d..c0e9c158a 100644 --- a/backend/crates/gateway-store/tests/postgres/observability/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/observability/mod.rs @@ -855,6 +855,30 @@ impl ProviderCooldownPort for StaticCooldowns { ) -> BoxFuture<'a, Result> { Box::pin(async { Ok(false) }) } + + fn record_capacity_failure<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _window: Duration, + _in_flight: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async { Ok(0) }) + } + + fn clear_after_success<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _through_revision: gateway_core::account::CredentialRevision, + ) -> BoxFuture<'a, Result<(), ProviderStoreError>> { + Box::pin(async { Ok(()) }) + } + + fn capacity_peak_in_flight<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + ) -> BoxFuture<'a, Result, ProviderStoreError>> { + Box::pin(async { Ok(None) }) + } } fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown { diff --git a/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs b/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs index 96b0e9e92..bfb3f12f0 100644 --- a/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/provider_accounts/mod.rs @@ -852,9 +852,9 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u sort: None, }, AccountRuntimeSnapshot { - rate_limited_until: BTreeMap::from([( + cooldown: BTreeMap::from([( "acct_alpha".to_owned(), - now + TimeDelta::minutes(5), + SystemTime::from(now + TimeDelta::minutes(5)).into(), )]), in_flight: None, }, @@ -3387,3 +3387,83 @@ async fn model_access_only_batch_update_preserves_other_settings_and_survives_re assert_eq!(loaded.summary.model_access, AccountModelAccess::all()); database.close().await; } + +#[tokio::test] +async fn adaptive_concurrency_uses_latest_locked_settings_without_overwriting_admin_fields() { + let Some(database) = TestDatabase::create("adaptive_concurrency").await else { + return; + }; + let repository = PgProviderAccountRepository::new(database.pool.clone()); + let store = admin_account_store(&database.pool); + let id = ProviderAccountId::new("acct_adaptive").expect("id"); + repository + .insert_provider_account(account(id.as_str(), "adaptive-user")) + .await + .expect("account"); + let group = "grp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, 'kept group', '#112233FF', now(), now())") + .bind(group).execute(&database.pool).await.expect("group"); + sqlx::query("insert into account_group_accounts (account_group_id, provider_account_id, created_at) values ($1, $2, now())") + .bind(group).bind(id.as_str()).execute(&database.pool).await.expect("membership"); + for (enabled, account_limit, default_limit, expected_limit, changes) in [ + (false, Some(8_i64), 10_i64, Some(8_i64), false), + (true, Some(2), 10, Some(2), false), + (true, None, 2, None, false), + (true, None, 10, Some(3), true), + ] { + let mut admin = database.pool.begin().await.expect("admin transaction"); + sqlx::query("update runtime_settings set max_concurrent_per_account = $1 where id = 1") + .bind(default_limit) + .execute(&mut *admin) + .await + .expect("latest default"); + let worker_store = store.clone(); + let worker_id = id.clone(); + let worker = tokio::spawn(async move { + worker_store + .lower_concurrency_limit( + &worker_id, + gateway_core::account::AccountConcurrencyLimit::new(3).expect("limit"), + &MutationContext { + actor: MutationActor::System, + request_id: "adaptive-worker".to_owned(), + }, + ) + .await + .expect("atomic adaptation") + }); + // 管理员持有设置锁时提交新的账号事实,worker 必须依据提交后的值判断。 + sqlx::query("update provider_accounts set enabled = $2, concurrency_limit = $3, weight = 7, notes = 'administrator edit' where id = $1") + .bind(id.as_str()).bind(enabled).bind(account_limit).execute(&mut *admin).await.expect("concurrent admin edit"); + admin.commit().await.expect("commit admin edit"); + let result = worker.await.expect("worker"); + assert_eq!(result.is_some(), changes); + let actual = repository + .load_provider_account(id.as_str()) + .await + .expect("read account") + .expect("account"); + assert_eq!(actual.summary.enabled, enabled); + assert_eq!( + actual + .summary + .concurrency_limit + .map(|value| i64::from(value.get())), + expected_limit + ); + assert_eq!(actual.summary.weight.get(), 7); + assert_eq!(actual.summary.notes.as_deref(), Some("administrator edit")); + assert_eq!( + account_group_ids(&database.pool, id.as_str()).await, + vec![group.to_owned()] + ); + } + let audited_fields: Vec> = sqlx::query_scalar( + "select changed_fields from admin_audit_events where action = 'adapt_concurrency'", + ) + .fetch_all(&database.pool) + .await + .expect("audit"); + assert_eq!(audited_fields, vec![vec!["concurrency_limit".to_owned()]]); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/runtime_settings.rs b/backend/crates/gateway-store/tests/postgres/runtime_settings.rs index 10d22bc53..988c163cd 100644 --- a/backend/crates/gateway-store/tests/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/tests/postgres/runtime_settings.rs @@ -27,6 +27,13 @@ fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettingsUpdate { usage_retention_days: 31, ops_event_retention_days: 30, audit_retention_days: 90, + account_auto_freeze_enabled: true, + account_auto_freeze_threshold: 12, + account_auto_freeze_window_seconds: 600, + account_auto_freeze_duration_seconds: 7_200, + account_auto_freeze_probe_enabled: true, + account_auto_freeze_probe_model: None, + account_auto_freeze_adaptive_concurrency: true, } } @@ -46,6 +53,30 @@ fn runtime_settings_reject_invalid_model_mapping() { assert!(settings.validate().is_err()); } +#[test] +fn runtime_settings_reject_out_of_range_auto_freeze() { + for update in [ + RuntimeSettingsUpdate { + account_auto_freeze_threshold: 1, + ..settings_with_margin(3_600) + }, + RuntimeSettingsUpdate { + account_auto_freeze_window_seconds: 59, + ..settings_with_margin(3_600) + }, + RuntimeSettingsUpdate { + account_auto_freeze_duration_seconds: 299, + ..settings_with_margin(3_600) + }, + RuntimeSettingsUpdate { + account_auto_freeze_probe_model: Some(" pad ".to_owned()), + ..settings_with_margin(3_600) + }, + ] { + assert!(update.validate().is_err()); + } +} + #[test] fn runtime_settings_reject_non_semver_client_min() { let settings = RuntimeSettingsUpdate { @@ -238,3 +269,52 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() { assert!(snapshot.config_revision > before.config_revision); database.close().await; } + +#[tokio::test] +async fn auto_freeze_defaults_off_and_migration_preserves_explicit_opt_in() { + let Some(database) = TestDatabase::create("freeze_opt_in").await else { + return; + }; + let repository = PgRuntimeSettingsRepository::new(database.pool.clone()); + assert!( + !repository + .load_runtime_settings() + .await + .expect("default settings") + .account_auto_freeze_enabled + ); + sqlx::query("update runtime_settings set account_auto_freeze_enabled = true where id = 1") + .execute(&database.pool) + .await + .expect("old default"); + let migration = include_str!("../../../../migrations/0011_account_auto_freeze_opt_in.sql"); + sqlx::raw_sql(migration) + .execute(&database.pool) + .await + .expect("migrate untouched default"); + assert!( + !repository + .load_runtime_settings() + .await + .expect("settings") + .account_auto_freeze_enabled + ); + sqlx::query("update runtime_settings set account_auto_freeze_enabled = true where id = 1") + .execute(&database.pool) + .await + .expect("explicit opt-in"); + sqlx::query("insert into admin_audit_events (id, actor_kind, actor_ref, action, entity_kind, entity_ref, changed_fields, created_at) values ('freeze-opt-in', 'system', 'test', 'settings.replace', 'runtime_settings', '1', array['account_auto_freeze'], now())") + .execute(&database.pool).await.expect("saved settings audit"); + sqlx::raw_sql(migration) + .execute(&database.pool) + .await + .expect("preserve opt-in"); + assert!( + repository + .load_runtime_settings() + .await + .expect("settings") + .account_auto_freeze_enabled + ); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/redis/credential_cooldown.rs b/backend/crates/gateway-store/tests/redis/credential_cooldown.rs index 0593dddfb..c5a1b2317 100644 --- a/backend/crates/gateway-store/tests/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/tests/redis/credential_cooldown.rs @@ -3,7 +3,9 @@ use std::time::{Duration as StdDuration, SystemTime}; use chrono::{DateTime, Duration, Utc}; use gateway_core::{ account::{CredentialRevision, ProviderAccountId}, - provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown}, + provider_ports::{ + ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown, + }, }; use gateway_store::{ Revision, @@ -18,6 +20,7 @@ fn credential_cooldown_is_revision_fenced() { provider_account_id: "account-1".to_owned(), credential_revision: Revision::new(2).expect("positive revision"), cooldown_until: Utc::now() + Duration::seconds(30), + kind: ProviderCooldownKind::RateLimit, }; assert_eq!(cooldown.credential_revision.get(), 2); } @@ -159,6 +162,7 @@ async fn credential_cooldown_read_removes_expired_grace_key() { provider_account_id: "acct_cooldown_expiry".to_owned(), credential_revision: Revision::new(1).expect("positive revision"), cooldown_until: millisecond_precision(cooldown_until), + kind: ProviderCooldownKind::RateLimit, }; repository .cache_credential_cooldown(&cooldown) @@ -284,6 +288,7 @@ fn cooldown(account_id: &str, revision: u64, seconds: i64) -> CredentialCooldown provider_account_id: account_id.to_owned(), credential_revision: Revision::new(revision).expect("positive revision"), cooldown_until: millisecond_precision(Utc::now() + Duration::seconds(seconds)), + kind: ProviderCooldownKind::RateLimit, } } @@ -311,3 +316,233 @@ async fn namespace_keys(connection: &mut ConnectionManager, namespace: &str) -> .await .expect("list isolated cooldown keys") } + +fn runtime_store( + repository: &RedisCredentialCooldownRepository, + connection: ConnectionManager, + namespace: &str, +) -> gateway_store::redis::RedisAdminAccountRuntimeStore { + gateway_store::redis::RedisAdminAccountRuntimeStore::new( + repository.clone(), + gateway_store::redis::RedisCredentialLeaseRepository::new(connection, namespace) + .expect("lease repository"), + ) +} + +#[tokio::test] +async fn capacity_freeze_survives_due_time_and_runtime_restart_until_probe_succeeds() { + use gateway_admin::ports::store::AccountRuntimeStore as _; + let Some((repository, mut connection, namespace)) = repository().await else { + return; + }; + let frozen = CredentialCooldown { + kind: ProviderCooldownKind::CapacityFreezeProbe, + cooldown_until: millisecond_precision(Utc::now() - Duration::seconds(90)), + ..cooldown("acct_probe_due", 1, 30) + }; + repository + .cache_credential_cooldown(&frozen) + .await + .expect("write due freeze"); + let runtime = runtime_store(&repository, connection.clone(), &namespace); + let snapshot = runtime.active_rate_limits().await.expect("status snapshot"); + assert!(snapshot.cooldown[&frozen.provider_account_id].is_active(SystemTime::now())); + assert!( + snapshot.cooldown[&frozen.provider_account_id] + .kind + .requires_probe() + ); + let key = namespace_keys(&mut connection, &namespace) + .await + .into_iter() + .find(|key| key.ends_with(":cooldown")) + .expect("freeze key"); + let ttl: i64 = redis::cmd("PTTL") + .arg(key) + .query_async(&mut connection) + .await + .expect("TTL"); + assert_eq!( + ttl, -1, + "awaiting probe must not expire while worker is unavailable" + ); + drop(runtime); + let restarted = runtime_store(&repository, connection, &namespace); + let freezes = restarted + .active_freezes() + .await + .expect("due freezes after restart"); + assert_eq!(freezes.len(), 1); + assert!( + restarted + .finish_freeze( + &frozen.provider_account_id, + &freezes[&frozen.provider_account_id], + None + ) + .await + .expect("probe completion") + ); + assert!( + repository + .read_credential_cooldown(&frozen.provider_account_id) + .await + .expect("read") + .is_none() + ); +} + +#[tokio::test] +async fn stale_probe_cannot_recreate_manual_recovery_or_clear_or_extend_new_freeze() { + use gateway_admin::ports::store::AccountRuntimeStore as _; + let Some((repository, connection, namespace)) = repository().await else { + return; + }; + let frozen = CredentialCooldown { + kind: ProviderCooldownKind::CapacityFreezeProbe, + ..cooldown("acct_probe_generation", 1, 30) + }; + let runtime = runtime_store(&repository, connection, &namespace); + repository + .cache_credential_cooldown(&frozen) + .await + .expect("freeze"); + let original = + runtime.active_freezes().await.expect("snapshot")[&frozen.provider_account_id].clone(); + repository + .delete_account_cooldowns(&frozen.provider_account_id) + .await + .expect("manual recovery"); + let postponed = Utc::now() + Duration::hours(2); + assert!( + !runtime + .finish_freeze(&frozen.provider_account_id, &original, Some(postponed)) + .await + .expect("stale failure") + ); + assert!( + repository + .read_credential_cooldown(&frozen.provider_account_id) + .await + .expect("read") + .is_none() + ); + // 同凭据版本、相同截止时间的新冻结仍必须视为不同代次。 + repository + .cache_credential_cooldown(&frozen) + .await + .expect("new freeze"); + assert!( + !runtime + .finish_freeze(&frozen.provider_account_id, &original, None) + .await + .expect("stale success") + ); + assert!( + !runtime + .finish_freeze(&frozen.provider_account_id, &original, Some(postponed)) + .await + .expect("stale failure") + ); + let current = + runtime.active_freezes().await.expect("new snapshot")[&frozen.provider_account_id].clone(); + assert_ne!(current.generation, original.generation); + assert!( + runtime + .finish_freeze(&frozen.provider_account_id, ¤t, Some(postponed)) + .await + .expect("postpone current freeze") + ); + assert!( + !runtime + .finish_freeze(&frozen.provider_account_id, ¤t, None) + .await + .expect("result from previous probe cycle") + ); + let extended = runtime.active_freezes().await.expect("extended snapshot") + [&frozen.provider_account_id] + .clone(); + assert!(extended.until >= current.until); + assert!( + runtime + .finish_freeze(&frozen.provider_account_id, &extended, None) + .await + .expect("recover current cycle") + ); +} + +#[tokio::test] +async fn ordinary_success_and_later_429_preserve_capacity_freeze_and_peak_evidence() { + let Some((repository, _connection, _namespace)) = repository().await else { + return; + }; + let id = ProviderAccountId::new("acct_late_success").expect("account id"); + let revision = CredentialRevision::new(1).expect("revision"); + repository + .record_capacity_failure(&id, StdDuration::from_secs(600), 8) + .await + .expect("failure evidence"); + let frozen = CredentialCooldown { + kind: ProviderCooldownKind::CapacityFreezeProbe, + ..cooldown(id.as_str(), 1, 30) + }; + repository + .cache_credential_cooldown(&frozen) + .await + .expect("freeze"); + repository + .clear_after_success(&id, revision) + .await + .expect("late success"); + assert_eq!( + repository.capacity_peak_in_flight(&id).await.expect("peak"), + Some(8) + ); + assert!( + !repository + .cache_credential_cooldown(&cooldown(id.as_str(), 1, 60)) + .await + .expect("429 must not downgrade freeze") + ); + assert_eq!( + repository + .read_credential_cooldown(id.as_str()) + .await + .expect("read"), + Some(frozen) + ); +} + +#[tokio::test] +async fn ordinary_success_still_clears_rate_limit_and_unfrozen_failure_evidence() { + let Some((repository, _connection, _namespace)) = repository().await else { + return; + }; + let id = ProviderAccountId::new("acct_success_429").expect("id"); + repository + .record_capacity_failure(&id, StdDuration::from_secs(600), 3) + .await + .expect("failure evidence"); + repository + .cache_credential_cooldown(&cooldown(id.as_str(), 1, 30)) + .await + .expect("429"); + repository + .clear_after_success(&id, CredentialRevision::new(1).expect("revision")) + .await + .expect("success"); + assert!( + repository + .read_credential_cooldown(id.as_str()) + .await + .expect("read") + .is_none() + ); + assert!( + repository + .capacity_peak_in_flight(&id) + .await + .expect("peak") + .is_none() + ); +} diff --git a/backend/crates/providers/openai/src/credential/quota/mod.rs b/backend/crates/providers/openai/src/credential/quota/mod.rs index 43b4ab9d1..b31b4d0a2 100644 --- a/backend/crates/providers/openai/src/credential/quota/mod.rs +++ b/backend/crates/providers/openai/src/credential/quota/mod.rs @@ -24,7 +24,10 @@ use gateway_core::account::{ QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaObservationTouch, QuotaState, QuotaWriteOutcome, }; -use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort}; +use gateway_core::provider_ports::{ + ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderFreezePolicy, + ProviderLeasePort, ProviderRuntimePolicyPort, +}; use gateway_protocol::openai::events::{ ParsedRateLimits, RateLimitDetails, RateLimitWindow, parse_rate_limit_headers, }; @@ -184,10 +187,17 @@ pub struct CodexCredentialQuotaService { http: Client, base_url: String, cooldowns: Arc, + leases: Arc, + runtime_policy: Arc, + /// 冻结策略的短 TTL 缓存:失败路径热读,避免每个容量错误都查询设置。 + freeze_policy_cache: Mutex>, scheduling: CodexQuotaSchedulingProjection, reset_consume_locks: Mutex>>>, } +/// 冻结策略缓存活跃期;过期后下一次容量错误重新读取运行时设置。 +const FREEZE_POLICY_CACHE_TTL: Duration = Duration::from_secs(30); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum QuotaRefreshAuthority { ObserveAccess, @@ -481,6 +491,8 @@ impl CodexCredentialQuotaService { http: Client, base_url: String, cooldowns: Arc, + leases: Arc, + runtime_policy: Arc, ) -> Self { Self { store: Arc::clone(repository.store()), @@ -489,11 +501,89 @@ impl CodexCredentialQuotaService { http, base_url, cooldowns, + leases, + runtime_policy, + freeze_policy_cache: Mutex::new(None), scheduling: CodexQuotaSchedulingProjection::default(), reset_consume_locks: Mutex::new(HashMap::new()), } } + /// 读取容量熔断策略(带短 TTL 缓存);读取失败退化为关闭,熔断不得放大失败。 + async fn freeze_policy(&self) -> ProviderFreezePolicy { + { + let cache = self.freeze_policy_cache.lock().await; + if let Some((policy, loaded_at)) = cache.as_ref() + && loaded_at.elapsed() < FREEZE_POLICY_CACHE_TTL + { + return policy.clone(); + } + } + let loaded = self.runtime_policy.load_freeze_policy().await.ok(); + let mut cache = self.freeze_policy_cache.lock().await; + if let Some(policy) = loaded { + *cache = Some((policy.clone(), Instant::now())); + return policy; + } + cache + .as_ref() + .filter(|(_, loaded_at)| loaded_at.elapsed() < FREEZE_POLICY_CACHE_TTL) + .map_or_else(ProviderFreezePolicy::disabled, |(policy, _)| policy.clone()) + } + + /// 容量熔断入口:滑动窗口内累计容量类失败,达到阈值即写入带 + /// `CapacityFreeze` 类别的账号级冷却。调度侧立即屏蔽该账号,恢复由 + /// freeze-recovery worker 处理;本路径只依赖 Redis 可丢失事实。 + pub async fn apply_capacity_failure(&self, account: &ProviderAccount, observed_at: SystemTime) { + let policy = self.freeze_policy().await; + if !policy.enabled() { + return; + } + let in_flight = self.current_in_flight(account.id()).await; + let Ok(count) = self + .cooldowns + .record_capacity_failure(account.id(), policy.window(), in_flight) + .await + else { + return; + }; + if count < policy.threshold() { + return; + } + let Some(until) = observed_at.checked_add(policy.freeze_duration()) else { + return; + }; + let cooldown = ProviderCooldown::new_with_kind( + account.id().clone(), + account.revision(), + until, + if policy.probe_enabled() { + ProviderCooldownKind::CapacityFreezeProbe + } else { + ProviderCooldownKind::CapacityFreeze + }, + ); + if self.cooldowns.put_if_later(cooldown).await.is_ok() { + tracing::warn!( + account_id = account.id().as_str(), + threshold = policy.threshold(), + window_seconds = policy.window().as_secs(), + freeze_seconds = policy.freeze_duration().as_secs(), + peak_in_flight = in_flight, + "账号容量熔断触发:冻结该账号一段时间", + ); + } + } + + async fn current_in_flight(&self, account_id: &ProviderAccountId) -> u32 { + self.leases + .account_in_flight(std::slice::from_ref(account_id)) + .await + .ok() + .and_then(|signals| signals.get(account_id).copied()) + .unwrap_or(0) + } + /// 查询当前账号由 Codex Desktop 暴露的主动额度重置卡。 pub async fn list_reset_credits( &self, @@ -607,7 +697,7 @@ impl CodexCredentialQuotaService { } /// 成功推理是额度可访问的权威证据,同时解除账号级 429 冷却。 - pub(crate) async fn record_successful_inference( + pub async fn record_successful_inference( &self, account: &ProviderAccount, observed_at: SystemTime, @@ -628,7 +718,7 @@ impl CodexCredentialQuotaService { } } self.cooldowns - .clear(account.id(), account.revision()) + .clear_after_success(account.id(), account.revision()) .await .map_err(|error| CodexCredentialQuotaError::Store { detail: error.to_string(), @@ -975,11 +1065,11 @@ impl CodexCredentialQuotaService { Ok(()) } - /// 读取账号当前是否处于临时限流(429)冷却,及到期时间。 - pub async fn rate_limited_until( + /// 读取有效的账号冷却事实;等待恢复探测的冻结到期后仍有效。 + pub async fn cooldown( &self, account_id: &ProviderAccountId, - ) -> Result, CodexCredentialQuotaError> { + ) -> Result, CodexCredentialQuotaError> { let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| { CodexCredentialQuotaError::Store { detail: error.to_string(), @@ -988,11 +1078,8 @@ impl CodexCredentialQuotaService { else { return Ok(None); }; - let until = cooldown.until(); - if until <= SystemTime::now() { - return Ok(None); - } - Ok(Some(until)) + let state = cooldown.scheduling_state(); + Ok(state.is_active(SystemTime::now()).then_some(state)) } /// 读取单账号最后一次落库的 Provider quota,并由 Codex 域解析展示窗口。 diff --git a/backend/crates/providers/openai/src/credential/selector.rs b/backend/crates/providers/openai/src/credential/selector.rs index e08945f23..db683bbc6 100644 --- a/backend/crates/providers/openai/src/credential/selector.rs +++ b/backend/crates/providers/openai/src/credential/selector.rs @@ -451,11 +451,7 @@ impl CodexCredentialSelector { let mut rate_limits = HashMap::with_capacity(accounts.len()); if !diagnostic { for account in &accounts { - let until = self - .quota - .rate_limited_until(account.id()) - .await - .unwrap_or(None); + let until = self.quota.cooldown(account.id()).await.unwrap_or(None); rate_limits.insert(account.id().clone(), until); } } @@ -487,7 +483,7 @@ impl CodexCredentialSelector { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }) @@ -1330,7 +1326,7 @@ fn affinity_selection_for_bound_account( }; match candidate .account - .status_projection(now, candidate.signals.rate_limited_until) + .status_projection(now, candidate.signals.cooldown) .status { AccountStatus::Normal => AffinitySelection::preferred(account_id), @@ -1349,7 +1345,7 @@ fn affinity_unavailable_reason( ) -> AffinityEscapeReason { match candidate .account - .status_projection(now, candidate.signals.rate_limited_until) + .status_projection(now, candidate.signals.cooldown) .status { AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted, diff --git a/backend/crates/providers/openai/src/lib.rs b/backend/crates/providers/openai/src/lib.rs index dbc98ec8d..49f030037 100644 --- a/backend/crates/providers/openai/src/lib.rs +++ b/backend/crates/providers/openai/src/lib.rs @@ -112,6 +112,8 @@ pub async fn initialize( http.clone(), config.base_url().to_owned(), ports.cooldowns(), + Arc::clone(&leases), + Arc::clone(&runtime_policy), )); let profile_statistics = Arc::new(CodexCredentialProfileService::new( repository.clone(), diff --git a/backend/crates/providers/openai/src/provider/execution.rs b/backend/crates/providers/openai/src/provider/execution.rs index 162fec634..0f6953e66 100644 --- a/backend/crates/providers/openai/src/provider/execution.rs +++ b/backend/crates/providers/openai/src/provider/execution.rs @@ -403,6 +403,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea response_origin: &request.response_origin, cyber_policy_scope: None, allows_account_state_mutation, + allows_capacity_feedback: !request.context.is_diagnostic_required_account(), }; let active_account = request.lease.account().clone(); let cookie_header = build_cookie_header(request.lease.cookies())?; @@ -577,6 +578,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream { response_origin: &response_origin, cyber_policy_scope: cyber_policy_scope.as_ref(), allows_account_state_mutation, + allows_capacity_feedback: !context.is_diagnostic_required_account(), }; let mut active_account = lease.account().clone(); let cookie_header = build_cookie_header(lease.cookies())?; diff --git a/backend/crates/providers/openai/src/provider/failure.rs b/backend/crates/providers/openai/src/provider/failure.rs index 93024dd4f..75de8acab 100644 --- a/backend/crates/providers/openai/src/provider/failure.rs +++ b/backend/crates/providers/openai/src/provider/failure.rs @@ -1,13 +1,15 @@ //! OpenAI 上游失败分类、恢复决策与稳定错误投影。 use super::*; -use crate::transport::diagnostics::is_capacity_error; /// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集。 /// /// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type。 /// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集。 +/// 容量拒绝影响短期调度健康度,但不证明账号凭据或额度失效。 const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[ + "server_is_overloaded", + "slow_down", "rate_limit_exceeded", "rate_limit_error", "server_error", @@ -32,17 +34,6 @@ fn openai_account_score_failure_reason(error: &ProviderError) -> Option<&str> { /// 返回该 OpenAI 失败是否属于 Smart 账号计分闭集。 #[doc(hidden)] pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool { - let client_error = error.client_visible_upstream_error(); - if is_capacity_error( - error - .upstream_code() - .map(OpaqueUpstreamValue::as_str) - .or_else(|| client_error.and_then(ClientVisibleUpstreamError::code)), - client_error.and_then(ClientVisibleUpstreamError::error_type), - client_error.map(ClientVisibleUpstreamError::message), - ) { - return false; - } openai_account_score_failure_reason(error).is_some_and(is_openai_account_score_failure_reason) } @@ -53,6 +44,7 @@ pub(super) struct MappedProviderFailure { /// 原始上游错误描述,仅在凭据错误状态下持久化。 pub(super) error_message: Option, pub(super) cyber_policy_failure: bool, + pub(super) upstream_capacity_failure: bool, pub(super) set_cookie_headers: Vec, pub(super) rate_limit_headers: Vec<(String, String)>, pub(super) observation: Option, @@ -67,6 +59,7 @@ impl MappedProviderFailure { account_failure: None, error_message: None, cyber_policy_failure: false, + upstream_capacity_failure: false, set_cookie_headers: Vec::new(), rate_limit_headers: Vec::new(), observation: None, @@ -211,6 +204,7 @@ pub(super) struct OpenAiFailureContext<'a> { pub(super) response_origin: &'a Url, pub(super) cyber_policy_scope: Option<&'a CodexCyberPolicyScope>, pub(super) allows_account_state_mutation: bool, + pub(super) allows_capacity_feedback: bool, } #[derive(Clone, Copy)] @@ -410,6 +404,13 @@ pub(super) async fn apply_failure( "Failed to persist OpenAI response cookies" ); } + // 探测只验证恢复,不能把探测自身的单请求并发当作业务负载证据。 + if context.allows_capacity_feedback && failure.upstream_capacity_failure { + context + .quota + .apply_capacity_failure(account, SystemTime::now()) + .await; + } } pub(super) fn schedule_authoritative_quota_refresh_after_failure( @@ -1269,6 +1270,12 @@ pub(super) fn map_upstream_failure( ), error_message: failure.client_message, cyber_policy_failure, + // 仅上游明确返回的容量/服务不可用计数;本地连接保护也会映射为 + // Unavailable,不能用通用 ProviderErrorKind 推断容量证据。 + upstream_capacity_failure: matches!( + category, + CodexFailureCategory::CapacityUnavailable | CodexFailureCategory::Unavailable + ), set_cookie_headers: failure.set_cookie_headers, rate_limit_headers: failure.rate_limit_headers, observation, diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index b494bf6ce..0b9e41d3f 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -1591,6 +1591,30 @@ impl ProviderCooldownPort for TestCooldown { ) -> BoxFuture<'a, Result> { Box::pin(async { Ok(false) }) } + + fn record_capacity_failure<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _window: Duration, + _in_flight: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async { Ok(0) }) + } + + fn clear_after_success<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _through_revision: gateway_core::account::CredentialRevision, + ) -> BoxFuture<'a, Result<(), ProviderStoreError>> { + Box::pin(async { Ok(()) }) + } + + fn capacity_peak_in_flight<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + ) -> BoxFuture<'a, Result, ProviderStoreError>> { + Box::pin(async { Ok(None) }) + } } struct TestRuntimePolicy; diff --git a/backend/crates/providers/openai/tests/credential/contract.rs b/backend/crates/providers/openai/tests/credential/contract.rs index 373b8b06b..50bc14fba 100644 --- a/backend/crates/providers/openai/tests/credential/contract.rs +++ b/backend/crates/providers/openai/tests/credential/contract.rs @@ -20,7 +20,8 @@ use gateway_core::engine::{ use gateway_core::lifecycle::CancellationToken; use gateway_core::policy::ClientApiKeyId; use gateway_core::provider_ports::{ - ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort, + ProviderCooldownPort, ProviderLeasePort, ProviderSessionAffinityKey, + ProviderSessionAffinityPort, }; use gateway_core::routing::{ ClientRoutingScope, FrozenAccountScope, ProviderKind, RuntimeAccount, RuntimeAccountDirectory, @@ -178,6 +179,8 @@ fn selector_with_runtime( http, OFFICIAL_CODEX_BASE_URL.to_owned(), cooldowns, + Arc::clone(&leases) as Arc, + crate::support::runtime_policy(), )); CodexCredentialSelector::new( ProviderKind::new("openai").expect("provider"), diff --git a/backend/crates/providers/openai/tests/credential/quota/capacity_freeze.rs b/backend/crates/providers/openai/tests/credential/quota/capacity_freeze.rs new file mode 100644 index 000000000..546f7f537 --- /dev/null +++ b/backend/crates/providers/openai/tests/credential/quota/capacity_freeze.rs @@ -0,0 +1,195 @@ +//! 容量熔断触发:高频容量类失败按滑动窗口计数冻结账号,成功调用清空证据。 + +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use gateway_core::provider_ports::{ + ProviderCooldownKind, ProviderCooldownPort as _, ProviderFreezePolicy, +}; +use provider_openai::credential::CodexCredentialQuotaService; + +use super::{MemoryAccountStore, create_account, wire_profile}; +use crate::support::{MemoryCooldownPort, StaticFreezePolicy, TestLeaseCoordinator}; + +fn freeze_policy(threshold: u32) -> ProviderFreezePolicy { + ProviderFreezePolicy::try_new(true, threshold, 600, 7_200, true, None, true) + .expect("valid freeze policy") +} + +async fn freeze_service( + store: &Arc, + policy: ProviderFreezePolicy, +) -> (Arc, Arc) { + let cooldowns = Arc::new(MemoryCooldownPort::new()); + let quota = Arc::new(CodexCredentialQuotaService::new( + store.repository(), + wire_profile(), + reqwest::Client::builder() + .no_proxy() + .build() + .expect("client"), + provider_openai::OFFICIAL_CODEX_BASE_URL.to_owned(), + Arc::clone(&cooldowns) as Arc, + Arc::new(TestLeaseCoordinator::default()), + StaticFreezePolicy::policy_port(policy), + )); + (quota, cooldowns) +} + +#[tokio::test] +async fn capacity_failures_below_threshold_do_not_freeze() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_capacity_below").await; + let account = store + .account("acct_capacity_below") + .expect("created account"); + let (quota, cooldowns) = freeze_service(&store, freeze_policy(3)).await; + + for _ in 0..2 { + quota + .apply_capacity_failure(&account, SystemTime::now()) + .await; + } + + assert_eq!( + cooldowns + .capacity_evidence(account.id()) + .map(|(count, _)| count), + Some(2) + ); + assert!( + quota + .cooldown(account.id()) + .await + .expect("cooldown read") + .is_none() + ); +} + +#[tokio::test] +async fn capacity_failures_reaching_threshold_write_capacity_freeze_cooldown() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_capacity_freeze").await; + let account = store + .account("acct_capacity_freeze") + .expect("created account"); + let (quota, cooldowns) = freeze_service(&store, freeze_policy(3)).await; + + for _ in 0..3 { + quota + .apply_capacity_failure(&account, SystemTime::now()) + .await; + } + + let until = quota + .cooldown(account.id()) + .await + .expect("cooldown read") + .expect("freeze cooldown written"); + let remaining = until + .until + .duration_since(SystemTime::now()) + .expect("freeze in the future"); + assert!( + remaining > Duration::from_secs(7_000) && remaining <= Duration::from_secs(7_200), + "freeze should last about 2 hours, got {remaining:?}" + ); + let cooldown = cooldowns.read(account.id()).await.expect("read cooldown"); + assert_eq!( + cooldown.expect("cooldown").kind(), + ProviderCooldownKind::CapacityFreezeProbe + ); +} + +#[tokio::test] +async fn disabled_freeze_policy_records_nothing() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_capacity_disabled").await; + let account = store + .account("acct_capacity_disabled") + .expect("created account"); + let (quota, cooldowns) = freeze_service(&store, ProviderFreezePolicy::disabled()).await; + + for _ in 0..5 { + quota + .apply_capacity_failure(&account, SystemTime::now()) + .await; + } + + assert!(cooldowns.capacity_evidence(account.id()).is_none()); + assert!( + quota + .cooldown(account.id()) + .await + .expect("cooldown read") + .is_none() + ); +} + +#[tokio::test] +async fn successful_inference_clears_capacity_evidence() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_capacity_success").await; + let account = store + .account("acct_capacity_success") + .expect("created account"); + let (quota, cooldowns) = freeze_service(&store, freeze_policy(100)).await; + + for _ in 0..3 { + quota + .apply_capacity_failure(&account, SystemTime::now()) + .await; + } + assert!(cooldowns.capacity_evidence(account.id()).is_some()); + + quota + .record_successful_inference(&account, SystemTime::now()) + .await + .expect("record success"); + + assert!(cooldowns.capacity_evidence(account.id()).is_none()); +} + +/// 阈值与窗口边界使用与迁移 check 约束一致的构造校验。 +#[test] +fn freeze_policy_rejects_out_of_range_configuration() { + for (threshold, window, duration) in [(1, 600, 7_200), (12, 59, 7_200), (12, 600, 299)] { + assert!( + ProviderFreezePolicy::try_new(true, threshold, window, duration, true, None, true) + .is_err() + ); + } + assert!( + ProviderFreezePolicy::try_new(true, 12, 600, 7_200, true, Some(" ".to_owned()), true) + .is_err() + ); + assert!(ProviderFreezePolicy::try_new(true, 12, 600, 7_200, true, None, true).is_ok()); +} + +#[tokio::test] +async fn in_flight_success_does_not_release_a_new_capacity_freeze() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_freeze_late_success").await; + let account = store.account("acct_freeze_late_success").expect("account"); + let (quota, cooldowns) = freeze_service(&store, freeze_policy(2)).await; + for _ in 0..2 { + quota + .apply_capacity_failure(&account, SystemTime::now()) + .await; + } + let before = cooldowns.read(account.id()).await.expect("freeze"); + quota + .record_successful_inference(&account, SystemTime::now()) + .await + .expect("late success"); + assert_eq!( + cooldowns.read(account.id()).await.expect("freeze remains"), + before + ); + assert_eq!( + cooldowns + .capacity_evidence(account.id()) + .map(|(count, _)| count), + Some(2) + ); +} diff --git a/backend/crates/providers/openai/tests/credential/quota/mod.rs b/backend/crates/providers/openai/tests/credential/quota/mod.rs index 3bee3a98b..ae25554e5 100644 --- a/backend/crates/providers/openai/tests/credential/quota/mod.rs +++ b/backend/crates/providers/openai/tests/credential/quota/mod.rs @@ -1,5 +1,6 @@ //! OpenAI 额度事实边界与展示快照回归。 +mod capacity_freeze; mod recovery; mod refresh_timing; mod scheduling; @@ -68,6 +69,8 @@ fn quota_service_with_base_url( http, base_url, Arc::new(crate::support::MemoryCooldownPort::new()), + Arc::new(crate::support::TestLeaseCoordinator::default()), + crate::support::runtime_policy(), ) } diff --git a/backend/crates/providers/openai/tests/credential/quota/scheduling.rs b/backend/crates/providers/openai/tests/credential/quota/scheduling.rs index c59da5043..0ac7f7693 100644 --- a/backend/crates/providers/openai/tests/credential/quota/scheduling.rs +++ b/backend/crates/providers/openai/tests/credential/quota/scheduling.rs @@ -84,7 +84,7 @@ async fn old_weekly_quota_should_outweigh_one_in_flight_title_request() { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, } diff --git a/backend/crates/providers/openai/tests/provider/contract.rs b/backend/crates/providers/openai/tests/provider/contract.rs index 2d4b12636..2d4641099 100644 --- a/backend/crates/providers/openai/tests/provider/contract.rs +++ b/backend/crates/providers/openai/tests/provider/contract.rs @@ -32,6 +32,7 @@ use gateway_core::operation::{ OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest, }; use gateway_core::policy::ClientApiKeyId; +use gateway_core::provider_ports::ProviderLeasePort; use gateway_core::routing::{ ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ModelServiceTier, ProviderKind, ProviderModel, PublicModelId, RoutingContext, RuntimeAccount, @@ -227,6 +228,31 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases( leases: Arc, stream_max_retries: u32, ) -> (CodexProvider, Arc) { + let (provider, quota, _) = provider_and_quota_with_runtime_ports( + store, + session_affinity, + base_url, + leases, + stream_max_retries, + Arc::new(MemoryCooldownPort::new()), + crate::support::runtime_policy(), + ); + (provider, quota) +} + +fn provider_and_quota_with_runtime_ports( + store: &Arc, + session_affinity: Arc, + base_url: String, + leases: Arc, + stream_max_retries: u32, + cooldowns: Arc, + policy: Arc, +) -> ( + CodexProvider, + Arc, + Arc, +) { let profile = wire_profile(); let http = reqwest::Client::builder() .no_proxy() @@ -245,7 +271,9 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases( profile.clone(), http.clone(), base_url.clone(), - Arc::new(MemoryCooldownPort::new()), + cooldowns, + Arc::clone(&leases) as Arc, + policy, )); let account_feedback = Arc::new(AccountFeedbackStats::default()); let selector = Arc::new(CodexCredentialSelector::new( @@ -268,11 +296,11 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases( http, profile, base_url, - websocket_pool, + Arc::clone(&websocket_pool), stream_max_retries, ) .expect("official OpenAI provider"); - (provider, quota) + (provider, quota, websocket_pool) } async fn create_account(store: &Arc, id: &str) { @@ -5476,7 +5504,11 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret create_account(&store, account_id).await; let server = MockServer::start().await; let body = json!({"error": { - "code": if status == 503 { Some("server_is_overloaded") } else { None }, + "code": match status { + 429 => Some("slow_down"), + 503 => Some("server_is_overloaded"), + _ => None, + }, "type": "server_error", "message": "Selected model is at capacity. Please try a different model.", "extension": {"preserved": true} @@ -5518,7 +5550,7 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret max_retries, initial_delay, max_delay, }) if max_retries.get() == 3 && initial_delay == Duration::from_secs(8) && max_delay == Duration::from_secs(8)) ); - assert!(!provider_openai::openai_failure_affects_account_score( + assert!(provider_openai::openai_failure_affects_account_score( &error )); assert_eq!( @@ -5540,6 +5572,196 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret } } +fn capacity_websocket_operation() -> Operation { + let payload = ProtocolPayload::json_object( + "openai", + Map::from_iter([ + ("model".to_owned(), json!("gpt-5.4")), + ("input".to_owned(), json!("hello")), + ("store".to_owned(), json!(false)), + ("session_id".to_owned(), json!("capacity-feedback-session")), + ]), + ) + .expect("WebSocket payload") + .with_context(Map::from_iter([( + "downstream_websocket_connection_id".to_owned(), + json!("ws_capacity_feedback"), + )])); + Operation::Generate( + GenerateRequest::from_protocol_payload(payload).with_provider_session_state( + ProviderSessionState::new( + "openai", + Map::from_iter([ + ("account_id".to_owned(), json!("acct_provider_contract")), + ( + "conversation_id".to_owned(), + json!("capacity-feedback-session"), + ), + ("continuation_scope".to_owned(), json!("persisted")), + ]), + ) + .expect("WebSocket session scope"), + ), + ) +} + +fn provider_with_capacity_tracking( + store: &Arc, + base_url: String, + cooldowns: Arc, +) -> (CodexProvider, Arc) { + let (provider, _, pool) = provider_and_quota_with_runtime_ports( + store, + Arc::new(MemorySessionAffinity::default()), + base_url, + Arc::new(TestLeaseCoordinator::default()), + 0, + cooldowns, + crate::support::StaticFreezePolicy::policy_port( + gateway_core::provider_ports::ProviderFreezePolicy::try_new( + true, 12, 600, 7_200, true, None, true, + ) + .expect("freeze policy"), + ), + ); + (provider, pool) +} + +#[tokio::test] +async fn capacity_feedback_counts_business_rejections_but_excludes_diagnostic_probes() { + use gateway_core::provider_ports::ProviderCooldownPort as _; + + for websocket in [false, true] { + for (status, code) in [ + (429, "slow_down"), + (503, "server_is_overloaded"), + (500, "server_error"), + ] { + for diagnostic in [false, true] { + let store = Arc::new(MemoryAccountStore::default()); + let account_id = "acct_provider_contract"; + create_account(&store, account_id).await; + let account = store.account(account_id).expect("account"); + let cooldowns = Arc::new(MemoryCooldownPort::new()); + let server = MockServer::start().await; + Mock::given(method(if websocket { "GET" } else { "POST" })) + .and(path("/codex/responses")) + .respond_with( + ResponseTemplate::new(status).set_body_json(json!({"error": { + "code": code, + "message": "Upstream temporarily unavailable" + }})), + ) + .expect(2) + .mount(&server) + .await; + let (provider, _) = + provider_with_capacity_tracking(&store, server.uri(), Arc::clone(&cooldowns)); + // 同时验证窗口证据已过期与尚未过期:探测不能重建峰值,也不能改写原计数。 + for existing_evidence in [None, Some((4, 20))] { + if let Some((count, peak)) = existing_evidence { + for _ in 0..count { + cooldowns + .record_capacity_failure( + account.id(), + Duration::from_secs(600), + peak, + ) + .await + .expect("seed normal-request evidence"); + } + } + let before = cooldowns.capacity_evidence(account.id()); + let attempt = if diagnostic { + diagnostic_context("req_capacity_feedback_probe", account_id) + } else { + context("req_capacity_feedback_business", CancellationToken::new()) + }; + let operation = if websocket { + capacity_websocket_operation() + } else { + http_generate_operation() + }; + let mut stream = provider + .execute(planned_request("openai", operation), attempt) + .await + .expect("prepare upstream attempt"); + let error = loop { + match stream.next().await { + Some(Ok(_)) => {} + Some(Err(error)) => break error, + None => panic!("expected upstream rejection"), + } + }; + assert_eq!( + error.upstream_status(), + Some(status), + "unexpected upstream error: {error:?}" + ); + let after = cooldowns.capacity_evidence(account.id()); + if diagnostic { + assert_eq!(after, before, "probe must preserve capacity count and peak"); + } else { + assert_eq!( + after.map(|(count, _)| count), + Some(before.map_or(1, |(count, _)| count + 1)) + ); + } + } + } + } + } +} + +#[tokio::test] +async fn local_websocket_connection_cancellation_does_not_supply_capacity_evidence() { + let store = Arc::new(MemoryAccountStore::default()); + let account_id = "acct_provider_contract"; + create_account(&store, account_id).await; + let account = store.account(account_id).expect("account"); + let cooldowns = Arc::new(MemoryCooldownPort::new()); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let base_url = format!("http://{}", listener.local_addr().expect("address")); + let (provider, pool) = + provider_with_capacity_tracking(&store, base_url, Arc::clone(&cooldowns)); + let operation = capacity_websocket_operation(); + let mut stream = provider + .execute( + planned_request("openai", operation), + context("req_capacity_local", CancellationToken::new()), + ) + .await + .expect("prepare WebSocket attempt"); + let attempt = tokio::spawn(async move { + loop { + match stream.next().await { + Some(Ok(_)) => {} + Some(Err(error)) => break error, + None => panic!("cancelled opening must fail"), + } + } + }); + let (mut opening, _) = timeout(Duration::from_secs(5), listener.accept()) + .await + .expect("opening deadline") + .expect("opening"); + read_http_request(&mut opening).await; + // 复现账号更新驱逐正在建连的连接,未收到任何上游容量拒绝。 + pool.evict_account(account_id).await; + let error = timeout(Duration::from_secs(5), attempt) + .await + .expect("cancelled attempt deadline") + .expect("attempt task"); + assert_eq!(error.kind(), ProviderErrorKind::Unavailable); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + assert_eq!( + error.diagnostic().and_then(|diagnostic| diagnostic.code()), + Some("shared_connect_failed") + ); + assert_eq!(cooldowns.capacity_evidence(account.id()), None); + pool.shutdown().await; +} + #[tokio::test] async fn websocket_usage_limit_rejection_preserves_quota_state_for_account_rotation() { let store = Arc::new(MemoryAccountStore::default()); @@ -5578,7 +5800,12 @@ async fn websocket_usage_limit_rejection_preserves_quota_state_for_account_rotat #[tokio::test] async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_transports() { - for use_websocket in [false, true] { + for (use_websocket, code) in [ + (false, "server_is_overloaded"), + (true, "server_is_overloaded"), + (false, "slow_down"), + (true, "slow_down"), + ] { for semantic_output in [false, true] { let store = Arc::new(MemoryAccountStore::default()); create_account(&store, "acct_provider_contract").await; @@ -5588,7 +5815,7 @@ async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_t if semantic_output { events.push(json!({"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello"})); } - let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": "server_is_overloaded", "message": "Selected model is at capacity. Please try a different model."}}}); + let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": code, "message": "Selected model is at capacity. Please try a different model."}}}); events.push(original.clone()); let (base_url, _http_server, websocket_server) = if use_websocket { let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); @@ -5653,7 +5880,7 @@ async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_t assert_eq!(error.replay_is_safe(), !semantic_output); assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable); assert_eq!(error.pre_delivery_retry().is_some(), !semantic_output); - assert!(!provider_openai::openai_failure_affects_account_score( + assert!(provider_openai::openai_failure_affects_account_score( &error )); assert_eq!( diff --git a/backend/crates/providers/openai/tests/provider/failure.rs b/backend/crates/providers/openai/tests/provider/failure.rs index e0ac0aa3a..cd8aea2c6 100644 --- a/backend/crates/providers/openai/tests/provider/failure.rs +++ b/backend/crates/providers/openai/tests/provider/failure.rs @@ -1,4 +1,6 @@ +use std::num::NonZeroU32; use std::sync::Arc; +use std::time::Duration; use futures::{StreamExt, stream}; use gateway_core::account::{AccountFeedbackStats, ProviderAccountId}; @@ -45,7 +47,7 @@ fn deliver_error_with_openai_feedback( } #[test] -fn account_score_failure_filter_should_exclude_request_scoped_capacity_errors() { +fn account_score_failure_filter_should_accept_capacity_errors_with_scored_reasons() { for code in ["server_is_overloaded", "slow_down", "server_error"] { let error = sent_error(ProviderErrorKind::Unavailable, Some(code)) .with_client_visible_upstream_error(ClientVisibleUpstreamError::new( @@ -53,13 +55,15 @@ fn account_score_failure_filter_should_exclude_request_scoped_capacity_errors() Some("server_error".to_owned()), Some("server_error".to_owned()), )); - assert!(!openai_failure_affects_account_score(&error)); + assert!(openai_failure_affects_account_score(&error)); } } #[test] fn account_score_failure_filter_should_accept_the_closed_reason_list() { for code in [ + "server_is_overloaded", + "slow_down", "rate_limit_exceeded", "rate_limit_error", "server_error", @@ -82,14 +86,20 @@ fn account_score_failure_filter_should_normalize_case_and_whitespace() { #[test] fn account_score_failure_filter_should_use_a_structured_type_when_code_is_absent() { - let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent) - .with_client_visible_upstream_error(ClientVisibleUpstreamError::new( - "upstream unavailable", - None, - Some("SERVICE_UNAVAILABLE_ERROR".to_owned()), - )); + for error_type in [ + "SERVICE_UNAVAILABLE_ERROR", + "SERVER_IS_OVERLOADED", + "SLOW_DOWN", + ] { + let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent) + .with_client_visible_upstream_error(ClientVisibleUpstreamError::new( + "upstream unavailable", + None, + Some(error_type.to_owned()), + )); - assert!(openai_failure_affects_account_score(&error)); + assert!(openai_failure_affects_account_score(&error)); + } } #[test] @@ -185,17 +195,27 @@ fn openai_feedback_should_not_amplify_repeated_client_configuration_errors() { } #[test] -fn openai_feedback_should_not_penalize_an_account_for_server_overload() { - let feedback = Arc::new(AccountFeedbackStats::default()); - let provider = ProviderKind::new("openai").expect("provider"); - let account = ProviderAccountId::new("acct_overloaded").expect("account"); +fn openai_feedback_should_penalize_capacity_rejections_even_with_transient_retry() { + for code in ["server_is_overloaded", "slow_down"] { + let feedback = Arc::new(AccountFeedbackStats::default()); + let provider = ProviderKind::new("openai").expect("provider"); + let account = ProviderAccountId::new("acct_overloaded").expect("account"); + let error = sent_error(ProviderErrorKind::UpstreamCapacityUnavailable, Some(code)) + .with_replay_safe() + .with_transient_retry( + NonZeroU32::new(3).expect("retry count"), + Duration::from_millis(500), + Duration::from_secs(8), + ); - deliver_error_with_openai_feedback( - &feedback, - &provider, - &account, - sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded")), - ); + deliver_error_with_openai_feedback(&feedback, &provider, &account, error); - assert_eq!(feedback.scheduling_signals(&provider, &account).0, None); + assert!( + feedback + .scheduling_signals(&provider, &account) + .0 + .is_some_and(|failure_rate| failure_rate > 0), + "capacity rejection did not affect the account score: {code}" + ); + } } diff --git a/backend/crates/providers/openai/tests/support.rs b/backend/crates/providers/openai/tests/support.rs index cb51a8bc6..18f8c9b98 100644 --- a/backend/crates/providers/openai/tests/support.rs +++ b/backend/crates/providers/openai/tests/support.rs @@ -20,11 +20,11 @@ use gateway_core::error::{StoreError, StoreErrorKind}; use gateway_core::policy::ClientApiKeyId; use gateway_core::provider_ports::{ ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort, - ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest, - ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest, - ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey, - ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions, - ProviderStoreError, + ProviderCooldownScope, ProviderFreezePolicy, ProviderLeaseAcquisition, ProviderLeasePort, + ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort, + ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderScopedCooldown, + ProviderSessionAffinityKey, ProviderSessionAffinityPort, ProviderSessionExclusionPort, + ProviderSessionExclusions, ProviderStoreError, }; use gateway_core::routing::ProviderKind; use provider_openai::credential::{ @@ -615,7 +615,7 @@ impl ProviderLeasePort for TestLeaseCoordinator { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -956,6 +956,36 @@ pub(crate) fn runtime_policy() -> Arc { Arc::new(StaticRuntimePolicy) } +/// 冻结策略可配置的运行时策略 fake;容量熔断触发测试用。 +pub(crate) struct StaticFreezePolicy(ProviderFreezePolicy); + +impl StaticFreezePolicy { + /// 构造带固定冻结策略的端口句柄;命名沿用测试构造器语义。 + #[must_use] + pub(crate) fn policy_port(policy: ProviderFreezePolicy) -> Arc { + Arc::new(Self(policy)) + } +} + +impl ProviderRuntimePolicyPort for StaticFreezePolicy { + fn load_refresh_policy( + &self, + ) -> BoxFuture<'_, Result> { + Box::pin(async { + ProviderRefreshPolicy::try_new( + Duration::from_secs(60 * 60), + NonZeroU32::new(2).expect("positive concurrency"), + ) + }) + } + + fn load_freeze_policy( + &self, + ) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(self.0.clone()) }) + } +} + pub(crate) fn secret(access_token: &str) -> CodexOAuthSecret { CodexOAuthSecret { access_token: SecretString::from(access_token.to_owned()), @@ -972,11 +1002,13 @@ pub(crate) fn account_policy() -> gateway_core::account::AccountSelectionPolicy ) } -/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector -/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。 +/// 内存 `ProviderCooldownPort`:实现 `read`/`put_if_later` 与容量失败计数 +/// 证据(openai selector 429 冷却与容量熔断路径用),scope 变体测试不涉及, +/// 返回占位。 #[derive(Clone, Default)] pub(crate) struct MemoryCooldownPort { pub(crate) cooldowns: Arc>>, + capacity_failures: Arc>>, } impl MemoryCooldownPort { @@ -984,6 +1016,16 @@ impl MemoryCooldownPort { pub(crate) fn new() -> Self { Self::default() } + + /// 测试断言用:窗口内累计失败次数与在途峰值。 + #[must_use] + pub(crate) fn capacity_evidence(&self, account_id: &ProviderAccountId) -> Option<(u32, u32)> { + self.capacity_failures + .lock() + .expect("capacity failures lock") + .get(account_id) + .copied() + } } impl ProviderCooldownPort for MemoryCooldownPort { @@ -1057,4 +1099,57 @@ impl ProviderCooldownPort for MemoryCooldownPort { ) -> BoxFuture<'a, Result> { Box::pin(async { Ok(false) }) } + + fn record_capacity_failure<'a>( + &'a self, + account_id: &'a ProviderAccountId, + _window: Duration, + in_flight: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut failures = self + .capacity_failures + .lock() + .expect("capacity failures lock"); + let entry = failures.entry(account_id.clone()).or_insert((0, 0)); + entry.0 = entry.0.saturating_add(1); + entry.1 = entry.1.max(in_flight); + Ok(entry.0) + }) + } + + fn clear_after_success<'a>( + &'a self, + account_id: &'a ProviderAccountId, + _through_revision: gateway_core::account::CredentialRevision, + ) -> BoxFuture<'a, Result<(), ProviderStoreError>> { + Box::pin(async move { + let mut cooldowns = self.cooldowns.lock().expect("cooldown lock"); + if cooldowns.get(account_id).is_some_and(|value| { + value.kind().is_capacity_freeze() || value.credential_revision() > _through_revision + }) { + return Ok(()); + } + cooldowns.remove(account_id); + self.capacity_failures + .lock() + .expect("capacity failures lock") + .remove(account_id); + Ok(()) + }) + } + + fn capacity_peak_in_flight<'a>( + &'a self, + account_id: &'a ProviderAccountId, + ) -> BoxFuture<'a, Result, ProviderStoreError>> { + Box::pin(async move { + Ok(self + .capacity_failures + .lock() + .expect("capacity failures lock") + .get(account_id) + .map(|(_, peak)| *peak)) + }) + } } diff --git a/backend/crates/providers/xai/tests/admin.rs b/backend/crates/providers/xai/tests/admin.rs index f5fad18ff..d15610aa1 100644 --- a/backend/crates/providers/xai/tests/admin.rs +++ b/backend/crates/providers/xai/tests/admin.rs @@ -666,7 +666,7 @@ impl ProviderLeasePort for TestLeases { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, @@ -862,6 +862,30 @@ impl ProviderCooldownPort for TestCooldown { ) -> BoxFuture<'a, Result> { Box::pin(async { Ok(false) }) } + + fn record_capacity_failure<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _window: Duration, + _in_flight: u32, + ) -> BoxFuture<'a, Result> { + Box::pin(async { Ok(0) }) + } + + fn clear_after_success<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _through_revision: gateway_core::account::CredentialRevision, + ) -> BoxFuture<'a, Result<(), ProviderStoreError>> { + Box::pin(async { Ok(()) }) + } + + fn capacity_peak_in_flight<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + ) -> BoxFuture<'a, Result, ProviderStoreError>> { + Box::pin(async { Ok(None) }) + } } struct TestRuntimePolicy; diff --git a/backend/crates/providers/xai/tests/credential/selector.rs b/backend/crates/providers/xai/tests/credential/selector.rs index a673b2ced..e693b74f6 100644 --- a/backend/crates/providers/xai/tests/credential/selector.rs +++ b/backend/crates/providers/xai/tests/credential/selector.rs @@ -138,7 +138,7 @@ impl SelectorFixture { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/providers/xai/tests/provider/contract.rs b/backend/crates/providers/xai/tests/provider/contract.rs index 44063d0f4..633b1d0b7 100644 --- a/backend/crates/providers/xai/tests/provider/contract.rs +++ b/backend/crates/providers/xai/tests/provider/contract.rs @@ -1065,7 +1065,7 @@ impl ProviderLeasePort for DiagnosticLeasePort { last_started_at: None, quota_reset_at: None, quota_remaining_rank: None, - rate_limited_until: None, + cooldown: None, failure_rate_basis_points: None, first_output_latency_ms: None, }, diff --git a/backend/crates/providers/xai/tests/support.rs b/backend/crates/providers/xai/tests/support.rs index 4c3c84ea4..2b7ff0c7a 100644 --- a/backend/crates/providers/xai/tests/support.rs +++ b/backend/crates/providers/xai/tests/support.rs @@ -972,6 +972,30 @@ impl ProviderCooldownPort for MemoryCooldownPort { Ok(account_removed || scoped_removed) }) } + + fn record_capacity_failure<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _window: Duration, + _in_flight: u32, + ) -> futures::future::BoxFuture<'a, Result> { + Box::pin(async { Ok(0) }) + } + + fn clear_after_success<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + _through_revision: gateway_core::account::CredentialRevision, + ) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> { + Box::pin(async { Ok(()) }) + } + + fn capacity_peak_in_flight<'a>( + &'a self, + _account_id: &'a ProviderAccountId, + ) -> futures::future::BoxFuture<'a, Result, ProviderStoreError>> { + Box::pin(async { Ok(None) }) + } } impl ProviderRuntimePolicyPort for StaticRuntimePolicy { diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 1af694aee..e59e5c8d0 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -7,3 +7,5 @@ b7d3df7b48d835a00a3b74cbefc958a039d38fea5529cdad04bfbc703dca72aa 0001_initial.s 18e8c1b71c896a9d576d9e4ef9701506f654f55383af908898195b9eb2980fde 0007_account_notes.sql 36b720c9d3e6bf0bdb799b9b6a89b81638c642ac866992a880bd82357012a208 0008_concurrency_queues.sql 161df9ae43da34d51b340bbf95d71c61a8c1a2aa142382f693a9d127c834630d 0009_account_model_access.sql +dd4aa34e9bcf1698bb9688421964bbf410314d8e905448b8229460b972cff2f6 0010_account_auto_freeze.sql +9bd261b7e94872126754eb7f749c56187264a47c567b0bef5c394d6776b4f463 0011_account_auto_freeze_opt_in.sql diff --git a/backend/migrations/0010_account_auto_freeze.sql b/backend/migrations/0010_account_auto_freeze.sql new file mode 100644 index 000000000..9abaa3c9d --- /dev/null +++ b/backend/migrations/0010_account_auto_freeze.sql @@ -0,0 +1,22 @@ +-- 账号容量熔断:统计窗口内高频出现容量类上游错误时自动冻结账号一段时间; +-- 恢复前可选执行一次探测调用验证上游已恢复;冻结期间按观测到的在途并发峰值 +-- 下调账号并发上限,让账号带着更安全的并发配置自愈。默认启用,已有实例可关闭。 +alter table runtime_settings + add column account_auto_freeze_enabled boolean not null default true, + add column account_auto_freeze_threshold bigint not null default 12 + check (account_auto_freeze_threshold between 2 and 1000), + add column account_auto_freeze_window_seconds bigint not null default 600 + check (account_auto_freeze_window_seconds between 60 and 3600), + add column account_auto_freeze_duration_seconds bigint not null default 7200 + check (account_auto_freeze_duration_seconds between 300 and 604800), + add column account_auto_freeze_probe_enabled boolean not null default true, + add column account_auto_freeze_probe_model text + check ( + account_auto_freeze_probe_model is null + or ( + octet_length(account_auto_freeze_probe_model) between 1 and 128 + and account_auto_freeze_probe_model = btrim(account_auto_freeze_probe_model) + and account_auto_freeze_probe_model !~ '[[:cntrl:]]' + ) + ), + add column account_auto_freeze_adaptive_concurrency boolean not null default true; diff --git a/backend/migrations/0011_account_auto_freeze_opt_in.sql b/backend/migrations/0011_account_auto_freeze_opt_in.sql new file mode 100644 index 000000000..70f0dcb1b --- /dev/null +++ b/backend/migrations/0011_account_auto_freeze_opt_in.sql @@ -0,0 +1,12 @@ +-- 自动冻结默认关闭;保留已经通过设置页显式保存的冻结配置。 +alter table runtime_settings + alter column account_auto_freeze_enabled set default false; + +update runtime_settings +set account_auto_freeze_enabled = false +where account_auto_freeze_enabled = true + and not exists ( + select 1 from admin_audit_events + where action = 'settings.replace' + and changed_fields @> array['account_auto_freeze']::text[] + ); diff --git a/docs/api.md b/docs/api.md index b885c942d..131812788 100644 --- a/docs/api.md +++ b/docs/api.md @@ -253,7 +253,10 @@ Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。 OpenAI 明确返回 `server_is_overloaded`、`slow_down` 或模型容量不足错误时,代理在允许安全重放且 尚未交付输出的前提下,先做最多 3 次同账号指数退避,再通过现有调度换号。默认间隔从 500ms 开始, 上游 `Retry-After` 参与退避计算,单次等待不超过 8 秒;重试同时受请求总尝试次数和截止时间约束。 -容量不足不扣 Smart 账号健康分,不触发 Provider 全局熔断,也不作为账号额度耗尽写入冷却状态。 +`server_is_overloaded`、`slow_down` 与既有可计分的结构化错误按已发送的失败尝试计入 Smart 账号 +健康分;容量分类不再豁免计分。失败率沿用账号级平滑与时间衰减,影响后续普通选路,已有可用账号的 +会话亲和仍优先。容量不足不触发 Provider 全局熔断,也不作为账号额度耗尽;启用账号自动冻结时, +达到容量失败阈值会另外写入临时冷却。 最终交付的上游错误仍按上述透明边界保留原始状态码、错误码和正文。 明确额度耗尽继续走现有账号隔离与安全换号流程, 包括 WebSocket 握手返回的 429;不会因其长 `Retry-After` 而转入同账号传输恢复等待。 @@ -367,6 +370,11 @@ Token 明细、费用明细、用时/首字与状态。Token 和费用复用现 - `sortBy`: `email`、`status`、`planType`、`usage`、`lastUsedAt`、`expiresAt`; - `sortDirection`: `asc`、`desc`。 +账号限流详情在 `quota` 中返回:`rateLimitReason` 为 `upstream_rate_limit`(上游临时限流)、 +`capacity_freeze`(容量错误触发自动冻结)或 `null`。`recoveryProbeRequired` 表示解除冻结是否需要成功探测; +此时 `rateLimitedUntil` 是最早探测时间,到期后仍保持 `rate_limited`,直到探测成功或手动恢复。 +未要求探测时,该字段表示冷却结束时间。所有此类情况统一显示“限流中”,仅详情原因和恢复条件不同。 + 账号列表和详情返回 `notes`(无备注时为 `null`)。编辑时省略或 `null` 保留原备注;字符串最多 500 个 Unicode 字符,允许换行和制表符,保存时去除首尾空白,空字符串清空备注。备注独立于上游身份,导入时未显式提供备注、 重新授权、凭据刷新及批量调度更新均保留已有备注。 @@ -910,6 +918,13 @@ minCodexCliVersion usageRetentionDays opsEventRetentionDays auditRetentionDays +accountAutoFreezeEnabled +accountAutoFreezeThreshold +accountAutoFreezeWindowSeconds +accountAutoFreezeDurationSeconds +accountAutoFreezeProbeEnabled +accountAutoFreezeProbeModel +accountAutoFreezeAdaptiveConcurrency ``` `maxWaitingPerKey` 与 `maxWaitingPerAccount` 是全局统一的排队容量,取值 0~1,000,默认 0(关闭); @@ -922,6 +937,17 @@ auditRetentionDays `rotationStrategy` 可取 `smart`、`quota_reset_priority`、`round_robin`、`sticky`。 两个 `minCodex*Version` 字段为 `string | null`,只设置最低版本,不存在最大版本字段。 +账号自动冻结(`accountAutoFreezeEnabled`)默认关闭。启用后,在统计窗口内按尝试累计容量类上游错误(`server_is_overloaded` +等与 5xx 不可用),达到阈值后把该账号冻结为带恢复倒计时的 `rate_limited` 状态。`accountAutoFreezeThreshold` +取值 2~1,000(默认 12,按普通请求的 attempt 计数,含请求内同账号重试,不含诊断探测与本地连接保护错误);`accountAutoFreezeWindowSeconds` +取值 60~3,600(默认 600,随每次失败滑动顺延);`accountAutoFreezeDurationSeconds` 取值 300~604,800 +(默认 7,200,即 2 小时,探测失败后按该时长顺延)。`accountAutoFreezeProbeEnabled` 开启时恢复 worker +在到期后执行真实探测调用,成功才解冻;探测过程中和服务重启后继续阻止普通请求。关闭自动冻结或探测后, +已有冻结等待当前冷却结束再恢复调度。`accountAutoFreezeProbeModel` 为 `string | null`,留空时自动 +选择账号可用的第一个模型。`accountAutoFreezeAdaptiveConcurrency` 开启时冻结期间把账号并发上限下调到 +观测在途峰值的 80%(下限 2,只降不升)。这会持久修改账号并发设置;跟随全局默认的账号也会设为独立上限, +解冻后不自动恢复,管理员可手动改回。 + Windows 离线包接口固定解析 Microsoft Store Product ID `9PLM9XGG6VKS` 的 Retail 包,不接受调用方提供 产品 ID、上游地址、ring 或文件名。后端只返回通过包名、架构、Microsoft CDN host/path、scheme 和失效 时间校验的 `x64` / `arm64` MSIX 直链,不代理安装包字节。Store 内容通道返回 HTTP/80 临时地址时保留 diff --git a/docs/architecture.md b/docs/architecture.md index 455823a67..0d7428006 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -422,6 +422,10 @@ PostgreSQL 周期对账才是正确性基础。 `normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`。只有明确上游证据才能恢复或终态化账号, 本地时钟和不确定响应不能伪造事实。 +容量冻结与普通上游限流共享账号冷却投影,但恢复条件不同:需要探测的冻结在截止时间后仍阻止调度, +普通推理成功不能解除。Admin 恢复任务按冻结代次原子提交探测结果,避免旧结果覆盖手动恢复或新冻结; +自动降低并发通过现有管理事务与快照发布链路,仅按数据库最新设置更新并发上限。 + PostgreSQL schema 由迁移目录按编号管理。已应用迁移按字节冻结,后续 schema 变化只能新增编号迁移, 详见 [迁移规则](../backend/migrations/README.md)。 @@ -497,8 +501,19 @@ Worker 由各 Bundle 贡献、由 Host 统一监督: - Store:过期请求恢复、历史保留和 PostgreSQL/Redis 观测队列; - Core:`runtime` owner 的 RuntimeSnapshot 周期对账和 Redis change 订阅; - Admin:S3/R2 备份 daemon,负责调度、执行、删除收敛与保留清理; + 以及账号冻结恢复 worker(容量熔断的自适应并发下调与到期探测解冻); - Provider:credential refresh、quota/catalog 健康和官方版本/etag 检查。 +账号容量熔断默认关闭。启用后,仅普通请求收到的容量类上游错误(`server_is_overloaded` 等与 +5xx 不可用)按滑动窗口计数,并把当时观测到的在途并发并入峰值证据;本地连接保护与诊断探测 +不参与容量计数或峰值采样。这些错误不证明凭据或配额失效,不进入账号失败状态。达到阈值后写入 +账号级 Redis 容量冷却,调度立即跳过该账号;管理端沿用限流状态,通过原因区分容量冻结与上游限流。 +冻结与计数由 Redis 保存,不改变 PostgreSQL 账号状态。需要探测的冻结在到期后仍阻止调度,恢复 +worker 复用连接测试探针执行真实上游调用,并按冻结代次处理结果:成功清除,失败按配置时长顺延, +过期探测结果不得覆盖更新的冻结或管理员操作。关闭自动冻结或探测后,已有冻结在冷却到期后恢复。 +自适应并发下调按观测峰值的 80%(下限 2)原子更新最新有效账号上限,只降不升,不覆盖其他账号 +设置,审计标注为系统变更;该上限持久保存,恢复调度后不自动调高。 + 周期任务的 Redis lease 只保证单周期互斥,不构成多副本 leader 选举。备份 daemon 依赖单副本部署边界, 自更新也只替换处理请求的当前进程,因此整个应用必须保持单副本。 diff --git a/frontend/src/api/modules/accounts.ts b/frontend/src/api/modules/accounts.ts index 4f2a0c40b..7dafccf77 100644 --- a/frontend/src/api/modules/accounts.ts +++ b/frontend/src/api/modules/accounts.ts @@ -33,6 +33,8 @@ export interface AccountQuota { limitReached: boolean // 429 临时限流(Redis 冷却)到期时间;非限流中为 null。 rateLimitedUntil: string | null + rateLimitReason: 'upstream_rate_limit' | 'capacity_freeze' | null + recoveryProbeRequired: boolean windows: AccountQuotaWindow[] } diff --git a/frontend/src/api/modules/settings.ts b/frontend/src/api/modules/settings.ts index 437409e1e..34ff6a17e 100644 --- a/frontend/src/api/modules/settings.ts +++ b/frontend/src/api/modules/settings.ts @@ -18,6 +18,13 @@ export interface RuntimeSettings { usageRetentionDays: number opsEventRetentionDays: number auditRetentionDays: number + accountAutoFreezeEnabled: boolean + accountAutoFreezeThreshold: number + accountAutoFreezeWindowSeconds: number + accountAutoFreezeDurationSeconds: number + accountAutoFreezeProbeEnabled: boolean + accountAutoFreezeProbeModel: string | null + accountAutoFreezeAdaptiveConcurrency: boolean updatedAt: string } diff --git a/frontend/src/views/accounts/components/AccountConnectionTestModal.vue b/frontend/src/views/accounts/components/AccountConnectionTestModal.vue index 7bd3bf163..5519aba47 100644 --- a/frontend/src/views/accounts/components/AccountConnectionTestModal.vue +++ b/frontend/src/views/accounts/components/AccountConnectionTestModal.vue @@ -62,6 +62,9 @@ function connectionLogClass(tone: string) { :status="account.status" :error-reason="account.errorReason" :error-message="account.errorMessage" + :rate-limited-until="account.quota.rateLimitedUntil" + :rate-limit-reason="account.quota.rateLimitReason" + :recovery-probe-required="account.quota.recoveryProbeRequired" :next-refresh-at="account.nextRefreshAt" variant="pill" /> diff --git a/frontend/src/views/accounts/components/AccountStatusBadge/index.vue b/frontend/src/views/accounts/components/AccountStatusBadge/index.vue index e47d4f96c..ea5e94f40 100644 --- a/frontend/src/views/accounts/components/AccountStatusBadge/index.vue +++ b/frontend/src/views/accounts/components/AccountStatusBadge/index.vue @@ -13,6 +13,8 @@ const props = withDefaults( errorReason?: AccountErrorReason | null errorMessage?: string | null rateLimitedUntil?: string | null + rateLimitReason?: 'upstream_rate_limit' | 'capacity_freeze' | null + recoveryProbeRequired?: boolean nextRefreshAt?: string | null variant?: 'inline' | 'pill' }>(), @@ -20,6 +22,8 @@ const props = withDefaults( errorReason: null, errorMessage: null, rateLimitedUntil: null, + rateLimitReason: null, + recoveryProbeRequired: false, nextRefreshAt: null, variant: 'inline', }, @@ -32,6 +36,8 @@ const presentation = computed(() => errorReason: props.errorReason, errorMessage: props.errorMessage, rateLimitedUntil: props.rateLimitedUntil, + rateLimitReason: props.rateLimitReason, + recoveryProbeRequired: props.recoveryProbeRequired, nextRefreshAt: props.nextRefreshAt, now: now.value.getTime(), }), @@ -102,7 +108,7 @@ const presentation = computed(() => v-if="presentation.rateLimitRecovery" class="flex items-center justify-between gap-3 rounded-cp bg-cp-fill-quaternary px-3 py-2" > - 预计恢复 + {{ presentation.recoveryTimeLabel }} {{ presentation.rateLimitRecovery }} diff --git a/frontend/src/views/accounts/components/AccountStatusBadge/presenter.ts b/frontend/src/views/accounts/components/AccountStatusBadge/presenter.ts index 37fd95a92..c95ae5f31 100644 --- a/frontend/src/views/accounts/components/AccountStatusBadge/presenter.ts +++ b/frontend/src/views/accounts/components/AccountStatusBadge/presenter.ts @@ -37,6 +37,7 @@ export interface AccountStatusPresentation { errorText: string | null nextRefreshDisplay: string | null rateLimitRecovery: string | null + recoveryTimeLabel: string triggerLabel: string } @@ -45,6 +46,8 @@ export interface AccountStatusPresentationInput { errorReason: AccountErrorReason | null errorMessage: string | null rateLimitedUntil: string | null + rateLimitReason: 'upstream_rate_limit' | 'capacity_freeze' | null + recoveryProbeRequired: boolean nextRefreshAt: string | null now: number } @@ -146,19 +149,32 @@ export function resolveAccountStatusPresentation( && nextRefreshTimestamp !== null && nextRefreshTimestamp > input.now const mode: AccountStatusDisplayMode = isBackoff ? 'refresh_backoff' : input.status - const definition = displayDefinitions[mode] + const isFreeze = mode === 'rate_limited' && input.rateLimitReason === 'capacity_freeze' + const waitsForProbe = isFreeze && input.recoveryProbeRequired + const definition = isFreeze + ? { + ...displayDefinitions.rate_limited, + description: '容量类请求失败累计达到阈值,系统已暂停该账号的调度。', + recoveryHint: waitsForProbe + ? '冷却结束后进行恢复探测,成功后恢复调度;也可手动恢复账号。' + : '冷却结束后自动恢复调度,也可手动恢复账号。', + } + : displayDefinitions[mode] const nextRefreshDisplay = isBackoff ? formatDateTime(nextRefreshTimestamp) : null const reasonLabel = input.errorReason ? errorReasonLabels[input.errorReason] : null const title = definition.title ?? reasonLabel ?? definition.label - const rateLimitRecovery = input.status === 'rate_limited' - ? remainingTime(input.rateLimitedUntil, input.now) + const until = input.rateLimitedUntil ? parseTimestamp(input.rateLimitedUntil) : null + const rateLimitRecovery = mode === 'rate_limited' + ? waitsForProbe && until !== null && until <= input.now + ? '等待探测成功' + : remainingTime(input.rateLimitedUntil, input.now) : null const recoveryHint = mode !== 'refresh_backoff' && mode !== 'rate_limited' && input.errorReason ? errorRecoveryHints[input.errorReason] : definition.recoveryHint - const hasDetail = input.status === 'error' || rateLimitRecovery !== null || isBackoff + const hasDetail = input.status === 'error' || mode === 'rate_limited' || isBackoff const retry = nextRefreshDisplay ? ` 下次尝试:${nextRefreshDisplay}。` : '' const triggerLabel = `${title}。${definition.description}${retry} 点击或聚焦查看详情。` @@ -174,6 +190,7 @@ export function resolveAccountStatusPresentation( errorText: input.errorMessage || null, nextRefreshDisplay, rateLimitRecovery, + recoveryTimeLabel: waitsForProbe ? '恢复探测' : '预计恢复', triggerLabel, } } diff --git a/frontend/src/views/accounts/index.vue b/frontend/src/views/accounts/index.vue index 0e7f6d1b1..daa70104d 100644 --- a/frontend/src/views/accounts/index.vue +++ b/frontend/src/views/accounts/index.vue @@ -291,6 +291,8 @@ const { :error-reason="row.errorReason" :error-message="row.errorMessage" :rate-limited-until="row.quota.rateLimitedUntil" + :rate-limit-reason="row.quota.rateLimitReason" + :recovery-probe-required="row.quota.recoveryProbeRequired" :next-refresh-at="row.nextRefreshAt" /> diff --git a/frontend/src/views/settings/components/AccountAutoFreezeCard.vue b/frontend/src/views/settings/components/AccountAutoFreezeCard.vue new file mode 100644 index 000000000..bc838043a --- /dev/null +++ b/frontend/src/views/settings/components/AccountAutoFreezeCard.vue @@ -0,0 +1,135 @@ + + + diff --git a/frontend/src/views/settings/composables/useSettingsForm.ts b/frontend/src/views/settings/composables/useSettingsForm.ts index d78dd1bfe..89982f04f 100644 --- a/frontend/src/views/settings/composables/useSettingsForm.ts +++ b/frontend/src/views/settings/composables/useSettingsForm.ts @@ -30,9 +30,17 @@ export function useSettingsForm() { usageRetentionDays: 31, opsEventRetentionDays: 30, auditRetentionDays: 90, + + accountAutoFreezeEnabled: false, + accountAutoFreezeThreshold: null as number | null, + accountAutoFreezeWindowSeconds: null as number | null, + accountAutoFreezeDurationSeconds: null as number | null, + accountAutoFreezeProbeEnabled: true, + accountAutoFreezeProbeModel: '', + accountAutoFreezeAdaptiveConcurrency: true, }) - function numericModel(key: 'refreshMarginSeconds' | 'refreshConcurrency' | 'maxConcurrentPerAccount' | 'requestIntervalMs' | 'maxWaitingPerKey' | 'maxWaitingPerAccount' | 'concurrencyWaitTimeoutSeconds') { + function numericModel(key: 'refreshMarginSeconds' | 'refreshConcurrency' | 'maxConcurrentPerAccount' | 'requestIntervalMs' | 'maxWaitingPerKey' | 'maxWaitingPerAccount' | 'concurrencyWaitTimeoutSeconds' | 'accountAutoFreezeThreshold' | 'accountAutoFreezeWindowSeconds' | 'accountAutoFreezeDurationSeconds') { return computed({ get: () => (form[key] === null ? '' : String(form[key])), set: (value: string) => { @@ -53,6 +61,9 @@ export function useSettingsForm() { const maxWaitingPerKeyValue = numericModel('maxWaitingPerKey') const maxWaitingPerAccountValue = numericModel('maxWaitingPerAccount') const concurrencyWaitTimeoutSecondsValue = numericModel('concurrencyWaitTimeoutSeconds') + const accountAutoFreezeThresholdValue = numericModel('accountAutoFreezeThreshold') + const accountAutoFreezeWindowSecondsValue = numericModel('accountAutoFreezeWindowSeconds') + const accountAutoFreezeDurationSecondsValue = numericModel('accountAutoFreezeDurationSeconds') const minCodexDesktopVersionError = computed(() => versionError(form.minCodexDesktopVersion)) const minCodexCliVersionError = computed(() => versionError(form.minCodexCliVersion)) @@ -77,6 +88,13 @@ export function useSettingsForm() { form.usageRetentionDays = data.usageRetentionDays form.opsEventRetentionDays = data.opsEventRetentionDays form.auditRetentionDays = data.auditRetentionDays + form.accountAutoFreezeEnabled = data.accountAutoFreezeEnabled + form.accountAutoFreezeThreshold = data.accountAutoFreezeThreshold + form.accountAutoFreezeWindowSeconds = data.accountAutoFreezeWindowSeconds + form.accountAutoFreezeDurationSeconds = data.accountAutoFreezeDurationSeconds + form.accountAutoFreezeProbeEnabled = data.accountAutoFreezeProbeEnabled + form.accountAutoFreezeProbeModel = data.accountAutoFreezeProbeModel ?? '' + form.accountAutoFreezeAdaptiveConcurrency = data.accountAutoFreezeAdaptiveConcurrency mappings.value = Object.entries(data.modelMappings || {}).map(([requestedModel, upstreamModel]) => ({ requestedModel, upstreamModel: String(upstreamModel), @@ -132,7 +150,7 @@ export function useSettingsForm() { async function saveSettings() { if (saving.value || loading.value) return - const { refreshMarginSeconds, refreshConcurrency, maxConcurrentPerAccount, requestIntervalMs, rotationStrategy, maxWaitingPerKey, maxWaitingPerAccount, concurrencyWaitTimeoutSeconds } = form + const { refreshMarginSeconds, refreshConcurrency, maxConcurrentPerAccount, requestIntervalMs, rotationStrategy, maxWaitingPerKey, maxWaitingPerAccount, concurrencyWaitTimeoutSeconds, accountAutoFreezeThreshold, accountAutoFreezeWindowSeconds, accountAutoFreezeDurationSeconds } = form if (refreshMarginSeconds === null || refreshConcurrency === null || maxConcurrentPerAccount === null || requestIntervalMs === null || !rotationStrategy || maxWaitingPerKey === null || maxWaitingPerAccount === null || concurrencyWaitTimeoutSeconds === null) { toast.warning('请完整填写运行参数和调度策略') return @@ -146,6 +164,21 @@ export function useSettingsForm() { toast.warning('请修正客户端最低版本格式') return } + if (accountAutoFreezeThreshold === null || accountAutoFreezeWindowSeconds === null || accountAutoFreezeDurationSeconds === null) { + toast.warning('请完整填写账号自动冻结参数') + return + } + if (!Number.isInteger(accountAutoFreezeThreshold) || accountAutoFreezeThreshold < 2 || accountAutoFreezeThreshold > 1000 + || !Number.isInteger(accountAutoFreezeWindowSeconds) || accountAutoFreezeWindowSeconds < 60 || accountAutoFreezeWindowSeconds > 3600 + || !Number.isInteger(accountAutoFreezeDurationSeconds) || accountAutoFreezeDurationSeconds < 300 || accountAutoFreezeDurationSeconds > 604800) { + toast.warning('自动冻结阈值应为 2~1000,统计窗口为 60~3600 秒,冻结时长为 300~604800 秒') + return + } + const probeModel = form.accountAutoFreezeProbeModel.trim() + if (probeModel && (probeModel.length > 128 || probeModel !== probeModel.trim())) { + toast.warning('探测模型名称不能超过 128 个字符') + return + } await saveAction.run(async () => { const result = await updateSettings({ modelMappings: mappingPayload(), @@ -162,6 +195,13 @@ export function useSettingsForm() { usageRetentionDays: form.usageRetentionDays, opsEventRetentionDays: form.opsEventRetentionDays, auditRetentionDays: form.auditRetentionDays, + accountAutoFreezeEnabled: form.accountAutoFreezeEnabled, + accountAutoFreezeThreshold, + accountAutoFreezeWindowSeconds, + accountAutoFreezeDurationSeconds, + accountAutoFreezeProbeEnabled: form.accountAutoFreezeProbeEnabled, + accountAutoFreezeProbeModel: probeModel || null, + accountAutoFreezeAdaptiveConcurrency: form.accountAutoFreezeAdaptiveConcurrency, }) applySettings(result) toast.success('设置已保存') @@ -189,6 +229,9 @@ export function useSettingsForm() { maxWaitingPerKeyValue, maxWaitingPerAccountValue, concurrencyWaitTimeoutSecondsValue, + accountAutoFreezeThresholdValue, + accountAutoFreezeWindowSecondsValue, + accountAutoFreezeDurationSecondsValue, minCodexDesktopVersionError, minCodexCliVersionError, saveSettings, diff --git a/frontend/src/views/settings/index.vue b/frontend/src/views/settings/index.vue index 5a7197990..d17488ef0 100644 --- a/frontend/src/views/settings/index.vue +++ b/frontend/src/views/settings/index.vue @@ -8,6 +8,7 @@ import BaseConfirmModal from '@/components/base/BaseConfirmModal.vue' import BasePageHeader from '@/components/base/BasePageHeader.vue' import BaseSegmented from '@/components/base/BaseSegmented.vue' +import AccountAutoFreezeCard from './components/AccountAutoFreezeCard.vue' import AdminApiKeyCard from './components/AdminApiKeyCard.vue' import SettingsBackupSection from './components/backup/SettingsBackupSection.vue' import ClientVersionSettings from './components/client-version/index.vue' @@ -52,6 +53,9 @@ const { maxWaitingPerKeyValue, maxWaitingPerAccountValue, concurrencyWaitTimeoutSecondsValue, + accountAutoFreezeThresholdValue, + accountAutoFreezeWindowSecondsValue, + accountAutoFreezeDurationSecondsValue, minCodexDesktopVersionError, minCodexCliVersionError, @@ -136,6 +140,16 @@ watch( v-model:concurrency-wait-timeout-seconds="concurrencyWaitTimeoutSecondsValue" /> + +