mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
feat(accounts): add opt-in capacity cooldown and recovery (#126)
Add opt-in account capacity cooldown, recovery probes, and atomic adaptive concurrency updates. Include overload and slow_down in Smart scheduling feedback, expose rate-limit reasons, and keep probe results fenced by cooldown generation. Exclude diagnostic probes and local connection failures from capacity evidence. Default auto-freeze to disabled and preserve explicitly saved settings with a forward migration. Closes #120 Co-authored-by: zhangwentian <2han9wen71an@gmail.com> Co-authored-by: Zyy <zyycn@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Zyy
Codex
parent
ae2da31380
commit
499b43043e
@@ -83,6 +83,9 @@ API Key 持有者可在同一登录页切换登录身份,进入 `/key-usage`
|
||||
2. 按需建立账号分组,再创建客户端密钥并选择可用分组。**不选分组表示可使用全部账号**。
|
||||
3. 打开密钥的「使用密钥」,复制客户端配置。
|
||||
|
||||
账号自动冻结默认关闭,可在「系统设置 → 运行参数」中启用。启用后,账号频繁命中上游容量错误时
|
||||
会暂停调度并显示为限流中;若开启恢复前探测,冷却到期后需探测成功才恢复。
|
||||
|
||||
## 客户端接入
|
||||
|
||||
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入。
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
//! 容量熔断冻结的恢复编排:到期探测、失败顺延与自适应并发下调。
|
||||
//!
|
||||
//! 冻结事实保存在可丢失的 Redis 冷却中,本服务只组合 Admin 端口:
|
||||
//! 观测(`AccountRuntimeStore`)、探测(`AccountsService`)与写回
|
||||
//! (原子降低并发 / 按冻结代次结束探测)。旧观测不会覆盖管理员操作。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::StreamExt as _;
|
||||
use gateway_core::account::{AccountConcurrencyLimit, ProviderAccountId};
|
||||
use gateway_core::routing::UpstreamModelId;
|
||||
use gateway_core::task::{ScheduledTask, WorkerCycleContext, WorkerTaskError};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::accounts::{
|
||||
AccountConnectionTestEvent, AccountFreeze, AccountPageItem, AccountRuntimeSnapshot,
|
||||
};
|
||||
use crate::model::{MutationActor, MutationContext};
|
||||
use crate::ports::store::{AccountRuntimeStore, AccountStore, SettingsStore};
|
||||
use crate::use_case::accounts::AccountsService;
|
||||
|
||||
/// 自适应并发下调保留系数:下调到观测在途峰值的 80%。
|
||||
const ADAPTIVE_CONCURRENCY_FACTOR: f64 = 0.8;
|
||||
/// 自适应并发下限:过低的并发让账号几乎不可用,宁可保持冻结。
|
||||
const ADAPTIVE_CONCURRENCY_FLOOR: u32 = 2;
|
||||
/// worker 周期;冷却观测与探测都依赖该节奏。
|
||||
pub const FREEZE_RECOVERY_INTERVAL: Duration = Duration::from_secs(30);
|
||||
pub const FREEZE_RECOVERY_WORKER_OWNER: &str = "account-freeze-recovery";
|
||||
pub const WORKER_INITIAL_BACKOFF: Duration = Duration::from_secs(1);
|
||||
pub const WORKER_MAXIMUM_BACKOFF: Duration = Duration::from_secs(60);
|
||||
pub const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60);
|
||||
pub const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60);
|
||||
/// 系统审计上下文的固定请求 ID;账号并发更新审计由此溯源到自动冻结编排。
|
||||
const SYSTEM_REQUEST_ID: &str = "account-freeze-recovery";
|
||||
|
||||
/// 冻结恢复所需的管理端口组合。
|
||||
#[derive(Clone)]
|
||||
pub struct FreezeRecoveryDeps {
|
||||
pub accounts: Arc<dyn AccountsService>,
|
||||
pub store: Arc<dyn AccountStore>,
|
||||
pub runtime: Arc<dyn AccountRuntimeStore>,
|
||||
pub settings: Arc<dyn SettingsStore>,
|
||||
}
|
||||
|
||||
pub struct FreezeRecoveryTask {
|
||||
deps: FreezeRecoveryDeps,
|
||||
}
|
||||
|
||||
impl FreezeRecoveryTask {
|
||||
pub fn new(deps: FreezeRecoveryDeps) -> Self {
|
||||
Self { deps }
|
||||
}
|
||||
|
||||
fn system_context() -> MutationContext {
|
||||
MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: SYSTEM_REQUEST_ID.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 读取冻结策略;读取或校验失败跳过本轮,不能把未知配置当作允许解冻。
|
||||
async fn freeze_policy(&self) -> Option<gateway_core::provider_ports::ProviderFreezePolicy> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
|
||||
settings.account_auto_freeze_enabled,
|
||||
settings.account_auto_freeze_threshold,
|
||||
settings.account_auto_freeze_window_seconds,
|
||||
settings.account_auto_freeze_duration_seconds,
|
||||
settings.account_auto_freeze_probe_enabled,
|
||||
settings.account_auto_freeze_probe_model,
|
||||
settings.account_auto_freeze_adaptive_concurrency,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
async fn active_freezes(&self) -> BTreeMap<String, AccountFreeze> {
|
||||
self.deps.runtime.active_freezes().await.unwrap_or_default()
|
||||
}
|
||||
|
||||
async fn run_cycle_inner(&self) {
|
||||
let Some(policy) = self.freeze_policy().await else {
|
||||
return;
|
||||
};
|
||||
let freezes = self.active_freezes().await;
|
||||
if freezes.is_empty() {
|
||||
return;
|
||||
}
|
||||
if policy.enabled() && policy.adaptive_concurrency() {
|
||||
self.adapt_concurrency_limits(&freezes, &policy).await;
|
||||
}
|
||||
self.recover_due_freezes(&freezes, &policy).await;
|
||||
}
|
||||
|
||||
/// 对冻结中的账号执行自适应并发下调:目标为观测在途峰值的 80%(下限 2),
|
||||
/// 只降不升;未观测到峰值证据的账号保持现状。
|
||||
async fn adapt_concurrency_limits(
|
||||
&self,
|
||||
freezes: &BTreeMap<String, AccountFreeze>,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let account_ids = freezes.keys().cloned().collect::<Vec<_>>();
|
||||
let Ok(peaks) = self.deps.runtime.capacity_peaks(&account_ids).await else {
|
||||
return;
|
||||
};
|
||||
for (account_id, peak) in peaks {
|
||||
let Some(target) = adaptive_target(peak) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(account) = ProviderAccountId::new(account_id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(limit) = AccountConcurrencyLimit::new(target) else {
|
||||
continue;
|
||||
};
|
||||
match self
|
||||
.deps
|
||||
.accounts
|
||||
.lower_concurrency_limit(&Self::system_context(), account, limit)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => {
|
||||
tracing::info!(
|
||||
account_id,
|
||||
adapted_limit = target,
|
||||
freeze_seconds = policy.freeze_duration().as_secs(),
|
||||
"容量熔断:按观测在途峰值下调账号并发上限",
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
warn!(account_id, error = %error, "自适应并发下调失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> Option<Option<AccountPageItem>> {
|
||||
let account_id = ProviderAccountId::new(account_id.to_owned()).ok()?;
|
||||
self.deps
|
||||
.store
|
||||
.load_account(account_id.as_str(), AccountRuntimeSnapshot::default())
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// 到达探测时间后才恢复;关闭自动冻结或探测时,已有冻结仍等待原冷却结束。
|
||||
async fn recover_due_freezes(
|
||||
&self,
|
||||
freezes: &BTreeMap<String, AccountFreeze>,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
for (account_id, freeze) in freezes {
|
||||
if freeze.until > Utc::now() {
|
||||
continue;
|
||||
}
|
||||
if policy.enabled() && policy.probe_enabled() && freeze.requires_probe {
|
||||
self.probe_and_recover(account_id, freeze, policy).await;
|
||||
} else if let Err(error) = self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account_id, freeze, None)
|
||||
.await
|
||||
{
|
||||
warn!(account_id, error = %error, "解除到期冻结失败");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_and_recover(
|
||||
&self,
|
||||
account_id: &str,
|
||||
freeze: &AccountFreeze,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let Some(Some(item)) = self.load_account(account_id).await else {
|
||||
return;
|
||||
};
|
||||
if !item.account.enabled {
|
||||
// 停用账号没有自动恢复意义;解冻交给管理员手动恢复。
|
||||
return;
|
||||
}
|
||||
let Ok(account) = ProviderAccountId::new(account_id.to_owned()) else {
|
||||
return;
|
||||
};
|
||||
let Some(model) = self.resolve_probe_model(&account, policy).await else {
|
||||
// 没有可用探测模型时按失败处理,顺延冻结等待下一轮。
|
||||
self.postpone(&account, freeze, policy).await;
|
||||
return;
|
||||
};
|
||||
let probe_succeeded = match self
|
||||
.deps
|
||||
.accounts
|
||||
.test_connection(account.clone(), model)
|
||||
.await
|
||||
{
|
||||
Ok(events) => drain_probe(events).await,
|
||||
Err(error) => {
|
||||
tracing::debug!(account_id, error = %error, "冻结恢复探测发起失败");
|
||||
false
|
||||
}
|
||||
};
|
||||
if probe_succeeded {
|
||||
match self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account_id, freeze, None)
|
||||
.await
|
||||
{
|
||||
Ok(true) => tracing::info!(account_id, "冻结恢复探测成功:账号已解冻"),
|
||||
Ok(false) => {}
|
||||
Err(error) => warn!(account_id, error = %error, "解除冻结失败"),
|
||||
}
|
||||
} else {
|
||||
self.postpone(&account, freeze, policy).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// 探测失败只顺延本次仍存在的冻结,旧探测不能重新冻结已手动恢复的账号。
|
||||
async fn postpone(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
freeze: &AccountFreeze,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) {
|
||||
let Some(until) = SystemTime::now().checked_add(policy.freeze_duration()) else {
|
||||
return;
|
||||
};
|
||||
let until = DateTime::<Utc>::from(until);
|
||||
match self
|
||||
.deps
|
||||
.runtime
|
||||
.finish_freeze(account.as_str(), freeze, Some(until))
|
||||
.await
|
||||
{
|
||||
Ok(true) => {
|
||||
tracing::info!(
|
||||
account_id = account.as_str(),
|
||||
postpone_seconds = policy.freeze_duration().as_secs(),
|
||||
"冻结恢复探测失败:账号冻结顺延",
|
||||
);
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
account_id = account.as_str(),
|
||||
error = %error,
|
||||
"冻结顺延失败"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn resolve_probe_model(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
policy: &gateway_core::provider_ports::ProviderFreezePolicy,
|
||||
) -> Option<UpstreamModelId> {
|
||||
if let Some(model) = policy.probe_model() {
|
||||
return UpstreamModelId::new(model.to_owned()).ok();
|
||||
}
|
||||
let models = self.deps.accounts.models(account, false).await.ok()?;
|
||||
models.models.into_iter().next().map(|model| model.id)
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for FreezeRecoveryTask {
|
||||
fn run_cycle(
|
||||
&self,
|
||||
context: WorkerCycleContext,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_cycle_inner().await;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 自适应目标并发:观测峰值 × 0.8 向下取整,下限 2;无证据(峰值为 0)不调整。
|
||||
fn adaptive_target(peak_in_flight: u32) -> Option<u32> {
|
||||
if peak_in_flight == 0 {
|
||||
return None;
|
||||
}
|
||||
let scaled = (f64::from(peak_in_flight) * ADAPTIVE_CONCURRENCY_FACTOR).floor() as u32;
|
||||
Some(scaled.max(ADAPTIVE_CONCURRENCY_FLOOR))
|
||||
}
|
||||
|
||||
/// 消耗连接测试事件流并返回是否以 `Completed` 终止。
|
||||
async fn drain_probe(mut events: crate::model::accounts::AccountConnectionTestEventStream) -> bool {
|
||||
let mut completed = false;
|
||||
while let Some(event) = events.next().await {
|
||||
match event {
|
||||
AccountConnectionTestEvent::Completed => completed = true,
|
||||
AccountConnectionTestEvent::Failed { .. } => return false,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
completed
|
||||
}
|
||||
@@ -10,14 +10,15 @@ use gateway_core::{
|
||||
routing::ProviderKind,
|
||||
runtime::SnapshotControl,
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
|
||||
WorkerRunnable,
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerLeaseRequest,
|
||||
WorkerRegistration, WorkerRunnable, WorkerSchedule,
|
||||
},
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
pub mod freeze_recovery;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
mod use_case;
|
||||
@@ -210,6 +211,12 @@ impl AdminServices {
|
||||
self.key_usage.as_ref()
|
||||
}
|
||||
|
||||
/// 取得账号服务的共享句柄;后台编排(冻结恢复 worker)需要持有 Arc。
|
||||
#[must_use]
|
||||
pub fn accounts_handle(&self) -> Arc<dyn AccountsService> {
|
||||
Arc::clone(&self.accounts)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn proxies(&self) -> &dyn ProxiesService {
|
||||
self.proxies.as_ref()
|
||||
@@ -392,7 +399,7 @@ pub async fn initialize(
|
||||
registry.clone(),
|
||||
)),
|
||||
auth,
|
||||
accounts,
|
||||
accounts: accounts.clone(),
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
store.account_runtime(),
|
||||
@@ -419,6 +426,13 @@ pub async fn initialize(
|
||||
import_tasks,
|
||||
backups,
|
||||
};
|
||||
let freeze_recovery =
|
||||
freeze_recovery::FreezeRecoveryTask::new(freeze_recovery::FreezeRecoveryDeps {
|
||||
accounts: Arc::clone(&accounts) as Arc<dyn AccountsService>,
|
||||
store: store.accounts(),
|
||||
runtime: store.account_runtime(),
|
||||
settings: store.settings(),
|
||||
});
|
||||
let mut worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
let id = WorkerId::try_new(WorkerKind::AccountImport, "admin")
|
||||
.map_err(|_| AdminError::internal("导入 Worker ID 不合法"))?;
|
||||
@@ -433,6 +447,7 @@ pub async fn initialize(
|
||||
)
|
||||
.map_err(|_| AdminError::internal("导入 Worker 注册信息不合法"))?;
|
||||
worker_contributions.push(WorkerContribution::Registration(registration));
|
||||
worker_contributions.extend(freeze_recovery_worker_contribution(freeze_recovery)?);
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
@@ -458,6 +473,37 @@ fn backup_worker_contribution(
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
/// 冻结恢复 Worker 注册:按固定周期扫描活跃冻结,owner 固定。
|
||||
fn freeze_recovery_worker_contribution(
|
||||
task: freeze_recovery::FreezeRecoveryTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
freeze_recovery::FREEZE_RECOVERY_WORKER_OWNER,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker ID 不合法"))?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
freeze_recovery::FREEZE_RECOVERY_INTERVAL,
|
||||
freeze_recovery::WORKER_INITIAL_BACKOFF,
|
||||
freeze_recovery::WORKER_MAXIMUM_BACKOFF,
|
||||
freeze_recovery::WORKER_LEASE_TTL,
|
||||
freeze_recovery::WORKER_LEASE_RENEWAL,
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 调度配置不合法"))?;
|
||||
let lease = WorkerLeaseRequest::try_new(id.clone(), freeze_recovery::WORKER_LEASE_TTL)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 租约配置不合法"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Scheduled {
|
||||
schedule,
|
||||
lease: Some(lease),
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("冻结恢复 Worker 注册信息不合法"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
|
||||
ProviderKind::new(value).map_err(|_| AdminError::internal("内置 Provider 类型不合法"))
|
||||
}
|
||||
|
||||
@@ -69,11 +69,20 @@ pub struct AccountListQuery {
|
||||
/// Admin query service 从运行态存储取得的当前账号冷却快照。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountRuntimeSnapshot {
|
||||
pub rate_limited_until: BTreeMap<String, DateTime<Utc>>,
|
||||
pub cooldown: BTreeMap<String, gateway_core::account::AccountCooldown>,
|
||||
/// `None` 表示实时 lease 存储不可用;`Some` 中未出现的账号当前使用量为零。
|
||||
pub in_flight: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// 恢复任务读取的冻结快照;generation 将异步结果绑定到本次冻结。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountFreeze {
|
||||
pub credential_revision: Revision,
|
||||
pub until: DateTime<Utc>,
|
||||
pub generation: String,
|
||||
pub requires_probe: bool,
|
||||
}
|
||||
|
||||
/// Optional account membership filter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AccountGroupFilter {
|
||||
|
||||
@@ -32,6 +32,13 @@ pub struct RuntimeSettings {
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -52,6 +59,13 @@ pub struct ReplaceRuntimeSettings {
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
/// 明文管理员 API Key;按产品约束明文落库,但禁止 Debug 泄漏。
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! 端口按业务资源拆分,方法使用领域模型,不暴露连接池、事务或 Redis client。
|
||||
|
||||
use std::{net::IpAddr, sync::Arc, time::Duration};
|
||||
use std::{collections::BTreeMap, net::IpAddr, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -159,6 +159,14 @@ pub trait AccountStore: Send + Sync {
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult>;
|
||||
|
||||
/// 在事务内按最新启用状态、账号上限和全局默认值判断,只降低并发上限。
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &gateway_core::account::ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>>;
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &gateway_core::account::ProviderAccountId,
|
||||
@@ -193,6 +201,25 @@ pub trait AccountRuntimeStore: Send + Sync {
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot>;
|
||||
|
||||
/// 容量熔断自动冻结中的账号与其冻结截止时间;429 临时限流不包含在内。
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, crate::model::accounts::AccountFreeze>>;
|
||||
|
||||
/// 读取容量失败窗口内观测到的在途并发峰值(自适应并发下调的证据)。
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>>;
|
||||
|
||||
/// 仅当冻结快照仍匹配时解除或顺延;旧探测不得覆盖手动恢复或新一轮冻结。
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &crate::model::accounts::AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool>;
|
||||
}
|
||||
|
||||
/// 控制面凭据、统一会话、登录限流与管理员安全审计。
|
||||
|
||||
@@ -192,11 +192,7 @@ fn project_group_runtime(
|
||||
let mut accounts_by_group = BTreeMap::<&str, Vec<&str>>::new();
|
||||
for member in members {
|
||||
let mut status = member.status.clone();
|
||||
status.rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.get(&member.account_id)
|
||||
.copied()
|
||||
.map(Into::into);
|
||||
status.cooldown = runtime.cooldown.get(&member.account_id).copied();
|
||||
status_by_account
|
||||
.entry(member.account_id.as_str())
|
||||
.or_insert_with(|| resolve_account_status(&status, now).status);
|
||||
|
||||
@@ -73,6 +73,13 @@ pub trait AccountsService: Send + Sync {
|
||||
command: UpdateAccount,
|
||||
) -> Result<AccountUpdateResult, AdminError>;
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
) -> Result<Option<AccountUpdateResult>, AdminError>;
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
@@ -582,6 +589,27 @@ impl AccountsService for DefaultAccountsService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
) -> Result<Option<AccountUpdateResult>, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
let result = self
|
||||
.accounts
|
||||
.lower_concurrency_limit(&account_id, limit, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider account"))?;
|
||||
if let Some(result) = &result {
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
|
||||
@@ -125,7 +125,11 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
&& command.audit_retention_days > 0
|
||||
&& valid_client_version(command.min_codex_desktop_version.as_deref())
|
||||
&& valid_client_version(command.min_codex_cli_version.as_deref())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok();
|
||||
&& valid_probe_model(command.account_auto_freeze_probe_model.as_deref())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok()
|
||||
&& (2..=1_000).contains(&command.account_auto_freeze_threshold)
|
||||
&& (60..=3_600).contains(&command.account_auto_freeze_window_seconds)
|
||||
&& (300..=604_800).contains(&command.account_auto_freeze_duration_seconds);
|
||||
if valid {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -136,3 +140,12 @@ fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError>
|
||||
fn valid_client_version(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
|
||||
}
|
||||
|
||||
fn valid_probe_model(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 128
|
||||
&& value == value.trim()
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
MutationContext, PageSize, Revision,
|
||||
@@ -168,13 +168,35 @@ impl AccountRuntimeStore for FakeRuntimeStore {
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
*self.requested_accounts.lock().expect("requested accounts") = account_ids.to_vec();
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::from([(
|
||||
cooldown: BTreeMap::from([(
|
||||
"acct_limited".to_owned(),
|
||||
Utc::now() + Duration::minutes(5),
|
||||
std::time::SystemTime::from(Utc::now() + Duration::minutes(5)).into(),
|
||||
)]),
|
||||
in_flight: Some(BTreeMap::from([("acct_available".to_owned(), 2)])),
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
fn group_record() -> AccountGroupRecord {
|
||||
@@ -215,7 +237,7 @@ fn member(account_id: &str, total_slots: u64) -> AccountGroupMemberFact {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
|
||||
@@ -544,6 +544,8 @@ pub(super) struct FakeAccountStore {
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
|
||||
quota_forecast_history: Mutex<QuotaForecastHistory>,
|
||||
update_commands: Mutex<Vec<UpdateAccount>>,
|
||||
pub(super) lowered_limits: Mutex<Vec<(String, u32)>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
@@ -551,7 +553,7 @@ impl FakeAccountStore {
|
||||
Self::with_account(account_record(kind), events)
|
||||
}
|
||||
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
pub(super) fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
@@ -562,9 +564,18 @@ impl FakeAccountStore {
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
quota_window_queries: Mutex::new(Vec::new()),
|
||||
quota_forecast_history: Mutex::new(QuotaForecastHistory::default()),
|
||||
update_commands: Mutex::new(Vec::new()),
|
||||
lowered_limits: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn update_commands(&self) -> Vec<UpdateAccount> {
|
||||
self.update_commands
|
||||
.lock()
|
||||
.expect("update commands")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(super) fn fail_next_commit(&self) {
|
||||
*self.fail_commit.lock().expect("store failure") = true;
|
||||
}
|
||||
@@ -630,7 +641,7 @@ impl FakeAccountStore {
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
};
|
||||
@@ -852,12 +863,45 @@ impl AccountStore for FakeAccountStore {
|
||||
self.record("store.update_account");
|
||||
self.record_context(context);
|
||||
self.require_commit()?;
|
||||
self.update_commands
|
||||
.lock()
|
||||
.expect("update commands")
|
||||
.push(command.clone());
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: ProviderAccountId::new(command.account_id).expect("account ID"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
limit: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>> {
|
||||
let mut accounts = self.accounts.lock().expect("accounts");
|
||||
let Some(account) = accounts
|
||||
.iter_mut()
|
||||
.find(|account| account.id == account_id.as_str())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !account.enabled
|
||||
|| account.concurrency_limit.map_or(5, |value| value.get()) <= limit.get()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
account.concurrency_limit = Some(limit);
|
||||
self.lowered_limits
|
||||
.lock()
|
||||
.expect("lowered limits")
|
||||
.push((account.id.clone(), limit.get()));
|
||||
Ok(Some(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: account_id.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -944,6 +988,13 @@ impl SettingsStore for StaticSettingsStore {
|
||||
usage_retention_days: 30,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,425 @@
|
||||
//! 容量熔断冻结恢复编排回归:到期探测、失败顺延与自适应并发下调。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration as TimeDelta, Utc};
|
||||
use gateway_admin::freeze_recovery::{FreezeRecoveryDeps, FreezeRecoveryTask};
|
||||
use gateway_admin::model::MutationContext;
|
||||
use gateway_admin::model::accounts::{AccountFreeze, AccountRuntimeSnapshot};
|
||||
use gateway_admin::model::settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RuntimeSettings,
|
||||
};
|
||||
use gateway_admin::ports::store::{
|
||||
AccountRuntimeStore, AccountStore, AdminStoreError, AdminStoreResult, SettingsStore,
|
||||
};
|
||||
use gateway_core::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::task::{ScheduledTask as _, WorkerCycleContext, WorkerId, WorkerKind};
|
||||
|
||||
use super::AdminHarness;
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, account_record, events, revision};
|
||||
|
||||
fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> RuntimeSettings {
|
||||
RuntimeSettings {
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: enabled,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: probe_enabled,
|
||||
account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()),
|
||||
account_auto_freeze_adaptive_concurrency: adaptive,
|
||||
updated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
struct FreezeSettingsStore {
|
||||
settings: RuntimeSettings,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SettingsStore for FreezeSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(self.settings.clone())
|
||||
}
|
||||
|
||||
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn replace_runtime_settings(
|
||||
&self,
|
||||
_: ReplaceRuntimeSettings,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<RuntimeSettings> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
_: AdminApiKey,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
Err(store_unavailable())
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
gateway_admin::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"freeze recovery test",
|
||||
"unused",
|
||||
)
|
||||
}
|
||||
|
||||
/// 记录解冻/顺延调用的运行态 fake;冻结与峰值证据由测试预置。
|
||||
struct FreezeRuntimeStore {
|
||||
freezes: BTreeMap<String, AccountFreeze>,
|
||||
peaks: BTreeMap<String, u32>,
|
||||
cleared: Mutex<Vec<String>>,
|
||||
extended: Mutex<Vec<(String, DateTime<Utc>)>>,
|
||||
}
|
||||
|
||||
impl FreezeRuntimeStore {
|
||||
fn new(freezes: BTreeMap<String, AccountFreeze>, peaks: BTreeMap<String, u32>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
freezes,
|
||||
peaks,
|
||||
cleared: Mutex::new(Vec::new()),
|
||||
extended: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn cleared(&self) -> Vec<String> {
|
||||
self.cleared.lock().expect("cleared lock").clone()
|
||||
}
|
||||
|
||||
fn extended(&self) -> Vec<(String, DateTime<Utc>)> {
|
||||
self.extended.lock().expect("extended lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AccountRuntimeStore for FreezeRuntimeStore {
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn account_runtime(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn active_freezes(&self) -> AdminStoreResult<BTreeMap<String, AccountFreeze>> {
|
||||
Ok(self.freezes.clone())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(account_ids
|
||||
.iter()
|
||||
.filter_map(|id| self.peaks.get(id).map(|peak| (id.clone(), *peak)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
assert_eq!(self.freezes.get(account_id), Some(expected));
|
||||
if let Some(until) = postpone_until {
|
||||
self.extended
|
||||
.lock()
|
||||
.expect("extended")
|
||||
.push((account_id.to_owned(), until));
|
||||
} else {
|
||||
self.cleared
|
||||
.lock()
|
||||
.expect("cleared")
|
||||
.push(account_id.to_owned());
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
struct SuccessfulProbe;
|
||||
|
||||
impl AccountProbe for SuccessfulProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingProbe;
|
||||
|
||||
impl AccountProbe for FailingProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> futures::future::BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(AccountProbeError::new(
|
||||
gateway_core::error::GatewayError::new(
|
||||
gateway_core::error::GatewayErrorKind::RateLimited,
|
||||
"still overloaded",
|
||||
),
|
||||
gateway_core::engine::probe::AccountProbeErrorSource::Upstream,
|
||||
Some(gateway_core::upstream::UpstreamSendState::NotSent),
|
||||
None,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn recovery_task(
|
||||
settings: RuntimeSettings,
|
||||
runtime: Arc<FreezeRuntimeStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
|
||||
recovery_task_with_store(settings, runtime, probe).await
|
||||
}
|
||||
|
||||
async fn recovery_task_with_store(
|
||||
settings: RuntimeSettings,
|
||||
runtime: Arc<FreezeRuntimeStore>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> (FreezeRecoveryTask, Arc<FakeAccountStore>) {
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let services = AdminHarness::new()
|
||||
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
|
||||
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
|
||||
.settings(Arc::new(FreezeSettingsStore {
|
||||
settings: settings.clone(),
|
||||
}))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.probe(probe)
|
||||
.build()
|
||||
.await;
|
||||
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
|
||||
accounts: services.accounts_handle(),
|
||||
store: store.clone(),
|
||||
runtime,
|
||||
settings: Arc::new(FreezeSettingsStore {
|
||||
settings: settings.clone(),
|
||||
}),
|
||||
});
|
||||
(task, store)
|
||||
}
|
||||
|
||||
async fn run_cycle(task: &FreezeRecoveryTask) {
|
||||
let worker = WorkerId::try_new(WorkerKind::AccountFreezeRecovery, "test").expect("worker id");
|
||||
let context = WorkerCycleContext::new(worker, None, CancellationToken::new());
|
||||
task.run_cycle(context)
|
||||
.await
|
||||
.expect("freeze recovery cycle");
|
||||
}
|
||||
|
||||
fn freeze_until(until: DateTime<Utc>) -> BTreeMap<String, AccountFreeze> {
|
||||
BTreeMap::from([(
|
||||
"acct_test".to_owned(),
|
||||
AccountFreeze {
|
||||
credential_revision: revision(1),
|
||||
until,
|
||||
generation: "freeze-generation".to_owned(),
|
||||
requires_probe: true,
|
||||
},
|
||||
)])
|
||||
}
|
||||
|
||||
fn freeze_due() -> BTreeMap<String, AccountFreeze> {
|
||||
freeze_until(Utc::now() - TimeDelta::seconds(1))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_success_clears_freeze() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_failure_postpones_freeze() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(FailingProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
let extended = runtime.extended();
|
||||
assert_eq!(extended.len(), 1);
|
||||
let (account_id, until) = &extended[0];
|
||||
assert_eq!(account_id, "acct_test");
|
||||
let remaining = until.timestamp_millis() - Utc::now().timestamp_millis();
|
||||
assert!(
|
||||
remaining > 7_000_000 && remaining <= 7_200_000,
|
||||
"postponed freeze should be about 2 hours, got {remaining}ms"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_policy_releases_due_freeze_without_probing() {
|
||||
let runtime = FreezeRuntimeStore::new(freeze_due(), BTreeMap::new());
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(false, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert_eq!(runtime.cleared(), vec!["acct_test".to_owned()]);
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_lowers_limit_to_observed_peak() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
|
||||
);
|
||||
let (task, store) = recovery_task(
|
||||
runtime_settings(true, false, true),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
// 峰值 4 × 0.8 = 3(不低于下限 2),低于全局默认 5,应下调到 3。
|
||||
assert_eq!(
|
||||
*store.lowered_limits.lock().expect("lowered limits"),
|
||||
vec![("acct_test".to_owned(), 3)]
|
||||
);
|
||||
assert!(store.update_commands().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
assert!(runtime.cleared().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_never_raises_limit() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::from([("acct_test".to_owned(), 40_u32)]),
|
||||
);
|
||||
let (task, store) = recovery_task(
|
||||
runtime_settings(true, false, true),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
// 峰值 40 的目标 32 高于全局默认 5;只降不升,不下发任何更新。
|
||||
assert!(store.update_commands().is_empty());
|
||||
}
|
||||
|
||||
/// 到期时间还早(超过探测提前量)的冻结不发起探测。
|
||||
#[tokio::test]
|
||||
async fn probe_skips_freezes_far_from_expiry() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_until(Utc::now() + TimeDelta::hours(1)),
|
||||
BTreeMap::new(),
|
||||
);
|
||||
let (task, _store) = recovery_task(
|
||||
runtime_settings(true, true, false),
|
||||
Arc::clone(&runtime),
|
||||
Arc::new(SuccessfulProbe),
|
||||
)
|
||||
.await;
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
}
|
||||
|
||||
/// 停用账号不参与探测恢复;并发下调同样跳过。
|
||||
#[tokio::test]
|
||||
async fn disabled_accounts_are_skipped() {
|
||||
let runtime = FreezeRuntimeStore::new(
|
||||
freeze_due(),
|
||||
BTreeMap::from([("acct_test".to_owned(), 4_u32)]),
|
||||
);
|
||||
let mut record = account_record("openai");
|
||||
record.enabled = false;
|
||||
let store = FakeAccountStore::with_account(record, events());
|
||||
let services = AdminHarness::new()
|
||||
.accounts(Arc::clone(&store) as Arc<dyn AccountStore>)
|
||||
.account_runtime(Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>)
|
||||
.settings(Arc::new(FreezeSettingsStore {
|
||||
settings: runtime_settings(true, true, true),
|
||||
}))
|
||||
.provider(FakeProviderAdmin::new("openai", events()))
|
||||
.probe(Arc::new(SuccessfulProbe))
|
||||
.build()
|
||||
.await;
|
||||
let task = FreezeRecoveryTask::new(FreezeRecoveryDeps {
|
||||
accounts: services.accounts_handle(),
|
||||
store: store.clone(),
|
||||
runtime: Arc::clone(&runtime) as Arc<dyn AccountRuntimeStore>,
|
||||
settings: Arc::new(FreezeSettingsStore {
|
||||
settings: runtime_settings(true, true, true),
|
||||
}),
|
||||
});
|
||||
|
||||
run_cycle(&task).await;
|
||||
|
||||
assert!(runtime.cleared().is_empty());
|
||||
assert!(runtime.extended().is_empty());
|
||||
assert!(store.update_commands().is_empty());
|
||||
}
|
||||
@@ -4,6 +4,7 @@ mod auth;
|
||||
mod auth_key;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod freeze_recovery;
|
||||
mod import_tasks;
|
||||
mod observability;
|
||||
mod openai;
|
||||
@@ -82,6 +83,7 @@ use gateway_core::{
|
||||
routing::{ConfigRevision, ProviderKind},
|
||||
runtime::SnapshotControl,
|
||||
};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(super) struct AdminHarness {
|
||||
default_password: String,
|
||||
@@ -488,6 +490,15 @@ impl AccountStore for UnavailableStore {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
_: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -530,6 +541,28 @@ impl AccountRuntimeStore for UnavailableStore {
|
||||
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot::default())
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -769,6 +769,13 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
updated_at: Utc::now(),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,13 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -66,7 +66,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
|
||||
quota,
|
||||
} = item;
|
||||
let status = projection.status.as_str().to_owned();
|
||||
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
|
||||
let cooldown = projection.cooldown;
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
@@ -75,7 +75,7 @@ pub(super) fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> Ac
|
||||
if account.authentication_kind == "api_key" {
|
||||
usage.window_label_display = "通用额度".to_owned();
|
||||
}
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now);
|
||||
AccountView {
|
||||
id: account.id.clone(),
|
||||
name: account.name,
|
||||
@@ -189,7 +189,7 @@ fn forecast_usd_display(value: Option<f64>) -> String {
|
||||
|
||||
pub(super) fn account_quota_view(
|
||||
mut quota: ProviderQuota,
|
||||
rate_limited_until: Option<DateTime<Utc>>,
|
||||
cooldown: Option<gateway_core::account::AccountCooldown>,
|
||||
now: DateTime<Utc>,
|
||||
) -> (AccountQuotaView, Option<String>) {
|
||||
quota.apply_limit_reached_display();
|
||||
@@ -200,12 +200,22 @@ pub(super) fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
|
||||
let rate_limited_until = cooldown.map(|value| china_datetime(&value.until.into()));
|
||||
let rate_limit_reason = cooldown.map(|value| {
|
||||
if value.kind.is_capacity_freeze() {
|
||||
"capacity_freeze".to_owned()
|
||||
} else {
|
||||
"upstream_rate_limit".to_owned()
|
||||
}
|
||||
});
|
||||
let recovery_probe_required = cooldown.is_some_and(|value| value.kind.requires_probe());
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
rate_limit_reason,
|
||||
recovery_probe_required,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
|
||||
@@ -361,8 +361,10 @@ pub struct AccountGroupRefView {
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
/// 冷却到期或可开始恢复探测的时间;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub rate_limit_reason: Option<String>,
|
||||
pub recovery_probe_required: bool,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,13 @@ pub struct RuntimeSettingsView {
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -69,6 +76,13 @@ pub struct UpdateRuntimeSettingsRequest {
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl UpdateRuntimeSettingsRequest {
|
||||
@@ -110,6 +124,35 @@ impl UpdateRuntimeSettingsRequest {
|
||||
self.min_codex_cli_version.as_deref(),
|
||||
"minCodexCliVersion",
|
||||
)?;
|
||||
for (value, field) in [
|
||||
(
|
||||
self.account_auto_freeze_threshold,
|
||||
"accountAutoFreezeThreshold",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_window_seconds,
|
||||
"accountAutoFreezeWindowSeconds",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_duration_seconds,
|
||||
"accountAutoFreezeDurationSeconds",
|
||||
),
|
||||
] {
|
||||
require_positive_i64(value, field)?;
|
||||
}
|
||||
if !(2..=1_000).contains(&self.account_auto_freeze_threshold) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeThreshold"));
|
||||
}
|
||||
if !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeWindowSeconds"));
|
||||
}
|
||||
if !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeDurationSeconds"));
|
||||
}
|
||||
validate_optional_probe_model(
|
||||
self.account_auto_freeze_probe_model.as_deref(),
|
||||
"accountAutoFreezeProbeModel",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -136,6 +179,14 @@ impl UpdateRuntimeSettingsRequest {
|
||||
.map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?,
|
||||
audit_retention_days: u32::try_from(self.audit_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
|
||||
account_auto_freeze_enabled: self.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold)
|
||||
.map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?,
|
||||
account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: self.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -157,6 +208,14 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold),
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
updated_at: settings.updated_at,
|
||||
}
|
||||
}
|
||||
@@ -452,6 +511,22 @@ fn validate_optional_client_version(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 探测模型为可选自由文本:非空、去首尾空白后不变、无控制字符且不超过 128 字节。
|
||||
fn validate_optional_probe_model(
|
||||
value: Option<&str>,
|
||||
field: &'static str,
|
||||
) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| {
|
||||
value.is_empty()
|
||||
|| value.len() > 128
|
||||
|| value != value.trim()
|
||||
|| value.bytes().any(|byte| byte.is_ascii_control())
|
||||
}) {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(),
|
||||
@@ -459,6 +534,11 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
"settingsUsageRetentionOverflow" => "usageRetentionDays 不合法".to_owned(),
|
||||
"settingsOpsRetentionOverflow" => "opsEventRetentionDays 不合法".to_owned(),
|
||||
"settingsAuditRetentionOverflow" => "auditRetentionDays 不合法".to_owned(),
|
||||
"settingsFreezeThresholdOverflow" => "accountAutoFreezeThreshold 不合法".to_owned(),
|
||||
"accountAutoFreezeThreshold" => "账号自动冻结阈值应为 2~1000 的整数".to_owned(),
|
||||
"accountAutoFreezeWindowSeconds" => "账号自动冻结统计窗口应为 60~3600 秒".to_owned(),
|
||||
"accountAutoFreezeDurationSeconds" => "账号自动冻结时长应为 300~604800 秒".to_owned(),
|
||||
"accountAutoFreezeProbeModel" => "探测模型格式不合法".to_owned(),
|
||||
"minCodexDesktopVersion" => "Codex Desktop 最低版本格式不合法".to_owned(),
|
||||
"minCodexCliVersion" => "Codex CLI 最低版本格式不合法".to_owned(),
|
||||
field => format!("{field} 字段不合法"),
|
||||
|
||||
@@ -463,7 +463,7 @@ mod provider {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
};
|
||||
|
||||
@@ -417,6 +417,13 @@ impl SettingsStore for MemorySettingsStore {
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
*settings = updated.clone();
|
||||
@@ -587,7 +594,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
@@ -601,7 +608,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::default(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
@@ -896,6 +903,15 @@ impl AccountStore for UnusedStore {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
_: &gateway_core::account::ProviderAccountId,
|
||||
_: gateway_core::account::AccountConcurrencyLimit,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountUpdateResult>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -933,17 +949,39 @@ impl AccountStore for UnusedStore {
|
||||
impl AccountRuntimeStore for UnusedStore {
|
||||
async fn active_rate_limits(&self) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::new(),
|
||||
cooldown: BTreeMap::new(),
|
||||
in_flight: Some(BTreeMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn account_runtime(&self, _: &[String]) -> AdminStoreResult<AccountRuntimeSnapshot> {
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::new(),
|
||||
cooldown: BTreeMap::new(),
|
||||
in_flight: Some(BTreeMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
Ok(BTreeMap::new())
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
_postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -1245,6 +1283,13 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,14 @@ fn update_body() -> Value {
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
"accountAutoFreezeThreshold": 12,
|
||||
"accountAutoFreezeWindowSeconds": 600,
|
||||
"accountAutoFreezeDurationSeconds": 7200,
|
||||
"accountAutoFreezeProbeEnabled": true,
|
||||
"accountAutoFreezeProbeModel": null,
|
||||
"accountAutoFreezeAdaptiveConcurrency": true
|
||||
})
|
||||
}
|
||||
|
||||
@@ -116,6 +123,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
@@ -143,6 +157,13 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
"accountAutoFreezeThreshold": 12,
|
||||
"accountAutoFreezeWindowSeconds": 600,
|
||||
"accountAutoFreezeDurationSeconds": 7200,
|
||||
"accountAutoFreezeProbeEnabled": true,
|
||||
"accountAutoFreezeProbeModel": null,
|
||||
"accountAutoFreezeAdaptiveConcurrency": true,
|
||||
"updatedAt": "2026-08-02T10:30:00Z"
|
||||
})
|
||||
);
|
||||
@@ -195,6 +216,13 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
updated_at: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
|
||||
@@ -583,6 +583,72 @@ impl AccountErrorReason {
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号冷却的原因与恢复方式。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AccountCooldownKind {
|
||||
/// 上游 429 临时限流;默认类别,兼容升级前未标记的存量 key。
|
||||
#[default]
|
||||
RateLimit,
|
||||
/// 容量类错误高频触发后由熔断策略写入的自动冻结。
|
||||
CapacityFreeze,
|
||||
/// 到期后仍阻止调度,必须由恢复探测成功或管理员恢复解除。
|
||||
CapacityFreezeProbe,
|
||||
}
|
||||
|
||||
impl AccountCooldownKind {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::RateLimit => "rate_limit",
|
||||
Self::CapacityFreeze => "capacity_freeze",
|
||||
Self::CapacityFreezeProbe => "capacity_freeze_probe",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_capacity_freeze(self) -> bool {
|
||||
matches!(self, Self::CapacityFreeze | Self::CapacityFreezeProbe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn requires_probe(self) -> bool {
|
||||
matches!(self, Self::CapacityFreezeProbe)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"rate_limit" => Some(Self::RateLimit),
|
||||
"capacity_freeze" => Some(Self::CapacityFreeze),
|
||||
"capacity_freeze_probe" => Some(Self::CapacityFreezeProbe),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号级冷却投影;探测冻结的 until 是下次探测时间,不是自动放行时间。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountCooldown {
|
||||
pub until: SystemTime,
|
||||
pub kind: AccountCooldownKind,
|
||||
}
|
||||
|
||||
impl AccountCooldown {
|
||||
#[must_use]
|
||||
pub fn is_active(self, now: SystemTime) -> bool {
|
||||
self.kind.requires_probe() || self.until > now
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SystemTime> for AccountCooldown {
|
||||
fn from(until: SystemTime) -> Self {
|
||||
Self {
|
||||
until,
|
||||
kind: AccountCooldownKind::RateLimit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 唯一状态解析器的完整输入事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusFacts {
|
||||
@@ -590,7 +656,7 @@ pub struct AccountStatusFacts {
|
||||
pub credential_state: CredentialState,
|
||||
pub access_token_expires_at: Option<SystemTime>,
|
||||
pub quota: QuotaState,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub cooldown: Option<AccountCooldown>,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
}
|
||||
@@ -601,8 +667,8 @@ pub struct AccountStatusProjection {
|
||||
pub status: AccountStatus,
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub error_message: Option<String>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却截止时间。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
/// 仅 `rate_limited` 状态携带仍有效的运行时冷却事实。
|
||||
pub cooldown: Option<AccountCooldown>,
|
||||
}
|
||||
|
||||
/// 从独立事实派生唯一、互斥的对外状态。
|
||||
@@ -625,21 +691,21 @@ pub fn resolve_account_status(
|
||||
status: AccountStatus::Error,
|
||||
error_reason: Some(facts.last_error_reason.unwrap_or(default_reason)),
|
||||
error_message: facts.last_error_message.clone(),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
};
|
||||
}
|
||||
if facts.quota.is_exhausted() {
|
||||
return status_projection(AccountStatus::QuotaExhausted);
|
||||
}
|
||||
if facts
|
||||
.rate_limited_until
|
||||
.is_some_and(|rate_limited_until| rate_limited_until > now)
|
||||
.cooldown
|
||||
.is_some_and(|cooldown| cooldown.is_active(now))
|
||||
{
|
||||
return AccountStatusProjection {
|
||||
status: AccountStatus::RateLimited,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: facts.rate_limited_until,
|
||||
cooldown: facts.cooldown,
|
||||
};
|
||||
}
|
||||
status_projection(AccountStatus::Normal)
|
||||
@@ -650,7 +716,7 @@ const fn status_projection(status: AccountStatus) -> AccountStatusProjection {
|
||||
status,
|
||||
error_reason: None,
|
||||
error_message: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -905,7 +971,7 @@ impl ProviderAccount {
|
||||
pub fn status_projection(
|
||||
&self,
|
||||
now: SystemTime,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
cooldown: Option<AccountCooldown>,
|
||||
) -> AccountStatusProjection {
|
||||
resolve_account_status(
|
||||
&AccountStatusFacts {
|
||||
@@ -913,7 +979,7 @@ impl ProviderAccount {
|
||||
credential_state: self.credential_state,
|
||||
access_token_expires_at: self.access_token_expires_at,
|
||||
quota: self.quota,
|
||||
rate_limited_until,
|
||||
cooldown,
|
||||
last_error_reason: self.last_error_reason,
|
||||
last_error_message: self.last_error_message.clone(),
|
||||
},
|
||||
|
||||
@@ -106,7 +106,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
/// Provider 归一化的剩余额度基点:0 耗尽,10_000 全部可用。
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
pub cooldown: Option<super::AccountCooldown>,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -316,8 +316,8 @@ impl AccountRuntimeSignals {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_rate_limit(mut self, rate_limited_until: Option<SystemTime>) -> Self {
|
||||
self.rate_limited_until = rate_limited_until;
|
||||
pub const fn with_rate_limit(mut self, cooldown: Option<super::AccountCooldown>) -> Self {
|
||||
self.cooldown = cooldown;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -654,7 +654,7 @@ impl AccountSelector {
|
||||
if !context.eligibility.bypasses_local_eligibility() {
|
||||
let status = candidate
|
||||
.account
|
||||
.status_projection(context.now, candidate.signals.rate_limited_until)
|
||||
.status_projection(context.now, candidate.signals.cooldown)
|
||||
.status;
|
||||
if status != AccountStatus::Normal {
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
|
||||
@@ -181,6 +181,16 @@ pub trait ProviderLeasePort: Send + Sync {
|
||||
&self,
|
||||
request: ProviderLeaseRequest,
|
||||
) -> BoxFuture<'_, Result<ProviderLeaseAcquisition, ProviderStoreError>>;
|
||||
|
||||
/// 读取指定账号当前的在途请求数;只用于容量熔断的峰值证据,
|
||||
/// 支持租约信号的存储实现覆盖,否则视为不可观测(空映射)。
|
||||
fn account_in_flight<'a>(
|
||||
&'a self,
|
||||
account_ids: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<'a, Result<BTreeMap<ProviderAccountId, u32>, ProviderStoreError>> {
|
||||
let _ = account_ids;
|
||||
Box::pin(async move { Ok(BTreeMap::new()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 从原始会话锚点派生的不可逆亲和键。
|
||||
@@ -596,12 +606,16 @@ pub trait ProviderCredentialStatePort: Send + Sync {
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// 临时 cooldown 只保存可丢失的调度截止时间,不进入账号持久状态。
|
||||
/// 账号级 cooldown 的来源类别;调度状态统一按 `rate_limited` 处理。
|
||||
pub use crate::account::AccountCooldownKind as ProviderCooldownKind;
|
||||
|
||||
/// 可丢失的账号冷却事实,不进入持久状态;探测冻结到期后仍需确认恢复。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
}
|
||||
|
||||
impl ProviderCooldown {
|
||||
@@ -610,11 +624,27 @@ impl ProviderCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self::new_with_kind(
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
ProviderCooldownKind::RateLimit,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn new_with_kind(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
until: SystemTime,
|
||||
kind: ProviderCooldownKind,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
until,
|
||||
kind,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -632,6 +662,19 @@ impl ProviderCooldown {
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scheduling_state(&self) -> crate::account::AccountCooldown {
|
||||
crate::account::AccountCooldown {
|
||||
until: self.until,
|
||||
kind: self.kind,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> ProviderCooldownKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域。
|
||||
@@ -749,6 +792,29 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 记录一次容量类失败并返回滑动窗口内的累计次数,同时把观测到的账号
|
||||
/// 在途并发并入窗口峰值(`in_flight` 为 0 表示本次未观测,跳过峰值更新)。
|
||||
/// 只服务容量熔断触发器;调用频率受失败频率约束,不需要批量接口。
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>>;
|
||||
|
||||
/// 普通请求成功后原子清除临时限流及失败证据;必须保留任何容量冻结。
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>>;
|
||||
|
||||
/// 读取窗口内观测到的在途并发峰值;无证据时返回 `None`。
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -864,6 +930,118 @@ pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
fn load_refresh_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>>;
|
||||
|
||||
/// 读取账号容量熔断策略;默认关闭,只有实现运行时设置的存储需要覆盖。
|
||||
fn load_freeze_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(ProviderFreezePolicy::disabled()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号容量熔断(自动冻结)策略;来源于 `runtime_settings`,
|
||||
/// 由 Provider 触发路径与恢复 worker 共享同一份配置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderFreezePolicy {
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window: Duration,
|
||||
freeze_duration: Duration,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl ProviderFreezePolicy {
|
||||
/// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致;
|
||||
/// store 层写入前已校验,这里兜底防御越界配置。
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
threshold: u32,
|
||||
window_seconds: u64,
|
||||
freeze_duration_seconds: u64,
|
||||
probe_enabled: bool,
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if !(2..=1_000).contains(&threshold)
|
||||
|| !(60..=3_600).contains(&window_seconds)
|
||||
|| !(300..=604_800).contains(&freeze_duration_seconds)
|
||||
|| probe_model.as_deref().is_some_and(|model| {
|
||||
model.is_empty()
|
||||
|| model.len() > 128
|
||||
|| model != model.trim()
|
||||
|| model.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
{
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate freeze policy",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
enabled,
|
||||
threshold,
|
||||
window: Duration::from_secs(window_seconds),
|
||||
freeze_duration: Duration::from_secs(freeze_duration_seconds),
|
||||
probe_enabled,
|
||||
probe_model,
|
||||
adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
/// 功能关闭时的全零策略;触发路径与 worker 都以此短路。
|
||||
#[must_use]
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
threshold: 2,
|
||||
window: Duration::from_secs(60),
|
||||
freeze_duration: Duration::from_secs(300),
|
||||
probe_enabled: false,
|
||||
probe_model: None,
|
||||
adaptive_concurrency: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
/// 窗口内触发冻结的请求级失败次数阈值。
|
||||
#[must_use]
|
||||
pub const fn threshold(&self) -> u32 {
|
||||
self.threshold
|
||||
}
|
||||
|
||||
/// 失败计数滑动窗口;每次失败都会顺延窗口。
|
||||
#[must_use]
|
||||
pub const fn window(&self) -> Duration {
|
||||
self.window
|
||||
}
|
||||
|
||||
/// 冻结时长;探测失败后的顺延也使用该值。
|
||||
#[must_use]
|
||||
pub const fn freeze_duration(&self) -> Duration {
|
||||
self.freeze_duration
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn probe_enabled(&self) -> bool {
|
||||
self.probe_enabled
|
||||
}
|
||||
|
||||
/// 探测模型;`None` 表示由 worker 选择账号可用的第一个模型。
|
||||
#[must_use]
|
||||
pub fn probe_model(&self) -> Option<&str> {
|
||||
self.probe_model.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn adaptive_concurrency(&self) -> bool {
|
||||
self.adaptive_concurrency
|
||||
}
|
||||
}
|
||||
|
||||
/// OAuth pending flow 的原始绑定只在 Provider 与 Store 边界内短暂存在。
|
||||
|
||||
@@ -23,10 +23,11 @@ pub enum WorkerKind {
|
||||
OpsFlush,
|
||||
Backup,
|
||||
AccountImport,
|
||||
AccountFreezeRecovery,
|
||||
}
|
||||
|
||||
impl WorkerKind {
|
||||
pub const ALL: [Self; 9] = [
|
||||
pub const ALL: [Self; 10] = [
|
||||
Self::OAuthRefresh,
|
||||
Self::QuotaCatalogHealth,
|
||||
Self::RuntimeSnapshotReconciliation,
|
||||
@@ -36,6 +37,7 @@ impl WorkerKind {
|
||||
Self::OpsFlush,
|
||||
Self::Backup,
|
||||
Self::AccountImport,
|
||||
Self::AccountFreezeRecovery,
|
||||
];
|
||||
|
||||
#[must_use]
|
||||
@@ -50,6 +52,7 @@ impl WorkerKind {
|
||||
Self::OpsFlush => "ops_flush",
|
||||
Self::Backup => "backup",
|
||||
Self::AccountImport => "account_import",
|
||||
Self::AccountFreezeRecovery => "account_freeze_recovery",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -222,7 +222,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -240,7 +240,7 @@ fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -313,13 +313,13 @@ fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
|
||||
let active_until = now + Duration::from_secs(60);
|
||||
let account = account("acct_rate_limited");
|
||||
|
||||
let active = account.status_projection(now, Some(active_until));
|
||||
let active = account.status_projection(now, Some(active_until.into()));
|
||||
assert_eq!(active.status, AccountStatus::RateLimited);
|
||||
assert_eq!(active.rate_limited_until, Some(active_until));
|
||||
assert_eq!(active.cooldown, Some(active_until.into()));
|
||||
|
||||
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
|
||||
let elapsed = account.status_projection(now, Some((now - Duration::from_secs(1)).into()));
|
||||
assert_eq!(elapsed.status, AccountStatus::Normal);
|
||||
assert_eq!(elapsed.rate_limited_until, None);
|
||||
assert_eq!(elapsed.cooldown, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -492,8 +492,8 @@ fn highest_priority_affinity_should_still_obey_existing_scheduling_constraints()
|
||||
);
|
||||
}
|
||||
AccountSchedulingBlocker::LocalAvailability => {
|
||||
candidates[0].signals.rate_limited_until =
|
||||
Some(selection.now + Duration::from_secs(60))
|
||||
candidates[0].signals.cooldown =
|
||||
Some((selection.now + Duration::from_secs(60)).into())
|
||||
}
|
||||
_ => unreachable!("test cases"),
|
||||
}
|
||||
@@ -643,7 +643,7 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
@@ -1062,3 +1062,32 @@ fn elapsed_quota_reset_does_not_fabricate_recovery() {
|
||||
.expect("healthy candidate available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn due_probe_freeze_blocks_scheduling_and_remains_visible_as_rate_limited() {
|
||||
use gateway_core::account::{AccountCooldown, AccountCooldownKind};
|
||||
let mut candidates = [
|
||||
weighted_candidate("acct_frozen", 100, 0),
|
||||
weighted_candidate("acct_ready", 1, 0),
|
||||
];
|
||||
let selection = context(RotationStrategy::Smart);
|
||||
candidates[0].signals.cooldown = Some(AccountCooldown {
|
||||
until: selection.now - Duration::from_secs(120),
|
||||
kind: AccountCooldownKind::CapacityFreezeProbe,
|
||||
});
|
||||
let projection = candidates[0]
|
||||
.account
|
||||
.status_projection(selection.now, candidates[0].signals.cooldown);
|
||||
assert_eq!(projection.status, AccountStatus::RateLimited);
|
||||
assert!(
|
||||
projection
|
||||
.cooldown
|
||||
.expect("freeze details")
|
||||
.kind
|
||||
.requires_probe()
|
||||
);
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("available account");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_ready");
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -17,7 +17,7 @@ use gateway_core::task::{
|
||||
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
const ACTIVE_KINDS: [WorkerKind; 8] = [
|
||||
const ACTIVE_KINDS: [WorkerKind; 9] = [
|
||||
WorkerKind::OAuthRefresh,
|
||||
WorkerKind::QuotaCatalogHealth,
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
@@ -26,6 +26,7 @@ const ACTIVE_KINDS: [WorkerKind; 8] = [
|
||||
WorkerKind::Retention,
|
||||
WorkerKind::Backup,
|
||||
WorkerKind::AccountImport,
|
||||
WorkerKind::AccountFreezeRecovery,
|
||||
];
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -54,6 +54,14 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
account_auto_freeze_enabled: command.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: command.account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds: command.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: command.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: command.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: command.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: command
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
},
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
@@ -73,6 +81,7 @@ impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
"min_codex_desktop_version".to_owned(),
|
||||
"min_codex_cli_version".to_owned(),
|
||||
"retention".to_owned(),
|
||||
"account_auto_freeze".to_owned(),
|
||||
],
|
||||
),
|
||||
};
|
||||
@@ -181,6 +190,13 @@ pub(crate) fn admin_runtime_settings(
|
||||
usage_retention_days: settings.usage_retention_days,
|
||||
ops_event_retention_days: settings.ops_event_retention_days,
|
||||
audit_retention_days: settings.audit_retention_days,
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: settings.account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings.account_auto_freeze_adaptive_concurrency,
|
||||
updated_at: settings.updated_at,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message,
|
||||
},
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::*;
|
||||
|
||||
use crate::postgres::{
|
||||
PgProviderAccountRepository, ProviderAccountRepository, ProviderAccountSummary,
|
||||
account_status_projection, load_rate_limited_until,
|
||||
account_status_projection, load_cooldown,
|
||||
};
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
@@ -49,19 +49,15 @@ impl PgObservabilityRepository {
|
||||
)
|
||||
.await?;
|
||||
let now = observed_at.into();
|
||||
let rate_limited_until =
|
||||
load_rate_limited_until(self.cooldowns.as_deref(), &accounts, now).await;
|
||||
let cooldown = load_cooldown(self.cooldowns.as_deref(), &accounts, now).await;
|
||||
let mut metrics = ProviderAccountMetrics {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
..ProviderAccountMetrics::default()
|
||||
};
|
||||
let mut normal_accounts = Vec::new();
|
||||
for account in &accounts {
|
||||
let projection = account_status_projection(
|
||||
account,
|
||||
now,
|
||||
rate_limited_until.get(&account.id).copied(),
|
||||
);
|
||||
let projection =
|
||||
account_status_projection(account, now, cooldown.get(&account.id).copied());
|
||||
match projection.status {
|
||||
gateway_core::account::AccountStatus::Normal => {
|
||||
metrics.normal = metrics.normal.saturating_add(1);
|
||||
|
||||
@@ -407,15 +407,11 @@ impl AccountStore for PgAdminAccountStore {
|
||||
));
|
||||
}
|
||||
let now = Utc::now();
|
||||
let rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.into_iter()
|
||||
.map(|(account_id, until)| (account_id, until.into()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let cooldown = runtime.cooldown;
|
||||
let now_system_time = std::time::SystemTime::from(now);
|
||||
let active_rate_limited_ids = rate_limited_until
|
||||
let active_rate_limited_ids = cooldown
|
||||
.iter()
|
||||
.filter(|(_, until)| **until > now_system_time)
|
||||
.filter(|(_, cooldown)| cooldown.is_active(now_system_time))
|
||||
.map(|(account_id, _)| account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let page =
|
||||
@@ -434,7 +430,7 @@ impl AccountStore for PgAdminAccountStore {
|
||||
let projection = account_status_projection(
|
||||
&summary,
|
||||
now.into(),
|
||||
rate_limited_until.get(&account_id).copied(),
|
||||
cooldown.get(&account_id).copied(),
|
||||
);
|
||||
let mut account = admin_account_record(summary)?;
|
||||
account.groups = groups_by_account.remove(&account_id).unwrap_or_default();
|
||||
@@ -466,12 +462,8 @@ impl AccountStore for PgAdminAccountStore {
|
||||
return Ok(None);
|
||||
};
|
||||
let now = Utc::now();
|
||||
let rate_limited_until = runtime
|
||||
.rate_limited_until
|
||||
.get(account_id)
|
||||
.copied()
|
||||
.map(Into::into);
|
||||
let projection = account_status_projection(&record.summary, now.into(), rate_limited_until);
|
||||
let cooldown = runtime.cooldown.get(account_id).copied();
|
||||
let projection = account_status_projection(&record.summary, now.into(), cooldown);
|
||||
let account_id = record.summary.id.clone();
|
||||
let mut groups = self
|
||||
.account_groups_by_account(std::slice::from_ref(&account_id))
|
||||
@@ -745,6 +737,69 @@ impl AccountStore for PgAdminAccountStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn lower_concurrency_limit(
|
||||
&self,
|
||||
account_id: &CoreProviderAccountId,
|
||||
limit: AccountConcurrencyLimit,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Option<AccountUpdateResult>> {
|
||||
let mut transaction = self.pool.begin().await.map_err(|_| {
|
||||
admin_store_error(ENTITY, postgres_unavailable("begin concurrency reduction"))
|
||||
})?;
|
||||
let result =
|
||||
async {
|
||||
// 与管理写入采用相同锁顺序;锁住默认值后再检查账号最新设置,避免把旧快照写回。
|
||||
let default_limit: i64 = sqlx::query_scalar(
|
||||
"select max_concurrent_per_account from runtime_settings where id = 1 for update"
|
||||
).fetch_one(&mut *transaction).await
|
||||
.map_err(|_| postgres_unavailable("lock default concurrency"))?;
|
||||
let changed = sqlx::query_scalar::<_, String>(
|
||||
"update provider_accounts set concurrency_limit = $2, updated_at = now()
|
||||
where id = $1 and enabled = true and coalesce(concurrency_limit, $3) > $2
|
||||
returning id",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.bind(i64::from(limit.get()))
|
||||
.bind(default_limit)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("lower account concurrency"))?;
|
||||
if changed.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
|
||||
append_admin_audit_event_in_transaction(
|
||||
&mut transaction,
|
||||
mutation_audit(
|
||||
context,
|
||||
"adapt_concurrency",
|
||||
"provider_account",
|
||||
account_id.as_str(),
|
||||
vec!["concurrency_limit".to_owned()],
|
||||
),
|
||||
revision,
|
||||
)
|
||||
.await?;
|
||||
Ok(Some(revision))
|
||||
}
|
||||
.await;
|
||||
let revision = super::repository::finish_admin_transaction(
|
||||
transaction,
|
||||
result,
|
||||
"lower account concurrency",
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
revision
|
||||
.map(|revision| {
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: admin_revision(revision)?,
|
||||
account_id: account_id.clone(),
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &CoreProviderAccountId,
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::ProviderAccountSummary;
|
||||
pub(crate) fn account_status_projection(
|
||||
account: &ProviderAccountSummary,
|
||||
now: SystemTime,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
cooldown: Option<gateway_core::account::AccountCooldown>,
|
||||
) -> AccountStatusProjection {
|
||||
resolve_account_status(
|
||||
&AccountStatusFacts {
|
||||
@@ -24,7 +24,7 @@ pub(crate) fn account_status_projection(
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until,
|
||||
cooldown,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
},
|
||||
@@ -36,18 +36,21 @@ pub(crate) fn account_status_projection(
|
||||
///
|
||||
/// 冷却属于可丢失的账号级运行时事实:端口不可用、单条读取失败或脏账号 ID
|
||||
/// 都视为没有冷却,不污染持久账号状态。凭据轮换不会解除上游对同一账号的限流。
|
||||
pub(crate) async fn load_rate_limited_until(
|
||||
pub(crate) async fn load_cooldown(
|
||||
cooldowns: Option<&dyn ProviderCooldownPort>,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: SystemTime,
|
||||
) -> BTreeMap<String, SystemTime> {
|
||||
) -> BTreeMap<String, gateway_core::account::AccountCooldown> {
|
||||
let Some(cooldowns) = cooldowns else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
let reads = accounts.iter().map(|account| async move {
|
||||
let account_id = ProviderAccountId::new(account.id.clone()).ok()?;
|
||||
let cooldown = cooldowns.read(&account_id).await.ok().flatten()?;
|
||||
(cooldown.until() > now).then(|| (account.id.clone(), cooldown.until()))
|
||||
cooldown
|
||||
.scheduling_state()
|
||||
.is_active(now)
|
||||
.then(|| (account.id.clone(), cooldown.scheduling_state()))
|
||||
});
|
||||
join_all(reads).await.into_iter().flatten().collect()
|
||||
}
|
||||
|
||||
@@ -11,7 +11,8 @@ use sqlx::{PgPool, Postgres, Transaction};
|
||||
use gateway_core::account::RotationStrategy;
|
||||
use gateway_core::policy::CodexClientVersion;
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
ProviderFreezePolicy, ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderStoreError,
|
||||
ProviderStoreErrorKind,
|
||||
};
|
||||
|
||||
use crate::{Revision, StoreError, StoreResult, postgres_unavailable};
|
||||
@@ -34,6 +35,13 @@ pub struct RuntimeSettings {
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -60,6 +68,34 @@ impl fmt::Debug for RuntimeSettings {
|
||||
.field("usage_retention_days", &self.usage_retention_days)
|
||||
.field("ops_event_retention_days", &self.ops_event_retention_days)
|
||||
.field("audit_retention_days", &self.audit_retention_days)
|
||||
.field(
|
||||
"account_auto_freeze_enabled",
|
||||
&self.account_auto_freeze_enabled,
|
||||
)
|
||||
.field(
|
||||
"account_auto_freeze_threshold",
|
||||
&self.account_auto_freeze_threshold,
|
||||
)
|
||||
.field(
|
||||
"account_auto_freeze_window_seconds",
|
||||
&self.account_auto_freeze_window_seconds,
|
||||
)
|
||||
.field(
|
||||
"account_auto_freeze_duration_seconds",
|
||||
&self.account_auto_freeze_duration_seconds,
|
||||
)
|
||||
.field(
|
||||
"account_auto_freeze_probe_enabled",
|
||||
&self.account_auto_freeze_probe_enabled,
|
||||
)
|
||||
.field(
|
||||
"account_auto_freeze_probe_model",
|
||||
&self.account_auto_freeze_probe_model,
|
||||
)
|
||||
.field(
|
||||
"account_auto_freeze_adaptive_concurrency",
|
||||
&self.account_auto_freeze_adaptive_concurrency,
|
||||
)
|
||||
.field("updated_at", &self.updated_at)
|
||||
.finish()
|
||||
}
|
||||
@@ -82,6 +118,13 @@ pub struct RuntimeSettingsUpdate {
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RuntimeSettingsUpdate {
|
||||
@@ -109,9 +152,13 @@ impl RuntimeSettingsUpdate {
|
||||
|| self.usage_retention_days < 31
|
||||
|| self.ops_event_retention_days == 0
|
||||
|| self.audit_retention_days == 0
|
||||
|| !(2..=1_000).contains(&self.account_auto_freeze_threshold)
|
||||
|| !(60..=3_600).contains(&self.account_auto_freeze_window_seconds)
|
||||
|| !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds)
|
||||
|| !valid_model_mappings(&self.model_mappings)
|
||||
|| !valid_client_version(self.min_codex_desktop_version.as_deref())
|
||||
|| !valid_client_version(self.min_codex_cli_version.as_deref())
|
||||
|| !valid_probe_model(self.account_auto_freeze_probe_model.as_deref())
|
||||
|| RotationStrategy::parse(&self.rotation_strategy).is_none()
|
||||
{
|
||||
return Err(StoreError::InvalidData {
|
||||
@@ -173,7 +220,11 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
|
||||
refresh_concurrency, max_concurrent_per_account, request_interval_ms,
|
||||
rotation_strategy, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds
|
||||
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds,
|
||||
account_auto_freeze_enabled, account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds, account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled, account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency
|
||||
from runtime_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
@@ -202,6 +253,25 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn load_freeze_policy(
|
||||
&self,
|
||||
) -> futures::future::BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let settings = RuntimeSettingsRepository::load_runtime_settings(self)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("load freeze policy"))?;
|
||||
ProviderFreezePolicy::try_new(
|
||||
settings.account_auto_freeze_enabled,
|
||||
settings.account_auto_freeze_threshold,
|
||||
settings.account_auto_freeze_window_seconds,
|
||||
settings.account_auto_freeze_duration_seconds,
|
||||
settings.account_auto_freeze_probe_enabled,
|
||||
settings.account_auto_freeze_probe_model,
|
||||
settings.account_auto_freeze_adaptive_concurrency,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_settings_in_transaction(
|
||||
@@ -212,7 +282,11 @@ pub(crate) async fn load_runtime_settings_in_transaction(
|
||||
refresh_concurrency, max_concurrent_per_account, request_interval_ms,
|
||||
rotation_strategy, model_mappings_json, usage_retention_days, ops_event_retention_days,
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds
|
||||
min_codex_cli_version, updated_at, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds,
|
||||
account_auto_freeze_enabled, account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds, account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled, account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency
|
||||
from runtime_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
@@ -250,6 +324,13 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
max_waiting_per_key = $13,
|
||||
max_waiting_per_account = $14,
|
||||
concurrency_wait_timeout_seconds = $15,
|
||||
account_auto_freeze_enabled = $16,
|
||||
account_auto_freeze_threshold = $17,
|
||||
account_auto_freeze_window_seconds = $18,
|
||||
account_auto_freeze_duration_seconds = $19,
|
||||
account_auto_freeze_probe_enabled = $20,
|
||||
account_auto_freeze_probe_model = $21,
|
||||
account_auto_freeze_adaptive_concurrency = $22,
|
||||
updated_at = now()
|
||||
where id = 1
|
||||
returning config_revision",
|
||||
@@ -269,6 +350,16 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
.bind(i64::from(update.max_waiting_per_key))
|
||||
.bind(i64::from(update.max_waiting_per_account))
|
||||
.bind(i64::from(update.concurrency_wait_timeout_seconds))
|
||||
.bind(update.account_auto_freeze_enabled)
|
||||
.bind(i64::from(update.account_auto_freeze_threshold))
|
||||
.bind(i64::try_from(update.account_auto_freeze_window_seconds).map_err(|_| invalid_numeric())?)
|
||||
.bind(
|
||||
i64::try_from(update.account_auto_freeze_duration_seconds)
|
||||
.map_err(|_| invalid_numeric())?,
|
||||
)
|
||||
.bind(update.account_auto_freeze_probe_enabled)
|
||||
.bind(update.account_auto_freeze_probe_model.as_deref())
|
||||
.bind(update.account_auto_freeze_adaptive_concurrency)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("update runtime settings in transaction"))?
|
||||
@@ -335,6 +426,13 @@ struct RuntimeSettingsRow {
|
||||
max_waiting_per_key: i64,
|
||||
max_waiting_per_account: i64,
|
||||
concurrency_wait_timeout_seconds: i64,
|
||||
account_auto_freeze_enabled: bool,
|
||||
account_auto_freeze_threshold: i64,
|
||||
account_auto_freeze_window_seconds: i64,
|
||||
account_auto_freeze_duration_seconds: i64,
|
||||
account_auto_freeze_probe_enabled: bool,
|
||||
account_auto_freeze_probe_model: Option<String>,
|
||||
account_auto_freeze_adaptive_concurrency: bool,
|
||||
}
|
||||
|
||||
fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSettings> {
|
||||
@@ -356,6 +454,13 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSett
|
||||
max_waiting_per_key: to_u32(row.max_waiting_per_key)?,
|
||||
max_waiting_per_account: to_u32(row.max_waiting_per_account)?,
|
||||
concurrency_wait_timeout_seconds: to_u32(row.concurrency_wait_timeout_seconds)?,
|
||||
account_auto_freeze_enabled: row.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: to_u32(row.account_auto_freeze_threshold)?,
|
||||
account_auto_freeze_window_seconds: to_u64(row.account_auto_freeze_window_seconds)?,
|
||||
account_auto_freeze_duration_seconds: to_u64(row.account_auto_freeze_duration_seconds)?,
|
||||
account_auto_freeze_probe_enabled: row.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: row.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: row.account_auto_freeze_adaptive_concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -398,3 +503,12 @@ fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
fn valid_client_version(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
|
||||
}
|
||||
|
||||
fn valid_probe_model(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 128
|
||||
&& value == value.trim()
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::future::join_all;
|
||||
use gateway_admin::{
|
||||
model::accounts::AccountRuntimeSnapshot,
|
||||
@@ -49,15 +50,23 @@ impl AccountRuntimeStore for RedisAdminAccountRuntimeStore {
|
||||
.read_credential_cooldown(account_id)
|
||||
.await
|
||||
.map(|cooldown| {
|
||||
cooldown.map(|cooldown| (account_id.clone(), cooldown.cooldown_until))
|
||||
cooldown.map(|cooldown| {
|
||||
(
|
||||
account_id.clone(),
|
||||
gateway_core::account::AccountCooldown {
|
||||
until: cooldown.cooldown_until.into(),
|
||||
kind: cooldown.kind,
|
||||
},
|
||||
)
|
||||
})
|
||||
})
|
||||
});
|
||||
let mut rate_limited_until = BTreeMap::new();
|
||||
let mut cooldown = BTreeMap::new();
|
||||
for result in join_all(reads).await {
|
||||
if let Some((account_id, until)) =
|
||||
result.map_err(|error| crate::admin_store_error("account runtime", error))?
|
||||
{
|
||||
rate_limited_until.insert(account_id, until);
|
||||
cooldown.insert(account_id, until);
|
||||
}
|
||||
}
|
||||
let in_flight = self
|
||||
@@ -72,8 +81,50 @@ impl AccountRuntimeStore for RedisAdminAccountRuntimeStore {
|
||||
.collect()
|
||||
});
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until,
|
||||
cooldown,
|
||||
in_flight,
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_freezes(
|
||||
&self,
|
||||
) -> AdminStoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
self.cooldowns
|
||||
.active_freezes()
|
||||
.await
|
||||
.map_err(|error| crate::admin_store_error("account runtime", error))
|
||||
}
|
||||
|
||||
async fn capacity_peaks(
|
||||
&self,
|
||||
account_ids: &[String],
|
||||
) -> AdminStoreResult<BTreeMap<String, u32>> {
|
||||
let reads = account_ids.iter().map(|account_id| async move {
|
||||
self.cooldowns
|
||||
.read_capacity_peak(account_id)
|
||||
.await
|
||||
.map(|peak| peak.map(|value| (account_id.clone(), value)))
|
||||
});
|
||||
let mut peaks = BTreeMap::new();
|
||||
for result in join_all(reads).await {
|
||||
if let Some((account_id, peak)) =
|
||||
result.map_err(|error| crate::admin_store_error("account runtime", error))?
|
||||
{
|
||||
peaks.insert(account_id, peak);
|
||||
}
|
||||
}
|
||||
Ok(peaks)
|
||||
}
|
||||
|
||||
async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
self.cooldowns
|
||||
.finish_freeze(account_id, expected, postpone_until)
|
||||
.await
|
||||
.map_err(|error| crate::admin_store_error("account runtime", error))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,13 @@ use gateway_admin::model::accounts::AccountRuntimeSnapshot;
|
||||
use gateway_core::{
|
||||
account::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError, ProviderStoreErrorKind,
|
||||
ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope,
|
||||
ProviderScopedCooldown, ProviderStoreError, ProviderStoreErrorKind,
|
||||
},
|
||||
};
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::{Revision, StoreError, StoreResult, redis_unavailable, require_nonempty};
|
||||
|
||||
@@ -22,17 +24,21 @@ local incoming = tonumber(ARGV[1])
|
||||
local incoming_until = tonumber(ARGV[2])
|
||||
if current > incoming then return 0 end
|
||||
local current_until = tonumber(redis.call('HGET', KEYS[1], 'until_ms') or '0')
|
||||
if current == incoming and current_until >= incoming_until then return 0 end
|
||||
local current_kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit'
|
||||
if current == incoming then
|
||||
if current_kind ~= 'rate_limit' and ARGV[4] == 'rate_limit' then return 0 end
|
||||
if current_kind == ARGV[4] and current_until >= incoming_until then return 0 end
|
||||
incoming_until = math.max(current_until, incoming_until)
|
||||
end
|
||||
local clock = redis.call('TIME')
|
||||
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
|
||||
if incoming_until <= now_ms then
|
||||
redis.call('DEL', KEYS[1])
|
||||
if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[3]) end
|
||||
return 0
|
||||
if incoming_until <= now_ms and ARGV[4] ~= 'capacity_freeze_probe' then return 0 end
|
||||
redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'until_ms', incoming_until, 'kind', ARGV[4], 'generation', ARGV[5])
|
||||
if ARGV[4] == 'capacity_freeze_probe' then
|
||||
redis.call('PERSIST', KEYS[1])
|
||||
else
|
||||
redis.call('PEXPIRE', KEYS[1], incoming_until - now_ms + 60000)
|
||||
end
|
||||
redis.call('HSET', KEYS[1], 'revision', ARGV[1], 'until_ms', ARGV[2])
|
||||
local ttl = incoming_until - now_ms + 60000
|
||||
redis.call('PEXPIRE', KEYS[1], ttl)
|
||||
if #KEYS > 1 then redis.call('ZADD', KEYS[2], incoming_until, ARGV[3]) end
|
||||
return 1
|
||||
"#;
|
||||
@@ -40,19 +46,15 @@ return 1
|
||||
const READ_SCRIPT: &str = r#"
|
||||
local revision = redis.call('HGET', KEYS[1], 'revision')
|
||||
local until_ms = redis.call('HGET', KEYS[1], 'until_ms')
|
||||
if revision == false or until_ms == false then
|
||||
redis.call('DEL', KEYS[1])
|
||||
if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[1]) end
|
||||
return {0, '0', '0'}
|
||||
end
|
||||
local kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit'
|
||||
local clock = redis.call('TIME')
|
||||
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
|
||||
if tonumber(until_ms) <= now_ms then
|
||||
if revision == false or until_ms == false or (kind ~= 'capacity_freeze_probe' and tonumber(until_ms) <= now_ms) then
|
||||
redis.call('DEL', KEYS[1])
|
||||
if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[1]) end
|
||||
return {0, '0', '0'}
|
||||
return {0, '0', '0', 'rate_limit', ''}
|
||||
end
|
||||
return {1, revision, until_ms}
|
||||
return {1, revision, until_ms, kind, redis.call('HGET', KEYS[1], 'generation') or ''}
|
||||
"#;
|
||||
|
||||
const INVALIDATE_SCRIPT: &str = r#"
|
||||
@@ -63,11 +65,56 @@ if #KEYS > 1 then redis.call('ZREM', KEYS[2], ARGV[2]) end
|
||||
return 1
|
||||
"#;
|
||||
|
||||
const ACTIVE_COOLDOWNS_SCRIPT: &str = r#"
|
||||
local clock = redis.call('TIME')
|
||||
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms)
|
||||
return redis.call('ZRANGEBYSCORE', KEYS[1], '(' .. now_ms, '+inf', 'WITHSCORES')
|
||||
// 普通推理成功只能清除临时限流和未形成冻结的证据,判断与删除必须原子执行。
|
||||
const SUCCESS_SCRIPT: &str = r#"
|
||||
local current = tonumber(redis.call('HGET', KEYS[1], 'revision') or '0')
|
||||
local kind = redis.call('HGET', KEYS[1], 'kind') or 'rate_limit'
|
||||
if current > tonumber(ARGV[1]) or kind ~= 'rate_limit' then return 0 end
|
||||
redis.call('DEL', KEYS[1], KEYS[3], KEYS[4])
|
||||
redis.call('ZREM', KEYS[2], ARGV[2])
|
||||
return 1
|
||||
"#;
|
||||
|
||||
// 探测结果只能修改读到的这一代冻结;删除后重建同 revision 的冻结也不匹配。
|
||||
const FINISH_FREEZE_SCRIPT: &str = r#"
|
||||
if redis.call('HGET', KEYS[1], 'generation') ~= ARGV[2]
|
||||
or redis.call('HGET', KEYS[1], 'revision') ~= ARGV[1] then return 0 end
|
||||
local kind = redis.call('HGET', KEYS[1], 'kind')
|
||||
if kind ~= 'capacity_freeze' and kind ~= 'capacity_freeze_probe' then return 0 end
|
||||
if ARGV[4] == '' then
|
||||
redis.call('DEL', KEYS[1], KEYS[3], KEYS[4])
|
||||
redis.call('ZREM', KEYS[2], ARGV[3])
|
||||
else
|
||||
local until_ms = math.max(tonumber(redis.call('HGET', KEYS[1], 'until_ms')), tonumber(ARGV[4]))
|
||||
redis.call('HSET', KEYS[1], 'until_ms', until_ms, 'generation', ARGV[5])
|
||||
redis.call('ZADD', KEYS[2], until_ms, ARGV[3])
|
||||
if kind == 'capacity_freeze_probe' then
|
||||
redis.call('PERSIST', KEYS[1])
|
||||
else
|
||||
local clock = redis.call('TIME')
|
||||
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
|
||||
redis.call('PEXPIRE', KEYS[1], math.max(1, until_ms - now_ms + 60000))
|
||||
end
|
||||
end
|
||||
return 1
|
||||
"#;
|
||||
|
||||
// 每次容量失败都顺延窗口 TTL(与刷新退避计数同语义),并把本次观测到的
|
||||
// 在途并发并入峰值证据;峰值与计数共享同一窗口生命周期。
|
||||
const RECORD_CAPACITY_FAILURE_SCRIPT: &str = r#"
|
||||
local count = redis.call('INCR', KEYS[1])
|
||||
local ttl_ms = tonumber(ARGV[1])
|
||||
redis.call('PEXPIRE', KEYS[1], ttl_ms)
|
||||
local in_flight = tonumber(ARGV[2])
|
||||
if in_flight > 0 then
|
||||
local peak = tonumber(redis.call('GET', KEYS[2]) or '0')
|
||||
if in_flight > peak then
|
||||
redis.call('SET', KEYS[2], in_flight, 'PX', ttl_ms)
|
||||
else
|
||||
redis.call('PEXPIRE', KEYS[2], ttl_ms)
|
||||
end
|
||||
end
|
||||
return count
|
||||
"#;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -75,6 +122,7 @@ pub struct CredentialCooldown {
|
||||
pub provider_account_id: String,
|
||||
pub credential_revision: Revision,
|
||||
pub cooldown_until: DateTime<Utc>,
|
||||
pub kind: ProviderCooldownKind,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -116,6 +164,22 @@ impl RedisCredentialCooldownRepository {
|
||||
format!("{}:account:active-cooldowns", self.namespace)
|
||||
}
|
||||
|
||||
fn capacity_failures_key(&self, provider_account_id: &str) -> StoreResult<String> {
|
||||
let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?;
|
||||
Ok(format!(
|
||||
"{}:account:{fingerprint}:capacity-failures",
|
||||
self.namespace
|
||||
))
|
||||
}
|
||||
|
||||
fn capacity_peak_key(&self, provider_account_id: &str) -> StoreResult<String> {
|
||||
let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?;
|
||||
Ok(format!(
|
||||
"{}:account:{fingerprint}:capacity-peak-inflight",
|
||||
self.namespace
|
||||
))
|
||||
}
|
||||
|
||||
fn scoped_key(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
@@ -135,6 +199,7 @@ impl RedisCredentialCooldownRepository {
|
||||
key: String,
|
||||
credential_revision: Revision,
|
||||
cooldown_until: DateTime<Utc>,
|
||||
kind: ProviderCooldownKind,
|
||||
index_member: Option<&str>,
|
||||
) -> StoreResult<bool> {
|
||||
let until_ms = cooldown_until.timestamp_millis();
|
||||
@@ -149,6 +214,8 @@ impl RedisCredentialCooldownRepository {
|
||||
.arg(credential_revision.get())
|
||||
.arg(until_ms)
|
||||
.arg(index_member)
|
||||
.arg(kind.as_str())
|
||||
.arg(uuid::Uuid::new_v4().to_string())
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
} else {
|
||||
@@ -157,6 +224,8 @@ impl RedisCredentialCooldownRepository {
|
||||
.arg(credential_revision.get())
|
||||
.arg(until_ms)
|
||||
.arg("")
|
||||
.arg(kind.as_str())
|
||||
.arg(uuid::Uuid::new_v4().to_string())
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
}
|
||||
@@ -168,7 +237,7 @@ impl RedisCredentialCooldownRepository {
|
||||
&self,
|
||||
key: String,
|
||||
index_member: Option<&str>,
|
||||
) -> StoreResult<Option<(Revision, DateTime<Utc>)>> {
|
||||
) -> StoreResult<Option<(Revision, DateTime<Utc>, ProviderCooldownKind, String)>> {
|
||||
let mut connection = self.connection.clone();
|
||||
let result = if let Some(index_member) = index_member {
|
||||
Script::new(READ_SCRIPT)
|
||||
@@ -184,7 +253,7 @@ impl RedisCredentialCooldownRepository {
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
};
|
||||
let (present, revision, until_ms): (i64, String, String) =
|
||||
let (present, revision, until_ms, kind, generation): (i64, String, String, String, String) =
|
||||
result.map_err(|_| redis_unavailable("read credential cooldown"))?;
|
||||
if present == 0 {
|
||||
return Ok(None);
|
||||
@@ -197,7 +266,13 @@ impl RedisCredentialCooldownRepository {
|
||||
.map_err(|_| invalid("cached cooldown expiry is invalid"))?;
|
||||
let cooldown_until = DateTime::from_timestamp_millis(until_ms)
|
||||
.ok_or_else(|| invalid("cached cooldown expiry is invalid"))?;
|
||||
Ok(Some((Revision::new(revision)?, cooldown_until)))
|
||||
let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit);
|
||||
Ok(Some((
|
||||
Revision::new(revision)?,
|
||||
cooldown_until,
|
||||
kind,
|
||||
generation,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn invalidate_at_key(
|
||||
@@ -227,30 +302,107 @@ impl RedisCredentialCooldownRepository {
|
||||
Ok(removed == 1)
|
||||
}
|
||||
|
||||
pub(crate) async fn active_cooldowns(&self) -> StoreResult<AccountRuntimeSnapshot> {
|
||||
async fn indexed_accounts(&self) -> StoreResult<Vec<String>> {
|
||||
let mut connection = self.connection.clone();
|
||||
let values = Script::new(ACTIVE_COOLDOWNS_SCRIPT)
|
||||
.key(self.active_index_key())
|
||||
.invoke_async::<Vec<String>>(&mut connection)
|
||||
redis::cmd("ZRANGE")
|
||||
.arg(self.active_index_key())
|
||||
.arg(0)
|
||||
.arg(-1)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("list active credential cooldowns"))?;
|
||||
if values.len() % 2 != 0 {
|
||||
return Err(invalid("active cooldown index is invalid"));
|
||||
}
|
||||
let mut rate_limited_until = std::collections::BTreeMap::new();
|
||||
for pair in values.chunks_exact(2) {
|
||||
let until_ms = pair[1]
|
||||
.parse::<i64>()
|
||||
.map_err(|_| invalid("active cooldown expiry is invalid"))?;
|
||||
let until = DateTime::from_timestamp_millis(until_ms)
|
||||
.ok_or_else(|| invalid("active cooldown expiry is invalid"))?;
|
||||
rate_limited_until.insert(pair[0].clone(), until);
|
||||
.map_err(|_| redis_unavailable("list indexed cooldowns"))
|
||||
}
|
||||
|
||||
pub(crate) async fn active_cooldowns(&self) -> StoreResult<AccountRuntimeSnapshot> {
|
||||
let mut cooldown = BTreeMap::new();
|
||||
for account_id in self.indexed_accounts().await? {
|
||||
if let Some((_, until, kind, _)) = self
|
||||
.read_at_key(self.key(&account_id)?, Some(&account_id))
|
||||
.await?
|
||||
{
|
||||
cooldown.insert(
|
||||
account_id,
|
||||
gateway_core::account::AccountCooldown {
|
||||
until: until.into(),
|
||||
kind,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(AccountRuntimeSnapshot {
|
||||
rate_limited_until,
|
||||
cooldown,
|
||||
in_flight: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// 包含已到探测时间但尚未确认恢复的冻结;到期不能从 worker 工作集中移除。
|
||||
pub(crate) async fn active_freezes(
|
||||
&self,
|
||||
) -> StoreResult<BTreeMap<String, gateway_admin::model::accounts::AccountFreeze>> {
|
||||
let mut freezes = BTreeMap::new();
|
||||
for account_id in self.indexed_accounts().await? {
|
||||
if let Some((revision, until, kind, generation)) = self
|
||||
.read_at_key(self.key(&account_id)?, Some(&account_id))
|
||||
.await?
|
||||
&& kind.is_capacity_freeze()
|
||||
{
|
||||
freezes.insert(
|
||||
account_id,
|
||||
gateway_admin::model::accounts::AccountFreeze {
|
||||
credential_revision: gateway_admin::model::Revision::new(revision.get())
|
||||
.map_err(|_| invalid("freeze revision"))?,
|
||||
until,
|
||||
generation,
|
||||
requires_probe: kind.requires_probe(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(freezes)
|
||||
}
|
||||
|
||||
pub(crate) async fn finish_freeze(
|
||||
&self,
|
||||
account_id: &str,
|
||||
expected: &gateway_admin::model::accounts::AccountFreeze,
|
||||
postpone_until: Option<DateTime<Utc>>,
|
||||
) -> StoreResult<bool> {
|
||||
let mut connection = self.connection.clone();
|
||||
let changed: i64 = Script::new(FINISH_FREEZE_SCRIPT)
|
||||
.key(self.key(account_id)?)
|
||||
.key(self.active_index_key())
|
||||
.key(self.capacity_failures_key(account_id)?)
|
||||
.key(self.capacity_peak_key(account_id)?)
|
||||
.arg(expected.credential_revision.get())
|
||||
.arg(&expected.generation)
|
||||
.arg(account_id)
|
||||
.arg(
|
||||
postpone_until
|
||||
.map(|until| until.timestamp_millis().to_string())
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.arg(uuid::Uuid::new_v4().to_string())
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("finish capacity freeze"))?;
|
||||
Ok(changed == 1)
|
||||
}
|
||||
|
||||
/// 读取窗口内观测到的在途并发峰值;key 随窗口 TTL 过期,无需额外清理。
|
||||
pub(crate) async fn read_capacity_peak(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
) -> StoreResult<Option<u32>> {
|
||||
let mut connection = self.connection.clone();
|
||||
let peak: Option<i64> = redis::cmd("GET")
|
||||
.arg(self.capacity_peak_key(provider_account_id)?)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("read capacity peak in-flight"))?;
|
||||
peak.map(u32::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| invalid("capacity peak in-flight is invalid"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -265,6 +417,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
self.key(&cooldown.provider_account_id)?,
|
||||
cooldown.credential_revision,
|
||||
cooldown.cooldown_until,
|
||||
cooldown.kind,
|
||||
Some(&cooldown.provider_account_id),
|
||||
)
|
||||
.await
|
||||
@@ -282,11 +435,14 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
self.read_at_key(self.key(provider_account_id)?, Some(provider_account_id))
|
||||
.await
|
||||
.map(|value| {
|
||||
value.map(|(credential_revision, cooldown_until)| CredentialCooldown {
|
||||
provider_account_id: provider_account_id.to_owned(),
|
||||
credential_revision,
|
||||
cooldown_until,
|
||||
})
|
||||
value.map(
|
||||
|(credential_revision, cooldown_until, kind, _)| CredentialCooldown {
|
||||
provider_account_id: provider_account_id.to_owned(),
|
||||
credential_revision,
|
||||
cooldown_until,
|
||||
kind,
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -317,8 +473,11 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
// 账号删除:清除该账号的 account key 与全部 model-scoped key。
|
||||
// 用 SCAN 精确匹配命名空间内该账号前缀,避免 KEYS 阻塞。
|
||||
let mut connection = self.connection.clone();
|
||||
let account_key = self.key(provider_account_id)?;
|
||||
let mut keys = vec![account_key.clone()];
|
||||
let mut keys = vec![
|
||||
self.key(provider_account_id)?,
|
||||
self.capacity_failures_key(provider_account_id)?,
|
||||
self.capacity_peak_key(provider_account_id)?,
|
||||
];
|
||||
let pattern = format!(
|
||||
"{}:account:{}:cooldown:*",
|
||||
self.namespace,
|
||||
@@ -341,21 +500,21 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if keys.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
let removed: i64 = redis::cmd("DEL")
|
||||
.arg(keys)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("delete account cooldown keys"))?;
|
||||
let index_removed: i64 = redis::cmd("ZREM")
|
||||
.arg(self.active_index_key())
|
||||
.arg(provider_account_id)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("remove active account cooldown"))?;
|
||||
Ok(removed > 0 || index_removed > 0)
|
||||
// 删除与索引移除同属一个原子边界;否则新冻结可能在两步之间写入后丢失索引。
|
||||
let removed: i64 = Script::new(
|
||||
r#"
|
||||
local removed = 0
|
||||
for i = 2, #KEYS do removed = removed + redis.call('DEL', KEYS[i]) end
|
||||
return removed + redis.call('ZREM', KEYS[1], ARGV[1])
|
||||
"#,
|
||||
)
|
||||
.key(self.active_index_key())
|
||||
.key(keys)
|
||||
.arg(provider_account_id)
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("delete account cooldowns and index"))?;
|
||||
Ok(removed > 0)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -370,6 +529,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
credential_revision: Revision::new(cooldown.credential_revision().get())
|
||||
.map_err(|_| provider_invalid("encode credential cooldown"))?,
|
||||
cooldown_until: cooldown.until().into(),
|
||||
kind: cooldown.kind(),
|
||||
};
|
||||
CredentialCooldownRepository::cache_credential_cooldown(self, &record)
|
||||
.await
|
||||
@@ -390,10 +550,11 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| provider_invalid("decode credential cooldown"))?;
|
||||
let revision = CredentialRevision::new(record.credential_revision.get())
|
||||
.map_err(|_| provider_invalid("decode credential cooldown"))?;
|
||||
Ok(ProviderCooldown::new(
|
||||
Ok(ProviderCooldown::new_with_kind(
|
||||
account_id,
|
||||
revision,
|
||||
record.cooldown_until.into(),
|
||||
record.kind,
|
||||
))
|
||||
})
|
||||
.transpose()
|
||||
@@ -430,6 +591,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| provider_invalid("encode scoped credential cooldown"))?,
|
||||
revision,
|
||||
cooldown.until().into(),
|
||||
ProviderCooldownKind::RateLimit,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
@@ -451,7 +613,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("read scoped credential cooldown"))?
|
||||
.map(|(revision, until)| {
|
||||
.map(|(revision, until, _, _)| {
|
||||
Ok(ProviderScopedCooldown::new(
|
||||
account_id.clone(),
|
||||
CredentialRevision::new(revision.get())
|
||||
@@ -494,6 +656,75 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| provider_unavailable("clear all credential cooldowns"))
|
||||
})
|
||||
}
|
||||
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
window: Duration,
|
||||
in_flight: u32,
|
||||
) -> futures::future::BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut connection = self.connection.clone();
|
||||
let window_ms = u64::try_from(window.as_millis())
|
||||
.map_err(|_| provider_invalid("encode capacity failure window"))?;
|
||||
let count: i64 = Script::new(RECORD_CAPACITY_FAILURE_SCRIPT)
|
||||
.key(
|
||||
self.capacity_failures_key(account_id.as_str())
|
||||
.map_err(|_| provider_invalid("encode capacity failure key"))?,
|
||||
)
|
||||
.key(
|
||||
self.capacity_peak_key(account_id.as_str())
|
||||
.map_err(|_| provider_invalid("encode capacity peak key"))?,
|
||||
)
|
||||
.arg(i64::try_from(window_ms).unwrap_or(i64::MAX))
|
||||
.arg(i64::from(in_flight))
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("record capacity failure"))?;
|
||||
u32::try_from(count).map_err(|_| provider_invalid("decode capacity failure count"))
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut connection = self.connection.clone();
|
||||
let _: i64 = Script::new(SUCCESS_SCRIPT)
|
||||
.key(
|
||||
self.key(account_id.as_str())
|
||||
.map_err(|_| provider_invalid("cooldown key"))?,
|
||||
)
|
||||
.key(self.active_index_key())
|
||||
.key(
|
||||
self.capacity_failures_key(account_id.as_str())
|
||||
.map_err(|_| provider_invalid("capacity count key"))?,
|
||||
)
|
||||
.key(
|
||||
self.capacity_peak_key(account_id.as_str())
|
||||
.map_err(|_| provider_invalid("capacity peak key"))?,
|
||||
)
|
||||
.arg(through_revision.get())
|
||||
.arg(account_id.as_str())
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("clear cooldown after success"))?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> futures::future::BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
self.read_capacity_peak(account_id.as_str())
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("read capacity peak in-flight"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> StoreError {
|
||||
|
||||
@@ -524,7 +524,7 @@ impl RedisProviderLeaseCoordinator {
|
||||
last_started_at: signal.last_started_at.map(Into::into),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -645,6 +645,36 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn account_in_flight<'a>(
|
||||
&'a self,
|
||||
account_ids: &'a [ProviderAccountId],
|
||||
) -> futures::future::BoxFuture<
|
||||
'a,
|
||||
Result<std::collections::BTreeMap<ProviderAccountId, u32>, ProviderStoreError>,
|
||||
> {
|
||||
Box::pin(async move {
|
||||
if account_ids.is_empty() {
|
||||
return Ok(std::collections::BTreeMap::new());
|
||||
}
|
||||
let ids = account_ids
|
||||
.iter()
|
||||
.map(|account_id| account_id.as_str().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let signals = self
|
||||
.repository
|
||||
.credential_runtime_signals(&ids)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("load account in-flight signals"))?;
|
||||
Ok(signals
|
||||
.into_iter()
|
||||
.filter_map(|signal| {
|
||||
let account_id = ProviderAccountId::new(signal.resource_id).ok()?;
|
||||
Some((account_id, signal.in_flight))
|
||||
})
|
||||
.collect())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
|
||||
|
||||
@@ -855,6 +855,30 @@ impl ProviderCooldownPort for StaticCooldowns {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_window: Duration,
|
||||
_in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(0) })
|
||||
}
|
||||
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: gateway_core::account::CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
}
|
||||
|
||||
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
|
||||
|
||||
@@ -852,9 +852,9 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
sort: None,
|
||||
},
|
||||
AccountRuntimeSnapshot {
|
||||
rate_limited_until: BTreeMap::from([(
|
||||
cooldown: BTreeMap::from([(
|
||||
"acct_alpha".to_owned(),
|
||||
now + TimeDelta::minutes(5),
|
||||
SystemTime::from(now + TimeDelta::minutes(5)).into(),
|
||||
)]),
|
||||
in_flight: None,
|
||||
},
|
||||
@@ -3387,3 +3387,83 @@ async fn model_access_only_batch_update_preserves_other_settings_and_survives_re
|
||||
assert_eq!(loaded.summary.model_access, AccountModelAccess::all());
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adaptive_concurrency_uses_latest_locked_settings_without_overwriting_admin_fields() {
|
||||
let Some(database) = TestDatabase::create("adaptive_concurrency").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let store = admin_account_store(&database.pool);
|
||||
let id = ProviderAccountId::new("acct_adaptive").expect("id");
|
||||
repository
|
||||
.insert_provider_account(account(id.as_str(), "adaptive-user"))
|
||||
.await
|
||||
.expect("account");
|
||||
let group = "grp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, 'kept group', '#112233FF', now(), now())")
|
||||
.bind(group).execute(&database.pool).await.expect("group");
|
||||
sqlx::query("insert into account_group_accounts (account_group_id, provider_account_id, created_at) values ($1, $2, now())")
|
||||
.bind(group).bind(id.as_str()).execute(&database.pool).await.expect("membership");
|
||||
for (enabled, account_limit, default_limit, expected_limit, changes) in [
|
||||
(false, Some(8_i64), 10_i64, Some(8_i64), false),
|
||||
(true, Some(2), 10, Some(2), false),
|
||||
(true, None, 2, None, false),
|
||||
(true, None, 10, Some(3), true),
|
||||
] {
|
||||
let mut admin = database.pool.begin().await.expect("admin transaction");
|
||||
sqlx::query("update runtime_settings set max_concurrent_per_account = $1 where id = 1")
|
||||
.bind(default_limit)
|
||||
.execute(&mut *admin)
|
||||
.await
|
||||
.expect("latest default");
|
||||
let worker_store = store.clone();
|
||||
let worker_id = id.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
worker_store
|
||||
.lower_concurrency_limit(
|
||||
&worker_id,
|
||||
gateway_core::account::AccountConcurrencyLimit::new(3).expect("limit"),
|
||||
&MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "adaptive-worker".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("atomic adaptation")
|
||||
});
|
||||
// 管理员持有设置锁时提交新的账号事实,worker 必须依据提交后的值判断。
|
||||
sqlx::query("update provider_accounts set enabled = $2, concurrency_limit = $3, weight = 7, notes = 'administrator edit' where id = $1")
|
||||
.bind(id.as_str()).bind(enabled).bind(account_limit).execute(&mut *admin).await.expect("concurrent admin edit");
|
||||
admin.commit().await.expect("commit admin edit");
|
||||
let result = worker.await.expect("worker");
|
||||
assert_eq!(result.is_some(), changes);
|
||||
let actual = repository
|
||||
.load_provider_account(id.as_str())
|
||||
.await
|
||||
.expect("read account")
|
||||
.expect("account");
|
||||
assert_eq!(actual.summary.enabled, enabled);
|
||||
assert_eq!(
|
||||
actual
|
||||
.summary
|
||||
.concurrency_limit
|
||||
.map(|value| i64::from(value.get())),
|
||||
expected_limit
|
||||
);
|
||||
assert_eq!(actual.summary.weight.get(), 7);
|
||||
assert_eq!(actual.summary.notes.as_deref(), Some("administrator edit"));
|
||||
assert_eq!(
|
||||
account_group_ids(&database.pool, id.as_str()).await,
|
||||
vec![group.to_owned()]
|
||||
);
|
||||
}
|
||||
let audited_fields: Vec<Vec<String>> = sqlx::query_scalar(
|
||||
"select changed_fields from admin_audit_events where action = 'adapt_concurrency'",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.expect("audit");
|
||||
assert_eq!(audited_fields, vec![vec!["concurrency_limit".to_owned()]]);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -27,6 +27,13 @@ fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettingsUpdate {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +53,30 @@ fn runtime_settings_reject_invalid_model_mapping() {
|
||||
assert!(settings.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_reject_out_of_range_auto_freeze() {
|
||||
for update in [
|
||||
RuntimeSettingsUpdate {
|
||||
account_auto_freeze_threshold: 1,
|
||||
..settings_with_margin(3_600)
|
||||
},
|
||||
RuntimeSettingsUpdate {
|
||||
account_auto_freeze_window_seconds: 59,
|
||||
..settings_with_margin(3_600)
|
||||
},
|
||||
RuntimeSettingsUpdate {
|
||||
account_auto_freeze_duration_seconds: 299,
|
||||
..settings_with_margin(3_600)
|
||||
},
|
||||
RuntimeSettingsUpdate {
|
||||
account_auto_freeze_probe_model: Some(" pad ".to_owned()),
|
||||
..settings_with_margin(3_600)
|
||||
},
|
||||
] {
|
||||
assert!(update.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_reject_non_semver_client_min() {
|
||||
let settings = RuntimeSettingsUpdate {
|
||||
@@ -238,3 +269,52 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
|
||||
assert!(snapshot.config_revision > before.config_revision);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_freeze_defaults_off_and_migration_preserves_explicit_opt_in() {
|
||||
let Some(database) = TestDatabase::create("freeze_opt_in").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
assert!(
|
||||
!repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("default settings")
|
||||
.account_auto_freeze_enabled
|
||||
);
|
||||
sqlx::query("update runtime_settings set account_auto_freeze_enabled = true where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("old default");
|
||||
let migration = include_str!("../../../../migrations/0011_account_auto_freeze_opt_in.sql");
|
||||
sqlx::raw_sql(migration)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("migrate untouched default");
|
||||
assert!(
|
||||
!repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("settings")
|
||||
.account_auto_freeze_enabled
|
||||
);
|
||||
sqlx::query("update runtime_settings set account_auto_freeze_enabled = true where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("explicit opt-in");
|
||||
sqlx::query("insert into admin_audit_events (id, actor_kind, actor_ref, action, entity_kind, entity_ref, changed_fields, created_at) values ('freeze-opt-in', 'system', 'test', 'settings.replace', 'runtime_settings', '1', array['account_auto_freeze'], now())")
|
||||
.execute(&database.pool).await.expect("saved settings audit");
|
||||
sqlx::raw_sql(migration)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("preserve opt-in");
|
||||
assert!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("settings")
|
||||
.account_auto_freeze_enabled
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ use std::time::{Duration as StdDuration, SystemTime};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_core::{
|
||||
account::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown},
|
||||
provider_ports::{
|
||||
ProviderCooldownKind, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
},
|
||||
};
|
||||
use gateway_store::{
|
||||
Revision,
|
||||
@@ -18,6 +20,7 @@ fn credential_cooldown_is_revision_fenced() {
|
||||
provider_account_id: "account-1".to_owned(),
|
||||
credential_revision: Revision::new(2).expect("positive revision"),
|
||||
cooldown_until: Utc::now() + Duration::seconds(30),
|
||||
kind: ProviderCooldownKind::RateLimit,
|
||||
};
|
||||
assert_eq!(cooldown.credential_revision.get(), 2);
|
||||
}
|
||||
@@ -159,6 +162,7 @@ async fn credential_cooldown_read_removes_expired_grace_key() {
|
||||
provider_account_id: "acct_cooldown_expiry".to_owned(),
|
||||
credential_revision: Revision::new(1).expect("positive revision"),
|
||||
cooldown_until: millisecond_precision(cooldown_until),
|
||||
kind: ProviderCooldownKind::RateLimit,
|
||||
};
|
||||
repository
|
||||
.cache_credential_cooldown(&cooldown)
|
||||
@@ -284,6 +288,7 @@ fn cooldown(account_id: &str, revision: u64, seconds: i64) -> CredentialCooldown
|
||||
provider_account_id: account_id.to_owned(),
|
||||
credential_revision: Revision::new(revision).expect("positive revision"),
|
||||
cooldown_until: millisecond_precision(Utc::now() + Duration::seconds(seconds)),
|
||||
kind: ProviderCooldownKind::RateLimit,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,3 +316,233 @@ async fn namespace_keys(connection: &mut ConnectionManager, namespace: &str) ->
|
||||
.await
|
||||
.expect("list isolated cooldown keys")
|
||||
}
|
||||
|
||||
fn runtime_store(
|
||||
repository: &RedisCredentialCooldownRepository,
|
||||
connection: ConnectionManager,
|
||||
namespace: &str,
|
||||
) -> gateway_store::redis::RedisAdminAccountRuntimeStore {
|
||||
gateway_store::redis::RedisAdminAccountRuntimeStore::new(
|
||||
repository.clone(),
|
||||
gateway_store::redis::RedisCredentialLeaseRepository::new(connection, namespace)
|
||||
.expect("lease repository"),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_freeze_survives_due_time_and_runtime_restart_until_probe_succeeds() {
|
||||
use gateway_admin::ports::store::AccountRuntimeStore as _;
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let frozen = CredentialCooldown {
|
||||
kind: ProviderCooldownKind::CapacityFreezeProbe,
|
||||
cooldown_until: millisecond_precision(Utc::now() - Duration::seconds(90)),
|
||||
..cooldown("acct_probe_due", 1, 30)
|
||||
};
|
||||
repository
|
||||
.cache_credential_cooldown(&frozen)
|
||||
.await
|
||||
.expect("write due freeze");
|
||||
let runtime = runtime_store(&repository, connection.clone(), &namespace);
|
||||
let snapshot = runtime.active_rate_limits().await.expect("status snapshot");
|
||||
assert!(snapshot.cooldown[&frozen.provider_account_id].is_active(SystemTime::now()));
|
||||
assert!(
|
||||
snapshot.cooldown[&frozen.provider_account_id]
|
||||
.kind
|
||||
.requires_probe()
|
||||
);
|
||||
let key = namespace_keys(&mut connection, &namespace)
|
||||
.await
|
||||
.into_iter()
|
||||
.find(|key| key.ends_with(":cooldown"))
|
||||
.expect("freeze key");
|
||||
let ttl: i64 = redis::cmd("PTTL")
|
||||
.arg(key)
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.expect("TTL");
|
||||
assert_eq!(
|
||||
ttl, -1,
|
||||
"awaiting probe must not expire while worker is unavailable"
|
||||
);
|
||||
drop(runtime);
|
||||
let restarted = runtime_store(&repository, connection, &namespace);
|
||||
let freezes = restarted
|
||||
.active_freezes()
|
||||
.await
|
||||
.expect("due freezes after restart");
|
||||
assert_eq!(freezes.len(), 1);
|
||||
assert!(
|
||||
restarted
|
||||
.finish_freeze(
|
||||
&frozen.provider_account_id,
|
||||
&freezes[&frozen.provider_account_id],
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect("probe completion")
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.read_credential_cooldown(&frozen.provider_account_id)
|
||||
.await
|
||||
.expect("read")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_probe_cannot_recreate_manual_recovery_or_clear_or_extend_new_freeze() {
|
||||
use gateway_admin::ports::store::AccountRuntimeStore as _;
|
||||
let Some((repository, connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let frozen = CredentialCooldown {
|
||||
kind: ProviderCooldownKind::CapacityFreezeProbe,
|
||||
..cooldown("acct_probe_generation", 1, 30)
|
||||
};
|
||||
let runtime = runtime_store(&repository, connection, &namespace);
|
||||
repository
|
||||
.cache_credential_cooldown(&frozen)
|
||||
.await
|
||||
.expect("freeze");
|
||||
let original =
|
||||
runtime.active_freezes().await.expect("snapshot")[&frozen.provider_account_id].clone();
|
||||
repository
|
||||
.delete_account_cooldowns(&frozen.provider_account_id)
|
||||
.await
|
||||
.expect("manual recovery");
|
||||
let postponed = Utc::now() + Duration::hours(2);
|
||||
assert!(
|
||||
!runtime
|
||||
.finish_freeze(&frozen.provider_account_id, &original, Some(postponed))
|
||||
.await
|
||||
.expect("stale failure")
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.read_credential_cooldown(&frozen.provider_account_id)
|
||||
.await
|
||||
.expect("read")
|
||||
.is_none()
|
||||
);
|
||||
// 同凭据版本、相同截止时间的新冻结仍必须视为不同代次。
|
||||
repository
|
||||
.cache_credential_cooldown(&frozen)
|
||||
.await
|
||||
.expect("new freeze");
|
||||
assert!(
|
||||
!runtime
|
||||
.finish_freeze(&frozen.provider_account_id, &original, None)
|
||||
.await
|
||||
.expect("stale success")
|
||||
);
|
||||
assert!(
|
||||
!runtime
|
||||
.finish_freeze(&frozen.provider_account_id, &original, Some(postponed))
|
||||
.await
|
||||
.expect("stale failure")
|
||||
);
|
||||
let current =
|
||||
runtime.active_freezes().await.expect("new snapshot")[&frozen.provider_account_id].clone();
|
||||
assert_ne!(current.generation, original.generation);
|
||||
assert!(
|
||||
runtime
|
||||
.finish_freeze(&frozen.provider_account_id, ¤t, Some(postponed))
|
||||
.await
|
||||
.expect("postpone current freeze")
|
||||
);
|
||||
assert!(
|
||||
!runtime
|
||||
.finish_freeze(&frozen.provider_account_id, ¤t, None)
|
||||
.await
|
||||
.expect("result from previous probe cycle")
|
||||
);
|
||||
let extended = runtime.active_freezes().await.expect("extended snapshot")
|
||||
[&frozen.provider_account_id]
|
||||
.clone();
|
||||
assert!(extended.until >= current.until);
|
||||
assert!(
|
||||
runtime
|
||||
.finish_freeze(&frozen.provider_account_id, &extended, None)
|
||||
.await
|
||||
.expect("recover current cycle")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_success_and_later_429_preserve_capacity_freeze_and_peak_evidence() {
|
||||
let Some((repository, _connection, _namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let id = ProviderAccountId::new("acct_late_success").expect("account id");
|
||||
let revision = CredentialRevision::new(1).expect("revision");
|
||||
repository
|
||||
.record_capacity_failure(&id, StdDuration::from_secs(600), 8)
|
||||
.await
|
||||
.expect("failure evidence");
|
||||
let frozen = CredentialCooldown {
|
||||
kind: ProviderCooldownKind::CapacityFreezeProbe,
|
||||
..cooldown(id.as_str(), 1, 30)
|
||||
};
|
||||
repository
|
||||
.cache_credential_cooldown(&frozen)
|
||||
.await
|
||||
.expect("freeze");
|
||||
repository
|
||||
.clear_after_success(&id, revision)
|
||||
.await
|
||||
.expect("late success");
|
||||
assert_eq!(
|
||||
repository.capacity_peak_in_flight(&id).await.expect("peak"),
|
||||
Some(8)
|
||||
);
|
||||
assert!(
|
||||
!repository
|
||||
.cache_credential_cooldown(&cooldown(id.as_str(), 1, 60))
|
||||
.await
|
||||
.expect("429 must not downgrade freeze")
|
||||
);
|
||||
assert_eq!(
|
||||
repository
|
||||
.read_credential_cooldown(id.as_str())
|
||||
.await
|
||||
.expect("read"),
|
||||
Some(frozen)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_success_still_clears_rate_limit_and_unfrozen_failure_evidence() {
|
||||
let Some((repository, _connection, _namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let id = ProviderAccountId::new("acct_success_429").expect("id");
|
||||
repository
|
||||
.record_capacity_failure(&id, StdDuration::from_secs(600), 3)
|
||||
.await
|
||||
.expect("failure evidence");
|
||||
repository
|
||||
.cache_credential_cooldown(&cooldown(id.as_str(), 1, 30))
|
||||
.await
|
||||
.expect("429");
|
||||
repository
|
||||
.clear_after_success(&id, CredentialRevision::new(1).expect("revision"))
|
||||
.await
|
||||
.expect("success");
|
||||
assert!(
|
||||
repository
|
||||
.read_credential_cooldown(id.as_str())
|
||||
.await
|
||||
.expect("read")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.capacity_peak_in_flight(&id)
|
||||
.await
|
||||
.expect("peak")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -24,7 +24,10 @@ use gateway_core::account::{
|
||||
QuotaAccessChange, QuotaAccessState, QuotaEvidence, QuotaObservation, QuotaObservationTouch,
|
||||
QuotaState, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownKind, ProviderCooldownPort, ProviderFreezePolicy,
|
||||
ProviderLeasePort, ProviderRuntimePolicyPort,
|
||||
};
|
||||
use gateway_protocol::openai::events::{
|
||||
ParsedRateLimits, RateLimitDetails, RateLimitWindow, parse_rate_limit_headers,
|
||||
};
|
||||
@@ -184,10 +187,17 @@ pub struct CodexCredentialQuotaService {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
/// 冻结策略的短 TTL 缓存:失败路径热读,避免每个容量错误都查询设置。
|
||||
freeze_policy_cache: Mutex<Option<(ProviderFreezePolicy, Instant)>>,
|
||||
scheduling: CodexQuotaSchedulingProjection,
|
||||
reset_consume_locks: Mutex<HashMap<ProviderAccountId, Arc<Mutex<()>>>>,
|
||||
}
|
||||
|
||||
/// 冻结策略缓存活跃期;过期后下一次容量错误重新读取运行时设置。
|
||||
const FREEZE_POLICY_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum QuotaRefreshAuthority {
|
||||
ObserveAccess,
|
||||
@@ -481,6 +491,8 @@ impl CodexCredentialQuotaService {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store: Arc::clone(repository.store()),
|
||||
@@ -489,11 +501,89 @@ impl CodexCredentialQuotaService {
|
||||
http,
|
||||
base_url,
|
||||
cooldowns,
|
||||
leases,
|
||||
runtime_policy,
|
||||
freeze_policy_cache: Mutex::new(None),
|
||||
scheduling: CodexQuotaSchedulingProjection::default(),
|
||||
reset_consume_locks: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// 读取容量熔断策略(带短 TTL 缓存);读取失败退化为关闭,熔断不得放大失败。
|
||||
async fn freeze_policy(&self) -> ProviderFreezePolicy {
|
||||
{
|
||||
let cache = self.freeze_policy_cache.lock().await;
|
||||
if let Some((policy, loaded_at)) = cache.as_ref()
|
||||
&& loaded_at.elapsed() < FREEZE_POLICY_CACHE_TTL
|
||||
{
|
||||
return policy.clone();
|
||||
}
|
||||
}
|
||||
let loaded = self.runtime_policy.load_freeze_policy().await.ok();
|
||||
let mut cache = self.freeze_policy_cache.lock().await;
|
||||
if let Some(policy) = loaded {
|
||||
*cache = Some((policy.clone(), Instant::now()));
|
||||
return policy;
|
||||
}
|
||||
cache
|
||||
.as_ref()
|
||||
.filter(|(_, loaded_at)| loaded_at.elapsed() < FREEZE_POLICY_CACHE_TTL)
|
||||
.map_or_else(ProviderFreezePolicy::disabled, |(policy, _)| policy.clone())
|
||||
}
|
||||
|
||||
/// 容量熔断入口:滑动窗口内累计容量类失败,达到阈值即写入带
|
||||
/// `CapacityFreeze` 类别的账号级冷却。调度侧立即屏蔽该账号,恢复由
|
||||
/// freeze-recovery worker 处理;本路径只依赖 Redis 可丢失事实。
|
||||
pub async fn apply_capacity_failure(&self, account: &ProviderAccount, observed_at: SystemTime) {
|
||||
let policy = self.freeze_policy().await;
|
||||
if !policy.enabled() {
|
||||
return;
|
||||
}
|
||||
let in_flight = self.current_in_flight(account.id()).await;
|
||||
let Ok(count) = self
|
||||
.cooldowns
|
||||
.record_capacity_failure(account.id(), policy.window(), in_flight)
|
||||
.await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if count < policy.threshold() {
|
||||
return;
|
||||
}
|
||||
let Some(until) = observed_at.checked_add(policy.freeze_duration()) else {
|
||||
return;
|
||||
};
|
||||
let cooldown = ProviderCooldown::new_with_kind(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
until,
|
||||
if policy.probe_enabled() {
|
||||
ProviderCooldownKind::CapacityFreezeProbe
|
||||
} else {
|
||||
ProviderCooldownKind::CapacityFreeze
|
||||
},
|
||||
);
|
||||
if self.cooldowns.put_if_later(cooldown).await.is_ok() {
|
||||
tracing::warn!(
|
||||
account_id = account.id().as_str(),
|
||||
threshold = policy.threshold(),
|
||||
window_seconds = policy.window().as_secs(),
|
||||
freeze_seconds = policy.freeze_duration().as_secs(),
|
||||
peak_in_flight = in_flight,
|
||||
"账号容量熔断触发:冻结该账号一段时间",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn current_in_flight(&self, account_id: &ProviderAccountId) -> u32 {
|
||||
self.leases
|
||||
.account_in_flight(std::slice::from_ref(account_id))
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|signals| signals.get(account_id).copied())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// 查询当前账号由 Codex Desktop 暴露的主动额度重置卡。
|
||||
pub async fn list_reset_credits(
|
||||
&self,
|
||||
@@ -607,7 +697,7 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
|
||||
/// 成功推理是额度可访问的权威证据,同时解除账号级 429 冷却。
|
||||
pub(crate) async fn record_successful_inference(
|
||||
pub async fn record_successful_inference(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
observed_at: SystemTime,
|
||||
@@ -628,7 +718,7 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
}
|
||||
self.cooldowns
|
||||
.clear(account.id(), account.revision())
|
||||
.clear_after_success(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(|error| CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
@@ -975,11 +1065,11 @@ impl CodexCredentialQuotaService {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 读取账号当前是否处于临时限流(429)冷却,及到期时间。
|
||||
pub async fn rate_limited_until(
|
||||
/// 读取有效的账号冷却事实;等待恢复探测的冻结到期后仍有效。
|
||||
pub async fn cooldown(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
|
||||
) -> Result<Option<gateway_core::account::AccountCooldown>, CodexCredentialQuotaError> {
|
||||
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
|
||||
CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
@@ -988,11 +1078,8 @@ impl CodexCredentialQuotaService {
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let until = cooldown.until();
|
||||
if until <= SystemTime::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(until))
|
||||
let state = cooldown.scheduling_state();
|
||||
Ok(state.is_active(SystemTime::now()).then_some(state))
|
||||
}
|
||||
|
||||
/// 读取单账号最后一次落库的 Provider quota,并由 Codex 域解析展示窗口。
|
||||
|
||||
@@ -451,11 +451,7 @@ impl CodexCredentialSelector {
|
||||
let mut rate_limits = HashMap::with_capacity(accounts.len());
|
||||
if !diagnostic {
|
||||
for account in &accounts {
|
||||
let until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
let until = self.quota.cooldown(account.id()).await.unwrap_or(None);
|
||||
rate_limits.insert(account.id().clone(), until);
|
||||
}
|
||||
}
|
||||
@@ -487,7 +483,7 @@ impl CodexCredentialSelector {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
})
|
||||
@@ -1330,7 +1326,7 @@ fn affinity_selection_for_bound_account(
|
||||
};
|
||||
match candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.rate_limited_until)
|
||||
.status_projection(now, candidate.signals.cooldown)
|
||||
.status
|
||||
{
|
||||
AccountStatus::Normal => AffinitySelection::preferred(account_id),
|
||||
@@ -1349,7 +1345,7 @@ fn affinity_unavailable_reason(
|
||||
) -> AffinityEscapeReason {
|
||||
match candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.rate_limited_until)
|
||||
.status_projection(now, candidate.signals.cooldown)
|
||||
.status
|
||||
{
|
||||
AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
|
||||
@@ -112,6 +112,8 @@ pub async fn initialize(
|
||||
http.clone(),
|
||||
config.base_url().to_owned(),
|
||||
ports.cooldowns(),
|
||||
Arc::clone(&leases),
|
||||
Arc::clone(&runtime_policy),
|
||||
));
|
||||
let profile_statistics = Arc::new(CodexCredentialProfileService::new(
|
||||
repository.clone(),
|
||||
|
||||
@@ -403,6 +403,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
|
||||
response_origin: &request.response_origin,
|
||||
cyber_policy_scope: None,
|
||||
allows_account_state_mutation,
|
||||
allows_capacity_feedback: !request.context.is_diagnostic_required_account(),
|
||||
};
|
||||
let active_account = request.lease.account().clone();
|
||||
let cookie_header = build_cookie_header(request.lease.cookies())?;
|
||||
@@ -577,6 +578,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
response_origin: &response_origin,
|
||||
cyber_policy_scope: cyber_policy_scope.as_ref(),
|
||||
allows_account_state_mutation,
|
||||
allows_capacity_feedback: !context.is_diagnostic_required_account(),
|
||||
};
|
||||
let mut active_account = lease.account().clone();
|
||||
let cookie_header = build_cookie_header(lease.cookies())?;
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
//! OpenAI 上游失败分类、恢复决策与稳定错误投影。
|
||||
|
||||
use super::*;
|
||||
use crate::transport::diagnostics::is_capacity_error;
|
||||
|
||||
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集。
|
||||
///
|
||||
/// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type。
|
||||
/// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集。
|
||||
/// 容量拒绝影响短期调度健康度,但不证明账号凭据或额度失效。
|
||||
const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[
|
||||
"server_is_overloaded",
|
||||
"slow_down",
|
||||
"rate_limit_exceeded",
|
||||
"rate_limit_error",
|
||||
"server_error",
|
||||
@@ -32,17 +34,6 @@ fn openai_account_score_failure_reason(error: &ProviderError) -> Option<&str> {
|
||||
/// 返回该 OpenAI 失败是否属于 Smart 账号计分闭集。
|
||||
#[doc(hidden)]
|
||||
pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
|
||||
let client_error = error.client_visible_upstream_error();
|
||||
if is_capacity_error(
|
||||
error
|
||||
.upstream_code()
|
||||
.map(OpaqueUpstreamValue::as_str)
|
||||
.or_else(|| client_error.and_then(ClientVisibleUpstreamError::code)),
|
||||
client_error.and_then(ClientVisibleUpstreamError::error_type),
|
||||
client_error.map(ClientVisibleUpstreamError::message),
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
openai_account_score_failure_reason(error).is_some_and(is_openai_account_score_failure_reason)
|
||||
}
|
||||
|
||||
@@ -53,6 +44,7 @@ pub(super) struct MappedProviderFailure {
|
||||
/// 原始上游错误描述,仅在凭据错误状态下持久化。
|
||||
pub(super) error_message: Option<String>,
|
||||
pub(super) cyber_policy_failure: bool,
|
||||
pub(super) upstream_capacity_failure: bool,
|
||||
pub(super) set_cookie_headers: Vec<String>,
|
||||
pub(super) rate_limit_headers: Vec<(String, String)>,
|
||||
pub(super) observation: Option<ProviderResponseObservation>,
|
||||
@@ -67,6 +59,7 @@ impl MappedProviderFailure {
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
upstream_capacity_failure: false,
|
||||
set_cookie_headers: Vec::new(),
|
||||
rate_limit_headers: Vec::new(),
|
||||
observation: None,
|
||||
@@ -211,6 +204,7 @@ pub(super) struct OpenAiFailureContext<'a> {
|
||||
pub(super) response_origin: &'a Url,
|
||||
pub(super) cyber_policy_scope: Option<&'a CodexCyberPolicyScope>,
|
||||
pub(super) allows_account_state_mutation: bool,
|
||||
pub(super) allows_capacity_feedback: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -410,6 +404,13 @@ pub(super) async fn apply_failure(
|
||||
"Failed to persist OpenAI response cookies"
|
||||
);
|
||||
}
|
||||
// 探测只验证恢复,不能把探测自身的单请求并发当作业务负载证据。
|
||||
if context.allows_capacity_feedback && failure.upstream_capacity_failure {
|
||||
context
|
||||
.quota
|
||||
.apply_capacity_failure(account, SystemTime::now())
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn schedule_authoritative_quota_refresh_after_failure(
|
||||
@@ -1269,6 +1270,12 @@ pub(super) fn map_upstream_failure(
|
||||
),
|
||||
error_message: failure.client_message,
|
||||
cyber_policy_failure,
|
||||
// 仅上游明确返回的容量/服务不可用计数;本地连接保护也会映射为
|
||||
// Unavailable,不能用通用 ProviderErrorKind 推断容量证据。
|
||||
upstream_capacity_failure: matches!(
|
||||
category,
|
||||
CodexFailureCategory::CapacityUnavailable | CodexFailureCategory::Unavailable
|
||||
),
|
||||
set_cookie_headers: failure.set_cookie_headers,
|
||||
rate_limit_headers: failure.rate_limit_headers,
|
||||
observation,
|
||||
|
||||
@@ -1591,6 +1591,30 @@ impl ProviderCooldownPort for TestCooldown {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_window: Duration,
|
||||
_in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(0) })
|
||||
}
|
||||
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: gateway_core::account::CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
}
|
||||
|
||||
struct TestRuntimePolicy;
|
||||
|
||||
@@ -20,7 +20,8 @@ use gateway_core::engine::{
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
|
||||
ProviderCooldownPort, ProviderLeasePort, ProviderSessionAffinityKey,
|
||||
ProviderSessionAffinityPort,
|
||||
};
|
||||
use gateway_core::routing::{
|
||||
ClientRoutingScope, FrozenAccountScope, ProviderKind, RuntimeAccount, RuntimeAccountDirectory,
|
||||
@@ -178,6 +179,8 @@ fn selector_with_runtime(
|
||||
http,
|
||||
OFFICIAL_CODEX_BASE_URL.to_owned(),
|
||||
cooldowns,
|
||||
Arc::clone(&leases) as Arc<dyn ProviderLeasePort>,
|
||||
crate::support::runtime_policy(),
|
||||
));
|
||||
CodexCredentialSelector::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
//! 容量熔断触发:高频容量类失败按滑动窗口计数冻结账号,成功调用清空证据。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownKind, ProviderCooldownPort as _, ProviderFreezePolicy,
|
||||
};
|
||||
use provider_openai::credential::CodexCredentialQuotaService;
|
||||
|
||||
use super::{MemoryAccountStore, create_account, wire_profile};
|
||||
use crate::support::{MemoryCooldownPort, StaticFreezePolicy, TestLeaseCoordinator};
|
||||
|
||||
fn freeze_policy(threshold: u32) -> ProviderFreezePolicy {
|
||||
ProviderFreezePolicy::try_new(true, threshold, 600, 7_200, true, None, true)
|
||||
.expect("valid freeze policy")
|
||||
}
|
||||
|
||||
async fn freeze_service(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
policy: ProviderFreezePolicy,
|
||||
) -> (Arc<CodexCredentialQuotaService>, Arc<MemoryCooldownPort>) {
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let quota = Arc::new(CodexCredentialQuotaService::new(
|
||||
store.repository(),
|
||||
wire_profile(),
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
provider_openai::OFFICIAL_CODEX_BASE_URL.to_owned(),
|
||||
Arc::clone(&cooldowns) as Arc<dyn gateway_core::provider_ports::ProviderCooldownPort>,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
StaticFreezePolicy::policy_port(policy),
|
||||
));
|
||||
(quota, cooldowns)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_failures_below_threshold_do_not_freeze() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_capacity_below").await;
|
||||
let account = store
|
||||
.account("acct_capacity_below")
|
||||
.expect("created account");
|
||||
let (quota, cooldowns) = freeze_service(&store, freeze_policy(3)).await;
|
||||
|
||||
for _ in 0..2 {
|
||||
quota
|
||||
.apply_capacity_failure(&account, SystemTime::now())
|
||||
.await;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
cooldowns
|
||||
.capacity_evidence(account.id())
|
||||
.map(|(count, _)| count),
|
||||
Some(2)
|
||||
);
|
||||
assert!(
|
||||
quota
|
||||
.cooldown(account.id())
|
||||
.await
|
||||
.expect("cooldown read")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_failures_reaching_threshold_write_capacity_freeze_cooldown() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_capacity_freeze").await;
|
||||
let account = store
|
||||
.account("acct_capacity_freeze")
|
||||
.expect("created account");
|
||||
let (quota, cooldowns) = freeze_service(&store, freeze_policy(3)).await;
|
||||
|
||||
for _ in 0..3 {
|
||||
quota
|
||||
.apply_capacity_failure(&account, SystemTime::now())
|
||||
.await;
|
||||
}
|
||||
|
||||
let until = quota
|
||||
.cooldown(account.id())
|
||||
.await
|
||||
.expect("cooldown read")
|
||||
.expect("freeze cooldown written");
|
||||
let remaining = until
|
||||
.until
|
||||
.duration_since(SystemTime::now())
|
||||
.expect("freeze in the future");
|
||||
assert!(
|
||||
remaining > Duration::from_secs(7_000) && remaining <= Duration::from_secs(7_200),
|
||||
"freeze should last about 2 hours, got {remaining:?}"
|
||||
);
|
||||
let cooldown = cooldowns.read(account.id()).await.expect("read cooldown");
|
||||
assert_eq!(
|
||||
cooldown.expect("cooldown").kind(),
|
||||
ProviderCooldownKind::CapacityFreezeProbe
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_freeze_policy_records_nothing() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_capacity_disabled").await;
|
||||
let account = store
|
||||
.account("acct_capacity_disabled")
|
||||
.expect("created account");
|
||||
let (quota, cooldowns) = freeze_service(&store, ProviderFreezePolicy::disabled()).await;
|
||||
|
||||
for _ in 0..5 {
|
||||
quota
|
||||
.apply_capacity_failure(&account, SystemTime::now())
|
||||
.await;
|
||||
}
|
||||
|
||||
assert!(cooldowns.capacity_evidence(account.id()).is_none());
|
||||
assert!(
|
||||
quota
|
||||
.cooldown(account.id())
|
||||
.await
|
||||
.expect("cooldown read")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_inference_clears_capacity_evidence() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_capacity_success").await;
|
||||
let account = store
|
||||
.account("acct_capacity_success")
|
||||
.expect("created account");
|
||||
let (quota, cooldowns) = freeze_service(&store, freeze_policy(100)).await;
|
||||
|
||||
for _ in 0..3 {
|
||||
quota
|
||||
.apply_capacity_failure(&account, SystemTime::now())
|
||||
.await;
|
||||
}
|
||||
assert!(cooldowns.capacity_evidence(account.id()).is_some());
|
||||
|
||||
quota
|
||||
.record_successful_inference(&account, SystemTime::now())
|
||||
.await
|
||||
.expect("record success");
|
||||
|
||||
assert!(cooldowns.capacity_evidence(account.id()).is_none());
|
||||
}
|
||||
|
||||
/// 阈值与窗口边界使用与迁移 check 约束一致的构造校验。
|
||||
#[test]
|
||||
fn freeze_policy_rejects_out_of_range_configuration() {
|
||||
for (threshold, window, duration) in [(1, 600, 7_200), (12, 59, 7_200), (12, 600, 299)] {
|
||||
assert!(
|
||||
ProviderFreezePolicy::try_new(true, threshold, window, duration, true, None, true)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
ProviderFreezePolicy::try_new(true, 12, 600, 7_200, true, Some(" ".to_owned()), true)
|
||||
.is_err()
|
||||
);
|
||||
assert!(ProviderFreezePolicy::try_new(true, 12, 600, 7_200, true, None, true).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn in_flight_success_does_not_release_a_new_capacity_freeze() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_freeze_late_success").await;
|
||||
let account = store.account("acct_freeze_late_success").expect("account");
|
||||
let (quota, cooldowns) = freeze_service(&store, freeze_policy(2)).await;
|
||||
for _ in 0..2 {
|
||||
quota
|
||||
.apply_capacity_failure(&account, SystemTime::now())
|
||||
.await;
|
||||
}
|
||||
let before = cooldowns.read(account.id()).await.expect("freeze");
|
||||
quota
|
||||
.record_successful_inference(&account, SystemTime::now())
|
||||
.await
|
||||
.expect("late success");
|
||||
assert_eq!(
|
||||
cooldowns.read(account.id()).await.expect("freeze remains"),
|
||||
before
|
||||
);
|
||||
assert_eq!(
|
||||
cooldowns
|
||||
.capacity_evidence(account.id())
|
||||
.map(|(count, _)| count),
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! OpenAI 额度事实边界与展示快照回归。
|
||||
|
||||
mod capacity_freeze;
|
||||
mod recovery;
|
||||
mod refresh_timing;
|
||||
mod scheduling;
|
||||
@@ -68,6 +69,8 @@ fn quota_service_with_base_url(
|
||||
http,
|
||||
base_url,
|
||||
Arc::new(crate::support::MemoryCooldownPort::new()),
|
||||
Arc::new(crate::support::TestLeaseCoordinator::default()),
|
||||
crate::support::runtime_policy(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ async fn old_weekly_quota_should_outweigh_one_in_flight_title_request() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ use gateway_core::operation::{
|
||||
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::ProviderLeasePort;
|
||||
use gateway_core::routing::{
|
||||
ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ModelServiceTier,
|
||||
ProviderKind, ProviderModel, PublicModelId, RoutingContext, RuntimeAccount,
|
||||
@@ -227,6 +228,31 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
stream_max_retries: u32,
|
||||
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
|
||||
let (provider, quota, _) = provider_and_quota_with_runtime_ports(
|
||||
store,
|
||||
session_affinity,
|
||||
base_url,
|
||||
leases,
|
||||
stream_max_retries,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
crate::support::runtime_policy(),
|
||||
);
|
||||
(provider, quota)
|
||||
}
|
||||
|
||||
fn provider_and_quota_with_runtime_ports(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
stream_max_retries: u32,
|
||||
cooldowns: Arc<MemoryCooldownPort>,
|
||||
policy: Arc<dyn gateway_core::provider_ports::ProviderRuntimePolicyPort>,
|
||||
) -> (
|
||||
CodexProvider,
|
||||
Arc<CodexCredentialQuotaService>,
|
||||
Arc<CodexWebSocketPool>,
|
||||
) {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -245,7 +271,9 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
profile.clone(),
|
||||
http.clone(),
|
||||
base_url.clone(),
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
cooldowns,
|
||||
Arc::clone(&leases) as Arc<dyn ProviderLeasePort>,
|
||||
policy,
|
||||
));
|
||||
let account_feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
@@ -268,11 +296,11 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
http,
|
||||
profile,
|
||||
base_url,
|
||||
websocket_pool,
|
||||
Arc::clone(&websocket_pool),
|
||||
stream_max_retries,
|
||||
)
|
||||
.expect("official OpenAI provider");
|
||||
(provider, quota)
|
||||
(provider, quota, websocket_pool)
|
||||
}
|
||||
|
||||
async fn create_account(store: &Arc<MemoryAccountStore>, id: &str) {
|
||||
@@ -5476,7 +5504,11 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
|
||||
create_account(&store, account_id).await;
|
||||
let server = MockServer::start().await;
|
||||
let body = json!({"error": {
|
||||
"code": if status == 503 { Some("server_is_overloaded") } else { None },
|
||||
"code": match status {
|
||||
429 => Some("slow_down"),
|
||||
503 => Some("server_is_overloaded"),
|
||||
_ => None,
|
||||
},
|
||||
"type": "server_error",
|
||||
"message": "Selected model is at capacity. Please try a different model.",
|
||||
"extension": {"preserved": true}
|
||||
@@ -5518,7 +5550,7 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
|
||||
max_retries, initial_delay, max_delay,
|
||||
}) if max_retries.get() == 3 && initial_delay == Duration::from_secs(8) && max_delay == Duration::from_secs(8))
|
||||
);
|
||||
assert!(!provider_openai::openai_failure_affects_account_score(
|
||||
assert!(provider_openai::openai_failure_affects_account_score(
|
||||
&error
|
||||
));
|
||||
assert_eq!(
|
||||
@@ -5540,6 +5572,196 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
|
||||
}
|
||||
}
|
||||
|
||||
fn capacity_websocket_operation() -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
("store".to_owned(), json!(false)),
|
||||
("session_id".to_owned(), json!("capacity-feedback-session")),
|
||||
]),
|
||||
)
|
||||
.expect("WebSocket payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"downstream_websocket_connection_id".to_owned(),
|
||||
json!("ws_capacity_feedback"),
|
||||
)]));
|
||||
Operation::Generate(
|
||||
GenerateRequest::from_protocol_payload(payload).with_provider_session_state(
|
||||
ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("account_id".to_owned(), json!("acct_provider_contract")),
|
||||
(
|
||||
"conversation_id".to_owned(),
|
||||
json!("capacity-feedback-session"),
|
||||
),
|
||||
("continuation_scope".to_owned(), json!("persisted")),
|
||||
]),
|
||||
)
|
||||
.expect("WebSocket session scope"),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn provider_with_capacity_tracking(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
base_url: String,
|
||||
cooldowns: Arc<MemoryCooldownPort>,
|
||||
) -> (CodexProvider, Arc<CodexWebSocketPool>) {
|
||||
let (provider, _, pool) = provider_and_quota_with_runtime_ports(
|
||||
store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
base_url,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
0,
|
||||
cooldowns,
|
||||
crate::support::StaticFreezePolicy::policy_port(
|
||||
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
|
||||
true, 12, 600, 7_200, true, None, true,
|
||||
)
|
||||
.expect("freeze policy"),
|
||||
),
|
||||
);
|
||||
(provider, pool)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_feedback_counts_business_rejections_but_excludes_diagnostic_probes() {
|
||||
use gateway_core::provider_ports::ProviderCooldownPort as _;
|
||||
|
||||
for websocket in [false, true] {
|
||||
for (status, code) in [
|
||||
(429, "slow_down"),
|
||||
(503, "server_is_overloaded"),
|
||||
(500, "server_error"),
|
||||
] {
|
||||
for diagnostic in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_provider_contract";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("account");
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method(if websocket { "GET" } else { "POST" }))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(status).set_body_json(json!({"error": {
|
||||
"code": code,
|
||||
"message": "Upstream temporarily unavailable"
|
||||
}})),
|
||||
)
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (provider, _) =
|
||||
provider_with_capacity_tracking(&store, server.uri(), Arc::clone(&cooldowns));
|
||||
// 同时验证窗口证据已过期与尚未过期:探测不能重建峰值,也不能改写原计数。
|
||||
for existing_evidence in [None, Some((4, 20))] {
|
||||
if let Some((count, peak)) = existing_evidence {
|
||||
for _ in 0..count {
|
||||
cooldowns
|
||||
.record_capacity_failure(
|
||||
account.id(),
|
||||
Duration::from_secs(600),
|
||||
peak,
|
||||
)
|
||||
.await
|
||||
.expect("seed normal-request evidence");
|
||||
}
|
||||
}
|
||||
let before = cooldowns.capacity_evidence(account.id());
|
||||
let attempt = if diagnostic {
|
||||
diagnostic_context("req_capacity_feedback_probe", account_id)
|
||||
} else {
|
||||
context("req_capacity_feedback_business", CancellationToken::new())
|
||||
};
|
||||
let operation = if websocket {
|
||||
capacity_websocket_operation()
|
||||
} else {
|
||||
http_generate_operation()
|
||||
};
|
||||
let mut stream = provider
|
||||
.execute(planned_request("openai", operation), attempt)
|
||||
.await
|
||||
.expect("prepare upstream attempt");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("expected upstream rejection"),
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
error.upstream_status(),
|
||||
Some(status),
|
||||
"unexpected upstream error: {error:?}"
|
||||
);
|
||||
let after = cooldowns.capacity_evidence(account.id());
|
||||
if diagnostic {
|
||||
assert_eq!(after, before, "probe must preserve capacity count and peak");
|
||||
} else {
|
||||
assert_eq!(
|
||||
after.map(|(count, _)| count),
|
||||
Some(before.map_or(1, |(count, _)| count + 1))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_websocket_connection_cancellation_does_not_supply_capacity_evidence() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_provider_contract";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("account");
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener");
|
||||
let base_url = format!("http://{}", listener.local_addr().expect("address"));
|
||||
let (provider, pool) =
|
||||
provider_with_capacity_tracking(&store, base_url, Arc::clone(&cooldowns));
|
||||
let operation = capacity_websocket_operation();
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_capacity_local", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare WebSocket attempt");
|
||||
let attempt = tokio::spawn(async move {
|
||||
loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("cancelled opening must fail"),
|
||||
}
|
||||
}
|
||||
});
|
||||
let (mut opening, _) = timeout(Duration::from_secs(5), listener.accept())
|
||||
.await
|
||||
.expect("opening deadline")
|
||||
.expect("opening");
|
||||
read_http_request(&mut opening).await;
|
||||
// 复现账号更新驱逐正在建连的连接,未收到任何上游容量拒绝。
|
||||
pool.evict_account(account_id).await;
|
||||
let error = timeout(Duration::from_secs(5), attempt)
|
||||
.await
|
||||
.expect("cancelled attempt deadline")
|
||||
.expect("attempt task");
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Unavailable);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
|
||||
assert_eq!(
|
||||
error.diagnostic().and_then(|diagnostic| diagnostic.code()),
|
||||
Some("shared_connect_failed")
|
||||
);
|
||||
assert_eq!(cooldowns.capacity_evidence(account.id()), None);
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_usage_limit_rejection_preserves_quota_state_for_account_rotation() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -5578,7 +5800,12 @@ async fn websocket_usage_limit_rejection_preserves_quota_state_for_account_rotat
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_transports() {
|
||||
for use_websocket in [false, true] {
|
||||
for (use_websocket, code) in [
|
||||
(false, "server_is_overloaded"),
|
||||
(true, "server_is_overloaded"),
|
||||
(false, "slow_down"),
|
||||
(true, "slow_down"),
|
||||
] {
|
||||
for semantic_output in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
@@ -5588,7 +5815,7 @@ async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_t
|
||||
if semantic_output {
|
||||
events.push(json!({"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello"}));
|
||||
}
|
||||
let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": "server_is_overloaded", "message": "Selected model is at capacity. Please try a different model."}}});
|
||||
let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": code, "message": "Selected model is at capacity. Please try a different model."}}});
|
||||
events.push(original.clone());
|
||||
let (base_url, _http_server, websocket_server) = if use_websocket {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener");
|
||||
@@ -5653,7 +5880,7 @@ async fn capacity_stream_rejection_only_retries_before_semantic_output_on_both_t
|
||||
assert_eq!(error.replay_is_safe(), !semantic_output);
|
||||
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
assert_eq!(error.pre_delivery_retry().is_some(), !semantic_output);
|
||||
assert!(!provider_openai::openai_failure_affects_account_score(
|
||||
assert!(provider_openai::openai_failure_affects_account_score(
|
||||
&error
|
||||
));
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::{StreamExt, stream};
|
||||
use gateway_core::account::{AccountFeedbackStats, ProviderAccountId};
|
||||
@@ -45,7 +47,7 @@ fn deliver_error_with_openai_feedback(
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_exclude_request_scoped_capacity_errors() {
|
||||
fn account_score_failure_filter_should_accept_capacity_errors_with_scored_reasons() {
|
||||
for code in ["server_is_overloaded", "slow_down", "server_error"] {
|
||||
let error = sent_error(ProviderErrorKind::Unavailable, Some(code))
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
@@ -53,13 +55,15 @@ fn account_score_failure_filter_should_exclude_request_scoped_capacity_errors()
|
||||
Some("server_error".to_owned()),
|
||||
Some("server_error".to_owned()),
|
||||
));
|
||||
assert!(!openai_failure_affects_account_score(&error));
|
||||
assert!(openai_failure_affects_account_score(&error));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_accept_the_closed_reason_list() {
|
||||
for code in [
|
||||
"server_is_overloaded",
|
||||
"slow_down",
|
||||
"rate_limit_exceeded",
|
||||
"rate_limit_error",
|
||||
"server_error",
|
||||
@@ -82,14 +86,20 @@ fn account_score_failure_filter_should_normalize_case_and_whitespace() {
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_use_a_structured_type_when_code_is_absent() {
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"upstream unavailable",
|
||||
None,
|
||||
Some("SERVICE_UNAVAILABLE_ERROR".to_owned()),
|
||||
));
|
||||
for error_type in [
|
||||
"SERVICE_UNAVAILABLE_ERROR",
|
||||
"SERVER_IS_OVERLOADED",
|
||||
"SLOW_DOWN",
|
||||
] {
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"upstream unavailable",
|
||||
None,
|
||||
Some(error_type.to_owned()),
|
||||
));
|
||||
|
||||
assert!(openai_failure_affects_account_score(&error));
|
||||
assert!(openai_failure_affects_account_score(&error));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -185,17 +195,27 @@ fn openai_feedback_should_not_amplify_repeated_client_configuration_errors() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_feedback_should_not_penalize_an_account_for_server_overload() {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let account = ProviderAccountId::new("acct_overloaded").expect("account");
|
||||
fn openai_feedback_should_penalize_capacity_rejections_even_with_transient_retry() {
|
||||
for code in ["server_is_overloaded", "slow_down"] {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let account = ProviderAccountId::new("acct_overloaded").expect("account");
|
||||
let error = sent_error(ProviderErrorKind::UpstreamCapacityUnavailable, Some(code))
|
||||
.with_replay_safe()
|
||||
.with_transient_retry(
|
||||
NonZeroU32::new(3).expect("retry count"),
|
||||
Duration::from_millis(500),
|
||||
Duration::from_secs(8),
|
||||
);
|
||||
|
||||
deliver_error_with_openai_feedback(
|
||||
&feedback,
|
||||
&provider,
|
||||
&account,
|
||||
sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded")),
|
||||
);
|
||||
deliver_error_with_openai_feedback(&feedback, &provider, &account, error);
|
||||
|
||||
assert_eq!(feedback.scheduling_signals(&provider, &account).0, None);
|
||||
assert!(
|
||||
feedback
|
||||
.scheduling_signals(&provider, &account)
|
||||
.0
|
||||
.is_some_and(|failure_rate| failure_rate > 0),
|
||||
"capacity rejection did not affect the account score: {code}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,11 +20,11 @@ use gateway_core::error::{StoreError, StoreErrorKind};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort,
|
||||
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest,
|
||||
ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey,
|
||||
ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
ProviderStoreError,
|
||||
ProviderCooldownScope, ProviderFreezePolicy, ProviderLeaseAcquisition, ProviderLeasePort,
|
||||
ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
|
||||
ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderScopedCooldown,
|
||||
ProviderSessionAffinityKey, ProviderSessionAffinityPort, ProviderSessionExclusionPort,
|
||||
ProviderSessionExclusions, ProviderStoreError,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::credential::{
|
||||
@@ -615,7 +615,7 @@ impl ProviderLeasePort for TestLeaseCoordinator {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -956,6 +956,36 @@ pub(crate) fn runtime_policy() -> Arc<dyn ProviderRuntimePolicyPort> {
|
||||
Arc::new(StaticRuntimePolicy)
|
||||
}
|
||||
|
||||
/// 冻结策略可配置的运行时策略 fake;容量熔断触发测试用。
|
||||
pub(crate) struct StaticFreezePolicy(ProviderFreezePolicy);
|
||||
|
||||
impl StaticFreezePolicy {
|
||||
/// 构造带固定冻结策略的端口句柄;命名沿用测试构造器语义。
|
||||
#[must_use]
|
||||
pub(crate) fn policy_port(policy: ProviderFreezePolicy) -> Arc<dyn ProviderRuntimePolicyPort> {
|
||||
Arc::new(Self(policy))
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderRuntimePolicyPort for StaticFreezePolicy {
|
||||
fn load_refresh_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderRefreshPolicy, ProviderStoreError>> {
|
||||
Box::pin(async {
|
||||
ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(60 * 60),
|
||||
NonZeroU32::new(2).expect("positive concurrency"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn load_freeze_policy(
|
||||
&self,
|
||||
) -> BoxFuture<'_, Result<ProviderFreezePolicy, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(self.0.clone()) })
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn secret(access_token: &str) -> CodexOAuthSecret {
|
||||
CodexOAuthSecret {
|
||||
access_token: SecretString::from(access_token.to_owned()),
|
||||
@@ -972,11 +1002,13 @@ pub(crate) fn account_policy() -> gateway_core::account::AccountSelectionPolicy
|
||||
)
|
||||
}
|
||||
|
||||
/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector
|
||||
/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。
|
||||
/// 内存 `ProviderCooldownPort`:实现 `read`/`put_if_later` 与容量失败计数
|
||||
/// 证据(openai selector 429 冷却与容量熔断路径用),scope 变体测试不涉及,
|
||||
/// 返回占位。
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct MemoryCooldownPort {
|
||||
pub(crate) cooldowns: Arc<Mutex<BTreeMap<ProviderAccountId, ProviderCooldown>>>,
|
||||
capacity_failures: Arc<Mutex<BTreeMap<ProviderAccountId, (u32, u32)>>>,
|
||||
}
|
||||
|
||||
impl MemoryCooldownPort {
|
||||
@@ -984,6 +1016,16 @@ impl MemoryCooldownPort {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// 测试断言用:窗口内累计失败次数与在途峰值。
|
||||
#[must_use]
|
||||
pub(crate) fn capacity_evidence(&self, account_id: &ProviderAccountId) -> Option<(u32, u32)> {
|
||||
self.capacity_failures
|
||||
.lock()
|
||||
.expect("capacity failures lock")
|
||||
.get(account_id)
|
||||
.copied()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
@@ -1057,4 +1099,57 @@ impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
_window: Duration,
|
||||
in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut failures = self
|
||||
.capacity_failures
|
||||
.lock()
|
||||
.expect("capacity failures lock");
|
||||
let entry = failures.entry(account_id.clone()).or_insert((0, 0));
|
||||
entry.0 = entry.0.saturating_add(1);
|
||||
entry.1 = entry.1.max(in_flight);
|
||||
Ok(entry.0)
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
_through_revision: gateway_core::account::CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut cooldowns = self.cooldowns.lock().expect("cooldown lock");
|
||||
if cooldowns.get(account_id).is_some_and(|value| {
|
||||
value.kind().is_capacity_freeze() || value.credential_revision() > _through_revision
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
cooldowns.remove(account_id);
|
||||
self.capacity_failures
|
||||
.lock()
|
||||
.expect("capacity failures lock")
|
||||
.remove(account_id);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
Ok(self
|
||||
.capacity_failures
|
||||
.lock()
|
||||
.expect("capacity failures lock")
|
||||
.get(account_id)
|
||||
.map(|(_, peak)| *peak))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -666,7 +666,7 @@ impl ProviderLeasePort for TestLeases {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -862,6 +862,30 @@ impl ProviderCooldownPort for TestCooldown {
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_window: Duration,
|
||||
_in_flight: u32,
|
||||
) -> BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(0) })
|
||||
}
|
||||
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: gateway_core::account::CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
}
|
||||
|
||||
struct TestRuntimePolicy;
|
||||
|
||||
@@ -138,7 +138,7 @@ impl SelectorFixture {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -1065,7 +1065,7 @@ impl ProviderLeasePort for DiagnosticLeasePort {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
rate_limited_until: None,
|
||||
cooldown: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
|
||||
@@ -972,6 +972,30 @@ impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
Ok(account_removed || scoped_removed)
|
||||
})
|
||||
}
|
||||
|
||||
fn record_capacity_failure<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_window: Duration,
|
||||
_in_flight: u32,
|
||||
) -> futures::future::BoxFuture<'a, Result<u32, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(0) })
|
||||
}
|
||||
|
||||
fn clear_after_success<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: gateway_core::account::CredentialRevision,
|
||||
) -> futures::future::BoxFuture<'a, Result<(), ProviderStoreError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn capacity_peak_in_flight<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> futures::future::BoxFuture<'a, Result<Option<u32>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderRuntimePolicyPort for StaticRuntimePolicy {
|
||||
|
||||
@@ -7,3 +7,5 @@ b7d3df7b48d835a00a3b74cbefc958a039d38fea5529cdad04bfbc703dca72aa 0001_initial.s
|
||||
18e8c1b71c896a9d576d9e4ef9701506f654f55383af908898195b9eb2980fde 0007_account_notes.sql
|
||||
36b720c9d3e6bf0bdb799b9b6a89b81638c642ac866992a880bd82357012a208 0008_concurrency_queues.sql
|
||||
161df9ae43da34d51b340bbf95d71c61a8c1a2aa142382f693a9d127c834630d 0009_account_model_access.sql
|
||||
dd4aa34e9bcf1698bb9688421964bbf410314d8e905448b8229460b972cff2f6 0010_account_auto_freeze.sql
|
||||
9bd261b7e94872126754eb7f749c56187264a47c567b0bef5c394d6776b4f463 0011_account_auto_freeze_opt_in.sql
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
-- 账号容量熔断:统计窗口内高频出现容量类上游错误时自动冻结账号一段时间;
|
||||
-- 恢复前可选执行一次探测调用验证上游已恢复;冻结期间按观测到的在途并发峰值
|
||||
-- 下调账号并发上限,让账号带着更安全的并发配置自愈。默认启用,已有实例可关闭。
|
||||
alter table runtime_settings
|
||||
add column account_auto_freeze_enabled boolean not null default true,
|
||||
add column account_auto_freeze_threshold bigint not null default 12
|
||||
check (account_auto_freeze_threshold between 2 and 1000),
|
||||
add column account_auto_freeze_window_seconds bigint not null default 600
|
||||
check (account_auto_freeze_window_seconds between 60 and 3600),
|
||||
add column account_auto_freeze_duration_seconds bigint not null default 7200
|
||||
check (account_auto_freeze_duration_seconds between 300 and 604800),
|
||||
add column account_auto_freeze_probe_enabled boolean not null default true,
|
||||
add column account_auto_freeze_probe_model text
|
||||
check (
|
||||
account_auto_freeze_probe_model is null
|
||||
or (
|
||||
octet_length(account_auto_freeze_probe_model) between 1 and 128
|
||||
and account_auto_freeze_probe_model = btrim(account_auto_freeze_probe_model)
|
||||
and account_auto_freeze_probe_model !~ '[[:cntrl:]]'
|
||||
)
|
||||
),
|
||||
add column account_auto_freeze_adaptive_concurrency boolean not null default true;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- 自动冻结默认关闭;保留已经通过设置页显式保存的冻结配置。
|
||||
alter table runtime_settings
|
||||
alter column account_auto_freeze_enabled set default false;
|
||||
|
||||
update runtime_settings
|
||||
set account_auto_freeze_enabled = false
|
||||
where account_auto_freeze_enabled = true
|
||||
and not exists (
|
||||
select 1 from admin_audit_events
|
||||
where action = 'settings.replace'
|
||||
and changed_fields @> array['account_auto_freeze']::text[]
|
||||
);
|
||||
+27
-1
@@ -253,7 +253,10 @@ Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
|
||||
OpenAI 明确返回 `server_is_overloaded`、`slow_down` 或模型容量不足错误时,代理在允许安全重放且
|
||||
尚未交付输出的前提下,先做最多 3 次同账号指数退避,再通过现有调度换号。默认间隔从 500ms 开始,
|
||||
上游 `Retry-After` 参与退避计算,单次等待不超过 8 秒;重试同时受请求总尝试次数和截止时间约束。
|
||||
容量不足不扣 Smart 账号健康分,不触发 Provider 全局熔断,也不作为账号额度耗尽写入冷却状态。
|
||||
`server_is_overloaded`、`slow_down` 与既有可计分的结构化错误按已发送的失败尝试计入 Smart 账号
|
||||
健康分;容量分类不再豁免计分。失败率沿用账号级平滑与时间衰减,影响后续普通选路,已有可用账号的
|
||||
会话亲和仍优先。容量不足不触发 Provider 全局熔断,也不作为账号额度耗尽;启用账号自动冻结时,
|
||||
达到容量失败阈值会另外写入临时冷却。
|
||||
最终交付的上游错误仍按上述透明边界保留原始状态码、错误码和正文。
|
||||
明确额度耗尽继续走现有账号隔离与安全换号流程,
|
||||
包括 WebSocket 握手返回的 429;不会因其长 `Retry-After` 而转入同账号传输恢复等待。
|
||||
@@ -367,6 +370,11 @@ Token 明细、费用明细、用时/首字与状态。Token 和费用复用现
|
||||
- `sortBy`: `email`、`status`、`planType`、`usage`、`lastUsedAt`、`expiresAt`;
|
||||
- `sortDirection`: `asc`、`desc`。
|
||||
|
||||
账号限流详情在 `quota` 中返回:`rateLimitReason` 为 `upstream_rate_limit`(上游临时限流)、
|
||||
`capacity_freeze`(容量错误触发自动冻结)或 `null`。`recoveryProbeRequired` 表示解除冻结是否需要成功探测;
|
||||
此时 `rateLimitedUntil` 是最早探测时间,到期后仍保持 `rate_limited`,直到探测成功或手动恢复。
|
||||
未要求探测时,该字段表示冷却结束时间。所有此类情况统一显示“限流中”,仅详情原因和恢复条件不同。
|
||||
|
||||
账号列表和详情返回 `notes`(无备注时为 `null`)。编辑时省略或 `null` 保留原备注;字符串最多 500 个 Unicode
|
||||
字符,允许换行和制表符,保存时去除首尾空白,空字符串清空备注。备注独立于上游身份,导入时未显式提供备注、
|
||||
重新授权、凭据刷新及批量调度更新均保留已有备注。
|
||||
@@ -910,6 +918,13 @@ minCodexCliVersion
|
||||
usageRetentionDays
|
||||
opsEventRetentionDays
|
||||
auditRetentionDays
|
||||
accountAutoFreezeEnabled
|
||||
accountAutoFreezeThreshold
|
||||
accountAutoFreezeWindowSeconds
|
||||
accountAutoFreezeDurationSeconds
|
||||
accountAutoFreezeProbeEnabled
|
||||
accountAutoFreezeProbeModel
|
||||
accountAutoFreezeAdaptiveConcurrency
|
||||
```
|
||||
|
||||
`maxWaitingPerKey` 与 `maxWaitingPerAccount` 是全局统一的排队容量,取值 0~1,000,默认 0(关闭);
|
||||
@@ -922,6 +937,17 @@ auditRetentionDays
|
||||
`rotationStrategy` 可取 `smart`、`quota_reset_priority`、`round_robin`、`sticky`。
|
||||
两个 `minCodex*Version` 字段为 `string | null`,只设置最低版本,不存在最大版本字段。
|
||||
|
||||
账号自动冻结(`accountAutoFreezeEnabled`)默认关闭。启用后,在统计窗口内按尝试累计容量类上游错误(`server_is_overloaded`
|
||||
等与 5xx 不可用),达到阈值后把该账号冻结为带恢复倒计时的 `rate_limited` 状态。`accountAutoFreezeThreshold`
|
||||
取值 2~1,000(默认 12,按普通请求的 attempt 计数,含请求内同账号重试,不含诊断探测与本地连接保护错误);`accountAutoFreezeWindowSeconds`
|
||||
取值 60~3,600(默认 600,随每次失败滑动顺延);`accountAutoFreezeDurationSeconds` 取值 300~604,800
|
||||
(默认 7,200,即 2 小时,探测失败后按该时长顺延)。`accountAutoFreezeProbeEnabled` 开启时恢复 worker
|
||||
在到期后执行真实探测调用,成功才解冻;探测过程中和服务重启后继续阻止普通请求。关闭自动冻结或探测后,
|
||||
已有冻结等待当前冷却结束再恢复调度。`accountAutoFreezeProbeModel` 为 `string | null`,留空时自动
|
||||
选择账号可用的第一个模型。`accountAutoFreezeAdaptiveConcurrency` 开启时冻结期间把账号并发上限下调到
|
||||
观测在途峰值的 80%(下限 2,只降不升)。这会持久修改账号并发设置;跟随全局默认的账号也会设为独立上限,
|
||||
解冻后不自动恢复,管理员可手动改回。
|
||||
|
||||
Windows 离线包接口固定解析 Microsoft Store Product ID `9PLM9XGG6VKS` 的 Retail 包,不接受调用方提供
|
||||
产品 ID、上游地址、ring 或文件名。后端只返回通过包名、架构、Microsoft CDN host/path、scheme 和失效
|
||||
时间校验的 `x64` / `arm64` MSIX 直链,不代理安装包字节。Store 内容通道返回 HTTP/80 临时地址时保留
|
||||
|
||||
@@ -422,6 +422,10 @@ PostgreSQL 周期对账才是正确性基础。
|
||||
`normal`、`quota_exhausted`、`rate_limited`、`disabled`、`error`。只有明确上游证据才能恢复或终态化账号,
|
||||
本地时钟和不确定响应不能伪造事实。
|
||||
|
||||
容量冻结与普通上游限流共享账号冷却投影,但恢复条件不同:需要探测的冻结在截止时间后仍阻止调度,
|
||||
普通推理成功不能解除。Admin 恢复任务按冻结代次原子提交探测结果,避免旧结果覆盖手动恢复或新冻结;
|
||||
自动降低并发通过现有管理事务与快照发布链路,仅按数据库最新设置更新并发上限。
|
||||
|
||||
PostgreSQL schema 由迁移目录按编号管理。已应用迁移按字节冻结,后续 schema 变化只能新增编号迁移,
|
||||
详见 [迁移规则](../backend/migrations/README.md)。
|
||||
|
||||
@@ -497,8 +501,19 @@ Worker 由各 Bundle 贡献、由 Host 统一监督:
|
||||
- Store:过期请求恢复、历史保留和 PostgreSQL/Redis 观测队列;
|
||||
- Core:`runtime` owner 的 RuntimeSnapshot 周期对账和 Redis change 订阅;
|
||||
- Admin:S3/R2 备份 daemon,负责调度、执行、删除收敛与保留清理;
|
||||
以及账号冻结恢复 worker(容量熔断的自适应并发下调与到期探测解冻);
|
||||
- Provider:credential refresh、quota/catalog 健康和官方版本/etag 检查。
|
||||
|
||||
账号容量熔断默认关闭。启用后,仅普通请求收到的容量类上游错误(`server_is_overloaded` 等与
|
||||
5xx 不可用)按滑动窗口计数,并把当时观测到的在途并发并入峰值证据;本地连接保护与诊断探测
|
||||
不参与容量计数或峰值采样。这些错误不证明凭据或配额失效,不进入账号失败状态。达到阈值后写入
|
||||
账号级 Redis 容量冷却,调度立即跳过该账号;管理端沿用限流状态,通过原因区分容量冻结与上游限流。
|
||||
冻结与计数由 Redis 保存,不改变 PostgreSQL 账号状态。需要探测的冻结在到期后仍阻止调度,恢复
|
||||
worker 复用连接测试探针执行真实上游调用,并按冻结代次处理结果:成功清除,失败按配置时长顺延,
|
||||
过期探测结果不得覆盖更新的冻结或管理员操作。关闭自动冻结或探测后,已有冻结在冷却到期后恢复。
|
||||
自适应并发下调按观测峰值的 80%(下限 2)原子更新最新有效账号上限,只降不升,不覆盖其他账号
|
||||
设置,审计标注为系统变更;该上限持久保存,恢复调度后不自动调高。
|
||||
|
||||
周期任务的 Redis lease 只保证单周期互斥,不构成多副本 leader 选举。备份 daemon 依赖单副本部署边界,
|
||||
自更新也只替换处理请求的当前进程,因此整个应用必须保持单副本。
|
||||
|
||||
|
||||
@@ -33,6 +33,8 @@ export interface AccountQuota {
|
||||
limitReached: boolean
|
||||
// 429 临时限流(Redis 冷却)到期时间;非限流中为 null。
|
||||
rateLimitedUntil: string | null
|
||||
rateLimitReason: 'upstream_rate_limit' | 'capacity_freeze' | null
|
||||
recoveryProbeRequired: boolean
|
||||
windows: AccountQuotaWindow[]
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,13 @@ export interface RuntimeSettings {
|
||||
usageRetentionDays: number
|
||||
opsEventRetentionDays: number
|
||||
auditRetentionDays: number
|
||||
accountAutoFreezeEnabled: boolean
|
||||
accountAutoFreezeThreshold: number
|
||||
accountAutoFreezeWindowSeconds: number
|
||||
accountAutoFreezeDurationSeconds: number
|
||||
accountAutoFreezeProbeEnabled: boolean
|
||||
accountAutoFreezeProbeModel: string | null
|
||||
accountAutoFreezeAdaptiveConcurrency: boolean
|
||||
updatedAt: string
|
||||
}
|
||||
|
||||
|
||||
@@ -62,6 +62,9 @@ function connectionLogClass(tone: string) {
|
||||
:status="account.status"
|
||||
:error-reason="account.errorReason"
|
||||
:error-message="account.errorMessage"
|
||||
:rate-limited-until="account.quota.rateLimitedUntil"
|
||||
:rate-limit-reason="account.quota.rateLimitReason"
|
||||
:recovery-probe-required="account.quota.recoveryProbeRequired"
|
||||
:next-refresh-at="account.nextRefreshAt"
|
||||
variant="pill"
|
||||
/>
|
||||
|
||||
@@ -13,6 +13,8 @@ const props = withDefaults(
|
||||
errorReason?: AccountErrorReason | null
|
||||
errorMessage?: string | null
|
||||
rateLimitedUntil?: string | null
|
||||
rateLimitReason?: 'upstream_rate_limit' | 'capacity_freeze' | null
|
||||
recoveryProbeRequired?: boolean
|
||||
nextRefreshAt?: string | null
|
||||
variant?: 'inline' | 'pill'
|
||||
}>(),
|
||||
@@ -20,6 +22,8 @@ const props = withDefaults(
|
||||
errorReason: null,
|
||||
errorMessage: null,
|
||||
rateLimitedUntil: null,
|
||||
rateLimitReason: null,
|
||||
recoveryProbeRequired: false,
|
||||
nextRefreshAt: null,
|
||||
variant: 'inline',
|
||||
},
|
||||
@@ -32,6 +36,8 @@ const presentation = computed(() =>
|
||||
errorReason: props.errorReason,
|
||||
errorMessage: props.errorMessage,
|
||||
rateLimitedUntil: props.rateLimitedUntil,
|
||||
rateLimitReason: props.rateLimitReason,
|
||||
recoveryProbeRequired: props.recoveryProbeRequired,
|
||||
nextRefreshAt: props.nextRefreshAt,
|
||||
now: now.value.getTime(),
|
||||
}),
|
||||
@@ -102,7 +108,7 @@ const presentation = computed(() =>
|
||||
v-if="presentation.rateLimitRecovery"
|
||||
class="flex items-center justify-between gap-3 rounded-cp bg-cp-fill-quaternary px-3 py-2"
|
||||
>
|
||||
<span class="font-heavy text-cp-text-tertiary">预计恢复</span>
|
||||
<span class="font-heavy text-cp-text-tertiary">{{ presentation.recoveryTimeLabel }}</span>
|
||||
<span class="font-mono font-emphasis tabular-nums text-cp-text">
|
||||
{{ presentation.rateLimitRecovery }}
|
||||
</span>
|
||||
|
||||
@@ -37,6 +37,7 @@ export interface AccountStatusPresentation {
|
||||
errorText: string | null
|
||||
nextRefreshDisplay: string | null
|
||||
rateLimitRecovery: string | null
|
||||
recoveryTimeLabel: string
|
||||
triggerLabel: string
|
||||
}
|
||||
|
||||
@@ -45,6 +46,8 @@ export interface AccountStatusPresentationInput {
|
||||
errorReason: AccountErrorReason | null
|
||||
errorMessage: string | null
|
||||
rateLimitedUntil: string | null
|
||||
rateLimitReason: 'upstream_rate_limit' | 'capacity_freeze' | null
|
||||
recoveryProbeRequired: boolean
|
||||
nextRefreshAt: string | null
|
||||
now: number
|
||||
}
|
||||
@@ -146,19 +149,32 @@ export function resolveAccountStatusPresentation(
|
||||
&& nextRefreshTimestamp !== null
|
||||
&& nextRefreshTimestamp > input.now
|
||||
const mode: AccountStatusDisplayMode = isBackoff ? 'refresh_backoff' : input.status
|
||||
const definition = displayDefinitions[mode]
|
||||
const isFreeze = mode === 'rate_limited' && input.rateLimitReason === 'capacity_freeze'
|
||||
const waitsForProbe = isFreeze && input.recoveryProbeRequired
|
||||
const definition = isFreeze
|
||||
? {
|
||||
...displayDefinitions.rate_limited,
|
||||
description: '容量类请求失败累计达到阈值,系统已暂停该账号的调度。',
|
||||
recoveryHint: waitsForProbe
|
||||
? '冷却结束后进行恢复探测,成功后恢复调度;也可手动恢复账号。'
|
||||
: '冷却结束后自动恢复调度,也可手动恢复账号。',
|
||||
}
|
||||
: displayDefinitions[mode]
|
||||
const nextRefreshDisplay = isBackoff ? formatDateTime(nextRefreshTimestamp) : null
|
||||
const reasonLabel = input.errorReason ? errorReasonLabels[input.errorReason] : null
|
||||
const title = definition.title ?? reasonLabel ?? definition.label
|
||||
const rateLimitRecovery = input.status === 'rate_limited'
|
||||
? remainingTime(input.rateLimitedUntil, input.now)
|
||||
const until = input.rateLimitedUntil ? parseTimestamp(input.rateLimitedUntil) : null
|
||||
const rateLimitRecovery = mode === 'rate_limited'
|
||||
? waitsForProbe && until !== null && until <= input.now
|
||||
? '等待探测成功'
|
||||
: remainingTime(input.rateLimitedUntil, input.now)
|
||||
: null
|
||||
const recoveryHint = mode !== 'refresh_backoff'
|
||||
&& mode !== 'rate_limited'
|
||||
&& input.errorReason
|
||||
? errorRecoveryHints[input.errorReason]
|
||||
: definition.recoveryHint
|
||||
const hasDetail = input.status === 'error' || rateLimitRecovery !== null || isBackoff
|
||||
const hasDetail = input.status === 'error' || mode === 'rate_limited' || isBackoff
|
||||
const retry = nextRefreshDisplay ? ` 下次尝试:${nextRefreshDisplay}。` : ''
|
||||
const triggerLabel = `${title}。${definition.description}${retry} 点击或聚焦查看详情。`
|
||||
|
||||
@@ -174,6 +190,7 @@ export function resolveAccountStatusPresentation(
|
||||
errorText: input.errorMessage || null,
|
||||
nextRefreshDisplay,
|
||||
rateLimitRecovery,
|
||||
recoveryTimeLabel: waitsForProbe ? '恢复探测' : '预计恢复',
|
||||
triggerLabel,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -291,6 +291,8 @@ const {
|
||||
:error-reason="row.errorReason"
|
||||
:error-message="row.errorMessage"
|
||||
:rate-limited-until="row.quota.rateLimitedUntil"
|
||||
:rate-limit-reason="row.quota.rateLimitReason"
|
||||
:recovery-probe-required="row.quota.recoveryProbeRequired"
|
||||
:next-refresh-at="row.nextRefreshAt"
|
||||
/>
|
||||
</template>
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
<script setup lang="ts">
|
||||
import { Activity, Gauge, Snowflake, Timer } from '@lucide/vue'
|
||||
|
||||
import BaseCard from '@/components/base/BaseCard.vue'
|
||||
import BaseCheckbox from '@/components/base/BaseCheckbox.vue'
|
||||
import BaseFormItem from '@/components/base/BaseForm/FormItem.vue'
|
||||
import BaseForm from '@/components/base/BaseForm/index.vue'
|
||||
import BaseInput from '@/components/base/BaseInput.vue'
|
||||
import BaseSwitch from '@/components/base/BaseSwitch.vue'
|
||||
|
||||
const enabled = defineModel<boolean>('enabled', { required: true })
|
||||
const threshold = defineModel<string>('threshold', { required: true })
|
||||
const windowSeconds = defineModel<string>('windowSeconds', { required: true })
|
||||
const durationSeconds = defineModel<string>('durationSeconds', { required: true })
|
||||
const probeEnabled = defineModel<boolean>('probeEnabled', { required: true })
|
||||
const probeModel = defineModel<string>('probeModel', { required: true })
|
||||
const adaptiveConcurrency = defineModel<boolean>('adaptiveConcurrency', { required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<BaseCard
|
||||
title="账号自动冻结"
|
||||
description="容量类错误高频出现时暂停账号调度,在账号管理中显示为限流中"
|
||||
>
|
||||
<BaseForm class="max-w-6xl sm:grid-cols-2">
|
||||
<BaseSwitch
|
||||
v-model="enabled"
|
||||
class="col-span-full justify-self-start"
|
||||
label="启用自动冻结"
|
||||
show-label
|
||||
/>
|
||||
|
||||
<BaseFormItem
|
||||
label="触发阈值"
|
||||
description="窗口内累计失败尝试达到此次数后触发"
|
||||
>
|
||||
<BaseInput
|
||||
v-model="threshold"
|
||||
:disabled="!enabled"
|
||||
aria-label="触发阈值"
|
||||
type="number"
|
||||
min="2"
|
||||
max="1000"
|
||||
step="1"
|
||||
>
|
||||
<template #prefix>
|
||||
<Activity class="size-4" />
|
||||
</template>
|
||||
<template #suffix>
|
||||
<span class="text-cp-sm">次</span>
|
||||
</template>
|
||||
</BaseInput>
|
||||
</BaseFormItem>
|
||||
|
||||
<BaseFormItem
|
||||
label="统计窗口"
|
||||
description="每次失败后,计数有效期向后顺延"
|
||||
>
|
||||
<BaseInput
|
||||
v-model="windowSeconds"
|
||||
:disabled="!enabled"
|
||||
aria-label="统计窗口秒数"
|
||||
type="number"
|
||||
min="60"
|
||||
max="3600"
|
||||
step="1"
|
||||
>
|
||||
<template #prefix>
|
||||
<Timer class="size-4" />
|
||||
</template>
|
||||
<template #suffix>
|
||||
<span class="text-cp-sm">秒</span>
|
||||
</template>
|
||||
</BaseInput>
|
||||
</BaseFormItem>
|
||||
|
||||
<BaseFormItem
|
||||
label="冷却时长"
|
||||
description="探测失败时,按此时长延后恢复"
|
||||
>
|
||||
<BaseInput
|
||||
v-model="durationSeconds"
|
||||
:disabled="!enabled"
|
||||
aria-label="冷却时长秒数"
|
||||
type="number"
|
||||
min="300"
|
||||
max="604800"
|
||||
step="1"
|
||||
>
|
||||
<template #prefix>
|
||||
<Snowflake class="size-4" />
|
||||
</template>
|
||||
<template #suffix>
|
||||
<span class="text-cp-sm">秒</span>
|
||||
</template>
|
||||
</BaseInput>
|
||||
</BaseFormItem>
|
||||
<BaseFormItem
|
||||
label="恢复探测模型"
|
||||
description="启用后需探测成功才恢复,模型留空时选择首个可用模型"
|
||||
>
|
||||
<template #extra>
|
||||
<BaseCheckbox
|
||||
v-model="probeEnabled"
|
||||
:disabled="!enabled"
|
||||
label="恢复前探测"
|
||||
show-label
|
||||
/>
|
||||
</template>
|
||||
<BaseInput
|
||||
v-model="probeModel"
|
||||
:disabled="!enabled || !probeEnabled"
|
||||
aria-label="恢复探测模型"
|
||||
placeholder="留空自动选择"
|
||||
>
|
||||
<template #prefix>
|
||||
<Gauge class="size-4" />
|
||||
</template>
|
||||
</BaseInput>
|
||||
</BaseFormItem>
|
||||
|
||||
<div class="col-span-full flex flex-wrap items-center gap-x-3 gap-y-2">
|
||||
<BaseCheckbox
|
||||
v-model="adaptiveConcurrency"
|
||||
:disabled="!enabled"
|
||||
label="自适应并发下调"
|
||||
show-label
|
||||
/>
|
||||
<p class="m-0 text-cp-sm leading-5 text-cp-text-secondary">
|
||||
修改账号并发上限,恢复后不自动调高
|
||||
</p>
|
||||
</div>
|
||||
</BaseForm>
|
||||
</BaseCard>
|
||||
</template>
|
||||
@@ -30,9 +30,17 @@ export function useSettingsForm() {
|
||||
usageRetentionDays: 31,
|
||||
opsEventRetentionDays: 30,
|
||||
auditRetentionDays: 90,
|
||||
|
||||
accountAutoFreezeEnabled: false,
|
||||
accountAutoFreezeThreshold: null as number | null,
|
||||
accountAutoFreezeWindowSeconds: null as number | null,
|
||||
accountAutoFreezeDurationSeconds: null as number | null,
|
||||
accountAutoFreezeProbeEnabled: true,
|
||||
accountAutoFreezeProbeModel: '',
|
||||
accountAutoFreezeAdaptiveConcurrency: true,
|
||||
})
|
||||
|
||||
function numericModel(key: 'refreshMarginSeconds' | 'refreshConcurrency' | 'maxConcurrentPerAccount' | 'requestIntervalMs' | 'maxWaitingPerKey' | 'maxWaitingPerAccount' | 'concurrencyWaitTimeoutSeconds') {
|
||||
function numericModel(key: 'refreshMarginSeconds' | 'refreshConcurrency' | 'maxConcurrentPerAccount' | 'requestIntervalMs' | 'maxWaitingPerKey' | 'maxWaitingPerAccount' | 'concurrencyWaitTimeoutSeconds' | 'accountAutoFreezeThreshold' | 'accountAutoFreezeWindowSeconds' | 'accountAutoFreezeDurationSeconds') {
|
||||
return computed({
|
||||
get: () => (form[key] === null ? '' : String(form[key])),
|
||||
set: (value: string) => {
|
||||
@@ -53,6 +61,9 @@ export function useSettingsForm() {
|
||||
const maxWaitingPerKeyValue = numericModel('maxWaitingPerKey')
|
||||
const maxWaitingPerAccountValue = numericModel('maxWaitingPerAccount')
|
||||
const concurrencyWaitTimeoutSecondsValue = numericModel('concurrencyWaitTimeoutSeconds')
|
||||
const accountAutoFreezeThresholdValue = numericModel('accountAutoFreezeThreshold')
|
||||
const accountAutoFreezeWindowSecondsValue = numericModel('accountAutoFreezeWindowSeconds')
|
||||
const accountAutoFreezeDurationSecondsValue = numericModel('accountAutoFreezeDurationSeconds')
|
||||
|
||||
const minCodexDesktopVersionError = computed(() => versionError(form.minCodexDesktopVersion))
|
||||
const minCodexCliVersionError = computed(() => versionError(form.minCodexCliVersion))
|
||||
@@ -77,6 +88,13 @@ export function useSettingsForm() {
|
||||
form.usageRetentionDays = data.usageRetentionDays
|
||||
form.opsEventRetentionDays = data.opsEventRetentionDays
|
||||
form.auditRetentionDays = data.auditRetentionDays
|
||||
form.accountAutoFreezeEnabled = data.accountAutoFreezeEnabled
|
||||
form.accountAutoFreezeThreshold = data.accountAutoFreezeThreshold
|
||||
form.accountAutoFreezeWindowSeconds = data.accountAutoFreezeWindowSeconds
|
||||
form.accountAutoFreezeDurationSeconds = data.accountAutoFreezeDurationSeconds
|
||||
form.accountAutoFreezeProbeEnabled = data.accountAutoFreezeProbeEnabled
|
||||
form.accountAutoFreezeProbeModel = data.accountAutoFreezeProbeModel ?? ''
|
||||
form.accountAutoFreezeAdaptiveConcurrency = data.accountAutoFreezeAdaptiveConcurrency
|
||||
mappings.value = Object.entries(data.modelMappings || {}).map(([requestedModel, upstreamModel]) => ({
|
||||
requestedModel,
|
||||
upstreamModel: String(upstreamModel),
|
||||
@@ -132,7 +150,7 @@ export function useSettingsForm() {
|
||||
async function saveSettings() {
|
||||
if (saving.value || loading.value)
|
||||
return
|
||||
const { refreshMarginSeconds, refreshConcurrency, maxConcurrentPerAccount, requestIntervalMs, rotationStrategy, maxWaitingPerKey, maxWaitingPerAccount, concurrencyWaitTimeoutSeconds } = form
|
||||
const { refreshMarginSeconds, refreshConcurrency, maxConcurrentPerAccount, requestIntervalMs, rotationStrategy, maxWaitingPerKey, maxWaitingPerAccount, concurrencyWaitTimeoutSeconds, accountAutoFreezeThreshold, accountAutoFreezeWindowSeconds, accountAutoFreezeDurationSeconds } = form
|
||||
if (refreshMarginSeconds === null || refreshConcurrency === null || maxConcurrentPerAccount === null || requestIntervalMs === null || !rotationStrategy || maxWaitingPerKey === null || maxWaitingPerAccount === null || concurrencyWaitTimeoutSeconds === null) {
|
||||
toast.warning('请完整填写运行参数和调度策略')
|
||||
return
|
||||
@@ -146,6 +164,21 @@ export function useSettingsForm() {
|
||||
toast.warning('请修正客户端最低版本格式')
|
||||
return
|
||||
}
|
||||
if (accountAutoFreezeThreshold === null || accountAutoFreezeWindowSeconds === null || accountAutoFreezeDurationSeconds === null) {
|
||||
toast.warning('请完整填写账号自动冻结参数')
|
||||
return
|
||||
}
|
||||
if (!Number.isInteger(accountAutoFreezeThreshold) || accountAutoFreezeThreshold < 2 || accountAutoFreezeThreshold > 1000
|
||||
|| !Number.isInteger(accountAutoFreezeWindowSeconds) || accountAutoFreezeWindowSeconds < 60 || accountAutoFreezeWindowSeconds > 3600
|
||||
|| !Number.isInteger(accountAutoFreezeDurationSeconds) || accountAutoFreezeDurationSeconds < 300 || accountAutoFreezeDurationSeconds > 604800) {
|
||||
toast.warning('自动冻结阈值应为 2~1000,统计窗口为 60~3600 秒,冻结时长为 300~604800 秒')
|
||||
return
|
||||
}
|
||||
const probeModel = form.accountAutoFreezeProbeModel.trim()
|
||||
if (probeModel && (probeModel.length > 128 || probeModel !== probeModel.trim())) {
|
||||
toast.warning('探测模型名称不能超过 128 个字符')
|
||||
return
|
||||
}
|
||||
await saveAction.run(async () => {
|
||||
const result = await updateSettings({
|
||||
modelMappings: mappingPayload(),
|
||||
@@ -162,6 +195,13 @@ export function useSettingsForm() {
|
||||
usageRetentionDays: form.usageRetentionDays,
|
||||
opsEventRetentionDays: form.opsEventRetentionDays,
|
||||
auditRetentionDays: form.auditRetentionDays,
|
||||
accountAutoFreezeEnabled: form.accountAutoFreezeEnabled,
|
||||
accountAutoFreezeThreshold,
|
||||
accountAutoFreezeWindowSeconds,
|
||||
accountAutoFreezeDurationSeconds,
|
||||
accountAutoFreezeProbeEnabled: form.accountAutoFreezeProbeEnabled,
|
||||
accountAutoFreezeProbeModel: probeModel || null,
|
||||
accountAutoFreezeAdaptiveConcurrency: form.accountAutoFreezeAdaptiveConcurrency,
|
||||
})
|
||||
applySettings(result)
|
||||
toast.success('设置已保存')
|
||||
@@ -189,6 +229,9 @@ export function useSettingsForm() {
|
||||
maxWaitingPerKeyValue,
|
||||
maxWaitingPerAccountValue,
|
||||
concurrencyWaitTimeoutSecondsValue,
|
||||
accountAutoFreezeThresholdValue,
|
||||
accountAutoFreezeWindowSecondsValue,
|
||||
accountAutoFreezeDurationSecondsValue,
|
||||
minCodexDesktopVersionError,
|
||||
minCodexCliVersionError,
|
||||
saveSettings,
|
||||
|
||||
@@ -8,6 +8,7 @@ import BaseConfirmModal from '@/components/base/BaseConfirmModal.vue'
|
||||
import BasePageHeader from '@/components/base/BasePageHeader.vue'
|
||||
import BaseSegmented from '@/components/base/BaseSegmented.vue'
|
||||
|
||||
import AccountAutoFreezeCard from './components/AccountAutoFreezeCard.vue'
|
||||
import AdminApiKeyCard from './components/AdminApiKeyCard.vue'
|
||||
import SettingsBackupSection from './components/backup/SettingsBackupSection.vue'
|
||||
import ClientVersionSettings from './components/client-version/index.vue'
|
||||
@@ -52,6 +53,9 @@ const {
|
||||
maxWaitingPerKeyValue,
|
||||
maxWaitingPerAccountValue,
|
||||
concurrencyWaitTimeoutSecondsValue,
|
||||
accountAutoFreezeThresholdValue,
|
||||
accountAutoFreezeWindowSecondsValue,
|
||||
accountAutoFreezeDurationSecondsValue,
|
||||
|
||||
minCodexDesktopVersionError,
|
||||
minCodexCliVersionError,
|
||||
@@ -136,6 +140,16 @@ watch(
|
||||
v-model:concurrency-wait-timeout-seconds="concurrencyWaitTimeoutSecondsValue"
|
||||
/>
|
||||
|
||||
<AccountAutoFreezeCard
|
||||
v-model:enabled="form.accountAutoFreezeEnabled"
|
||||
v-model:threshold="accountAutoFreezeThresholdValue"
|
||||
v-model:window-seconds="accountAutoFreezeWindowSecondsValue"
|
||||
v-model:duration-seconds="accountAutoFreezeDurationSecondsValue"
|
||||
v-model:probe-enabled="form.accountAutoFreezeProbeEnabled"
|
||||
v-model:probe-model="form.accountAutoFreezeProbeModel"
|
||||
v-model:adaptive-concurrency="form.accountAutoFreezeAdaptiveConcurrency"
|
||||
/>
|
||||
|
||||
<ClientVersionSettings
|
||||
v-model:min-codex-desktop-version="form.minCodexDesktopVersion"
|
||||
v-model:min-codex-cli-version="form.minCodexCliVersion"
|
||||
|
||||
Reference in New Issue
Block a user