mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f174320e2a | ||
|
|
9db8528758 | ||
|
|
119032c067 | ||
|
|
3d1dc79f5d | ||
|
|
128ced4093 | ||
|
|
693025afa4 | ||
|
|
0bfeff28fc | ||
|
|
cb7daffb66 | ||
|
|
d922c3af60 | ||
|
|
366bc5f1ca | ||
|
|
a5e4041295 | ||
|
|
b27877c99b | ||
|
|
7505459ece | ||
|
|
102d064ab0 | ||
|
|
7616fdf4ab | ||
|
|
5cfe0c68cc |
Generated
+1
@@ -3238,6 +3238,7 @@ dependencies = [
|
||||
name = "provider-openai"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"allocation-counter",
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
|
||||
@@ -117,6 +117,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
probes,
|
||||
host.worker_health(),
|
||||
host.connection_lifecycle(),
|
||||
store.diagnostics(),
|
||||
)
|
||||
)
|
||||
.with_middleware({
|
||||
@@ -155,7 +156,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
));
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
host.start_workers(plan, store.worker_leader_lease())
|
||||
host.start_workers(plan, store.worker_leader_lease(), store.diagnostics())
|
||||
);
|
||||
host.report_startup_ready("Workers");
|
||||
let served = host.serve(api.router()).await;
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::{
|
||||
|
||||
impl From<BackupError> for WorkerTaskError {
|
||||
fn from(error: BackupError) -> Self {
|
||||
WorkerTaskError::safe(error.message())
|
||||
WorkerTaskError::safe(error.message()).with_source(error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,8 +125,9 @@ impl BackupTask {
|
||||
if cron.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let schedule = BackupSchedule::parse(cron, self.timezone)
|
||||
.map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?;
|
||||
let schedule = BackupSchedule::parse(cron, self.timezone).map_err(|source| {
|
||||
WorkerTaskError::safe("backup schedule is invalid").with_source(source)
|
||||
})?;
|
||||
|
||||
if settings.schedule_timezone.as_deref() != Some(timezone) {
|
||||
// 时区切换只初始化未来游标,不把旧时区计划当作应补跑任务
|
||||
@@ -685,9 +686,9 @@ fn status_transition(
|
||||
}
|
||||
|
||||
fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError {
|
||||
WorkerTaskError::safe(format!("backup store failed: {error}"))
|
||||
WorkerTaskError::safe("backup store failed").with_source(error)
|
||||
}
|
||||
|
||||
fn infra_error(error: BackupError) -> WorkerTaskError {
|
||||
WorkerTaskError::safe(error.message())
|
||||
WorkerTaskError::safe(error.message()).with_source(error)
|
||||
}
|
||||
|
||||
@@ -66,13 +66,13 @@ impl FreezeRecoveryTask {
|
||||
async fn freeze_policy(&self) -> Option<gateway_core::provider_ports::ProviderFreezePolicy> {
|
||||
let settings = self.deps.settings.load_runtime_settings().await.ok()?;
|
||||
gateway_core::provider_ports::ProviderFreezePolicy::try_new(
|
||||
settings.account_auto_freeze_enabled,
|
||||
settings.account_auto_freeze_threshold,
|
||||
settings.account_auto_freeze_window_seconds,
|
||||
settings.account_auto_freeze_duration_seconds,
|
||||
settings.account_auto_freeze_probe_enabled,
|
||||
settings.account_auto_freeze_probe_model,
|
||||
settings.account_auto_freeze_adaptive_concurrency,
|
||||
settings.values.account_auto_freeze_enabled,
|
||||
settings.values.account_auto_freeze_threshold,
|
||||
settings.values.account_auto_freeze_window_seconds,
|
||||
settings.values.account_auto_freeze_duration_seconds,
|
||||
settings.values.account_auto_freeze_probe_enabled,
|
||||
settings.values.account_auto_freeze_probe_model,
|
||||
settings.values.account_auto_freeze_adaptive_concurrency,
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
|
||||
@@ -104,10 +104,16 @@ impl fmt::Debug for InitialAdminPassword {
|
||||
#[derive(Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct AdminConfig {
|
||||
pub session_ttl_minutes: u64,
|
||||
#[serde(default = "default_absolute_session_ttl_minutes")]
|
||||
pub session_absolute_ttl_minutes: u64,
|
||||
pub default_username: String,
|
||||
pub default_password: InitialAdminPassword,
|
||||
}
|
||||
|
||||
fn default_absolute_session_ttl_minutes() -> u64 {
|
||||
30 * 24 * 60
|
||||
}
|
||||
|
||||
/// Client 登录域的通用启动配置
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct ClientConfig {
|
||||
@@ -151,6 +157,13 @@ impl AdminConfig {
|
||||
if self.session_ttl_minutes == 0 || i64::try_from(self.session_ttl_minutes).is_err() {
|
||||
return Err(AdminConfigError::InvalidField("admin.session_ttl_minutes"));
|
||||
}
|
||||
if self.session_absolute_ttl_minutes == 0
|
||||
|| i64::try_from(self.session_absolute_ttl_minutes).is_err()
|
||||
{
|
||||
return Err(AdminConfigError::InvalidField(
|
||||
"admin.session_absolute_ttl_minutes",
|
||||
));
|
||||
}
|
||||
let password = self.default_password.expose().trim();
|
||||
if password.len() < MINIMUM_INITIAL_PASSWORD_BYTES
|
||||
|| password.contains('$')
|
||||
@@ -167,6 +180,10 @@ impl fmt::Debug for AdminConfig {
|
||||
formatter
|
||||
.debug_struct("AdminConfig")
|
||||
.field("session_ttl_minutes", &self.session_ttl_minutes)
|
||||
.field(
|
||||
"session_absolute_ttl_minutes",
|
||||
&self.session_absolute_ttl_minutes,
|
||||
)
|
||||
.field("default_username", &self.default_username)
|
||||
.field("default_password", &"[REDACTED]")
|
||||
.finish()
|
||||
@@ -401,6 +418,7 @@ async fn initialize_inner(
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
config.default_username,
|
||||
config.session_ttl_minutes,
|
||||
config.session_absolute_ttl_minutes,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
|
||||
@@ -10,6 +10,23 @@ use gateway_core::{
|
||||
|
||||
use super::{PageSize, Revision, observability::DecimalAmount};
|
||||
|
||||
/// 名称与描述的共同约束,HTTP、用例和存储边界只映射错误
|
||||
pub fn validate_group_fields(
|
||||
name: &str,
|
||||
description: Option<&str>,
|
||||
) -> Result<(), super::AdminModelError> {
|
||||
if name.trim() != name
|
||||
|| name.is_empty()
|
||||
|| name.chars().count() > 100
|
||||
|| name.chars().any(char::is_control)
|
||||
|| description
|
||||
.is_some_and(|value| value.len() > 4096 || value.chars().any(char::is_control))
|
||||
{
|
||||
return Err(super::AdminModelError::InvalidAccountGroupFields);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 账号分组持久化使用的标准 `#RRGGBBAA` 颜色
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupColor(String);
|
||||
|
||||
@@ -133,40 +133,10 @@ pub struct AccountRecord {
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 单一货币的账号成本聚合
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountCost {
|
||||
pub currency: String,
|
||||
pub amount: super::observability::DecimalAmount,
|
||||
}
|
||||
|
||||
/// 账号在一个模型上的历史用量
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountModelUsage {
|
||||
pub model: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
pub image_request_failed_count: u64,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_coverage: super::observability::CostCoverage,
|
||||
pub costs: Vec<AccountCost>,
|
||||
pub last_used_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 账号在一个小时窗口内的请求数
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountRequestBucket {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
}
|
||||
pub use super::observability::{
|
||||
AccountModelUsage, AccountPoolMetrics as AccountSummary, AccountRequestBucket,
|
||||
CurrencyCost as AccountCost,
|
||||
};
|
||||
|
||||
/// 账号历史用量聚合
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -235,19 +205,6 @@ pub struct AccountCapacity {
|
||||
pub total_slots: Option<u64>,
|
||||
}
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响
|
||||
///
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号可编辑事实的一次性替换命令
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateAccount {
|
||||
|
||||
@@ -138,11 +138,12 @@ impl std::fmt::Debug for ChangePassword {
|
||||
}
|
||||
}
|
||||
|
||||
/// 两种登录方式共用的固定有效期会话
|
||||
/// 会话有效期与管理员续期上限;没有上限字段的旧会话保持固定有效期
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AuthSession {
|
||||
pub subject: SessionSubject,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub absolute_expires_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 安全审计事件类型
|
||||
|
||||
@@ -20,6 +20,20 @@ pub type ProviderRequestProfileOverrides =
|
||||
pub type ProviderRequestProfileOverrideUpdates =
|
||||
BTreeMap<ProviderKind, Option<gateway_core::account::OpaqueProviderData>>;
|
||||
|
||||
/// Client Key 关联分组的数量与唯一性约束
|
||||
pub fn validate_group_ids(group_ids: &[AccountGroupId]) -> Result<(), AdminModelError> {
|
||||
if group_ids.len() > 1000
|
||||
|| group_ids
|
||||
.iter()
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.len()
|
||||
!= group_ids.len()
|
||||
{
|
||||
return Err(AdminModelError::InvalidClientKeyGroups);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Client Key 列表保持旧 HTTP 合同允许的完整非零 `u16` 页大小
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct ClientKeyPageSize(NonZeroU16);
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
//! 单页 Key 用量的查询合同;身份范围不由调用方提供
|
||||
|
||||
use super::{
|
||||
PageSize,
|
||||
client_keys::ClientKeyRecord,
|
||||
observability::{
|
||||
HealthTimeline, OpsErrorPage, RequestMetricPoint, TimeRange, UsageOverview, UsagePage,
|
||||
HealthTimeline, ObservabilityPageSize, OpsErrorPage, RequestMetricPoint, TimeRange,
|
||||
UsageOverview, UsagePage,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -25,7 +25,7 @@ pub struct KeyUsageRecordsQuery {
|
||||
pub usage: KeyUsageQuery,
|
||||
pub kind: KeyUsageRecordKind,
|
||||
pub current_page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
}
|
||||
|
||||
pub struct KeyUsageOverview {
|
||||
|
||||
@@ -39,12 +39,31 @@ pub enum AdminErrorKind {
|
||||
Internal,
|
||||
}
|
||||
|
||||
/// 不携带基础设施细节、可安全返回给管理员的管理用例错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
impl AdminErrorKind {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Invalid => "invalid",
|
||||
Self::Forbidden => "forbidden",
|
||||
Self::Unauthorized => "unauthorized",
|
||||
Self::NotFound => "not_found",
|
||||
Self::Conflict => "conflict",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::BadGateway => "bad_gateway",
|
||||
Self::UpstreamResultUnknown => "upstream_result_unknown",
|
||||
Self::Unavailable => "unavailable",
|
||||
Self::Internal => "internal",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理用例的安全消息与内部原因分离,原始来源不进入响应
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct AdminError {
|
||||
kind: AdminErrorKind,
|
||||
message: String,
|
||||
source: Option<gateway_core::error::ErrorSource>,
|
||||
}
|
||||
|
||||
impl AdminError {
|
||||
@@ -53,9 +72,21 @@ impl AdminError {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<gateway_core::error::ErrorDetails> {
|
||||
gateway_core::error::ErrorDetails::capture(self.source.as_ref(), None, false)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> AdminErrorKind {
|
||||
self.kind
|
||||
@@ -184,6 +215,12 @@ pub enum AdminModelError {
|
||||
ZeroRevision,
|
||||
#[error("page size {0} is outside 1..=200")]
|
||||
InvalidPageSize(u16),
|
||||
#[error("observability page size {0} is outside 1..=100")]
|
||||
InvalidObservabilityPageSize(u16),
|
||||
#[error("account group name or description is invalid")]
|
||||
InvalidAccountGroupFields,
|
||||
#[error("client key group IDs are duplicated or exceed 1000 entries")]
|
||||
InvalidClientKeyGroups,
|
||||
#[error("client key page size must be inside 1..=65535")]
|
||||
InvalidClientKeyPageSize,
|
||||
#[error("request outcome must be 1..=256 bytes without control characters")]
|
||||
|
||||
@@ -1,10 +1,32 @@
|
||||
//! 用量、成本、健康与错误诊断的 UTC 语义事实
|
||||
|
||||
use std::str::FromStr;
|
||||
use std::{num::NonZeroU16, str::FromStr};
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
|
||||
use super::{AdminModelError, PageSize};
|
||||
use super::AdminModelError;
|
||||
|
||||
/// 观测记录查询的页大小,由 HTTP、管理用例和存储端口共同使用
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ObservabilityPageSize(NonZeroU16);
|
||||
|
||||
impl ObservabilityPageSize {
|
||||
pub const MAX: u16 = 100;
|
||||
|
||||
pub fn new(value: u16) -> Result<Self, AdminModelError> {
|
||||
if value > Self::MAX {
|
||||
return Err(AdminModelError::InvalidObservabilityPageSize(value));
|
||||
}
|
||||
NonZeroU16::new(value)
|
||||
.map(Self)
|
||||
.ok_or(AdminModelError::InvalidObservabilityPageSize(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn get(self) -> u16 {
|
||||
self.0.get()
|
||||
}
|
||||
}
|
||||
|
||||
/// 页面筛选表达自然日范围,具体 UTC 边界由部署时区解析
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -152,7 +174,7 @@ pub struct UsageQuery {
|
||||
pub range: TimeRange,
|
||||
pub filter: UsageFilter,
|
||||
pub current_page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
}
|
||||
|
||||
/// 运维错误过滤条件
|
||||
@@ -178,7 +200,7 @@ pub struct OpsErrorQuery {
|
||||
pub range: TimeRange,
|
||||
pub filter: OpsErrorFilter,
|
||||
pub current_page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
}
|
||||
|
||||
/// 用量诊断维度
|
||||
@@ -220,7 +242,7 @@ impl DecimalAmount {
|
||||
}
|
||||
|
||||
fn from_decimal(value: gateway_core::metering::Decimal) -> Option<Self> {
|
||||
Some(Self(value.canonical()))
|
||||
Self::from_str(&value.canonical()).ok()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -466,7 +488,7 @@ pub struct RequestMetricPoint {
|
||||
}
|
||||
|
||||
/// 账号池的统一五态统计
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub normal: u64,
|
||||
@@ -476,9 +498,9 @@ pub struct AccountPoolMetrics {
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 中一个账号的模型级用量事实
|
||||
/// 账号在查询时间范围内的模型级用量事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DashboardAccountModelUsage {
|
||||
pub struct AccountModelUsage {
|
||||
pub model: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -497,9 +519,9 @@ pub struct DashboardAccountModelUsage {
|
||||
pub last_used_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Dashboard 中账号的单小时请求数
|
||||
/// 账号在单小时窗口内的请求数
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DashboardAccountRequestBucket {
|
||||
pub struct AccountRequestBucket {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
}
|
||||
@@ -530,14 +552,14 @@ pub struct DashboardAccountUsage {
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
pub last_used_at: Option<DateTime<Utc>>,
|
||||
pub request_buckets: Vec<DashboardAccountRequestBucket>,
|
||||
pub request_buckets: Vec<AccountRequestBucket>,
|
||||
/// Provider 已持久化额度窗口投影出的代表性已用比例
|
||||
///
|
||||
/// `None` 表示上游未提供可比较的百分比,不应伪造为零
|
||||
pub quota_used_percent: Option<f64>,
|
||||
/// Provider 返回的代表窗口;缺失时保留未知语义
|
||||
pub quota_window: Option<super::provider_credentials::ProviderQuotaWindow>,
|
||||
pub models: Vec<DashboardAccountModelUsage>,
|
||||
pub models: Vec<AccountModelUsage>,
|
||||
}
|
||||
|
||||
/// Dashboard 当前账号池的可重建运行时槽位事实
|
||||
@@ -552,7 +574,7 @@ pub struct DashboardRuntimeSlots {
|
||||
}
|
||||
|
||||
/// 仪表盘卡片脚注展示的全历史累计
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
@@ -851,7 +873,7 @@ pub struct OpsError {
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: String,
|
||||
pub raw_upstream_error: Option<String>,
|
||||
pub error_details: Option<String>,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
|
||||
@@ -862,7 +862,7 @@ pub struct ProviderSubscription {
|
||||
}
|
||||
|
||||
/// 按需汇聚的个人信息;资料查询失败不丢弃可用的订阅结果
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountPersonalInfo {
|
||||
pub profile: Result<ProviderProfileStatistics, AdminError>,
|
||||
pub subscription: Option<ProviderSubscription>,
|
||||
|
||||
@@ -17,8 +17,26 @@ pub struct RetentionPolicy {
|
||||
}
|
||||
|
||||
impl RetentionPolicy {
|
||||
/// 保留期管理与全量设置保存共用的字段约束
|
||||
pub fn validate_values(
|
||||
usage_days: u32,
|
||||
ops_days: u32,
|
||||
audit_days: u32,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
(usage_days >= 31, "usage_retention_days"),
|
||||
(ops_days > 0, "ops_event_retention_days"),
|
||||
(audit_days > 0, "audit_retention_days"),
|
||||
] {
|
||||
if !valid {
|
||||
return Err(field);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn try_new(usage_days: u32, ops_days: u32, audit_days: u32) -> Result<Self, AdminError> {
|
||||
if usage_days < 31 || ops_days == 0 || audit_days == 0 {
|
||||
if Self::validate_values(usage_days, ops_days, audit_days).is_err() {
|
||||
return Err(AdminError::invalid(
|
||||
"请求保留期至少 31 天,事件与审计保留期必须为正数",
|
||||
));
|
||||
|
||||
@@ -11,6 +11,26 @@ use super::Revision;
|
||||
/// 客户端模型到上游模型的全局精确映射
|
||||
pub type ModelMappings = BTreeMap<PublicModelId, UpstreamModelId>;
|
||||
|
||||
/// 全局模型映射的数量与文本约束;调用方保留各自的键值表示
|
||||
pub fn validate_model_mappings<'a>(
|
||||
mappings: impl ExactSizeIterator<Item = (&'a str, &'a str)>,
|
||||
) -> Result<(), &'static str> {
|
||||
if mappings.len() > 512 {
|
||||
return Err("model_mappings");
|
||||
}
|
||||
for (requested, upstream) in mappings {
|
||||
for value in [requested, upstream] {
|
||||
if value.is_empty()
|
||||
|| value.len() > 256
|
||||
|| value.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err("model_mappings");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Provider-owned 请求画像选择;Provider ID 是唯一命名空间
|
||||
pub type ProviderRequestProfiles =
|
||||
BTreeMap<ProviderKind, gateway_core::account::OpaqueProviderData>;
|
||||
@@ -22,15 +42,11 @@ pub type ProviderRequestProfileUpdates =
|
||||
/// 账号调度策略;由 Core 拥有稳定值与 wire 映射
|
||||
pub use gateway_core::account::RotationStrategy;
|
||||
|
||||
/// 完整运行设置事实
|
||||
/// 运行设置的共同值;版本、Provider 画像更新和数据库秘密由外层类型拥有
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RuntimeSettings {
|
||||
pub request_profiles: ProviderRequestProfiles,
|
||||
pub config_revision: Revision,
|
||||
pub struct RuntimeSettingsValues {
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
pub max_concurrent_per_account: u32,
|
||||
@@ -39,9 +55,11 @@ pub struct RuntimeSettings {
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub rotation_strategy: RotationStrategy,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
@@ -57,6 +75,18 @@ pub struct RuntimeSettings {
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
}
|
||||
|
||||
/// 完整运行设置事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RuntimeSettings {
|
||||
#[serde(flatten)]
|
||||
pub values: RuntimeSettingsValues,
|
||||
pub request_profiles: ProviderRequestProfiles,
|
||||
pub config_revision: Revision,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub rotation_strategy: RotationStrategy,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -64,39 +94,14 @@ pub struct RuntimeSettings {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReplaceRuntimeSettings {
|
||||
#[serde(flatten)]
|
||||
pub values: RuntimeSettingsValues,
|
||||
/// 读取设置时的版本;与写入在同一事务内比较,防止覆盖并发更新
|
||||
pub expected_revision: Revision,
|
||||
/// 只覆盖提交的 Provider;未提交项保留当前持久值
|
||||
pub request_profile_updates: ProviderRequestProfileUpdates,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u32,
|
||||
pub max_concurrent_per_account: u32,
|
||||
pub request_interval_ms: u64,
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub rotation_strategy: RotationStrategy,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u32,
|
||||
pub ops_event_retention_days: u32,
|
||||
pub audit_retention_days: u32,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u32,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
}
|
||||
|
||||
/// 明文管理员 API Key;按产品约束明文落库,但禁止 Debug 泄漏
|
||||
@@ -174,36 +179,63 @@ impl From<RuntimeSettings> for ReplaceRuntimeSettings {
|
||||
.into_iter()
|
||||
.map(|(provider, value)| (provider, Some(value)))
|
||||
.collect(),
|
||||
request_location_enabled: settings.request_location_enabled,
|
||||
request_location: settings.request_location,
|
||||
values: settings.values,
|
||||
model_mappings: settings.model_mappings,
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: settings.refresh_concurrency,
|
||||
max_concurrent_per_account: settings.max_concurrent_per_account,
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: settings.smart_scheduling,
|
||||
rotation_strategy: settings.rotation_strategy,
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: settings.usage_retention_days,
|
||||
ops_event_retention_days: settings.ops_event_retention_days,
|
||||
audit_retention_days: settings.audit_retention_days,
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: settings.account_auto_freeze_threshold,
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
account_warmup_enabled: settings.account_warmup_enabled,
|
||||
account_warmup_schedule_time: settings.account_warmup_schedule_time,
|
||||
account_warmup_model: settings.account_warmup_model,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RuntimeSettingsValues {
|
||||
/// 校验共同事实,返回字段名供边界转换错误;SQL 类型转换仍由 Store 处理
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
self.request_location
|
||||
.validate()
|
||||
.map_err(|_| "request_location")?;
|
||||
gateway_core::settings::validate_request_limits(
|
||||
self.max_waiting_per_key,
|
||||
self.max_waiting_per_account,
|
||||
self.concurrency_wait_timeout_seconds,
|
||||
self.max_account_rotations,
|
||||
self.openai_session_affinity_ttl_hours,
|
||||
)?;
|
||||
gateway_core::settings::response_body_limit(self.responses_max_decompressed_body_bytes)?;
|
||||
gateway_core::settings::client_min_versions(
|
||||
self.min_codex_desktop_version.as_deref(),
|
||||
self.min_codex_cli_version.as_deref(),
|
||||
)?;
|
||||
for (valid, field) in [
|
||||
(
|
||||
self.refresh_margin_seconds > 0
|
||||
&& i64::try_from(self.refresh_margin_seconds).is_ok(),
|
||||
"refresh_margin_seconds",
|
||||
),
|
||||
(self.refresh_concurrency > 0, "refresh_concurrency"),
|
||||
(
|
||||
i64::try_from(self.request_interval_ms).is_ok(),
|
||||
"request_interval_ms",
|
||||
),
|
||||
] {
|
||||
if !valid {
|
||||
return Err(field);
|
||||
}
|
||||
}
|
||||
super::retention::RetentionPolicy::validate_values(
|
||||
self.usage_retention_days,
|
||||
self.ops_event_retention_days,
|
||||
self.audit_retention_days,
|
||||
)?;
|
||||
gateway_core::provider_ports::ProviderFreezePolicy::validate_values(
|
||||
self.account_auto_freeze_threshold,
|
||||
self.account_auto_freeze_window_seconds,
|
||||
self.account_auto_freeze_duration_seconds,
|
||||
self.account_auto_freeze_probe_model.as_deref(),
|
||||
)?;
|
||||
gateway_core::provider_ports::ProviderWarmupPolicy::validate_values(
|
||||
self.account_warmup_enabled,
|
||||
&self.account_warmup_schedule_time,
|
||||
self.account_warmup_model.as_deref(),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,11 +46,12 @@ pub enum ProviderAdminErrorKind {
|
||||
/// `message` 是 Provider 局部诊断,可能包含原始上游正文;通用管理用例不得自动把它作为公开文案
|
||||
/// 只有明确拥有原始诊断合同的调用方才能读取,`Debug` 始终只记录是否存在
|
||||
/// `public_message` 则是明确标记为可公开的静态提示,不允许携带动态上游材料
|
||||
#[derive(Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Clone, thiserror::Error)]
|
||||
#[error("provider admin operation failed: {kind:?}")]
|
||||
pub struct ProviderAdminError {
|
||||
kind: ProviderAdminErrorKind,
|
||||
message: Option<String>,
|
||||
#[source]
|
||||
diagnostic: Option<Arc<ProviderAdminDiagnostic>>,
|
||||
public_message: Option<&'static str>,
|
||||
}
|
||||
|
||||
@@ -59,7 +60,7 @@ impl std::fmt::Debug for ProviderAdminError {
|
||||
formatter
|
||||
.debug_struct("ProviderAdminError")
|
||||
.field("kind", &self.kind)
|
||||
.field("message", &self.message.as_ref().map(|_| "<redacted>"))
|
||||
.field("message", &self.message().map(|_| "<redacted>"))
|
||||
.field("public_message", &self.public_message)
|
||||
.finish()
|
||||
}
|
||||
@@ -70,14 +71,22 @@ impl ProviderAdminError {
|
||||
pub const fn new(kind: ProviderAdminErrorKind) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: None,
|
||||
diagnostic: None,
|
||||
public_message: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
|
||||
Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).source =
|
||||
Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_message(mut self, message: impl Into<String>) -> Self {
|
||||
self.message = Some(message.into());
|
||||
Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).message =
|
||||
Some(message.into());
|
||||
self
|
||||
}
|
||||
|
||||
@@ -100,10 +109,20 @@ impl ProviderAdminError {
|
||||
|
||||
#[must_use]
|
||||
pub fn message(&self) -> Option<&str> {
|
||||
self.message.as_deref()
|
||||
self.diagnostic
|
||||
.as_ref()
|
||||
.and_then(|diagnostic| diagnostic.message.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
// Provider 局部消息与 typed 来源共用一个不公开的上下文,避免跨用例转换后丢失
|
||||
#[derive(Debug, Clone, Default, thiserror::Error)]
|
||||
#[error("{}", message.as_deref().unwrap_or("provider admin dependency failed"))]
|
||||
struct ProviderAdminDiagnostic {
|
||||
message: Option<String>,
|
||||
source: Option<gateway_core::error::ErrorSource>,
|
||||
}
|
||||
|
||||
/// 一个具体 Provider 对管理控制面提供的解析、验证、上游交互与运行时资源回收能力
|
||||
///
|
||||
/// 数据变更由 Provider 返回 prepared facts;config revision、审计与 PostgreSQL 事务
|
||||
|
||||
@@ -52,12 +52,13 @@ pub enum AdminStoreErrorKind {
|
||||
}
|
||||
|
||||
/// 隐藏数据库实现细节的持久化错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("{resource} store operation failed: {message}")]
|
||||
pub struct AdminStoreError {
|
||||
kind: AdminStoreErrorKind,
|
||||
resource: &'static str,
|
||||
message: String,
|
||||
source: Option<gateway_core::error::ErrorSource>,
|
||||
}
|
||||
|
||||
impl AdminStoreError {
|
||||
@@ -71,9 +72,27 @@ impl AdminStoreError {
|
||||
kind,
|
||||
resource,
|
||||
message: message.into(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 来源留在内部诊断链,公开消息不展开基础设施错误
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// 清理失败不覆盖最初操作的分类与原因
|
||||
#[must_use]
|
||||
pub fn with_cleanup(mut self, cleanup: impl Into<gateway_core::error::ErrorSource>) -> Self {
|
||||
self.source = Some(gateway_core::error::ErrorSource::cleanup(
|
||||
self.source.take(),
|
||||
cleanup,
|
||||
));
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> AdminStoreErrorKind {
|
||||
self.kind
|
||||
@@ -274,6 +293,14 @@ pub trait AuthStore: Send + Sync {
|
||||
|
||||
async fn store_session(&self, session_id: &str, session: &AuthSession) -> AdminStoreResult<()>;
|
||||
|
||||
/// 只延长仍存在且匹配的会话;并发续期返回当前值,已退出或过期时不重建
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> AdminStoreResult<Option<AuthSession>>;
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AuthSession>>;
|
||||
|
||||
async fn client_key_enabled(
|
||||
|
||||
@@ -135,6 +135,11 @@ impl AccountGroupService for DefaultAccountGroupService {
|
||||
context: &MutationContext,
|
||||
command: CreateAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
crate::model::account_groups::validate_group_fields(
|
||||
&command.name,
|
||||
command.description.as_deref(),
|
||||
)
|
||||
.map_err(|_| AdminError::invalid("账号组请求不合法"))?;
|
||||
let id = AccountGroupId::new(format!("grp_{}", Uuid::now_v7().simple()))
|
||||
.map_err(|_| AdminError::internal("创建账号组 ID 失败"))?;
|
||||
self.publish(
|
||||
@@ -159,6 +164,11 @@ impl AccountGroupService for DefaultAccountGroupService {
|
||||
context: &MutationContext,
|
||||
command: UpdateAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
crate::model::account_groups::validate_group_fields(
|
||||
&command.name,
|
||||
command.description.as_deref(),
|
||||
)
|
||||
.map_err(|_| AdminError::invalid("账号组请求不合法"))?;
|
||||
self.publish(self.store.update_account_group(command, context).await)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -36,6 +36,10 @@ pub trait AuthService: Send + Sync {
|
||||
) -> Result<(), AdminError>;
|
||||
async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminError>;
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn resolve_admin_user_id(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
@@ -58,6 +62,7 @@ const LOGIN_ATTEMPTS_GLOBAL: u32 = 200;
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
admin_session_ttl: Duration,
|
||||
admin_session_absolute_ttl: Duration,
|
||||
key_session_ttl: Duration,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
@@ -68,6 +73,7 @@ impl DefaultAuthService {
|
||||
pub(crate) fn new(
|
||||
default_admin_user_id: impl Into<String>,
|
||||
admin_session_ttl_minutes: u64,
|
||||
admin_session_absolute_ttl_minutes: u64,
|
||||
key_session_ttl_minutes: u64,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
@@ -75,6 +81,7 @@ impl DefaultAuthService {
|
||||
Self {
|
||||
default_admin_user_id: default_admin_user_id.into(),
|
||||
admin_session_ttl: session_ttl(admin_session_ttl_minutes),
|
||||
admin_session_absolute_ttl: session_ttl(admin_session_absolute_ttl_minutes),
|
||||
key_session_ttl: session_ttl(key_session_ttl_minutes),
|
||||
store,
|
||||
verifier,
|
||||
@@ -232,8 +239,12 @@ impl AuthService for DefaultAuthService {
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if session.expires_at <= Utc::now() {
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
if session.expires_at <= Utc::now()
|
||||
|| session
|
||||
.absolute_expires_at
|
||||
.is_some_and(|expiry| expiry <= Utc::now())
|
||||
{
|
||||
// Redis 的 TTL 负责过期清理,避免旧读取删除另一个请求刚续期的会话
|
||||
return Ok(None);
|
||||
}
|
||||
if let SessionSubject::Admin {
|
||||
@@ -266,6 +277,37 @@ impl AuthService for DefaultAuthService {
|
||||
Ok(Some(session))
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session_id) = session_id else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(session) = self.session(Some(session_id)).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Key 与升级前的会话保留原有期限,不能凭续期获得新的最长登录窗口
|
||||
let Some(absolute) = session
|
||||
.absolute_expires_at
|
||||
.filter(|_| matches!(session.subject, SessionSubject::Admin { .. }))
|
||||
else {
|
||||
return Ok(Some(session));
|
||||
};
|
||||
let now = Utc::now();
|
||||
let expires_at = (now + self.admin_session_ttl).min(absolute);
|
||||
let interval = (self.admin_session_ttl / 4).min(Duration::minutes(1));
|
||||
if expires_at <= session.expires_at
|
||||
|| session.expires_at - now > self.admin_session_ttl - interval
|
||||
{
|
||||
return Ok(Some(session));
|
||||
}
|
||||
self.store
|
||||
.renew_session(session_id, &session, expires_at)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session renewal"))
|
||||
}
|
||||
|
||||
async fn resolve_admin_user_id(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
@@ -325,9 +367,13 @@ impl AuthService for DefaultAuthService {
|
||||
SessionSubject::Admin { .. } => self.admin_session_ttl,
|
||||
SessionSubject::Key { .. } => self.key_session_ttl,
|
||||
};
|
||||
let now = Utc::now();
|
||||
let absolute_expires_at = matches!(subject, SessionSubject::Admin { .. })
|
||||
.then_some(now + self.admin_session_absolute_ttl);
|
||||
let session = AuthSession {
|
||||
subject,
|
||||
expires_at: Utc::now() + ttl,
|
||||
expires_at: (now + ttl).min(absolute_expires_at.unwrap_or(now + ttl)),
|
||||
absolute_expires_at,
|
||||
};
|
||||
let session_id = random_session_token();
|
||||
self.store
|
||||
|
||||
@@ -157,6 +157,8 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
context: &MutationContext,
|
||||
command: CreateClientKey,
|
||||
) -> Result<CreatedClientKey, AdminError> {
|
||||
crate::model::client_keys::validate_group_ids(&command.group_ids)
|
||||
.map_err(|_| AdminError::invalid("Client API Key 分组不合法"))?;
|
||||
for (provider, profile) in &command.request_profile_overrides {
|
||||
self.providers
|
||||
.require(provider)
|
||||
@@ -199,6 +201,8 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
context: &MutationContext,
|
||||
command: UpdateClientKey,
|
||||
) -> Result<ClientKeyMutation, AdminError> {
|
||||
crate::model::client_keys::validate_group_ids(&command.group_ids)
|
||||
.map_err(|_| AdminError::invalid("Client API Key 分组不合法"))?;
|
||||
for (provider, profile) in &command.request_profile_override_updates {
|
||||
if let Some(profile) = profile {
|
||||
self.providers
|
||||
|
||||
@@ -51,7 +51,6 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError
|
||||
| AdminStoreErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
AdminStoreErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
};
|
||||
tracing::warn!(resource, error_kind = ?error.kind(), "admin store operation failed");
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "请求参数不合法",
|
||||
AdminErrorKind::NotFound => "请求的资源不存在",
|
||||
@@ -59,7 +58,10 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError
|
||||
AdminErrorKind::Unavailable => "依赖服务暂不可用",
|
||||
_ => "服务内部错误",
|
||||
};
|
||||
AdminError::new(kind, message)
|
||||
AdminError::new(kind, message).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
source: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn map_provider_error(
|
||||
@@ -79,7 +81,6 @@ fn map_provider_error(
|
||||
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
|
||||
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
tracing::warn!(resource, error_kind = ?error.kind(), "admin provider operation failed");
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "Provider 请求不合法",
|
||||
AdminErrorKind::NotFound => "Provider 资源不存在",
|
||||
@@ -90,7 +91,10 @@ fn map_provider_error(
|
||||
AdminErrorKind::Internal => "服务内部错误",
|
||||
_ => "Provider 操作失败",
|
||||
};
|
||||
AdminError::new(kind, error.public_message().unwrap_or(message))
|
||||
AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
source: error.into(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn publish_committed(
|
||||
@@ -454,3 +458,11 @@ async fn delete_credentials(
|
||||
account_ids,
|
||||
})
|
||||
}
|
||||
|
||||
// 用例的操作名补充端口资源上下文,底层分类与原始原因仍由来源持有
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("admin operation failed: {operation}")]
|
||||
struct OperationFailure {
|
||||
operation: &'static str,
|
||||
source: gateway_core::error::ErrorSource,
|
||||
}
|
||||
|
||||
@@ -204,7 +204,7 @@ impl DefaultObservabilityService {
|
||||
let wire_profiles = self
|
||||
.providers
|
||||
.dashboard_wire_profiles(&settings.request_profiles);
|
||||
let max_concurrent_per_account = u64::from(settings.max_concurrent_per_account);
|
||||
let max_concurrent_per_account = u64::from(settings.values.max_concurrent_per_account);
|
||||
let (inherited_accounts, overridden_slots) = runtime_slots
|
||||
.as_ref()
|
||||
.map_or((observation.provider_accounts.normal, 0), |slots| {
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_core::policy::CodexClientVersion;
|
||||
use gateway_core::runtime::SnapshotControl;
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
|
||||
@@ -103,50 +102,17 @@ impl DefaultSettingsService {
|
||||
}
|
||||
|
||||
fn validate_settings(command: &ReplaceRuntimeSettings) -> Result<(), AdminError> {
|
||||
let valid = command.request_location.validate().is_ok()
|
||||
&& command.responses_max_decompressed_body_bytes > 0
|
||||
&& isize::try_from(command.responses_max_decompressed_body_bytes).is_ok()
|
||||
&& command.refresh_margin_seconds > 0
|
||||
&& command.refresh_concurrency > 0
|
||||
&& command.max_waiting_per_key <= 1_000
|
||||
&& command.max_waiting_per_account <= 1_000
|
||||
&& (1..=120).contains(&command.concurrency_wait_timeout_seconds)
|
||||
&& crate::model::retention::RetentionPolicy::try_new(
|
||||
command.usage_retention_days,
|
||||
command.ops_event_retention_days,
|
||||
command.audit_retention_days,
|
||||
)
|
||||
.is_ok()
|
||||
&& valid_client_version(command.min_codex_desktop_version.as_deref())
|
||||
&& valid_client_version(command.min_codex_cli_version.as_deref())
|
||||
&& valid_probe_model(command.account_auto_freeze_probe_model.as_deref())
|
||||
&& gateway_core::provider_ports::valid_warmup_schedule_time(
|
||||
&command.account_warmup_schedule_time,
|
||||
)
|
||||
&& valid_probe_model(command.account_warmup_model.as_deref())
|
||||
&& (!command.account_warmup_enabled || command.account_warmup_model.is_some())
|
||||
&& i64::try_from(command.request_interval_ms).is_ok()
|
||||
&& (2..=1_000).contains(&command.account_auto_freeze_threshold)
|
||||
&& (60..=3_600).contains(&command.account_auto_freeze_window_seconds)
|
||||
&& (300..=604_800).contains(&command.account_auto_freeze_duration_seconds);
|
||||
if valid {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AdminError::invalid("运行时设置不满足约束"))
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_client_version(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| CodexClientVersion::parse(value).is_ok())
|
||||
}
|
||||
|
||||
fn valid_probe_model(value: Option<&str>) -> bool {
|
||||
value.is_none_or(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 128
|
||||
&& value == value.trim()
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
crate::model::settings::validate_model_mappings(
|
||||
command
|
||||
.model_mappings
|
||||
.iter()
|
||||
.map(|(requested, upstream)| (requested.as_str(), upstream.as_str())),
|
||||
)
|
||||
.map_err(|_| AdminError::invalid("运行时设置不满足约束"))?;
|
||||
command
|
||||
.values
|
||||
.validate()
|
||||
.map_err(|_| AdminError::invalid("运行时设置不满足约束"))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -683,6 +683,15 @@ impl AuthStore for FakeAuthStore {
|
||||
) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn renew_session(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &gateway_admin::model::auth::AuthSession,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AuthSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn delete_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
//! 验证分组文本共用规则中的字符数、字节数和控制字符边界
|
||||
|
||||
use gateway_admin::model::account_groups::validate_group_fields;
|
||||
|
||||
#[test]
|
||||
fn group_fields_distinguish_name_characters_from_description_bytes() {
|
||||
let name = "组".repeat(100);
|
||||
let description = "é".repeat(2048);
|
||||
assert!(validate_group_fields(&name, Some(&description)).is_ok());
|
||||
assert!(validate_group_fields(&"组".repeat(101), None).is_err());
|
||||
assert!(validate_group_fields("组", Some(&"é".repeat(2049))).is_err());
|
||||
for name in ["", " 组", "组 ", "组\n名"] {
|
||||
assert!(validate_group_fields(name, None).is_err());
|
||||
}
|
||||
assert!(validate_group_fields("组", Some("含\t制表符")).is_err());
|
||||
assert!(validate_group_fields("组", Some("")).is_ok());
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! 验证 Client Key 分组集合的共同上限与唯一性规则
|
||||
|
||||
use gateway_admin::model::client_keys::validate_group_ids;
|
||||
use gateway_core::routing::AccountGroupId;
|
||||
|
||||
#[test]
|
||||
fn client_key_groups_reject_duplicates_and_more_than_one_thousand_ids() {
|
||||
assert!(validate_group_ids(&[]).is_ok());
|
||||
let mut groups: Vec<_> = (0..1000)
|
||||
.map(|index| AccountGroupId::new(format!("grp_{index:032x}")).unwrap())
|
||||
.collect();
|
||||
assert!(validate_group_ids(&groups).is_ok());
|
||||
groups.push(AccountGroupId::new("grp_ffffffffffffffffffffffffffffffff").unwrap());
|
||||
assert!(validate_group_ids(&groups).is_err());
|
||||
assert!(validate_group_ids(&[groups[0].clone(), groups[0].clone()]).is_err());
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
//! 模型模块测试:镜像 src/model/ 中带逻辑的模块
|
||||
|
||||
mod account_groups;
|
||||
mod audit;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod observability;
|
||||
mod plugins;
|
||||
mod provider_credentials;
|
||||
mod quota_forecast;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
//! 验证观测查询和数值类型在 HTTP、管理及持久化边界共用的约束
|
||||
|
||||
use gateway_admin::model::{
|
||||
PageSize,
|
||||
observability::{DecimalAmount, ObservabilityPageSize, PercentileMilliseconds},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn observability_page_size_has_its_own_limit() {
|
||||
for value in [1, 100] {
|
||||
assert_eq!(ObservabilityPageSize::new(value).unwrap().get(), value);
|
||||
}
|
||||
for value in [0, 101, 150, 200, u16::MAX] {
|
||||
assert!(ObservabilityPageSize::new(value).is_err(), "{value}");
|
||||
}
|
||||
assert!(PageSize::new(200).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decimal_amount_preserves_numeric_precision_and_normalization() {
|
||||
for (input, expected) in [
|
||||
("0", "0"),
|
||||
(" 00012.34000 ", "12.34"),
|
||||
("00000.00000", "0"),
|
||||
("9999999999.9999999999", "9999999999.9999999999"),
|
||||
("0.0000000001", "0.0000000001"),
|
||||
] {
|
||||
assert_eq!(input.parse::<DecimalAmount>().unwrap().as_str(), expected);
|
||||
}
|
||||
for input in [
|
||||
"",
|
||||
"-1",
|
||||
"+1",
|
||||
"NaN",
|
||||
"inf",
|
||||
"1e3",
|
||||
".1",
|
||||
"1.",
|
||||
"1.2.3",
|
||||
"10000000000",
|
||||
"0.00000000001",
|
||||
"1",
|
||||
] {
|
||||
assert!(input.parse::<DecimalAmount>().is_err(), "{input:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decimal_arithmetic_keeps_the_shared_storage_precision() {
|
||||
let max: DecimalAmount = "9999999999.9999999999".parse().unwrap();
|
||||
let unit: DecimalAmount = "0.0000000001".parse().unwrap();
|
||||
assert!(max.checked_add(&unit).is_none());
|
||||
let one: DecimalAmount = "1".parse().unwrap();
|
||||
assert_eq!(one.checked_div_u64(3).unwrap().as_str(), "0.3333333333");
|
||||
assert!(one.checked_div_u64(0).is_none());
|
||||
assert_eq!(one.checked_add(&unit).unwrap().as_str(), "1.0000000001");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentile_preserves_interpolation_and_rejects_invalid_values() {
|
||||
for value in [0.0, 0.125, 1.5, 95.75] {
|
||||
assert_eq!(PercentileMilliseconds::new(value).unwrap().as_f64(), value);
|
||||
}
|
||||
for value in [-1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
|
||||
assert!(PercentileMilliseconds::new(value).is_err());
|
||||
}
|
||||
}
|
||||
@@ -223,8 +223,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() {
|
||||
assert_eq!(
|
||||
PendingAuthorizationMutation::from_storage_v1(Value::Object(
|
||||
expected.to_storage_v1()
|
||||
)),
|
||||
Ok(expected.clone()),
|
||||
))
|
||||
.unwrap(),
|
||||
expected.clone(),
|
||||
);
|
||||
let with_proxy = expected
|
||||
.with_outbound_proxy(Some(
|
||||
@@ -237,8 +238,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() {
|
||||
assert_eq!(
|
||||
PendingAuthorizationMutation::from_storage_v1(Value::Object(
|
||||
with_proxy.to_storage_v1()
|
||||
)),
|
||||
Ok(with_proxy)
|
||||
))
|
||||
.unwrap(),
|
||||
with_proxy
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,3 +297,46 @@ fn unused() -> AdminStoreError {
|
||||
"unused test operation",
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_group_fields_are_rejected_before_store_mutations() {
|
||||
use gateway_admin::model::{AdminErrorKind, MutationActor, account_groups::CreateAccountGroup};
|
||||
let service = AdminHarness::new()
|
||||
.account_groups(Arc::new(FakeGroupStore::default()))
|
||||
.build()
|
||||
.await;
|
||||
let context = MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "invalid-group".to_owned(),
|
||||
};
|
||||
let record = group_record();
|
||||
let created = service
|
||||
.account_groups()
|
||||
.create(
|
||||
&context,
|
||||
CreateAccountGroup {
|
||||
fast_mode: FastMode::Default,
|
||||
name: " invalid".to_owned(),
|
||||
description: None,
|
||||
color: record.color.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let updated = service
|
||||
.account_groups()
|
||||
.update(
|
||||
&context,
|
||||
UpdateAccountGroup {
|
||||
fast_mode: None,
|
||||
id: record.id,
|
||||
name: "valid".to_owned(),
|
||||
description: Some("invalid\n".to_owned()),
|
||||
color: record.color,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(created.kind(), AdminErrorKind::Invalid);
|
||||
assert_eq!(updated.kind(), AdminErrorKind::Invalid);
|
||||
}
|
||||
|
||||
@@ -1110,37 +1110,42 @@ impl SettingsStore for StaticSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 1,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 30,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 1,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 30,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ async fn password_change_obeys_rate_limit_and_rejects_unbound_legacy_sessions()
|
||||
admin_user_id: "admin".into(),
|
||||
credential_fingerprint: String::new(),
|
||||
},
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + TimeDelta::hours(1),
|
||||
},
|
||||
);
|
||||
@@ -137,6 +138,25 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AuthSession>> {
|
||||
if self.reject_delete.lock().unwrap().as_deref() == Some(session_id) {
|
||||
return Err(super::unavailable("delete session"));
|
||||
@@ -262,7 +282,7 @@ async fn login_with_huge_session_ttl_should_clamp_expiry_instead_of_panicking()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_sessions_are_removed_and_store_outages_are_not_treated_as_logout() {
|
||||
async fn expired_sessions_are_rejected_and_store_outages_are_not_treated_as_logout() {
|
||||
use gateway_admin::model::{AdminErrorKind, auth::SessionSubject};
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new().auth(store.clone()).build().await;
|
||||
@@ -273,6 +293,7 @@ async fn expired_sessions_are_removed_and_store_outages_are_not_treated_as_logou
|
||||
credential_fingerprint: String::new(),
|
||||
admin_user_id: "admin".to_owned(),
|
||||
},
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() - TimeDelta::seconds(1),
|
||||
},
|
||||
);
|
||||
@@ -284,7 +305,6 @@ async fn expired_sessions_are_removed_and_store_outages_are_not_treated_as_logou
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
store
|
||||
.unavailable
|
||||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
@@ -370,3 +390,110 @@ async fn failed_rotation_discards_the_new_session_and_leaves_the_old_session_ret
|
||||
services.auth().logout(&original.session_id).await.unwrap();
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_is_throttled_and_capped_by_the_original_absolute_expiry() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.auth(store.clone())
|
||||
.session_ttl_minutes(60)
|
||||
.build()
|
||||
.await;
|
||||
let auth = services.auth();
|
||||
let login = auth
|
||||
.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(login.session.clone())
|
||||
);
|
||||
let limit = Utc::now() + TimeDelta::minutes(30);
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(&login.session_id).unwrap();
|
||||
session.expires_at = Utc::now() + TimeDelta::minutes(1);
|
||||
session.absolute_expires_at = Some(limit);
|
||||
}
|
||||
let renewed = auth
|
||||
.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(renewed.expires_at, limit);
|
||||
assert_eq!(renewed.absolute_expires_at, Some(limit));
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(renewed)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_never_revives_expired_revoked_or_legacy_sessions() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = auth
|
||||
.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut session = login.session.clone();
|
||||
session.absolute_expires_at = None;
|
||||
session.expires_at = Utc::now() + TimeDelta::seconds(30);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session.clone());
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(session.clone())
|
||||
);
|
||||
session.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session);
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let mut session = login.session;
|
||||
session.absolute_expires_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session);
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
auth.logout(&login.session_id).await.unwrap();
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
@@ -285,6 +285,25 @@ impl AuthStore for MemoryKeyAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AuthSession>> {
|
||||
Ok(self
|
||||
.sessions
|
||||
|
||||
@@ -414,3 +414,41 @@ fn unused() -> AdminStoreError {
|
||||
"unused in this test",
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn duplicate_group_ids_are_rejected_before_key_creation_or_update() {
|
||||
let store = Arc::new(TestClientKeyStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.client_keys(store.clone())
|
||||
.build()
|
||||
.await;
|
||||
let group =
|
||||
gateway_core::routing::AccountGroupId::new("grp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap();
|
||||
let mut command = create_command(None);
|
||||
command.group_ids = vec![group.clone(), group.clone()];
|
||||
let created = services
|
||||
.client_keys()
|
||||
.create(&mutation_context(), command)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let updated = services
|
||||
.client_keys()
|
||||
.update(
|
||||
&mutation_context(),
|
||||
UpdateClientKey {
|
||||
request_profile_override_updates: Default::default(),
|
||||
id: ClientApiKeyId::new("key_existing").unwrap(),
|
||||
name: "Migration".to_owned(),
|
||||
label: None,
|
||||
group_ids: vec![group.clone(), group],
|
||||
limits: RateLimits::unlimited(),
|
||||
daily_limit_usd: None,
|
||||
weekly_limit_usd: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(created.kind(), AdminErrorKind::Invalid);
|
||||
assert_eq!(updated.kind(), AdminErrorKind::Invalid);
|
||||
assert!(store.plaintexts.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
@@ -26,37 +26,42 @@ use super::accounts::{FakeAccountStore, FakeProviderAdmin, account_record, event
|
||||
fn runtime_settings(enabled: bool, probe_enabled: bool, adaptive: bool) -> RuntimeSettings {
|
||||
RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: revision(1),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: gateway_admin::model::settings::RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: enabled,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: probe_enabled,
|
||||
account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()),
|
||||
account_auto_freeze_adaptive_concurrency: adaptive,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: enabled,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: probe_enabled,
|
||||
account_auto_freeze_probe_model: Some("gpt-5.5".to_owned()),
|
||||
account_auto_freeze_adaptive_concurrency: adaptive,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -246,6 +246,7 @@ impl AdminHarness {
|
||||
pub(super) async fn build_bundle(self) -> gateway_admin::AdminBundle {
|
||||
gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
session_absolute_ttl_minutes: 30 * 24 * 60,
|
||||
session_ttl_minutes: self.session_ttl_minutes,
|
||||
default_username: "admin".to_owned(),
|
||||
default_password: InitialAdminPassword::new(self.default_password),
|
||||
@@ -364,6 +365,15 @@ impl AuthStore for BootstrapAuthStore {
|
||||
Err(unavailable("admin session"))
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &gateway_admin::model::auth::AuthSession,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AuthSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn delete_session(&self, _: &str) -> AdminStoreResult<Option<AuthSession>> {
|
||||
Err(unavailable("admin session"))
|
||||
}
|
||||
|
||||
@@ -15,15 +15,15 @@ use chrono::{DateTime, Duration, Timelike as _, Utc};
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::{
|
||||
MutationContext, PageSize, Revision,
|
||||
MutationContext, Revision,
|
||||
observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, CostCoverage, CurrencyCost, DashboardAccountUsage,
|
||||
DashboardObservation, DashboardRuntimeSlots, DiagnosticDimension,
|
||||
DiagnosticObservation, DiagnosticsObservation, Granularity, HealthStatus,
|
||||
LatencyPercentiles, OpsErrorPage, OpsErrorQuery, PercentileMilliseconds,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, TrendKind, UsageBilling,
|
||||
UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageListRecord, UsageOverview,
|
||||
UsagePage, UsageQuery,
|
||||
LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
||||
PercentileMilliseconds, RequestMetricPoint, RequestMetrics, TimeRange, TrendKind,
|
||||
UsageBilling, UsageCalculatedBillingFact, UsageDetail, UsageFilter, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageQuery,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -989,37 +989,42 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<RuntimeSettings> {
|
||||
Ok(RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(1).expect("revision"),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: self.max_concurrent_per_account,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
updated_at: Utc::now(),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 300,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: self.max_concurrent_per_account,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1084,7 +1089,7 @@ fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
range: observation_range(now),
|
||||
filter: UsageFilter::default(),
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(50).expect("page size"),
|
||||
page_size: ObservabilityPageSize::new(50).expect("page size"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -163,7 +163,37 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
_ => false,
|
||||
})
|
||||
.unwrap_or_else(|| panic!("缺少合同类型 {name}"));
|
||||
if let Item::Struct(item) = item
|
||||
let mut item = item.clone();
|
||||
// SDK 保持独立的平铺合同,不暴露宿主内部的值对象组合
|
||||
if matches!(*name, "RuntimeSettings" | "ReplaceRuntimeSettings") {
|
||||
let values = items
|
||||
.iter()
|
||||
.find_map(|item| match item {
|
||||
Item::Struct(item) if item.ident == "RuntimeSettingsValues" => {
|
||||
Some(&item.fields)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.expect("settings values");
|
||||
let Item::Struct(record) = &mut item else {
|
||||
unreachable!()
|
||||
};
|
||||
let syn::Fields::Named(fields) = &mut record.fields else {
|
||||
unreachable!()
|
||||
};
|
||||
fields.named = fields
|
||||
.named
|
||||
.iter()
|
||||
.flat_map(|field| {
|
||||
if field.ident.as_ref().is_some_and(|ident| ident == "values") {
|
||||
values.iter().cloned().collect::<Vec<_>>()
|
||||
} else {
|
||||
vec![field.clone()]
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
if let Item::Struct(item) = &item
|
||||
&& item.ident == "ReplaceRuntimeSettings"
|
||||
{
|
||||
replacement_fields = item
|
||||
@@ -173,7 +203,7 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
.collect();
|
||||
}
|
||||
types.push(contract_type(
|
||||
item.clone(),
|
||||
item,
|
||||
if *name == "SmartSchedulingValues" {
|
||||
"SmartSchedulingConfig"
|
||||
} else {
|
||||
@@ -273,6 +303,7 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
pub type ModelMappings = BTreeMap<String, String>;
|
||||
pub type Revision = NonZeroU64;
|
||||
pub type RotationStrategy = String;
|
||||
pub type AccountAffinity = String;
|
||||
pub type PricingOverrides = BTreeMap<String, BTreeMap<String, ModelPriceOverride>>;
|
||||
#(#types)*
|
||||
#(#operations)*
|
||||
|
||||
@@ -277,35 +277,40 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
ReplaceRuntimeSettings {
|
||||
expected_revision: gateway_admin::model::Revision::new(1).unwrap(),
|
||||
request_profile_updates: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
model_mappings: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
max_concurrent_per_account: 1,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 0,
|
||||
refresh_concurrency: 1,
|
||||
max_concurrent_per_account: 1,
|
||||
request_interval_ms: 0,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 30,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -28,6 +28,27 @@ use super::{
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
/// 账号、代理与 Client Key 共用的分组引用
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountGroupRefView {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub color: String,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
impl From<gateway_admin::model::account_groups::AccountGroupRef> for AccountGroupRefView {
|
||||
fn from(group: gateway_admin::model::account_groups::AccountGroupRef) -> Self {
|
||||
Self {
|
||||
id: group.id.to_string(),
|
||||
name: group.name,
|
||||
color: group.color.as_str().to_owned(),
|
||||
enabled: group.enabled,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_PAGE_SIZE: u32 = 50;
|
||||
const MAX_PAGE_SIZE: u32 = 200;
|
||||
|
||||
@@ -420,16 +441,8 @@ fn group_color(value: &str) -> Result<AccountGroupColor, AdminError> {
|
||||
}
|
||||
|
||||
fn validate_group_fields(name: &str, description: Option<&str>) -> Result<(), AdminError> {
|
||||
if name.trim() != name
|
||||
|| name.is_empty()
|
||||
|| name.chars().count() > 100
|
||||
|| name.chars().any(char::is_control)
|
||||
|| description
|
||||
.is_some_and(|value| value.len() > 4096 || value.chars().any(char::is_control))
|
||||
{
|
||||
return Err(AdminError::bad_request("账号组请求不合法"));
|
||||
}
|
||||
Ok(())
|
||||
gateway_admin::model::account_groups::validate_group_fields(name, description)
|
||||
.map_err(|_| AdminError::bad_request("账号组请求不合法"))
|
||||
}
|
||||
|
||||
fn map_wire_error(_: WireValidationError) -> AdminError {
|
||||
|
||||
@@ -96,12 +96,7 @@ pub(super) fn account_view(
|
||||
groups: account
|
||||
.groups
|
||||
.into_iter()
|
||||
.map(|group| AccountGroupRefView {
|
||||
id: group.id.to_string(),
|
||||
name: group.name,
|
||||
color: group.color.as_str().to_owned(),
|
||||
enabled: group.enabled,
|
||||
})
|
||||
.map(AccountGroupRefView::from)
|
||||
.collect(),
|
||||
resource_ref: account.id,
|
||||
email: account.email,
|
||||
|
||||
@@ -378,14 +378,7 @@ pub struct QuotaForecastSourceView {
|
||||
pub usd_display: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountGroupRefView {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub color: String,
|
||||
pub enabled: bool,
|
||||
}
|
||||
pub use crate::admin::account_groups::AccountGroupRefView;
|
||||
|
||||
/// Provider quota 安全视图
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
|
||||
@@ -29,6 +29,7 @@ use axum::{
|
||||
routing::{get, post},
|
||||
};
|
||||
|
||||
use super::account_groups::AccountGroupRefView;
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminJson, AdminQuery, AdminResponse,
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
@@ -342,7 +343,7 @@ pub struct ClientKeyView {
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
routing_scope: &'static str,
|
||||
groups: Vec<ClientKeyGroupView>,
|
||||
groups: Vec<AccountGroupRefView>,
|
||||
provider_kinds: Vec<String>,
|
||||
prefix: String,
|
||||
enabled: bool,
|
||||
@@ -365,15 +366,6 @@ pub struct ClientKeyView {
|
||||
last_used_at_full_display: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ClientKeyGroupView {
|
||||
id: String,
|
||||
name: String,
|
||||
color: String,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl From<(ClientKeyRecord, crate::time::TimePresenter)> for ClientKeyView {
|
||||
fn from((record, time): (ClientKeyRecord, crate::time::TimePresenter)) -> Self {
|
||||
let routing_scope = if record.groups.is_empty() {
|
||||
@@ -399,12 +391,7 @@ impl From<(ClientKeyRecord, crate::time::TimePresenter)> for ClientKeyView {
|
||||
groups: record
|
||||
.groups
|
||||
.into_iter()
|
||||
.map(|group| ClientKeyGroupView {
|
||||
id: group.id.to_string(),
|
||||
name: group.name,
|
||||
color: group.color.as_str().to_owned(),
|
||||
enabled: group.enabled,
|
||||
})
|
||||
.map(AccountGroupRefView::from)
|
||||
.collect(),
|
||||
provider_kinds: record
|
||||
.provider_kinds
|
||||
@@ -847,19 +834,13 @@ fn validate_required_text(value: &str, field: &'static str) -> Result<(), WireVa
|
||||
}
|
||||
|
||||
fn validate_group_ids(values: Vec<String>) -> Result<Vec<AccountGroupId>, WireValidationError> {
|
||||
if values.len() > 1000
|
||||
|| values
|
||||
.iter()
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.len()
|
||||
!= values.len()
|
||||
{
|
||||
return Err(WireValidationError::new("groupIds"));
|
||||
}
|
||||
values
|
||||
let groups = values
|
||||
.into_iter()
|
||||
.map(|value| AccountGroupId::new(value).map_err(|_| WireValidationError::new("groupIds")))
|
||||
.collect()
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
gateway_admin::model::client_keys::validate_group_ids(&groups)
|
||||
.map_err(|_| WireValidationError::new("groupIds"))?;
|
||||
Ok(groups)
|
||||
}
|
||||
|
||||
fn validate_optional_text(
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
//! 管理请求在 HTTP 出口记录一次受控详情,响应只保留安全投影
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
extract::{Request, State},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure};
|
||||
use tower_http::request_id::RequestId;
|
||||
|
||||
pub(crate) async fn record_failure(
|
||||
State(diagnostics): State<Arc<dyn OperationalDiagnostics>>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
let correlation_id = request
|
||||
.extensions()
|
||||
.get::<RequestId>()
|
||||
.and_then(|id| id.header_value().to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let mut response = next.run(request).await;
|
||||
if let Some(error) = response
|
||||
.extensions_mut()
|
||||
.remove::<gateway_admin::model::AdminError>()
|
||||
&& let Some(details) = error.error_details()
|
||||
{
|
||||
let mut failure = OperationalFailure::new(
|
||||
"admin",
|
||||
"http_request",
|
||||
error.kind().as_str(),
|
||||
error.message(),
|
||||
);
|
||||
failure.correlation_id = correlation_id;
|
||||
failure.details = Some(details);
|
||||
if diagnostics.record_failure(failure).await.is_err() {
|
||||
tracing::warn!("admin diagnostic could not be recorded");
|
||||
}
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -15,6 +15,7 @@ pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backups;
|
||||
pub mod client_keys;
|
||||
pub(crate) mod diagnostics;
|
||||
mod extract;
|
||||
pub mod observability;
|
||||
mod plugins;
|
||||
|
||||
@@ -4,7 +4,9 @@ use std::collections::BTreeMap;
|
||||
|
||||
use axum::{Router, extract::State, http::StatusCode, response::IntoResponse, routing::get};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
|
||||
use gateway_admin::model::observability::{
|
||||
self as domain, ObservabilityPageSize as DomainPageSize,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
|
||||
@@ -617,7 +617,7 @@ pub(crate) fn ops_error_view(
|
||||
subagent_kind: error.subagent_kind,
|
||||
compact: error.compact,
|
||||
message: error.message,
|
||||
raw_upstream_error: error.raw_upstream_error,
|
||||
error_details: error.error_details,
|
||||
metadata: OpsErrorMetadataView {
|
||||
source: error.source,
|
||||
component: error.component,
|
||||
|
||||
@@ -5,7 +5,7 @@ use super::*;
|
||||
/// 观测列表默认页大小
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小
|
||||
pub const MAX_PAGE_SIZE: u16 = 100;
|
||||
pub const MAX_PAGE_SIZE: u16 = DomainPageSize::MAX;
|
||||
|
||||
/// Dashboard 查询参数
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
@@ -59,7 +59,7 @@ impl UsageQuery {
|
||||
return Err(WireValidationError::new("currentPage"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
if DomainPageSize::new(page_size).is_err() {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok((current_page, page_size))
|
||||
@@ -136,7 +136,7 @@ impl OpsQuery {
|
||||
return Err(WireValidationError::new("currentPage"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
if DomainPageSize::new(page_size).is_err() {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok((current_page, page_size))
|
||||
|
||||
@@ -814,8 +814,8 @@ pub struct OpsErrorView {
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub message: String,
|
||||
/// 上游错误正文或 WebSocket close/error frame 原文;不做脱敏
|
||||
pub raw_upstream_error: Option<String>,
|
||||
/// 受控错误详情,分别承载本地原因链与上游错误正文
|
||||
pub error_details: Option<String>,
|
||||
pub metadata: OpsErrorMetadataView,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub created_at_display: String,
|
||||
|
||||
@@ -324,12 +324,7 @@ where
|
||||
groups: account
|
||||
.groups
|
||||
.into_iter()
|
||||
.map(|group| AccountGroupRefView {
|
||||
id: group.id.to_string(),
|
||||
name: group.name,
|
||||
color: group.color.as_str().to_owned(),
|
||||
enabled: group.enabled,
|
||||
})
|
||||
.map(AccountGroupRefView::from)
|
||||
.collect(),
|
||||
enabled: account.enabled,
|
||||
})
|
||||
|
||||
@@ -18,7 +18,6 @@ use gateway_admin::model::client_distribution::{
|
||||
use gateway_admin::model::settings::{
|
||||
ModelMappings as DomainModelMappings, ReplaceRuntimeSettings, RotationStrategy, RuntimeSettings,
|
||||
};
|
||||
use gateway_core::policy::CodexClientVersion;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -33,20 +32,12 @@ pub type ProviderRequestProfiles = BTreeMap<String, serde_json::Map<String, serd
|
||||
pub type ProviderRequestProfileUpdates =
|
||||
BTreeMap<String, Option<serde_json::Map<String, serde_json::Value>>>;
|
||||
|
||||
/// 运行配置投影与设置页字段的聚合响应
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
/// HTTP 读写共用的设置字段;仅保留 wire 数值类型与命名
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RuntimeSettingsView {
|
||||
pub config_revision: u64,
|
||||
pub smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig,
|
||||
pub provider_request_profiles: ProviderRequestProfiles,
|
||||
/// 固定兼容字段;值始终从 provider_request_profiles 派生
|
||||
pub openai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
/// 固定兼容字段;值始终从 provider_request_profiles 派生
|
||||
pub xai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
pub struct RuntimeSettingsFields {
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
pub max_concurrent_per_account: u64,
|
||||
@@ -55,9 +46,11 @@ pub struct RuntimeSettingsView {
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub openai_account_affinity: gateway_core::account::AccountAffinity,
|
||||
pub max_account_rotations: u32,
|
||||
pub openai_session_affinity_ttl_hours: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub rotation_strategy: String,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u64,
|
||||
@@ -73,6 +66,23 @@ pub struct RuntimeSettingsView {
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
}
|
||||
|
||||
/// 运行配置投影与设置页字段的聚合响应
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RuntimeSettingsView {
|
||||
#[serde(flatten)]
|
||||
pub values: RuntimeSettingsFields,
|
||||
pub config_revision: u64,
|
||||
pub smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig,
|
||||
pub provider_request_profiles: ProviderRequestProfiles,
|
||||
/// 固定兼容字段;值始终从 provider_request_profiles 派生
|
||||
pub openai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
/// 固定兼容字段;值始终从 provider_request_profiles 派生
|
||||
pub xai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub rotation_strategy: String,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
pub updated_at_display: String,
|
||||
}
|
||||
@@ -81,6 +91,8 @@ pub struct RuntimeSettingsView {
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateRuntimeSettingsRequest {
|
||||
#[serde(flatten)]
|
||||
pub values: RuntimeSettingsFields,
|
||||
pub config_revision: u64,
|
||||
#[serde(default)]
|
||||
pub provider_request_profiles: ProviderRequestProfileUpdates,
|
||||
@@ -90,128 +102,22 @@ pub struct UpdateRuntimeSettingsRequest {
|
||||
/// 兼容既有 wire;与泛化字段冲突时拒绝整个请求
|
||||
#[serde(default, deserialize_with = "deserialize_profile_update")]
|
||||
pub xai_client_profile: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
pub request_location_enabled: bool,
|
||||
pub request_location: gateway_core::account::RequestLocation,
|
||||
pub model_mappings: ModelMappings,
|
||||
pub refresh_margin_seconds: u64,
|
||||
pub refresh_concurrency: u64,
|
||||
pub max_concurrent_per_account: u64,
|
||||
pub request_interval_ms: u64,
|
||||
pub max_waiting_per_key: u32,
|
||||
pub max_waiting_per_account: u32,
|
||||
pub concurrency_wait_timeout_seconds: u32,
|
||||
pub openai_guardian_reserved_concurrency: u32,
|
||||
pub responses_max_decompressed_body_bytes: u64,
|
||||
pub smart_scheduling: gateway_core::account::SmartSchedulingConfig,
|
||||
pub rotation_strategy: String,
|
||||
pub min_codex_desktop_version: Option<String>,
|
||||
pub min_codex_cli_version: Option<String>,
|
||||
pub usage_retention_days: u64,
|
||||
pub ops_event_retention_days: u64,
|
||||
pub audit_retention_days: u64,
|
||||
pub account_auto_freeze_enabled: bool,
|
||||
pub account_auto_freeze_threshold: u64,
|
||||
pub account_auto_freeze_window_seconds: u64,
|
||||
pub account_auto_freeze_duration_seconds: u64,
|
||||
pub account_auto_freeze_probe_enabled: bool,
|
||||
pub account_auto_freeze_probe_model: Option<String>,
|
||||
pub account_auto_freeze_adaptive_concurrency: bool,
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
}
|
||||
|
||||
impl UpdateRuntimeSettingsRequest {
|
||||
/// 校验公共运行参数
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
self.request_location
|
||||
.validate()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?;
|
||||
validate_model_mappings(&self.model_mappings)?;
|
||||
for (value, field) in [
|
||||
(self.max_waiting_per_key, "maxWaitingPerKey"),
|
||||
(self.max_waiting_per_account, "maxWaitingPerAccount"),
|
||||
] {
|
||||
if value > 1_000 {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
}
|
||||
if self.responses_max_decompressed_body_bytes == 0
|
||||
|| isize::try_from(self.responses_max_decompressed_body_bytes).is_err()
|
||||
{
|
||||
return Err(WireValidationError::new(
|
||||
"responsesMaxDecompressedBodyBytes",
|
||||
));
|
||||
}
|
||||
if !(1..=120).contains(&self.concurrency_wait_timeout_seconds) {
|
||||
return Err(WireValidationError::new("concurrencyWaitTimeoutSeconds"));
|
||||
}
|
||||
for (value, field) in [
|
||||
(self.refresh_margin_seconds, "refreshMarginSeconds"),
|
||||
(self.refresh_concurrency, "refreshConcurrency"),
|
||||
(self.usage_retention_days, "usageRetentionDays"),
|
||||
(self.ops_event_retention_days, "opsEventRetentionDays"),
|
||||
(self.audit_retention_days, "auditRetentionDays"),
|
||||
] {
|
||||
require_positive_i64(value, field)?;
|
||||
}
|
||||
if i64::try_from(self.request_interval_ms).is_err() {
|
||||
return Err(WireValidationError::new("requestIntervalMs"));
|
||||
}
|
||||
if RotationStrategy::parse(&self.rotation_strategy).is_none() {
|
||||
return Err(WireValidationError::new("rotationStrategy"));
|
||||
}
|
||||
validate_optional_client_version(
|
||||
self.min_codex_desktop_version.as_deref(),
|
||||
"minCodexDesktopVersion",
|
||||
)?;
|
||||
validate_optional_client_version(
|
||||
self.min_codex_cli_version.as_deref(),
|
||||
"minCodexCliVersion",
|
||||
)?;
|
||||
for (value, field) in [
|
||||
(
|
||||
self.account_auto_freeze_threshold,
|
||||
"accountAutoFreezeThreshold",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_window_seconds,
|
||||
"accountAutoFreezeWindowSeconds",
|
||||
),
|
||||
(
|
||||
self.account_auto_freeze_duration_seconds,
|
||||
"accountAutoFreezeDurationSeconds",
|
||||
),
|
||||
] {
|
||||
require_positive_i64(value, field)?;
|
||||
}
|
||||
if !(2..=1_000).contains(&self.account_auto_freeze_threshold) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeThreshold"));
|
||||
}
|
||||
if !(60..=3_600).contains(&self.account_auto_freeze_window_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeWindowSeconds"));
|
||||
}
|
||||
if !(300..=604_800).contains(&self.account_auto_freeze_duration_seconds) {
|
||||
return Err(WireValidationError::new("accountAutoFreezeDurationSeconds"));
|
||||
}
|
||||
validate_optional_probe_model(
|
||||
self.account_auto_freeze_probe_model.as_deref(),
|
||||
"accountAutoFreezeProbeModel",
|
||||
)?;
|
||||
if !gateway_core::provider_ports::valid_warmup_schedule_time(
|
||||
&self.account_warmup_schedule_time,
|
||||
) {
|
||||
return Err(WireValidationError::new("accountWarmupScheduleTime"));
|
||||
}
|
||||
validate_optional_probe_model(self.account_warmup_model.as_deref(), "accountWarmupModel")?;
|
||||
if self.account_warmup_enabled && self.account_warmup_model.is_none() {
|
||||
return Err(WireValidationError::new("accountWarmupModel"));
|
||||
}
|
||||
Ok(())
|
||||
self.values.clone().into_values().map(|_| ())
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<ReplaceRuntimeSettings, WireValidationError> {
|
||||
self.validate()?;
|
||||
validate_model_mappings(&self.model_mappings)?;
|
||||
let request_profile_updates = normalize_request_profile_updates(
|
||||
self.provider_request_profiles,
|
||||
self.openai_client_profile,
|
||||
@@ -221,45 +127,10 @@ impl UpdateRuntimeSettingsRequest {
|
||||
expected_revision: gateway_admin::model::Revision::new(self.config_revision)
|
||||
.map_err(|_| WireValidationError::new("configRevision"))?,
|
||||
request_profile_updates,
|
||||
request_location_enabled: self.request_location_enabled,
|
||||
request_location: self
|
||||
.request_location
|
||||
.normalized()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?,
|
||||
model_mappings: domain_model_mappings(self.model_mappings)?,
|
||||
refresh_margin_seconds: self.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(self.refresh_concurrency)
|
||||
.map_err(|_| WireValidationError::new("settingsRefreshConcurrencyOverflow"))?,
|
||||
max_concurrent_per_account: u32::try_from(self.max_concurrent_per_account)
|
||||
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
|
||||
request_interval_ms: self.request_interval_ms,
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
|
||||
responses_max_decompressed_body_bytes: self.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: self.smart_scheduling,
|
||||
rotation_strategy: RotationStrategy::parse(&self.rotation_strategy)
|
||||
.ok_or_else(|| WireValidationError::new("rotationStrategy"))?,
|
||||
min_codex_desktop_version: self.min_codex_desktop_version,
|
||||
min_codex_cli_version: self.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsUsageRetentionOverflow"))?,
|
||||
ops_event_retention_days: u32::try_from(self.ops_event_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?,
|
||||
audit_retention_days: u32::try_from(self.audit_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
|
||||
account_auto_freeze_enabled: self.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold)
|
||||
.map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?,
|
||||
account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: self.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency,
|
||||
account_warmup_enabled: self.account_warmup_enabled,
|
||||
account_warmup_schedule_time: self.account_warmup_schedule_time,
|
||||
account_warmup_model: self.account_warmup_model,
|
||||
values: self.values.into_values()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -276,39 +147,12 @@ impl From<(RuntimeSettings, crate::time::TimePresenter)> for RuntimeSettingsView
|
||||
openai_client_profile: provider_request_profiles.get("openai").cloned(),
|
||||
xai_client_profile: provider_request_profiles.get("xai").cloned(),
|
||||
provider_request_profiles,
|
||||
request_location_enabled: settings.request_location_enabled,
|
||||
request_location: settings.request_location,
|
||||
model_mappings: wire_model_mappings(settings.model_mappings),
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: settings.smart_scheduling,
|
||||
smart_scheduling_defaults: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold),
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
account_warmup_enabled: settings.account_warmup_enabled,
|
||||
account_warmup_schedule_time: settings.account_warmup_schedule_time,
|
||||
account_warmup_model: settings.account_warmup_model,
|
||||
updated_at_display: time.datetime(&settings.updated_at),
|
||||
updated_at: settings.updated_at,
|
||||
values: settings.values.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -699,23 +543,13 @@ where
|
||||
))
|
||||
}
|
||||
|
||||
fn require_positive_i64(value: u64, field: &'static str) -> Result<(), WireValidationError> {
|
||||
if value == 0 || i64::try_from(value).is_err() {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_model_mappings(mappings: &ModelMappings) -> Result<(), WireValidationError> {
|
||||
if mappings.len() > 512 {
|
||||
return Err(WireValidationError::new("modelMappings"));
|
||||
}
|
||||
for (requested, upstream) in mappings {
|
||||
if !valid_model_name(requested, 256) || !valid_model_name(upstream, 256) {
|
||||
return Err(WireValidationError::new("modelMappings"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
gateway_admin::model::settings::validate_model_mappings(
|
||||
mappings
|
||||
.iter()
|
||||
.map(|(requested, upstream)| (requested.as_str(), upstream.as_str())),
|
||||
)
|
||||
.map_err(|_| WireValidationError::new("modelMappings"))
|
||||
}
|
||||
|
||||
fn domain_model_mappings(
|
||||
@@ -741,38 +575,6 @@ fn wire_model_mappings(mappings: DomainModelMappings) -> ModelMappings {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= max_len
|
||||
&& !value.bytes().any(|byte| byte.is_ascii_control())
|
||||
}
|
||||
|
||||
fn validate_optional_client_version(
|
||||
value: Option<&str>,
|
||||
field: &'static str,
|
||||
) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| CodexClientVersion::parse(value).is_err()) {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 探测模型为可选自由文本:非空、去首尾空白后不变、无控制字符且不超过 128 字节
|
||||
fn validate_optional_probe_model(
|
||||
value: Option<&str>,
|
||||
field: &'static str,
|
||||
) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| {
|
||||
value.is_empty()
|
||||
|| value.len() > 128
|
||||
|| value != value.trim()
|
||||
|| value.bytes().any(|byte| byte.is_ascii_control())
|
||||
}) {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(),
|
||||
@@ -911,3 +713,127 @@ fn client_profile_preview_view(
|
||||
}
|
||||
profile
|
||||
}
|
||||
|
||||
impl RuntimeSettingsFields {
|
||||
fn into_values(
|
||||
self,
|
||||
) -> Result<gateway_admin::model::settings::RuntimeSettingsValues, WireValidationError> {
|
||||
let values = gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: self.request_location_enabled,
|
||||
request_location: self
|
||||
.request_location
|
||||
.normalized()
|
||||
.map_err(|_| WireValidationError::new("requestLocation"))?,
|
||||
refresh_margin_seconds: self.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(self.refresh_concurrency)
|
||||
.map_err(|_| WireValidationError::new("settingsRefreshConcurrencyOverflow"))?,
|
||||
max_concurrent_per_account: u32::try_from(self.max_concurrent_per_account)
|
||||
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
|
||||
request_interval_ms: self.request_interval_ms,
|
||||
max_waiting_per_key: self.max_waiting_per_key,
|
||||
max_waiting_per_account: self.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: self.openai_account_affinity,
|
||||
max_account_rotations: self.max_account_rotations,
|
||||
openai_session_affinity_ttl_hours: self.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: self.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: self.smart_scheduling,
|
||||
min_codex_desktop_version: self.min_codex_desktop_version,
|
||||
min_codex_cli_version: self.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsUsageRetentionOverflow"))?,
|
||||
ops_event_retention_days: u32::try_from(self.ops_event_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsOpsRetentionOverflow"))?,
|
||||
audit_retention_days: u32::try_from(self.audit_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
|
||||
account_auto_freeze_enabled: self.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u32::try_from(self.account_auto_freeze_threshold)
|
||||
.map_err(|_| WireValidationError::new("settingsFreezeThresholdOverflow"))?,
|
||||
account_auto_freeze_window_seconds: self.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: self.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: self.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: self.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: self.account_auto_freeze_adaptive_concurrency,
|
||||
account_warmup_enabled: self.account_warmup_enabled,
|
||||
account_warmup_schedule_time: self.account_warmup_schedule_time,
|
||||
account_warmup_model: self.account_warmup_model,
|
||||
};
|
||||
values.validate().map_err(settings_validation_error)?;
|
||||
Ok(values)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSettingsFields {
|
||||
fn from(settings: gateway_admin::model::settings::RuntimeSettingsValues) -> Self {
|
||||
crate::admin::settings::RuntimeSettingsFields {
|
||||
request_location_enabled: settings.request_location_enabled,
|
||||
request_location: settings.request_location,
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
max_waiting_per_key: settings.max_waiting_per_key,
|
||||
max_waiting_per_account: settings.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: settings.openai_account_affinity,
|
||||
max_account_rotations: settings.max_account_rotations,
|
||||
openai_session_affinity_ttl_hours: settings.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: settings.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: settings.smart_scheduling,
|
||||
min_codex_desktop_version: settings.min_codex_desktop_version,
|
||||
min_codex_cli_version: settings.min_codex_cli_version,
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
account_auto_freeze_enabled: settings.account_auto_freeze_enabled,
|
||||
account_auto_freeze_threshold: u64::from(settings.account_auto_freeze_threshold),
|
||||
account_auto_freeze_window_seconds: settings.account_auto_freeze_window_seconds,
|
||||
account_auto_freeze_duration_seconds: settings.account_auto_freeze_duration_seconds,
|
||||
account_auto_freeze_probe_enabled: settings.account_auto_freeze_probe_enabled,
|
||||
account_auto_freeze_probe_model: settings.account_auto_freeze_probe_model,
|
||||
account_auto_freeze_adaptive_concurrency: settings
|
||||
.account_auto_freeze_adaptive_concurrency,
|
||||
account_warmup_enabled: settings.account_warmup_enabled,
|
||||
account_warmup_schedule_time: settings.account_warmup_schedule_time,
|
||||
account_warmup_model: settings.account_warmup_model,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn settings_validation_error(field: &'static str) -> WireValidationError {
|
||||
WireValidationError::new(match field {
|
||||
"account_auto_freeze_adaptive_concurrency" => "accountAutoFreezeAdaptiveConcurrency",
|
||||
"account_auto_freeze_duration_seconds" => "accountAutoFreezeDurationSeconds",
|
||||
"account_auto_freeze_enabled" => "accountAutoFreezeEnabled",
|
||||
"account_auto_freeze_probe_enabled" => "accountAutoFreezeProbeEnabled",
|
||||
"account_auto_freeze_probe_model" => "accountAutoFreezeProbeModel",
|
||||
"account_auto_freeze_threshold" => "accountAutoFreezeThreshold",
|
||||
"account_auto_freeze_window_seconds" => "accountAutoFreezeWindowSeconds",
|
||||
"account_warmup_enabled" => "accountWarmupEnabled",
|
||||
"account_warmup_model" => "accountWarmupModel",
|
||||
"account_warmup_schedule_time" => "accountWarmupScheduleTime",
|
||||
"audit_retention_days" => "auditRetentionDays",
|
||||
"concurrency_wait_timeout_seconds" => "concurrencyWaitTimeoutSeconds",
|
||||
"max_account_rotations" => "maxAccountRotations",
|
||||
"max_concurrent_per_account" => "maxConcurrentPerAccount",
|
||||
"max_waiting_per_account" => "maxWaitingPerAccount",
|
||||
"max_waiting_per_key" => "maxWaitingPerKey",
|
||||
"min_codex_cli_version" => "minCodexCliVersion",
|
||||
"min_codex_desktop_version" => "minCodexDesktopVersion",
|
||||
"openai_account_affinity" => "openaiAccountAffinity",
|
||||
"openai_guardian_reserved_concurrency" => "openaiGuardianReservedConcurrency",
|
||||
"openai_session_affinity_ttl_hours" => "openaiSessionAffinityTtlHours",
|
||||
"ops_event_retention_days" => "opsEventRetentionDays",
|
||||
"refresh_concurrency" => "refreshConcurrency",
|
||||
"refresh_margin_seconds" => "refreshMarginSeconds",
|
||||
"request_interval_ms" => "requestIntervalMs",
|
||||
"request_location" => "requestLocation",
|
||||
"request_location_enabled" => "requestLocationEnabled",
|
||||
"responses_max_decompressed_body_bytes" => "responsesMaxDecompressedBodyBytes",
|
||||
"smart_scheduling" => "smartScheduling",
|
||||
"usage_retention_days" => "usageRetentionDays",
|
||||
_ => "settings",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -170,6 +170,7 @@ pub struct AdminError {
|
||||
status: StatusCode,
|
||||
body: AdminErrorBody,
|
||||
retry_after_seconds: Option<u64>,
|
||||
diagnostic: Option<gateway_admin::model::AdminError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -266,6 +267,7 @@ impl AdminError {
|
||||
status,
|
||||
body: AdminErrorBody::new(code, message),
|
||||
retry_after_seconds: None,
|
||||
diagnostic: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -380,12 +382,16 @@ pub(crate) fn map_admin_service_error(error: gateway_admin::model::AdminError) -
|
||||
{
|
||||
response.body.message = error.message().to_owned();
|
||||
}
|
||||
response.diagnostic = Some(error);
|
||||
response
|
||||
}
|
||||
|
||||
impl IntoResponse for AdminError {
|
||||
fn into_response(self) -> Response {
|
||||
let mut response = (self.status, Json(self.body)).into_response();
|
||||
if let Some(error) = self.diagnostic {
|
||||
response.extensions_mut().insert(error);
|
||||
}
|
||||
if let Some(seconds) = self.retry_after_seconds
|
||||
&& let Ok(value) = axum::http::HeaderValue::from_str(&seconds.to_string())
|
||||
{
|
||||
|
||||
@@ -114,6 +114,7 @@ where
|
||||
Router::new()
|
||||
.route("/api/auth/login", post(login::<S>))
|
||||
.route("/api/auth/status", get(session_status::<S>))
|
||||
.route("/api/auth/refresh", post(refresh_session::<S>))
|
||||
.route("/api/auth/logout", post(logout::<S>))
|
||||
.route("/api/auth/password", post(change_password::<S>))
|
||||
.route("/api/auth", any(not_found))
|
||||
@@ -141,32 +142,16 @@ where
|
||||
)
|
||||
.await
|
||||
.map_err(map_login_error)?;
|
||||
let max_age = result
|
||||
.session
|
||||
.expires_at
|
||||
.signed_duration_since(chrono::Utc::now())
|
||||
.num_seconds()
|
||||
.max(1);
|
||||
let expires = result
|
||||
.session
|
||||
.expires_at
|
||||
.format("%a, %d %b %Y %H:%M:%S GMT");
|
||||
let cookie = format!(
|
||||
"{}={}; {}; Max-Age={max_age}; Expires={expires}",
|
||||
session_cookie::NAME,
|
||||
result.session_id,
|
||||
session_cookie::attributes(&headers)
|
||||
);
|
||||
let mut response = AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(SessionData::from(&result.session)),
|
||||
set_session_cookie(
|
||||
AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(SessionData::from(&result.session)),
|
||||
)
|
||||
.into_response(),
|
||||
&headers,
|
||||
&result.session_id,
|
||||
&result.session,
|
||||
)
|
||||
.into_response();
|
||||
response.headers_mut().insert(
|
||||
header::SET_COOKIE,
|
||||
HeaderValue::from_str(&cookie).map_err(|_| AdminError::internal())?,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn session_status<S>(
|
||||
@@ -182,13 +167,64 @@ where
|
||||
.session(session_cookie::value(&headers).as_deref())
|
||||
.await
|
||||
.map_err(map_admin_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
Ok(session_response(session.as_ref()))
|
||||
}
|
||||
|
||||
async fn refresh_session<S>(
|
||||
State(state): State<S>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Response, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let session_id = session_cookie::value(&headers);
|
||||
let session = state
|
||||
.admin_services()
|
||||
.auth()
|
||||
.renew_session(session_id.as_deref())
|
||||
.await
|
||||
.map_err(map_admin_service_error)?;
|
||||
let response = session_response(session.as_ref());
|
||||
if let (Some(session_id), Some(session)) = (session_id, session) {
|
||||
return set_session_cookie(response, &headers, &session_id, &session);
|
||||
}
|
||||
// 失效响应不清 Cookie,避免晚到的请求覆盖另一个标签页刚建立的会话
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn session_response(session: Option<&AuthSession>) -> Response {
|
||||
AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(SessionStatusData {
|
||||
authenticated: session.is_some(),
|
||||
session: session.as_ref().map(SessionData::from),
|
||||
session: session.map(SessionData::from),
|
||||
}),
|
||||
))
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn set_session_cookie(
|
||||
mut response: Response,
|
||||
headers: &HeaderMap,
|
||||
session_id: &str,
|
||||
session: &AuthSession,
|
||||
) -> Result<Response, AdminError> {
|
||||
let max_age = session
|
||||
.expires_at
|
||||
.signed_duration_since(chrono::Utc::now())
|
||||
.num_seconds()
|
||||
.max(1);
|
||||
let expires = session.expires_at.format("%a, %d %b %Y %H:%M:%S GMT");
|
||||
let cookie = format!(
|
||||
"{}={session_id}; {}; Max-Age={max_age}; Expires={expires}",
|
||||
session_cookie::NAME,
|
||||
session_cookie::attributes(headers)
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
header::SET_COOKIE,
|
||||
HeaderValue::from_str(&cookie).map_err(|_| AdminError::internal())?,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn logout<S>(State(state): State<S>, headers: HeaderMap) -> Result<Response, AdminError>
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
use axum::http::StatusCode;
|
||||
use chrono::Duration;
|
||||
use gateway_admin::model::{
|
||||
PageSize,
|
||||
key_usage::{KeyUsageQuery, KeyUsageRecordKind, KeyUsageRecordsQuery},
|
||||
observability::ObservabilityPageSize,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
@@ -89,13 +89,14 @@ impl RecordsQuery {
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> Result<KeyUsageRecordsQuery, AdminError> {
|
||||
let current_page = self.current_page.unwrap_or(1);
|
||||
let page_size = self.page_size.unwrap_or(20);
|
||||
if current_page == 0 || !(1..=100).contains(&page_size) {
|
||||
return Err(AdminError::invalid_request(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"页码必须大于 0,每页数量为 1–100",
|
||||
));
|
||||
let invalid_page = || {
|
||||
AdminError::invalid_request(StatusCode::BAD_REQUEST, "页码必须大于 0,每页数量为 1–100")
|
||||
};
|
||||
if current_page == 0 {
|
||||
return Err(invalid_page());
|
||||
}
|
||||
let page_size =
|
||||
ObservabilityPageSize::new(self.page_size.unwrap_or(20)).map_err(|_| invalid_page())?;
|
||||
Ok(KeyUsageRecordsQuery {
|
||||
usage: OverviewQuery {
|
||||
period: self.period,
|
||||
@@ -110,9 +111,7 @@ impl RecordsQuery {
|
||||
RecordKind::Error => KeyUsageRecordKind::Error,
|
||||
},
|
||||
current_page,
|
||||
page_size: PageSize::new(page_size).map_err(|_| {
|
||||
AdminError::invalid_request(StatusCode::BAD_REQUEST, "每页数量不合法")
|
||||
})?,
|
||||
page_size,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,6 +170,7 @@ pub fn initialize(
|
||||
probes: Vec<Arc<dyn HealthProbe>>,
|
||||
worker_health: Arc<dyn WorkerHealthSource>,
|
||||
lifecycle: Arc<dyn ConnectionLifecycle>,
|
||||
diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
|
||||
) -> Result<ApiBundle, ApiError> {
|
||||
// 配置加载已解析环境变量与相对路径,初始化只校验,避免覆盖最终目录
|
||||
config.validate().map_err(ApiError::Config)?;
|
||||
@@ -256,6 +257,10 @@ pub fn initialize(
|
||||
headers_ms = latency.as_millis(), "HTTP response headers ready");
|
||||
});
|
||||
let router = router
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
diagnostics,
|
||||
admin::diagnostics::record_failure,
|
||||
))
|
||||
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
|
||||
.layer(trace_layer)
|
||||
.layer(SetRequestIdLayer::new(
|
||||
|
||||
@@ -165,8 +165,11 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
"no upstream provider is currently available for this request",
|
||||
)
|
||||
}
|
||||
EngineError::Store(_)
|
||||
| EngineError::ProviderMetadataMismatch
|
||||
EngineError::Store(source) => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
|
||||
.with_source(source.clone())
|
||||
}
|
||||
EngineError::ProviderMetadataMismatch
|
||||
| EngineError::ContinuationPinMismatch
|
||||
| EngineError::RequiredAccountMismatch
|
||||
| EngineError::AccountOutsideClientScope
|
||||
|
||||
@@ -134,6 +134,8 @@ pub(crate) async fn execute_prepared_responses(
|
||||
ClientTransport::HttpJson
|
||||
};
|
||||
let model_hint = Some(decoded.metadata().requested_model().to_owned());
|
||||
// 终端会重新校验中间件改写后的正文,初步解码不必保留到响应交付
|
||||
drop(decoded);
|
||||
let execution = service.execution();
|
||||
let request_id = prepared.request_id().clone();
|
||||
let cancellation = prepared.cancellation();
|
||||
|
||||
@@ -214,8 +214,9 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
}
|
||||
request_count = request_count.saturating_add(1);
|
||||
let correlation_id = Arc::<str>::from(service.next_request_id());
|
||||
let decoded = match decode_response_create_with_context(&payload, &request_headers) {
|
||||
Ok(decoded) => decoded,
|
||||
// 初步解码只保留路由事实,避免整份正文跨越准入等待和响应交付
|
||||
let model_hint = match decode_response_create_with_context(&payload, &request_headers) {
|
||||
Ok(decoded) => Some(decoded.metadata().requested_model().to_owned()),
|
||||
Err(error) => {
|
||||
trace_rejected_request(
|
||||
&correlation_id,
|
||||
@@ -298,9 +299,9 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
protocol: "openai".to_owned(),
|
||||
operation: Some(OperationKind::Generate),
|
||||
transport: ClientTransport::WebSocket,
|
||||
model_hint: Some(decoded.metadata().requested_model().to_owned()),
|
||||
model_hint,
|
||||
headers: encode_headers(&raw_headers),
|
||||
body: Bytes::from(payload.clone()),
|
||||
body: Bytes::from(payload),
|
||||
};
|
||||
let service_for_terminal = service.clone();
|
||||
let replay_for_terminal = replay.clone();
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! 验证管理端来源经用例和 HTTP 出口到达受控记录,公开响应和扩展不泄漏
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use tower::ServiceExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_failure_is_recorded_once_with_the_incoming_request_id() {
|
||||
let fixture = super::AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let diagnostics = Arc::new(crate::support::RecordingDiagnostics::default());
|
||||
let bundle = crate::openai::api_bundle_with_diagnostics(
|
||||
fixture.services,
|
||||
gateway_api::ApiConfig {
|
||||
asset_directory: std::env::temp_dir(),
|
||||
request_timeout_seconds: None,
|
||||
cors_allowed_origins: Vec::new(),
|
||||
request_id_header: "x-request-id".to_owned(),
|
||||
},
|
||||
diagnostics.clone(),
|
||||
);
|
||||
let response = bundle
|
||||
.router()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/accounts")
|
||||
.header("cookie", "cpr_session=valid-session")
|
||||
.header("x-request-id", "admin-native-cause-test")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(
|
||||
response
|
||||
.extensions()
|
||||
.get::<gateway_admin::model::AdminError>()
|
||||
.is_none()
|
||||
);
|
||||
let bytes = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
assert!(!String::from_utf8_lossy(&bytes).contains("PRIVATE_ADMIN_NATIVE_CAUSE"));
|
||||
let failures = diagnostics.0.lock().unwrap();
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert_eq!(failures[0].kind, "unavailable");
|
||||
assert_eq!(
|
||||
failures[0].correlation_id.as_deref(),
|
||||
Some("admin-native-cause-test")
|
||||
);
|
||||
assert!(
|
||||
failures[0]
|
||||
.details
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.contains("PRIVATE_ADMIN_NATIVE_CAUSE")
|
||||
);
|
||||
}
|
||||
@@ -86,6 +86,7 @@ mod account_groups;
|
||||
mod accounts;
|
||||
mod auth;
|
||||
mod client_keys;
|
||||
mod error_diagnostics;
|
||||
mod errors;
|
||||
mod observability;
|
||||
mod proxies;
|
||||
@@ -185,6 +186,7 @@ impl AdminTestFixture {
|
||||
];
|
||||
let bundle = gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
session_absolute_ttl_minutes: 30 * 24 * 60,
|
||||
session_ttl_minutes: 60,
|
||||
default_username: "admin_1".to_owned(),
|
||||
default_password: InitialAdminPassword::new("strong-admin-password"),
|
||||
@@ -313,6 +315,7 @@ impl MemoryAuthStore {
|
||||
},
|
||||
admin_user_id: "admin_1".to_owned(),
|
||||
},
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + Duration::hours(1),
|
||||
},
|
||||
);
|
||||
@@ -326,6 +329,15 @@ impl MemoryAuthStore {
|
||||
self.fail_audit.store(fail, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn set_session_expiry(&self, session_id: &str, expires_at: chrono::DateTime<Utc>) {
|
||||
self.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(session_id)
|
||||
.unwrap()
|
||||
.expires_at = expires_at;
|
||||
}
|
||||
|
||||
pub fn session_count(&self) -> usize {
|
||||
self.sessions.lock().expect("sessions").len()
|
||||
}
|
||||
@@ -397,6 +409,25 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AuthSession>> {
|
||||
Ok(self.sessions.lock().expect("sessions").remove(session_id))
|
||||
}
|
||||
@@ -562,37 +593,46 @@ impl SettingsStore for MemorySettingsStore {
|
||||
}
|
||||
let updated = RuntimeSettings {
|
||||
request_profiles,
|
||||
request_location_enabled: command.request_location_enabled,
|
||||
request_location: command.request_location,
|
||||
config_revision: next_revision(settings.config_revision),
|
||||
model_mappings: command.model_mappings,
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
refresh_concurrency: command.refresh_concurrency,
|
||||
max_concurrent_per_account: command.max_concurrent_per_account,
|
||||
request_interval_ms: command.request_interval_ms,
|
||||
max_waiting_per_key: command.max_waiting_per_key,
|
||||
max_waiting_per_account: command.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: command.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: command.openai_guardian_reserved_concurrency,
|
||||
responses_max_decompressed_body_bytes: command.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: command.smart_scheduling,
|
||||
rotation_strategy: command.rotation_strategy,
|
||||
min_codex_desktop_version: command.min_codex_desktop_version,
|
||||
min_codex_cli_version: command.min_codex_cli_version,
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: command.values.request_location_enabled,
|
||||
request_location: command.values.request_location,
|
||||
refresh_margin_seconds: command.values.refresh_margin_seconds,
|
||||
refresh_concurrency: command.values.refresh_concurrency,
|
||||
max_concurrent_per_account: command.values.max_concurrent_per_account,
|
||||
request_interval_ms: command.values.request_interval_ms,
|
||||
max_waiting_per_key: command.values.max_waiting_per_key,
|
||||
max_waiting_per_account: command.values.max_waiting_per_account,
|
||||
concurrency_wait_timeout_seconds: command.values.concurrency_wait_timeout_seconds,
|
||||
openai_guardian_reserved_concurrency: command
|
||||
.values
|
||||
.openai_guardian_reserved_concurrency,
|
||||
openai_account_affinity: command.values.openai_account_affinity,
|
||||
max_account_rotations: command.values.max_account_rotations,
|
||||
openai_session_affinity_ttl_hours: command.values.openai_session_affinity_ttl_hours,
|
||||
responses_max_decompressed_body_bytes: command
|
||||
.values
|
||||
.responses_max_decompressed_body_bytes,
|
||||
smart_scheduling: command.values.smart_scheduling,
|
||||
min_codex_desktop_version: command.values.min_codex_desktop_version,
|
||||
min_codex_cli_version: command.values.min_codex_cli_version,
|
||||
usage_retention_days: command.values.usage_retention_days,
|
||||
ops_event_retention_days: command.values.ops_event_retention_days,
|
||||
audit_retention_days: command.values.audit_retention_days,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
};
|
||||
*settings = updated.clone();
|
||||
Ok(updated)
|
||||
@@ -1571,37 +1611,42 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
]);
|
||||
RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: mappings,
|
||||
refresh_margin_seconds: 3_600,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: RotationStrategy::Smart,
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
updated_at: Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 3_600,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: None,
|
||||
min_codex_cli_version: None,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1643,6 +1688,7 @@ fn unavailable(resource: &'static str) -> AdminStoreError {
|
||||
resource,
|
||||
"unused test port",
|
||||
)
|
||||
.with_source(std::io::Error::other("PRIVATE_ADMIN_NATIVE_CAUSE"))
|
||||
}
|
||||
|
||||
fn not_found(resource: &'static str) -> AdminStoreError {
|
||||
|
||||
@@ -466,7 +466,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
raw_upstream_error: Some(
|
||||
error_details: Some(
|
||||
r#"{"error":{"code":"upstream","message":"raw upstream marker"}}"#.to_owned(),
|
||||
),
|
||||
client_ip: Some("203.0.113.8".to_owned()),
|
||||
@@ -526,7 +526,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"userAgent": value["data"]["items"][0]["userAgent"],
|
||||
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
|
||||
"upstreamSendState": value["data"]["items"][0]["upstreamSendState"],
|
||||
"rawUpstreamError": value["data"]["items"][0]["rawUpstreamError"],
|
||||
"errorDetails": value["data"]["items"][0]["errorDetails"],
|
||||
"continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"],
|
||||
"upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"],
|
||||
"upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"],
|
||||
@@ -547,7 +547,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"userAgent": "codex-cli/0.144.0",
|
||||
"providerErrorCode": "upstream",
|
||||
"upstreamSendState": "sent",
|
||||
"rawUpstreamError": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
|
||||
"errorDetails": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
|
||||
"continuationUnavailableReason": "reused_connection_lost",
|
||||
"upstreamConnectionExitReason": "tcp_reset",
|
||||
"upstreamConnectionAgeMs": 12000,
|
||||
|
||||
@@ -53,6 +53,9 @@ fn update_body() -> Value {
|
||||
"maxWaitingPerKey": 0,
|
||||
"maxWaitingPerAccount": 0,
|
||||
"openaiGuardianReservedConcurrency": 0,
|
||||
"openaiAccountAffinity": "relaxed",
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
@@ -203,8 +206,6 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
@@ -216,36 +217,43 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
UpstreamModelId::new("grok-4.5").expect("upstream model"),
|
||||
),
|
||||
]),
|
||||
refresh_margin_seconds: 1800,
|
||||
refresh_concurrency: 4,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 25,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
rotation_strategy: RotationStrategy::RoundRobin,
|
||||
min_codex_desktop_version: Some("26.825.6671".to_owned()),
|
||||
min_codex_cli_version: Some("0.40.0".to_owned()),
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
.expect("timestamp"),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: 1800,
|
||||
refresh_concurrency: 4,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 25,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: Some("26.825.6671".to_owned()),
|
||||
min_codex_cli_version: Some("0.40.0".to_owned()),
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(RuntimeSettingsView::from((
|
||||
@@ -273,6 +281,9 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
"maxWaitingPerKey": 0,
|
||||
"maxWaitingPerAccount": 0,
|
||||
"openaiGuardianReservedConcurrency": 0,
|
||||
"openaiAccountAffinity": "relaxed",
|
||||
"maxAccountRotations": 3,
|
||||
"openaiSessionAffinityTtlHours": 24,
|
||||
"concurrencyWaitTimeoutSeconds": 30,
|
||||
"responsesMaxDecompressedBodyBytes": 67108864,
|
||||
"rotationStrategy": "round_robin",
|
||||
@@ -321,8 +332,6 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
@@ -335,34 +344,43 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, gateway_core::error::IdentifierError>>()
|
||||
.expect("valid model mappings"),
|
||||
refresh_margin_seconds: request.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(request.refresh_concurrency).expect("u32"),
|
||||
max_concurrent_per_account: u32::try_from(request.max_concurrent_per_account).expect("u32"),
|
||||
request_interval_ms: request.request_interval_ms,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: request.smart_scheduling,
|
||||
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
|
||||
.expect("fixture rotation strategy"),
|
||||
min_codex_desktop_version: request.min_codex_desktop_version,
|
||||
min_codex_cli_version: request.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
updated_at: chrono::Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: request.values.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(request.values.refresh_concurrency).expect("u32"),
|
||||
max_concurrent_per_account: u32::try_from(request.values.max_concurrent_per_account)
|
||||
.expect("u32"),
|
||||
request_interval_ms: request.values.request_interval_ms,
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: request.values.smart_scheduling,
|
||||
min_codex_desktop_version: request.values.min_codex_desktop_version,
|
||||
min_codex_cli_version: request.values.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(request.values.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.values.ops_event_retention_days)
|
||||
.expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.values.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
},
|
||||
};
|
||||
|
||||
let response_fields: BTreeSet<String> = serde_json::to_value(RuntimeSettingsView::from((
|
||||
@@ -386,6 +404,48 @@ fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
assert_eq!(response_fields, expected_fields);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_accepts_json_bytes_with_flattened_fields() {
|
||||
serde_json::from_slice::<UpdateRuntimeSettingsRequest>(
|
||||
&serde_json::to_vec(&update_body()).unwrap(),
|
||||
)
|
||||
.expect("settings JSON bytes must decode like the HTTP request body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
use gateway_admin::model::settings::{ReplaceRuntimeSettings, RuntimeSettings};
|
||||
|
||||
let current = super::test_runtime_settings();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<RuntimeSettings>(&serde_json::to_vec(¤t).unwrap()).unwrap(),
|
||||
current
|
||||
);
|
||||
let mut record = serde_json::to_value(¤t).unwrap();
|
||||
assert!(record.get("values").is_none());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<RuntimeSettings>(record.clone()).unwrap(),
|
||||
current
|
||||
);
|
||||
record["unexpected_setting"] = json!(true);
|
||||
assert!(serde_json::from_value::<RuntimeSettings>(record).is_err());
|
||||
|
||||
let command = ReplaceRuntimeSettings::from(current);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<ReplaceRuntimeSettings>(&serde_json::to_vec(&command).unwrap())
|
||||
.unwrap(),
|
||||
command
|
||||
);
|
||||
let mut wire = serde_json::to_value(&command).unwrap();
|
||||
assert!(wire.get("values").is_none());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ReplaceRuntimeSettings>(wire.clone()).unwrap(),
|
||||
command
|
||||
);
|
||||
wire["unexpected_setting"] = json!(true);
|
||||
assert!(serde_json::from_value::<ReplaceRuntimeSettings>(wire).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
let mut body = update_body();
|
||||
@@ -1329,3 +1389,76 @@ async fn guardian_reservation_round_trips_and_rejects_invalid_values() {
|
||||
.remove("openaiGuardianReservedConcurrency");
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(omitted).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_affinity_and_rotation_budget_round_trip_and_reject_invalid_values() {
|
||||
for (mode, rotations, ttl) in [("relaxed", 0, 1), ("preferred", 3, 24), ("strict", 31, 720)] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
body["openaiAccountAffinity"] = json!(mode);
|
||||
body["maxAccountRotations"] = json!(rotations);
|
||||
body["openaiSessionAffinityTtlHours"] = json!(ttl);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = response_json(response).await;
|
||||
assert_eq!(response["data"]["openaiAccountAffinity"], mode);
|
||||
assert_eq!(response["data"]["maxAccountRotations"], rotations);
|
||||
assert_eq!(response["data"]["openaiSessionAffinityTtlHours"], ttl);
|
||||
}
|
||||
// 类型错误沿用 AdminJson 的 422,合法类型越界由字段校验返回 400
|
||||
for (field, value, expected_status) in [
|
||||
(
|
||||
"openaiAccountAffinity",
|
||||
json!("unknown"),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
(
|
||||
"maxAccountRotations",
|
||||
json!(-1),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
("maxAccountRotations", json!(32), StatusCode::BAD_REQUEST),
|
||||
(
|
||||
"maxAccountRotations",
|
||||
json!(1.5),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
(
|
||||
"openaiSessionAffinityTtlHours",
|
||||
json!(0),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
"openaiSessionAffinityTtlHours",
|
||||
json!(721),
|
||||
StatusCode::BAD_REQUEST,
|
||||
),
|
||||
(
|
||||
"openaiSessionAffinityTtlHours",
|
||||
json!(1.5),
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
body[field] = value;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), expected_status, "{field}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/admin/auth.rs",
|
||||
"src/admin/backups.rs",
|
||||
"src/admin/client_keys.rs",
|
||||
"src/admin/diagnostics.rs",
|
||||
"src/admin/extract.rs",
|
||||
"src/admin/mod.rs",
|
||||
"src/admin/observability/mod.rs",
|
||||
@@ -124,6 +125,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/admin/auth.rs",
|
||||
"tests/admin/client_keys.rs",
|
||||
"tests/admin/errors.rs",
|
||||
"tests/admin/error_diagnostics.rs",
|
||||
"tests/admin/mod.rs",
|
||||
"tests/admin/observability/mod.rs",
|
||||
"tests/admin/observability/query.rs",
|
||||
|
||||
@@ -535,3 +535,97 @@ async fn key_auth_should_issue_restore_and_clear_a_unified_cookie() {
|
||||
.expect("logged out");
|
||||
assert_eq!(response_json(status).await["data"]["authenticated"], false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_renews_admin_cookie_and_preserves_fixed_key_expiry() {
|
||||
let (app, fixture) = auth_app().await;
|
||||
for body in [
|
||||
json!({"mode": "admin", "password": "strong-admin-password"}),
|
||||
json!({"mode": "key", "apiKey": RAW_KEY}),
|
||||
] {
|
||||
let response = login(&app, body.clone(), None).await;
|
||||
let cookie = session_cookie(&response);
|
||||
let original = response_json(response).await["data"]["expiresAt"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
if body["mode"] == "admin" {
|
||||
fixture.set_session_expiry(
|
||||
cookie.split_once('=').unwrap().1,
|
||||
chrono::Utc::now() + chrono::Duration::minutes(1),
|
||||
);
|
||||
}
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/refresh", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
assert_eq!(session_cookie(&response), cookie);
|
||||
assert!(
|
||||
response.headers()[header::SET_COOKIE]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.contains("HttpOnly")
|
||||
);
|
||||
let data = response_json(response).await;
|
||||
assert_eq!(data["data"]["authenticated"], true);
|
||||
let expiry = data["data"]["session"]["expiresAt"].as_str().unwrap();
|
||||
if body["mode"] == "key" {
|
||||
assert_eq!(expiry, original);
|
||||
} else {
|
||||
assert!(
|
||||
chrono::DateTime::parse_from_rfc3339(expiry).unwrap()
|
||||
> chrono::Utc::now() + chrono::Duration::minutes(1)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_after_logout_or_password_change_cannot_recreate_a_session() {
|
||||
let (app, _) = auth_app().await;
|
||||
for password_change in [false, true] {
|
||||
let cookie = session_cookie(
|
||||
&login(
|
||||
&app,
|
||||
json!({"mode": "admin", "password": "strong-admin-password"}),
|
||||
None,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
let response = if password_change {
|
||||
change_password(
|
||||
&app,
|
||||
Some(&cookie),
|
||||
"strong-admin-password",
|
||||
"new-strong-admin-password",
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
app.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/logout", &cookie))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/refresh", &cookie))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(header::SET_COOKIE));
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
assert_eq!(
|
||||
get(&app, "/api/admin/system/version", &cookie)
|
||||
.await
|
||||
.status(),
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,7 +183,7 @@ fn error_record() -> OpsError {
|
||||
upstream_request_id: Some("private-sentinel".to_owned()),
|
||||
latency_ms: Some(420),
|
||||
message: "private-sentinel".to_owned(),
|
||||
raw_upstream_error: Some("private-sentinel".to_owned()),
|
||||
error_details: Some("private-sentinel".to_owned()),
|
||||
client_ip: Some("192.0.2.42".to_owned()),
|
||||
user_agent: Some("key-usage-test/1.0".to_owned()),
|
||||
reasoning_effort: Some("xhigh".to_owned()),
|
||||
|
||||
@@ -479,6 +479,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission(
|
||||
Arc::new(super::UnusedAdmissions),
|
||||
Arc::new(super::UnusedContinuation),
|
||||
Arc::new(super::IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
));
|
||||
let admin = crate::admin::AdminTestFixture::new().await;
|
||||
let bundle = gateway_api::initialize(
|
||||
@@ -493,6 +494,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission(
|
||||
vec![],
|
||||
Arc::new(super::EmptyWorkerHealth),
|
||||
Arc::new(super::TestLifecycle::default()),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.unwrap();
|
||||
let baseline = bundle.dispatcher();
|
||||
|
||||
@@ -304,12 +304,14 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::AccountCapacityUnavailable,
|
||||
"all eligible upstream accounts are temporarily busy"
|
||||
)
|
||||
projected.kind(),
|
||||
GatewayErrorKind::AccountCapacityUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
projected.safe_message(),
|
||||
"all eligible upstream accounts are temporarily busy"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -352,12 +354,14 @@ fn provider_infrastructure_failure_should_have_a_distinct_client_contract() {
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider account infrastructure is temporarily unavailable"
|
||||
)
|
||||
projected.kind(),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
projected.safe_message(),
|
||||
"provider account infrastructure is temporarily unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -368,12 +372,11 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::NoAvailableProvider);
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request"
|
||||
)
|
||||
projected.safe_message(),
|
||||
"no upstream provider is currently available for this request"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -499,13 +502,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() {
|
||||
] {
|
||||
let error = EngineError::Provider(ProviderError::new(kind, UpstreamSendState::Ambiguous));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"upstream service is unavailable"
|
||||
)
|
||||
);
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert_eq!(projected.safe_message(), "upstream service is unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -513,10 +512,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() {
|
||||
fn engine_store_error_should_collapse_to_safe_internal_error() {
|
||||
let error = EngineError::Store(StoreError::new(StoreErrorKind::Unavailable));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
|
||||
);
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::Internal);
|
||||
assert_eq!(projected.safe_message(), "gateway execution failed");
|
||||
}
|
||||
#[test]
|
||||
fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
@@ -669,6 +667,7 @@ mod model_routing {
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
);
|
||||
let response = crate::openai::api_router(Arc::new(execution))
|
||||
.await
|
||||
|
||||
@@ -138,6 +138,7 @@ async fn blocked_relay(
|
||||
Vec::new(),
|
||||
Arc::new(crate::openai::EmptyWorkerHealth),
|
||||
Arc::new(Lifecycle(trace.clone())),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.unwrap()
|
||||
.router();
|
||||
|
||||
@@ -92,6 +92,7 @@ pub(super) fn api_router_with_admin_and_client(
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
));
|
||||
api_router_with_admin_and_execution(admin, execution)
|
||||
}
|
||||
@@ -106,6 +107,18 @@ pub(super) fn api_router_with_config(
|
||||
pub(super) fn api_bundle(
|
||||
admin: gateway_admin::AdminServices,
|
||||
config: gateway_api::ApiConfig,
|
||||
) -> gateway_api::ApiBundle {
|
||||
api_bundle_with_diagnostics(
|
||||
admin,
|
||||
config,
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn api_bundle_with_diagnostics(
|
||||
admin: gateway_admin::AdminServices,
|
||||
config: gateway_api::ApiConfig,
|
||||
diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
|
||||
) -> gateway_api::ApiBundle {
|
||||
let execution = Arc::new(DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot("unused-client-route-key", "openai")),
|
||||
@@ -114,6 +127,7 @@ pub(super) fn api_bundle(
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
));
|
||||
gateway_api::initialize(
|
||||
config,
|
||||
@@ -122,6 +136,7 @@ pub(super) fn api_bundle(
|
||||
Vec::new(),
|
||||
Arc::new(EmptyWorkerHealth),
|
||||
Arc::new(TestLifecycle::default()),
|
||||
diagnostics,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
@@ -138,6 +153,7 @@ fn api_router_with_config_and_execution(
|
||||
Vec::new(),
|
||||
Arc::new(EmptyWorkerHealth),
|
||||
Arc::new(TestLifecycle::default()),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.expect("API bundle")
|
||||
.router()
|
||||
@@ -180,6 +196,7 @@ async fn api_router_with_origins_and_worker_health(
|
||||
Vec::new(),
|
||||
worker_health,
|
||||
Arc::new(TestLifecycle::default()),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.expect("API bundle")
|
||||
.router()
|
||||
@@ -214,6 +231,7 @@ pub(super) fn authenticated_client_for_provider_with_limit(
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
);
|
||||
source
|
||||
.authenticate(plaintext)
|
||||
@@ -244,6 +262,7 @@ pub(super) fn authenticated_client_with_min_versions(
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
);
|
||||
source
|
||||
.authenticate(plaintext)
|
||||
|
||||
@@ -165,6 +165,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake
|
||||
Arc::new(crate::openai::UnusedAdmissions),
|
||||
Arc::new(crate::openai::UnusedContinuation),
|
||||
Arc::new(crate::openai::IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
));
|
||||
let admin = crate::admin::AdminTestFixture::new().await;
|
||||
let plan = Arc::new(SettingsPlan::default());
|
||||
@@ -187,6 +188,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake
|
||||
vec![],
|
||||
Arc::new(crate::openai::EmptyWorkerHealth),
|
||||
Arc::new(crate::openai::TestLifecycle::default()),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.unwrap()
|
||||
.with_middleware(move |_| Some(frozen.clone()))
|
||||
|
||||
@@ -761,6 +761,7 @@ async fn start_active_response_with_middleware(
|
||||
Vec::new(),
|
||||
Arc::new(crate::openai::EmptyWorkerHealth),
|
||||
Arc::new(crate::openai::TestLifecycle::default()),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.unwrap()
|
||||
.with_middleware(move |_| Some(plan.clone()))
|
||||
|
||||
@@ -162,6 +162,7 @@ async fn interrupt_is_live_scoped_and_keeps_queued_creates_serial() {
|
||||
admissions.clone(),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
));
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -1108,6 +1108,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele
|
||||
ports.clone(),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.with_budget(ports.clone()),
|
||||
);
|
||||
@@ -1125,6 +1126,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele
|
||||
Vec::new(),
|
||||
Arc::new(EmptyWorkerHealth),
|
||||
lifecycle.clone(),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.unwrap()
|
||||
.router();
|
||||
|
||||
@@ -159,3 +159,18 @@ impl ClientKeyVerifier for AcceptingVerifier {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RecordingDiagnostics(
|
||||
pub std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>,
|
||||
);
|
||||
#[async_trait]
|
||||
impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
|
||||
async fn record_failure(
|
||||
&self,
|
||||
failure: gateway_core::diagnostics::OperationalFailure,
|
||||
) -> Result<(), gateway_core::error::StoreError> {
|
||||
self.0.lock().unwrap().push(failure);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,43 @@ impl RotationStrategy {
|
||||
}
|
||||
}
|
||||
|
||||
/// 单请求换号预算不能超过路由尝试总数减一
|
||||
pub const MAX_ACCOUNT_ROTATIONS: u32 = 31;
|
||||
|
||||
/// 会话绑定与轮次关联共用的保留时长上限,单位小时
|
||||
pub const MAX_SESSION_AFFINITY_TTL_HOURS: u32 = 720;
|
||||
|
||||
/// OpenAI 会话账号亲和;请求身份和模式行为由 Provider 解释
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AccountAffinity {
|
||||
Relaxed,
|
||||
Preferred,
|
||||
#[default]
|
||||
Strict,
|
||||
}
|
||||
|
||||
impl AccountAffinity {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Relaxed => "relaxed",
|
||||
Self::Preferred => "preferred",
|
||||
Self::Strict => "strict",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"relaxed" => Some(Self::Relaxed),
|
||||
"preferred" => Some(Self::Preferred),
|
||||
"strict" => Some(Self::Strict),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `runtime_settings` 冻结到一次请求计划的账号调度策略
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSelectionPolicy {
|
||||
@@ -55,6 +92,9 @@ pub struct AccountSelectionPolicy {
|
||||
request_interval: Duration,
|
||||
queue_policy: ConcurrencyQueuePolicy,
|
||||
openai_guardian_reserved_concurrency: u32,
|
||||
openai_account_affinity: AccountAffinity,
|
||||
max_account_rotations: u32,
|
||||
openai_session_affinity_ttl: Duration,
|
||||
}
|
||||
|
||||
impl AccountSelectionPolicy {
|
||||
@@ -70,6 +110,9 @@ impl AccountSelectionPolicy {
|
||||
max_concurrent_per_account: max_concurrent_per_account.into(),
|
||||
request_interval,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: AccountAffinity::default(),
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl: Duration::from_secs(24 * 3600),
|
||||
queue_policy: ConcurrencyQueuePolicy {
|
||||
max_waiting: 0,
|
||||
timeout: Duration::ZERO,
|
||||
@@ -77,6 +120,39 @@ impl AccountSelectionPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_openai_account_affinity(mut self, affinity: AccountAffinity) -> Self {
|
||||
self.openai_account_affinity = affinity;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn openai_account_affinity(self) -> AccountAffinity {
|
||||
self.openai_account_affinity
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_max_account_rotations(mut self, rotations: u32) -> Self {
|
||||
self.max_account_rotations = rotations;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn max_account_rotations(self) -> u32 {
|
||||
self.max_account_rotations
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_openai_session_affinity_ttl(mut self, ttl: Duration) -> Self {
|
||||
self.openai_session_affinity_ttl = ttl;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn openai_session_affinity_ttl(self) -> Duration {
|
||||
self.openai_session_affinity_ttl
|
||||
}
|
||||
|
||||
/// 只传递冻结的运行设置,Guardian 分类与预留策略由 OpenAI Provider 解释
|
||||
#[must_use]
|
||||
pub const fn with_openai_guardian_reserved_concurrency(mut self, reserved: u32) -> Self {
|
||||
|
||||
@@ -2,13 +2,24 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(try_from = "SmartSchedulingValues", into = "SmartSchedulingValues")]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(into = "SmartSchedulingValues")]
|
||||
pub struct SmartSchedulingConfig {
|
||||
weights: [u8; 6],
|
||||
prefer_higher_weight: bool,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SmartSchedulingConfig {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
// flatten 会缓冲字段,arbitrary_precision 的小数需先恢复为 JSON Number
|
||||
// 随后仍由同一类型检查完整字段、数值范围和十分位精度
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
let values = serde_json::from_value::<SmartSchedulingValues>(value)
|
||||
.map_err(serde::de::Error::custom)?;
|
||||
Self::try_from(values).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SmartSchedulingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
//! 控制面与后台操作的受控故障记录,不伪造模型请求或上游发送事实
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::{
|
||||
account::ProviderAccountId,
|
||||
error::{ErrorDetails, OpaqueUpstreamValue, StoreError},
|
||||
identity::ProviderKind,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct OperationalFailure {
|
||||
pub occurred_at: std::time::SystemTime,
|
||||
pub component: &'static str,
|
||||
pub operation: &'static str,
|
||||
pub kind: &'static str,
|
||||
pub message: String,
|
||||
pub correlation_id: Option<String>,
|
||||
pub provider_kind: Option<ProviderKind>,
|
||||
pub account_id: Option<ProviderAccountId>,
|
||||
pub upstream_status: Option<u16>,
|
||||
pub upstream_code: Option<OpaqueUpstreamValue>,
|
||||
pub details: Option<ErrorDetails>,
|
||||
}
|
||||
|
||||
impl OperationalFailure {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
component: &'static str,
|
||||
operation: &'static str,
|
||||
kind: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
occurred_at: std::time::SystemTime::now(),
|
||||
component,
|
||||
operation,
|
||||
kind,
|
||||
message: message.into(),
|
||||
correlation_id: None,
|
||||
provider_kind: None,
|
||||
account_id: None,
|
||||
upstream_status: None,
|
||||
upstream_code: None,
|
||||
details: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录遵循既有观测队列的 best-effort 合同,失败不应替换业务结果
|
||||
#[async_trait]
|
||||
pub trait OperationalDiagnostics: Send + Sync {
|
||||
async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError>;
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
//! 有界、请求局部的诊断证据;不参与路由、重试或客户端协议决策
|
||||
//! 请求诊断与操作故障的观测合同,不参与路由、重试或客户端协议决策
|
||||
|
||||
mod capture;
|
||||
mod failure;
|
||||
mod selection;
|
||||
mod stream;
|
||||
mod trace;
|
||||
|
||||
pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json};
|
||||
pub use failure::{OperationalDiagnostics, OperationalFailure};
|
||||
pub use stream::{StreamCapture, StreamFormat};
|
||||
pub use trace::TraceContext;
|
||||
|
||||
@@ -10,6 +10,8 @@ use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::error::ProviderError;
|
||||
|
||||
use super::capture::{bounded, diagnostic_event_json, diagnostic_event_type};
|
||||
use super::{body_fingerprint, diagnostic_headers};
|
||||
|
||||
@@ -123,6 +125,22 @@ impl TraceContext {
|
||||
self.push(stage, data, false);
|
||||
}
|
||||
|
||||
/// 诊断正文与导出分类来自同一失败,导出路径不接收任意 JSON 或上游文案
|
||||
pub fn record_provider_failure(&self, error: &ProviderError) {
|
||||
if !self.is_enabled() {
|
||||
return;
|
||||
}
|
||||
let data = json!({
|
||||
"kind": error.kind().as_str(), "sendState": error.send_state().as_str(),
|
||||
"diagnostic": error.diagnostic().map(|diagnostic| json!({
|
||||
"stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(),
|
||||
})),
|
||||
"upstreamStatus": error.upstream_status(),
|
||||
"upstreamRequestId": error.upstream_request_id().map(|id| id.as_str()),
|
||||
});
|
||||
self.push("attempt.failed", data, false);
|
||||
}
|
||||
|
||||
/// 同一个头部边界只采集一次:默认脱敏,显式开启 dump 时另存完整字节
|
||||
/// facts 必须为调用方构造的安全对象;仅受控诊断头可明文保留,完整头部另存 dump
|
||||
pub fn headers<'a>(
|
||||
@@ -243,7 +261,7 @@ impl TraceContext {
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
Some(json!({
|
||||
"schemaVersion": 1, "requestId": state.request_id,
|
||||
"schemaVersion": 2, "requestId": state.request_id,
|
||||
"startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence,
|
||||
"droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS,
|
||||
"captureMode": "sanitized", "events": state.events,
|
||||
@@ -258,10 +276,35 @@ impl TraceContext {
|
||||
let Some(state) = &self.state else { return };
|
||||
let mut encoded = serde_json::to_vec(&data).unwrap_or_default();
|
||||
if encoded.len() > MAX_DATA_BYTES {
|
||||
data = json!({"truncated": true, "summary": body_fingerprint(&encoded),
|
||||
let mut summary = json!({"truncated": true, "summary": body_fingerprint(&encoded),
|
||||
"eventType": data.get("eventType"), "body": data.get("body"),
|
||||
"jsonValid": data.get("jsonValid"),
|
||||
});
|
||||
if stage == "attempt.failed" {
|
||||
// 过长诊断的指纹不能挤掉已经由 Provider 分类的失败原因
|
||||
// 保持有界,完整诊断仍由终态错误记录保存
|
||||
summary = json!({"truncated": true, "summary": body_fingerprint(&encoded)});
|
||||
for key in ["kind", "sendState"] {
|
||||
if let Some(value) = data.get(key).and_then(Value::as_str) {
|
||||
summary[key] = json!(bounded(value, 64));
|
||||
}
|
||||
}
|
||||
summary["upstreamStatus"] =
|
||||
json!(data.get("upstreamStatus").and_then(Value::as_u64));
|
||||
if let Some(diagnostic) = data.get("diagnostic").and_then(Value::as_object) {
|
||||
let mut kept = json!({});
|
||||
for (key, limit) in [("stage", 96), ("code", 96), ("message", 256)] {
|
||||
if let Some(value) = diagnostic.get(key).and_then(Value::as_str) {
|
||||
kept[key] = json!(bounded(value, limit));
|
||||
if value.chars().count() > limit {
|
||||
kept["truncated"] = json!(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
summary["diagnostic"] = kept;
|
||||
}
|
||||
}
|
||||
data = summary;
|
||||
encoded = serde_json::to_vec(&data).unwrap_or_default();
|
||||
}
|
||||
let mut state = state
|
||||
|
||||
@@ -57,9 +57,9 @@ pub struct ClientAdmissionRestoreResult {
|
||||
pub restored_running_requests: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("client admission store is unavailable")]
|
||||
pub struct ClientAdmissionError;
|
||||
pub struct ClientAdmissionError(#[source] pub Option<crate::error::ErrorSource>);
|
||||
|
||||
pub trait ClientAdmissionPort: Send + Sync {
|
||||
/// 仅续期已经准入的并发槽位,不重复消费 RPM
|
||||
@@ -118,10 +118,10 @@ pub async fn restore_client_admission_startup(
|
||||
let expired = execution
|
||||
.recover_expired(now)
|
||||
.await
|
||||
.map_err(|_| ClientAdmissionError)?;
|
||||
.map_err(|source| ClientAdmissionError(Some(source.into())))?;
|
||||
let since = now
|
||||
.checked_sub(Duration::from_secs(61))
|
||||
.ok_or(ClientAdmissionError)?;
|
||||
.ok_or(ClientAdmissionError(None))?;
|
||||
let recoveries = recovery.load_recovery(since).await?;
|
||||
let restored_clients = u64::try_from(recoveries.len()).unwrap_or(u64::MAX);
|
||||
let mut restored_recent_requests = 0_u64;
|
||||
|
||||
@@ -39,9 +39,9 @@ pub struct ClientBudgetCharge {
|
||||
pub completed_at: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, thiserror::Error)]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("client budget store is unavailable")]
|
||||
pub struct ClientBudgetError;
|
||||
pub struct ClientBudgetError(#[source] pub Option<crate::error::ErrorSource>);
|
||||
|
||||
pub trait ClientBudgetPort: Send + Sync {
|
||||
/// 原子检查当前限额与已用金额,不创建预扣费或待结算记录
|
||||
|
||||
@@ -398,7 +398,7 @@ struct FailureFinalization {
|
||||
upstream_status_code: Option<u16>,
|
||||
upstream_request_id: Option<String>,
|
||||
provider_error_code: Option<String>,
|
||||
raw_upstream_error: Option<String>,
|
||||
error_details: Option<String>,
|
||||
retry_after_ms: Option<u64>,
|
||||
observation: ModelRequestFailureObservation,
|
||||
}
|
||||
@@ -464,7 +464,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
/// 已发生的换号次数:选中账号与上一 attempt 不同的路由 attempt 计一次。
|
||||
/// 首个 attempt 不计;同账号钉选重试(瞬态退避、传输恢复、凭据恢复重放、
|
||||
/// continuation 精确重连)不消耗。预算耗尽后所有必然换号的重试门关闭,
|
||||
/// 换号深度由 [`crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS`] 封顶。
|
||||
/// 换号深度由请求冻结的调度策略封顶
|
||||
account_rotations: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
@@ -523,6 +523,10 @@ where
|
||||
self.trace.clone()
|
||||
}
|
||||
|
||||
pub(super) fn request_id(&self) -> &ModelRequestId {
|
||||
&self.request_id
|
||||
}
|
||||
|
||||
/// 读取下一条 canonical event;首条未提交事件会携带 commit 要求
|
||||
///
|
||||
/// # Errors
|
||||
@@ -956,7 +960,7 @@ where
|
||||
upstream_status_code: None,
|
||||
upstream_request_id: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
@@ -1026,6 +1030,9 @@ where
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
self.trace
|
||||
.attempt(self.attempts)
|
||||
.record_provider_failure(&error);
|
||||
self.finish_provider_error(&error).await?;
|
||||
return Err(provider_engine_error(error));
|
||||
}
|
||||
@@ -1101,7 +1108,7 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
record_trace_error(&attempt_trace, &error);
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
let continuation_retry = !error.retry_is_prohibited()
|
||||
&& self.prepare_unavailable_native_continuation_replay(&error);
|
||||
let candidate_retry = !error.retry_is_prohibited() && !continuation_retry
|
||||
@@ -1122,7 +1129,7 @@ where
|
||||
)
|
||||
// 跨 Provider 候选推进必然换号(账号行按 Provider 隔离),
|
||||
// 预算耗尽后不再推进,交回容量类失败的原有终态语义。
|
||||
&& self.account_rotations < crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS
|
||||
&& self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
|
||||
&& self.advance_provider_candidate();
|
||||
let retryable = self
|
||||
.apply_retry_policy(super::policy::RetryFacts {
|
||||
@@ -1200,7 +1207,7 @@ where
|
||||
upstream_status_code: None,
|
||||
upstream_request_id: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
@@ -1233,7 +1240,7 @@ where
|
||||
upstream_status_code: None,
|
||||
upstream_request_id: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
@@ -1256,7 +1263,7 @@ where
|
||||
upstream_status_code: None,
|
||||
upstream_request_id: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
@@ -1282,7 +1289,7 @@ where
|
||||
upstream_status_code: None,
|
||||
upstream_request_id: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
@@ -1438,7 +1445,9 @@ where
|
||||
) -> Result<StreamErrorOutcome, EngineError> {
|
||||
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
|
||||
// 均只接触已剥离的稳定错误字段
|
||||
record_trace_error(&self.trace.attempt(self.attempts), &error);
|
||||
self.trace
|
||||
.attempt(self.attempts)
|
||||
.record_provider_failure(&error);
|
||||
let mut atomic_client_events = error.take_atomic_client_events();
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if self.request_observation.is_some() {
|
||||
@@ -1596,7 +1605,8 @@ where
|
||||
&& transient_retry.is_none()
|
||||
&& transport_recovery.is_none()
|
||||
&& (ordinary_retry || account_rotation_retry)
|
||||
&& self.account_rotations < crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS;
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
let retryable = !error.retry_is_prohibited()
|
||||
&& (continuation_retry
|
||||
|| same_account_retry
|
||||
@@ -1839,7 +1849,8 @@ where
|
||||
// 两个排除臂都必然换号;预算耗尽后不再排除当前账号做跨账号续写重放,
|
||||
// 落回不可重试路径以原始上游错误终态。
|
||||
ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny
|
||||
if self.account_rotations >= crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS =>
|
||||
if self.account_rotations
|
||||
>= self.plan.account_selection_policy().max_account_rotations() =>
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -1890,9 +1901,8 @@ where
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// Native 期间所有 attempt 都在 pin 账号上,到这里预算必未耗尽;
|
||||
// 与其余换号路径保持同一预算门,防止状态机演化后破坏换号上限。
|
||||
if self.account_rotations >= crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS {
|
||||
// Native 期间仍可能配置为禁止换号,跨账号重放共用请求冻结的预算
|
||||
if self.account_rotations >= self.plan.account_selection_policy().max_account_rotations() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1961,7 +1971,7 @@ where
|
||||
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
failure_observation: ModelRequestFailureObservation::default(),
|
||||
retry_after_ms: None,
|
||||
usage: self.observation.usage.clone(),
|
||||
@@ -2002,9 +2012,7 @@ where
|
||||
upstream_status_code: error.upstream_status(),
|
||||
upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()),
|
||||
provider_error_code: error.upstream_code().map(|code| code.as_str().to_owned()),
|
||||
raw_upstream_error: error
|
||||
.raw_upstream_error()
|
||||
.map(|raw| raw.as_str().to_owned()),
|
||||
error_details: error.error_details(),
|
||||
retry_after_ms: error.retry_after().map(duration_ms),
|
||||
observation: ModelRequestFailureObservation {
|
||||
continuation_unavailable_reason: error
|
||||
@@ -2065,7 +2073,7 @@ where
|
||||
upstream_status_code: None,
|
||||
upstream_request_id: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
error_details: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
@@ -2128,7 +2136,7 @@ where
|
||||
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
raw_upstream_error: finalization.raw_upstream_error,
|
||||
error_details: finalization.error_details,
|
||||
failure_observation: finalization.observation,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: self.observation.usage.clone(),
|
||||
@@ -2489,19 +2497,3 @@ const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> Upst
|
||||
fn duration_ms(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn record_trace_error(trace: &TraceContext, error: &ProviderError) {
|
||||
trace.record("attempt.failed", json!({
|
||||
"kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()),
|
||||
"diagnostic": error.diagnostic().map(|diagnostic| json!({
|
||||
"stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(),
|
||||
})),
|
||||
"upstreamStatus": error.upstream_status(),
|
||||
"upstreamRequestId": error.upstream_request_id().map(|id| id.as_str()),
|
||||
"upstreamCode": error.upstream_code().map(|code| code.as_str()),
|
||||
"rawError": error.raw_upstream_error().map(|raw| {
|
||||
let value = serde_json::from_str(raw.as_str()).unwrap_or_else(|_| json!(raw.as_str()));
|
||||
crate::diagnostics::diagnostic_json(&value)
|
||||
}),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -739,6 +739,7 @@ pub struct DefaultExecutionService {
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
/// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events
|
||||
observations: Arc<dyn ExecutionStore>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
providers: ProviderRegistry,
|
||||
admissions: Arc<dyn ClientAdmissionPort>,
|
||||
admission_waiting: ConcurrencyWaitQueue<ClientApiKeyId>,
|
||||
@@ -763,10 +764,12 @@ impl DefaultExecutionService {
|
||||
admissions: Arc<dyn ClientAdmissionPort>,
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
) -> Self {
|
||||
Self {
|
||||
snapshots,
|
||||
observations: execution,
|
||||
diagnostics,
|
||||
providers,
|
||||
admissions,
|
||||
admission_waiting: ConcurrencyWaitQueue::default(),
|
||||
@@ -1310,8 +1313,12 @@ impl DefaultExecutionService {
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(write, timeout);
|
||||
select_biased! {
|
||||
result = write => { if result.is_err() { tracing::warn!("入口拒绝观测写入失败"); } },
|
||||
_ = timeout => tracing::warn!("入口拒绝观测写入超时"),
|
||||
result = write => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(request_id = request_id.as_str(), operation = "record_entry_rejection", error_kind = ?error.kind(), "入口拒绝观测写入失败");
|
||||
}
|
||||
},
|
||||
_ = timeout => tracing::warn!(request_id = request_id.as_str(), operation = "record_entry_rejection", "入口拒绝观测写入超时"),
|
||||
}
|
||||
}
|
||||
if let (Some(observation), Err(error)) = (&request_observation, &result) {
|
||||
@@ -1321,7 +1328,12 @@ impl DefaultExecutionService {
|
||||
&& let Some(guard) = start_guard.take()
|
||||
{
|
||||
guard
|
||||
.release_failed(self.budget.as_deref(), request_id, budget_key_id)
|
||||
.release_failed(
|
||||
self.budget.as_deref(),
|
||||
request_id,
|
||||
budget_key_id,
|
||||
self.diagnostics.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
result
|
||||
@@ -1494,6 +1506,7 @@ impl DefaultExecutionService {
|
||||
amount_usd: crate::metering::Decimal::ZERO,
|
||||
completed_at: SystemTime::now(),
|
||||
},
|
||||
self.diagnostics.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1514,6 +1527,7 @@ impl DefaultExecutionService {
|
||||
active_request,
|
||||
Arc::clone(&self.continuation),
|
||||
self.budget.clone(),
|
||||
self.diagnostics.clone(),
|
||||
)),
|
||||
})
|
||||
}
|
||||
@@ -1868,6 +1882,7 @@ impl DefaultExecutionService {
|
||||
CapacityWait::new(&self.admission_waiting, policy, deadline_at.at(), budget);
|
||||
let mut admission = AdmissionLease {
|
||||
port: Arc::clone(&self.admissions),
|
||||
diagnostics: self.diagnostics.clone(),
|
||||
client_api_key_id: key.clone(),
|
||||
model_request_id: request_id.clone(),
|
||||
armed: false,
|
||||
@@ -1898,7 +1913,7 @@ impl DefaultExecutionService {
|
||||
pin_mut!(acquire, timeout, cancelled);
|
||||
let decision = select_biased! {
|
||||
() = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "request admission was cancelled")),
|
||||
result = acquire => result.map_err(|_| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable"))?,
|
||||
result = acquire => result.map_err(|source| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable").with_source(source))?,
|
||||
_ = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")),
|
||||
};
|
||||
match decision {
|
||||
@@ -2091,7 +2106,8 @@ impl DefaultExecutionService {
|
||||
failure_kind = provider_error.kind().as_str(),
|
||||
send_state = ?provider_error.send_state(),
|
||||
upstream_status = ?provider_error.upstream_status(),
|
||||
provider_error_code = ?provider_error.upstream_code().map(|code| code.as_str()),
|
||||
diagnostic_stage = ?provider_error.diagnostic().and_then(|diagnostic| diagnostic.stage()),
|
||||
diagnostic_code = ?provider_error.diagnostic().and_then(|diagnostic| diagnostic.code()),
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
@@ -2472,13 +2488,26 @@ struct AdmissionLease {
|
||||
renewal: Option<Box<dyn LeaseGuard>>,
|
||||
armed: bool,
|
||||
port: Arc<dyn ClientAdmissionPort>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
client_api_key_id: ClientApiKeyId,
|
||||
model_request_id: ModelRequestId,
|
||||
}
|
||||
|
||||
async fn settle_budget(port: &dyn ClientBudgetPort, charge: ClientBudgetCharge) {
|
||||
async fn settle_budget(
|
||||
port: &dyn ClientBudgetPort,
|
||||
charge: ClientBudgetCharge,
|
||||
diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
|
||||
) {
|
||||
let request_id = charge.request_id.clone();
|
||||
if let Err(error) = port.settle(charge).await {
|
||||
tracing::error!(%error, "Client budget settlement failed; storage will retry on the next request");
|
||||
record_resource_failure(
|
||||
diagnostics,
|
||||
&request_id,
|
||||
"settle_client_budget",
|
||||
"Client budget settlement failed; storage will retry on the next request",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2490,7 +2519,14 @@ impl AdmissionLease {
|
||||
.release(&self.client_api_key_id, &self.model_request_id)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(%error, "Client admission 释放失败,依赖租约 TTL 收敛");
|
||||
record_resource_failure(
|
||||
self.diagnostics.as_ref(),
|
||||
&self.model_request_id,
|
||||
"release_client_admission",
|
||||
"Client admission release failed; lease TTL remains active",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.armed = false;
|
||||
}
|
||||
@@ -2531,6 +2567,7 @@ impl ExecutionStartGuard {
|
||||
budget: Option<&dyn ClientBudgetPort>,
|
||||
request_id: ModelRequestId,
|
||||
key_id: ClientApiKeyId,
|
||||
diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
|
||||
) {
|
||||
if let Some(active_request) = self.active_request {
|
||||
active_request.release();
|
||||
@@ -2544,6 +2581,7 @@ impl ExecutionStartGuard {
|
||||
amount_usd: crate::metering::Decimal::ZERO,
|
||||
completed_at: SystemTime::now(),
|
||||
},
|
||||
diagnostics,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -2559,6 +2597,7 @@ struct DefaultExecutionSession {
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
continuation_recorded: bool,
|
||||
budget: Option<Arc<dyn ClientBudgetPort>>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
}
|
||||
|
||||
impl DefaultExecutionSession {
|
||||
@@ -2568,6 +2607,7 @@ impl DefaultExecutionSession {
|
||||
active_request: Option<ActiveRequestLease>,
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
budget: Option<Arc<dyn ClientBudgetPort>>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
) -> Self {
|
||||
Self {
|
||||
core,
|
||||
@@ -2577,6 +2617,7 @@ impl DefaultExecutionSession {
|
||||
continuation,
|
||||
continuation_recorded: false,
|
||||
budget,
|
||||
diagnostics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2589,12 +2630,13 @@ impl DefaultExecutionSession {
|
||||
}
|
||||
let budget = self.budget.take();
|
||||
let charge = self.core.budget_charge();
|
||||
let diagnostics = self.diagnostics.clone();
|
||||
// 在首次 await 前把完整清理责任留在会话内
|
||||
// 事件等待被取消后,后续 poll
|
||||
// 或 detach 继续同一个 future,既不丢失费用,也不重启已完成的结算
|
||||
self.cleanup = Some(Box::pin(async move {
|
||||
if let Some(budget) = budget {
|
||||
settle_budget(budget.as_ref(), charge).await;
|
||||
settle_budget(budget.as_ref(), charge, diagnostics.as_ref()).await;
|
||||
}
|
||||
admission.release().await;
|
||||
}));
|
||||
@@ -2621,7 +2663,7 @@ impl DefaultExecutionSession {
|
||||
|
||||
async fn finalize_detached(&mut self) {
|
||||
if let Err(error) = self.core.cancel_and_finalize().await {
|
||||
tracing::warn!(%error, "Detached execution 终态收敛失败");
|
||||
tracing::warn!(request_id = self.core.request_id().as_str(), operation = "finalize_detached_execution", %error, "Detached execution 终态收敛失败");
|
||||
}
|
||||
self.settle_if_finalized().await;
|
||||
}
|
||||
@@ -2917,6 +2959,34 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
"no provider is available",
|
||||
)
|
||||
}
|
||||
EngineError::Store(source) => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "request execution failed")
|
||||
.with_source(source.clone())
|
||||
}
|
||||
_ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_resource_failure(
|
||||
diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
|
||||
request_id: &ModelRequestId,
|
||||
operation: &'static str,
|
||||
message: &'static str,
|
||||
source: impl Into<crate::error::ErrorSource>,
|
||||
) {
|
||||
let mut failure = crate::diagnostics::OperationalFailure::new(
|
||||
"core",
|
||||
operation,
|
||||
"resource_unavailable",
|
||||
message,
|
||||
);
|
||||
failure.correlation_id = Some(request_id.as_str().to_owned());
|
||||
failure.details = crate::error::ErrorDetails::capture(Some(&source.into()), None, false);
|
||||
if diagnostics.record_failure(failure).await.is_err() {
|
||||
tracing::warn!(
|
||||
request_id = request_id.as_str(),
|
||||
operation,
|
||||
"resource diagnostic could not be recorded"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -934,7 +934,7 @@ pub struct ModelRequestFinalization {
|
||||
pub error: Option<GatewayError>,
|
||||
pub provider_error_code: Option<String>,
|
||||
/// Provider 返回的原始错误正文或 WebSocket close/error frame
|
||||
pub raw_upstream_error: Option<String>,
|
||||
pub error_details: Option<String>,
|
||||
pub failure_observation: ModelRequestFailureObservation,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub usage: Usage,
|
||||
|
||||
@@ -840,11 +840,11 @@ impl ResponseObservation {
|
||||
fn observe_event_timing(timings: &mut ModelRequestTimings, event: &GatewayEvent, elapsed_ms: u64) {
|
||||
timings.first_event_ms.get_or_insert(elapsed_ms);
|
||||
match event {
|
||||
GatewayEvent::ReasoningDelta(_) => {
|
||||
GatewayEvent::ReasoningDelta(delta) if !delta.text.is_empty() => {
|
||||
timings.first_reasoning_ms.get_or_insert(elapsed_ms);
|
||||
timings.first_token_ms.get_or_insert(elapsed_ms);
|
||||
}
|
||||
GatewayEvent::TextDelta(_) => {
|
||||
GatewayEvent::TextDelta(delta) if !delta.text.is_empty() => {
|
||||
timings.first_text_ms.get_or_insert(elapsed_ms);
|
||||
timings.first_token_ms.get_or_insert(elapsed_ms);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
//! 原因链与上游正文的受控诊断投影,不参与普通错误格式化
|
||||
|
||||
use super::{ErrorSource, RawUpstreamError};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ErrorDetails(String);
|
||||
|
||||
impl ErrorDetails {
|
||||
#[must_use]
|
||||
pub fn capture(
|
||||
source: Option<&ErrorSource>,
|
||||
upstream: Option<&RawUpstreamError>,
|
||||
redacted: bool,
|
||||
) -> Option<Self> {
|
||||
if source.is_none() && upstream.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(Self(
|
||||
serde_json::json!({
|
||||
"causes": source.map(ErrorSource::snapshot),
|
||||
"upstream": upstream.map(RawUpstreamError::as_str),
|
||||
"redacted": redacted,
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn into_string(self) -> String {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ErrorDetails {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("ErrorDetails(<restricted>)")
|
||||
}
|
||||
}
|
||||
+83
-20
@@ -1,5 +1,11 @@
|
||||
//! 网关核心使用的稳定错误分类
|
||||
|
||||
mod details;
|
||||
mod source;
|
||||
pub use details::ErrorDetails;
|
||||
|
||||
pub use source::ErrorSource;
|
||||
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::time::Duration;
|
||||
@@ -231,6 +237,8 @@ impl ProviderDiagnostic {
|
||||
while let Some(error) = source {
|
||||
if let Some(error) = error.downcast_ref::<std::io::Error>() {
|
||||
let code = match error.kind() {
|
||||
std::io::ErrorKind::NotFound => Some("io_not_found"),
|
||||
std::io::ErrorKind::PermissionDenied => Some("io_permission_denied"),
|
||||
std::io::ErrorKind::ConnectionRefused => Some("connection_refused"),
|
||||
std::io::ErrorKind::ConnectionReset => Some("tcp_reset"),
|
||||
std::io::ErrorKind::ConnectionAborted => Some("connection_aborted"),
|
||||
@@ -244,6 +252,15 @@ impl ProviderDiagnostic {
|
||||
if let Some(code) = code {
|
||||
self.message.push_str("; I/O cause: ");
|
||||
self.message.push_str(code);
|
||||
if let Some(os_code) = error.raw_os_error() {
|
||||
// 只重建操作系统错误,不能格式化可能包含 URL、路径或正文的自定义 I/O 错误
|
||||
use std::fmt::Write as _;
|
||||
let _ = write!(
|
||||
self.message,
|
||||
"; OS cause: {}",
|
||||
std::io::Error::from_raw_os_error(os_code)
|
||||
);
|
||||
}
|
||||
self.code = Some(code);
|
||||
break;
|
||||
}
|
||||
@@ -277,7 +294,7 @@ impl fmt::Debug for ProviderDiagnostic {
|
||||
|
||||
/// 运维错误详情中按原样展示的上游错误返回
|
||||
///
|
||||
/// 该值只接收响应方向的错误正文或 WebSocket close/error frame;不得放入请求
|
||||
/// 该值只接收响应方向的错误正文、WebSocket close/error frame 或上游适配插件 fault;不得放入请求
|
||||
/// Authorization、Cookie 或客户端输入
|
||||
/// 它会被持久化并由管理端显式返回,但普通
|
||||
/// `Debug`/`Display` 不输出正文,避免在非运维日志中重复扩散
|
||||
@@ -427,8 +444,9 @@ impl fmt::Debug for ClientVisibleUpstreamResponse {
|
||||
///
|
||||
/// 稳定诊断字段不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// code/request ID/response ID
|
||||
/// Adapter 若捕获到可能含 secret 的上下文,只能调用
|
||||
/// [`ProviderError::redact_sensitive_context`] 丢弃正文并留下脱敏标记
|
||||
/// 原始错误来源与上游响应正文仅由 [`ProviderError::error_details`] 提取到受控运维详情
|
||||
/// 请求方向的凭据等敏感上下文必须先移除,并通过
|
||||
/// [`ProviderError::redact_sensitive_context`] 留下脱敏标记
|
||||
///
|
||||
/// 唯一例外是 [`ProviderError::with_atomic_client_events`]:它只承载尚未交付客户端、
|
||||
/// 必须与本错误一起完成重试判断的协议事件
|
||||
@@ -450,6 +468,7 @@ pub struct ProviderError {
|
||||
retry_same_account: bool,
|
||||
sensitive_context_redacted: bool,
|
||||
diagnostic: Option<Box<ProviderDiagnostic>>,
|
||||
source: Option<ErrorSource>,
|
||||
raw_upstream_error: Option<Box<RawUpstreamError>>,
|
||||
client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
|
||||
client_visible_upstream_response: Option<Box<ClientVisibleUpstreamResponse>>,
|
||||
@@ -491,6 +510,7 @@ impl ProviderError {
|
||||
retry_same_account: false,
|
||||
sensitive_context_redacted: false,
|
||||
diagnostic: None,
|
||||
source: None,
|
||||
raw_upstream_error: None,
|
||||
client_visible_upstream_error: None,
|
||||
client_visible_upstream_response: None,
|
||||
@@ -507,7 +527,7 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为安全的上游错误 code
|
||||
/// 保留原始上游错误 code,不参与普通格式化或替代稳定分类
|
||||
#[must_use]
|
||||
pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().code = Some(code);
|
||||
@@ -845,6 +865,24 @@ impl ProviderError {
|
||||
self.diagnostic.as_deref()
|
||||
}
|
||||
|
||||
/// 包装底层失败;来源只供显式诊断访问,不改变分类和发送事实
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<crate::error::ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// 运维详情的唯一投影入口,来源链与上游正文分别保留,不进入公共响应或诊断包
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<String> {
|
||||
ErrorDetails::capture(
|
||||
self.source.as_ref(),
|
||||
self.raw_upstream_error(),
|
||||
self.sensitive_context_was_redacted(),
|
||||
)
|
||||
.map(ErrorDetails::into_string)
|
||||
}
|
||||
|
||||
/// 返回将由运维错误详情原样展示的上游错误返回
|
||||
#[must_use]
|
||||
pub fn raw_upstream_error(&self) -> Option<&RawUpstreamError> {
|
||||
@@ -882,7 +920,7 @@ impl ProviderError {
|
||||
|
||||
/// 复制可安全进入诊断和持久化边界的稳定错误事实
|
||||
///
|
||||
/// 请求局部的原始上游响应与尚未提交的客户端事件不会进入快照
|
||||
/// 保留原因链和上游错误详情;供协议透传的完整响应与尚未提交的客户端事件不进入快照
|
||||
#[must_use]
|
||||
pub fn stable_snapshot(&self) -> Self {
|
||||
Self {
|
||||
@@ -901,6 +939,7 @@ impl ProviderError {
|
||||
retry_same_account: self.retry_same_account,
|
||||
sensitive_context_redacted: self.sensitive_context_redacted,
|
||||
diagnostic: self.diagnostic.clone(),
|
||||
source: self.source.clone(),
|
||||
raw_upstream_error: self.raw_upstream_error.clone(),
|
||||
client_visible_upstream_error: self.client_visible_upstream_error.clone(),
|
||||
client_visible_upstream_response: None,
|
||||
@@ -916,10 +955,7 @@ impl fmt::Debug for ProviderError {
|
||||
.field("kind", &self.kind)
|
||||
.field("send_state", &self.send_state)
|
||||
.field("upstream_status", &self.upstream_status)
|
||||
.field(
|
||||
"upstream_code",
|
||||
&self.upstream_code().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field("upstream_code", &self.upstream_code().map(|_| "<opaque>"))
|
||||
.field(
|
||||
"upstream_request_id",
|
||||
&self.upstream_request_id().map(|_| "<classified-safe>"),
|
||||
@@ -986,7 +1022,11 @@ impl fmt::Display for ProviderError {
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ProviderError {}
|
||||
impl std::error::Error for ProviderError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
self.source.as_deref().map(|source| source as _)
|
||||
}
|
||||
}
|
||||
|
||||
/// 对客户端协议稳定的网关错误分类
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -1051,14 +1091,15 @@ impl GatewayErrorKind {
|
||||
}
|
||||
|
||||
/// 协议无关、可安全暴露的网关错误
|
||||
#[derive(Clone, PartialEq, Eq, Error)]
|
||||
#[derive(Clone, Error)]
|
||||
#[error("{message}")]
|
||||
pub struct GatewayError {
|
||||
kind: GatewayErrorKind,
|
||||
message: &'static str,
|
||||
diagnostic: Option<Box<ProviderDiagnostic>>,
|
||||
client_visible_upstream_error: Option<ClientVisibleUpstreamError>,
|
||||
client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
|
||||
client_details: Option<Box<GatewayClientErrorDetails>>,
|
||||
source: Option<ErrorSource>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
@@ -1077,9 +1118,21 @@ impl GatewayError {
|
||||
diagnostic: None,
|
||||
client_visible_upstream_error: None,
|
||||
client_details: None,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<ErrorDetails> {
|
||||
ErrorDetails::capture(self.source.as_ref(), None, false)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_client_code(mut self, code: &'static str) -> Self {
|
||||
self.client_details
|
||||
@@ -1189,7 +1242,7 @@ impl GatewayError {
|
||||
/// 附加只供请求方协议展示的结构化上游错误
|
||||
#[must_use]
|
||||
pub fn with_client_visible_upstream_error(mut self, error: ClientVisibleUpstreamError) -> Self {
|
||||
self.client_visible_upstream_error = Some(error);
|
||||
self.client_visible_upstream_error = Some(Box::new(error));
|
||||
self
|
||||
}
|
||||
|
||||
@@ -1216,7 +1269,7 @@ impl GatewayError {
|
||||
#[must_use]
|
||||
pub fn client_message(&self) -> &str {
|
||||
self.client_visible_upstream_error
|
||||
.as_ref()
|
||||
.as_deref()
|
||||
.map_or(self.message, ClientVisibleUpstreamError::message)
|
||||
}
|
||||
|
||||
@@ -1224,7 +1277,7 @@ impl GatewayError {
|
||||
#[must_use]
|
||||
pub fn client_error_type(&self) -> Option<&str> {
|
||||
self.client_visible_upstream_error
|
||||
.as_ref()
|
||||
.as_deref()
|
||||
.and_then(ClientVisibleUpstreamError::error_type)
|
||||
}
|
||||
|
||||
@@ -1232,7 +1285,7 @@ impl GatewayError {
|
||||
#[must_use]
|
||||
pub fn client_error_code(&self) -> Option<&str> {
|
||||
self.client_visible_upstream_error
|
||||
.as_ref()
|
||||
.as_deref()
|
||||
.and_then(ClientVisibleUpstreamError::code)
|
||||
.or_else(|| {
|
||||
self.client_details
|
||||
@@ -1277,18 +1330,28 @@ pub enum StoreErrorKind {
|
||||
InvalidData,
|
||||
}
|
||||
|
||||
/// Core port 返回的脱敏存储错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
/// Core port 返回的存储错误,普通格式化不展开原始来源
|
||||
#[derive(Debug, Clone, Error)]
|
||||
#[error("execution store failed: {kind:?}")]
|
||||
pub struct StoreError {
|
||||
kind: StoreErrorKind,
|
||||
source: Option<ErrorSource>,
|
||||
}
|
||||
|
||||
impl StoreError {
|
||||
/// 创建不携带数据库正文的 store 错误
|
||||
/// 创建只有分类的本地存储错误;包装实际异常使用 caused_by
|
||||
#[must_use]
|
||||
pub const fn new(kind: StoreErrorKind) -> Self {
|
||||
Self { kind }
|
||||
Self { kind, source: None }
|
||||
}
|
||||
|
||||
/// 包装已有存储错误并保留来源
|
||||
#[must_use]
|
||||
pub fn caused_by(kind: StoreErrorKind, source: impl Into<crate::error::ErrorSource>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
source: Some(source.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回稳定错误分类
|
||||
@@ -0,0 +1,164 @@
|
||||
//! 跨领域边界共享原始错误来源,不在普通格式化中展开详情
|
||||
|
||||
use std::{error::Error, fmt, ops::Deref, sync::Arc};
|
||||
|
||||
/// 保留底层错误的类型和来源链,避免领域端口依赖具体基础设施
|
||||
#[derive(Clone)]
|
||||
pub struct ErrorSource(Arc<dyn Error + Send + Sync>);
|
||||
|
||||
impl ErrorSource {
|
||||
#[must_use]
|
||||
pub fn new(error: impl Error + Send + Sync + 'static) -> Self {
|
||||
Self(Arc::new(error))
|
||||
}
|
||||
|
||||
/// 清理失败是附属事实,标准来源链仍指向最初失败
|
||||
#[must_use]
|
||||
pub fn with_cleanup(self, cleanup: impl Into<Self>) -> Self {
|
||||
Self::new(CleanupFailure {
|
||||
primary: Some(self),
|
||||
cleanup: cleanup.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 附属清理错误与可选的原始底层原因分别保留
|
||||
#[must_use]
|
||||
pub fn cleanup(primary: Option<Self>, cleanup: impl Into<Self>) -> Self {
|
||||
Self::new(CleanupFailure {
|
||||
primary,
|
||||
cleanup: cleanup.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 仅供受控错误详情落盘,限制链深度和总文本大小并显式标记缺口
|
||||
pub(super) fn snapshot(&self) -> serde_json::Value {
|
||||
snapshot(self.0.as_ref(), &mut 32, &mut (64 * 1024))
|
||||
}
|
||||
|
||||
/// 观测队列估算来源占用,包含附属清理失败;过深或循环的来源按超限处理
|
||||
#[must_use]
|
||||
pub fn estimated_chain_bytes(source: Option<&(dyn Error + 'static)>) -> usize {
|
||||
struct ByteCount(usize);
|
||||
impl fmt::Write for ByteCount {
|
||||
fn write_str(&mut self, value: &str) -> fmt::Result {
|
||||
self.0 = self.0.saturating_add(value.len());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut pending = Vec::from_iter(source);
|
||||
let mut bytes = ByteCount(0);
|
||||
for _ in 0..32 {
|
||||
let Some(error) = pending.pop() else {
|
||||
return bytes.0;
|
||||
};
|
||||
bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error));
|
||||
if fmt::write(&mut bytes, format_args!("{error}")).is_err() {
|
||||
return usize::MAX;
|
||||
}
|
||||
if let Some(cleanup) = error.downcast_ref::<CleanupFailure>() {
|
||||
pending.push(cleanup.cleanup.0.as_ref());
|
||||
}
|
||||
if let Some(source) = error.source() {
|
||||
pending.push(source);
|
||||
}
|
||||
}
|
||||
if pending.is_empty() {
|
||||
bytes.0
|
||||
} else {
|
||||
usize::MAX
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: Error + Send + Sync + 'static> From<E> for ErrorSource {
|
||||
fn from(error: E) -> Self {
|
||||
Self::new(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("operation failed and cleanup also failed")]
|
||||
struct CleanupFailure {
|
||||
#[source]
|
||||
primary: Option<ErrorSource>,
|
||||
cleanup: ErrorSource,
|
||||
}
|
||||
|
||||
fn snapshot(
|
||||
error: &(dyn Error + 'static),
|
||||
nodes: &mut usize,
|
||||
bytes: &mut usize,
|
||||
) -> serde_json::Value {
|
||||
let mut messages = Vec::new();
|
||||
let mut cleanup_sources = Vec::new();
|
||||
let mut source = Some(error);
|
||||
let mut truncated = false;
|
||||
while *nodes > 0 {
|
||||
let Some(error) = source else { break };
|
||||
*nodes -= 1;
|
||||
if let Some(failure) = error.downcast_ref::<CleanupFailure>() {
|
||||
cleanup_sources.push(failure.cleanup.0.as_ref());
|
||||
source = failure
|
||||
.primary
|
||||
.as_ref()
|
||||
.map(|primary| primary.0.as_ref() as &(dyn Error + 'static));
|
||||
continue;
|
||||
}
|
||||
let mut message = BoundedMessage {
|
||||
text: String::new(),
|
||||
remaining: *bytes,
|
||||
};
|
||||
if fmt::write(&mut message, format_args!("{error}")).is_err() {
|
||||
truncated = true;
|
||||
}
|
||||
*bytes = message.remaining;
|
||||
messages.push(message.text);
|
||||
source = error.source();
|
||||
if truncated || *bytes == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut result =
|
||||
serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()});
|
||||
if !cleanup_sources.is_empty() {
|
||||
result["cleanup"] = serde_json::Value::Array(
|
||||
cleanup_sources
|
||||
.into_iter()
|
||||
.map(|error| snapshot(error, nodes, bytes))
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
struct BoundedMessage {
|
||||
text: String,
|
||||
remaining: usize,
|
||||
}
|
||||
|
||||
impl fmt::Write for BoundedMessage {
|
||||
fn write_str(&mut self, value: &str) -> fmt::Result {
|
||||
let end = value.floor_char_boundary(self.remaining.min(value.len()));
|
||||
self.text.push_str(&value[..end]);
|
||||
self.remaining -= end;
|
||||
if end == value.len() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(fmt::Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for ErrorSource {
|
||||
type Target = dyn Error + Send + Sync;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
self.0.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ErrorSource {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ErrorSource(<restricted>)")
|
||||
}
|
||||
}
|
||||
@@ -479,7 +479,11 @@ impl UpstreamHttpVersion {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider transport 边界测得的阶段耗时
|
||||
/// Provider transport 边界测得的时间
|
||||
///
|
||||
/// `first_*_ms` 是相对 `AttemptContext::timing_started_at()` 的请求级偏移,
|
||||
/// 与 Core 总耗时使用同一起点并包含选号与重试等待;其余字段是各阶段独立耗时
|
||||
/// 首字边界由 Provider 协议定义,缺失时保留 `None`,不使用首包代替
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct ProviderResponseTimings {
|
||||
pub transport_decision_wait_ms: Option<u64>,
|
||||
|
||||
@@ -53,6 +53,7 @@ use task::{WorkerContribution, WorkerDefinitionError};
|
||||
#[derive(Clone)]
|
||||
pub struct CoreStorePorts {
|
||||
execution: Arc<dyn ExecutionStore>,
|
||||
diagnostics: Arc<dyn diagnostics::OperationalDiagnostics>,
|
||||
admissions: Arc<dyn ClientAdmissionPort>,
|
||||
admission_recovery: Arc<dyn ClientAdmissionRecoveryPort>,
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
@@ -77,9 +78,11 @@ impl CoreStorePorts {
|
||||
Arc<dyn SnapshotSubscriptionPort>,
|
||||
),
|
||||
client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
|
||||
diagnostics: Arc<dyn diagnostics::OperationalDiagnostics>,
|
||||
) -> Self {
|
||||
Self {
|
||||
execution,
|
||||
diagnostics,
|
||||
admissions,
|
||||
admission_recovery,
|
||||
continuation,
|
||||
@@ -201,6 +204,7 @@ fn build_execution_service(
|
||||
Arc::clone(&ports.admissions),
|
||||
Arc::clone(&ports.continuation),
|
||||
Arc::clone(&ports.client_api_key_usage),
|
||||
Arc::clone(&ports.diagnostics),
|
||||
);
|
||||
if let Some(budget) = &ports.budget {
|
||||
service = service.with_budget(Arc::clone(budget));
|
||||
|
||||
@@ -28,17 +28,36 @@ pub enum ProviderStoreErrorKind {
|
||||
}
|
||||
|
||||
/// Provider 存储端口的脱敏错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("provider store {operation} failed: {kind:?}")]
|
||||
pub struct ProviderStoreError {
|
||||
kind: ProviderStoreErrorKind,
|
||||
operation: &'static str,
|
||||
source: Option<crate::error::ErrorSource>,
|
||||
}
|
||||
|
||||
impl ProviderStoreError {
|
||||
#[must_use]
|
||||
pub const fn new(kind: ProviderStoreErrorKind, operation: &'static str) -> Self {
|
||||
Self { kind, operation }
|
||||
Self {
|
||||
kind,
|
||||
operation,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 包装基础设施失败,分类与原始来源分别保留
|
||||
#[must_use]
|
||||
pub fn caused_by(
|
||||
kind: ProviderStoreErrorKind,
|
||||
operation: &'static str,
|
||||
source: impl Into<crate::error::ErrorSource>,
|
||||
) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
operation,
|
||||
source: Some(source.into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -278,6 +297,7 @@ impl ProviderSessionBinding {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderSessionAlias {
|
||||
pub session_key: ProviderSessionAffinityKey,
|
||||
pub root_session_key: Option<ProviderSessionAffinityKey>,
|
||||
pub follow_only: bool,
|
||||
}
|
||||
|
||||
@@ -1091,6 +1111,38 @@ pub struct ProviderFreezePolicy {
|
||||
}
|
||||
|
||||
impl ProviderFreezePolicy {
|
||||
/// 管理写入与运行策略构造共用的冻结约束
|
||||
pub fn validate_values(
|
||||
threshold: u32,
|
||||
window_seconds: u64,
|
||||
duration_seconds: u64,
|
||||
probe_model: Option<&str>,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
(
|
||||
(2..=1_000).contains(&threshold),
|
||||
"account_auto_freeze_threshold",
|
||||
),
|
||||
(
|
||||
(60..=3_600).contains(&window_seconds),
|
||||
"account_auto_freeze_window_seconds",
|
||||
),
|
||||
(
|
||||
(300..=604_800).contains(&duration_seconds),
|
||||
"account_auto_freeze_duration_seconds",
|
||||
),
|
||||
(
|
||||
valid_optional_probe_model(probe_model),
|
||||
"account_auto_freeze_probe_model",
|
||||
),
|
||||
] {
|
||||
if !valid {
|
||||
return Err(field);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 边界与迁移 `0010_account_auto_freeze.sql` 的 check 约束一致;
|
||||
/// store 层写入前已校验,这里兜底防御越界配置
|
||||
pub fn try_new(
|
||||
@@ -1102,15 +1154,13 @@ impl ProviderFreezePolicy {
|
||||
probe_model: Option<String>,
|
||||
adaptive_concurrency: bool,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if !(2..=1_000).contains(&threshold)
|
||||
|| !(60..=3_600).contains(&window_seconds)
|
||||
|| !(300..=604_800).contains(&freeze_duration_seconds)
|
||||
|| probe_model.as_deref().is_some_and(|model| {
|
||||
model.is_empty()
|
||||
|| model.len() > 128
|
||||
|| model != model.trim()
|
||||
|| model.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
if Self::validate_values(
|
||||
threshold,
|
||||
window_seconds,
|
||||
freeze_duration_seconds,
|
||||
probe_model.as_deref(),
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
@@ -1219,20 +1269,27 @@ pub struct ProviderWarmupPolicy {
|
||||
}
|
||||
|
||||
impl ProviderWarmupPolicy {
|
||||
/// 保存设置与创建预热策略使用相同的时间表和模型约束
|
||||
pub fn validate_values(
|
||||
enabled: bool,
|
||||
schedule_time: &str,
|
||||
model: Option<&str>,
|
||||
) -> Result<(), &'static str> {
|
||||
if !valid_warmup_schedule_time(schedule_time) {
|
||||
return Err("account_warmup_schedule_time");
|
||||
}
|
||||
if (enabled && model.is_none()) || !valid_optional_probe_model(model) {
|
||||
return Err("account_warmup_model");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
schedule_time: String,
|
||||
model: Option<String>,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if !valid_warmup_schedule_time(&schedule_time)
|
||||
|| (enabled && model.is_none())
|
||||
|| model.as_deref().is_some_and(|m| {
|
||||
m.is_empty()
|
||||
|| m.len() > 128
|
||||
|| m != m.trim()
|
||||
|| m.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
{
|
||||
if Self::validate_values(enabled, &schedule_time, model.as_deref()).is_err() {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate warmup policy",
|
||||
@@ -1475,6 +1532,7 @@ pub struct ProviderStorePorts {
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
oauth_pending: Arc<dyn OAuthPendingFlowPort>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
}
|
||||
|
||||
impl ProviderStorePorts {
|
||||
@@ -1492,6 +1550,7 @@ impl ProviderStorePorts {
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
oauth_pending: Arc<dyn OAuthPendingFlowPort>,
|
||||
diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
@@ -1505,9 +1564,15 @@ impl ProviderStorePorts {
|
||||
cooldowns,
|
||||
runtime_policy,
|
||||
oauth_pending,
|
||||
diagnostics,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn diagnostics(&self) -> Arc<dyn crate::diagnostics::OperationalDiagnostics> {
|
||||
self.diagnostics.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn accounts(&self) -> Arc<dyn ProviderAccountStore> {
|
||||
Arc::clone(&self.accounts)
|
||||
@@ -1569,3 +1634,12 @@ impl fmt::Debug for ProviderStorePorts {
|
||||
formatter.write_str("ProviderStorePorts([CAPABILITIES])")
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_optional_probe_model(model: Option<&str>) -> bool {
|
||||
model.is_none_or(|model| {
|
||||
!model.is_empty()
|
||||
&& model.len() <= 128
|
||||
&& model == model.trim()
|
||||
&& !model.bytes().any(|byte| byte.is_ascii_control())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -24,11 +24,7 @@ use crate::account::{AccountSelectionPolicy, FastMode};
|
||||
use crate::operation::{CapabilityRequirements, Feature, OperationKind};
|
||||
use crate::validation::{IdentifierError, RoutingError, validate_text};
|
||||
|
||||
const MAX_REQUEST_ATTEMPTS: u32 = 32;
|
||||
|
||||
/// 单请求内允许的换号次数上限,选中账号与上一 attempt 不同的路由 attempt 记一次
|
||||
/// 初始账号和同账号重试不计入换号次数,所有尝试仍受总路由预算限制
|
||||
pub(crate) const MAX_ACCOUNT_ROTATION_ATTEMPTS: u32 = 3;
|
||||
const MAX_REQUEST_ATTEMPTS: u32 = crate::account::MAX_ACCOUNT_ROTATIONS + 1;
|
||||
|
||||
/// 客户端请求中的模型名称
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
|
||||
|
||||
@@ -454,7 +454,10 @@ impl ScheduledTask for RuntimeSnapshotReconciliationTask {
|
||||
.refresh_with_mode(RefreshMode::Reconcile)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|_| WorkerTaskError::safe("runtime snapshot reconciliation failed"))
|
||||
.map_err(|source| {
|
||||
WorkerTaskError::safe("runtime snapshot reconciliation failed")
|
||||
.with_source(source)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use super::{InvalidSettings, SettingsValues};
|
||||
use crate::{
|
||||
account::{AccountConcurrency, AccountSelectionPolicy, RotationStrategy},
|
||||
concurrency::ConcurrencyQueuePolicy,
|
||||
policy::{CodexClientMinVersions, CodexClientVersion},
|
||||
policy::CodexClientMinVersions,
|
||||
};
|
||||
|
||||
// 持久快照和请求覆盖共用同一编译结果,避免参数校验与派生规则分叉
|
||||
@@ -22,34 +22,24 @@ pub(crate) struct CompiledSettings {
|
||||
|
||||
impl CompiledSettings {
|
||||
pub(crate) fn new(settings: SettingsValues) -> Result<Self, InvalidSettings> {
|
||||
if settings.max_waiting_per_key > 1_000
|
||||
|| settings.max_waiting_per_account > 1_000
|
||||
|| !(1..=120).contains(&settings.concurrency_wait_timeout_seconds)
|
||||
{
|
||||
return Err(InvalidSettings);
|
||||
}
|
||||
super::validate_request_limits(
|
||||
settings.max_waiting_per_key,
|
||||
settings.max_waiting_per_account,
|
||||
settings.concurrency_wait_timeout_seconds,
|
||||
settings.max_account_rotations,
|
||||
settings.openai_session_affinity_ttl_hours,
|
||||
)
|
||||
.map_err(|_| InvalidSettings)?;
|
||||
let queue_timeout =
|
||||
Duration::from_secs(u64::from(settings.concurrency_wait_timeout_seconds));
|
||||
let responses_max_decompressed_body_bytes =
|
||||
isize::try_from(settings.responses_max_decompressed_body_bytes)
|
||||
.ok()
|
||||
.and_then(|bytes| usize::try_from(bytes).ok())
|
||||
.and_then(std::num::NonZeroUsize::new)
|
||||
.ok_or(InvalidSettings)?;
|
||||
let min_codex_client_versions = CodexClientMinVersions::new(
|
||||
settings
|
||||
.min_codex_desktop_version
|
||||
.as_deref()
|
||||
.map(CodexClientVersion::parse)
|
||||
.transpose()
|
||||
.map_err(|_| InvalidSettings)?,
|
||||
settings
|
||||
.min_codex_cli_version
|
||||
.as_deref()
|
||||
.map(CodexClientVersion::parse)
|
||||
.transpose()
|
||||
.map_err(|_| InvalidSettings)?,
|
||||
);
|
||||
super::response_body_limit(settings.responses_max_decompressed_body_bytes)
|
||||
.map_err(|_| InvalidSettings)?;
|
||||
let min_codex_client_versions = super::client_min_versions(
|
||||
settings.min_codex_desktop_version.as_deref(),
|
||||
settings.min_codex_cli_version.as_deref(),
|
||||
)
|
||||
.map_err(|_| InvalidSettings)?;
|
||||
let strategy =
|
||||
RotationStrategy::parse(&settings.rotation_strategy).ok_or(InvalidSettings)?;
|
||||
let account_selection_policy = AccountSelectionPolicy::new(
|
||||
@@ -58,6 +48,11 @@ impl CompiledSettings {
|
||||
Duration::from_millis(settings.request_interval_ms),
|
||||
)
|
||||
.with_openai_guardian_reserved_concurrency(settings.openai_guardian_reserved_concurrency)
|
||||
.with_openai_account_affinity(settings.openai_account_affinity)
|
||||
.with_max_account_rotations(settings.max_account_rotations)
|
||||
.with_openai_session_affinity_ttl(Duration::from_secs(
|
||||
u64::from(settings.openai_session_affinity_ttl_hours) * 3600,
|
||||
))
|
||||
.with_smart_scheduling(settings.smart_scheduling)
|
||||
.with_queue(ConcurrencyQueuePolicy {
|
||||
max_waiting: settings.max_waiting_per_account,
|
||||
|
||||
@@ -15,7 +15,9 @@ use crate::routing::RuntimeSnapshot;
|
||||
|
||||
pub(crate) mod compiled;
|
||||
mod values;
|
||||
pub use values::SettingsValues;
|
||||
pub use values::{
|
||||
SettingsValues, client_min_versions, response_body_limit, validate_request_limits,
|
||||
};
|
||||
|
||||
/// 一次模型调用的有效设置;改写只影响当前请求,不发布配置或修改持久化 revision
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
||||
@@ -17,6 +17,9 @@ pub struct SettingsValues {
|
||||
pub(crate) max_waiting_per_account: u32,
|
||||
pub(crate) concurrency_wait_timeout_seconds: u32,
|
||||
pub(crate) openai_guardian_reserved_concurrency: u32,
|
||||
pub(crate) openai_account_affinity: crate::account::AccountAffinity,
|
||||
pub(crate) max_account_rotations: u32,
|
||||
pub(crate) openai_session_affinity_ttl_hours: u32,
|
||||
pub(crate) responses_max_decompressed_body_bytes: u64,
|
||||
pub(crate) request_interval_ms: u64,
|
||||
pub(crate) smart_scheduling: crate::account::SmartSchedulingConfig,
|
||||
@@ -27,6 +30,27 @@ pub struct SettingsValues {
|
||||
}
|
||||
|
||||
impl SettingsValues {
|
||||
#[must_use]
|
||||
pub const fn with_openai_session_affinity_ttl_hours(mut self, hours: u32) -> Self {
|
||||
self.openai_session_affinity_ttl_hours = hours;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_openai_account_affinity(
|
||||
mut self,
|
||||
affinity: crate::account::AccountAffinity,
|
||||
) -> Self {
|
||||
self.openai_account_affinity = affinity;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_max_account_rotations(mut self, rotations: u32) -> Self {
|
||||
self.max_account_rotations = rotations;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_openai_guardian_reserved_concurrency(mut self, reserved: u32) -> Self {
|
||||
self.openai_guardian_reserved_concurrency = reserved;
|
||||
@@ -114,6 +138,9 @@ impl SettingsValues {
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: crate::account::AccountAffinity::default(),
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
request_interval_ms,
|
||||
smart_scheduling: crate::account::SmartSchedulingConfig::default(),
|
||||
@@ -140,3 +167,60 @@ impl SettingsValues {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求排队与账号选择的数值约束;返回不合法的设置字段
|
||||
pub fn validate_request_limits(
|
||||
max_waiting_per_key: u32,
|
||||
max_waiting_per_account: u32,
|
||||
timeout_seconds: u32,
|
||||
max_account_rotations: u32,
|
||||
affinity_ttl_hours: u32,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
(max_waiting_per_key <= 1_000, "max_waiting_per_key"),
|
||||
(max_waiting_per_account <= 1_000, "max_waiting_per_account"),
|
||||
(
|
||||
(1..=120).contains(&timeout_seconds),
|
||||
"concurrency_wait_timeout_seconds",
|
||||
),
|
||||
(
|
||||
max_account_rotations <= crate::account::MAX_ACCOUNT_ROTATIONS,
|
||||
"max_account_rotations",
|
||||
),
|
||||
(
|
||||
(1..=crate::account::MAX_SESSION_AFFINITY_TTL_HOURS).contains(&affinity_ttl_hours),
|
||||
"openai_session_affinity_ttl_hours",
|
||||
),
|
||||
] {
|
||||
if !valid {
|
||||
return Err(field);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 解压上限必须能作为非零缓冲区长度使用
|
||||
pub fn response_body_limit(bytes: u64) -> Result<std::num::NonZeroUsize, &'static str> {
|
||||
isize::try_from(bytes)
|
||||
.ok()
|
||||
.and_then(|bytes| usize::try_from(bytes).ok())
|
||||
.and_then(std::num::NonZeroUsize::new)
|
||||
.ok_or("responses_max_decompressed_body_bytes")
|
||||
}
|
||||
|
||||
/// 管理写入与请求覆盖共用客户端最低版本解析
|
||||
pub fn client_min_versions(
|
||||
desktop: Option<&str>,
|
||||
cli: Option<&str>,
|
||||
) -> Result<crate::policy::CodexClientMinVersions, &'static str> {
|
||||
use crate::policy::{CodexClientMinVersions, CodexClientVersion};
|
||||
Ok(CodexClientMinVersions::new(
|
||||
desktop
|
||||
.map(CodexClientVersion::parse)
|
||||
.transpose()
|
||||
.map_err(|_| "min_codex_desktop_version")?,
|
||||
cli.map(CodexClientVersion::parse)
|
||||
.transpose()
|
||||
.map_err(|_| "min_codex_cli_version")?,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -485,10 +485,11 @@ pub enum WorkerDefinitionError {
|
||||
}
|
||||
|
||||
/// 不暴露基础设施原文或 lease resource 的错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct WorkerLeaseError {
|
||||
message: String,
|
||||
source: Option<crate::error::ErrorSource>,
|
||||
}
|
||||
|
||||
impl WorkerLeaseError {
|
||||
@@ -496,6 +497,7 @@ impl WorkerLeaseError {
|
||||
pub fn safe(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
message: message.into(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -503,13 +505,24 @@ impl WorkerLeaseError {
|
||||
pub fn as_safe_str(&self) -> &str {
|
||||
&self.message
|
||||
}
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<crate::error::ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<crate::error::ErrorDetails> {
|
||||
crate::error::ErrorDetails::capture(self.source.as_ref(), None, false)
|
||||
}
|
||||
}
|
||||
|
||||
/// 不暴露 Provider 原文或存储细节的后台任务错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct WorkerTaskError {
|
||||
message: String,
|
||||
source: Option<crate::error::ErrorSource>,
|
||||
}
|
||||
|
||||
impl WorkerTaskError {
|
||||
@@ -517,6 +530,7 @@ impl WorkerTaskError {
|
||||
pub fn safe(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
message: message.into(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -524,6 +538,25 @@ impl WorkerTaskError {
|
||||
pub fn as_safe_str(&self) -> &str {
|
||||
&self.message
|
||||
}
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<crate::error::ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<crate::error::ErrorDetails> {
|
||||
crate::error::ErrorDetails::capture(self.source.as_ref(), None, false)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_cleanup(mut self, cleanup: impl Into<crate::error::ErrorSource>) -> Self {
|
||||
self.source = Some(crate::error::ErrorSource::cleanup(
|
||||
self.source.take(),
|
||||
cleanup,
|
||||
));
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_owner(owner: &str) -> bool {
|
||||
|
||||
@@ -6,6 +6,8 @@ use std::{
|
||||
};
|
||||
|
||||
use gateway_core::diagnostics::{TraceContext, body_fingerprint};
|
||||
use gateway_core::error::{ProviderDiagnostic, ProviderError, ProviderErrorKind, RawUpstreamError};
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
use serde_json::json;
|
||||
use tracing::{
|
||||
Event, Metadata, Subscriber,
|
||||
@@ -47,6 +49,55 @@ fn bounded_history_preserves_start_failure_and_final_result() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_failure_trace_keeps_classification_when_diagnostic_is_truncated() {
|
||||
let trace = TraceContext::new("req_export");
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_diagnostic(
|
||||
ProviderDiagnostic::new("Safe diagnostic".repeat(500))
|
||||
.with_classification("prepare", "account_proxy_invalid"),
|
||||
)
|
||||
.with_raw_upstream_error(RawUpstreamError::new("PRIVATE_RAW"));
|
||||
trace.attempt(3).record_provider_failure(&error);
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let data = &snapshot["events"][0]["data"];
|
||||
assert_eq!(data["kind"], "unavailable");
|
||||
assert_eq!(data["sendState"], "not_sent");
|
||||
assert_eq!(data["diagnostic"]["stage"], "prepare");
|
||||
assert_eq!(data["diagnostic"]["code"], "account_proxy_invalid");
|
||||
assert_eq!(data["diagnostic"]["truncated"], true);
|
||||
assert!(!snapshot.to_string().contains("PRIVATE_RAW"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_failure_keeps_classification_and_marks_truncated_message() {
|
||||
let trace = TraceContext::new("req_oversized_failure");
|
||||
trace.attempt(2).record("attempt.failed", json!({
|
||||
"kind": "transport", "sendState": "not_sent", "upstreamStatus": 502,
|
||||
"diagnostic": {"stage": "connect", "code": "connection_refused", "message": "连接被拒绝".repeat(2000)},
|
||||
"rawError": {"summary": "x".repeat(10_000)},
|
||||
}));
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let event = &snapshot["events"][0];
|
||||
assert_eq!(event["attemptIndex"], 2);
|
||||
let data = &event["data"];
|
||||
assert_eq!(data["truncated"], true);
|
||||
assert_eq!(data["kind"], "transport");
|
||||
assert_eq!(data["sendState"], "not_sent");
|
||||
assert_eq!(data["upstreamStatus"], 502);
|
||||
assert_eq!(data["diagnostic"]["stage"], "connect");
|
||||
assert_eq!(data["diagnostic"]["code"], "connection_refused");
|
||||
assert_eq!(data["diagnostic"]["truncated"], true);
|
||||
assert!(
|
||||
data["diagnostic"]["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("连接被拒绝")
|
||||
);
|
||||
assert!(serde_json::to_vec(data).unwrap().len() <= 4096);
|
||||
assert!(data.get("rawError").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_attempts_and_exchanges_coalesce_delta_without_losing_metadata() {
|
||||
let trace = TraceContext::new("req_trace");
|
||||
@@ -314,6 +365,18 @@ fn ordinary_trace_logs_and_serialized_snapshots_share_the_sanitized_capture_boun
|
||||
);
|
||||
exchange.capture_named_event("upstream.event", b"{}", Some("PRIVATE_SSE_EVENT"));
|
||||
exchange.wire_event("openai", Some("PRIVATE_WIRE_EVENT"));
|
||||
exchange.record_provider_failure(
|
||||
&ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_upstream_code(gateway_core::error::OpaqueUpstreamValue::new(
|
||||
"PRIVATE_CODE",
|
||||
))
|
||||
.with_raw_upstream_error(RawUpstreamError::new("PRIVATE_UPSTREAM_BODY"))
|
||||
.with_source(std::io::Error::other("PRIVATE_NATIVE_CAUSE"))
|
||||
.with_diagnostic(
|
||||
ProviderDiagnostic::new("upstream rejected request")
|
||||
.with_classification("upstream", "upstream_failure"),
|
||||
),
|
||||
);
|
||||
exchange.headers(
|
||||
"upstream.response.headers",
|
||||
json!({"status": 429}),
|
||||
|
||||
@@ -72,7 +72,7 @@ fn client_admission_startup_recovery_should_fail_closed_at_each_boundary() {
|
||||
assert!(
|
||||
restore_client_admission_startup(
|
||||
&RecoveryStore::success(0),
|
||||
&ScriptedRecoveries::new(Err(ClientAdmissionError)),
|
||||
&ScriptedRecoveries::new(Err(ClientAdmissionError(None))),
|
||||
&RecordingAdmissions::default(),
|
||||
now,
|
||||
)
|
||||
@@ -212,7 +212,7 @@ impl ClientAdmissionPort for RecordingAdmissions {
|
||||
) -> BoxFuture<'_, Result<ClientAdmissionRestoreResult, ClientAdmissionError>> {
|
||||
Box::pin(async move {
|
||||
if self.fail_restore {
|
||||
return Err(ClientAdmissionError);
|
||||
return Err(ClientAdmissionError(None));
|
||||
}
|
||||
self.restored.lock().expect("restored").push(recovery);
|
||||
Ok(ClientAdmissionRestoreResult {
|
||||
|
||||
@@ -65,6 +65,7 @@ struct FinalState {
|
||||
image_output_tokens: Option<u64>,
|
||||
image_generation_succeeded: Option<bool>,
|
||||
provider_error_code: Option<String>,
|
||||
error_details: Option<String>,
|
||||
retry_after_ms: Option<u64>,
|
||||
latency_ms: Option<u64>,
|
||||
client_response_id: Option<String>,
|
||||
@@ -232,6 +233,7 @@ impl ExecutionStore for FakeStore {
|
||||
image_output_tokens: finalization.usage.image_output_tokens,
|
||||
image_generation_succeeded: finalization.image_generation_succeeded,
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
error_details: finalization.error_details,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
latency_ms: finalization.timings.latency_ms,
|
||||
client_response_id: finalization.client_response_id,
|
||||
@@ -297,6 +299,7 @@ struct ScriptedProvider {
|
||||
profile_generation: AtomicUsize,
|
||||
default_profile_calls: AtomicUsize,
|
||||
default_profile: Mutex<Option<gateway_core::account::OpaqueProviderData>>,
|
||||
default_profile_error: Mutex<Option<ProviderError>>,
|
||||
scripts: Mutex<VecDeque<Script>>,
|
||||
contexts: Mutex<Vec<AttemptContext>>,
|
||||
operations: Mutex<Vec<Operation>>,
|
||||
@@ -323,6 +326,7 @@ impl ScriptedProvider {
|
||||
profile_generation: AtomicUsize::new(1),
|
||||
default_profile_calls: AtomicUsize::new(0),
|
||||
default_profile: Mutex::new(None),
|
||||
default_profile_error: Mutex::new(None),
|
||||
scripts: Mutex::new(scripts.into()),
|
||||
contexts: Mutex::new(Vec::new()),
|
||||
operations: Mutex::new(Vec::new()),
|
||||
@@ -349,6 +353,9 @@ impl Provider for ScriptedProvider {
|
||||
&self,
|
||||
) -> Result<Option<gateway_core::account::OpaqueProviderData>, ProviderError> {
|
||||
self.default_profile_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(error) = self.default_profile_error.lock().unwrap().take() {
|
||||
return Err(error);
|
||||
}
|
||||
Ok(self.default_profile.lock().unwrap().clone())
|
||||
}
|
||||
|
||||
@@ -606,6 +613,42 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
|
||||
]
|
||||
}
|
||||
|
||||
const ROTATION_TEST_ACCOUNTS: [&str; 33] = [
|
||||
"acct_rotation_0",
|
||||
"acct_rotation_1",
|
||||
"acct_rotation_2",
|
||||
"acct_rotation_3",
|
||||
"acct_rotation_4",
|
||||
"acct_rotation_5",
|
||||
"acct_rotation_6",
|
||||
"acct_rotation_7",
|
||||
"acct_rotation_8",
|
||||
"acct_rotation_9",
|
||||
"acct_rotation_10",
|
||||
"acct_rotation_11",
|
||||
"acct_rotation_12",
|
||||
"acct_rotation_13",
|
||||
"acct_rotation_14",
|
||||
"acct_rotation_15",
|
||||
"acct_rotation_16",
|
||||
"acct_rotation_17",
|
||||
"acct_rotation_18",
|
||||
"acct_rotation_19",
|
||||
"acct_rotation_20",
|
||||
"acct_rotation_21",
|
||||
"acct_rotation_22",
|
||||
"acct_rotation_23",
|
||||
"acct_rotation_24",
|
||||
"acct_rotation_25",
|
||||
"acct_rotation_26",
|
||||
"acct_rotation_27",
|
||||
"acct_rotation_28",
|
||||
"acct_rotation_29",
|
||||
"acct_rotation_30",
|
||||
"acct_rotation_31",
|
||||
"acct_rotation_32",
|
||||
];
|
||||
|
||||
fn plan(operation: &Operation) -> RoutingPlan {
|
||||
plan_with_policy(
|
||||
operation,
|
||||
@@ -680,6 +723,7 @@ fn plan_with_profiles(
|
||||
"acct_wrong",
|
||||
]
|
||||
.into_iter()
|
||||
.chain(ROTATION_TEST_ACCOUNTS)
|
||||
.map(|id| {
|
||||
(
|
||||
ProviderAccountId::new(id).expect("account"),
|
||||
@@ -699,6 +743,17 @@ fn plan_with_profiles(
|
||||
None,
|
||||
)
|
||||
.with_smart_scheduling(account_selection_policy.smart_scheduling())
|
||||
.with_max_account_rotations(account_selection_policy.max_account_rotations())
|
||||
.with_openai_account_affinity(account_selection_policy.openai_account_affinity())
|
||||
.with_openai_session_affinity_ttl_hours(
|
||||
u32::try_from(
|
||||
account_selection_policy
|
||||
.openai_session_affinity_ttl()
|
||||
.as_secs()
|
||||
/ 3600,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.with_request_location(request_location, true),
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
@@ -1454,6 +1509,63 @@ fn empty_tool_call_delta_should_not_preempt_provider_first_token_timing() {
|
||||
assert_eq!(state.finalizations[0].first_token_ms, Some(37));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_content_deltas_do_not_create_first_token_timings() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let mut items = complete_stream(None);
|
||||
items.splice(
|
||||
1..1,
|
||||
[
|
||||
Ok(GatewayEvent::ContentAdded(ContentItem::new(
|
||||
0,
|
||||
ContentKind::Text,
|
||||
))),
|
||||
Ok(GatewayEvent::TextDelta(gateway_core::event::TextDelta {
|
||||
content_index: 0,
|
||||
text: String::new(),
|
||||
})),
|
||||
Ok(GatewayEvent::ContentAdded(ContentItem::new(
|
||||
1,
|
||||
ContentKind::Reasoning,
|
||||
))),
|
||||
Ok(GatewayEvent::ReasoningDelta(
|
||||
gateway_core::event::ReasoningDelta {
|
||||
content_index: 1,
|
||||
text: String::new(),
|
||||
},
|
||||
)),
|
||||
],
|
||||
);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_observed",
|
||||
items,
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
|
||||
let state = store.state.lock().unwrap();
|
||||
let timings = &state.finalizations[0];
|
||||
assert!(timings.first_event_ms.is_some());
|
||||
assert_eq!(
|
||||
(
|
||||
timings.first_token_ms,
|
||||
timings.first_text_ms,
|
||||
timings.first_reasoning_ms
|
||||
),
|
||||
(None, None, None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_wire_event_before_response_identity_is_discarded_with_retried_attempt() {
|
||||
let operation = generate_operation();
|
||||
@@ -1531,7 +1643,7 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
.into_iter()
|
||||
.map(canonical_provider_event),
|
||||
);
|
||||
let (coordinator, store, _) = coordinator(vec![
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
items: vec![
|
||||
@@ -1607,6 +1719,12 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
assert_eq!(finalization.connect_ms, None);
|
||||
assert_eq!(finalization.headers_ms, None);
|
||||
assert_eq!(finalization.provider_processing_ms, None);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[0].timing_started_at(),
|
||||
contexts[1].timing_started_at()
|
||||
);
|
||||
assert!(
|
||||
finalization
|
||||
.first_event_ms
|
||||
@@ -2786,6 +2904,7 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_source(std::io::Error::other("ORIGINAL_RATE_LIMIT_CAUSE"))
|
||||
.with_status(429)
|
||||
.with_upstream_code(OpaqueUpstreamValue::new("rate_limit_exceeded"))
|
||||
.with_retry_after(Duration::from_secs(30))
|
||||
@@ -2831,6 +2950,12 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
|
||||
state.finalizations[0].provider_error_code.as_deref(),
|
||||
Some("rate_limit_exceeded")
|
||||
);
|
||||
let details: Value =
|
||||
serde_json::from_str(state.finalizations[0].error_details.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
details["causes"]["messages"],
|
||||
json!(["ORIGINAL_RATE_LIMIT_CAUSE"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4842,6 +4967,59 @@ fn global_location_and_group_fast_policy_reach_every_account_retry() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_preparation_failure_is_traced_and_finalized_without_starting_an_attempt() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![]);
|
||||
*provider.default_profile_error.lock().unwrap() = Some(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_diagnostic(
|
||||
gateway_core::error::ProviderDiagnostic::new("Profile release unavailable")
|
||||
.with_classification("prepare", "request_profile_release_unavailable"),
|
||||
),
|
||||
);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(block_on(session.collect_uncommitted()).is_err());
|
||||
assert!(provider.contexts.lock().unwrap().is_empty());
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.finalizations.len(), 1);
|
||||
let trace: Value = serde_json::from_str(
|
||||
state.finalizations[0]
|
||||
.diagnostic_trace_json
|
||||
.as_deref()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let events = trace["events"].as_array().unwrap();
|
||||
let failure = events
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "attempt.failed")
|
||||
.unwrap();
|
||||
assert_eq!(failure["attemptIndex"], 0);
|
||||
assert_eq!(
|
||||
failure["data"]["diagnostic"]["code"],
|
||||
"request_profile_release_unavailable"
|
||||
);
|
||||
assert_eq!(
|
||||
failure["data"]["diagnostic"]["message"],
|
||||
"Profile release unavailable"
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.all(|event| event["stage"] != "attempt.started")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_resolved_profile_is_frozen_across_account_retries() {
|
||||
use gateway_core::account::OpaqueProviderData;
|
||||
@@ -5505,3 +5683,138 @@ fn prepare_failure_can_forbid_provider_fallback_without_affecting_ordinary_empty
|
||||
assert_eq!(xai.contexts.lock().unwrap().len(), usize::from(!prohibited));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_rotation_budget_limits_actual_account_changes() {
|
||||
for budget in [0, 1, 31] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan_with_policy(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).unwrap(),
|
||||
Duration::ZERO,
|
||||
)
|
||||
.with_max_account_rotations(budget)
|
||||
.with_openai_session_affinity_ttl(Duration::from_secs(168 * 3600)),
|
||||
);
|
||||
let (coordinator, store, provider) = coordinator(
|
||||
ROTATION_TEST_ACCOUNTS
|
||||
.iter()
|
||||
.map(|id| Script::Stream {
|
||||
account_id: id,
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(block_on(session.collect_uncommitted()).is_err());
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), (budget + 1) as usize);
|
||||
assert!(contexts.iter().all(|context| {
|
||||
context.account_selection_policy().max_account_rotations() == budget
|
||||
&& context
|
||||
.account_selection_policy()
|
||||
.openai_session_affinity_ttl()
|
||||
== Duration::from_secs(168 * 3600)
|
||||
}));
|
||||
assert_eq!(
|
||||
store.state.lock().unwrap().finalizations[0].attempt_count,
|
||||
budget + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn larger_rotation_budget_reaches_a_healthy_account_after_four_failures() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan_with_policy(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).unwrap(),
|
||||
Duration::ZERO,
|
||||
)
|
||||
.with_max_account_rotations(4),
|
||||
);
|
||||
let mut scripts = ROTATION_TEST_ACCOUNTS[..4]
|
||||
.iter()
|
||||
.map(|id| Script::Stream {
|
||||
account_id: id,
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_replay_safe())],
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
scripts.push(Script::Stream {
|
||||
account_id: ROTATION_TEST_ACCOUNTS[4],
|
||||
items: complete_stream(None),
|
||||
});
|
||||
let (coordinator, _, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_rotation_budget_still_allows_retrying_the_same_account() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan_with_policy(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).unwrap(),
|
||||
Duration::ZERO,
|
||||
)
|
||||
.with_max_account_rotations(0),
|
||||
);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(transient_rejection())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[1].required_account(),
|
||||
Some(&ProviderAccountId::new("acct_first").unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user