From 2abbfb1fac6d8202ac1e7b0e9fb3c8dc611ad727 Mon Sep 17 00:00:00 2001 From: Zyy Date: Mon, 16 Nov 2026 17:45:40 +0800 Subject: [PATCH] fix(openai): honor account rotation limits on WebSocket connection failures Reserve a connection attempt for another eligible account when a new WebSocket request repeatedly fails before sending. Use the same rotation budget as HTTP while preserving session and replay restrictions. Co-authored-by: Codex --- .../gateway-core/src/engine/coordinator.rs | 25 +++--- backend/crates/gateway-core/src/error/mod.rs | 24 ++++- .../providers/openai/src/provider/failure.rs | 43 +++++++-- .../openai/tests/provider/contract/mod.rs | 5 ++ .../provider/contract/selection_retry.rs | 88 ++++++++++++++++++- docs/api.md | 9 +- docs/architecture.md | 7 +- 7 files changed, 172 insertions(+), 29 deletions(-) diff --git a/backend/crates/gateway-core/src/engine/coordinator.rs b/backend/crates/gateway-core/src/engine/coordinator.rs index ad12c29b0..4804ff35d 100644 --- a/backend/crates/gateway-core/src/engine/coordinator.rs +++ b/backend/crates/gateway-core/src/engine/coordinator.rs @@ -1019,6 +1019,10 @@ where } } + fn has_account_rotation_budget(&self) -> bool { + self.account_rotations < self.plan.account_selection_policy().max_account_rotations() + } + async fn prepare_attempt(&mut self) -> Result, EngineError> { let pending_retry = self.pending_retry.take(); let is_transport_recovery = pending_retry @@ -1264,7 +1268,7 @@ where ) // 跨 Provider 候选推进必然换号(账号行按 Provider 隔离), // 预算耗尽后不再推进,交回容量类失败的原有终态语义。 - && self.account_rotations < self.plan.account_selection_policy().max_account_rotations() + && self.has_account_rotation_budget() && self.advance_provider_candidate(); let retryable = self .apply_retry_policy(super::policy::RetryFacts { @@ -1668,10 +1672,8 @@ where && self.routing_attempts < self.plan.max_attempts().get(); // 只有尚未发送的逻辑请求进入新增策略;已有发送后安全恢复保持原有路由规则 let connection_retry_requested = self.current_send_state() == UpstreamSendState::NotSent - && matches!( - error.pre_delivery_retry(), - Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. }) - ); + && self.continuation_attempt == ContinuationAttempt::None + && error.allows_connection_rotation(); // 服务器建议不可截短 // 超出上限,或上游拒绝继续重试当前账号时,跳过同账号等待 // 较短的 Retry-After 也直接沿用安全换号或原错误出口 @@ -1729,9 +1731,8 @@ where let can_rotate_connection = connection_retry_requested && self.account_selection.required_account().is_none() && self.routing_attempts < self.plan.max_attempts().get() - && self.account_rotations - < self.plan.account_selection_policy().max_account_rotations(); - // A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening + && self.has_account_rotation_budget(); + // A 先恢复一次,再利用预留预算重选;下一次传输仍由 Provider 的恢复策略决定 let connection_rotation = transport_recovery .filter(|_| can_rotate_connection && current.connection_rotation_reserved); let reserve_connection_rotation = can_rotate_connection @@ -1808,8 +1809,7 @@ where && !is_return_error_retry(&error) && transport_recovery.is_none() && (ordinary_retry || account_rotation_retry) - && self.account_rotations - < self.plan.account_selection_policy().max_account_rotations(); + && self.has_account_rotation_budget(); let retryable = !error.retry_is_prohibited() && (continuation_retry || same_account_retry @@ -2209,8 +2209,7 @@ where // 两个排除臂都必然换号;预算耗尽后不再排除当前账号做跨账号续写重放, // 落回不可重试路径以原始上游错误终态。 ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny - if self.account_rotations - >= self.plan.account_selection_policy().max_account_rotations() => + if !self.has_account_rotation_budget() => { return false; } @@ -2262,7 +2261,7 @@ where return false; } // Native 期间仍可能配置为禁止换号,跨账号重放共用请求冻结的预算 - if self.account_rotations >= self.plan.account_selection_policy().max_account_rotations() { + if !self.has_account_rotation_budget() { return false; } diff --git a/backend/crates/gateway-core/src/error/mod.rs b/backend/crates/gateway-core/src/error/mod.rs index dcf7a9481..91cb9915b 100644 --- a/backend/crates/gateway-core/src/error/mod.rs +++ b/backend/crates/gateway-core/src/error/mod.rs @@ -135,6 +135,7 @@ pub enum TransientRetryPolicy { } /// Provider 在下游尚未提交事件时请求的隐藏恢复方式 +/// 传输恢复默认保持原账号;明确允许建连换号时,Core 可在同账号恢复失败后重选 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PreDeliveryRetry { /// 排除本次账号,按普通调度策略重新选号 @@ -152,7 +153,7 @@ pub enum PreDeliveryRetry { /// 同账号恢复策略,不提供额外安全重放资格 policy: TransientRetryPolicy, }, - /// 固定本次账号,并按 Provider 给出的序号重试当前传输 + /// 按 Provider 给出的序号重试当前传输 /// 可用于发送结果不明确时的有界尽力恢复,不等于上游未执行的证明 SameAccountTransportRetry { /// Provider-owned 传输重试序号,从 1 开始 @@ -165,7 +166,7 @@ pub enum PreDeliveryRetry { /// Provider 明确指定实际失败的传输,避免 HTTP 恢复重新进入 WS transport: crate::upstream::AttemptTransport, }, - /// 固定本次账号,并要求 Provider 使用备用传输 + /// 要求 Provider 使用备用传输 SameAccountTransportFallback, } @@ -517,6 +518,7 @@ pub struct ProviderError { replay_safe: bool, retry_prohibited: bool, same_account_retry_blocked: bool, + connection_rotation_allowed: bool, pre_delivery_retry: Option>, credential_recovery_required: bool, retry_same_account: bool, @@ -561,6 +563,7 @@ impl ProviderError { replay_safe: false, retry_prohibited: false, same_account_retry_blocked: false, + connection_rotation_allowed: false, pre_delivery_retry: None, credential_recovery_required: false, retry_same_account: false, @@ -758,9 +761,21 @@ impl ProviderError { self.pre_delivery_retry = Some(Box::new(PreDeliveryRetry::SameAccountConnectionRetry { transport, })); + self.allow_connection_rotation(); self } + /// 允许 Core 为瞬时建连失败预留换号机会,不授予重放资格或增加传输重试次数 + pub fn allow_connection_rotation(&mut self) { + self.connection_rotation_allowed = true; + } + + /// 供 Core 在传输仍可恢复时为换号预留建连预算 + #[must_use] + pub const fn allows_connection_rotation(&self) -> bool { + self.connection_rotation_allowed + } + /// 要求 Core 在账号凭据已恢复后仅对原账号重放一次 #[must_use] pub const fn with_same_account_retry(mut self) -> Self { @@ -1027,6 +1042,7 @@ impl ProviderError { replay_safe: self.replay_safe, retry_prohibited: self.retry_prohibited, same_account_retry_blocked: self.same_account_retry_blocked, + connection_rotation_allowed: self.connection_rotation_allowed, pre_delivery_retry: self.pre_delivery_retry.clone(), credential_recovery_required: self.credential_recovery_required, retry_same_account: self.retry_same_account, @@ -1070,6 +1086,10 @@ impl fmt::Debug for ProviderError { ) .field("connection_observation", &self.connection_observation()) .field("replay_safe", &self.replay_safe) + .field( + "connection_rotation_allowed", + &self.connection_rotation_allowed, + ) .field("pre_delivery_retry", &self.pre_delivery_retry) .field( "credential_recovery_required", diff --git a/backend/crates/providers/openai/src/provider/failure.rs b/backend/crates/providers/openai/src/provider/failure.rs index 0a2590533..f5684fd85 100644 --- a/backend/crates/providers/openai/src/provider/failure.rs +++ b/backend/crates/providers/openai/src/provider/failure.rs @@ -442,6 +442,7 @@ pub(super) fn schedule_authoritative_quota_refresh_after_failure( #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum WebSocketFailurePolicy { Budgeted, + TransientConnect, ImmediateFallback, } @@ -470,6 +471,9 @@ pub(super) fn websocket_client_failure_policy( } if *status == reqwest::StatusCode::UPGRADE_REQUIRED => { Some(WebSocketFailurePolicy::ImmediateFallback) } + CodexClientError::WebSocket(error) if transient_websocket_connect(error) => { + Some(WebSocketFailurePolicy::TransientConnect) + } CodexClientError::WebSocket(error) if !matches!( error.classified(), @@ -485,6 +489,28 @@ pub(super) fn websocket_client_failure_policy( } } +fn transient_websocket_connect(error: &CodexWebSocketExchangeError) -> bool { + use std::io::ErrorKind; + match error.classified() { + CodexWebSocketExchangeError::ConnectTimeout { .. } => true, + CodexWebSocketExchangeError::Connect { + source: tungstenite::Error::Io(io), + .. + } => matches!( + io.kind(), + ErrorKind::ConnectionRefused + | ErrorKind::NetworkUnreachable + | ErrorKind::HostUnreachable + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::BrokenPipe + | ErrorKind::UnexpectedEof + | ErrorKind::TimedOut + ), + _ => false, + } +} + pub(super) fn apply_websocket_recovery_policy( failure: &mut MappedProviderFailure, context: WebSocketRecoveryContext<'_>, @@ -545,6 +571,15 @@ pub(super) fn apply_websocket_recovery_policy( let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback || context.retry_count >= context.max_retries || session_budget_exhausted; + if fallback_now && !context.requirement.allows_pre_send_http_fallback() { + return; + } + // 只为无续接依赖的新请求保留重选机会;是否重试 WS 或退回 HTTP 仍由协议预算决定 + if context.policy == WebSocketFailurePolicy::TransientConnect + && context.requirement.allows_connection_restart() + { + failure.error.allow_connection_rotation(); + } if !fallback_now { let Some(retry_index) = context.retry_count.checked_add(1).and_then(NonZeroU32::new) else { return; @@ -574,14 +609,11 @@ pub(super) fn apply_websocket_recovery_policy( upstream_send_state = ?failure.error.send_state(), replay_proved_safe = failure.error.replay_is_safe(), transport_requirement = context.requirement.as_str(), - "OpenAI upstream WebSocket failed; retrying the same account" + "OpenAI upstream WebSocket failed; requesting transport retry" ); return; } - if !context.requirement.allows_pre_send_http_fallback() { - return; - } failure.error.set_pre_delivery_transport_fallback(); if let Some(key) = context.session_transport_key { context.session_transport_recovery.disable_websocket(key); @@ -601,7 +633,8 @@ pub(super) fn apply_websocket_recovery_policy( websocket_retry_count = context.retry_count, websocket_max_retries = context.max_retries, websocket_fallback_reason = match context.policy { - WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted", + WebSocketFailurePolicy::Budgeted | WebSocketFailurePolicy::TransientConnect => + "retry_budget_exhausted", WebSocketFailurePolicy::ImmediateFallback => "upgrade_required", }, session_affinity_present = context.session_transport_key.is_some(), diff --git a/backend/crates/providers/openai/tests/provider/contract/mod.rs b/backend/crates/providers/openai/tests/provider/contract/mod.rs index ff70fe933..a06f8e90b 100644 --- a/backend/crates/providers/openai/tests/provider/contract/mod.rs +++ b/backend/crates/providers/openai/tests/provider/contract/mod.rs @@ -4389,6 +4389,10 @@ async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent }; let diagnostic = error.diagnostic().expect("connection diagnosis"); assert_eq!(diagnostic.proxy_hint(), proxy_hint); + assert_eq!( + error.allows_connection_rotation(), + proxy_hint != Some(ProxyConnectionHint::TunnelFailed) + ); if let Some(rejection) = rejection { rejection.await.unwrap(); } @@ -4438,6 +4442,7 @@ async fn abrupt_websocket_disconnect_preserves_diagnosis_and_ambiguous_send_stat server.await.unwrap(); assert_eq!(error.kind(), ProviderErrorKind::Transport); assert_eq!(error.send_state(), UpstreamSendState::Ambiguous); + assert!(!error.allows_connection_rotation()); assert!(!error.replay_is_safe()); assert!( matches!(error.pre_delivery_retry(), Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, .. }) if retry_index.get() == 1) diff --git a/backend/crates/providers/openai/tests/provider/contract/selection_retry.rs b/backend/crates/providers/openai/tests/provider/contract/selection_retry.rs index a19d9329d..2e63a4eae 100644 --- a/backend/crates/providers/openai/tests/provider/contract/selection_retry.rs +++ b/backend/crates/providers/openai/tests/provider/contract/selection_retry.rs @@ -428,6 +428,81 @@ async fn streaming_not_found_preserves_accounts_and_stops_gateway_retries() { } } +#[tokio::test] +async fn refused_connections_respect_the_same_rotation_limit_over_http_and_websocket() { + let upstream = MockServer::start().await; + for websocket in [false, true] { + for rotations in [0, 2] { + let store = Arc::new(MemoryAccountStore::default()); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = listener.local_addr().unwrap(); + drop(listener); + let proxy = + gateway_core::account::OutboundProxy::parse(&format!("socks5h://{endpoint}")) + .unwrap(); + for id in ["acct_provider_contract", "acct_subagent_b"] { + create_account(&store, id).await; + store.set_egress(id, Some(proxy.clone()), None); + } + let operation = if websocket { + capacity_websocket_operation() + } else { + http_generate_operation() + }; + let leases = Arc::new(TestLeaseCoordinator::default()); + let provider = provider_with_affinity_and_base_url_and_leases( + &store, + Arc::new(MemorySessionAffinity::default()), + upstream.uri(), + Arc::clone(&leases), + ); + let providers = ProviderRegistry::new([ + provider as Arc + ]) + .unwrap(); + let coordinator = AttemptCoordinator::new(GatewayEngine::new( + Arc::new(NoopExecutionStore), + providers, + )); + let settings = SettingsValues::new(3, "smart", Default::default(), None, None) + .with_max_account_rotations(rotations); + let plan = execution_plan_with_settings(&operation, settings); + let mut session = coordinator + .start( + execution_request("req_refused", &operation, Duration::from_secs(15)), + operation, + plan, + None, + None, + CancellationToken::new(), + ) + .await + .unwrap(); + let result = session.collect_uncommitted().await; + assert!( + matches!(result, Err(EngineError::Provider(_))), + "ws={websocket}, rotations={rotations}: {result:?}" + ); + let requests = leases.requests.lock().unwrap(); + let accounts: Vec<_> = requests.iter().map(|r| r.account_id()).collect(); + assert!( + (3..=4).contains(&accounts.len()), + "ws={websocket}, rotations={rotations}: {accounts:?}, {result:?}" + ); + assert_eq!(accounts[0], accounts[1]); + if rotations == 0 { + assert!( + accounts.iter().all(|a| *a == accounts[0]), + "ws={websocket}: {accounts:?}" + ); + } else { + assert_ne!(accounts[1], accounts[2], "ws={websocket}: {accounts:?}"); + } + } + } + assert!(upstream.received_requests().await.unwrap().is_empty()); +} + #[tokio::test] async fn stalled_http_proxy_retries_same_account_before_using_reserved_rotation_budget() { stalled_proxy_recovery(false).await; @@ -536,11 +611,20 @@ async fn stalled_proxy_recovery(websocket: bool) { } fn execution_plan(operation: &Operation) -> gateway_core::routing::RoutingPlan { + execution_plan_with_settings( + operation, + SettingsValues::new(3, "smart", Default::default(), None, None), + ) +} + +fn execution_plan_with_settings( + operation: &Operation, + settings: SettingsValues, +) -> gateway_core::routing::RoutingPlan { let kind = ProviderKind::new("openai").unwrap(); let snapshot = RuntimeSnapshot::new( ConfigRevision::new(1).unwrap(), - SettingsValues::new(3, "smart", Default::default(), None, None) - .with_request_location(global_request_location(), true), + settings.with_request_location(global_request_location(), true), vec![kind.clone()], vec![ProviderModel::new( kind, diff --git a/docs/api.md b/docs/api.md index 66333b1fd..7e9bb4e0e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -769,8 +769,9 @@ xAI 的 `reasoning.effort` 接受 `none / minimal / low / medium / high / xhigh #### 容量拒绝与重试 -OpenAI HTTP 明确未发送的瞬时建连失败(含 TLS 握手 EOF)在单请求内共享最多 4 次 opening、30 秒预算。 -先在原账号上退避恢复;允许换号且至少剩余两次 opening 时,本次恢复最多使用一半剩余时间,并保留一次 opening。 +OpenAI HTTP 与 WebSocket 在单请求内共享最多 4 次建连、30 秒预算。 +HTTP 瞬时建连失败(含 TLS 握手 EOF),或 WebSocket 新请求在发送前遇到连接拒绝、网络不可达等瞬时建连失败时,使用同一换号规则: +先在原账号上退避重试;允许换号且至少还能建连两次时,这次重试最多使用一半剩余时间,给重选账号留一次建连机会。 再次建连失败才按调度规则重选账号,选号不可用时使用原账号的剩余预算;固定账号或换号预算不足时仅在原账号恢复。 证书、TLS 协议及配置错误不进入该恢复,取消与请求截止时间优先终止等待 @@ -802,7 +803,7 @@ SSE 注释和 WebSocket Ping/Pong 不重置官方 Codex 客户端的事件空闲 其 `stream_idle_timeout_ms` 默认是 300 秒;长请求需持续收到真实事件, 或同时调整客户端与上游传输的空闲等待预算,单独缩短心跳间隔不能延长业务等待 -WebSocket 在交付有效输出前断线或超时,可按 `stream_max_retries` 做同账号、有退避的尽力重试 +WebSocket 已连接、尚未交付有效输出时断线或超时,可按 `stream_max_retries` 做同账号、有退避的尽力重试 (默认最多 5 次),耗尽后仅在请求允许时改用 HTTP。已发送或发送结果不明不单独阻止这项恢复; 次数限制不保证上游只执行一次,诊断保留原始发送状态。精确连接续接、显式终止拒绝、已观测的 外部副作用和下游提交仍限制恢复;取消与请求截止时间随时终止等待 @@ -2335,7 +2336,7 @@ Chat Completions 和 Responses 通过原生 OpenAI Provider 选号时,缺少 `maxAccountRotations` 是单请求最大换号次数,默认 3,取值 0~31;0 表示不换号。 首次选号、同账号重试和选号时过滤不可用候选不计入换号次数,总路由尝试仍最多 32 次。 -提高该值允许请求尝试更多账号,但不放宽安全重放或交付后的重试限制。 +例如设为 2,A 重试后换到 B 只计 1 次,即使 A、B 使用同一代理。实际换号次数仍受可用账号、建连预算和安全重放条件限制。 设置更新请求须包含账号亲和、亲和时长与最大换号次数,保存后对新请求生效,执行中请求及其重试沿用冻结值 ### 请求解压上限 diff --git a/docs/architecture.md b/docs/architecture.md index bf553c7bb..33a2a1fd3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -436,10 +436,11 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别客户端,Cor - OpenAI 的自动恢复限于明确识别的账号、容量、`503 + error.code=server_error`、续接和传输场景; 其他上游拒绝保留响应与重试提示,交由客户端决策。WS opening 尚未发送业务 payload 也不扩大该恢复范围,明确的 `426` 仍允许 HTTP 回退 - `not_sent`、`sent`、`ambiguous` 是单调的上游发送边界;结果不明确时不能假定上游未收到请求 -- OpenAI HTTP 明确未发送的瞬时建连失败共享请求级 opening 预算,先在原账号、HTTP 传输上退避恢复; - 允许换号且预算足够时,为重选保留时间和连接次数;再次建连失败才排除原账号并以 HTTP 重选,选号不可用时回原账号使用剩余预算; +- OpenAI HTTP 和无续接依赖的 WS 新请求遇到明确未发送的瞬时建连失败时,先在原账号恢复; + Core 按请求级建连与换号预算为重选保留时间和连接次数,再次建连失败才排除原账号;选号不可用时回原账号使用剩余预算。 + Provider 决定恢复传输:HTTP 继续使用 HTTP,WS 按协议重试或回退; 外部指定账号或换号预算不足时保留同账号恢复,账号与容量拒绝仍按各自的安全重放规则判断换号; - 证书、TLS 协议及配置错误不进入同账号建连恢复,取消和执行 deadline 优先终止等待 + 证书、TLS 协议及配置错误不触发这项建连换号策略,取消和执行 deadline 优先终止等待 - downstream commit 记录不可撤回的首次交付;不可重放事件交付后禁止换号、重试和 fallback HTTP 保活可先提交响应头,Core 记录该首字节时间与实际状态,但首个业务批次仍独立提交; SSE 注释不构成业务输出或上游已发送的证明,也不改变重放资格