fix(openai): honor account rotation limits on WebSocket connection failures

Reserve a connection attempt for another eligible account when a new WebSocket request repeatedly fails before sending. Use the same rotation budget as HTTP while preserving session and replay restrictions.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-11-16 17:45:40 +08:00
co-authored by Codex
parent 5b7f307bb2
commit 2abbfb1fac
7 changed files with 172 additions and 29 deletions
@@ -1019,6 +1019,10 @@ where
}
}
fn has_account_rotation_budget(&self) -> bool {
self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
}
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
let pending_retry = self.pending_retry.take();
let is_transport_recovery = pending_retry
@@ -1264,7 +1268,7 @@ where
)
// 跨 Provider 候选推进必然换号(账号行按 Provider 隔离),
// 预算耗尽后不再推进,交回容量类失败的原有终态语义。
&& self.account_rotations < self.plan.account_selection_policy().max_account_rotations()
&& self.has_account_rotation_budget()
&& self.advance_provider_candidate();
let retryable = self
.apply_retry_policy(super::policy::RetryFacts {
@@ -1668,10 +1672,8 @@ where
&& self.routing_attempts < self.plan.max_attempts().get();
// 只有尚未发送的逻辑请求进入新增策略;已有发送后安全恢复保持原有路由规则
let connection_retry_requested = self.current_send_state() == UpstreamSendState::NotSent
&& matches!(
error.pre_delivery_retry(),
Some(crate::error::PreDeliveryRetry::SameAccountConnectionRetry { .. })
);
&& self.continuation_attempt == ContinuationAttempt::None
&& error.allows_connection_rotation();
// 服务器建议不可截短
// 超出上限,或上游拒绝继续重试当前账号时,跳过同账号等待
// 较短的 Retry-After 也直接沿用安全换号或原错误出口
@@ -1729,9 +1731,8 @@ where
let can_rotate_connection = connection_retry_requested
&& self.account_selection.required_account().is_none()
&& self.routing_attempts < self.plan.max_attempts().get()
&& self.account_rotations
< self.plan.account_selection_policy().max_account_rotations();
// A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening
&& self.has_account_rotation_budget();
// A 先恢复一次,再利用预留预算重选;下一次传输仍由 Provider 的恢复策略决定
let connection_rotation = transport_recovery
.filter(|_| can_rotate_connection && current.connection_rotation_reserved);
let reserve_connection_rotation = can_rotate_connection
@@ -1808,8 +1809,7 @@ where
&& !is_return_error_retry(&error)
&& transport_recovery.is_none()
&& (ordinary_retry || account_rotation_retry)
&& self.account_rotations
< self.plan.account_selection_policy().max_account_rotations();
&& self.has_account_rotation_budget();
let retryable = !error.retry_is_prohibited()
&& (continuation_retry
|| same_account_retry
@@ -2209,8 +2209,7 @@ where
// 两个排除臂都必然换号;预算耗尽后不再排除当前账号做跨账号续写重放,
// 落回不可重试路径以原始上游错误终态。
ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny
if self.account_rotations
>= self.plan.account_selection_policy().max_account_rotations() =>
if !self.has_account_rotation_budget() =>
{
return false;
}
@@ -2262,7 +2261,7 @@ where
return false;
}
// Native 期间仍可能配置为禁止换号,跨账号重放共用请求冻结的预算
if self.account_rotations >= self.plan.account_selection_policy().max_account_rotations() {
if !self.has_account_rotation_budget() {
return false;
}
+22 -2
View File
@@ -135,6 +135,7 @@ pub enum TransientRetryPolicy {
}
/// Provider 在下游尚未提交事件时请求的隐藏恢复方式
/// 传输恢复默认保持原账号;明确允许建连换号时,Core 可在同账号恢复失败后重选
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum PreDeliveryRetry {
/// 排除本次账号,按普通调度策略重新选号
@@ -152,7 +153,7 @@ pub enum PreDeliveryRetry {
/// 同账号恢复策略,不提供额外安全重放资格
policy: TransientRetryPolicy,
},
/// 固定本次账号,并按 Provider 给出的序号重试当前传输
/// 按 Provider 给出的序号重试当前传输
/// 可用于发送结果不明确时的有界尽力恢复,不等于上游未执行的证明
SameAccountTransportRetry {
/// Provider-owned 传输重试序号,从 1 开始
@@ -165,7 +166,7 @@ pub enum PreDeliveryRetry {
/// Provider 明确指定实际失败的传输,避免 HTTP 恢复重新进入 WS
transport: crate::upstream::AttemptTransport,
},
/// 固定本次账号,并要求 Provider 使用备用传输
/// 要求 Provider 使用备用传输
SameAccountTransportFallback,
}
@@ -517,6 +518,7 @@ pub struct ProviderError {
replay_safe: bool,
retry_prohibited: bool,
same_account_retry_blocked: bool,
connection_rotation_allowed: bool,
pre_delivery_retry: Option<Box<PreDeliveryRetry>>,
credential_recovery_required: bool,
retry_same_account: bool,
@@ -561,6 +563,7 @@ impl ProviderError {
replay_safe: false,
retry_prohibited: false,
same_account_retry_blocked: false,
connection_rotation_allowed: false,
pre_delivery_retry: None,
credential_recovery_required: false,
retry_same_account: false,
@@ -758,9 +761,21 @@ impl ProviderError {
self.pre_delivery_retry = Some(Box::new(PreDeliveryRetry::SameAccountConnectionRetry {
transport,
}));
self.allow_connection_rotation();
self
}
/// 允许 Core 为瞬时建连失败预留换号机会,不授予重放资格或增加传输重试次数
pub fn allow_connection_rotation(&mut self) {
self.connection_rotation_allowed = true;
}
/// 供 Core 在传输仍可恢复时为换号预留建连预算
#[must_use]
pub const fn allows_connection_rotation(&self) -> bool {
self.connection_rotation_allowed
}
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次
#[must_use]
pub const fn with_same_account_retry(mut self) -> Self {
@@ -1027,6 +1042,7 @@ impl ProviderError {
replay_safe: self.replay_safe,
retry_prohibited: self.retry_prohibited,
same_account_retry_blocked: self.same_account_retry_blocked,
connection_rotation_allowed: self.connection_rotation_allowed,
pre_delivery_retry: self.pre_delivery_retry.clone(),
credential_recovery_required: self.credential_recovery_required,
retry_same_account: self.retry_same_account,
@@ -1070,6 +1086,10 @@ impl fmt::Debug for ProviderError {
)
.field("connection_observation", &self.connection_observation())
.field("replay_safe", &self.replay_safe)
.field(
"connection_rotation_allowed",
&self.connection_rotation_allowed,
)
.field("pre_delivery_retry", &self.pre_delivery_retry)
.field(
"credential_recovery_required",
@@ -442,6 +442,7 @@ pub(super) fn schedule_authoritative_quota_refresh_after_failure(
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum WebSocketFailurePolicy {
Budgeted,
TransientConnect,
ImmediateFallback,
}
@@ -470,6 +471,9 @@ pub(super) fn websocket_client_failure_policy(
} if *status == reqwest::StatusCode::UPGRADE_REQUIRED => {
Some(WebSocketFailurePolicy::ImmediateFallback)
}
CodexClientError::WebSocket(error) if transient_websocket_connect(error) => {
Some(WebSocketFailurePolicy::TransientConnect)
}
CodexClientError::WebSocket(error)
if !matches!(
error.classified(),
@@ -485,6 +489,28 @@ pub(super) fn websocket_client_failure_policy(
}
}
fn transient_websocket_connect(error: &CodexWebSocketExchangeError) -> bool {
use std::io::ErrorKind;
match error.classified() {
CodexWebSocketExchangeError::ConnectTimeout { .. } => true,
CodexWebSocketExchangeError::Connect {
source: tungstenite::Error::Io(io),
..
} => matches!(
io.kind(),
ErrorKind::ConnectionRefused
| ErrorKind::NetworkUnreachable
| ErrorKind::HostUnreachable
| ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted
| ErrorKind::BrokenPipe
| ErrorKind::UnexpectedEof
| ErrorKind::TimedOut
),
_ => false,
}
}
pub(super) fn apply_websocket_recovery_policy(
failure: &mut MappedProviderFailure,
context: WebSocketRecoveryContext<'_>,
@@ -545,6 +571,15 @@ pub(super) fn apply_websocket_recovery_policy(
let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback
|| context.retry_count >= context.max_retries
|| session_budget_exhausted;
if fallback_now && !context.requirement.allows_pre_send_http_fallback() {
return;
}
// 只为无续接依赖的新请求保留重选机会;是否重试 WS 或退回 HTTP 仍由协议预算决定
if context.policy == WebSocketFailurePolicy::TransientConnect
&& context.requirement.allows_connection_restart()
{
failure.error.allow_connection_rotation();
}
if !fallback_now {
let Some(retry_index) = context.retry_count.checked_add(1).and_then(NonZeroU32::new) else {
return;
@@ -574,14 +609,11 @@ pub(super) fn apply_websocket_recovery_policy(
upstream_send_state = ?failure.error.send_state(),
replay_proved_safe = failure.error.replay_is_safe(),
transport_requirement = context.requirement.as_str(),
"OpenAI upstream WebSocket failed; retrying the same account"
"OpenAI upstream WebSocket failed; requesting transport retry"
);
return;
}
if !context.requirement.allows_pre_send_http_fallback() {
return;
}
failure.error.set_pre_delivery_transport_fallback();
if let Some(key) = context.session_transport_key {
context.session_transport_recovery.disable_websocket(key);
@@ -601,7 +633,8 @@ pub(super) fn apply_websocket_recovery_policy(
websocket_retry_count = context.retry_count,
websocket_max_retries = context.max_retries,
websocket_fallback_reason = match context.policy {
WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted",
WebSocketFailurePolicy::Budgeted | WebSocketFailurePolicy::TransientConnect =>
"retry_budget_exhausted",
WebSocketFailurePolicy::ImmediateFallback => "upgrade_required",
},
session_affinity_present = context.session_transport_key.is_some(),
@@ -4389,6 +4389,10 @@ async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent
};
let diagnostic = error.diagnostic().expect("connection diagnosis");
assert_eq!(diagnostic.proxy_hint(), proxy_hint);
assert_eq!(
error.allows_connection_rotation(),
proxy_hint != Some(ProxyConnectionHint::TunnelFailed)
);
if let Some(rejection) = rejection {
rejection.await.unwrap();
}
@@ -4438,6 +4442,7 @@ async fn abrupt_websocket_disconnect_preserves_diagnosis_and_ambiguous_send_stat
server.await.unwrap();
assert_eq!(error.kind(), ProviderErrorKind::Transport);
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
assert!(!error.allows_connection_rotation());
assert!(!error.replay_is_safe());
assert!(
matches!(error.pre_delivery_retry(), Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, .. }) if retry_index.get() == 1)
@@ -428,6 +428,81 @@ async fn streaming_not_found_preserves_accounts_and_stops_gateway_retries() {
}
}
#[tokio::test]
async fn refused_connections_respect_the_same_rotation_limit_over_http_and_websocket() {
let upstream = MockServer::start().await;
for websocket in [false, true] {
for rotations in [0, 2] {
let store = Arc::new(MemoryAccountStore::default());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = listener.local_addr().unwrap();
drop(listener);
let proxy =
gateway_core::account::OutboundProxy::parse(&format!("socks5h://{endpoint}"))
.unwrap();
for id in ["acct_provider_contract", "acct_subagent_b"] {
create_account(&store, id).await;
store.set_egress(id, Some(proxy.clone()), None);
}
let operation = if websocket {
capacity_websocket_operation()
} else {
http_generate_operation()
};
let leases = Arc::new(TestLeaseCoordinator::default());
let provider = provider_with_affinity_and_base_url_and_leases(
&store,
Arc::new(MemorySessionAffinity::default()),
upstream.uri(),
Arc::clone(&leases),
);
let providers = ProviderRegistry::new([
provider as Arc<dyn gateway_core::engine::provider::Provider>
])
.unwrap();
let coordinator = AttemptCoordinator::new(GatewayEngine::new(
Arc::new(NoopExecutionStore),
providers,
));
let settings = SettingsValues::new(3, "smart", Default::default(), None, None)
.with_max_account_rotations(rotations);
let plan = execution_plan_with_settings(&operation, settings);
let mut session = coordinator
.start(
execution_request("req_refused", &operation, Duration::from_secs(15)),
operation,
plan,
None,
None,
CancellationToken::new(),
)
.await
.unwrap();
let result = session.collect_uncommitted().await;
assert!(
matches!(result, Err(EngineError::Provider(_))),
"ws={websocket}, rotations={rotations}: {result:?}"
);
let requests = leases.requests.lock().unwrap();
let accounts: Vec<_> = requests.iter().map(|r| r.account_id()).collect();
assert!(
(3..=4).contains(&accounts.len()),
"ws={websocket}, rotations={rotations}: {accounts:?}, {result:?}"
);
assert_eq!(accounts[0], accounts[1]);
if rotations == 0 {
assert!(
accounts.iter().all(|a| *a == accounts[0]),
"ws={websocket}: {accounts:?}"
);
} else {
assert_ne!(accounts[1], accounts[2], "ws={websocket}: {accounts:?}");
}
}
}
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[tokio::test]
async fn stalled_http_proxy_retries_same_account_before_using_reserved_rotation_budget() {
stalled_proxy_recovery(false).await;
@@ -536,11 +611,20 @@ async fn stalled_proxy_recovery(websocket: bool) {
}
fn execution_plan(operation: &Operation) -> gateway_core::routing::RoutingPlan {
execution_plan_with_settings(
operation,
SettingsValues::new(3, "smart", Default::default(), None, None),
)
}
fn execution_plan_with_settings(
operation: &Operation,
settings: SettingsValues,
) -> gateway_core::routing::RoutingPlan {
let kind = ProviderKind::new("openai").unwrap();
let snapshot = RuntimeSnapshot::new(
ConfigRevision::new(1).unwrap(),
SettingsValues::new(3, "smart", Default::default(), None, None)
.with_request_location(global_request_location(), true),
settings.with_request_location(global_request_location(), true),
vec![kind.clone()],
vec![ProviderModel::new(
kind,
+5 -4
View File
@@ -769,8 +769,9 @@ xAI 的 `reasoning.effort` 接受 `none / minimal / low / medium / high / xhigh
#### 容量拒绝与重试
OpenAI HTTP 明确未发送的瞬时建连失败(含 TLS 握手 EOF)在单请求内共享最多 4 次 opening、30 秒预算。
先在原账号上退避恢复;允许换号且至少剩余两次 opening 时,本次恢复最多使用一半剩余时间,并保留一次 opening。
OpenAI HTTP 与 WebSocket 在单请求内共享最多 4 次建连、30 秒预算。
HTTP 瞬时建连失败(含 TLS 握手 EOF),或 WebSocket 新请求在发送前遇到连接拒绝、网络不可达等瞬时建连失败时,使用同一换号规则:
先在原账号上退避重试;允许换号且至少还能建连两次时,这次重试最多使用一半剩余时间,给重选账号留一次建连机会。
再次建连失败才按调度规则重选账号,选号不可用时使用原账号的剩余预算;固定账号或换号预算不足时仅在原账号恢复。
证书、TLS 协议及配置错误不进入该恢复,取消与请求截止时间优先终止等待
@@ -802,7 +803,7 @@ SSE 注释和 WebSocket Ping/Pong 不重置官方 Codex 客户端的事件空闲
其 `stream_idle_timeout_ms` 默认是 300 秒;长请求需持续收到真实事件,
或同时调整客户端与上游传输的空闲等待预算,单独缩短心跳间隔不能延长业务等待
WebSocket 在交付有效输出前断线或超时,可按 `stream_max_retries` 做同账号、有退避的尽力重试
WebSocket 已连接、尚未交付有效输出时断线或超时,可按 `stream_max_retries` 做同账号、有退避的尽力重试
(默认最多 5 次),耗尽后仅在请求允许时改用 HTTP。已发送或发送结果不明不单独阻止这项恢复;
次数限制不保证上游只执行一次,诊断保留原始发送状态。精确连接续接、显式终止拒绝、已观测的
外部副作用和下游提交仍限制恢复;取消与请求截止时间随时终止等待
@@ -2335,7 +2336,7 @@ Chat Completions 和 Responses 通过原生 OpenAI Provider 选号时,缺少
`maxAccountRotations` 是单请求最大换号次数,默认 3,取值 0~31;0 表示不换号。
首次选号、同账号重试和选号时过滤不可用候选不计入换号次数,总路由尝试仍最多 32 次。
提高该值允许请求尝试更多账号,但不放宽安全重放或交付后的重试限制。
例如设为 2,A 重试后换到 B 只计 1 次,即使 A、B 使用同一代理。实际换号次数仍受可用账号、建连预算和安全重放条件限制。
设置更新请求须包含账号亲和、亲和时长与最大换号次数,保存后对新请求生效,执行中请求及其重试沿用冻结值
### 请求解压上限
+4 -3
View File
@@ -436,10 +436,11 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别客户端,Cor
- OpenAI 的自动恢复限于明确识别的账号、容量、`503 + error.code=server_error`、续接和传输场景;
其他上游拒绝保留响应与重试提示,交由客户端决策。WS opening 尚未发送业务 payload 也不扩大该恢复范围,明确的 `426` 仍允许 HTTP 回退
- `not_sent`、`sent`、`ambiguous` 是单调的上游发送边界;结果不明确时不能假定上游未收到请求
- OpenAI HTTP 明确未发送的瞬时建连失败共享请求级 opening 预算,先在原账号、HTTP 传输上退避恢复;
允许换号且预算足够时,为重选保留时间和连接次数;再次建连失败才排除原账号并以 HTTP 重选,选号不可用时回原账号使用剩余预算;
- OpenAI HTTP 和无续接依赖的 WS 新请求遇到明确未发送的瞬时建连失败时,先在原账号恢复;
Core 按请求级建连与换号预算为重选保留时间和连接次数,再次建连失败才排除原账号;选号不可用时回原账号使用剩余预算。
Provider 决定恢复传输:HTTP 继续使用 HTTP,WS 按协议重试或回退;
外部指定账号或换号预算不足时保留同账号恢复,账号与容量拒绝仍按各自的安全重放规则判断换号;
证书、TLS 协议及配置错误不进入同账号建连恢复,取消和执行 deadline 优先终止等待
证书、TLS 协议及配置错误不触发这项建连换号策略,取消和执行 deadline 优先终止等待
- downstream commit 记录不可撤回的首次交付;不可重放事件交付后禁止换号、重试和 fallback
HTTP 保活可先提交响应头,Core 记录该首字节时间与实际状态,但首个业务批次仍独立提交;
SSE 注释不构成业务输出或上游已发送的证明,也不改变重放资格