fix(openai): classify upstream close 1009 as message_too_big request fault

Upstream (RFC 6455 close code 1009) rejects the current request because its
message exceeds the upstream WebSocket size limit. Previously this surfaced
as a generic Transport failure, which:

- counted toward the provider-level circuit breaker, so three such closes
  amplified into 30s of 503 'no provider can execute this request' for every
  unrelated request, even though healthy accounts remained schedulable;
- returned an opaque 502 to the client with no actionable error semantics,
  so clients kept resending the same oversized payload.

Now the close code is detected through the post-send ambiguous wrapper, and:

- ProviderErrorKind::MessageTooBig is a request-scoped fault that never
  affects the provider circuit breaker and never penalizes accounts;
- GatewayErrorKind::MessageTooBig maps to HTTP 413 with the structured
  invalid_request_error/message_too_big code on both HTTP and WebSocket
  client paths, matching the upstream close reason.
This commit is contained in:
zhangwentian
2026-09-24 16:08:00 +08:00
parent 94dca85085
commit fb6ef1d080
10 changed files with 83 additions and 7 deletions
@@ -327,6 +327,11 @@ pub const fn gateway_error_contract(
"server_error",
"upstream_unavailable",
),
GatewayErrorKind::MessageTooBig => (
StatusCode::PAYLOAD_TOO_LARGE,
"invalid_request_error",
"message_too_big",
),
GatewayErrorKind::Timeout => (
StatusCode::GATEWAY_TIMEOUT,
"server_error",
+12
View File
@@ -48,6 +48,9 @@ pub enum ProviderErrorKind {
Transport,
/// 上游协议不合法。
Protocol,
/// 上游以 message too big 拒收当前请求(如 WebSocket close 1009)。
/// 这是请求自身的问题:换账号或暂停 Provider 都无法让它成功。
MessageTooBig,
/// Provider 暂不可用。
Unavailable,
/// 上游明确拒绝当前请求的模型容量;不证明整个 Provider 的基础设施故障。
@@ -79,6 +82,7 @@ impl ProviderErrorKind {
Self::Timeout => "timeout",
Self::Transport => "transport",
Self::Protocol => "protocol",
Self::MessageTooBig => "message_too_big",
Self::Unavailable => "unavailable",
Self::UpstreamCapacityUnavailable => "upstream_capacity_unavailable",
Self::Cancelled => "cancelled",
@@ -978,6 +982,9 @@ pub enum GatewayErrorKind {
RateLimited,
/// 上游暂不可用。
UpstreamUnavailable,
/// 上游拒收当前请求:消息过大(如 WebSocket close 1009)。
/// 客户端必须缩小或重建请求;重试或换账号都无法成功。
MessageTooBig,
/// 请求超时。
Timeout,
/// 请求取消。
@@ -1003,6 +1010,7 @@ impl GatewayErrorKind {
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
Self::RateLimited => "rate_limited",
Self::UpstreamUnavailable => "upstream_unavailable",
Self::MessageTooBig => "message_too_big",
Self::Timeout => "timeout",
Self::Cancelled => "cancelled",
Self::Internal => "internal_error",
@@ -1113,6 +1121,10 @@ impl GatewayError {
ProviderErrorKind::Timeout => {
Self::new(GatewayErrorKind::Timeout, "upstream request timed out")
}
ProviderErrorKind::MessageTooBig => Self::new(
GatewayErrorKind::MessageTooBig,
"upstream rejected the request because the message is too large",
),
ProviderErrorKind::Cancelled => {
Self::new(GatewayErrorKind::Cancelled, "request was cancelled")
}
@@ -2832,6 +2832,10 @@ fn only_provider_attributable_failures_should_affect_circuit() {
assert!(!provider_failure_affects_circuit(
ProviderErrorKind::UpstreamCapacityUnavailable
));
// 上游 close 1009(message too big)是请求自身的问题:不得熔断 provider。
assert!(!provider_failure_affects_circuit(
ProviderErrorKind::MessageTooBig
));
}
#[test]
@@ -203,6 +203,26 @@ fn continuation_recovery_should_keep_its_internal_classification_and_client_cont
);
}
#[test]
fn message_too_big_should_map_to_a_request_scoped_client_contract() {
let error = ProviderError::new(ProviderErrorKind::MessageTooBig, UpstreamSendState::Ambiguous)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"upstream websocket message too big",
Some("message_too_big".to_owned()),
Some("invalid_request_error".to_owned()),
));
let gateway = GatewayError::from_provider(&error);
assert_eq!(error.kind().as_str(), "message_too_big");
assert_eq!(gateway.kind().as_str(), "message_too_big");
assert_eq!(
gateway.safe_message(),
"upstream rejected the request because the message is too large"
);
assert_eq!(gateway.client_error_code(), Some("message_too_big"));
assert_eq!(gateway.client_error_type(), Some("invalid_request_error"));
}
#[test]
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
let message = format!("\0{}\n", "m".repeat(9_000));
@@ -854,7 +854,7 @@ fn project_wire(wire: ProtocolWireEvent) -> Result<WireEvent, PluginFault> {
pub(super) fn gateway_fault(error: GatewayError) -> PluginFault {
let code = match error.kind() {
GatewayErrorKind::InvalidRequest => ErrorCode::InvalidInput,
GatewayErrorKind::InvalidRequest | GatewayErrorKind::MessageTooBig => ErrorCode::InvalidInput,
GatewayErrorKind::Unsupported | GatewayErrorKind::ModelNotFound => ErrorCode::Unsupported,
GatewayErrorKind::Unauthorized => ErrorCode::PermissionDenied,
GatewayErrorKind::PolicyDenied => ErrorCode::Rejected,
@@ -1227,6 +1227,22 @@ pub(super) fn websocket_client_visible_error(
));
}
let close = error.close_before_terminal()?;
// RFC 6455 1009 = message too big:客户端必须能识别并缩小/重建请求,
// 因此投影成结构化的 invalid_request_error/message_too_big,而不是裸 close 码。
if close.code() == Some(WEBSOCKET_CLOSE_MESSAGE_TOO_BIG) {
let message = close
.reason()
.filter(|reason| !reason.is_empty())
.map_or_else(
|| "upstream websocket message too big".to_owned(),
str::to_owned,
);
return Some(ClientVisibleUpstreamError::new(
message,
Some("message_too_big".to_owned()),
Some("invalid_request_error".to_owned()),
));
}
let message = close
.reason()
.filter(|reason| !reason.is_empty())
@@ -1487,11 +1503,24 @@ pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> Provi
CodexWebSocketExchangeError::ConnectionLimitReached(_) => ProviderErrorKind::RateLimited,
CodexWebSocketExchangeError::Transport(_)
| CodexWebSocketExchangeError::Connect(_)
| CodexWebSocketExchangeError::PostSendAmbiguous { .. }
| CodexWebSocketExchangeError::ClosedBeforeTerminal(_)
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. } => {
ProviderErrorKind::Transport
}
// 上游 close 1009 是 RFC 6455 的 "message too big":请求自身超出上游
// WS 消息上限。换账号或熔断 Provider 都无法让该请求成功,必须让客户端
// 感知到可行动的 message_too_big 语义,且不得计入 provider 熔断。
// live 流错误会先被 post_send_ambiguous 包装,需从 source 解出 close 码。
CodexWebSocketExchangeError::ClosedBeforeTerminal(_)
| CodexWebSocketExchangeError::PostSendAmbiguous { .. } => {
if error
.close_before_terminal()
.is_some_and(|close| close.code() == Some(WEBSOCKET_CLOSE_MESSAGE_TOO_BIG))
{
ProviderErrorKind::MessageTooBig
} else {
ProviderErrorKind::Transport
}
}
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
@@ -86,7 +86,7 @@ use crate::transport::request::{
};
use crate::transport::session::CodexSessionIdentity;
use crate::transport::usage::normalize_service_tier;
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason};
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason, WEBSOCKET_CLOSE_MESSAGE_TOO_BIG};
use crate::transport::{
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
@@ -6,6 +6,9 @@ use gateway_protocol::openai::sse::SseError;
use thiserror::Error;
use uuid::Uuid;
/// RFC 6455 close code 1009:上游拒收超出大小限制的消息。
pub(crate) const WEBSOCKET_CLOSE_MESSAGE_TOO_BIG: u16 = 1009;
use crate::transport::client::CodexClientVisibleUpstreamResponse;
use crate::transport::diagnostics::CodexUpstreamDiagnostics;
use crate::transport::diagnostics::CodexUpstreamSendPhase;
@@ -41,5 +41,6 @@ pub(crate) use self::{
execute_prepared_response_create_request_stream, post_send_ambiguous,
prepare_response_create_request_with_pool,
},
error::WEBSOCKET_CLOSE_MESSAGE_TOO_BIG,
pool::DEFAULT_STREAM_IDLE_TIMEOUT,
};
@@ -3592,7 +3592,9 @@ async fn websocket_close_after_delivery_preserves_details_and_reconnects_through
assert!(!format!("{error:?}").contains("message too big"));
assert!(!error.to_string().contains("message too big"));
assert_eq!(error.kind(), ProviderErrorKind::Transport);
// 上游 close 1009 是 RFC 6455 "message too big":必须归因为请求自身问题,
// 而不是 provider 传输故障(否则会被熔断器和换号逻辑误伤其他请求)。
assert_eq!(error.kind(), ProviderErrorKind::MessageTooBig);
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
assert!(
!error.allows_pre_delivery_retry(),
@@ -3602,8 +3604,8 @@ async fn websocket_close_after_delivery_preserves_details_and_reconnects_through
.client_visible_upstream_error()
.expect("WebSocket close detail");
assert_eq!(detail.message(), "message too big");
assert_eq!(detail.code(), Some("1009"));
assert_eq!(detail.error_type(), Some("websocket_close_error"));
assert_eq!(detail.code(), Some("message_too_big"));
assert_eq!(detail.error_type(), Some("invalid_request_error"));
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
Some("websocket_close_1009")