mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
fix(openai): classify upstream close 1009 as message_too_big request fault
Upstream (RFC 6455 close code 1009) rejects the current request because its message exceeds the upstream WebSocket size limit. Previously this surfaced as a generic Transport failure, which: - counted toward the provider-level circuit breaker, so three such closes amplified into 30s of 503 'no provider can execute this request' for every unrelated request, even though healthy accounts remained schedulable; - returned an opaque 502 to the client with no actionable error semantics, so clients kept resending the same oversized payload. Now the close code is detected through the post-send ambiguous wrapper, and: - ProviderErrorKind::MessageTooBig is a request-scoped fault that never affects the provider circuit breaker and never penalizes accounts; - GatewayErrorKind::MessageTooBig maps to HTTP 413 with the structured invalid_request_error/message_too_big code on both HTTP and WebSocket client paths, matching the upstream close reason.
This commit is contained in:
@@ -327,6 +327,11 @@ pub const fn gateway_error_contract(
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MessageTooBig => (
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"invalid_request_error",
|
||||
"message_too_big",
|
||||
),
|
||||
GatewayErrorKind::Timeout => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"server_error",
|
||||
|
||||
@@ -48,6 +48,9 @@ pub enum ProviderErrorKind {
|
||||
Transport,
|
||||
/// 上游协议不合法。
|
||||
Protocol,
|
||||
/// 上游以 message too big 拒收当前请求(如 WebSocket close 1009)。
|
||||
/// 这是请求自身的问题:换账号或暂停 Provider 都无法让它成功。
|
||||
MessageTooBig,
|
||||
/// Provider 暂不可用。
|
||||
Unavailable,
|
||||
/// 上游明确拒绝当前请求的模型容量;不证明整个 Provider 的基础设施故障。
|
||||
@@ -79,6 +82,7 @@ impl ProviderErrorKind {
|
||||
Self::Timeout => "timeout",
|
||||
Self::Transport => "transport",
|
||||
Self::Protocol => "protocol",
|
||||
Self::MessageTooBig => "message_too_big",
|
||||
Self::Unavailable => "unavailable",
|
||||
Self::UpstreamCapacityUnavailable => "upstream_capacity_unavailable",
|
||||
Self::Cancelled => "cancelled",
|
||||
@@ -978,6 +982,9 @@ pub enum GatewayErrorKind {
|
||||
RateLimited,
|
||||
/// 上游暂不可用。
|
||||
UpstreamUnavailable,
|
||||
/// 上游拒收当前请求:消息过大(如 WebSocket close 1009)。
|
||||
/// 客户端必须缩小或重建请求;重试或换账号都无法成功。
|
||||
MessageTooBig,
|
||||
/// 请求超时。
|
||||
Timeout,
|
||||
/// 请求取消。
|
||||
@@ -1003,6 +1010,7 @@ impl GatewayErrorKind {
|
||||
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::UpstreamUnavailable => "upstream_unavailable",
|
||||
Self::MessageTooBig => "message_too_big",
|
||||
Self::Timeout => "timeout",
|
||||
Self::Cancelled => "cancelled",
|
||||
Self::Internal => "internal_error",
|
||||
@@ -1113,6 +1121,10 @@ impl GatewayError {
|
||||
ProviderErrorKind::Timeout => {
|
||||
Self::new(GatewayErrorKind::Timeout, "upstream request timed out")
|
||||
}
|
||||
ProviderErrorKind::MessageTooBig => Self::new(
|
||||
GatewayErrorKind::MessageTooBig,
|
||||
"upstream rejected the request because the message is too large",
|
||||
),
|
||||
ProviderErrorKind::Cancelled => {
|
||||
Self::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
}
|
||||
|
||||
@@ -2832,6 +2832,10 @@ fn only_provider_attributable_failures_should_affect_circuit() {
|
||||
assert!(!provider_failure_affects_circuit(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable
|
||||
));
|
||||
// 上游 close 1009(message too big)是请求自身的问题:不得熔断 provider。
|
||||
assert!(!provider_failure_affects_circuit(
|
||||
ProviderErrorKind::MessageTooBig
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -203,6 +203,26 @@ fn continuation_recovery_should_keep_its_internal_classification_and_client_cont
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_too_big_should_map_to_a_request_scoped_client_contract() {
|
||||
let error = ProviderError::new(ProviderErrorKind::MessageTooBig, UpstreamSendState::Ambiguous)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"upstream websocket message too big",
|
||||
Some("message_too_big".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
));
|
||||
let gateway = GatewayError::from_provider(&error);
|
||||
|
||||
assert_eq!(error.kind().as_str(), "message_too_big");
|
||||
assert_eq!(gateway.kind().as_str(), "message_too_big");
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"upstream rejected the request because the message is too large"
|
||||
);
|
||||
assert_eq!(gateway.client_error_code(), Some("message_too_big"));
|
||||
assert_eq!(gateway.client_error_type(), Some("invalid_request_error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
|
||||
let message = format!("\0{}\n", "m".repeat(9_000));
|
||||
|
||||
@@ -854,7 +854,7 @@ fn project_wire(wire: ProtocolWireEvent) -> Result<WireEvent, PluginFault> {
|
||||
|
||||
pub(super) fn gateway_fault(error: GatewayError) -> PluginFault {
|
||||
let code = match error.kind() {
|
||||
GatewayErrorKind::InvalidRequest => ErrorCode::InvalidInput,
|
||||
GatewayErrorKind::InvalidRequest | GatewayErrorKind::MessageTooBig => ErrorCode::InvalidInput,
|
||||
GatewayErrorKind::Unsupported | GatewayErrorKind::ModelNotFound => ErrorCode::Unsupported,
|
||||
GatewayErrorKind::Unauthorized => ErrorCode::PermissionDenied,
|
||||
GatewayErrorKind::PolicyDenied => ErrorCode::Rejected,
|
||||
|
||||
@@ -1227,6 +1227,22 @@ pub(super) fn websocket_client_visible_error(
|
||||
));
|
||||
}
|
||||
let close = error.close_before_terminal()?;
|
||||
// RFC 6455 1009 = message too big:客户端必须能识别并缩小/重建请求,
|
||||
// 因此投影成结构化的 invalid_request_error/message_too_big,而不是裸 close 码。
|
||||
if close.code() == Some(WEBSOCKET_CLOSE_MESSAGE_TOO_BIG) {
|
||||
let message = close
|
||||
.reason()
|
||||
.filter(|reason| !reason.is_empty())
|
||||
.map_or_else(
|
||||
|| "upstream websocket message too big".to_owned(),
|
||||
str::to_owned,
|
||||
);
|
||||
return Some(ClientVisibleUpstreamError::new(
|
||||
message,
|
||||
Some("message_too_big".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
));
|
||||
}
|
||||
let message = close
|
||||
.reason()
|
||||
.filter(|reason| !reason.is_empty())
|
||||
@@ -1487,11 +1503,24 @@ pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> Provi
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached(_) => ProviderErrorKind::RateLimited,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
| CodexWebSocketExchangeError::Connect(_)
|
||||
| CodexWebSocketExchangeError::PostSendAmbiguous { .. }
|
||||
| CodexWebSocketExchangeError::ClosedBeforeTerminal(_)
|
||||
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. } => {
|
||||
ProviderErrorKind::Transport
|
||||
}
|
||||
// 上游 close 1009 是 RFC 6455 的 "message too big":请求自身超出上游
|
||||
// WS 消息上限。换账号或熔断 Provider 都无法让该请求成功,必须让客户端
|
||||
// 感知到可行动的 message_too_big 语义,且不得计入 provider 熔断。
|
||||
// live 流错误会先被 post_send_ambiguous 包装,需从 source 解出 close 码。
|
||||
CodexWebSocketExchangeError::ClosedBeforeTerminal(_)
|
||||
| CodexWebSocketExchangeError::PostSendAmbiguous { .. } => {
|
||||
if error
|
||||
.close_before_terminal()
|
||||
.is_some_and(|close| close.code() == Some(WEBSOCKET_CLOSE_MESSAGE_TOO_BIG))
|
||||
{
|
||||
ProviderErrorKind::MessageTooBig
|
||||
} else {
|
||||
ProviderErrorKind::Transport
|
||||
}
|
||||
}
|
||||
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
|
||||
unreachable!("classified websocket errors never retain observation wrappers")
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ use crate::transport::request::{
|
||||
};
|
||||
use crate::transport::session::CodexSessionIdentity;
|
||||
use crate::transport::usage::normalize_service_tier;
|
||||
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason};
|
||||
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason, WEBSOCKET_CLOSE_MESSAGE_TOO_BIG};
|
||||
use crate::transport::{
|
||||
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
|
||||
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
|
||||
|
||||
@@ -6,6 +6,9 @@ use gateway_protocol::openai::sse::SseError;
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// RFC 6455 close code 1009:上游拒收超出大小限制的消息。
|
||||
pub(crate) const WEBSOCKET_CLOSE_MESSAGE_TOO_BIG: u16 = 1009;
|
||||
|
||||
use crate::transport::client::CodexClientVisibleUpstreamResponse;
|
||||
use crate::transport::diagnostics::CodexUpstreamDiagnostics;
|
||||
use crate::transport::diagnostics::CodexUpstreamSendPhase;
|
||||
|
||||
@@ -41,5 +41,6 @@ pub(crate) use self::{
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool,
|
||||
},
|
||||
error::WEBSOCKET_CLOSE_MESSAGE_TOO_BIG,
|
||||
pool::DEFAULT_STREAM_IDLE_TIMEOUT,
|
||||
};
|
||||
|
||||
@@ -3592,7 +3592,9 @@ async fn websocket_close_after_delivery_preserves_details_and_reconnects_through
|
||||
|
||||
assert!(!format!("{error:?}").contains("message too big"));
|
||||
assert!(!error.to_string().contains("message too big"));
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Transport);
|
||||
// 上游 close 1009 是 RFC 6455 "message too big":必须归因为请求自身问题,
|
||||
// 而不是 provider 传输故障(否则会被熔断器和换号逻辑误伤其他请求)。
|
||||
assert_eq!(error.kind(), ProviderErrorKind::MessageTooBig);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
|
||||
assert!(
|
||||
!error.allows_pre_delivery_retry(),
|
||||
@@ -3602,8 +3604,8 @@ async fn websocket_close_after_delivery_preserves_details_and_reconnects_through
|
||||
.client_visible_upstream_error()
|
||||
.expect("WebSocket close detail");
|
||||
assert_eq!(detail.message(), "message too big");
|
||||
assert_eq!(detail.code(), Some("1009"));
|
||||
assert_eq!(detail.error_type(), Some("websocket_close_error"));
|
||||
assert_eq!(detail.code(), Some("message_too_big"));
|
||||
assert_eq!(detail.error_type(), Some("invalid_request_error"));
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some("websocket_close_1009")
|
||||
|
||||
Reference in New Issue
Block a user