fix(openai): release websocket resources under backpressure

Make event delivery respond to downstream cancellation and pool shutdown. Return or discard sockets before publishing terminal events while preserving normal connection reuse.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-11-13 15:38:22 +08:00
co-authored by Codex
parent e1f84c210b
commit f85d80d756
3 changed files with 269 additions and 62 deletions
@@ -97,12 +97,14 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
pool_return,
reused_connection,
stream_idle_timeout,
trace,
shutdown,
downstream_cancellation,
delivery: WebSocketStreamDelivery {
trace,
shutdown,
downstream_cancellation,
tx,
},
rate_limit_updates: rate_limit_updates_for_task,
response_metadata_updates: response_metadata_updates_for_task,
tx,
})
.await;
};
@@ -133,33 +135,69 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
}
struct WebSocketStreamForwardState {
trace: TraceContext,
websocket: PumpedWebSocket,
metadata: CodexWebSocketConnectionMetadata,
pool_return: Option<WebSocketStreamPoolReturn>,
reused_connection: bool,
stream_idle_timeout: Option<Duration>,
shutdown: CancellationToken,
downstream_cancellation: Option<gateway_core::lifecycle::CancellationToken>,
rate_limit_updates: CodexWebSocketRateLimitUpdates,
response_metadata_updates: CodexWebSocketResponseMetadataUpdates,
delivery: WebSocketStreamDelivery,
}
struct WebSocketStreamDelivery {
trace: TraceContext,
shutdown: CancellationToken,
downstream_cancellation: Option<gateway_core::lifecycle::CancellationToken>,
tx: mpsc::Sender<Result<Bytes, CodexWebSocketExchangeError>>,
}
impl WebSocketStreamDelivery {
async fn cancelled(&self) -> StreamWebSocketDiscardReason {
let (reason, discard_reason) = tokio::select! {
biased;
() = async {
match &self.downstream_cancellation {
Some(cancellation) => cancellation.cancelled().await,
None => std::future::pending().await,
}
} => ("downstream_connection_closed", StreamWebSocketDiscardReason::ClientCancelled),
() = self.tx.closed() => ("receiver_dropped", StreamWebSocketDiscardReason::ClientCancelled),
() = self.shutdown.cancelled() => ("pool_shutdown", StreamWebSocketDiscardReason::PoolShutdown),
};
self.trace
.record("upstream.cancelled", json!({"reason": reason}));
discard_reason
}
async fn send(
&self,
event: Result<Bytes, CodexWebSocketExchangeError>,
) -> Result<(), StreamWebSocketDiscardReason> {
// 终态与错误也可能被满队列阻塞,所有交付等待都必须响应取消和停机
tokio::select! {
biased;
reason = self.cancelled() => Err(reason),
result = self.tx.send(event) => result.map_err(|_| {
self.trace.record("upstream.forward.failed", json!({"reason": "receiver_dropped"}));
StreamWebSocketDiscardReason::DownstreamSendFailed
}),
}
}
}
async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
let WebSocketStreamForwardState {
trace,
mut websocket,
mut metadata,
pool_return,
reused_connection,
stream_idle_timeout,
shutdown,
downstream_cancellation,
rate_limit_updates,
response_metadata_updates,
tx,
delivery,
} = state;
let trace = &delivery.trace;
let mut pool_return = pool_return;
let mut continuation = pool_return
.as_mut()
@@ -172,38 +210,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
loop {
let message = tokio::select! {
biased;
() = async {
match &downstream_cancellation {
Some(cancellation) => cancellation.cancelled().await,
None => std::future::pending().await,
}
} => {
trace.record("upstream.cancelled", json!({"reason": "downstream_connection_closed"}));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::ClientCancelled,
).await;
return;
}
// 客户端断开下游 SSE 流:立即丢弃连接并释放池 slot,
// 不再傻等上游 idle 超时(否则同会话后续请求会一直 bypass/busy)
() = tx.closed() => {
trace.record("upstream.cancelled", json!({"reason": "receiver_dropped"}));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::ClientCancelled,
).await;
return;
}
() = shutdown.cancelled() => {
trace.record("upstream.cancelled", json!({"reason": "pool_shutdown"}));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::PoolShutdown,
).await;
reason = delivery.cancelled() => {
discard_stream_websocket(websocket, pool_return, reason).await;
return;
}
message = next_websocket_message(
@@ -239,7 +247,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
} else {
error
};
let _ = tx.send(Err(error)).await;
let _ = delivery.send(Err(error)).await;
return;
}
};
@@ -258,7 +266,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
StreamWebSocketDiscardReason::UnexpectedBinaryEvent,
)
.await;
let _ = tx
let _ = delivery
.send(Err(error.with_connection_observation(observation)))
.await;
return;
@@ -293,7 +301,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
} else {
error
};
let _ = tx.send(Err(error)).await;
let _ = delivery.send(Err(error)).await;
return;
}
_ => continue,
@@ -315,7 +323,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
Some(observation) => error.with_connection_observation(observation),
None => error,
};
let _ = tx.send(Err(error)).await;
let _ = delivery.send(Err(error)).await;
return;
}
};
@@ -336,19 +344,6 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
ExchangeAction::Forward { terminal } => terminal,
ExchangeAction::Ignore => continue,
};
if tx.send(Ok(Bytes::from(raw))).await.is_err() {
trace.record(
"upstream.forward.failed",
json!({"reason": "receiver_dropped"}),
);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::DownstreamSendFailed,
)
.await;
return;
}
if let Some(terminal) = terminal {
trace.record(
"upstream.terminal",
@@ -377,6 +372,12 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
.await;
}
}
// 先归还续接状态或释放 socket,终态交付不能延迟资源收尾
let _ = delivery.send(Ok(Bytes::from(raw))).await;
return;
}
if let Err(reason) = delivery.send(Ok(Bytes::from(raw))).await {
discard_stream_websocket(websocket, pool_return, reason).await;
return;
}
}
@@ -427,7 +428,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
} else {
error
};
let _ = tx.send(Err(error)).await;
let _ = delivery.send(Err(error)).await;
}
async fn finish_stream_websocket(
@@ -1,8 +1,213 @@
//! 验证上游停止读请求时,发送超时仍能返回并释放连接
//! 验证上下游背压、取消与终态下的连接和任务回收
use super::*;
use gateway_core::{
diagnostics::TraceContext, engine::response_control::ResponseControl,
lifecycle::CancellationToken,
};
use std::assert_matches;
async fn wait_for_forwarded_frames(trace: &TraceContext, stage: &str, count: u64) {
timeout(Duration::from_secs(2), async {
while trace.snapshot().unwrap()["stageCounts"][stage]
.as_u64()
.unwrap_or_default()
< count
{
tokio::task::yield_now().await;
}
})
.await
.expect("forwarder should reach the backpressured frame");
}
fn response_tail(kind: &str) -> Message {
match kind {
"binary" => Message::Binary(vec![0].into()),
"delta" => Message::Text(
json!({"type":"response.output_text.delta", "delta":"x"})
.to_string()
.into(),
),
"completed" => {
Message::Text(completed_websocket_response("resp_backpressure", 2, 1).into())
}
kind => Message::Text(
json!({"type":format!("response.{kind}"), "response":{
"id":"resp_backpressure", "status":kind, "output":[]
}})
.to_string()
.into(),
),
}
}
async fn send_backpressured_response(
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
tail: &str,
) {
// 先填满交付队列,后续帧由诊断计数确认已到达 forwarder,不靠 sleep 猜测调度
for _ in 0..16 {
websocket.send(response_tail("delta")).await.unwrap();
}
websocket.send(response_tail(tail)).await.unwrap();
}
async fn expect_upstream_socket_closed(
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
) {
let close = timeout(Duration::from_secs(2), websocket.next())
.await
.expect("cancellation must release the socket while the receiver remains alive");
assert_matches!(
close,
None | Some(Ok(Message::Close(_)))
| Some(Err(tungstenite::Error::Protocol(
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake
)))
);
}
#[tokio::test]
async fn backpressured_delivery_should_stop_on_downstream_cancel_or_pool_shutdown() {
for cancel_downstream in [true, false] {
for tail in ["delta", "binary", "completed", "failed", "incomplete"] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let mut server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.unwrap();
let mut websocket = accept_codex_test_websocket(socket).await;
websocket.next().await.unwrap().unwrap();
send_backpressured_response(&mut websocket, tail).await;
expect_upstream_socket_closed(&mut websocket).await;
});
let pool = Arc::new(CodexWebSocketPool::default());
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
)
.with_websocket_pool(Arc::clone(&pool));
let cancellation = CancellationToken::new();
let control =
ResponseControl::default().with_connection_cancellation(cancellation.clone());
let trace = TraceContext::new("backpressured_delivery");
let mut context = request_context("backpressured_delivery", Some("chatgpt-account"))
.with_response_control(Some(&control));
context.trace = Some(&trace);
let mut request = pooled_websocket_request("backpressured_delivery");
request.downstream_websocket_connection_id = Some("downstream_backpressure".into());
let mut response = backend
.create_response_stream(&request, context)
.await
.unwrap();
let (stage, count) = if tail == "binary" {
("upstream.binary", 1)
} else {
("upstream.event", 17)
};
wait_for_forwarded_frames(&trace, stage, count).await;
if cancel_downstream {
cancellation.cancel();
// 必须在 shutdown 前看到物理连接关闭,避免停机掩盖下游取消失效
(&mut server).await.unwrap();
}
timeout(Duration::from_secs(2), pool.shutdown())
.await
.expect("shutdown must join delivery even with a full receiver");
if !cancel_downstream {
(&mut server).await.unwrap();
}
// 收尾前不消费也不丢弃 body,已交付的事件仍可正常读完
let mut buffered = 0;
while response.body.next().await.transpose().unwrap().is_some() {
buffered += 1;
}
assert_eq!(
buffered, 16,
"tail={tail}, cancel_downstream={cancel_downstream}"
);
}
}
}
#[tokio::test]
async fn terminal_should_release_or_reuse_socket_before_delivery() {
for tail in ["completed", "failed", "incomplete"] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let mut server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.unwrap();
let mut websocket = accept_codex_test_websocket(socket).await;
websocket.next().await.unwrap().unwrap();
send_backpressured_response(&mut websocket, tail).await;
if tail == "completed" {
let next = timeout(Duration::from_secs(2), websocket.next())
.await
.expect("continuation should reuse the completed socket")
.unwrap()
.unwrap();
let Message::Text(next) = next else {
panic!("expected continuation request");
};
assert_eq!(
serde_json::from_str::<Value>(&next).unwrap()["previous_response_id"],
"resp_backpressure"
);
websocket
.send(Message::Text(
completed_websocket_response("resp_next", 2, 1).into(),
))
.await
.unwrap();
}
expect_upstream_socket_closed(&mut websocket).await;
});
let pool = Arc::new(CodexWebSocketPool::default());
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
)
.with_websocket_pool(Arc::clone(&pool));
let trace = TraceContext::new("terminal_delivery");
let mut context = request_context("terminal_delivery", Some("chatgpt-account"));
context.trace = Some(&trace);
let mut request = pooled_websocket_request("terminal_delivery");
let mut response = backend
.create_response_stream(&request, context)
.await
.unwrap();
wait_for_forwarded_frames(&trace, "upstream.event", 17).await;
if tail == "completed" {
request.set_previous_response_id(Some("resp_backpressure".into()));
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let next = backend
.create_response(
&request,
request_context("terminal_next", Some("chatgpt-account")),
)
.await
.expect("completed continuation should be reusable before terminal consumption");
assert!(next.websocket_pool_decision.unwrap().is_reuse());
} else {
// 失败终态必须释放 socket,不能依赖客户端读走终态
(&mut server).await.unwrap();
}
let mut events = Vec::new();
while let Some(chunk) = response.body.next().await {
events.push(serde_json::from_slice::<Value>(&chunk.unwrap()).unwrap());
}
assert_eq!(events.len(), 17);
assert_eq!(events.last().unwrap()["type"], format!("response.{tail}"));
pool.shutdown().await;
if tail == "completed" {
(&mut server).await.unwrap();
}
}
}
#[tokio::test]
async fn send_timeout_should_abort_stalled_pump_and_allow_next_request() {
let socket = tokio::net::TcpSocket::new_v4().unwrap();
+1
View File
@@ -531,6 +531,7 @@ OpenAI 的连接池监督该取消信号,按当前归属保护绑定连接;
重新绑定或切到 HTTP 时移除失去用途的旧连接,失效 reservation 不能重新归池。
普通空闲期限复用池槽位的 `idle_since`,只在归池时重置;心跳不刷新业务空闲,建连与在途不参与空闲回收。
回收先移出可复用集合,再在锁外有界关闭并等待 pump/socket 释放,物理容量随 socket 归还;批量关闭并行执行。
上游转发等待下游消费时仍响应连接取消和停机;终态先完成连接归还或销毁,再交付下游。
Codex OAuth 上游 WS 在首个请求写入后记录搜索声明,握手完成但尚未发送请求的连接仍未初始化。
后续首次声明搜索时,连接池回收不兼容的原连接;普通新链重新建连,连接内续接交由客户端完整重放。
该判断共用于 HTTP 与 WS 入站,不修改客户端工具声明,也不保留对话正文。