mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-21 08:01:41 +00:00
fix(openai): release websocket resources under backpressure
Make event delivery respond to downstream cancellation and pool shutdown. Return or discard sockets before publishing terminal events while preserving normal connection reuse. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -97,12 +97,14 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
|
||||
pool_return,
|
||||
reused_connection,
|
||||
stream_idle_timeout,
|
||||
trace,
|
||||
shutdown,
|
||||
downstream_cancellation,
|
||||
delivery: WebSocketStreamDelivery {
|
||||
trace,
|
||||
shutdown,
|
||||
downstream_cancellation,
|
||||
tx,
|
||||
},
|
||||
rate_limit_updates: rate_limit_updates_for_task,
|
||||
response_metadata_updates: response_metadata_updates_for_task,
|
||||
tx,
|
||||
})
|
||||
.await;
|
||||
};
|
||||
@@ -133,33 +135,69 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
|
||||
}
|
||||
|
||||
struct WebSocketStreamForwardState {
|
||||
trace: TraceContext,
|
||||
websocket: PumpedWebSocket,
|
||||
metadata: CodexWebSocketConnectionMetadata,
|
||||
pool_return: Option<WebSocketStreamPoolReturn>,
|
||||
reused_connection: bool,
|
||||
stream_idle_timeout: Option<Duration>,
|
||||
shutdown: CancellationToken,
|
||||
downstream_cancellation: Option<gateway_core::lifecycle::CancellationToken>,
|
||||
rate_limit_updates: CodexWebSocketRateLimitUpdates,
|
||||
response_metadata_updates: CodexWebSocketResponseMetadataUpdates,
|
||||
delivery: WebSocketStreamDelivery,
|
||||
}
|
||||
|
||||
struct WebSocketStreamDelivery {
|
||||
trace: TraceContext,
|
||||
shutdown: CancellationToken,
|
||||
downstream_cancellation: Option<gateway_core::lifecycle::CancellationToken>,
|
||||
tx: mpsc::Sender<Result<Bytes, CodexWebSocketExchangeError>>,
|
||||
}
|
||||
|
||||
impl WebSocketStreamDelivery {
|
||||
async fn cancelled(&self) -> StreamWebSocketDiscardReason {
|
||||
let (reason, discard_reason) = tokio::select! {
|
||||
biased;
|
||||
() = async {
|
||||
match &self.downstream_cancellation {
|
||||
Some(cancellation) => cancellation.cancelled().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
} => ("downstream_connection_closed", StreamWebSocketDiscardReason::ClientCancelled),
|
||||
() = self.tx.closed() => ("receiver_dropped", StreamWebSocketDiscardReason::ClientCancelled),
|
||||
() = self.shutdown.cancelled() => ("pool_shutdown", StreamWebSocketDiscardReason::PoolShutdown),
|
||||
};
|
||||
self.trace
|
||||
.record("upstream.cancelled", json!({"reason": reason}));
|
||||
discard_reason
|
||||
}
|
||||
|
||||
async fn send(
|
||||
&self,
|
||||
event: Result<Bytes, CodexWebSocketExchangeError>,
|
||||
) -> Result<(), StreamWebSocketDiscardReason> {
|
||||
// 终态与错误也可能被满队列阻塞,所有交付等待都必须响应取消和停机
|
||||
tokio::select! {
|
||||
biased;
|
||||
reason = self.cancelled() => Err(reason),
|
||||
result = self.tx.send(event) => result.map_err(|_| {
|
||||
self.trace.record("upstream.forward.failed", json!({"reason": "receiver_dropped"}));
|
||||
StreamWebSocketDiscardReason::DownstreamSendFailed
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
let WebSocketStreamForwardState {
|
||||
trace,
|
||||
mut websocket,
|
||||
mut metadata,
|
||||
pool_return,
|
||||
reused_connection,
|
||||
stream_idle_timeout,
|
||||
shutdown,
|
||||
downstream_cancellation,
|
||||
rate_limit_updates,
|
||||
response_metadata_updates,
|
||||
tx,
|
||||
delivery,
|
||||
} = state;
|
||||
let trace = &delivery.trace;
|
||||
let mut pool_return = pool_return;
|
||||
let mut continuation = pool_return
|
||||
.as_mut()
|
||||
@@ -172,38 +210,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
biased;
|
||||
() = async {
|
||||
match &downstream_cancellation {
|
||||
Some(cancellation) => cancellation.cancelled().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
} => {
|
||||
trace.record("upstream.cancelled", json!({"reason": "downstream_connection_closed"}));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::ClientCancelled,
|
||||
).await;
|
||||
return;
|
||||
}
|
||||
// 客户端断开下游 SSE 流:立即丢弃连接并释放池 slot,
|
||||
// 不再傻等上游 idle 超时(否则同会话后续请求会一直 bypass/busy)
|
||||
() = tx.closed() => {
|
||||
trace.record("upstream.cancelled", json!({"reason": "receiver_dropped"}));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::ClientCancelled,
|
||||
).await;
|
||||
return;
|
||||
}
|
||||
() = shutdown.cancelled() => {
|
||||
trace.record("upstream.cancelled", json!({"reason": "pool_shutdown"}));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::PoolShutdown,
|
||||
).await;
|
||||
reason = delivery.cancelled() => {
|
||||
discard_stream_websocket(websocket, pool_return, reason).await;
|
||||
return;
|
||||
}
|
||||
message = next_websocket_message(
|
||||
@@ -239,7 +247,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
} else {
|
||||
error
|
||||
};
|
||||
let _ = tx.send(Err(error)).await;
|
||||
let _ = delivery.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -258,7 +266,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
StreamWebSocketDiscardReason::UnexpectedBinaryEvent,
|
||||
)
|
||||
.await;
|
||||
let _ = tx
|
||||
let _ = delivery
|
||||
.send(Err(error.with_connection_observation(observation)))
|
||||
.await;
|
||||
return;
|
||||
@@ -293,7 +301,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
} else {
|
||||
error
|
||||
};
|
||||
let _ = tx.send(Err(error)).await;
|
||||
let _ = delivery.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
_ => continue,
|
||||
@@ -315,7 +323,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
Some(observation) => error.with_connection_observation(observation),
|
||||
None => error,
|
||||
};
|
||||
let _ = tx.send(Err(error)).await;
|
||||
let _ = delivery.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -336,19 +344,6 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
ExchangeAction::Forward { terminal } => terminal,
|
||||
ExchangeAction::Ignore => continue,
|
||||
};
|
||||
if tx.send(Ok(Bytes::from(raw))).await.is_err() {
|
||||
trace.record(
|
||||
"upstream.forward.failed",
|
||||
json!({"reason": "receiver_dropped"}),
|
||||
);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::DownstreamSendFailed,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
if let Some(terminal) = terminal {
|
||||
trace.record(
|
||||
"upstream.terminal",
|
||||
@@ -377,6 +372,12 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
.await;
|
||||
}
|
||||
}
|
||||
// 先归还续接状态或释放 socket,终态交付不能延迟资源收尾
|
||||
let _ = delivery.send(Ok(Bytes::from(raw))).await;
|
||||
return;
|
||||
}
|
||||
if let Err(reason) = delivery.send(Ok(Bytes::from(raw))).await {
|
||||
discard_stream_websocket(websocket, pool_return, reason).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -427,7 +428,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
} else {
|
||||
error
|
||||
};
|
||||
let _ = tx.send(Err(error)).await;
|
||||
let _ = delivery.send(Err(error)).await;
|
||||
}
|
||||
|
||||
async fn finish_stream_websocket(
|
||||
|
||||
@@ -1,8 +1,213 @@
|
||||
//! 验证上游停止读请求时,发送超时仍能返回并释放连接
|
||||
//! 验证上下游背压、取消与终态下的连接和任务回收
|
||||
|
||||
use super::*;
|
||||
use gateway_core::{
|
||||
diagnostics::TraceContext, engine::response_control::ResponseControl,
|
||||
lifecycle::CancellationToken,
|
||||
};
|
||||
use std::assert_matches;
|
||||
|
||||
async fn wait_for_forwarded_frames(trace: &TraceContext, stage: &str, count: u64) {
|
||||
timeout(Duration::from_secs(2), async {
|
||||
while trace.snapshot().unwrap()["stageCounts"][stage]
|
||||
.as_u64()
|
||||
.unwrap_or_default()
|
||||
< count
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("forwarder should reach the backpressured frame");
|
||||
}
|
||||
|
||||
fn response_tail(kind: &str) -> Message {
|
||||
match kind {
|
||||
"binary" => Message::Binary(vec![0].into()),
|
||||
"delta" => Message::Text(
|
||||
json!({"type":"response.output_text.delta", "delta":"x"})
|
||||
.to_string()
|
||||
.into(),
|
||||
),
|
||||
"completed" => {
|
||||
Message::Text(completed_websocket_response("resp_backpressure", 2, 1).into())
|
||||
}
|
||||
kind => Message::Text(
|
||||
json!({"type":format!("response.{kind}"), "response":{
|
||||
"id":"resp_backpressure", "status":kind, "output":[]
|
||||
}})
|
||||
.to_string()
|
||||
.into(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_backpressured_response(
|
||||
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
|
||||
tail: &str,
|
||||
) {
|
||||
// 先填满交付队列,后续帧由诊断计数确认已到达 forwarder,不靠 sleep 猜测调度
|
||||
for _ in 0..16 {
|
||||
websocket.send(response_tail("delta")).await.unwrap();
|
||||
}
|
||||
websocket.send(response_tail(tail)).await.unwrap();
|
||||
}
|
||||
|
||||
async fn expect_upstream_socket_closed(
|
||||
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
|
||||
) {
|
||||
let close = timeout(Duration::from_secs(2), websocket.next())
|
||||
.await
|
||||
.expect("cancellation must release the socket while the receiver remains alive");
|
||||
assert_matches!(
|
||||
close,
|
||||
None | Some(Ok(Message::Close(_)))
|
||||
| Some(Err(tungstenite::Error::Protocol(
|
||||
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake
|
||||
)))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backpressured_delivery_should_stop_on_downstream_cancel_or_pool_shutdown() {
|
||||
for cancel_downstream in [true, false] {
|
||||
for tail in ["delta", "binary", "completed", "failed", "incomplete"] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let mut server = tokio::spawn(async move {
|
||||
let (socket, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(socket).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
send_backpressured_response(&mut websocket, tail).await;
|
||||
expect_upstream_socket_closed(&mut websocket).await;
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let cancellation = CancellationToken::new();
|
||||
let control =
|
||||
ResponseControl::default().with_connection_cancellation(cancellation.clone());
|
||||
let trace = TraceContext::new("backpressured_delivery");
|
||||
let mut context = request_context("backpressured_delivery", Some("chatgpt-account"))
|
||||
.with_response_control(Some(&control));
|
||||
context.trace = Some(&trace);
|
||||
let mut request = pooled_websocket_request("backpressured_delivery");
|
||||
request.downstream_websocket_connection_id = Some("downstream_backpressure".into());
|
||||
let mut response = backend
|
||||
.create_response_stream(&request, context)
|
||||
.await
|
||||
.unwrap();
|
||||
let (stage, count) = if tail == "binary" {
|
||||
("upstream.binary", 1)
|
||||
} else {
|
||||
("upstream.event", 17)
|
||||
};
|
||||
wait_for_forwarded_frames(&trace, stage, count).await;
|
||||
|
||||
if cancel_downstream {
|
||||
cancellation.cancel();
|
||||
// 必须在 shutdown 前看到物理连接关闭,避免停机掩盖下游取消失效
|
||||
(&mut server).await.unwrap();
|
||||
}
|
||||
timeout(Duration::from_secs(2), pool.shutdown())
|
||||
.await
|
||||
.expect("shutdown must join delivery even with a full receiver");
|
||||
if !cancel_downstream {
|
||||
(&mut server).await.unwrap();
|
||||
}
|
||||
// 收尾前不消费也不丢弃 body,已交付的事件仍可正常读完
|
||||
let mut buffered = 0;
|
||||
while response.body.next().await.transpose().unwrap().is_some() {
|
||||
buffered += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
buffered, 16,
|
||||
"tail={tail}, cancel_downstream={cancel_downstream}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_should_release_or_reuse_socket_before_delivery() {
|
||||
for tail in ["completed", "failed", "incomplete"] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let mut server = tokio::spawn(async move {
|
||||
let (socket, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(socket).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
send_backpressured_response(&mut websocket, tail).await;
|
||||
if tail == "completed" {
|
||||
let next = timeout(Duration::from_secs(2), websocket.next())
|
||||
.await
|
||||
.expect("continuation should reuse the completed socket")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let Message::Text(next) = next else {
|
||||
panic!("expected continuation request");
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&next).unwrap()["previous_response_id"],
|
||||
"resp_backpressure"
|
||||
);
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_next", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
expect_upstream_socket_closed(&mut websocket).await;
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let trace = TraceContext::new("terminal_delivery");
|
||||
let mut context = request_context("terminal_delivery", Some("chatgpt-account"));
|
||||
context.trace = Some(&trace);
|
||||
let mut request = pooled_websocket_request("terminal_delivery");
|
||||
let mut response = backend
|
||||
.create_response_stream(&request, context)
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_forwarded_frames(&trace, "upstream.event", 17).await;
|
||||
if tail == "completed" {
|
||||
request.set_previous_response_id(Some("resp_backpressure".into()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
let next = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("terminal_next", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("completed continuation should be reusable before terminal consumption");
|
||||
assert!(next.websocket_pool_decision.unwrap().is_reuse());
|
||||
} else {
|
||||
// 失败终态必须释放 socket,不能依赖客户端读走终态
|
||||
(&mut server).await.unwrap();
|
||||
}
|
||||
let mut events = Vec::new();
|
||||
while let Some(chunk) = response.body.next().await {
|
||||
events.push(serde_json::from_slice::<Value>(&chunk.unwrap()).unwrap());
|
||||
}
|
||||
assert_eq!(events.len(), 17);
|
||||
assert_eq!(events.last().unwrap()["type"], format!("response.{tail}"));
|
||||
pool.shutdown().await;
|
||||
if tail == "completed" {
|
||||
(&mut server).await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_timeout_should_abort_stalled_pump_and_allow_next_request() {
|
||||
let socket = tokio::net::TcpSocket::new_v4().unwrap();
|
||||
|
||||
@@ -531,6 +531,7 @@ OpenAI 的连接池监督该取消信号,按当前归属保护绑定连接;
|
||||
重新绑定或切到 HTTP 时移除失去用途的旧连接,失效 reservation 不能重新归池。
|
||||
普通空闲期限复用池槽位的 `idle_since`,只在归池时重置;心跳不刷新业务空闲,建连与在途不参与空闲回收。
|
||||
回收先移出可复用集合,再在锁外有界关闭并等待 pump/socket 释放,物理容量随 socket 归还;批量关闭并行执行。
|
||||
上游转发等待下游消费时仍响应连接取消和停机;终态先完成连接归还或销毁,再交付下游。
|
||||
Codex OAuth 上游 WS 在首个请求写入后记录搜索声明,握手完成但尚未发送请求的连接仍未初始化。
|
||||
后续首次声明搜索时,连接池回收不兼容的原连接;普通新链重新建连,连接内续接交由客户端完整重放。
|
||||
该判断共用于 HTTP 与 WS 入站,不修改客户端工具声明,也不保留对话正文。
|
||||
|
||||
Reference in New Issue
Block a user