diff --git a/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs b/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs index c223bcb67..53fe39660 100644 --- a/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs +++ b/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs @@ -97,12 +97,14 @@ pub(in crate::transport::websocket) fn stream_websocket_response( pool_return, reused_connection, stream_idle_timeout, - trace, - shutdown, - downstream_cancellation, + delivery: WebSocketStreamDelivery { + trace, + shutdown, + downstream_cancellation, + tx, + }, rate_limit_updates: rate_limit_updates_for_task, response_metadata_updates: response_metadata_updates_for_task, - tx, }) .await; }; @@ -133,33 +135,69 @@ pub(in crate::transport::websocket) fn stream_websocket_response( } struct WebSocketStreamForwardState { - trace: TraceContext, websocket: PumpedWebSocket, metadata: CodexWebSocketConnectionMetadata, pool_return: Option, reused_connection: bool, stream_idle_timeout: Option, - shutdown: CancellationToken, - downstream_cancellation: Option, rate_limit_updates: CodexWebSocketRateLimitUpdates, response_metadata_updates: CodexWebSocketResponseMetadataUpdates, + delivery: WebSocketStreamDelivery, +} + +struct WebSocketStreamDelivery { + trace: TraceContext, + shutdown: CancellationToken, + downstream_cancellation: Option, tx: mpsc::Sender>, } +impl WebSocketStreamDelivery { + async fn cancelled(&self) -> StreamWebSocketDiscardReason { + let (reason, discard_reason) = tokio::select! { + biased; + () = async { + match &self.downstream_cancellation { + Some(cancellation) => cancellation.cancelled().await, + None => std::future::pending().await, + } + } => ("downstream_connection_closed", StreamWebSocketDiscardReason::ClientCancelled), + () = self.tx.closed() => ("receiver_dropped", StreamWebSocketDiscardReason::ClientCancelled), + () = self.shutdown.cancelled() => ("pool_shutdown", StreamWebSocketDiscardReason::PoolShutdown), + }; + self.trace + .record("upstream.cancelled", json!({"reason": reason})); + discard_reason + } + + async fn send( + &self, + event: Result, + ) -> Result<(), StreamWebSocketDiscardReason> { + // 终态与错误也可能被满队列阻塞,所有交付等待都必须响应取消和停机 + tokio::select! { + biased; + reason = self.cancelled() => Err(reason), + result = self.tx.send(event) => result.map_err(|_| { + self.trace.record("upstream.forward.failed", json!({"reason": "receiver_dropped"})); + StreamWebSocketDiscardReason::DownstreamSendFailed + }), + } + } +} + async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { let WebSocketStreamForwardState { - trace, mut websocket, mut metadata, pool_return, reused_connection, stream_idle_timeout, - shutdown, - downstream_cancellation, rate_limit_updates, response_metadata_updates, - tx, + delivery, } = state; + let trace = &delivery.trace; let mut pool_return = pool_return; let mut continuation = pool_return .as_mut() @@ -172,38 +210,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { loop { let message = tokio::select! { biased; - () = async { - match &downstream_cancellation { - Some(cancellation) => cancellation.cancelled().await, - None => std::future::pending().await, - } - } => { - trace.record("upstream.cancelled", json!({"reason": "downstream_connection_closed"})); - discard_stream_websocket( - websocket, - pool_return, - StreamWebSocketDiscardReason::ClientCancelled, - ).await; - return; - } - // 客户端断开下游 SSE 流:立即丢弃连接并释放池 slot, - // 不再傻等上游 idle 超时(否则同会话后续请求会一直 bypass/busy) - () = tx.closed() => { - trace.record("upstream.cancelled", json!({"reason": "receiver_dropped"})); - discard_stream_websocket( - websocket, - pool_return, - StreamWebSocketDiscardReason::ClientCancelled, - ).await; - return; - } - () = shutdown.cancelled() => { - trace.record("upstream.cancelled", json!({"reason": "pool_shutdown"})); - discard_stream_websocket( - websocket, - pool_return, - StreamWebSocketDiscardReason::PoolShutdown, - ).await; + reason = delivery.cancelled() => { + discard_stream_websocket(websocket, pool_return, reason).await; return; } message = next_websocket_message( @@ -239,7 +247,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { } else { error }; - let _ = tx.send(Err(error)).await; + let _ = delivery.send(Err(error)).await; return; } }; @@ -258,7 +266,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { StreamWebSocketDiscardReason::UnexpectedBinaryEvent, ) .await; - let _ = tx + let _ = delivery .send(Err(error.with_connection_observation(observation))) .await; return; @@ -293,7 +301,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { } else { error }; - let _ = tx.send(Err(error)).await; + let _ = delivery.send(Err(error)).await; return; } _ => continue, @@ -315,7 +323,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { Some(observation) => error.with_connection_observation(observation), None => error, }; - let _ = tx.send(Err(error)).await; + let _ = delivery.send(Err(error)).await; return; } }; @@ -336,19 +344,6 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { ExchangeAction::Forward { terminal } => terminal, ExchangeAction::Ignore => continue, }; - if tx.send(Ok(Bytes::from(raw))).await.is_err() { - trace.record( - "upstream.forward.failed", - json!({"reason": "receiver_dropped"}), - ); - discard_stream_websocket( - websocket, - pool_return, - StreamWebSocketDiscardReason::DownstreamSendFailed, - ) - .await; - return; - } if let Some(terminal) = terminal { trace.record( "upstream.terminal", @@ -377,6 +372,12 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { .await; } } + // 先归还续接状态或释放 socket,终态交付不能延迟资源收尾 + let _ = delivery.send(Ok(Bytes::from(raw))).await; + return; + } + if let Err(reason) = delivery.send(Ok(Bytes::from(raw))).await { + discard_stream_websocket(websocket, pool_return, reason).await; return; } } @@ -427,7 +428,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { } else { error }; - let _ = tx.send(Err(error)).await; + let _ = delivery.send(Err(error)).await; } async fn finish_stream_websocket( diff --git a/backend/crates/providers/openai/tests/transport/websocket/cleanup.rs b/backend/crates/providers/openai/tests/transport/websocket/cleanup.rs index aa4028b5a..734f619fe 100644 --- a/backend/crates/providers/openai/tests/transport/websocket/cleanup.rs +++ b/backend/crates/providers/openai/tests/transport/websocket/cleanup.rs @@ -1,8 +1,213 @@ -//! 验证上游停止读请求时,发送超时仍能返回并释放连接 +//! 验证上下游背压、取消与终态下的连接和任务回收 use super::*; +use gateway_core::{ + diagnostics::TraceContext, engine::response_control::ResponseControl, + lifecycle::CancellationToken, +}; use std::assert_matches; +async fn wait_for_forwarded_frames(trace: &TraceContext, stage: &str, count: u64) { + timeout(Duration::from_secs(2), async { + while trace.snapshot().unwrap()["stageCounts"][stage] + .as_u64() + .unwrap_or_default() + < count + { + tokio::task::yield_now().await; + } + }) + .await + .expect("forwarder should reach the backpressured frame"); +} + +fn response_tail(kind: &str) -> Message { + match kind { + "binary" => Message::Binary(vec![0].into()), + "delta" => Message::Text( + json!({"type":"response.output_text.delta", "delta":"x"}) + .to_string() + .into(), + ), + "completed" => { + Message::Text(completed_websocket_response("resp_backpressure", 2, 1).into()) + } + kind => Message::Text( + json!({"type":format!("response.{kind}"), "response":{ + "id":"resp_backpressure", "status":kind, "output":[] + }}) + .to_string() + .into(), + ), + } +} + +async fn send_backpressured_response( + websocket: &mut tokio_tungstenite::WebSocketStream, + tail: &str, +) { + // 先填满交付队列,后续帧由诊断计数确认已到达 forwarder,不靠 sleep 猜测调度 + for _ in 0..16 { + websocket.send(response_tail("delta")).await.unwrap(); + } + websocket.send(response_tail(tail)).await.unwrap(); +} + +async fn expect_upstream_socket_closed( + websocket: &mut tokio_tungstenite::WebSocketStream, +) { + let close = timeout(Duration::from_secs(2), websocket.next()) + .await + .expect("cancellation must release the socket while the receiver remains alive"); + assert_matches!( + close, + None | Some(Ok(Message::Close(_))) + | Some(Err(tungstenite::Error::Protocol( + tungstenite::error::ProtocolError::ResetWithoutClosingHandshake + ))) + ); +} + +#[tokio::test] +async fn backpressured_delivery_should_stop_on_downstream_cancel_or_pool_shutdown() { + for cancel_downstream in [true, false] { + for tail in ["delta", "binary", "completed", "failed", "incomplete"] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut websocket = accept_codex_test_websocket(socket).await; + websocket.next().await.unwrap().unwrap(); + send_backpressured_response(&mut websocket, tail).await; + expect_upstream_socket_closed(&mut websocket).await; + }); + let pool = Arc::new(CodexWebSocketPool::default()); + let backend = CodexBackendClient::new( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{addr}"), + test_wire_profile(), + ) + .with_websocket_pool(Arc::clone(&pool)); + let cancellation = CancellationToken::new(); + let control = + ResponseControl::default().with_connection_cancellation(cancellation.clone()); + let trace = TraceContext::new("backpressured_delivery"); + let mut context = request_context("backpressured_delivery", Some("chatgpt-account")) + .with_response_control(Some(&control)); + context.trace = Some(&trace); + let mut request = pooled_websocket_request("backpressured_delivery"); + request.downstream_websocket_connection_id = Some("downstream_backpressure".into()); + let mut response = backend + .create_response_stream(&request, context) + .await + .unwrap(); + let (stage, count) = if tail == "binary" { + ("upstream.binary", 1) + } else { + ("upstream.event", 17) + }; + wait_for_forwarded_frames(&trace, stage, count).await; + + if cancel_downstream { + cancellation.cancel(); + // 必须在 shutdown 前看到物理连接关闭,避免停机掩盖下游取消失效 + (&mut server).await.unwrap(); + } + timeout(Duration::from_secs(2), pool.shutdown()) + .await + .expect("shutdown must join delivery even with a full receiver"); + if !cancel_downstream { + (&mut server).await.unwrap(); + } + // 收尾前不消费也不丢弃 body,已交付的事件仍可正常读完 + let mut buffered = 0; + while response.body.next().await.transpose().unwrap().is_some() { + buffered += 1; + } + assert_eq!( + buffered, 16, + "tail={tail}, cancel_downstream={cancel_downstream}" + ); + } + } +} + +#[tokio::test] +async fn terminal_should_release_or_reuse_socket_before_delivery() { + for tail in ["completed", "failed", "incomplete"] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut websocket = accept_codex_test_websocket(socket).await; + websocket.next().await.unwrap().unwrap(); + send_backpressured_response(&mut websocket, tail).await; + if tail == "completed" { + let next = timeout(Duration::from_secs(2), websocket.next()) + .await + .expect("continuation should reuse the completed socket") + .unwrap() + .unwrap(); + let Message::Text(next) = next else { + panic!("expected continuation request"); + }; + assert_eq!( + serde_json::from_str::(&next).unwrap()["previous_response_id"], + "resp_backpressure" + ); + websocket + .send(Message::Text( + completed_websocket_response("resp_next", 2, 1).into(), + )) + .await + .unwrap(); + } + expect_upstream_socket_closed(&mut websocket).await; + }); + let pool = Arc::new(CodexWebSocketPool::default()); + let backend = CodexBackendClient::new( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{addr}"), + test_wire_profile(), + ) + .with_websocket_pool(Arc::clone(&pool)); + let trace = TraceContext::new("terminal_delivery"); + let mut context = request_context("terminal_delivery", Some("chatgpt-account")); + context.trace = Some(&trace); + let mut request = pooled_websocket_request("terminal_delivery"); + let mut response = backend + .create_response_stream(&request, context) + .await + .unwrap(); + wait_for_forwarded_frames(&trace, "upstream.event", 17).await; + if tail == "completed" { + request.set_previous_response_id(Some("resp_backpressure".into())); + request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal); + let next = backend + .create_response( + &request, + request_context("terminal_next", Some("chatgpt-account")), + ) + .await + .expect("completed continuation should be reusable before terminal consumption"); + assert!(next.websocket_pool_decision.unwrap().is_reuse()); + } else { + // 失败终态必须释放 socket,不能依赖客户端读走终态 + (&mut server).await.unwrap(); + } + let mut events = Vec::new(); + while let Some(chunk) = response.body.next().await { + events.push(serde_json::from_slice::(&chunk.unwrap()).unwrap()); + } + assert_eq!(events.len(), 17); + assert_eq!(events.last().unwrap()["type"], format!("response.{tail}")); + pool.shutdown().await; + if tail == "completed" { + (&mut server).await.unwrap(); + } + } +} + #[tokio::test] async fn send_timeout_should_abort_stalled_pump_and_allow_next_request() { let socket = tokio::net::TcpSocket::new_v4().unwrap(); diff --git a/docs/architecture.md b/docs/architecture.md index ba20675f8..066df8375 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -531,6 +531,7 @@ OpenAI 的连接池监督该取消信号,按当前归属保护绑定连接; 重新绑定或切到 HTTP 时移除失去用途的旧连接,失效 reservation 不能重新归池。 普通空闲期限复用池槽位的 `idle_since`,只在归池时重置;心跳不刷新业务空闲,建连与在途不参与空闲回收。 回收先移出可复用集合,再在锁外有界关闭并等待 pump/socket 释放,物理容量随 socket 归还;批量关闭并行执行。 +上游转发等待下游消费时仍响应连接取消和停机;终态先完成连接归还或销毁,再交付下游。 Codex OAuth 上游 WS 在首个请求写入后记录搜索声明,握手完成但尚未发送请求的连接仍未初始化。 后续首次声明搜索时,连接池回收不兼容的原连接;普通新链重新建连,连接内续接交由客户端完整重放。 该判断共用于 HTTP 与 WS 入站,不修改客户端工具声明,也不保留对话正文。