diff --git a/src/proxy/dispatch/chat.rs b/src/proxy/dispatch/chat.rs index 796a8addb..3cedf81e9 100644 --- a/src/proxy/dispatch/chat.rs +++ b/src/proxy/dispatch/chat.rs @@ -240,6 +240,7 @@ impl ChatDispatchService { &request, request_id, &account, + started_at, ) .await; self.account_pool.release(&release_account_id).await; @@ -397,6 +398,7 @@ impl ChatDispatchService { "responseId": response_id, "stream": false, "transport": backend_transport_name(response.transport), + "firstTokenMs": response.first_token_ms, "usage": response.usage, }), rate_limit_headers: &response.rate_limit_headers, diff --git a/src/proxy/dispatch/responses.rs b/src/proxy/dispatch/responses.rs index 9388bb591..7768e8fc4 100644 --- a/src/proxy/dispatch/responses.rs +++ b/src/proxy/dispatch/responses.rs @@ -63,8 +63,9 @@ use crate::{ events::{extract_sse_usage, extract_usage, TokenUsage}, responses::{ apply_response_model_options, completed_response_metadata, - reconvert_responses_sse_event_tuple_values, response_from_codex_sse, - CodexCompactRequest, CodexResponsesRequest, CollectedResponse, ResponsesSseFailure, + reconvert_responses_sse_event_tuple_values, response_body_has_first_event, + response_from_codex_sse, CodexCompactRequest, CodexResponsesRequest, + CollectedResponse, ResponsesSseFailure, }, sse::{ encode_sse_event, parse_sse_events, sse_body_has_done, SseError, DONE_SSE_FRAME, @@ -570,6 +571,7 @@ impl ResponseDispatchService { &request, request_id, &account, + started_at, ) .await; self.account_pool.release(&release_account_id).await; @@ -892,6 +894,7 @@ impl ResponseDispatchService { "responseId": response_id, "stream": false, "transport": backend_transport_name(response.transport), + "firstTokenMs": response.first_token_ms, "usage": response.usage, }), rate_limit_headers: &response.rate_limit_headers, @@ -2177,90 +2180,11 @@ fn update_first_token_ms( body_bytes: &[u8], first_token_ms: &mut Option, ) { - if first_token_ms.is_none() && response_body_has_first_token(body_bytes) { + if first_token_ms.is_none() && response_body_has_first_event(body_bytes) { *first_token_ms = Some(elapsed_millis_i64(context.started_at).max(1)); } } -fn response_body_has_first_token(body_bytes: &[u8]) -> bool { - let body = String::from_utf8_lossy(body_bytes); - let Some(event_end) = body.rfind("\n\n") else { - return false; - }; - parse_sse_events(&body[..event_end + 2]).is_ok_and(|events| { - events.iter().any(|event| { - serde_json::from_str::(&event.data) - .is_ok_and(|value| response_event_has_first_token(event.event.as_deref(), &value)) - }) - }) -} - -fn response_event_has_first_token(event_name: Option<&str>, value: &Value) -> bool { - match response_stream_event_type(event_name, value) { - Some( - "response.output_text.delta" - | "response.reasoning_summary_text.delta" - | "response.reasoning_text.delta" - | "response.function_call_arguments.delta" - | "response.custom_tool_call_input.delta", - ) => non_empty_string_field(value, &["delta"]), - Some("response.output_text.done" | "response.reasoning_text.done") => { - non_empty_string_field(value, &["text"]) - } - Some("response.output_item.added" | "response.output_item.done") => value - .get("item") - .is_some_and(output_item_has_generated_content), - Some("response.completed") => value - .get("response") - .is_some_and(completed_response_has_generated_content), - _ => false, - } -} - -fn response_stream_event_type<'a>( - event_name: Option<&'a str>, - value: &'a Value, -) -> Option<&'a str> { - event_name.or_else(|| value.get("type").and_then(Value::as_str)) -} - -fn completed_response_has_generated_content(response: &Value) -> bool { - non_empty_string_field(response, &["output_text"]) - || response - .get("output") - .and_then(Value::as_array) - .is_some_and(|items| items.iter().any(output_item_has_generated_content)) - || response - .pointer("/usage/output_tokens") - .and_then(Value::as_i64) - .is_some_and(|tokens| tokens > 0) -} - -fn output_item_has_generated_content(item: &Value) -> bool { - non_empty_string_field(item, &["content", "text", "arguments", "input"]) - || item - .get("content") - .and_then(Value::as_array) - .is_some_and(|parts| parts.iter().any(output_content_part_has_text)) - || item - .get("summary") - .and_then(Value::as_array) - .is_some_and(|parts| parts.iter().any(output_content_part_has_text)) -} - -fn output_content_part_has_text(part: &Value) -> bool { - non_empty_string_field(part, &["text", "content"]) -} - -fn non_empty_string_field(value: &Value, fields: &[&str]) -> bool { - fields.iter().any(|field| { - value - .get(field) - .and_then(Value::as_str) - .is_some_and(|text| !text.trim().is_empty()) - }) -} - async fn send_live_response_stream_chunk( sender: &mpsc::Sender>, body_bytes: &mut Vec, @@ -3349,43 +3273,49 @@ fn elapsed_millis_i64(started_at: Instant) -> i64 { #[cfg(test)] mod tests { - use super::response_body_has_first_token; + use crate::upstream::protocol::responses::response_body_has_first_event; #[test] - fn first_token_detection_should_match_text_delta() { + fn first_event_detection_should_match_text_delta() { let body = br#"event: response.output_text.delta data: {"delta":"hello"} "#; - assert!(response_body_has_first_token(body)); + assert!(response_body_has_first_event(body)); } #[test] - fn first_token_detection_should_match_output_item_done_text() { - let body = br#"event: response.output_item.done -data: {"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hello"}]}} + fn first_event_detection_should_match_response_created() { + let body = br#"event: response.created +data: {"type":"response.created","response":{"id":"resp_1","status":"in_progress"}} "#; - assert!(response_body_has_first_token(body)); + assert!(response_body_has_first_event(body)); } #[test] - fn first_token_detection_should_fallback_to_completed_usage_output_tokens() { - let body = br#"event: response.completed -data: {"response":{"id":"resp_1","status":"completed","usage":{"input_tokens":3,"output_tokens":4}}} + fn first_event_detection_should_ignore_done_frame() { + let body = br#"data: [DONE] "#; - assert!(response_body_has_first_token(body)); + assert!(!response_body_has_first_event(body)); } #[test] - fn first_token_detection_should_ignore_incomplete_frames() { + fn first_event_detection_should_ignore_incomplete_frames() { let body = br#"event: response.output_text.delta data: {"delta":"hello"}"#; - assert!(!response_body_has_first_token(body)); + assert!(!response_body_has_first_event(body)); + } + + #[test] + fn first_event_detection_should_match_crlf_frames() { + let body = b"event: response.created\r\ndata: {\"type\":\"response.created\"}\r\n\r\n"; + + assert!(response_body_has_first_event(body)); } } diff --git a/src/proxy/dispatch/upstream.rs b/src/proxy/dispatch/upstream.rs index 21fd503b9..112dcc354 100644 --- a/src/proxy/dispatch/upstream.rs +++ b/src/proxy/dispatch/upstream.rs @@ -1,5 +1,7 @@ //! 调度层共享的账号级 Codex 上游调用。 +use std::time::Instant; + use serde_json::Value; use crate::{ @@ -130,6 +132,7 @@ pub(crate) async fn create_response_with_account( request: &CodexResponsesRequest, request_id: &str, account: &Account, + started_at: Instant, ) -> Result { let cookie_header = cloudflare .cookie_header_for_request(&account.id, "/codex/responses") @@ -140,7 +143,7 @@ pub(crate) async fn create_response_with_account( &account.id, ); codex - .create_response( + .create_response_with_pool_account_started_at( request, CodexRequestContext { access_token: &account.access_token, @@ -157,6 +160,8 @@ pub(crate) async fn create_response_with_account( installation_id, session_id: identity.conversation_id.as_deref(), }, + Some(&account.id), + started_at, ) .await } @@ -178,7 +183,7 @@ async fn create_response_stream_with_account( &account.id, ); codex - .create_response_stream( + .create_response_stream_with_pool_account( request, CodexRequestContext { access_token: &account.access_token, @@ -195,6 +200,7 @@ async fn create_response_stream_with_account( installation_id, session_id: identity.conversation_id.as_deref(), }, + Some(&account.id), ) .await } @@ -241,6 +247,7 @@ pub(crate) async fn create_response_with_account_retrying_5xx( request: &CodexResponsesRequest, request_id: &str, account: &Account, + started_at: Instant, ) -> Result { let mut retries = 0; loop { @@ -251,6 +258,7 @@ pub(crate) async fn create_response_with_account_retrying_5xx( request, request_id, account, + started_at, ) .await; match result { diff --git a/src/proxy/openai/chat.rs b/src/proxy/openai/chat.rs index 8fff4331e..38eebb849 100644 --- a/src/proxy/openai/chat.rs +++ b/src/proxy/openai/chat.rs @@ -484,6 +484,7 @@ pub fn translate_chat_to_codex( .as_deref() .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()); + let use_websocket = user.is_some(); let codex = CodexResponsesRequest::new_http_sse( request.model, String::new(), @@ -506,7 +507,7 @@ pub fn translate_chat_to_codex( service_tier: request.service_tier, prompt_cache_key: user.clone(), client_conversation_id: user, - force_http_sse: true, + use_websocket, ..codex }) } diff --git a/src/upstream/protocol/responses.rs b/src/upstream/protocol/responses.rs index 12c65f52b..d11c8ba96 100644 --- a/src/upstream/protocol/responses.rs +++ b/src/upstream/protocol/responses.rs @@ -140,6 +140,30 @@ pub fn http_sse_fallback_allowed(request: &CodexResponsesRequest) -> bool { ) } +/// 判断已收到的 Responses SSE 内容是否包含首个完整的有效数据事件。 +pub fn response_body_has_first_event(body_bytes: &[u8]) -> bool { + let body = String::from_utf8_lossy(body_bytes); + let Some(complete_body) = complete_sse_body_prefix(&body) else { + return false; + }; + parse_sse_events(complete_body).is_ok_and(|events| { + events.iter().any(|event| { + let data = event.data.trim(); + !data.is_empty() && data != "[DONE]" + }) + }) +} + +fn complete_sse_body_prefix(body: &str) -> Option<&str> { + let lf_end = body.rfind("\n\n").map(|index| index + 2); + let crlf_end = body.rfind("\r\n\r\n").map(|index| index + 4); + lf_end + .into_iter() + .chain(crlf_end) + .max() + .map(|end| &body[..end]) +} + /// 从 Codex SSE 收集出的非流式 Responses 结果。 #[derive(Debug, Clone, PartialEq)] pub enum CollectedResponse { diff --git a/src/upstream/transport/client.rs b/src/upstream/transport/client.rs index dcd319cc1..04229a407 100644 --- a/src/upstream/transport/client.rs +++ b/src/upstream/transport/client.rs @@ -4,7 +4,7 @@ use std::{ collections::HashMap, pin::Pin, sync::{Arc, Mutex, OnceLock}, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use async_trait::async_trait; @@ -26,8 +26,8 @@ use crate::upstream::fingerprint::Fingerprint; use crate::upstream::models::BackendModelEntry; use crate::upstream::protocol::events::extract_sse_usage; use crate::upstream::protocol::responses::{ - http_sse_fallback_allowed, transport_for_request, CodexCompactRequest, CodexResponsesRequest, - CodexTransport, + http_sse_fallback_allowed, response_body_has_first_event, transport_for_request, + CodexCompactRequest, CodexResponsesRequest, CodexTransport, }; use crate::upstream::protocol::sse::SseError; use crate::upstream::protocol::websocket::{ @@ -41,10 +41,9 @@ use super::headers::{ }; use super::tls::{build_reqwest_client_with_custom_ca, custom_ca_env_cache_key, CustomCaError}; use super::websocket::{ - execute_response_create_request, execute_response_create_request_stream_with_pool, - execute_response_create_request_with_pool, write_websocket_audit_artifact_from_env, - CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketRateLimitHeaderUpdates, - CodexWebSocketTurnStateUpdate, + execute_response_create_request_stream_with_pool, execute_response_create_request_with_pool, + write_websocket_audit_artifact_from_env, CodexWebSocketConnection, CodexWebSocketExchangeError, + CodexWebSocketRateLimitHeaderUpdates, CodexWebSocketTurnStateUpdate, }; use super::websocket_pool::{CodexWebSocketPool, CodexWebSocketPoolKey}; @@ -244,6 +243,8 @@ pub struct CodexBackendResponse { pub set_cookie_headers: Vec, /// 上游透传的限流头。 pub rate_limit_headers: Vec<(String, String)>, + /// 首个有效上游 SSE/WebSocket 事件到达代理的耗时。 + pub first_token_ms: Option, } /// Codex Responses 实际使用的上游传输。 @@ -345,6 +346,7 @@ impl CodexBackendClient { &self, upstream_request: &CodexResponsesRequest, context: CodexRequestContext<'_>, + started_at: Instant, ) -> CodexClientResult { let headers = self.request_headers_for_http_response(upstream_request, context)?; let response = self @@ -371,7 +373,14 @@ impl CodexBackendClient { }); } - let body = response.text().await?; + let mut body_bytes = Vec::new(); + let mut first_token_ms = None; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.try_next().await? { + body_bytes.extend_from_slice(&chunk); + update_first_token_ms(started_at, &body_bytes, &mut first_token_ms); + } + let body = String::from_utf8_lossy(&body_bytes).into_owned(); let usage = extract_sse_usage(&body).map_err(CodexClientError::InvalidSse)?; Ok(CodexBackendResponse { body, @@ -380,6 +389,7 @@ impl CodexBackendClient { turn_state, set_cookie_headers, rate_limit_headers, + first_token_ms, }) } @@ -429,16 +439,42 @@ impl CodexBackendClient { &self, request: &CodexResponsesRequest, context: CodexRequestContext<'_>, + ) -> CodexClientResult { + self.create_response_started_at(request, context, Instant::now()) + .await + } + + pub async fn create_response_started_at( + &self, + request: &CodexResponsesRequest, + context: CodexRequestContext<'_>, + started_at: Instant, + ) -> CodexClientResult { + self.create_response_with_pool_account_started_at(request, context, None, started_at) + .await + } + + pub async fn create_response_with_pool_account_started_at( + &self, + request: &CodexResponsesRequest, + context: CodexRequestContext<'_>, + pool_account_id: Option<&str>, + started_at: Instant, ) -> CodexClientResult { let upstream_request = response_upstream_request(request, context); match transport_for_request(&upstream_request) { CodexTransport::HttpSse => { - self.create_response_http_sse(&upstream_request, context) + self.create_response_http_sse(&upstream_request, context, started_at) .await } CodexTransport::WebSocketPreferred | CodexTransport::WebSocketRequired => { match self - .create_response_websocket(&upstream_request, context) + .create_response_websocket( + &upstream_request, + context, + pool_account_id, + started_at, + ) .await { Ok(response) => Ok(response), @@ -447,7 +483,7 @@ impl CodexBackendClient { && websocket_error_allows_http_fallback(&error) => { tracing::warn!(error = %error, "websocket response failed; falling back to HTTP SSE"); - self.create_response_http_sse(&upstream_request, context) + self.create_response_http_sse(&upstream_request, context, started_at) .await } Err(error) => Err(error), @@ -461,6 +497,16 @@ impl CodexBackendClient { &self, request: &CodexResponsesRequest, context: CodexRequestContext<'_>, + ) -> CodexClientResult { + self.create_response_stream_with_pool_account(request, context, None) + .await + } + + pub async fn create_response_stream_with_pool_account( + &self, + request: &CodexResponsesRequest, + context: CodexRequestContext<'_>, + pool_account_id: Option<&str>, ) -> CodexClientResult { let upstream_request = response_upstream_request(request, context); match transport_for_request(&upstream_request) { @@ -470,7 +516,7 @@ impl CodexBackendClient { } CodexTransport::WebSocketPreferred | CodexTransport::WebSocketRequired => { match self - .create_response_websocket_stream(&upstream_request, context) + .create_response_websocket_stream(&upstream_request, context, pool_account_id) .await { Ok(response) => Ok(response), @@ -492,6 +538,8 @@ impl CodexBackendClient { &self, upstream_request: &CodexResponsesRequest, context: CodexRequestContext<'_>, + pool_account_id: Option<&str>, + started_at: Instant, ) -> CodexClientResult { let websocket_request = websocket_upstream_request(upstream_request); let headers = self.request_headers_for_http_response(&websocket_request, context)?; @@ -510,12 +558,13 @@ impl CodexBackendClient { if let Err(error) = write_websocket_audit_artifact_from_env(&artifact).await { tracing::warn!(error = %error, "failed to write Codex WebSocket audit artifact"); } - let pool_key = self.websocket_pool_key(upstream_request, context); + let pool_key = self.websocket_pool_key(upstream_request, context, pool_account_id); let exchange = match (self.websocket_pool.as_deref(), pool_key) { (Some(pool), Some(key)) => { - execute_response_create_request_with_pool(&prepared, Some((pool, key))).await + execute_response_create_request_with_pool(&prepared, Some((pool, key)), started_at) + .await } - _ => execute_response_create_request(&prepared).await, + _ => execute_response_create_request_with_pool(&prepared, None, started_at).await, } .map_err(websocket_exchange_error_to_client_error)?; @@ -526,6 +575,7 @@ impl CodexBackendClient { turn_state: exchange.turn_state, set_cookie_headers: exchange.set_cookie_headers, rate_limit_headers: exchange.rate_limit_headers, + first_token_ms: exchange.first_token_ms, }) } @@ -533,6 +583,7 @@ impl CodexBackendClient { &self, upstream_request: &CodexResponsesRequest, context: CodexRequestContext<'_>, + pool_account_id: Option<&str>, ) -> CodexClientResult { let websocket_request = websocket_upstream_request(upstream_request); let headers = self.request_headers_for_http_response(&websocket_request, context)?; @@ -551,7 +602,7 @@ impl CodexBackendClient { if let Err(error) = write_websocket_audit_artifact_from_env(&artifact).await { tracing::warn!(error = %error, "failed to write Codex WebSocket audit artifact"); } - let pool_key = self.websocket_pool_key(upstream_request, context); + let pool_key = self.websocket_pool_key(upstream_request, context, pool_account_id); let exchange = match (self.websocket_pool.as_deref(), pool_key) { (Some(pool), Some(key)) => { execute_response_create_request_stream_with_pool(&prepared, Some((pool, key))).await @@ -579,8 +630,9 @@ impl CodexBackendClient { &self, request: &CodexResponsesRequest, context: CodexRequestContext<'_>, + pool_account_id: Option<&str>, ) -> Option { - let account_id = context.account_id?; + let account_id = pool_account_id.or(context.account_id)?; let conversation_id = request .prompt_cache_key .as_deref() @@ -866,6 +918,16 @@ fn turn_state(response: &ReqwestResponse) -> Option { .map(ToString::to_string) } +fn update_first_token_ms(started_at: Instant, body_bytes: &[u8], first_token_ms: &mut Option) { + if first_token_ms.is_none() && response_body_has_first_event(body_bytes) { + *first_token_ms = Some(elapsed_millis_i64(started_at).max(1)); + } +} + +fn elapsed_millis_i64(started_at: Instant) -> i64 { + started_at.elapsed().as_millis().min(i64::MAX as u128) as i64 +} + fn set_cookie_headers(response: &ReqwestResponse) -> Vec { response .headers() diff --git a/src/upstream/transport/websocket.rs b/src/upstream/transport/websocket.rs index 971d4cc20..f57bf5554 100644 --- a/src/upstream/transport/websocket.rs +++ b/src/upstream/transport/websocket.rs @@ -26,7 +26,7 @@ use tungstenite::{ use crate::infra::time::china_filename_timestamp_millis; use crate::upstream::protocol::events::{self, TokenUsage}; -use crate::upstream::protocol::responses::CodexResponsesRequest; +use crate::upstream::protocol::responses::{response_body_has_first_event, CodexResponsesRequest}; use crate::upstream::protocol::sse::SseError; use crate::upstream::protocol::websocket::{ classify_websocket_error_frame, is_terminal_websocket_event, @@ -113,6 +113,8 @@ pub struct CodexWebSocketExchange { pub rate_limit_headers: Vec<(String, String)>, /// 打开握手响应状态码。 pub handshake_status: u16, + /// 首个有效上游 WebSocket 事件到达代理的耗时。 + pub first_token_ms: Option, } /// Responses WebSocket live SSE exchange result. @@ -394,6 +396,7 @@ fn request_line_for_endpoint(endpoint: &str) -> String { async fn execute_fresh_response_create_request( request: &CodexWebSocketRequest, + started_at: Instant, ) -> Result { let (mut websocket, response) = connect_websocket(request.connection()).await?; websocket @@ -402,7 +405,7 @@ async fn execute_fresh_response_create_request( let metadata = websocket_connection_metadata(&response); let (exchange, _websocket, _metadata) = - collect_websocket_response(websocket, metadata, false).await?; + collect_websocket_response(websocket, metadata, false, started_at).await?; Ok(exchange) } @@ -425,31 +428,40 @@ async fn execute_fresh_response_create_request_stream( pub(crate) async fn execute_response_create_request_with_pool( request: &CodexWebSocketRequest, pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>, + started_at: Instant, ) -> Result { let Some((pool, key)) = pool else { - return execute_fresh_response_create_request(request).await; + return execute_fresh_response_create_request(request, started_at).await; }; match pool.acquire(&key).await { WebSocketPoolAcquire::Reused(connection) => { let result = - execute_pooled_response_create_request(request, pool, key, *connection).await; + execute_pooled_response_create_request(request, pool, key, *connection, started_at) + .await; match result { Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstFrame { .. - }) => execute_fresh_response_create_request(request).await, + }) => execute_fresh_response_create_request(request, started_at).await, other => other, } } WebSocketPoolAcquire::FreshReserved => { - let result = - execute_fresh_pooled_response_create_request(request, pool, key.clone()).await; + let result = execute_fresh_pooled_response_create_request( + request, + pool, + key.clone(), + started_at, + ) + .await; if result.is_err() { pool.discard(&key).await; } result } - WebSocketPoolAcquire::Bypass => execute_fresh_response_create_request(request).await, + WebSocketPoolAcquire::Bypass => { + execute_fresh_response_create_request(request, started_at).await + } } } @@ -497,6 +509,7 @@ async fn execute_fresh_pooled_response_create_request( request: &CodexWebSocketRequest, pool: &CodexWebSocketPool, key: CodexWebSocketPoolKey, + started_at: Instant, ) -> Result { let (mut websocket, response) = connect_websocket(request.connection()).await?; websocket @@ -506,7 +519,7 @@ async fn execute_fresh_pooled_response_create_request( let now = Instant::now(); let metadata = websocket_connection_metadata(&response); let (exchange, websocket, metadata) = - collect_websocket_response(websocket, metadata, false).await?; + collect_websocket_response(websocket, metadata, false, started_at).await?; let last_activity_at = Instant::now(); pool.put( key, @@ -552,6 +565,7 @@ async fn execute_pooled_response_create_request( pool: &CodexWebSocketPool, key: CodexWebSocketPoolKey, mut connection: PooledWebSocketConnection, + started_at: Instant, ) -> Result { if let Err(error) = connection .websocket @@ -568,7 +582,9 @@ async fn execute_pooled_response_create_request( let created_at = connection.created_at; let last_ping_at = connection.last_ping_at; - match collect_websocket_response(connection.websocket, connection.metadata, true).await { + match collect_websocket_response(connection.websocket, connection.metadata, true, started_at) + .await + { Ok((exchange, websocket, metadata)) => { let last_activity_at = Instant::now(); pool.put( @@ -663,7 +679,7 @@ async fn first_reused_stream_text( pub async fn execute_response_create_request( request: &CodexWebSocketRequest, ) -> Result { - execute_response_create_request_with_pool(request, None).await + execute_response_create_request_with_pool(request, None, Instant::now()).await } // --------------------------------------------------------------------------- @@ -674,6 +690,7 @@ async fn collect_websocket_response( mut websocket: CodexWsStream, mut metadata: CodexWebSocketConnectionMetadata, reused_connection: bool, + started_at: Instant, ) -> Result< ( CodexWebSocketExchange, @@ -684,6 +701,7 @@ async fn collect_websocket_response( > { let mut body = String::new(); let mut saw_response_frame = false; + let mut first_token_ms = None; loop { let message = match next_websocket_message(&mut websocket).await { @@ -743,6 +761,7 @@ async fn collect_websocket_response( let forwarded = if let Some(frame) = websocket_event_to_sse_frame(&raw) { body.push_str(&frame); saw_response_frame = true; + update_first_token_ms(started_at, body.as_bytes(), &mut first_token_ms); true } else { false @@ -756,6 +775,7 @@ async fn collect_websocket_response( set_cookie_headers: metadata.set_cookie_headers.clone(), rate_limit_headers: metadata.rate_limit_headers.clone(), handshake_status: metadata.handshake_status, + first_token_ms, }; return Ok((exchange, websocket, metadata)); } @@ -780,6 +800,16 @@ fn reused_connection_died_before_first_frame( } } +fn update_first_token_ms(started_at: Instant, body_bytes: &[u8], first_token_ms: &mut Option) { + if first_token_ms.is_none() && response_body_has_first_event(body_bytes) { + *first_token_ms = Some(elapsed_millis_i64(started_at).max(1)); + } +} + +fn elapsed_millis_i64(started_at: Instant) -> i64 { + started_at.elapsed().as_millis().min(i64::MAX as u128) as i64 +} + fn websocket_connection_metadata( response: &WsResponse>>, ) -> CodexWebSocketConnectionMetadata { diff --git a/tests/proxy/dispatch/chat_upstream/chat_routes.rs b/tests/proxy/dispatch/chat_upstream/chat_routes.rs index da0c2b314..e04027b8c 100644 --- a/tests/proxy/dispatch/chat_upstream/chat_routes.rs +++ b/tests/proxy/dispatch/chat_upstream/chat_routes.rs @@ -94,6 +94,122 @@ async fn chat_completions_should_dispatch_to_codex_and_return_openai_response() assert_eq!(metadata["usage"]["outputTokens"], 3); } +#[tokio::test] +async fn chat_completions_with_user_should_use_and_reuse_websocket() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let upstream = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut websocket = accept_async(stream).await.unwrap(); + let first_payload = send_chat_websocket_response( + &mut websocket, + "websocket chat first", + "resp_chat_ws_first", + ) + .await; + + loop { + tokio::select! { + message = websocket.next() => { + match message { + Some(Ok(message)) if message.is_text() => { + let second_payload = send_chat_websocket_response_after_message( + &mut websocket, + message, + "websocket chat second", + "resp_chat_ws_second", + ) + .await; + websocket.close(None).await.unwrap(); + break (true, first_payload, second_payload); + } + Some(_) => {} + None => { + let second_payload = accept_chat_websocket_response( + &listener, + "websocket chat second", + "resp_chat_ws_second", + ) + .await; + break (false, first_payload, second_payload); + } + } + } + accepted = listener.accept() => { + let (stream, _) = accepted.unwrap(); + let mut second_websocket = accept_async(stream).await.unwrap(); + let second_payload = send_chat_websocket_response( + &mut second_websocket, + "websocket chat second", + "resp_chat_ws_second", + ) + .await; + second_websocket.close(None).await.unwrap(); + break (false, first_payload, second_payload); + } + } + } + }); + let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(base_url).await; + + let first_response = app + .clone() + .oneshot(chat_json_request( + &api_key, + "req_chat_ws_first", + "chat-ws-user", + "Say hello over websocket", + )) + .await + .unwrap(); + assert_eq!(first_response.status(), StatusCode::OK); + let first_body = response_json(first_response).await; + assert_eq!( + first_body["choices"][0]["message"]["content"], + "websocket chat first" + ); + + let second_response = app + .oneshot(chat_json_request( + &api_key, + "req_chat_ws_second", + "chat-ws-user", + "Continue over the same websocket", + )) + .await + .unwrap(); + assert_eq!(second_response.status(), StatusCode::OK); + let second_body = response_json(second_response).await; + assert_eq!( + second_body["choices"][0]["message"]["content"], + "websocket chat second" + ); + let (reused_connection, first_payload, second_payload) = upstream.await.unwrap(); + let event = latest_usage_record(&pool, "v1.chat").await; + let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); + + assert!( + reused_connection, + "second chat request opened a new websocket" + ); + assert_eq!(first_payload["type"], "response.create"); + assert_eq!(second_payload["type"], "response.create"); + assert_eq!( + first_payload["prompt_cache_key"], second_payload["prompt_cache_key"], + "same chat user should keep the same account-scoped conversation key" + ); + assert!(first_payload["prompt_cache_key"] + .as_str() + .is_some_and(|key| key.starts_with("cp_"))); + assert_eq!(metadata["transport"], "websocket"); + assert!( + metadata["firstTokenMs"] + .as_i64() + .is_some_and(|value| value > 0), + "chat websocket usage metadata should include first token latency: {metadata:?}", + ); +} + #[tokio::test] async fn chat_completions_stream_should_translate_codex_sse_to_openai_chunks() { let server = MockServer::start().await; @@ -912,3 +1028,70 @@ async fn chat_completions_should_return_quota_error_when_402_fallback_is_exhaust assert_eq!(body["error"]["code"], "upstream_error"); assert_eq!(account_status.0, "quota_exhausted"); } + +fn chat_json_request(api_key: &str, request_id: &str, user: &str, content: &str) -> Request { + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .header("x-request-id", request_id) + .body(Body::from( + json!({ + "model": "gpt-5.5", + "user": user, + "messages": [{"role": "user", "content": content}] + }) + .to_string(), + )) + .unwrap() +} + +async fn accept_chat_websocket_response( + listener: &TcpListener, + content: &str, + response_id: &str, +) -> Value { + let (stream, _) = listener.accept().await.unwrap(); + let mut websocket = accept_async(stream).await.unwrap(); + let payload = send_chat_websocket_response(&mut websocket, content, response_id).await; + websocket.close(None).await.unwrap(); + payload +} + +async fn send_chat_websocket_response( + websocket: &mut WebSocketStream, + content: &str, + response_id: &str, +) -> Value { + let message = websocket.next().await.unwrap().unwrap(); + send_chat_websocket_response_after_message(websocket, message, content, response_id).await +} + +async fn send_chat_websocket_response_after_message( + websocket: &mut WebSocketStream, + message: Message, + content: &str, + response_id: &str, +) -> Value { + let payload = serde_json::from_str::(&message.into_text().unwrap()) + .expect("websocket payload should be json"); + websocket + .send(Message::Text( + json!({ + "type": "response.output_text.delta", + "delta": content + }) + .to_string() + .into(), + )) + .await + .unwrap(); + websocket + .send(Message::Text( + response_completed_websocket_message(response_id).into(), + )) + .await + .unwrap(); + payload +} diff --git a/tests/proxy/dispatch/chat_upstream/usage_logging.rs b/tests/proxy/dispatch/chat_upstream/usage_logging.rs index 54bb750ff..2ec71a655 100644 --- a/tests/proxy/dispatch/chat_upstream/usage_logging.rs +++ b/tests/proxy/dispatch/chat_upstream/usage_logging.rs @@ -85,6 +85,57 @@ async fn responses_should_use_imported_account_record_usage_cookie_and_usage_rec assert_eq!(metadata["usage"]["inputTokens"], 7); } +#[tokio::test] +async fn chat_completions_non_stream_should_record_first_token_latency() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(RESPONSES_COMPLETED_USAGE_SSE), + ) + .expect(1) + .mount(&server) + .await; + + let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(server.uri()).await; + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .header("x-request-id", "req_chat_non_stream_first_token") + .body(Body::from( + json!({ + "model": "gpt-5.5", + "messages": [{"role": "user", "content": "Say hello"}] + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = response_json(response).await; + let event = latest_usage_record(&pool, "v1.chat").await; + let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); + + assert_eq!(status, StatusCode::OK); + assert_eq!(body["usage"]["prompt_tokens"], 7); + assert_eq!(metadata["stream"], false); + assert_eq!(metadata["transport"], "http_sse"); + assert!( + metadata["firstTokenMs"] + .as_i64() + .is_some_and(|value| value > 0), + "chat non-stream usage metadata should include first token latency: {metadata:?}", + ); +} + #[tokio::test] async fn responses_usage_record_should_store_resolved_upstream_model_after_alias_mapping() { let server = MockServer::start().await; diff --git a/tests/upstream/protocol/openai_chat.rs b/tests/upstream/protocol/openai_chat.rs index 3d89a7e91..ad1509b65 100644 --- a/tests/upstream/protocol/openai_chat.rs +++ b/tests/upstream/protocol/openai_chat.rs @@ -37,7 +37,8 @@ fn chat_completion_request_should_translate_to_codex_request() { .expect("chat request should translate"); assert_eq!(codex.model, "gpt-5.5"); - assert!(codex.force_http_sse); + assert!(codex.use_websocket); + assert!(!codex.force_http_sse); assert_eq!(codex.prompt_cache_key.as_deref(), Some("client-123")); assert_eq!(codex.client_conversation_id.as_deref(), Some("client-123")); } diff --git a/tests/upstream/transport/websocket_pool.rs b/tests/upstream/transport/websocket_pool.rs index ac7590e3e..62fb1e41b 100644 --- a/tests/upstream/transport/websocket_pool.rs +++ b/tests/upstream/transport/websocket_pool.rs @@ -72,6 +72,68 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and assert_eq!(accepted_connections.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn codex_backend_client_should_not_reuse_pooled_websocket_across_local_accounts() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let accepted_connections = Arc::new(AtomicUsize::new(0)); + let accepted_connections_for_server = Arc::clone(&accepted_connections); + let server = tokio::spawn(async move { + for response_id in ["resp_local_a", "resp_local_b"] { + let (stream, _) = listener.accept().await.unwrap(); + accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); + let mut websocket = accept_codex_test_websocket(stream).await; + let _message = websocket.next().await.unwrap().unwrap(); + websocket + .send(Message::Text( + completed_websocket_response(response_id, 3, 1).into(), + )) + .await + .unwrap(); + websocket.close(None).await.unwrap(); + } + }); + let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1))); + let backend = CodexBackendClient::new( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{addr}"), + crate::support::fingerprint::test_fingerprint(), + ) + .with_websocket_pool(Arc::clone(&pool)); + let mut request = + codex_proxy_rs::upstream::protocol::responses::CodexResponsesRequest::new_http_sse( + "gpt-5.5", + "be brief", + Vec::new(), + ); + request.previous_response_id = Some("resp_previous".to_string()); + request.prompt_cache_key = Some("conversation-pool".to_string()); + + let first = backend + .create_response_with_pool_account_started_at( + &request, + request_context("req_pool_local_a", Some("same-chatgpt-account")), + Some("acct_local_a"), + std::time::Instant::now(), + ) + .await + .expect("first local account websocket response should succeed"); + let second = backend + .create_response_with_pool_account_started_at( + &request, + request_context("req_pool_local_b", Some("same-chatgpt-account")), + Some("acct_local_b"), + std::time::Instant::now(), + ) + .await + .expect("second local account websocket response should succeed"); + server.await.unwrap(); + + assert!(first.body.contains("resp_local_a")); + assert!(second.body.contains("resp_local_b")); + assert_eq!(accepted_connections.load(Ordering::SeqCst), 2); +} + #[tokio::test] async fn websocket_pool_should_bypass_busy_key_with_one_shot_connections() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();