feat: align codex websocket transport

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-06-29 14:44:01 +08:00
co-authored by Codex
parent f3dc8297a7
commit 982bc47dc7
11 changed files with 481 additions and 127 deletions
+2
View File
@@ -240,6 +240,7 @@ impl ChatDispatchService {
&request,
request_id,
&account,
started_at,
)
.await;
self.account_pool.release(&release_account_id).await;
@@ -397,6 +398,7 @@ impl ChatDispatchService {
"responseId": response_id,
"stream": false,
"transport": backend_transport_name(response.transport),
"firstTokenMs": response.first_token_ms,
"usage": response.usage,
}),
rate_limit_headers: &response.rate_limit_headers,
+25 -95
View File
@@ -63,8 +63,9 @@ use crate::{
events::{extract_sse_usage, extract_usage, TokenUsage},
responses::{
apply_response_model_options, completed_response_metadata,
reconvert_responses_sse_event_tuple_values, response_from_codex_sse,
CodexCompactRequest, CodexResponsesRequest, CollectedResponse, ResponsesSseFailure,
reconvert_responses_sse_event_tuple_values, response_body_has_first_event,
response_from_codex_sse, CodexCompactRequest, CodexResponsesRequest,
CollectedResponse, ResponsesSseFailure,
},
sse::{
encode_sse_event, parse_sse_events, sse_body_has_done, SseError, DONE_SSE_FRAME,
@@ -570,6 +571,7 @@ impl ResponseDispatchService {
&request,
request_id,
&account,
started_at,
)
.await;
self.account_pool.release(&release_account_id).await;
@@ -892,6 +894,7 @@ impl ResponseDispatchService {
"responseId": response_id,
"stream": false,
"transport": backend_transport_name(response.transport),
"firstTokenMs": response.first_token_ms,
"usage": response.usage,
}),
rate_limit_headers: &response.rate_limit_headers,
@@ -2177,90 +2180,11 @@ fn update_first_token_ms(
body_bytes: &[u8],
first_token_ms: &mut Option<i64>,
) {
if first_token_ms.is_none() && response_body_has_first_token(body_bytes) {
if first_token_ms.is_none() && response_body_has_first_event(body_bytes) {
*first_token_ms = Some(elapsed_millis_i64(context.started_at).max(1));
}
}
fn response_body_has_first_token(body_bytes: &[u8]) -> bool {
let body = String::from_utf8_lossy(body_bytes);
let Some(event_end) = body.rfind("\n\n") else {
return false;
};
parse_sse_events(&body[..event_end + 2]).is_ok_and(|events| {
events.iter().any(|event| {
serde_json::from_str::<Value>(&event.data)
.is_ok_and(|value| response_event_has_first_token(event.event.as_deref(), &value))
})
})
}
fn response_event_has_first_token(event_name: Option<&str>, value: &Value) -> bool {
match response_stream_event_type(event_name, value) {
Some(
"response.output_text.delta"
| "response.reasoning_summary_text.delta"
| "response.reasoning_text.delta"
| "response.function_call_arguments.delta"
| "response.custom_tool_call_input.delta",
) => non_empty_string_field(value, &["delta"]),
Some("response.output_text.done" | "response.reasoning_text.done") => {
non_empty_string_field(value, &["text"])
}
Some("response.output_item.added" | "response.output_item.done") => value
.get("item")
.is_some_and(output_item_has_generated_content),
Some("response.completed") => value
.get("response")
.is_some_and(completed_response_has_generated_content),
_ => false,
}
}
fn response_stream_event_type<'a>(
event_name: Option<&'a str>,
value: &'a Value,
) -> Option<&'a str> {
event_name.or_else(|| value.get("type").and_then(Value::as_str))
}
fn completed_response_has_generated_content(response: &Value) -> bool {
non_empty_string_field(response, &["output_text"])
|| response
.get("output")
.and_then(Value::as_array)
.is_some_and(|items| items.iter().any(output_item_has_generated_content))
|| response
.pointer("/usage/output_tokens")
.and_then(Value::as_i64)
.is_some_and(|tokens| tokens > 0)
}
fn output_item_has_generated_content(item: &Value) -> bool {
non_empty_string_field(item, &["content", "text", "arguments", "input"])
|| item
.get("content")
.and_then(Value::as_array)
.is_some_and(|parts| parts.iter().any(output_content_part_has_text))
|| item
.get("summary")
.and_then(Value::as_array)
.is_some_and(|parts| parts.iter().any(output_content_part_has_text))
}
fn output_content_part_has_text(part: &Value) -> bool {
non_empty_string_field(part, &["text", "content"])
}
fn non_empty_string_field(value: &Value, fields: &[&str]) -> bool {
fields.iter().any(|field| {
value
.get(field)
.and_then(Value::as_str)
.is_some_and(|text| !text.trim().is_empty())
})
}
async fn send_live_response_stream_chunk(
sender: &mpsc::Sender<Result<Bytes, ResponseDispatchStreamError>>,
body_bytes: &mut Vec<u8>,
@@ -3349,43 +3273,49 @@ fn elapsed_millis_i64(started_at: Instant) -> i64 {
#[cfg(test)]
mod tests {
use super::response_body_has_first_token;
use crate::upstream::protocol::responses::response_body_has_first_event;
#[test]
fn first_token_detection_should_match_text_delta() {
fn first_event_detection_should_match_text_delta() {
let body = br#"event: response.output_text.delta
data: {"delta":"hello"}
"#;
assert!(response_body_has_first_token(body));
assert!(response_body_has_first_event(body));
}
#[test]
fn first_token_detection_should_match_output_item_done_text() {
let body = br#"event: response.output_item.done
data: {"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hello"}]}}
fn first_event_detection_should_match_response_created() {
let body = br#"event: response.created
data: {"type":"response.created","response":{"id":"resp_1","status":"in_progress"}}
"#;
assert!(response_body_has_first_token(body));
assert!(response_body_has_first_event(body));
}
#[test]
fn first_token_detection_should_fallback_to_completed_usage_output_tokens() {
let body = br#"event: response.completed
data: {"response":{"id":"resp_1","status":"completed","usage":{"input_tokens":3,"output_tokens":4}}}
fn first_event_detection_should_ignore_done_frame() {
let body = br#"data: [DONE]
"#;
assert!(response_body_has_first_token(body));
assert!(!response_body_has_first_event(body));
}
#[test]
fn first_token_detection_should_ignore_incomplete_frames() {
fn first_event_detection_should_ignore_incomplete_frames() {
let body = br#"event: response.output_text.delta
data: {"delta":"hello"}"#;
assert!(!response_body_has_first_token(body));
assert!(!response_body_has_first_event(body));
}
#[test]
fn first_event_detection_should_match_crlf_frames() {
let body = b"event: response.created\r\ndata: {\"type\":\"response.created\"}\r\n\r\n";
assert!(response_body_has_first_event(body));
}
}
+10 -2
View File
@@ -1,5 +1,7 @@
//! 调度层共享的账号级 Codex 上游调用。
use std::time::Instant;
use serde_json::Value;
use crate::{
@@ -130,6 +132,7 @@ pub(crate) async fn create_response_with_account(
request: &CodexResponsesRequest,
request_id: &str,
account: &Account,
started_at: Instant,
) -> Result<CodexBackendResponse, CodexClientError> {
let cookie_header = cloudflare
.cookie_header_for_request(&account.id, "/codex/responses")
@@ -140,7 +143,7 @@ pub(crate) async fn create_response_with_account(
&account.id,
);
codex
.create_response(
.create_response_with_pool_account_started_at(
request,
CodexRequestContext {
access_token: &account.access_token,
@@ -157,6 +160,8 @@ pub(crate) async fn create_response_with_account(
installation_id,
session_id: identity.conversation_id.as_deref(),
},
Some(&account.id),
started_at,
)
.await
}
@@ -178,7 +183,7 @@ async fn create_response_stream_with_account(
&account.id,
);
codex
.create_response_stream(
.create_response_stream_with_pool_account(
request,
CodexRequestContext {
access_token: &account.access_token,
@@ -195,6 +200,7 @@ async fn create_response_stream_with_account(
installation_id,
session_id: identity.conversation_id.as_deref(),
},
Some(&account.id),
)
.await
}
@@ -241,6 +247,7 @@ pub(crate) async fn create_response_with_account_retrying_5xx(
request: &CodexResponsesRequest,
request_id: &str,
account: &Account,
started_at: Instant,
) -> Result<CodexBackendResponse, CodexClientError> {
let mut retries = 0;
loop {
@@ -251,6 +258,7 @@ pub(crate) async fn create_response_with_account_retrying_5xx(
request,
request_id,
account,
started_at,
)
.await;
match result {
+2 -1
View File
@@ -484,6 +484,7 @@ pub fn translate_chat_to_codex(
.as_deref()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty());
let use_websocket = user.is_some();
let codex = CodexResponsesRequest::new_http_sse(
request.model,
String::new(),
@@ -506,7 +507,7 @@ pub fn translate_chat_to_codex(
service_tier: request.service_tier,
prompt_cache_key: user.clone(),
client_conversation_id: user,
force_http_sse: true,
use_websocket,
..codex
})
}
+24
View File
@@ -140,6 +140,30 @@ pub fn http_sse_fallback_allowed(request: &CodexResponsesRequest) -> bool {
)
}
/// 判断已收到的 Responses SSE 内容是否包含首个完整的有效数据事件。
pub fn response_body_has_first_event(body_bytes: &[u8]) -> bool {
let body = String::from_utf8_lossy(body_bytes);
let Some(complete_body) = complete_sse_body_prefix(&body) else {
return false;
};
parse_sse_events(complete_body).is_ok_and(|events| {
events.iter().any(|event| {
let data = event.data.trim();
!data.is_empty() && data != "[DONE]"
})
})
}
fn complete_sse_body_prefix(body: &str) -> Option<&str> {
let lf_end = body.rfind("\n\n").map(|index| index + 2);
let crlf_end = body.rfind("\r\n\r\n").map(|index| index + 4);
lf_end
.into_iter()
.chain(crlf_end)
.max()
.map(|end| &body[..end])
}
/// 从 Codex SSE 收集出的非流式 Responses 结果。
#[derive(Debug, Clone, PartialEq)]
pub enum CollectedResponse {
+79 -17
View File
@@ -4,7 +4,7 @@ use std::{
collections::HashMap,
pin::Pin,
sync::{Arc, Mutex, OnceLock},
time::{Duration, SystemTime, UNIX_EPOCH},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use async_trait::async_trait;
@@ -26,8 +26,8 @@ use crate::upstream::fingerprint::Fingerprint;
use crate::upstream::models::BackendModelEntry;
use crate::upstream::protocol::events::extract_sse_usage;
use crate::upstream::protocol::responses::{
http_sse_fallback_allowed, transport_for_request, CodexCompactRequest, CodexResponsesRequest,
CodexTransport,
http_sse_fallback_allowed, response_body_has_first_event, transport_for_request,
CodexCompactRequest, CodexResponsesRequest, CodexTransport,
};
use crate::upstream::protocol::sse::SseError;
use crate::upstream::protocol::websocket::{
@@ -41,10 +41,9 @@ use super::headers::{
};
use super::tls::{build_reqwest_client_with_custom_ca, custom_ca_env_cache_key, CustomCaError};
use super::websocket::{
execute_response_create_request, execute_response_create_request_stream_with_pool,
execute_response_create_request_with_pool, write_websocket_audit_artifact_from_env,
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketRateLimitHeaderUpdates,
CodexWebSocketTurnStateUpdate,
execute_response_create_request_stream_with_pool, execute_response_create_request_with_pool,
write_websocket_audit_artifact_from_env, CodexWebSocketConnection, CodexWebSocketExchangeError,
CodexWebSocketRateLimitHeaderUpdates, CodexWebSocketTurnStateUpdate,
};
use super::websocket_pool::{CodexWebSocketPool, CodexWebSocketPoolKey};
@@ -244,6 +243,8 @@ pub struct CodexBackendResponse {
pub set_cookie_headers: Vec<String>,
/// 上游透传的限流头。
pub rate_limit_headers: Vec<(String, String)>,
/// 首个有效上游 SSE/WebSocket 事件到达代理的耗时。
pub first_token_ms: Option<i64>,
}
/// Codex Responses 实际使用的上游传输。
@@ -345,6 +346,7 @@ impl CodexBackendClient {
&self,
upstream_request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
started_at: Instant,
) -> CodexClientResult<CodexBackendResponse> {
let headers = self.request_headers_for_http_response(upstream_request, context)?;
let response = self
@@ -371,7 +373,14 @@ impl CodexBackendClient {
});
}
let body = response.text().await?;
let mut body_bytes = Vec::new();
let mut first_token_ms = None;
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.try_next().await? {
body_bytes.extend_from_slice(&chunk);
update_first_token_ms(started_at, &body_bytes, &mut first_token_ms);
}
let body = String::from_utf8_lossy(&body_bytes).into_owned();
let usage = extract_sse_usage(&body).map_err(CodexClientError::InvalidSse)?;
Ok(CodexBackendResponse {
body,
@@ -380,6 +389,7 @@ impl CodexBackendClient {
turn_state,
set_cookie_headers,
rate_limit_headers,
first_token_ms,
})
}
@@ -429,16 +439,42 @@ impl CodexBackendClient {
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<CodexBackendResponse> {
self.create_response_started_at(request, context, Instant::now())
.await
}
pub async fn create_response_started_at(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
started_at: Instant,
) -> CodexClientResult<CodexBackendResponse> {
self.create_response_with_pool_account_started_at(request, context, None, started_at)
.await
}
pub async fn create_response_with_pool_account_started_at(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
started_at: Instant,
) -> CodexClientResult<CodexBackendResponse> {
let upstream_request = response_upstream_request(request, context);
match transport_for_request(&upstream_request) {
CodexTransport::HttpSse => {
self.create_response_http_sse(&upstream_request, context)
self.create_response_http_sse(&upstream_request, context, started_at)
.await
}
CodexTransport::WebSocketPreferred | CodexTransport::WebSocketRequired => {
match self
.create_response_websocket(&upstream_request, context)
.create_response_websocket(
&upstream_request,
context,
pool_account_id,
started_at,
)
.await
{
Ok(response) => Ok(response),
@@ -447,7 +483,7 @@ impl CodexBackendClient {
&& websocket_error_allows_http_fallback(&error) =>
{
tracing::warn!(error = %error, "websocket response failed; falling back to HTTP SSE");
self.create_response_http_sse(&upstream_request, context)
self.create_response_http_sse(&upstream_request, context, started_at)
.await
}
Err(error) => Err(error),
@@ -461,6 +497,16 @@ impl CodexBackendClient {
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<CodexBackendStreamingResponse> {
self.create_response_stream_with_pool_account(request, context, None)
.await
}
pub async fn create_response_stream_with_pool_account(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
) -> CodexClientResult<CodexBackendStreamingResponse> {
let upstream_request = response_upstream_request(request, context);
match transport_for_request(&upstream_request) {
@@ -470,7 +516,7 @@ impl CodexBackendClient {
}
CodexTransport::WebSocketPreferred | CodexTransport::WebSocketRequired => {
match self
.create_response_websocket_stream(&upstream_request, context)
.create_response_websocket_stream(&upstream_request, context, pool_account_id)
.await
{
Ok(response) => Ok(response),
@@ -492,6 +538,8 @@ impl CodexBackendClient {
&self,
upstream_request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
started_at: Instant,
) -> CodexClientResult<CodexBackendResponse> {
let websocket_request = websocket_upstream_request(upstream_request);
let headers = self.request_headers_for_http_response(&websocket_request, context)?;
@@ -510,12 +558,13 @@ impl CodexBackendClient {
if let Err(error) = write_websocket_audit_artifact_from_env(&artifact).await {
tracing::warn!(error = %error, "failed to write Codex WebSocket audit artifact");
}
let pool_key = self.websocket_pool_key(upstream_request, context);
let pool_key = self.websocket_pool_key(upstream_request, context, pool_account_id);
let exchange = match (self.websocket_pool.as_deref(), pool_key) {
(Some(pool), Some(key)) => {
execute_response_create_request_with_pool(&prepared, Some((pool, key))).await
execute_response_create_request_with_pool(&prepared, Some((pool, key)), started_at)
.await
}
_ => execute_response_create_request(&prepared).await,
_ => execute_response_create_request_with_pool(&prepared, None, started_at).await,
}
.map_err(websocket_exchange_error_to_client_error)?;
@@ -526,6 +575,7 @@ impl CodexBackendClient {
turn_state: exchange.turn_state,
set_cookie_headers: exchange.set_cookie_headers,
rate_limit_headers: exchange.rate_limit_headers,
first_token_ms: exchange.first_token_ms,
})
}
@@ -533,6 +583,7 @@ impl CodexBackendClient {
&self,
upstream_request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
) -> CodexClientResult<CodexBackendStreamingResponse> {
let websocket_request = websocket_upstream_request(upstream_request);
let headers = self.request_headers_for_http_response(&websocket_request, context)?;
@@ -551,7 +602,7 @@ impl CodexBackendClient {
if let Err(error) = write_websocket_audit_artifact_from_env(&artifact).await {
tracing::warn!(error = %error, "failed to write Codex WebSocket audit artifact");
}
let pool_key = self.websocket_pool_key(upstream_request, context);
let pool_key = self.websocket_pool_key(upstream_request, context, pool_account_id);
let exchange = match (self.websocket_pool.as_deref(), pool_key) {
(Some(pool), Some(key)) => {
execute_response_create_request_stream_with_pool(&prepared, Some((pool, key))).await
@@ -579,8 +630,9 @@ impl CodexBackendClient {
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
) -> Option<CodexWebSocketPoolKey> {
let account_id = context.account_id?;
let account_id = pool_account_id.or(context.account_id)?;
let conversation_id = request
.prompt_cache_key
.as_deref()
@@ -866,6 +918,16 @@ fn turn_state(response: &ReqwestResponse) -> Option<String> {
.map(ToString::to_string)
}
fn update_first_token_ms(started_at: Instant, body_bytes: &[u8], first_token_ms: &mut Option<i64>) {
if first_token_ms.is_none() && response_body_has_first_event(body_bytes) {
*first_token_ms = Some(elapsed_millis_i64(started_at).max(1));
}
}
fn elapsed_millis_i64(started_at: Instant) -> i64 {
started_at.elapsed().as_millis().min(i64::MAX as u128) as i64
}
fn set_cookie_headers(response: &ReqwestResponse) -> Vec<String> {
response
.headers()
+41 -11
View File
@@ -26,7 +26,7 @@ use tungstenite::{
use crate::infra::time::china_filename_timestamp_millis;
use crate::upstream::protocol::events::{self, TokenUsage};
use crate::upstream::protocol::responses::CodexResponsesRequest;
use crate::upstream::protocol::responses::{response_body_has_first_event, CodexResponsesRequest};
use crate::upstream::protocol::sse::SseError;
use crate::upstream::protocol::websocket::{
classify_websocket_error_frame, is_terminal_websocket_event,
@@ -113,6 +113,8 @@ pub struct CodexWebSocketExchange {
pub rate_limit_headers: Vec<(String, String)>,
/// 打开握手响应状态码。
pub handshake_status: u16,
/// 首个有效上游 WebSocket 事件到达代理的耗时。
pub first_token_ms: Option<i64>,
}
/// Responses WebSocket live SSE exchange result.
@@ -394,6 +396,7 @@ fn request_line_for_endpoint(endpoint: &str) -> String {
async fn execute_fresh_response_create_request(
request: &CodexWebSocketRequest,
started_at: Instant,
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
let (mut websocket, response) = connect_websocket(request.connection()).await?;
websocket
@@ -402,7 +405,7 @@ async fn execute_fresh_response_create_request(
let metadata = websocket_connection_metadata(&response);
let (exchange, _websocket, _metadata) =
collect_websocket_response(websocket, metadata, false).await?;
collect_websocket_response(websocket, metadata, false, started_at).await?;
Ok(exchange)
}
@@ -425,31 +428,40 @@ async fn execute_fresh_response_create_request_stream(
pub(crate) async fn execute_response_create_request_with_pool(
request: &CodexWebSocketRequest,
pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>,
started_at: Instant,
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
let Some((pool, key)) = pool else {
return execute_fresh_response_create_request(request).await;
return execute_fresh_response_create_request(request, started_at).await;
};
match pool.acquire(&key).await {
WebSocketPoolAcquire::Reused(connection) => {
let result =
execute_pooled_response_create_request(request, pool, key, *connection).await;
execute_pooled_response_create_request(request, pool, key, *connection, started_at)
.await;
match result {
Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstFrame {
..
}) => execute_fresh_response_create_request(request).await,
}) => execute_fresh_response_create_request(request, started_at).await,
other => other,
}
}
WebSocketPoolAcquire::FreshReserved => {
let result =
execute_fresh_pooled_response_create_request(request, pool, key.clone()).await;
let result = execute_fresh_pooled_response_create_request(
request,
pool,
key.clone(),
started_at,
)
.await;
if result.is_err() {
pool.discard(&key).await;
}
result
}
WebSocketPoolAcquire::Bypass => execute_fresh_response_create_request(request).await,
WebSocketPoolAcquire::Bypass => {
execute_fresh_response_create_request(request, started_at).await
}
}
}
@@ -497,6 +509,7 @@ async fn execute_fresh_pooled_response_create_request(
request: &CodexWebSocketRequest,
pool: &CodexWebSocketPool,
key: CodexWebSocketPoolKey,
started_at: Instant,
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
let (mut websocket, response) = connect_websocket(request.connection()).await?;
websocket
@@ -506,7 +519,7 @@ async fn execute_fresh_pooled_response_create_request(
let now = Instant::now();
let metadata = websocket_connection_metadata(&response);
let (exchange, websocket, metadata) =
collect_websocket_response(websocket, metadata, false).await?;
collect_websocket_response(websocket, metadata, false, started_at).await?;
let last_activity_at = Instant::now();
pool.put(
key,
@@ -552,6 +565,7 @@ async fn execute_pooled_response_create_request(
pool: &CodexWebSocketPool,
key: CodexWebSocketPoolKey,
mut connection: PooledWebSocketConnection,
started_at: Instant,
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
if let Err(error) = connection
.websocket
@@ -568,7 +582,9 @@ async fn execute_pooled_response_create_request(
let created_at = connection.created_at;
let last_ping_at = connection.last_ping_at;
match collect_websocket_response(connection.websocket, connection.metadata, true).await {
match collect_websocket_response(connection.websocket, connection.metadata, true, started_at)
.await
{
Ok((exchange, websocket, metadata)) => {
let last_activity_at = Instant::now();
pool.put(
@@ -663,7 +679,7 @@ async fn first_reused_stream_text(
pub async fn execute_response_create_request(
request: &CodexWebSocketRequest,
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
execute_response_create_request_with_pool(request, None).await
execute_response_create_request_with_pool(request, None, Instant::now()).await
}
// ---------------------------------------------------------------------------
@@ -674,6 +690,7 @@ async fn collect_websocket_response(
mut websocket: CodexWsStream,
mut metadata: CodexWebSocketConnectionMetadata,
reused_connection: bool,
started_at: Instant,
) -> Result<
(
CodexWebSocketExchange,
@@ -684,6 +701,7 @@ async fn collect_websocket_response(
> {
let mut body = String::new();
let mut saw_response_frame = false;
let mut first_token_ms = None;
loop {
let message = match next_websocket_message(&mut websocket).await {
@@ -743,6 +761,7 @@ async fn collect_websocket_response(
let forwarded = if let Some(frame) = websocket_event_to_sse_frame(&raw) {
body.push_str(&frame);
saw_response_frame = true;
update_first_token_ms(started_at, body.as_bytes(), &mut first_token_ms);
true
} else {
false
@@ -756,6 +775,7 @@ async fn collect_websocket_response(
set_cookie_headers: metadata.set_cookie_headers.clone(),
rate_limit_headers: metadata.rate_limit_headers.clone(),
handshake_status: metadata.handshake_status,
first_token_ms,
};
return Ok((exchange, websocket, metadata));
}
@@ -780,6 +800,16 @@ fn reused_connection_died_before_first_frame(
}
}
fn update_first_token_ms(started_at: Instant, body_bytes: &[u8], first_token_ms: &mut Option<i64>) {
if first_token_ms.is_none() && response_body_has_first_event(body_bytes) {
*first_token_ms = Some(elapsed_millis_i64(started_at).max(1));
}
}
fn elapsed_millis_i64(started_at: Instant) -> i64 {
started_at.elapsed().as_millis().min(i64::MAX as u128) as i64
}
fn websocket_connection_metadata(
response: &WsResponse<Option<Vec<u8>>>,
) -> CodexWebSocketConnectionMetadata {
@@ -94,6 +94,122 @@ async fn chat_completions_should_dispatch_to_codex_and_return_openai_response()
assert_eq!(metadata["usage"]["outputTokens"], 3);
}
#[tokio::test]
async fn chat_completions_with_user_should_use_and_reuse_websocket() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let base_url = format!("http://{}", listener.local_addr().unwrap());
let upstream = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut websocket = accept_async(stream).await.unwrap();
let first_payload = send_chat_websocket_response(
&mut websocket,
"websocket chat first",
"resp_chat_ws_first",
)
.await;
loop {
tokio::select! {
message = websocket.next() => {
match message {
Some(Ok(message)) if message.is_text() => {
let second_payload = send_chat_websocket_response_after_message(
&mut websocket,
message,
"websocket chat second",
"resp_chat_ws_second",
)
.await;
websocket.close(None).await.unwrap();
break (true, first_payload, second_payload);
}
Some(_) => {}
None => {
let second_payload = accept_chat_websocket_response(
&listener,
"websocket chat second",
"resp_chat_ws_second",
)
.await;
break (false, first_payload, second_payload);
}
}
}
accepted = listener.accept() => {
let (stream, _) = accepted.unwrap();
let mut second_websocket = accept_async(stream).await.unwrap();
let second_payload = send_chat_websocket_response(
&mut second_websocket,
"websocket chat second",
"resp_chat_ws_second",
)
.await;
second_websocket.close(None).await.unwrap();
break (false, first_payload, second_payload);
}
}
}
});
let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(base_url).await;
let first_response = app
.clone()
.oneshot(chat_json_request(
&api_key,
"req_chat_ws_first",
"chat-ws-user",
"Say hello over websocket",
))
.await
.unwrap();
assert_eq!(first_response.status(), StatusCode::OK);
let first_body = response_json(first_response).await;
assert_eq!(
first_body["choices"][0]["message"]["content"],
"websocket chat first"
);
let second_response = app
.oneshot(chat_json_request(
&api_key,
"req_chat_ws_second",
"chat-ws-user",
"Continue over the same websocket",
))
.await
.unwrap();
assert_eq!(second_response.status(), StatusCode::OK);
let second_body = response_json(second_response).await;
assert_eq!(
second_body["choices"][0]["message"]["content"],
"websocket chat second"
);
let (reused_connection, first_payload, second_payload) = upstream.await.unwrap();
let event = latest_usage_record(&pool, "v1.chat").await;
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
assert!(
reused_connection,
"second chat request opened a new websocket"
);
assert_eq!(first_payload["type"], "response.create");
assert_eq!(second_payload["type"], "response.create");
assert_eq!(
first_payload["prompt_cache_key"], second_payload["prompt_cache_key"],
"same chat user should keep the same account-scoped conversation key"
);
assert!(first_payload["prompt_cache_key"]
.as_str()
.is_some_and(|key| key.starts_with("cp_")));
assert_eq!(metadata["transport"], "websocket");
assert!(
metadata["firstTokenMs"]
.as_i64()
.is_some_and(|value| value > 0),
"chat websocket usage metadata should include first token latency: {metadata:?}",
);
}
#[tokio::test]
async fn chat_completions_stream_should_translate_codex_sse_to_openai_chunks() {
let server = MockServer::start().await;
@@ -912,3 +1028,70 @@ async fn chat_completions_should_return_quota_error_when_402_fallback_is_exhaust
assert_eq!(body["error"]["code"], "upstream_error");
assert_eq!(account_status.0, "quota_exhausted");
}
fn chat_json_request(api_key: &str, request_id: &str, user: &str, content: &str) -> Request<Body> {
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("authorization", format!("Bearer {api_key}"))
.header("content-type", "application/json")
.header("x-request-id", request_id)
.body(Body::from(
json!({
"model": "gpt-5.5",
"user": user,
"messages": [{"role": "user", "content": content}]
})
.to_string(),
))
.unwrap()
}
async fn accept_chat_websocket_response(
listener: &TcpListener,
content: &str,
response_id: &str,
) -> Value {
let (stream, _) = listener.accept().await.unwrap();
let mut websocket = accept_async(stream).await.unwrap();
let payload = send_chat_websocket_response(&mut websocket, content, response_id).await;
websocket.close(None).await.unwrap();
payload
}
async fn send_chat_websocket_response(
websocket: &mut WebSocketStream<TcpStream>,
content: &str,
response_id: &str,
) -> Value {
let message = websocket.next().await.unwrap().unwrap();
send_chat_websocket_response_after_message(websocket, message, content, response_id).await
}
async fn send_chat_websocket_response_after_message(
websocket: &mut WebSocketStream<TcpStream>,
message: Message,
content: &str,
response_id: &str,
) -> Value {
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
.expect("websocket payload should be json");
websocket
.send(Message::Text(
json!({
"type": "response.output_text.delta",
"delta": content
})
.to_string()
.into(),
))
.await
.unwrap();
websocket
.send(Message::Text(
response_completed_websocket_message(response_id).into(),
))
.await
.unwrap();
payload
}
@@ -85,6 +85,57 @@ async fn responses_should_use_imported_account_record_usage_cookie_and_usage_rec
assert_eq!(metadata["usage"]["inputTokens"], 7);
}
#[tokio::test]
async fn chat_completions_non_stream_should_record_first_token_latency() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("content-type", "text/event-stream")
.set_body_string(RESPONSES_COMPLETED_USAGE_SSE),
)
.expect(1)
.mount(&server)
.await;
let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(server.uri()).await;
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("authorization", format!("Bearer {api_key}"))
.header("content-type", "application/json")
.header("x-request-id", "req_chat_non_stream_first_token")
.body(Body::from(
json!({
"model": "gpt-5.5",
"messages": [{"role": "user", "content": "Say hello"}]
})
.to_string(),
))
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = response_json(response).await;
let event = latest_usage_record(&pool, "v1.chat").await;
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
assert_eq!(status, StatusCode::OK);
assert_eq!(body["usage"]["prompt_tokens"], 7);
assert_eq!(metadata["stream"], false);
assert_eq!(metadata["transport"], "http_sse");
assert!(
metadata["firstTokenMs"]
.as_i64()
.is_some_and(|value| value > 0),
"chat non-stream usage metadata should include first token latency: {metadata:?}",
);
}
#[tokio::test]
async fn responses_usage_record_should_store_resolved_upstream_model_after_alias_mapping() {
let server = MockServer::start().await;
+2 -1
View File
@@ -37,7 +37,8 @@ fn chat_completion_request_should_translate_to_codex_request() {
.expect("chat request should translate");
assert_eq!(codex.model, "gpt-5.5");
assert!(codex.force_http_sse);
assert!(codex.use_websocket);
assert!(!codex.force_http_sse);
assert_eq!(codex.prompt_cache_key.as_deref(), Some("client-123"));
assert_eq!(codex.client_conversation_id.as_deref(), Some("client-123"));
}
@@ -72,6 +72,68 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn codex_backend_client_should_not_reuse_pooled_websocket_across_local_accounts() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let accepted_connections = Arc::new(AtomicUsize::new(0));
let accepted_connections_for_server = Arc::clone(&accepted_connections);
let server = tokio::spawn(async move {
for response_id in ["resp_local_a", "resp_local_b"] {
let (stream, _) = listener.accept().await.unwrap();
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
let mut websocket = accept_codex_test_websocket(stream).await;
let _message = websocket.next().await.unwrap().unwrap();
websocket
.send(Message::Text(
completed_websocket_response(response_id, 3, 1).into(),
))
.await
.unwrap();
websocket.close(None).await.unwrap();
}
});
let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1)));
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
crate::support::fingerprint::test_fingerprint(),
)
.with_websocket_pool(Arc::clone(&pool));
let mut request =
codex_proxy_rs::upstream::protocol::responses::CodexResponsesRequest::new_http_sse(
"gpt-5.5",
"be brief",
Vec::new(),
);
request.previous_response_id = Some("resp_previous".to_string());
request.prompt_cache_key = Some("conversation-pool".to_string());
let first = backend
.create_response_with_pool_account_started_at(
&request,
request_context("req_pool_local_a", Some("same-chatgpt-account")),
Some("acct_local_a"),
std::time::Instant::now(),
)
.await
.expect("first local account websocket response should succeed");
let second = backend
.create_response_with_pool_account_started_at(
&request,
request_context("req_pool_local_b", Some("same-chatgpt-account")),
Some("acct_local_b"),
std::time::Instant::now(),
)
.await
.expect("second local account websocket response should succeed");
server.await.unwrap();
assert!(first.body.contains("resp_local_a"));
assert!(second.body.contains("resp_local_b"));
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn websocket_pool_should_bypass_busy_key_with_one_shot_connections() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();