mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-17 08:07:05 +00:00
feat: align codex websocket transport
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -240,6 +240,7 @@ impl ChatDispatchService {
|
||||
&request,
|
||||
request_id,
|
||||
&account,
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
@@ -397,6 +398,7 @@ impl ChatDispatchService {
|
||||
"responseId": response_id,
|
||||
"stream": false,
|
||||
"transport": backend_transport_name(response.transport),
|
||||
"firstTokenMs": response.first_token_ms,
|
||||
"usage": response.usage,
|
||||
}),
|
||||
rate_limit_headers: &response.rate_limit_headers,
|
||||
|
||||
@@ -63,8 +63,9 @@ use crate::{
|
||||
events::{extract_sse_usage, extract_usage, TokenUsage},
|
||||
responses::{
|
||||
apply_response_model_options, completed_response_metadata,
|
||||
reconvert_responses_sse_event_tuple_values, response_from_codex_sse,
|
||||
CodexCompactRequest, CodexResponsesRequest, CollectedResponse, ResponsesSseFailure,
|
||||
reconvert_responses_sse_event_tuple_values, response_body_has_first_event,
|
||||
response_from_codex_sse, CodexCompactRequest, CodexResponsesRequest,
|
||||
CollectedResponse, ResponsesSseFailure,
|
||||
},
|
||||
sse::{
|
||||
encode_sse_event, parse_sse_events, sse_body_has_done, SseError, DONE_SSE_FRAME,
|
||||
@@ -570,6 +571,7 @@ impl ResponseDispatchService {
|
||||
&request,
|
||||
request_id,
|
||||
&account,
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
self.account_pool.release(&release_account_id).await;
|
||||
@@ -892,6 +894,7 @@ impl ResponseDispatchService {
|
||||
"responseId": response_id,
|
||||
"stream": false,
|
||||
"transport": backend_transport_name(response.transport),
|
||||
"firstTokenMs": response.first_token_ms,
|
||||
"usage": response.usage,
|
||||
}),
|
||||
rate_limit_headers: &response.rate_limit_headers,
|
||||
@@ -2177,90 +2180,11 @@ fn update_first_token_ms(
|
||||
body_bytes: &[u8],
|
||||
first_token_ms: &mut Option<i64>,
|
||||
) {
|
||||
if first_token_ms.is_none() && response_body_has_first_token(body_bytes) {
|
||||
if first_token_ms.is_none() && response_body_has_first_event(body_bytes) {
|
||||
*first_token_ms = Some(elapsed_millis_i64(context.started_at).max(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn response_body_has_first_token(body_bytes: &[u8]) -> bool {
|
||||
let body = String::from_utf8_lossy(body_bytes);
|
||||
let Some(event_end) = body.rfind("\n\n") else {
|
||||
return false;
|
||||
};
|
||||
parse_sse_events(&body[..event_end + 2]).is_ok_and(|events| {
|
||||
events.iter().any(|event| {
|
||||
serde_json::from_str::<Value>(&event.data)
|
||||
.is_ok_and(|value| response_event_has_first_token(event.event.as_deref(), &value))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn response_event_has_first_token(event_name: Option<&str>, value: &Value) -> bool {
|
||||
match response_stream_event_type(event_name, value) {
|
||||
Some(
|
||||
"response.output_text.delta"
|
||||
| "response.reasoning_summary_text.delta"
|
||||
| "response.reasoning_text.delta"
|
||||
| "response.function_call_arguments.delta"
|
||||
| "response.custom_tool_call_input.delta",
|
||||
) => non_empty_string_field(value, &["delta"]),
|
||||
Some("response.output_text.done" | "response.reasoning_text.done") => {
|
||||
non_empty_string_field(value, &["text"])
|
||||
}
|
||||
Some("response.output_item.added" | "response.output_item.done") => value
|
||||
.get("item")
|
||||
.is_some_and(output_item_has_generated_content),
|
||||
Some("response.completed") => value
|
||||
.get("response")
|
||||
.is_some_and(completed_response_has_generated_content),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn response_stream_event_type<'a>(
|
||||
event_name: Option<&'a str>,
|
||||
value: &'a Value,
|
||||
) -> Option<&'a str> {
|
||||
event_name.or_else(|| value.get("type").and_then(Value::as_str))
|
||||
}
|
||||
|
||||
fn completed_response_has_generated_content(response: &Value) -> bool {
|
||||
non_empty_string_field(response, &["output_text"])
|
||||
|| response
|
||||
.get("output")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|items| items.iter().any(output_item_has_generated_content))
|
||||
|| response
|
||||
.pointer("/usage/output_tokens")
|
||||
.and_then(Value::as_i64)
|
||||
.is_some_and(|tokens| tokens > 0)
|
||||
}
|
||||
|
||||
fn output_item_has_generated_content(item: &Value) -> bool {
|
||||
non_empty_string_field(item, &["content", "text", "arguments", "input"])
|
||||
|| item
|
||||
.get("content")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|parts| parts.iter().any(output_content_part_has_text))
|
||||
|| item
|
||||
.get("summary")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|parts| parts.iter().any(output_content_part_has_text))
|
||||
}
|
||||
|
||||
fn output_content_part_has_text(part: &Value) -> bool {
|
||||
non_empty_string_field(part, &["text", "content"])
|
||||
}
|
||||
|
||||
fn non_empty_string_field(value: &Value, fields: &[&str]) -> bool {
|
||||
fields.iter().any(|field| {
|
||||
value
|
||||
.get(field)
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|text| !text.trim().is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
async fn send_live_response_stream_chunk(
|
||||
sender: &mpsc::Sender<Result<Bytes, ResponseDispatchStreamError>>,
|
||||
body_bytes: &mut Vec<u8>,
|
||||
@@ -3349,43 +3273,49 @@ fn elapsed_millis_i64(started_at: Instant) -> i64 {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::response_body_has_first_token;
|
||||
use crate::upstream::protocol::responses::response_body_has_first_event;
|
||||
|
||||
#[test]
|
||||
fn first_token_detection_should_match_text_delta() {
|
||||
fn first_event_detection_should_match_text_delta() {
|
||||
let body = br#"event: response.output_text.delta
|
||||
data: {"delta":"hello"}
|
||||
|
||||
"#;
|
||||
|
||||
assert!(response_body_has_first_token(body));
|
||||
assert!(response_body_has_first_event(body));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_token_detection_should_match_output_item_done_text() {
|
||||
let body = br#"event: response.output_item.done
|
||||
data: {"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hello"}]}}
|
||||
fn first_event_detection_should_match_response_created() {
|
||||
let body = br#"event: response.created
|
||||
data: {"type":"response.created","response":{"id":"resp_1","status":"in_progress"}}
|
||||
|
||||
"#;
|
||||
|
||||
assert!(response_body_has_first_token(body));
|
||||
assert!(response_body_has_first_event(body));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_token_detection_should_fallback_to_completed_usage_output_tokens() {
|
||||
let body = br#"event: response.completed
|
||||
data: {"response":{"id":"resp_1","status":"completed","usage":{"input_tokens":3,"output_tokens":4}}}
|
||||
fn first_event_detection_should_ignore_done_frame() {
|
||||
let body = br#"data: [DONE]
|
||||
|
||||
"#;
|
||||
|
||||
assert!(response_body_has_first_token(body));
|
||||
assert!(!response_body_has_first_event(body));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_token_detection_should_ignore_incomplete_frames() {
|
||||
fn first_event_detection_should_ignore_incomplete_frames() {
|
||||
let body = br#"event: response.output_text.delta
|
||||
data: {"delta":"hello"}"#;
|
||||
|
||||
assert!(!response_body_has_first_token(body));
|
||||
assert!(!response_body_has_first_event(body));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_event_detection_should_match_crlf_frames() {
|
||||
let body = b"event: response.created\r\ndata: {\"type\":\"response.created\"}\r\n\r\n";
|
||||
|
||||
assert!(response_body_has_first_event(body));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 调度层共享的账号级 Codex 上游调用。
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
@@ -130,6 +132,7 @@ pub(crate) async fn create_response_with_account(
|
||||
request: &CodexResponsesRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexBackendResponse, CodexClientError> {
|
||||
let cookie_header = cloudflare
|
||||
.cookie_header_for_request(&account.id, "/codex/responses")
|
||||
@@ -140,7 +143,7 @@ pub(crate) async fn create_response_with_account(
|
||||
&account.id,
|
||||
);
|
||||
codex
|
||||
.create_response(
|
||||
.create_response_with_pool_account_started_at(
|
||||
request,
|
||||
CodexRequestContext {
|
||||
access_token: &account.access_token,
|
||||
@@ -157,6 +160,8 @@ pub(crate) async fn create_response_with_account(
|
||||
installation_id,
|
||||
session_id: identity.conversation_id.as_deref(),
|
||||
},
|
||||
Some(&account.id),
|
||||
started_at,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -178,7 +183,7 @@ async fn create_response_stream_with_account(
|
||||
&account.id,
|
||||
);
|
||||
codex
|
||||
.create_response_stream(
|
||||
.create_response_stream_with_pool_account(
|
||||
request,
|
||||
CodexRequestContext {
|
||||
access_token: &account.access_token,
|
||||
@@ -195,6 +200,7 @@ async fn create_response_stream_with_account(
|
||||
installation_id,
|
||||
session_id: identity.conversation_id.as_deref(),
|
||||
},
|
||||
Some(&account.id),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -241,6 +247,7 @@ pub(crate) async fn create_response_with_account_retrying_5xx(
|
||||
request: &CodexResponsesRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexBackendResponse, CodexClientError> {
|
||||
let mut retries = 0;
|
||||
loop {
|
||||
@@ -251,6 +258,7 @@ pub(crate) async fn create_response_with_account_retrying_5xx(
|
||||
request,
|
||||
request_id,
|
||||
account,
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
|
||||
@@ -484,6 +484,7 @@ pub fn translate_chat_to_codex(
|
||||
.as_deref()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
let use_websocket = user.is_some();
|
||||
let codex = CodexResponsesRequest::new_http_sse(
|
||||
request.model,
|
||||
String::new(),
|
||||
@@ -506,7 +507,7 @@ pub fn translate_chat_to_codex(
|
||||
service_tier: request.service_tier,
|
||||
prompt_cache_key: user.clone(),
|
||||
client_conversation_id: user,
|
||||
force_http_sse: true,
|
||||
use_websocket,
|
||||
..codex
|
||||
})
|
||||
}
|
||||
|
||||
@@ -140,6 +140,30 @@ pub fn http_sse_fallback_allowed(request: &CodexResponsesRequest) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// 判断已收到的 Responses SSE 内容是否包含首个完整的有效数据事件。
|
||||
pub fn response_body_has_first_event(body_bytes: &[u8]) -> bool {
|
||||
let body = String::from_utf8_lossy(body_bytes);
|
||||
let Some(complete_body) = complete_sse_body_prefix(&body) else {
|
||||
return false;
|
||||
};
|
||||
parse_sse_events(complete_body).is_ok_and(|events| {
|
||||
events.iter().any(|event| {
|
||||
let data = event.data.trim();
|
||||
!data.is_empty() && data != "[DONE]"
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn complete_sse_body_prefix(body: &str) -> Option<&str> {
|
||||
let lf_end = body.rfind("\n\n").map(|index| index + 2);
|
||||
let crlf_end = body.rfind("\r\n\r\n").map(|index| index + 4);
|
||||
lf_end
|
||||
.into_iter()
|
||||
.chain(crlf_end)
|
||||
.max()
|
||||
.map(|end| &body[..end])
|
||||
}
|
||||
|
||||
/// 从 Codex SSE 收集出的非流式 Responses 结果。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum CollectedResponse {
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::{
|
||||
collections::HashMap,
|
||||
pin::Pin,
|
||||
sync::{Arc, Mutex, OnceLock},
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -26,8 +26,8 @@ use crate::upstream::fingerprint::Fingerprint;
|
||||
use crate::upstream::models::BackendModelEntry;
|
||||
use crate::upstream::protocol::events::extract_sse_usage;
|
||||
use crate::upstream::protocol::responses::{
|
||||
http_sse_fallback_allowed, transport_for_request, CodexCompactRequest, CodexResponsesRequest,
|
||||
CodexTransport,
|
||||
http_sse_fallback_allowed, response_body_has_first_event, transport_for_request,
|
||||
CodexCompactRequest, CodexResponsesRequest, CodexTransport,
|
||||
};
|
||||
use crate::upstream::protocol::sse::SseError;
|
||||
use crate::upstream::protocol::websocket::{
|
||||
@@ -41,10 +41,9 @@ use super::headers::{
|
||||
};
|
||||
use super::tls::{build_reqwest_client_with_custom_ca, custom_ca_env_cache_key, CustomCaError};
|
||||
use super::websocket::{
|
||||
execute_response_create_request, execute_response_create_request_stream_with_pool,
|
||||
execute_response_create_request_with_pool, write_websocket_audit_artifact_from_env,
|
||||
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketRateLimitHeaderUpdates,
|
||||
CodexWebSocketTurnStateUpdate,
|
||||
execute_response_create_request_stream_with_pool, execute_response_create_request_with_pool,
|
||||
write_websocket_audit_artifact_from_env, CodexWebSocketConnection, CodexWebSocketExchangeError,
|
||||
CodexWebSocketRateLimitHeaderUpdates, CodexWebSocketTurnStateUpdate,
|
||||
};
|
||||
use super::websocket_pool::{CodexWebSocketPool, CodexWebSocketPoolKey};
|
||||
|
||||
@@ -244,6 +243,8 @@ pub struct CodexBackendResponse {
|
||||
pub set_cookie_headers: Vec<String>,
|
||||
/// 上游透传的限流头。
|
||||
pub rate_limit_headers: Vec<(String, String)>,
|
||||
/// 首个有效上游 SSE/WebSocket 事件到达代理的耗时。
|
||||
pub first_token_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Codex Responses 实际使用的上游传输。
|
||||
@@ -345,6 +346,7 @@ impl CodexBackendClient {
|
||||
&self,
|
||||
upstream_request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
started_at: Instant,
|
||||
) -> CodexClientResult<CodexBackendResponse> {
|
||||
let headers = self.request_headers_for_http_response(upstream_request, context)?;
|
||||
let response = self
|
||||
@@ -371,7 +373,14 @@ impl CodexBackendClient {
|
||||
});
|
||||
}
|
||||
|
||||
let body = response.text().await?;
|
||||
let mut body_bytes = Vec::new();
|
||||
let mut first_token_ms = None;
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.try_next().await? {
|
||||
body_bytes.extend_from_slice(&chunk);
|
||||
update_first_token_ms(started_at, &body_bytes, &mut first_token_ms);
|
||||
}
|
||||
let body = String::from_utf8_lossy(&body_bytes).into_owned();
|
||||
let usage = extract_sse_usage(&body).map_err(CodexClientError::InvalidSse)?;
|
||||
Ok(CodexBackendResponse {
|
||||
body,
|
||||
@@ -380,6 +389,7 @@ impl CodexBackendClient {
|
||||
turn_state,
|
||||
set_cookie_headers,
|
||||
rate_limit_headers,
|
||||
first_token_ms,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -429,16 +439,42 @@ impl CodexBackendClient {
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<CodexBackendResponse> {
|
||||
self.create_response_started_at(request, context, Instant::now())
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_response_started_at(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
started_at: Instant,
|
||||
) -> CodexClientResult<CodexBackendResponse> {
|
||||
self.create_response_with_pool_account_started_at(request, context, None, started_at)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_response_with_pool_account_started_at(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
started_at: Instant,
|
||||
) -> CodexClientResult<CodexBackendResponse> {
|
||||
let upstream_request = response_upstream_request(request, context);
|
||||
match transport_for_request(&upstream_request) {
|
||||
CodexTransport::HttpSse => {
|
||||
self.create_response_http_sse(&upstream_request, context)
|
||||
self.create_response_http_sse(&upstream_request, context, started_at)
|
||||
.await
|
||||
}
|
||||
CodexTransport::WebSocketPreferred | CodexTransport::WebSocketRequired => {
|
||||
match self
|
||||
.create_response_websocket(&upstream_request, context)
|
||||
.create_response_websocket(
|
||||
&upstream_request,
|
||||
context,
|
||||
pool_account_id,
|
||||
started_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => Ok(response),
|
||||
@@ -447,7 +483,7 @@ impl CodexBackendClient {
|
||||
&& websocket_error_allows_http_fallback(&error) =>
|
||||
{
|
||||
tracing::warn!(error = %error, "websocket response failed; falling back to HTTP SSE");
|
||||
self.create_response_http_sse(&upstream_request, context)
|
||||
self.create_response_http_sse(&upstream_request, context, started_at)
|
||||
.await
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
@@ -461,6 +497,16 @@ impl CodexBackendClient {
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
self.create_response_stream_with_pool_account(request, context, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_response_stream_with_pool_account(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
let upstream_request = response_upstream_request(request, context);
|
||||
match transport_for_request(&upstream_request) {
|
||||
@@ -470,7 +516,7 @@ impl CodexBackendClient {
|
||||
}
|
||||
CodexTransport::WebSocketPreferred | CodexTransport::WebSocketRequired => {
|
||||
match self
|
||||
.create_response_websocket_stream(&upstream_request, context)
|
||||
.create_response_websocket_stream(&upstream_request, context, pool_account_id)
|
||||
.await
|
||||
{
|
||||
Ok(response) => Ok(response),
|
||||
@@ -492,6 +538,8 @@ impl CodexBackendClient {
|
||||
&self,
|
||||
upstream_request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
started_at: Instant,
|
||||
) -> CodexClientResult<CodexBackendResponse> {
|
||||
let websocket_request = websocket_upstream_request(upstream_request);
|
||||
let headers = self.request_headers_for_http_response(&websocket_request, context)?;
|
||||
@@ -510,12 +558,13 @@ impl CodexBackendClient {
|
||||
if let Err(error) = write_websocket_audit_artifact_from_env(&artifact).await {
|
||||
tracing::warn!(error = %error, "failed to write Codex WebSocket audit artifact");
|
||||
}
|
||||
let pool_key = self.websocket_pool_key(upstream_request, context);
|
||||
let pool_key = self.websocket_pool_key(upstream_request, context, pool_account_id);
|
||||
let exchange = match (self.websocket_pool.as_deref(), pool_key) {
|
||||
(Some(pool), Some(key)) => {
|
||||
execute_response_create_request_with_pool(&prepared, Some((pool, key))).await
|
||||
execute_response_create_request_with_pool(&prepared, Some((pool, key)), started_at)
|
||||
.await
|
||||
}
|
||||
_ => execute_response_create_request(&prepared).await,
|
||||
_ => execute_response_create_request_with_pool(&prepared, None, started_at).await,
|
||||
}
|
||||
.map_err(websocket_exchange_error_to_client_error)?;
|
||||
|
||||
@@ -526,6 +575,7 @@ impl CodexBackendClient {
|
||||
turn_state: exchange.turn_state,
|
||||
set_cookie_headers: exchange.set_cookie_headers,
|
||||
rate_limit_headers: exchange.rate_limit_headers,
|
||||
first_token_ms: exchange.first_token_ms,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -533,6 +583,7 @@ impl CodexBackendClient {
|
||||
&self,
|
||||
upstream_request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
let websocket_request = websocket_upstream_request(upstream_request);
|
||||
let headers = self.request_headers_for_http_response(&websocket_request, context)?;
|
||||
@@ -551,7 +602,7 @@ impl CodexBackendClient {
|
||||
if let Err(error) = write_websocket_audit_artifact_from_env(&artifact).await {
|
||||
tracing::warn!(error = %error, "failed to write Codex WebSocket audit artifact");
|
||||
}
|
||||
let pool_key = self.websocket_pool_key(upstream_request, context);
|
||||
let pool_key = self.websocket_pool_key(upstream_request, context, pool_account_id);
|
||||
let exchange = match (self.websocket_pool.as_deref(), pool_key) {
|
||||
(Some(pool), Some(key)) => {
|
||||
execute_response_create_request_stream_with_pool(&prepared, Some((pool, key))).await
|
||||
@@ -579,8 +630,9 @@ impl CodexBackendClient {
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> Option<CodexWebSocketPoolKey> {
|
||||
let account_id = context.account_id?;
|
||||
let account_id = pool_account_id.or(context.account_id)?;
|
||||
let conversation_id = request
|
||||
.prompt_cache_key
|
||||
.as_deref()
|
||||
@@ -866,6 +918,16 @@ fn turn_state(response: &ReqwestResponse) -> Option<String> {
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn update_first_token_ms(started_at: Instant, body_bytes: &[u8], first_token_ms: &mut Option<i64>) {
|
||||
if first_token_ms.is_none() && response_body_has_first_event(body_bytes) {
|
||||
*first_token_ms = Some(elapsed_millis_i64(started_at).max(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn elapsed_millis_i64(started_at: Instant) -> i64 {
|
||||
started_at.elapsed().as_millis().min(i64::MAX as u128) as i64
|
||||
}
|
||||
|
||||
fn set_cookie_headers(response: &ReqwestResponse) -> Vec<String> {
|
||||
response
|
||||
.headers()
|
||||
|
||||
@@ -26,7 +26,7 @@ use tungstenite::{
|
||||
|
||||
use crate::infra::time::china_filename_timestamp_millis;
|
||||
use crate::upstream::protocol::events::{self, TokenUsage};
|
||||
use crate::upstream::protocol::responses::CodexResponsesRequest;
|
||||
use crate::upstream::protocol::responses::{response_body_has_first_event, CodexResponsesRequest};
|
||||
use crate::upstream::protocol::sse::SseError;
|
||||
use crate::upstream::protocol::websocket::{
|
||||
classify_websocket_error_frame, is_terminal_websocket_event,
|
||||
@@ -113,6 +113,8 @@ pub struct CodexWebSocketExchange {
|
||||
pub rate_limit_headers: Vec<(String, String)>,
|
||||
/// 打开握手响应状态码。
|
||||
pub handshake_status: u16,
|
||||
/// 首个有效上游 WebSocket 事件到达代理的耗时。
|
||||
pub first_token_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Responses WebSocket live SSE exchange result.
|
||||
@@ -394,6 +396,7 @@ fn request_line_for_endpoint(endpoint: &str) -> String {
|
||||
|
||||
async fn execute_fresh_response_create_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let (mut websocket, response) = connect_websocket(request.connection()).await?;
|
||||
websocket
|
||||
@@ -402,7 +405,7 @@ async fn execute_fresh_response_create_request(
|
||||
|
||||
let metadata = websocket_connection_metadata(&response);
|
||||
let (exchange, _websocket, _metadata) =
|
||||
collect_websocket_response(websocket, metadata, false).await?;
|
||||
collect_websocket_response(websocket, metadata, false, started_at).await?;
|
||||
Ok(exchange)
|
||||
}
|
||||
|
||||
@@ -425,31 +428,40 @@ async fn execute_fresh_response_create_request_stream(
|
||||
pub(crate) async fn execute_response_create_request_with_pool(
|
||||
request: &CodexWebSocketRequest,
|
||||
pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let Some((pool, key)) = pool else {
|
||||
return execute_fresh_response_create_request(request).await;
|
||||
return execute_fresh_response_create_request(request, started_at).await;
|
||||
};
|
||||
|
||||
match pool.acquire(&key).await {
|
||||
WebSocketPoolAcquire::Reused(connection) => {
|
||||
let result =
|
||||
execute_pooled_response_create_request(request, pool, key, *connection).await;
|
||||
execute_pooled_response_create_request(request, pool, key, *connection, started_at)
|
||||
.await;
|
||||
match result {
|
||||
Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstFrame {
|
||||
..
|
||||
}) => execute_fresh_response_create_request(request).await,
|
||||
}) => execute_fresh_response_create_request(request, started_at).await,
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::FreshReserved => {
|
||||
let result =
|
||||
execute_fresh_pooled_response_create_request(request, pool, key.clone()).await;
|
||||
let result = execute_fresh_pooled_response_create_request(
|
||||
request,
|
||||
pool,
|
||||
key.clone(),
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
if result.is_err() {
|
||||
pool.discard(&key).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
WebSocketPoolAcquire::Bypass => execute_fresh_response_create_request(request).await,
|
||||
WebSocketPoolAcquire::Bypass => {
|
||||
execute_fresh_response_create_request(request, started_at).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,6 +509,7 @@ async fn execute_fresh_pooled_response_create_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
pool: &CodexWebSocketPool,
|
||||
key: CodexWebSocketPoolKey,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let (mut websocket, response) = connect_websocket(request.connection()).await?;
|
||||
websocket
|
||||
@@ -506,7 +519,7 @@ async fn execute_fresh_pooled_response_create_request(
|
||||
let now = Instant::now();
|
||||
let metadata = websocket_connection_metadata(&response);
|
||||
let (exchange, websocket, metadata) =
|
||||
collect_websocket_response(websocket, metadata, false).await?;
|
||||
collect_websocket_response(websocket, metadata, false, started_at).await?;
|
||||
let last_activity_at = Instant::now();
|
||||
pool.put(
|
||||
key,
|
||||
@@ -552,6 +565,7 @@ async fn execute_pooled_response_create_request(
|
||||
pool: &CodexWebSocketPool,
|
||||
key: CodexWebSocketPoolKey,
|
||||
mut connection: PooledWebSocketConnection,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
if let Err(error) = connection
|
||||
.websocket
|
||||
@@ -568,7 +582,9 @@ async fn execute_pooled_response_create_request(
|
||||
|
||||
let created_at = connection.created_at;
|
||||
let last_ping_at = connection.last_ping_at;
|
||||
match collect_websocket_response(connection.websocket, connection.metadata, true).await {
|
||||
match collect_websocket_response(connection.websocket, connection.metadata, true, started_at)
|
||||
.await
|
||||
{
|
||||
Ok((exchange, websocket, metadata)) => {
|
||||
let last_activity_at = Instant::now();
|
||||
pool.put(
|
||||
@@ -663,7 +679,7 @@ async fn first_reused_stream_text(
|
||||
pub async fn execute_response_create_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
execute_response_create_request_with_pool(request, None).await
|
||||
execute_response_create_request_with_pool(request, None, Instant::now()).await
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -674,6 +690,7 @@ async fn collect_websocket_response(
|
||||
mut websocket: CodexWsStream,
|
||||
mut metadata: CodexWebSocketConnectionMetadata,
|
||||
reused_connection: bool,
|
||||
started_at: Instant,
|
||||
) -> Result<
|
||||
(
|
||||
CodexWebSocketExchange,
|
||||
@@ -684,6 +701,7 @@ async fn collect_websocket_response(
|
||||
> {
|
||||
let mut body = String::new();
|
||||
let mut saw_response_frame = false;
|
||||
let mut first_token_ms = None;
|
||||
|
||||
loop {
|
||||
let message = match next_websocket_message(&mut websocket).await {
|
||||
@@ -743,6 +761,7 @@ async fn collect_websocket_response(
|
||||
let forwarded = if let Some(frame) = websocket_event_to_sse_frame(&raw) {
|
||||
body.push_str(&frame);
|
||||
saw_response_frame = true;
|
||||
update_first_token_ms(started_at, body.as_bytes(), &mut first_token_ms);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -756,6 +775,7 @@ async fn collect_websocket_response(
|
||||
set_cookie_headers: metadata.set_cookie_headers.clone(),
|
||||
rate_limit_headers: metadata.rate_limit_headers.clone(),
|
||||
handshake_status: metadata.handshake_status,
|
||||
first_token_ms,
|
||||
};
|
||||
return Ok((exchange, websocket, metadata));
|
||||
}
|
||||
@@ -780,6 +800,16 @@ fn reused_connection_died_before_first_frame(
|
||||
}
|
||||
}
|
||||
|
||||
fn update_first_token_ms(started_at: Instant, body_bytes: &[u8], first_token_ms: &mut Option<i64>) {
|
||||
if first_token_ms.is_none() && response_body_has_first_event(body_bytes) {
|
||||
*first_token_ms = Some(elapsed_millis_i64(started_at).max(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn elapsed_millis_i64(started_at: Instant) -> i64 {
|
||||
started_at.elapsed().as_millis().min(i64::MAX as u128) as i64
|
||||
}
|
||||
|
||||
fn websocket_connection_metadata(
|
||||
response: &WsResponse<Option<Vec<u8>>>,
|
||||
) -> CodexWebSocketConnectionMetadata {
|
||||
|
||||
@@ -94,6 +94,122 @@ async fn chat_completions_should_dispatch_to_codex_and_return_openai_response()
|
||||
assert_eq!(metadata["usage"]["outputTokens"], 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_completions_with_user_should_use_and_reuse_websocket() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let upstream = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_async(stream).await.unwrap();
|
||||
let first_payload = send_chat_websocket_response(
|
||||
&mut websocket,
|
||||
"websocket chat first",
|
||||
"resp_chat_ws_first",
|
||||
)
|
||||
.await;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
message = websocket.next() => {
|
||||
match message {
|
||||
Some(Ok(message)) if message.is_text() => {
|
||||
let second_payload = send_chat_websocket_response_after_message(
|
||||
&mut websocket,
|
||||
message,
|
||||
"websocket chat second",
|
||||
"resp_chat_ws_second",
|
||||
)
|
||||
.await;
|
||||
websocket.close(None).await.unwrap();
|
||||
break (true, first_payload, second_payload);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
let second_payload = accept_chat_websocket_response(
|
||||
&listener,
|
||||
"websocket chat second",
|
||||
"resp_chat_ws_second",
|
||||
)
|
||||
.await;
|
||||
break (false, first_payload, second_payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
accepted = listener.accept() => {
|
||||
let (stream, _) = accepted.unwrap();
|
||||
let mut second_websocket = accept_async(stream).await.unwrap();
|
||||
let second_payload = send_chat_websocket_response(
|
||||
&mut second_websocket,
|
||||
"websocket chat second",
|
||||
"resp_chat_ws_second",
|
||||
)
|
||||
.await;
|
||||
second_websocket.close(None).await.unwrap();
|
||||
break (false, first_payload, second_payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(base_url).await;
|
||||
|
||||
let first_response = app
|
||||
.clone()
|
||||
.oneshot(chat_json_request(
|
||||
&api_key,
|
||||
"req_chat_ws_first",
|
||||
"chat-ws-user",
|
||||
"Say hello over websocket",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first_response.status(), StatusCode::OK);
|
||||
let first_body = response_json(first_response).await;
|
||||
assert_eq!(
|
||||
first_body["choices"][0]["message"]["content"],
|
||||
"websocket chat first"
|
||||
);
|
||||
|
||||
let second_response = app
|
||||
.oneshot(chat_json_request(
|
||||
&api_key,
|
||||
"req_chat_ws_second",
|
||||
"chat-ws-user",
|
||||
"Continue over the same websocket",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second_response.status(), StatusCode::OK);
|
||||
let second_body = response_json(second_response).await;
|
||||
assert_eq!(
|
||||
second_body["choices"][0]["message"]["content"],
|
||||
"websocket chat second"
|
||||
);
|
||||
let (reused_connection, first_payload, second_payload) = upstream.await.unwrap();
|
||||
let event = latest_usage_record(&pool, "v1.chat").await;
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
|
||||
assert!(
|
||||
reused_connection,
|
||||
"second chat request opened a new websocket"
|
||||
);
|
||||
assert_eq!(first_payload["type"], "response.create");
|
||||
assert_eq!(second_payload["type"], "response.create");
|
||||
assert_eq!(
|
||||
first_payload["prompt_cache_key"], second_payload["prompt_cache_key"],
|
||||
"same chat user should keep the same account-scoped conversation key"
|
||||
);
|
||||
assert!(first_payload["prompt_cache_key"]
|
||||
.as_str()
|
||||
.is_some_and(|key| key.starts_with("cp_")));
|
||||
assert_eq!(metadata["transport"], "websocket");
|
||||
assert!(
|
||||
metadata["firstTokenMs"]
|
||||
.as_i64()
|
||||
.is_some_and(|value| value > 0),
|
||||
"chat websocket usage metadata should include first token latency: {metadata:?}",
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_completions_stream_should_translate_codex_sse_to_openai_chunks() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -912,3 +1028,70 @@ async fn chat_completions_should_return_quota_error_when_402_fallback_is_exhaust
|
||||
assert_eq!(body["error"]["code"], "upstream_error");
|
||||
assert_eq!(account_status.0, "quota_exhausted");
|
||||
}
|
||||
|
||||
fn chat_json_request(api_key: &str, request_id: &str, user: &str, content: &str) -> Request<Body> {
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", request_id)
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"user": user,
|
||||
"messages": [{"role": "user", "content": content}]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn accept_chat_websocket_response(
|
||||
listener: &TcpListener,
|
||||
content: &str,
|
||||
response_id: &str,
|
||||
) -> Value {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_async(stream).await.unwrap();
|
||||
let payload = send_chat_websocket_response(&mut websocket, content, response_id).await;
|
||||
websocket.close(None).await.unwrap();
|
||||
payload
|
||||
}
|
||||
|
||||
async fn send_chat_websocket_response(
|
||||
websocket: &mut WebSocketStream<TcpStream>,
|
||||
content: &str,
|
||||
response_id: &str,
|
||||
) -> Value {
|
||||
let message = websocket.next().await.unwrap().unwrap();
|
||||
send_chat_websocket_response_after_message(websocket, message, content, response_id).await
|
||||
}
|
||||
|
||||
async fn send_chat_websocket_response_after_message(
|
||||
websocket: &mut WebSocketStream<TcpStream>,
|
||||
message: Message,
|
||||
content: &str,
|
||||
response_id: &str,
|
||||
) -> Value {
|
||||
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
|
||||
.expect("websocket payload should be json");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": content
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
response_completed_websocket_message(response_id).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
payload
|
||||
}
|
||||
|
||||
@@ -85,6 +85,57 @@ async fn responses_should_use_imported_account_record_usage_cookie_and_usage_rec
|
||||
assert_eq!(metadata["usage"]["inputTokens"], 7);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_completions_non_stream_should_record_first_token_latency() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(RESPONSES_COMPLETED_USAGE_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(server.uri()).await;
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/chat/completions")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", "req_chat_non_stream_first_token")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"messages": [{"role": "user", "content": "Say hello"}]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response_json(response).await;
|
||||
let event = latest_usage_record(&pool, "v1.chat").await;
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body["usage"]["prompt_tokens"], 7);
|
||||
assert_eq!(metadata["stream"], false);
|
||||
assert_eq!(metadata["transport"], "http_sse");
|
||||
assert!(
|
||||
metadata["firstTokenMs"]
|
||||
.as_i64()
|
||||
.is_some_and(|value| value > 0),
|
||||
"chat non-stream usage metadata should include first token latency: {metadata:?}",
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_usage_record_should_store_resolved_upstream_model_after_alias_mapping() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -37,7 +37,8 @@ fn chat_completion_request_should_translate_to_codex_request() {
|
||||
.expect("chat request should translate");
|
||||
|
||||
assert_eq!(codex.model, "gpt-5.5");
|
||||
assert!(codex.force_http_sse);
|
||||
assert!(codex.use_websocket);
|
||||
assert!(!codex.force_http_sse);
|
||||
assert_eq!(codex.prompt_cache_key.as_deref(), Some("client-123"));
|
||||
assert_eq!(codex.client_conversation_id.as_deref(), Some("client-123"));
|
||||
}
|
||||
|
||||
@@ -72,6 +72,68 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_not_reuse_pooled_websocket_across_local_accounts() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
let accepted_connections_for_server = Arc::clone(&accepted_connections);
|
||||
let server = tokio::spawn(async move {
|
||||
for response_id in ["resp_local_a", "resp_local_b"] {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _message = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response(response_id, 3, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
}
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
crate::support::fingerprint::test_fingerprint(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let mut request =
|
||||
codex_proxy_rs::upstream::protocol::responses::CodexResponsesRequest::new_http_sse(
|
||||
"gpt-5.5",
|
||||
"be brief",
|
||||
Vec::new(),
|
||||
);
|
||||
request.previous_response_id = Some("resp_previous".to_string());
|
||||
request.prompt_cache_key = Some("conversation-pool".to_string());
|
||||
|
||||
let first = backend
|
||||
.create_response_with_pool_account_started_at(
|
||||
&request,
|
||||
request_context("req_pool_local_a", Some("same-chatgpt-account")),
|
||||
Some("acct_local_a"),
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.expect("first local account websocket response should succeed");
|
||||
let second = backend
|
||||
.create_response_with_pool_account_started_at(
|
||||
&request,
|
||||
request_context("req_pool_local_b", Some("same-chatgpt-account")),
|
||||
Some("acct_local_b"),
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.expect("second local account websocket response should succeed");
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(first.body.contains("resp_local_a"));
|
||||
assert!(second.body.contains("resp_local_b"));
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_pool_should_bypass_busy_key_with_one_shot_connections() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user