Compare commits

...
16 Commits
Author SHA1 Message Date
ZyyandCodex c2332bc8d0 chore: bump version to 3.3.23
Co-authored-by: Codex <noreply@openai.com>
2026-09-04 19:16:46 +08:00
ZyyandCodex 8f242e2875 feat(logging): add opt-in request dumps
Capture complete OpenAI Responses headers and payloads in a dedicated rotated log, keep the target out of normal sinks, and default the sensitive switch to off.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 19:15:28 +08:00
ZyyandCodex 51e9981bee fix(openai): resume metadata-close websocket requests
Treat metadata turn state as a same-request checkpoint, retry on the original account over WebSocket, and keep ambiguous requests without a checkpoint fail-closed.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 18:48:36 +08:00
ZyyandCodex 65fbbcb3ff chore: bump version to 3.3.22
Co-authored-by: Codex <noreply@openai.com>
2026-09-04 16:52:27 +08:00
ZyyandCodex bd086a7e3c fix(observability): show reasoning token usage
Expose aggregate reasoning tokens on account responses and use them in the token structure panel while retaining cache-write fields for API compatibility.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 16:51:18 +08:00
ZyyandCodex 3554ba13db fix(openai): preserve websocket turn state on failure
Bridge both metadata event names, project sticky state per response.create, and return captured state on pre-terminal failures without replaying ambiguous requests.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 16:51:01 +08:00
ZyyandCodex bcccebc9fa chore: bump version to 3.3.21
Co-authored-by: Codex <noreply@openai.com>
2026-09-04 11:52:25 +08:00
ZyyandCodex d67b46a9e2 fix(frontend): default Codex websocket config off
Keep inline switch text legible in the off state and omit WebSocket settings from generated Codex and CCSwitch configuration unless explicitly enabled.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 11:50:47 +08:00
ZyyandCodex 20c6d33cf4 fix(observability): retain recovered transport failures
Link successful same-session HTTP fallbacks to pending WebSocket failures, keep recovered records visible in Ops diagnostics, and label their recovery state.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 11:50:41 +08:00
ZyyandCodex cf0e84c548 fix(openai): isolate websocket continuation lanes
Include the downstream connection identity in pooled upstream WebSocket keys so concurrent clients cannot contend for connection-local response state.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 11:50:34 +08:00
ZyyandCodex 2332af9a08 chore: bump version to 3.3.20
Co-authored-by: Codex <noreply@openai.com>
2026-09-04 00:47:35 +08:00
ZyyandCodex 024fc9fcf0 fix(openai): fall back after websocket fast-path misses
Treat the 800 ms fast-path budget as a routing decision instead of a provider error, while preserving background preconnect for the next request.

Co-authored-by: Codex <noreply@openai.com>
2026-09-04 00:36:50 +08:00
ZyyandCodex 4220f4e095 fix(openai): recover websocket sessions after ambiguous failures
Co-authored-by: Codex <noreply@openai.com>
2026-09-03 23:33:26 +08:00
ZyyandCodex a709acb149 chore: bump version to 3.3.19
Co-authored-by: Codex <noreply@openai.com>
2026-09-03 17:56:10 +08:00
ZyyandCodex 466842acd8 fix(observability): search usage by API key
Resolve sk_ searches through client key references while only sending the visible key prefix from the frontend, and clarify the request detail search hint.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 17:38:29 +08:00
ZyyandCodex b91f28f95b fix(frontend): align configuration and usage copy
Generate the displayed Codex files and CCSwitch import payload from one source, carry the full settings through deep links, and clarify the quota percentage label.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 16:37:53 +08:00
62 changed files with 3276 additions and 459 deletions
@@ -230,6 +230,8 @@ pub(super) fn account_usage_view(
output_tokens_display: display_optional_tokens(usage.output_tokens),
cached_tokens: usage.cached_tokens,
cached_tokens_display: display_optional_tokens(usage.cached_tokens),
reasoning_tokens: usage.reasoning_tokens,
reasoning_tokens_display: display_optional_tokens(usage.reasoning_tokens),
image_input_tokens: usage.image_input_tokens,
image_input_tokens_display: display_optional_tokens(usage.image_input_tokens),
image_output_tokens: usage.image_output_tokens,
@@ -347,6 +349,8 @@ pub(super) fn empty_account_usage() -> AccountUsageView {
output_tokens_display: "—".to_owned(),
cached_tokens: None,
cached_tokens_display: "—".to_owned(),
reasoning_tokens: None,
reasoning_tokens_display: "—".to_owned(),
image_input_tokens: None,
image_input_tokens_display: "—".to_owned(),
image_output_tokens: None,
@@ -276,6 +276,8 @@ pub struct AccountUsageView {
pub output_tokens_display: String,
pub cached_tokens: Option<u64>,
pub cached_tokens_display: String,
pub reasoning_tokens: Option<u64>,
pub reasoning_tokens_display: String,
pub image_input_tokens: Option<u64>,
pub image_input_tokens_display: String,
pub image_output_tokens: Option<u64>,
@@ -14,8 +14,8 @@ use axum::{
response::{IntoResponse, Response},
};
use futures::stream;
use gateway_core::engine::CommitRequirement;
use gateway_core::engine::execution::{ClientTransport, ExecutionSession, StartedExecution};
use gateway_core::engine::{CommitRequirement, EngineError};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
use gateway_core::lifecycle::ConnectionGuard;
@@ -83,6 +83,19 @@ pub(crate) async fn responses(
Ok(started) => started,
Err(error) => return gateway_error_response(&error),
};
tracing::info!(
target: "request_dump",
provider = "openai",
request_id = %started.request_id,
request_direction = "downstream",
request_transport = if streaming { "http_sse" } else { "http_json" },
request_method = "POST",
request_path = "/v1/responses",
request_headers = ?headers,
request_body = %String::from_utf8_lossy(&body),
contains_sensitive_data = true,
"request dump (unredacted)"
);
let StartedExecution {
stream, session, ..
} = started;
@@ -152,7 +165,8 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
let events = match session.collect_uncommitted().await {
Ok(events) => events,
Err(error) => {
let response = engine_error_response(&error);
let response_headers = session.response_headers().to_vec();
let response = engine_error_response_with_headers(&error, &response_headers);
return execution.record_response_status(response).await;
}
};
@@ -178,7 +192,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
.await
{
execution.cancel_and_finalize().await;
let response = engine_error_response(&error);
let response = engine_error_response_with_headers(&error, &response_headers);
return execution.record_response_status(response).await;
}
if !session.is_finalized() {
@@ -246,6 +260,13 @@ fn apply_response_headers(
response
}
fn engine_error_response_with_headers(
error: &EngineError,
response_headers: &[ProviderResponseHeader],
) -> Response {
apply_response_headers(engine_error_response(error), response_headers)
}
/// 编码首个 SSE frame 后提交下游,再持续驱动同一执行会话。
pub async fn stream_execution_response(
session: Box<dyn ExecutionSession>,
@@ -267,7 +288,8 @@ pub async fn stream_execution_response(
return response;
}
Err(error) => {
let response = engine_error_response(&error);
let response_headers = session.response_headers().to_vec();
let response = engine_error_response_with_headers(&error, &response_headers);
return execution.record_response_status(response).await;
}
};
@@ -299,7 +321,7 @@ pub async fn stream_execution_response(
.await
{
execution.cancel_and_finalize().await;
let response = engine_error_response(&error);
let response = engine_error_response_with_headers(&error, &response_headers);
return execution.record_response_status(response).await;
}
let Some(session) = execution.into_session() else {
@@ -16,6 +16,7 @@ const OPENAI_PROTOCOL: &str = "openai";
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
const DOWNSTREAM_WEBSOCKET_CONNECTION_ID_CONTEXT_KEY: &str = "downstream_websocket_connection_id";
/// Responses 请求进入共享解码内核时的下游传输来源。
#[derive(Clone, Copy)]
@@ -35,6 +36,7 @@ pub struct OpenAiRequestHeaders {
include_timing_metrics: Option<String>,
codex_window_id: Option<String>,
parent_thread_id: Option<String>,
downstream_websocket_connection_id: Option<String>,
conversation_id: Option<String>,
session_id: Option<String>,
@@ -60,6 +62,7 @@ impl OpenAiRequestHeaders {
include_timing_metrics: header_string(headers, "x-responsesapi-include-timing-metrics"),
codex_window_id: header_string(headers, "x-codex-window-id"),
parent_thread_id: header_string(headers, "x-codex-parent-thread-id"),
downstream_websocket_connection_id: None,
conversation_id: header_string(headers, "conversation-id")
.or_else(|| header_string(headers, "conversation_id")),
session_id: header_string(headers, "session-id")
@@ -74,6 +77,16 @@ impl OpenAiRequestHeaders {
}
}
/// 绑定代理为当前下游 WebSocket 分配的连接身份。
#[must_use]
pub(super) fn with_downstream_websocket_connection_id(
mut self,
connection_id: impl Into<String>,
) -> Self {
self.downstream_websocket_connection_id = Some(connection_id.into());
self
}
fn apply_subagent(&self, body: &mut Map<String, Value>) {
if let Some(subagent) = self.subagent.as_deref() {
inject_subagent_metadata(body, subagent);
@@ -109,6 +122,11 @@ impl OpenAiRequestHeaders {
"parent_thread_id",
self.parent_thread_id.as_ref(),
);
insert_protocol_context(
&mut context,
DOWNSTREAM_WEBSOCKET_CONNECTION_ID_CONTEXT_KEY,
self.downstream_websocket_connection_id.as_ref(),
);
insert_protocol_context(
&mut context,
"conversation_id",
@@ -77,8 +77,8 @@ pub(super) async fn forward_execution(
return send_gateway_error(connection, &error, &request_id).await;
}
ActiveInput::Event(Err(error)) => {
let error = gateway_error_from_engine(&error);
return send_gateway_error(connection, &error, &request_id).await;
return send_initial_engine_error(connection, &mut execution, &error, &request_id)
.await;
}
ActiveInput::Control => continue,
ActiveInput::Disconnect => return ForwardOutcome::Disconnect,
@@ -271,6 +271,30 @@ pub(super) async fn forward_execution(
}
}
async fn send_initial_engine_error(
connection: &mut ResponsesWebSocketConnection,
execution: &mut PendingExecution,
error: &EngineError,
request_id: &Arc<str>,
) -> ForwardOutcome {
let response_headers = execution
.session_mut()
.map(|session| session.response_headers().to_vec())
.unwrap_or_default();
if !response_headers.is_empty()
&& !send_metadata(
connection,
request_id,
response_metadata_event(request_id, &response_headers),
)
.await
{
return ForwardOutcome::Disconnect;
}
let error = gateway_error_from_engine(error);
send_gateway_error(connection, &error, request_id).await
}
fn commit_connection_replay(
replay: &mut ConnectionReplaySnapshot,
encoder: &OpenAiResponsesEncoder,
@@ -60,12 +60,14 @@ pub(crate) async fn responses_websocket(
&headers,
connect_info.map(|Extension(ConnectInfo(address))| address),
);
let request_headers = OpenAiRequestHeaders::from_headers(&headers);
ResponsesWebSocketAdapter::new(service).upgrade_with_client_context(
websocket,
client,
client_ip,
user_agent,
OpenAiRequestHeaders::from_headers(&headers),
request_headers,
headers,
)
}
@@ -89,15 +91,29 @@ impl ResponsesWebSocketAdapter {
client_ip: Option<IpAddr>,
user_agent: Option<String>,
request_headers: OpenAiRequestHeaders,
raw_headers: HeaderMap,
) -> Response {
let connection_guard = match self.service.try_register_connection() {
Ok(guard) => guard,
Err(_) => return runtime_unavailable_response().into_response(),
};
let connection_id = self.service.next_request_id().replacen("req_", "ws_", 1);
tracing::info!(
target: "request_dump",
provider = "openai",
websocket_connection_id = %connection_id,
request_direction = "downstream",
request_transport = "websocket_handshake",
request_method = "GET",
request_path = "/v1/responses",
request_headers = ?raw_headers,
contains_sensitive_data = true,
"request dump (unredacted)"
);
let session = ResponsesWebSocketSession {
service: self.service.clone(),
client,
connection_id: self.service.next_request_id().replacen("req_", "ws_", 1),
connection_id,
client_ip,
user_agent,
request_headers,
@@ -143,6 +159,8 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
);
let _connection_guard = connection_guard;
let cancellation = lifecycle.cancellation();
let request_headers =
request_headers.with_downstream_websocket_connection_id(connection_id.clone());
let mut connection = ResponsesWebSocketConnection::new(socket, connection_id, cancellation);
let mut request_count = 0_u64;
let mut replay = ConnectionReplaySnapshot::default();
@@ -169,6 +187,18 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
}
request_count = request_count.saturating_add(1);
let correlation_id = Arc::<str>::from(service.next_request_id());
tracing::info!(
target: "request_dump",
provider = "openai",
websocket_connection_id = connection.id(),
correlation_id = %correlation_id,
request_index = request_count,
request_direction = "downstream",
request_transport = "websocket",
request_payload = %payload,
contains_sensitive_data = true,
"request dump (unredacted)"
);
let decoded = match decode_response_create_with_context(&payload, &request_headers) {
Ok(decoded) => decoded.with_client_context(client_ip, user_agent.clone()),
Err(error) => {
@@ -217,6 +247,14 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
continue;
}
};
tracing::info!(
target: "request_dump",
provider = "openai",
websocket_connection_id = connection.id(),
correlation_id = %correlation_id,
request_id = %started.request_id,
"request dump correlation"
);
if forward_execution(&mut connection, started, &mut replay).await
== ForwardOutcome::Disconnect
@@ -348,6 +348,8 @@ mod response {
output_tokens_display: "-".to_owned(),
cached_tokens: None,
cached_tokens_display: "-".to_owned(),
reasoning_tokens: None,
reasoning_tokens_display: "-".to_owned(),
image_input_tokens: None,
image_input_tokens_display: "-".to_owned(),
image_output_tokens: None,
@@ -374,6 +376,8 @@ mod response {
let value = serde_json::to_value(view).expect("serialize account usage");
assert!(value["inputTokens"].is_null());
assert!(value["totalTokens"].is_null());
assert!(value["reasoningTokens"].is_null());
assert_eq!(value["reasoningTokensDisplay"], "-");
assert_eq!(value["createdTokensDisplay"], "-");
}
@@ -893,6 +893,41 @@ async fn streaming_rate_limit_before_first_frame_should_persist_the_returned_429
assert_eq!(trace.client_statuses(), vec![429]);
}
#[tokio::test]
async fn streaming_failure_before_first_frame_should_return_safe_observed_headers() {
let trace = Arc::new(Trace::default());
let session = FakeSession::streaming(
Arc::clone(&trace),
vec![NextStep::Error(EngineError::Provider(ProviderError::new(
ProviderErrorKind::Unavailable,
UpstreamSendState::Ambiguous,
)))],
)
.with_response_headers(vec![
ProviderResponseHeader::new(
"x-codex-turn-state",
Bytes::from_static(b"turn-state-before-close"),
),
ProviderResponseHeader::new(
"authorization",
Bytes::from_static(b"must-not-cross-boundary"),
),
]);
let response = stream_execution_response(Box::new(session), None).await;
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
assert_eq!(
response
.headers()
.get("x-codex-turn-state")
.and_then(|value| value.to_str().ok()),
Some("turn-state-before-close")
);
assert!(response.headers().get("authorization").is_none());
assert_eq!(trace.client_statuses(), vec![502]);
}
#[tokio::test]
async fn streaming_upstream_http_failure_before_first_frame_should_preserve_raw_response() {
let trace = Arc::new(Trace::default());
@@ -979,11 +1014,29 @@ async fn buffered_upstream_http_failure_should_preserve_raw_response() {
raw_body.clone(),
));
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
.with_collect_error(EngineError::Provider(error));
.with_collect_error(EngineError::Provider(error))
.with_response_headers(vec![
ProviderResponseHeader::new(
"x-codex-turn-state",
Bytes::from_static(b"turn-state-before-close"),
),
ProviderResponseHeader::new(
"authorization",
Bytes::from_static(b"must-not-cross-boundary"),
),
]);
let response = collect_execution_response(Box::new(session)).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
response
.headers()
.get("x-codex-turn-state")
.and_then(|value| value.to_str().ok()),
Some("turn-state-before-close")
);
assert!(response.headers().get("authorization").is_none());
assert_eq!(
to_bytes(response.into_body(), usize::MAX)
.await
@@ -428,6 +428,7 @@ async fn idle_connection_reaches_the_official_limit_without_starting_an_executio
client: authenticated_client("sk_ws_atomic"),
trace: Arc::clone(&trace),
response_headers: Vec::new(),
fail_before_first_event: false,
});
let app = api_router(execution).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
@@ -2,7 +2,7 @@ mod connection;
mod protocol;
use std::sync::{
Arc,
Arc, Mutex,
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use std::time::{Duration, SystemTime};
@@ -280,17 +280,26 @@ struct AtomicFailureTrace {
next_calls: AtomicUsize,
committed: AtomicBool,
finalized: AtomicBool,
downstream_websocket_connection_ids: Mutex<Vec<String>>,
}
struct AtomicFailureSession {
trace: Arc<AtomicFailureTrace>,
response_headers: Vec<ProviderResponseHeader>,
fail_before_first_event: bool,
}
impl ExecutionSession for AtomicFailureSession {
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
Box::pin(async move {
match self.trace.next_calls.fetch_add(1, Ordering::AcqRel) {
let next_call = self.trace.next_calls.fetch_add(1, Ordering::AcqRel);
if self.fail_before_first_event && next_call == 0 {
return Err(EngineError::Provider(ProviderError::new(
ProviderErrorKind::Unavailable,
UpstreamSendState::Ambiguous,
)));
}
match next_call {
0 => Ok(Some(atomic_failure_batch())),
1 => {
self.trace.finalized.store(true, Ordering::Release);
@@ -340,6 +349,7 @@ struct AtomicFailureExecution {
client: AuthenticatedClient,
trace: Arc<AtomicFailureTrace>,
response_headers: Vec<ProviderResponseHeader>,
fail_before_first_event: bool,
}
impl ExecutionService for AtomicFailureExecution {
@@ -365,6 +375,19 @@ impl ExecutionService for AtomicFailureExecution {
request: StartExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
self.trace.starts.fetch_add(1, Ordering::AcqRel);
if let Operation::Generate(generate) = &request.operation
&& let Some(connection_id) = generate
.protocol_payload()
.context()
.get("downstream_websocket_connection_id")
.and_then(Value::as_str)
{
self.trace
.downstream_websocket_connection_ids
.lock()
.expect("downstream connection trace lock")
.push(connection_id.to_owned());
}
Box::pin(async move {
Ok(StartedExecution {
request_id: gateway_core::engine::ModelRequestId::new("req_ws_atomic")
@@ -374,6 +397,7 @@ impl ExecutionService for AtomicFailureExecution {
session: Box::new(AtomicFailureSession {
trace: Arc::clone(&self.trace),
response_headers: self.response_headers.clone(),
fail_before_first_event: self.fail_before_first_event,
}),
})
})
@@ -439,6 +463,7 @@ async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
client: authenticated_client("sk_ws_atomic"),
trace: Arc::clone(&trace),
response_headers: Vec::new(),
fail_before_first_event: false,
});
let app = api_router(execution).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
@@ -510,6 +535,14 @@ async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
assert!(trace.committed.load(Ordering::Acquire));
assert_eq!(trace.next_calls.load(Ordering::Acquire), 2);
assert!(trace.finalized.load(Ordering::Acquire));
{
let downstream_connection_ids = trace
.downstream_websocket_connection_ids
.lock()
.expect("downstream connection trace lock");
assert_eq!(downstream_connection_ids.len(), 1);
assert!(downstream_connection_ids[0].starts_with("ws_"));
}
socket.close(None).await.expect("close WebSocket");
server.abort();
@@ -540,6 +573,7 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req_upstream")),
],
fail_before_first_event: false,
});
let app = api_router(execution).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
@@ -624,6 +658,91 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
server.abort();
}
#[tokio::test]
async fn websocket_failure_before_first_event_should_return_observed_metadata_then_error() {
let trace = Arc::new(AtomicFailureTrace::default());
let execution = Arc::new(AtomicFailureExecution {
client: authenticated_client("sk_ws_atomic"),
trace: Arc::clone(&trace),
response_headers: vec![
ProviderResponseHeader::new(
"x-codex-turn-state",
Bytes::from_static(b"turn-state-before-close"),
),
ProviderResponseHeader::new(
"authorization",
Bytes::from_static(b"must-not-cross-boundary"),
),
],
fail_before_first_event: true,
});
let app = api_router(execution).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind WebSocket test server");
let address = listener.local_addr().expect("test server address");
let server = tokio::spawn(async move {
axum::serve(listener, app)
.await
.expect("serve WebSocket test router");
});
let mut request = format!("ws://{address}/v1/responses")
.into_client_request()
.expect("WebSocket request");
request.headers_mut().insert(
AUTHORIZATION,
"Bearer sk_ws_atomic".parse().expect("authorization"),
);
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
socket
.send(ClientMessage::Text(
json!({
"type": "response.create",
"model": "model-a",
"input": "hello"
})
.to_string()
.into(),
))
.await
.expect("send response.create");
let mut messages = Vec::<Value>::new();
while messages.len() < 2 {
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
.await
.expect("WebSocket response timeout")
.expect("WebSocket remains open")
.expect("valid WebSocket frame");
let ClientMessage::Text(text) = message else {
continue;
};
messages.push(serde_json::from_str(&text).expect("JSON WebSocket event"));
}
assert_eq!(
messages
.iter()
.filter_map(|message| message.get("type").and_then(Value::as_str))
.collect::<Vec<_>>(),
vec!["response.metadata", "error"]
);
let headers = messages[0]
.get("headers")
.and_then(Value::as_object)
.expect("response metadata headers");
assert_eq!(
headers.get("x-codex-turn-state"),
Some(&json!("turn-state-before-close"))
);
assert!(!headers.contains_key("authorization"));
assert!(!trace.committed.load(Ordering::Acquire));
socket.close(None).await.expect("close WebSocket");
server.abort();
}
#[tokio::test]
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
let trace = Arc::new(AtomicFailureTrace::default());
@@ -631,6 +750,7 @@ async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit(
client: authenticated_client("sk_ws_atomic"),
trace,
response_headers: Vec::new(),
fail_before_first_event: false,
});
let app = api_router(execution).await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
@@ -210,6 +210,8 @@ struct CurrentAttempt {
index: NonZeroU32,
started_at: SystemTime,
send_observed: bool,
/// Provider 已为当前 attempt 交付可供同请求恢复的不透明状态检查点。
session_checkpoint_observed: bool,
response_observation: Option<ProviderResponseObservation>,
}
@@ -570,6 +572,9 @@ where
return Err(EngineError::Deadline);
}
PollBoundary::Item(Some(Ok(mut event))) => {
if let Some(state) = event.session_update() {
self.observe_session_checkpoint(state);
}
if let Some(observation) = event.take_observation() {
self.observe_response(observation);
}
@@ -931,6 +936,7 @@ where
index: next_attempt,
started_at: SystemTime::now(),
send_observed: false,
session_checkpoint_observed: false,
response_observation: None,
});
Ok(None)
@@ -972,6 +978,17 @@ where
self.mark_send_observed().await;
}
fn observe_session_checkpoint(&mut self, state: &ProviderSessionState) {
let Some(current) = self.current.as_mut() else {
return;
};
if state.provider() != current.metadata.provider().as_str() {
return;
}
current.session_checkpoint_observed = true;
self.operation.set_provider_session_state(state.clone());
}
async fn mark_send_observed(&mut self) {
let Some(current) = self.current.as_mut() else {
return;
@@ -1071,6 +1088,11 @@ where
.await;
}
let provider_proved_replay_safe = provider_proved_replay_safe(&error);
// Ambiguous 仍是默认 fail-closed。唯一例外是 Provider 在同一 attempt 中先
// 交付了不透明恢复检查点,并明确证明从该检查点重放安全;该例外只用于
// 固定原账号的 transport recovery,不放开普通换号或 continuation 重放。
let checkpointed_transport_replay =
current.session_checkpoint_observed && error.replay_is_safe();
let continuation_retry = self.prepare_continuation_retry(
&current,
&error,
@@ -1093,14 +1115,16 @@ where
delay,
}) if self.downstream_committed_at.is_none()
&& !self.delivery_pending
&& attempt_send_state != UpstreamSendState::Ambiguous =>
&& (attempt_send_state != UpstreamSendState::Ambiguous
|| checkpointed_transport_replay) =>
{
Some((AttemptTransport::Retry(retry_index), delay))
}
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
if self.downstream_committed_at.is_none()
&& !self.delivery_pending
&& attempt_send_state != UpstreamSendState::Ambiguous =>
&& (attempt_send_state != UpstreamSendState::Ambiguous
|| checkpointed_transport_replay) =>
{
Some((AttemptTransport::Fallback, Duration::ZERO))
}
+9 -1
View File
@@ -623,13 +623,21 @@ impl ProviderError {
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
}
/// 标记 Provider 已证明本次拒绝没有执行生成,可在下游提交前重放。
/// 标记 Provider 已证明本次失败可在下游提交前安全重放。
///
/// 证明可以来自明确的“未执行”拒绝,也可以来自同 attempt 已交付、可恢复
/// 同一业务请求的状态检查点;后者仍须由 Core 的同账号恢复门禁单独校验。
#[must_use]
pub const fn with_replay_safe(mut self) -> Self {
self.replay_safe = true;
self
}
/// 原地标记 Provider 已证明本次失败可从已交付的状态检查点安全恢复。
pub const fn set_replay_safe(&mut self) {
self.replay_safe = true;
}
/// 允许 Core 仅在客户端尚未收到任何事件时执行一次受预算约束的换号恢复。
///
/// 该标记不证明上游未执行,也不等价于 [`Self::with_replay_safe`];它只表达
+2 -1
View File
@@ -763,7 +763,8 @@ impl ProviderEvent {
}
}
/// 把本事件标记为 Provider 连接内状态的提交边界。
/// 附加 Provider 私有状态检查点;Core 可将其用于同请求恢复,协议连接可在
/// terminal 事件上持有它供下一轮使用。
pub fn attach_session_update(&mut self, state: ProviderSessionState) {
self.session_update = Some(Box::new(state));
}
@@ -270,6 +270,7 @@ enum Script {
struct ScriptedProvider {
scripts: Mutex<VecDeque<Script>>,
contexts: Mutex<Vec<AttemptContext>>,
operations: Mutex<Vec<Operation>>,
}
impl ScriptedProvider {
@@ -277,6 +278,7 @@ impl ScriptedProvider {
Self {
scripts: Mutex::new(scripts.into()),
contexts: Mutex::new(Vec::new()),
operations: Mutex::new(Vec::new()),
}
}
}
@@ -302,6 +304,10 @@ impl Provider for ScriptedProvider {
request: ProviderRequest,
context: AttemptContext,
) -> Result<ProviderStream, ProviderError> {
self.operations
.lock()
.expect("operations lock")
.push(request.operation().clone());
self.contexts.lock().expect("contexts lock").push(context);
let script = self
.scripts
@@ -2838,14 +2844,25 @@ fn ambiguous_send_state_stops_retry() {
fn ambiguous_pre_delivery_retry_marker_does_not_rotate_account() {
let operation = generate_operation();
let route_plan = plan(&operation);
let checkpoint = ProviderSessionState::new(
"openai",
Map::from_iter([("checkpoint".to_owned(), json!("opaque-state"))]),
)
.expect("provider checkpoint");
let mut checkpoint_event = ProviderEvent::observation(ProviderResponseObservation::new(
UpstreamTransport::new("websocket").expect("transport"),
));
checkpoint_event.attach_session_update(checkpoint);
let (coordinator, store, provider) = coordinator(vec![
Script::Stream {
Script::ObservedStream {
account_id: "acct_first",
items: vec![Err(ProviderError::new(
ProviderErrorKind::Transport,
UpstreamSendState::Ambiguous,
)
.with_pre_delivery_retry())],
items: vec![
Ok(checkpoint_event),
Err(
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
.with_pre_delivery_retry(),
),
],
},
Script::Stream {
account_id: "acct_second",
@@ -2874,6 +2891,70 @@ fn ambiguous_pre_delivery_retry_marker_does_not_rotate_account() {
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
}
#[test]
fn ambiguous_transport_retry_uses_provider_session_checkpoint_on_same_account() {
let operation = generate_operation();
let route_plan = plan(&operation);
let checkpoint = ProviderSessionState::new(
"openai",
Map::from_iter([("checkpoint".to_owned(), json!("opaque-state"))]),
)
.expect("provider checkpoint");
let mut checkpoint_event = ProviderEvent::observation(ProviderResponseObservation::new(
UpstreamTransport::new("websocket").expect("transport"),
));
checkpoint_event.attach_session_update(checkpoint.clone());
let retry_index = NonZeroU32::new(1).expect("retry index");
let (coordinator, store, provider) = coordinator(vec![
Script::ObservedStream {
account_id: "acct_first",
items: vec![
Ok(checkpoint_event),
Err(
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
.with_replay_safe()
.with_pre_delivery_transport_retry(retry_index, Duration::ZERO),
),
],
},
Script::Stream {
account_id: "acct_first",
items: complete_stream(None),
},
]);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
None,
CancellationToken::new(),
))
.expect("start execution");
block_on(session.collect_uncommitted()).expect("checkpointed retry succeeds");
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
let original = ProviderAccountId::new("acct_first").expect("account id");
let contexts = provider.contexts.lock().expect("contexts lock");
assert_eq!(contexts.len(), 2);
assert_eq!(
contexts[1].transport(),
AttemptTransport::Retry(retry_index)
);
assert_eq!(contexts[1].required_account(), Some(&original));
drop(contexts);
let operations = provider.operations.lock().expect("operations lock");
assert_eq!(
operations[1].provider_session_state("openai"),
Some(&checkpoint)
);
let state = store.state.lock().expect("store lock");
assert_eq!(state.intermediate_failures, 1);
assert_eq!(state.finalizations[0].attempt_count, 2);
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
}
#[test]
fn pre_delivery_transport_fallback_retries_the_same_account_with_fallback_transport() {
let operation = generate_operation();
@@ -118,6 +118,9 @@ pub struct LoggingConfig {
pub level: String,
pub stdout: bool,
pub file: FileLoggingConfig,
/// 将完整请求原文写入独立诊断日志;默认关闭,因为内容包含凭据。
#[serde(default)]
pub request_dump: bool,
}
impl LoggingConfig {
+68 -26
View File
@@ -6,16 +6,19 @@ use std::io::{self, Write};
use std::path::{Path, PathBuf};
use chrono::Utc;
use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
use tracing_appender::non_blocking::{NonBlocking, NonBlockingBuilder, WorkerGuard};
use tracing_subscriber::EnvFilter;
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _};
use crate::config::LoggingConfig;
use crate::config::{FileLoggingConfig, LoggingConfig};
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump";
/// 请求转储含完整凭据与正文,只允许在显式开启时写入独立文件。
const REQUEST_DUMP_LOG_TARGET: &str = "request_dump";
/// non-blocking 日志 writer 的进程级守卫。
pub struct LogGuard {
@@ -39,6 +42,7 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
let file_filter = application_file_filter(&directive)?;
let recovery_file_filter = oauth_recovery_file_filter();
let request_dump_file_filter = request_dump_file_filter();
let stdout_filter =
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
.map_err(|_| LogError::InvalidFilter)?;
@@ -52,37 +56,30 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
writer
});
let (file_writer, recovery_file_writer) = if config.file.enabled {
let maximum_bytes = config
.file
.max_file_size_mb
.checked_mul(1024 * 1024)
.ok_or(LogError::SizeOverflow)?;
let file_writer = RotatingLogWriter::open(
config.file.directory.clone(),
LOG_FILE_PREFIX,
maximum_bytes,
config.file.retention_days,
config.file.max_files,
)?;
let recovery_file_writer = RotatingLogWriter::open(
config.file.directory.clone(),
let (file_writer, file_guard) =
rotating_file_writer(&config.file, LOG_FILE_PREFIX, "gateway-log-file")?;
let (recovery_file_writer, recovery_file_guard) = rotating_file_writer(
&config.file,
OAUTH_RECOVERY_LOG_FILE_PREFIX,
maximum_bytes,
config.file.retention_days,
config.file.max_files,
"gateway-log-oauth-recovery-file",
)?;
let (file_writer, file_guard) = NonBlockingBuilder::default()
.thread_name("gateway-log-file")
.finish(file_writer);
let (recovery_file_writer, recovery_file_guard) = NonBlockingBuilder::default()
.thread_name("gateway-log-oauth-recovery-file")
.finish(recovery_file_writer);
guards.push(file_guard);
guards.push(recovery_file_guard);
(Some(file_writer), Some(recovery_file_writer))
} else {
(None, None)
};
let request_dump_file_writer = if config.request_dump {
let (writer, guard) = rotating_file_writer(
&config.file,
REQUEST_DUMP_LOG_FILE_PREFIX,
"gateway-log-request-dump-file",
)?;
guards.push(guard);
Some(writer)
} else {
None
};
let stdout_layer = stdout_writer.map(|writer| {
tracing_subscriber::fmt::layer()
@@ -118,10 +115,24 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
.with_span_list(true)
.with_filter(recovery_file_filter)
});
let request_dump_file_layer = request_dump_file_writer.map(|writer| {
tracing_subscriber::fmt::layer()
.json()
.with_writer(writer)
.with_target(true)
.with_file(true)
.with_line_number(true)
.with_thread_ids(true)
.with_thread_names(true)
.with_current_span(true)
.with_span_list(true)
.with_filter(request_dump_file_filter)
});
tracing_subscriber::registry()
.with(stdout_layer)
.with(file_layer)
.with(recovery_file_layer)
.with(request_dump_file_layer)
.try_init()
.map_err(|_| LogError::AlreadyInitialized)?;
Ok(LogGuard { _writers: guards })
@@ -131,8 +142,12 @@ fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
.parse()
.expect("static OAuth recovery log directive is valid");
let request_dump_directive = format!("{REQUEST_DUMP_LOG_TARGET}=off")
.parse()
.expect("static request dump log directive is valid");
EnvFilter::try_new(directive)
.map(|filter| filter.add_directive(recovery_directive))
.map(|filter| filter.add_directive(request_dump_directive))
.map_err(|_| LogError::InvalidFilter)
}
@@ -142,14 +157,41 @@ fn oauth_recovery_file_filter() -> EnvFilter {
.expect("static OAuth recovery log filter is valid")
}
fn request_dump_file_filter() -> EnvFilter {
format!("off,{REQUEST_DUMP_LOG_TARGET}=info")
.parse()
.expect("static request dump log filter is valid")
}
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
if !persistent_log_enabled {
directive.to_owned()
format!("{directive},{REQUEST_DUMP_LOG_TARGET}=off")
} else {
"off,gateway_startup=info".to_owned()
}
}
fn rotating_file_writer(
config: &FileLoggingConfig,
file_prefix: &'static str,
thread_name: &'static str,
) -> Result<(NonBlocking, WorkerGuard), LogError> {
let maximum_bytes = config
.max_file_size_mb
.checked_mul(1024 * 1024)
.ok_or(LogError::SizeOverflow)?;
let writer = RotatingLogWriter::open(
config.directory.clone(),
file_prefix,
maximum_bytes,
config.retention_days,
config.max_files,
)?;
Ok(NonBlockingBuilder::default()
.thread_name(thread_name)
.finish(writer))
}
struct RotatingLogWriter {
directory: PathBuf,
file_prefix: &'static str,
@@ -103,6 +103,7 @@ fn valid_config() -> HostConfig {
max_file_size_mb: 100,
max_files: 30,
},
request_dump: false,
},
system_update,
drain_timeout_seconds: 30,
+76 -7
View File
@@ -8,13 +8,18 @@ use gateway_host::system_update::SystemUpdateConfig;
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
const REQUEST_DUMP_ENABLED_ENV: &str = "CPR_LOGGING_TEST_REQUEST_DUMP_ENABLED";
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump.";
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
const REQUEST_DUMP_LOG_TARGET: &str = "request_dump";
const REQUEST_DUMP_LOG_MARKER: &str = "request-dump-file-filter-test";
const REQUEST_DUMP_SECRET: &str = "unredacted-authorization-value";
#[test]
fn logging_requires_at_least_one_sink() {
@@ -34,6 +39,7 @@ fn logging_requires_at_least_one_sink() {
max_file_size_mb: 100,
max_files: 30,
},
request_dump: false,
},
system_update: SystemUpdateConfig::default(),
drain_timeout_seconds: 30,
@@ -48,11 +54,12 @@ fn logging_requires_at_least_one_sink() {
}
#[test]
fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
fn sensitive_file_logging_is_separate_and_overrides_global_log_level() {
if env::var_os(CHILD_PROCESS_ENV).is_some() {
write_oauth_recovery_log(PathBuf::from(
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
));
write_sensitive_logs(
PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).expect("child log directory")),
env::var(REQUEST_DUMP_ENABLED_ENV).is_ok_and(|value| value == "true"),
);
return;
}
@@ -60,14 +67,15 @@ fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
let output = Command::new(env::current_exe().expect("current test executable"))
.args([
"--exact",
"logging::oauth_recovery_file_logging_is_separate_and_overrides_global_log_level",
"logging::sensitive_file_logging_is_separate_and_overrides_global_log_level",
"--nocapture",
])
.env(CHILD_PROCESS_ENV, "1")
.env(LOG_DIRECTORY_ENV, directory.path())
.env(REQUEST_DUMP_ENABLED_ENV, "true")
.env(
"RUST_LOG",
"off,logging_test_application=info,oauth_recovery=off",
"off,logging_test_application=info,oauth_recovery=off,request_dump=trace",
)
.output()
.expect("run logging child process");
@@ -84,6 +92,8 @@ fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
assert!(application_log.contains(APPLICATION_LOG_MARKER));
assert!(!application_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
assert!(!application_log.contains(OAUTH_RECOVERY_LOG_MARKER));
assert!(!application_log.contains(&json_target_field(REQUEST_DUMP_LOG_TARGET)));
assert!(!application_log.contains(REQUEST_DUMP_SECRET));
let recovery_log = read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX);
assert!(recovery_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
@@ -91,9 +101,49 @@ fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
assert!(!recovery_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
assert!(!recovery_log.contains(APPLICATION_LOG_MARKER));
assert!(recovery_log.contains(&format!(r#""provider":"{OAUTH_RECOVERY_PROVIDER}""#)));
assert!(!recovery_log.contains(&json_target_field(REQUEST_DUMP_LOG_TARGET)));
assert!(!recovery_log.contains(REQUEST_DUMP_SECRET));
let request_dump_log = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
assert!(request_dump_log.contains(&json_target_field(REQUEST_DUMP_LOG_TARGET)));
assert!(request_dump_log.contains(REQUEST_DUMP_LOG_MARKER));
assert!(request_dump_log.contains(REQUEST_DUMP_SECRET));
assert!(!request_dump_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
assert!(!request_dump_log.contains(APPLICATION_LOG_MARKER));
assert!(!request_dump_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
assert!(!request_dump_log.contains(OAUTH_RECOVERY_LOG_MARKER));
let disabled_directory = tempfile::tempdir().expect("create disabled log directory");
let output = Command::new(env::current_exe().expect("current test executable"))
.args([
"--exact",
"logging::sensitive_file_logging_is_separate_and_overrides_global_log_level",
"--nocapture",
])
.env(CHILD_PROCESS_ENV, "1")
.env(LOG_DIRECTORY_ENV, disabled_directory.path())
.env(REQUEST_DUMP_ENABLED_ENV, "false")
.env(
"RUST_LOG",
"off,logging_test_application=info,oauth_recovery=off,request_dump=trace",
)
.output()
.expect("run logging child process with request dump disabled");
assert!(
output.status.success(),
"logging child process failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
assert!(!log_file_set_exists(
disabled_directory.path(),
REQUEST_DUMP_LOG_FILE_PREFIX
));
let application_log = read_log_file_set(disabled_directory.path(), APPLICATION_LOG_FILE_PREFIX);
assert!(!application_log.contains(REQUEST_DUMP_SECRET));
}
fn write_oauth_recovery_log(directory: PathBuf) {
fn write_sensitive_logs(directory: PathBuf, request_dump: bool) {
let config = HostConfig {
listen: ListenConfig {
host: "127.0.0.1".to_owned(),
@@ -110,6 +160,7 @@ fn write_oauth_recovery_log(directory: PathBuf) {
max_file_size_mb: 1,
max_files: 1,
},
request_dump,
},
system_update: SystemUpdateConfig::default(),
drain_timeout_seconds: 30,
@@ -134,6 +185,12 @@ fn write_oauth_recovery_log(directory: PathBuf) {
marker = OAUTH_RECOVERY_LOG_MARKER,
"OAuth recovery test record"
);
tracing::info!(
target: REQUEST_DUMP_LOG_TARGET,
marker = REQUEST_DUMP_LOG_MARKER,
authorization = REQUEST_DUMP_SECRET,
"request dump test record"
);
drop(bundle);
}
@@ -156,6 +213,18 @@ fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
.collect()
}
fn log_file_set_exists(directory: &Path, file_prefix: &str) -> bool {
fs::read_dir(directory)
.expect("read log directory")
.filter_map(Result::ok)
.any(|entry| {
entry
.file_name()
.to_str()
.is_some_and(|name| name.starts_with(file_prefix))
})
}
fn json_target_field(target: &str) -> String {
format!(r#""target":"{target}""#)
}
@@ -728,7 +728,8 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_connection_idle_ms = $47
where id = $1 and outcome = 'running'
returning id, client_api_key_ref, continuation_affinity_hash,
continuation_requested, outcome, started_at, completed_at
continuation_requested, provider_kind, upstream_transport,
outcome, started_at, completed_at
), recovery_target as (
select prior.id
from model_requests prior
@@ -775,9 +776,47 @@ impl ModelRequestRepository for PgExecutionStore {
where prior.id = target.id
and prior.recovered_at is null
returning prior.id
), session_transport_recovery_target as (
select prior.id
from model_requests prior
cross join finalized current
where current.outcome = 'succeeded'
and current.provider_kind = 'openai'
and current.upstream_transport = 'http_sse'
and current.continuation_affinity_hash is not null
and prior.client_api_key_ref = current.client_api_key_ref
and prior.continuation_affinity_hash = current.continuation_affinity_hash
and prior.provider_kind = current.provider_kind
and prior.outcome = 'failed'
and prior.error_kind = 'upstream_unavailable'
and prior.upstream_transport = 'websocket'
and prior.upstream_send_state = 'ambiguous'
and prior.downstream_committed_at is null
and prior.recovered_at is null
and current.started_at >= prior.completed_at
and current.started_at <= prior.completed_at + interval '30 seconds'
), session_transport_recovery_update as (
update model_requests prior
set recovery_attempt_count = prior.recovery_attempt_count + 1,
recovery_request_id = current.id,
recovered_at = current.completed_at,
recovery_retry_delay_ms = greatest(
0,
floor(extract(epoch from (current.started_at - prior.completed_at)) * 1000)
)::bigint,
recovery_total_latency_ms = greatest(
0,
floor(extract(epoch from (current.completed_at - prior.completed_at)) * 1000)
)::bigint
from finalized current
join session_transport_recovery_target target on true
where prior.id = target.id
and prior.recovered_at is null
returning prior.id
)
select (select count(*) from finalized)::bigint
+ (select count(*) * 0 from recovery_update)::bigint",
+ (select count(*) * 0 from recovery_update)::bigint
+ (select count(*) * 0 from session_transport_recovery_update)::bigint",
)
.bind(&finalization.model_request_id)
.bind(finalization.outcome.as_str())
@@ -128,7 +128,6 @@ fn push_request_error_predicates(
range: ObservabilityRange,
filter: &OpsErrorFilter,
) {
statement.push(" and mr.recovered_at is null");
push_range(statement, "mr.completed_at", range);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
@@ -171,7 +170,6 @@ fn push_ops_event_predicates(
range: ObservabilityRange,
filter: &OpsErrorFilter,
) {
statement.push(" and (mr.id is null or mr.recovered_at is null)");
push_range(statement, "oe.created_at", range);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
@@ -88,7 +88,19 @@ pub(crate) fn push_usage_filter(
query.push(format!(" escape '\\' or {alias}.{column} like "));
query.push_bind(pattern.clone());
}
query.push(" escape '\\')");
if value.starts_with("sk_") {
query.push(format!(
" escape '\\' or exists (
select 1 from client_api_keys searched_client_key
where searched_client_key.id = {alias}.client_api_key_ref
and searched_client_key.key like "
));
query.push_bind(pattern);
query.push(" escape '\\')");
} else {
query.push(" escape '\\'");
}
query.push(")");
}
}
@@ -719,6 +719,165 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered(
database.close().await;
}
#[tokio::test]
async fn successful_http_downgrade_should_mark_pending_websocket_failures_recovered() {
let Some(database) = TestDatabase::create("execution_transport_downgrade_recovered").await
else {
return;
};
let affinity_hash = "d".repeat(64);
let first_failure_at = Utc::now() - Duration::seconds(8);
let second_failure_at = first_failure_at + Duration::seconds(1);
seed_websocket_transport_failure(
&database.pool,
"req_websocket_failed_first",
&affinity_hash,
first_failure_at,
)
.await;
seed_websocket_transport_failure(
&database.pool,
"req_websocket_failed_second",
&affinity_hash,
second_failure_at,
)
.await;
seed_websocket_transport_failure(
&database.pool,
"req_websocket_failed_after_delivery",
&affinity_hash,
second_failure_at,
)
.await;
sqlx::query(
"update model_requests
set downstream_committed_at = completed_at
where id = 'req_websocket_failed_after_delivery'",
)
.execute(&database.pool)
.await
.expect("mark control failure delivered downstream");
let recovery_started_at = first_failure_at + Duration::seconds(4);
seed_transport_recovery_request(
&database.pool,
"req_http_fallback_succeeded",
&affinity_hash,
recovery_started_at,
)
.await;
let store = PgExecutionStore::new(database.pool.clone());
let mut recovery = successful_core_finalization("req_http_fallback_succeeded");
recovery.upstream_transport = Some("http_sse".to_owned());
recovery.downstream_committed_at = Some(std::time::SystemTime::now());
ExecutionStore::finalize_model_request(&store, recovery)
.await
.expect("finalize successful HTTP fallback");
type PersistedRecovery = (
String,
Option<String>,
Option<chrono::DateTime<Utc>>,
i32,
Option<i64>,
Option<i64>,
);
let recovered: Vec<PersistedRecovery> = sqlx::query_as(
"select id, recovery_request_id, recovered_at, recovery_attempt_count,
recovery_retry_delay_ms, recovery_total_latency_ms
from model_requests
where id like 'req_websocket_failed_%'
order by id",
)
.fetch_all(&database.pool)
.await
.expect("load WebSocket recovery observations");
assert_eq!(recovered.len(), 3);
let delivered = &recovered[0];
assert_eq!(delivered.0, "req_websocket_failed_after_delivery");
assert_eq!((&delivered.1, delivered.2, delivered.3), (&None, None, 0));
for (row, expected_delay_ms) in recovered[1..].iter().zip([4_000, 3_000]) {
assert_eq!(row.1.as_deref(), Some("req_http_fallback_succeeded"));
assert!(row.2.is_some());
assert_eq!(row.3, 1);
assert_eq!(row.4, Some(expected_delay_ms));
assert!(row.5.is_some_and(|latency| latency >= expected_delay_ms));
}
database.close().await;
}
#[tokio::test]
async fn websocket_success_or_late_http_success_should_not_hide_transport_failure() {
let Some(database) = TestDatabase::create("execution_transport_downgrade_unresolved").await
else {
return;
};
let failure_at = Utc::now() - Duration::seconds(60);
let store = PgExecutionStore::new(database.pool.clone());
let websocket_affinity = "e".repeat(64);
seed_websocket_transport_failure(
&database.pool,
"req_websocket_control_failure",
&websocket_affinity,
failure_at,
)
.await;
seed_transport_recovery_request(
&database.pool,
"req_websocket_control_success",
&websocket_affinity,
failure_at + Duration::seconds(4),
)
.await;
ExecutionStore::finalize_model_request(
&store,
successful_core_finalization("req_websocket_control_success"),
)
.await
.expect("finalize successful WebSocket control request");
let late_affinity = "f".repeat(64);
seed_websocket_transport_failure(
&database.pool,
"req_websocket_late_failure",
&late_affinity,
failure_at,
)
.await;
seed_transport_recovery_request(
&database.pool,
"req_http_late_success",
&late_affinity,
failure_at + Duration::seconds(31),
)
.await;
let mut late_recovery = successful_core_finalization("req_http_late_success");
late_recovery.upstream_transport = Some("http_sse".to_owned());
ExecutionStore::finalize_model_request(&store, late_recovery)
.await
.expect("finalize late HTTP request");
type UnresolvedTransportFailure = (String, Option<String>, Option<chrono::DateTime<Utc>>, i32);
let unresolved: Vec<UnresolvedTransportFailure> = sqlx::query_as(
"select id, recovery_request_id, recovered_at, recovery_attempt_count
from model_requests
where id in ('req_websocket_control_failure', 'req_websocket_late_failure')
order by id",
)
.fetch_all(&database.pool)
.await
.expect("load unresolved transport failures");
assert_eq!(unresolved.len(), 2);
assert!(
unresolved
.iter()
.all(|row| row.1.is_none() && row.2.is_none() && row.3 == 0)
);
database.close().await;
}
#[tokio::test]
async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() {
let Some(database) = TestDatabase::create("execution_continuation_unresolved").await else {
@@ -842,6 +1001,61 @@ async fn seed_recovery_request(
.expect("prepare seeded recovery request");
}
async fn seed_websocket_transport_failure(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
completed_at: chrono::DateTime<Utc>,
) {
seed_running_request(pool, id)
.await
.expect("seed WebSocket transport failure request");
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_transport_recovery',
client_transport = 'http_sse',
continuation_affinity_hash = $2,
attempt_count = 1, upstream_send_state = 'ambiguous',
upstream_transport = 'websocket', outcome = 'failed',
error_kind = 'upstream_unavailable',
provider_error_code = 'websocket_close_1000',
started_at = $3 - interval '1 second', deadline_at = $3,
completed_at = $3
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind(completed_at)
.execute(pool)
.await
.expect("finalize seeded WebSocket transport failure");
}
async fn seed_transport_recovery_request(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
started_at: chrono::DateTime<Utc>,
) {
seed_running_request(pool, id)
.await
.expect("seed transport recovery request");
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_transport_recovery',
client_transport = 'http_sse',
continuation_affinity_hash = $2,
started_at = $3, deadline_at = $3 + interval '1 minute'
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind(started_at)
.execute(pool)
.await
.expect("prepare seeded transport recovery request");
}
async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx::Error> {
sqlx::query(
"insert into model_requests (
@@ -186,6 +186,48 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
database.close().await;
}
#[tokio::test]
async fn usage_search_should_match_client_api_key_prefix() {
let Some(database) = TestDatabase::create("usage_client_api_key_search").await else {
return;
};
let now = Utc::now();
seed_observability_facts(&database.pool, now)
.await
.expect("seed observability facts");
let plaintext_key = format!("sk_{}", "K".repeat(43));
sqlx::query(
"insert into client_api_keys (id, name, key, enabled, created_at, updated_at)
values ('key_observe', 'usage search', $1, true, $2, $2)",
)
.bind(&plaintext_key)
.bind(now)
.execute(&database.pool)
.await
.expect("seed searchable client API key");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
for search in [plaintext_key.as_str(), &plaintext_key[..10]] {
let page = observability_repository(&database.pool)
.list_usage_records(UsageRecordQuery {
range,
filter: UsageRecordFilter {
search: Some(search.to_owned()),
..UsageRecordFilter::default()
},
current_page: 1,
page_size: ObservabilityPageSize::new(10).expect("page size"),
})
.await
.expect("search usage by client API key");
assert_eq!(page.total, 1);
assert_eq!(page.items[0].id, "req_observe_success");
}
database.close().await;
}
#[tokio::test]
async fn ops_search_should_treat_sql_wildcards_as_literals() {
let Some(database) = TestDatabase::create("ops_literal_wildcard_search").await else {
@@ -216,7 +258,8 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
}
#[tokio::test]
async fn recovered_continuation_failure_should_be_hidden_from_default_business_metrics() {
async fn recovered_continuation_failure_should_be_visible_in_ops_but_hidden_from_business_metrics()
{
let Some(database) = TestDatabase::create("observability_recovered_continuation").await else {
return;
};
@@ -276,8 +319,13 @@ async fn recovered_continuation_failure_should_be_hidden_from_default_business_m
page_size: ObservabilityPageSize::new(10).expect("page size"),
})
.await
.expect("ops errors without recovered intermediates");
assert_eq!(errors.total, 0);
.expect("ops errors retain recovered failures");
assert_eq!(errors.total, 2);
assert!(errors.items.iter().all(|error| {
error.recovery_request_id.as_deref() == Some("req_observe_success")
&& error.recovered_at.is_some()
&& error.recovery_attempt_count == 1
}));
let retained: (String, Option<String>) = sqlx::query_as(
"select outcome, recovery_request_id
@@ -16,8 +16,8 @@ pub mod token_client;
mod types;
pub(crate) use affinity::{
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
derive_codex_session_affinity, derive_previous_response_id_hash,
CodexSessionAffinity, derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
derive_previous_response_id_hash,
};
pub(crate) use oauth::oauth_owner_ref;
pub(crate) use types::parse_access_token_expiration;
@@ -147,6 +147,7 @@ pub(super) struct ColdResponse {
pub(super) request: CodexResponsesRequest,
pub(super) upstream_model: UpstreamModelId,
pub(super) transport_policy: CodexProviderTransport,
pub(super) websocket_connection_preference: WebSocketConnectionPreference,
pub(super) context: AttemptContext,
pub(super) selector: Arc<CodexCredentialSelector>,
pub(super) quota: Arc<CodexCredentialQuotaService>,
@@ -155,7 +156,8 @@ pub(super) struct ColdResponse {
pub(super) output_started_at: Instant,
pub(super) session_affinity_key: Option<ProviderSessionAffinityKey>,
pub(super) session_affinity_key_hash: Option<String>,
pub(super) session_transport_fallbacks: CodexSessionTransportFallbacks,
pub(super) session_transport_recovery: CodexSessionTransportRecovery,
pub(super) session_recovery_probe: Option<CodexSessionWebSocketProbe>,
pub(super) websocket_retry_count: u32,
pub(super) stream_max_retries: u32,
pub(super) session_capture: Option<OpenAiSessionCapture>,
@@ -184,6 +186,13 @@ pub(super) struct OpenAiSessionState {
#[serde(default)]
pub(super) client_turn_id: Option<String>,
pub(super) continuation_scope: OpenAiContinuationScope,
/// 该状态来自同一业务请求的 metadata 检查点,只允许被紧随其后的恢复 attempt 消费。
#[serde(default, skip_serializing_if = "is_false")]
pub(super) retry_checkpoint: bool,
}
const fn is_false(value: &bool) -> bool {
!*value
}
#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -230,6 +239,26 @@ pub(super) fn encode_openai_session_state(
.map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::Sent))
}
fn encode_openai_session_capture(
capture: &OpenAiSessionCapture,
retry_checkpoint: bool,
) -> Result<ProviderSessionState, ProviderError> {
let Some(continuation_scope) = capture.continuation_scope else {
return Err(provider_error(
ProviderErrorKind::Protocol,
UpstreamSendState::Sent,
));
};
encode_openai_session_state(OpenAiSessionState {
account_id: capture.account_id.clone(),
conversation_id: capture.conversation_id.clone(),
turn_state: capture.turn_state.clone(),
client_turn_id: capture.client_turn_id.clone(),
continuation_scope,
retry_checkpoint,
})
}
pub(super) fn attach_openai_session_update(
events: &mut [ProviderEvent],
capture: &mut Option<OpenAiSessionCapture>,
@@ -243,16 +272,7 @@ pub(super) fn attach_openai_session_update(
let Some(capture) = capture.take() else {
return;
};
let Some(continuation_scope) = capture.continuation_scope else {
return;
};
let Ok(update) = encode_openai_session_state(OpenAiSessionState {
account_id: capture.account_id,
conversation_id: capture.conversation_id,
turn_state: capture.turn_state,
client_turn_id: capture.client_turn_id,
continuation_scope,
}) else {
let Ok(update) = encode_openai_session_capture(&capture, false) else {
return;
};
events[terminal_index].attach_session_update(update);
@@ -292,6 +312,7 @@ pub(super) async fn create_response_attempt(
account_id: &str,
deadline: SystemTime,
cancellation: &CancellationToken,
connection_preference: WebSocketConnectionPreference,
) -> Result<CodexBackendStreamingResponse, CodexHandshakeAttemptError> {
let Some(handshake_deadline) = remaining(deadline) else {
return Err(CodexHandshakeAttemptError::Timeout);
@@ -304,6 +325,7 @@ pub(super) async fn create_response_attempt(
request,
request_context,
Some(account_id),
connection_preference,
) => response.map_err(CodexHandshakeAttemptError::Client),
}
}
@@ -472,6 +494,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
request,
upstream_model,
transport_policy,
websocket_connection_preference,
context,
selector,
quota,
@@ -480,7 +503,8 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
output_started_at,
session_affinity_key,
session_affinity_key_hash,
session_transport_fallbacks,
session_transport_recovery,
mut session_recovery_probe,
websocket_retry_count,
stream_max_retries,
mut session_capture,
@@ -530,6 +554,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
active_account.id().as_str(),
context.deadline(),
&cancellation,
websocket_connection_preference,
)
.await;
let websocket_failure_policy = match &response {
@@ -556,6 +581,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
WebSocketRecoveryContext {
policy,
requirement: request_transport_requirement,
replay_checkpoint_available: false,
retry_count: websocket_retry_count,
max_retries: stream_max_retries,
request_id: context.request_id().as_str(),
@@ -563,7 +589,8 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
account_id: active_account.id().as_str(),
session_affinity_key: session_affinity_key.as_ref(),
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
session_transport_fallbacks: &session_transport_fallbacks,
session_transport_recovery: &session_transport_recovery,
session_recovery_probe: &mut session_recovery_probe,
},
);
}
@@ -586,6 +613,9 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
Err(failure.error)?;
return;
}
let mut turn_state_checkpoint_observed = response.transport
== CodexBackendTransport::WebSocket
&& response.turn_state.is_some();
if let Some(capture) = session_capture.as_mut() {
capture.continuation_scope = Some(if capture.response_store {
OpenAiContinuationScope::Persisted
@@ -699,15 +729,62 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
continue;
}
Err(mut failure) => {
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
let rate_limits_changed = if updates.is_empty() {
false
} else {
passive_quota_observation.observe(&updates);
let update_headers = rate_limit_update_headers(&updates);
observation_state.merge_rate_limit_headers(&update_headers)
};
let turn_state_merge = merge_turn_state_update(
turn_state_updates.as_ref(),
&mut session_capture,
&mut observation_state,
)
.await;
turn_state_checkpoint_observed |= turn_state_merge.is_some();
let retry_checkpoint_update = if turn_state_checkpoint_observed
&& failure.websocket_checkpoint_retryable
{
session_capture
.as_ref()
.and_then(|capture| encode_openai_session_capture(capture, true).ok())
} else {
None
};
let mut observation_event = if rate_limits_changed
|| turn_state_merge.is_some()
|| retry_checkpoint_update.is_some()
{
observation_state
.observation()
.map(ProviderEvent::observation)
} else {
None
};
if let (Some(event), Some(update)) =
(observation_event.as_mut(), retry_checkpoint_update)
{
event.attach_session_update(update);
}
let replay_checkpoint_available = observation_event
.as_ref()
.and_then(ProviderEvent::session_update)
.is_some();
if failure.websocket_transport_retryable
&& response_transport == CodexBackendTransport::WebSocket
&& !pre_commit_events.is_committed()
&& (matches!(
failure.error.send_state(),
UpstreamSendState::Sent | UpstreamSendState::Ambiguous
) || !pre_commit_events.is_committed())
{
apply_websocket_recovery_policy(
&mut failure,
WebSocketRecoveryContext {
policy: WebSocketFailurePolicy::Budgeted,
requirement: request_transport_requirement,
replay_checkpoint_available,
retry_count: websocket_retry_count,
max_retries: stream_max_retries,
request_id: context.request_id().as_str(),
@@ -715,19 +792,13 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
account_id: active_account.id().as_str(),
session_affinity_key: session_affinity_key.as_ref(),
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
session_transport_fallbacks: &session_transport_fallbacks,
session_transport_recovery: &session_transport_recovery,
session_recovery_probe: &mut session_recovery_probe,
},
);
}
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
if !updates.is_empty() {
passive_quota_observation.observe(&updates);
let update_headers = rate_limit_update_headers(&updates);
if observation_state.merge_rate_limit_headers(&update_headers)
&& let Some(observation) = observation_state.observation()
{
yield ProviderEvent::observation(observation);
}
if let Some(event) = observation_event {
yield event;
}
if allows_account_state_mutation {
synchronize_passive_quota(
@@ -751,16 +822,14 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
passive_quota_observation.observe(&updates);
observation_state.merge_rate_limit_headers(&rate_limit_update_headers(&updates))
};
let turn_state_changed = if let Some(updates) = turn_state_updates.as_ref()
&& let Some(turn_state) = updates.lock().await.take()
{
if let Some(capture) = session_capture.as_mut() {
capture.turn_state = Some(turn_state.clone());
}
observation_state.merge_client_header("x-codex-turn-state", &turn_state)
} else {
false
};
let turn_state_merge = merge_turn_state_update(
turn_state_updates.as_ref(),
&mut session_capture,
&mut observation_state,
)
.await;
turn_state_checkpoint_observed |= turn_state_merge.is_some();
let turn_state_changed = turn_state_merge.unwrap_or(false);
let first_event_changed =
observation_state.observe_stream_chunk(&chunk, output_started_at);
let chunk_len = chunk.len();
@@ -808,6 +877,21 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
.any(|event| matches!(event, GatewayEvent::Completed(_)));
let terminal_changed = completed
&& observation_state.mark_completed(terminal_response_is_incomplete(&events));
if response_transport == CodexBackendTransport::WebSocket
&& completed
&& terminal_failure.is_none()
&& let Some(probe) = session_recovery_probe.take()
{
let session_recovery_cleared = probe.succeed();
tracing::info!(
request_id = %context.request_id(),
attempt_index = context.attempt_index().get(),
session_affinity_key_hash = session_affinity_key_hash.as_deref().unwrap_or(""),
session_transport_action = "websocket_recovered",
session_recovery_cleared,
"OpenAI session fresh WebSocket probe reached a terminal response"
);
}
if allows_account_state_mutation && (completed || terminal_failure.is_some()) {
synchronize_passive_quota(
&quota,
@@ -922,16 +1006,13 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
apply_failure(&failure_context, &active_account, failure)
.await;
}
let turn_state_changed = if let Some(updates) = turn_state_updates.as_ref()
&& let Some(turn_state) = updates.lock().await.take()
{
if let Some(capture) = session_capture.as_mut() {
capture.turn_state = Some(turn_state.clone());
}
observation_state.merge_client_header("x-codex-turn-state", &turn_state)
} else {
false
};
let turn_state_changed = merge_turn_state_update(
turn_state_updates.as_ref(),
&mut session_capture,
&mut observation_state,
)
.await
.unwrap_or(false);
attach_openai_session_update(&mut events, &mut session_capture);
let completed = events
.iter()
@@ -939,6 +1020,21 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
.any(|event| matches!(event, GatewayEvent::Completed(_)));
let terminal_changed = completed
&& observation_state.mark_completed(terminal_response_is_incomplete(&events));
if response_transport == CodexBackendTransport::WebSocket
&& completed
&& terminal_failure.is_none()
&& let Some(probe) = session_recovery_probe.take()
{
let session_recovery_cleared = probe.succeed();
tracing::info!(
request_id = %context.request_id(),
attempt_index = context.attempt_index().get(),
session_affinity_key_hash = session_affinity_key_hash.as_deref().unwrap_or(""),
session_transport_action = "websocket_recovered",
session_recovery_cleared,
"OpenAI session fresh WebSocket probe reached a terminal response"
);
}
if allows_account_state_mutation && completed && terminal_failure.is_none() {
// 同上:尾部 finish() 也可能产出 completed,亲和记录必须先于任何下游 yield。
selector
@@ -982,3 +1078,16 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
}
})
}
async fn merge_turn_state_update(
updates: Option<&CodexTurnStateUpdate>,
session_capture: &mut Option<OpenAiSessionCapture>,
observation_state: &mut OpenAiResponseObservationState,
) -> Option<bool> {
let updates = updates?;
let turn_state = updates.lock().await.take()?;
if let Some(capture) = session_capture.as_mut() {
capture.turn_state = Some(turn_state.clone());
}
Some(observation_state.merge_client_header("x-codex-turn-state", &turn_state))
}
@@ -39,6 +39,8 @@ pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
pub(super) struct MappedProviderFailure {
pub(super) error: ProviderError,
pub(super) websocket_transport_retryable: bool,
/// 兼容少数链路在 metadata 后、终态前关闭;仅在捕获 turn-state 时允许安全恢复。
pub(super) websocket_checkpoint_retryable: bool,
pub(super) account_failure: Option<CodexAccountFailure>,
/// 原始上游错误描述,仅在凭据错误状态下持久化。
pub(super) error_message: Option<String>,
@@ -54,6 +56,7 @@ impl MappedProviderFailure {
Self {
error,
websocket_transport_retryable: false,
websocket_checkpoint_retryable: false,
account_failure: None,
error_message: None,
cyber_policy_failure: false,
@@ -451,6 +454,8 @@ pub(super) enum WebSocketFailurePolicy {
pub(super) struct WebSocketRecoveryContext<'a> {
pub(super) policy: WebSocketFailurePolicy,
pub(super) requirement: TransportRequirement,
/// 本 attempt 已从 metadata 捕获可恢复同一请求的 turn-state 检查点。
pub(super) replay_checkpoint_available: bool,
pub(super) retry_count: u32,
pub(super) max_retries: u32,
pub(super) request_id: &'a str,
@@ -458,7 +463,8 @@ pub(super) struct WebSocketRecoveryContext<'a> {
pub(super) account_id: &'a str,
pub(super) session_affinity_key: Option<&'a ProviderSessionAffinityKey>,
pub(super) session_affinity_key_hash: Option<&'a str>,
pub(super) session_transport_fallbacks: &'a CodexSessionTransportFallbacks,
pub(super) session_transport_recovery: &'a CodexSessionTransportRecovery,
pub(super) session_recovery_probe: &'a mut Option<CodexSessionWebSocketProbe>,
}
pub(super) fn websocket_client_failure_policy(
@@ -504,15 +510,24 @@ pub(super) fn apply_websocket_recovery_policy(
context: WebSocketRecoveryContext<'_>,
) {
let send_state = failure.error.send_state();
if matches!(
let post_send = matches!(
send_state,
UpstreamSendState::Sent | UpstreamSendState::Ambiguous
) {
);
if post_send && !context.replay_checkpoint_available {
// payload 已经发送或发送结果不确定时,代理不能用同一业务输入自动重放。
// 只为客户端下一次完整请求记住 HTTP 偏好,当前请求保持终态失败。
let sticky_state_transition = context
.session_affinity_key
.map(|key| context.session_transport_fallbacks.disable_websocket(key));
// 当前请求保持终态失败;恢复状态只裁决客户端主动发起的下一次请求。
let session_recovery_transition = match context.session_recovery_probe.take() {
Some(probe) => Some(probe.post_send_failed()),
None => context.session_affinity_key.map(|key| {
context
.session_transport_recovery
.record_post_send_failure(key)
}),
};
let session_http_cooldown_ms = session_recovery_transition
.and_then(CodexSessionRecoveryTransition::cooldown)
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX));
tracing::warn!(
request_id = context.request_id,
attempt_index = context.attempt_index,
@@ -527,8 +542,12 @@ pub(super) fn apply_websocket_recovery_policy(
continuation_recovery_action = "client_replay_required",
session_affinity_present = context.session_affinity_key.is_some(),
session_affinity_key_hash = context.session_affinity_key_hash.unwrap_or(""),
sticky_http_enabled = sticky_state_transition.is_some(),
sticky_state_transition = sticky_state_transition.unwrap_or(false),
session_transport_action = session_recovery_transition
.map_or("untracked", CodexSessionRecoveryTransition::action),
session_websocket_failure_count = session_recovery_transition
.map_or(0, CodexSessionRecoveryTransition::failure_count),
session_http_cooldown_ms = session_http_cooldown_ms.unwrap_or_default(),
session_http_cooldown_present = session_http_cooldown_ms.is_some(),
"OpenAI upstream WebSocket failed after payload send; proxy replay was suppressed"
);
return;
@@ -542,6 +561,11 @@ pub(super) fn apply_websocket_recovery_policy(
// 传输恢复必须保持原账号可调度;最终 HTTP attempt 若仍失败,再按真实 HTTP
// 结果更新账号健康度,避免中间 WS 错误把同账号钉选提前冷却掉。
failure.account_failure = None;
if post_send {
// Core 对 Ambiguous 默认仍然 fail-closed;只有同一 attempt 的 session
// checkpoint 与这个 Provider proof 同时存在时,才允许同账号传输恢复。
failure.error.set_replay_safe();
}
let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback
|| context.retry_count >= context.max_retries;
if !fallback_now {
@@ -570,15 +594,28 @@ pub(super) fn apply_websocket_recovery_policy(
websocket_max_retries = context.max_retries,
websocket_retry_delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
transport_requirement = context.requirement.as_str(),
replay_checkpoint_available = context.replay_checkpoint_available,
"OpenAI upstream WebSocket failed; retrying the same account"
);
return;
}
failure.error.set_pre_delivery_transport_fallback();
let sticky_state_transition = context
.session_affinity_key
.map(|key| context.session_transport_fallbacks.disable_websocket(key));
let session_fallback = match context.policy {
WebSocketFailurePolicy::Budgeted => SessionWebSocketFallback::RetryBudgetExhausted,
WebSocketFailurePolicy::ImmediateFallback => SessionWebSocketFallback::UpgradeRequired,
};
let session_recovery_transition = match context.session_recovery_probe.take() {
Some(probe) => Some(probe.fallback(session_fallback)),
None => context.session_affinity_key.map(|key| {
context
.session_transport_recovery
.record_websocket_fallback(key, session_fallback)
}),
};
let session_http_cooldown_ms = session_recovery_transition
.and_then(CodexSessionRecoveryTransition::cooldown)
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX));
tracing::warn!(
request_id = context.request_id,
attempt_index = context.attempt_index,
@@ -596,11 +633,16 @@ pub(super) fn apply_websocket_recovery_policy(
WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted",
WebSocketFailurePolicy::ImmediateFallback => "upgrade_required",
},
replay_checkpoint_available = context.replay_checkpoint_available,
session_affinity_present = context.session_affinity_key.is_some(),
session_affinity_key_hash = context.session_affinity_key_hash.unwrap_or(""),
sticky_http_enabled = sticky_state_transition.is_some(),
sticky_state_transition = sticky_state_transition.unwrap_or(false),
"OpenAI upstream WebSocket disabled for the current session"
session_transport_action =
session_recovery_transition.map_or("untracked", CodexSessionRecoveryTransition::action),
session_websocket_failure_count =
session_recovery_transition.map_or(0, CodexSessionRecoveryTransition::failure_count),
session_http_cooldown_ms = session_http_cooldown_ms.unwrap_or_default(),
session_http_cooldown_present = session_http_cooldown_ms.is_some(),
"OpenAI upstream WebSocket entered session HTTP cooldown"
);
}
@@ -644,14 +686,31 @@ pub(super) fn continuation_replay_required_error(reason: &'static str) -> Provid
pub(super) fn map_stream_error(error: CodexClientError) -> MappedProviderFailure {
let allows_pre_delivery_retry = stream_transport_allows_pre_delivery_retry(&error);
let websocket_failure = error.transport() == Some(CodexBackendTransport::WebSocket);
let websocket_checkpoint_retryable = websocket_metadata_close_can_resume(&error);
let mut failure = map_client_error(error, UpstreamSendState::Sent, false);
failure.websocket_transport_retryable = allows_pre_delivery_retry && websocket_failure;
failure.websocket_checkpoint_retryable =
allows_pre_delivery_retry && websocket_checkpoint_retryable;
if allows_pre_delivery_retry && !websocket_failure {
failure.error = failure.error.with_pre_delivery_retry();
}
failure
}
fn websocket_metadata_close_can_resume(error: &CodexClientError) -> bool {
let CodexClientError::WebSocket(error) = error else {
return false;
};
let Some(close) = error.close_before_terminal() else {
return false;
};
close.code() == Some(1000)
&& matches!(
close.last_event_type(),
Some("codex.response.metadata" | "response.metadata")
)
}
pub(super) fn stream_transport_allows_pre_delivery_retry(error: &CodexClientError) -> bool {
match error {
CodexClientError::Http(_)
@@ -870,9 +929,6 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
CodexWebSocketExchangeError::ConnectTimeout { timeout } => {
format!("OpenAI WebSocket connection timed out after {timeout:?}")
}
CodexWebSocketExchangeError::FastPathTimeout { timeout } => {
format!("OpenAI WebSocket fast-path budget expired after {timeout:?}")
}
CodexWebSocketExchangeError::OriginCircuitOpen => {
"OpenAI WebSocket origin circuit is open".to_owned()
}
@@ -1042,6 +1098,7 @@ pub(super) fn map_upstream_failure(
MappedProviderFailure {
error,
websocket_transport_retryable: false,
websocket_checkpoint_retryable: false,
account_failure: account_failure(
category,
failure.retry_after_seconds,
@@ -1147,7 +1204,6 @@ pub(super) fn websocket_send_state(error: &CodexWebSocketExchangeError) -> Upstr
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::Connect(_)
| CodexWebSocketExchangeError::ConnectTimeout { .. }
| CodexWebSocketExchangeError::FastPathTimeout { .. }
| CodexWebSocketExchangeError::OriginCircuitOpen
| CodexWebSocketExchangeError::OriginHalfOpenBusy
| CodexWebSocketExchangeError::SharedConnectFailed
@@ -1175,7 +1231,6 @@ pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> Provi
| CodexWebSocketExchangeError::InvalidSse(_)
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => ProviderErrorKind::Protocol,
CodexWebSocketExchangeError::ConnectTimeout { .. }
| CodexWebSocketExchangeError::FastPathTimeout { .. }
| CodexWebSocketExchangeError::SendTimeout { .. }
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
| CodexWebSocketExchangeError::InitialEventTimeout { .. } => ProviderErrorKind::Timeout,
@@ -61,7 +61,10 @@ use crate::credential::{
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
derive_previous_response_id_hash,
};
use crate::session_transport::CodexSessionTransportFallbacks;
use crate::session_transport::{
CodexSessionRecoveryTransition, CodexSessionTransportDecision, CodexSessionTransportRecovery,
CodexSessionWebSocketProbe, SessionWebSocketFallback,
};
use crate::transport::canonical::{
CodexCanonicalDecoder, CodexCanonicalError, CodexCanonicalOutcome,
};
@@ -90,7 +93,8 @@ use crate::transport::{
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
CodexBackendJsonResponse, CodexBackendStreamingResponse, CodexBackendTransport,
CodexClientError, CodexRateLimitUpdates, CodexRequestContext, CodexResponseMetadata,
CodexTransportMetrics, CodexUpstreamDiagnostics, CodexWebSocketPool, endpoint_url,
CodexTransportMetrics, CodexTurnStateUpdate, CodexUpstreamDiagnostics, CodexWebSocketPool,
WebSocketConnectionPreference, endpoint_url,
};
mod execution;
@@ -145,7 +149,7 @@ pub struct CodexProvider {
image_edits_url: Url,
search_url: Url,
session_identity: Option<CodexSessionIdentity>,
session_transport_fallbacks: CodexSessionTransportFallbacks,
session_transport_recovery: CodexSessionTransportRecovery,
stream_max_retries: u32,
}
@@ -185,7 +189,7 @@ impl CodexProvider {
image_edits_url,
search_url,
session_identity: None,
session_transport_fallbacks: CodexSessionTransportFallbacks::default(),
session_transport_recovery: CodexSessionTransportRecovery::default(),
stream_max_retries,
})
}
@@ -316,15 +320,22 @@ impl Provider for CodexProvider {
if let Some(identity) = &self.session_identity {
identity.prepare_local_conversation(&mut upstream_request);
}
if previous_session.as_ref().is_some_and(|state| {
same_client_turn(
state.client_turn_id.as_deref(),
upstream_request.client_turn_id.as_deref(),
)
}) {
upstream_request.turn_state = previous_session
.as_ref()
.and_then(|state| state.turn_state.clone());
if let Some(previous_session) = previous_session.as_ref() {
upstream_request.turn_state = if previous_session.retry_checkpoint
|| same_client_turn(
previous_session.client_turn_id.as_deref(),
upstream_request.client_turn_id.as_deref(),
) {
// 客户端可能携带 metadata-close 失败后才拿到的新状态;同一
// turn 内显式回传值比最后一次成功响应保存的状态更新。同请求
// 内部恢复由一次性 retry checkpoint 明确证明,不依赖客户端 turn ID。
upstream_request
.turn_state
.take()
.or_else(|| previous_session.turn_state.clone())
} else {
None
};
}
let session_affinity =
derive_codex_session_affinity(&upstream_request, context.client_api_key_ref());
@@ -442,26 +453,46 @@ impl Provider for CodexProvider {
);
let requirement = transport_requirement(&upstream_request);
let requested_transport = selected_transport(&upstream_request);
let sticky_http_fallback = context.transport() == AttemptTransport::Default
&& requested_transport == CodexProviderTransport::PreferWebSocket
&& requirement.allows_sticky_http_fallback()
&& session_affinity.as_ref().is_some_and(|affinity| {
self.session_transport_fallbacks
.is_http_only(affinity.key())
});
let session_transport_decision = if matches!(
context.transport(),
AttemptTransport::Default | AttemptTransport::Retry(_)
) && requested_transport
== CodexProviderTransport::PreferWebSocket
&& requirement.allows_session_transport_recovery()
&& let Some(affinity) = session_affinity.as_ref()
{
self.session_transport_recovery.decide(affinity.key())
} else {
CodexSessionTransportDecision::Default
};
let session_transport_action = session_transport_decision.action();
let session_websocket_failure_count = session_transport_decision.failure_count();
let session_http_cooldown_ms = session_transport_decision
.retry_after()
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX));
let mut websocket_connection_preference = WebSocketConnectionPreference::ReuseOrConnect;
let mut session_recovery_probe = None;
let transport = if requirement.requires_websocket() {
CodexProviderTransport::PreferWebSocket
} else {
match context.transport() {
AttemptTransport::Default if sticky_http_fallback => {
CodexProviderTransport::HttpOnly
AttemptTransport::Default | AttemptTransport::Retry(_) => {
match session_transport_decision {
CodexSessionTransportDecision::Default => requested_transport,
CodexSessionTransportDecision::FreshWebSocket { probe, .. } => {
websocket_connection_preference = WebSocketConnectionPreference::Fresh;
session_recovery_probe = Some(probe);
CodexProviderTransport::PreferWebSocket
}
CodexSessionTransportDecision::HttpSse { .. } => {
CodexProviderTransport::HttpOnly
}
}
}
AttemptTransport::Default => requested_transport,
AttemptTransport::Retry(_) => requested_transport,
AttemptTransport::Fallback => CodexProviderTransport::HttpOnly,
}
};
if sticky_http_fallback {
if let Some(session_transport_action) = session_transport_action {
tracing::info!(
request_id = %context.request_id(),
attempt_index = context.attempt_index().get(),
@@ -469,7 +500,11 @@ impl Provider for CodexProvider {
.as_ref()
.map_or("", CodexSessionAffinity::key_hash),
transport_requirement = requirement.as_str(),
"Reusing session-sticky OpenAI upstream HTTP fallback"
session_transport_action,
session_websocket_failure_count = session_websocket_failure_count.unwrap_or_default(),
session_http_cooldown_ms = session_http_cooldown_ms.unwrap_or_default(),
session_http_cooldown_present = session_http_cooldown_ms.is_some(),
"Applied OpenAI session transport recovery decision"
);
}
apply_transport(&mut upstream_request, transport);
@@ -518,7 +553,9 @@ impl Provider for CodexProvider {
output_started_at,
session_affinity_key,
session_affinity_key_hash,
session_transport_fallbacks: self.session_transport_fallbacks.clone(),
websocket_connection_preference,
session_transport_recovery: self.session_transport_recovery.clone(),
session_recovery_probe,
websocket_retry_count,
stream_max_retries: self.stream_max_retries,
session_capture,
@@ -1,104 +1,457 @@
//! OpenAI 根会话级上游传输降级状态。
//! OpenAI 根会话级上游传输恢复状态。
use std::collections::HashMap;
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, Instant};
use std::time::Duration;
use gateway_core::provider_ports::ProviderSessionAffinityKey;
use tokio::time::Instant;
use uuid::Uuid;
use crate::credential::CODEX_ROOT_SESSION_TTL;
/// 未成功恢复的失败链需要跨过最长冷却,避免 4 小时档在半开前被清理。
const SESSION_RECOVERY_RETENTION: Duration = Duration::from_secs(8 * 60 * 60);
const HTTP_COOLDOWNS: [Duration; 4] = [
Duration::from_secs(5 * 60),
Duration::from_secs(30 * 60),
Duration::from_secs(60 * 60),
Duration::from_secs(4 * 60 * 60),
];
/// Provider 是进程级单例;限制不可显式观测结束的客户端会话占用。
const MAX_SESSION_HTTP_FALLBACKS: usize = 16_384;
const MAX_SESSION_RECOVERIES: usize = 16_384;
#[derive(Clone)]
pub(crate) struct CodexSessionTransportFallbacks {
state: Arc<Mutex<SessionTransportFallbackState>>,
ttl: Duration,
pub(crate) struct CodexSessionTransportRecovery {
state: Arc<Mutex<SessionTransportRecoveryState>>,
http_cooldowns: [Duration; 4],
retention: Duration,
capacity: usize,
}
#[derive(Default)]
struct SessionTransportFallbackState {
http_only: HashMap<ProviderSessionAffinityKey, Instant>,
struct SessionTransportRecoveryState {
sessions: HashMap<ProviderSessionAffinityKey, SessionCircuit>,
}
impl Default for CodexSessionTransportFallbacks {
enum SessionCircuit {
FreshPending {
failure_count: u8,
last_failure_at: Instant,
},
Open {
failure_count: u8,
last_failure_at: Instant,
until: Instant,
},
HalfOpen {
failure_count: u8,
last_failure_at: Instant,
probe_id: Uuid,
},
}
impl SessionCircuit {
const fn failure_count(&self) -> u8 {
match self {
Self::FreshPending { failure_count, .. }
| Self::Open { failure_count, .. }
| Self::HalfOpen { failure_count, .. } => *failure_count,
}
}
const fn last_failure_at(&self) -> Instant {
match self {
Self::FreshPending {
last_failure_at, ..
}
| Self::Open {
last_failure_at, ..
}
| Self::HalfOpen {
last_failure_at, ..
} => *last_failure_at,
}
}
}
impl Default for CodexSessionTransportRecovery {
fn default() -> Self {
Self::with_limits(CODEX_ROOT_SESSION_TTL, MAX_SESSION_HTTP_FALLBACKS)
Self {
state: Arc::new(Mutex::new(SessionTransportRecoveryState::default())),
http_cooldowns: HTTP_COOLDOWNS,
retention: SESSION_RECOVERY_RETENTION,
capacity: MAX_SESSION_RECOVERIES,
}
}
}
impl CodexSessionTransportFallbacks {
fn with_limits(ttl: Duration, capacity: usize) -> Self {
debug_assert!(!ttl.is_zero());
debug_assert!(capacity > 0);
#[derive(Default)]
pub(crate) enum CodexSessionTransportDecision {
#[default]
Default,
FreshWebSocket {
failure_count: u8,
probe: CodexSessionWebSocketProbe,
},
HttpSse {
reason: SessionHttpFallbackReason,
failure_count: u8,
retry_after: Option<Duration>,
},
}
impl CodexSessionTransportDecision {
pub(crate) const fn action(&self) -> Option<&'static str> {
match self {
Self::Default => None,
Self::FreshWebSocket { .. } => Some("fresh_websocket_probe"),
Self::HttpSse {
reason: SessionHttpFallbackReason::Cooldown,
..
} => Some("http_cooldown"),
Self::HttpSse {
reason: SessionHttpFallbackReason::ProbeInFlight,
..
} => Some("http_probe_in_flight"),
}
}
pub(crate) const fn failure_count(&self) -> Option<u8> {
match self {
Self::Default => None,
Self::FreshWebSocket { failure_count, .. } | Self::HttpSse { failure_count, .. } => {
Some(*failure_count)
}
}
}
pub(crate) const fn retry_after(&self) -> Option<Duration> {
match self {
Self::HttpSse { retry_after, .. } => *retry_after,
Self::Default | Self::FreshWebSocket { .. } => None,
}
}
}
#[derive(Clone, Copy)]
pub(crate) enum SessionWebSocketFallback {
RetryBudgetExhausted,
UpgradeRequired,
}
impl SessionWebSocketFallback {
const fn minimum_failure_count(self) -> u8 {
match self {
Self::RetryBudgetExhausted => 2,
Self::UpgradeRequired => 5,
}
}
}
#[derive(Clone, Copy)]
pub(crate) enum SessionHttpFallbackReason {
Cooldown,
ProbeInFlight,
}
#[derive(Clone, Copy)]
pub(crate) struct CodexSessionRecoveryTransition {
failure_count: u8,
cooldown: Option<Duration>,
}
impl CodexSessionRecoveryTransition {
pub(crate) const fn action(self) -> &'static str {
if self.cooldown.is_some() {
"http_cooldown"
} else {
"fresh_websocket_next"
}
}
pub(crate) const fn failure_count(self) -> u8 {
self.failure_count
}
pub(crate) const fn cooldown(self) -> Option<Duration> {
self.cooldown
}
}
/// 一次根会话级 fresh WebSocket 许可。
///
/// 许可未得到明确结果就被释放时,恢复状态会重新允许下一次半开探测。
pub(crate) struct CodexSessionWebSocketProbe {
recovery: CodexSessionTransportRecovery,
key: ProviderSessionAffinityKey,
probe_id: Uuid,
armed: bool,
}
impl CodexSessionWebSocketProbe {
fn new(
recovery: CodexSessionTransportRecovery,
key: ProviderSessionAffinityKey,
probe_id: Uuid,
) -> Self {
Self {
state: Arc::new(Mutex::new(SessionTransportFallbackState::default())),
ttl,
capacity,
recovery,
key,
probe_id,
armed: true,
}
}
/// 返回该根会话是否已粘性切到 HTTP,并为活跃会话续期。
pub(crate) fn is_http_only(&self, key: &ProviderSessionAffinityKey) -> bool {
self.is_http_only_at(key, Instant::now())
pub(crate) fn succeed(mut self) -> bool {
let recovered = self.recovery.record_probe_success(&self.key, self.probe_id);
self.armed = false;
recovered
}
/// 在当前 WebSocket 重试预算耗尽后,为根会话启用粘性 HTTP。
///
/// 返回 `true` 表示本次发生了禁用状态转换;已禁用会话只续期。
pub(crate) fn disable_websocket(&self, key: &ProviderSessionAffinityKey) -> bool {
self.disable_websocket_at(key, Instant::now())
pub(crate) fn post_send_failed(mut self) -> CodexSessionRecoveryTransition {
let transition = self.recovery.record_post_send_failure(&self.key);
self.armed = false;
transition
}
fn is_http_only_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
pub(crate) fn fallback(
mut self,
fallback: SessionWebSocketFallback,
) -> CodexSessionRecoveryTransition {
let transition = self.recovery.record_websocket_fallback(&self.key, fallback);
self.armed = false;
transition
}
}
impl Drop for CodexSessionWebSocketProbe {
fn drop(&mut self) {
if self.armed {
self.recovery.cancel_probe(&self.key, self.probe_id);
}
}
}
impl CodexSessionTransportRecovery {
pub(crate) fn decide(&self, key: &ProviderSessionAffinityKey) -> CodexSessionTransportDecision {
self.decide_at(key, Instant::now())
}
pub(crate) fn record_post_send_failure(
&self,
key: &ProviderSessionAffinityKey,
) -> CodexSessionRecoveryTransition {
self.record_post_send_failure_at(key, Instant::now())
}
pub(crate) fn record_websocket_fallback(
&self,
key: &ProviderSessionAffinityKey,
fallback: SessionWebSocketFallback,
) -> CodexSessionRecoveryTransition {
self.record_websocket_fallback_at(key, fallback, Instant::now())
}
fn decide_at(
&self,
key: &ProviderSessionAffinityKey,
now: Instant,
) -> CodexSessionTransportDecision {
let mut state = self.lock();
let Some(last_seen) = state.http_only.get_mut(key) else {
return false;
let stale = state.sessions.get(key).is_some_and(|circuit| {
now.saturating_duration_since(circuit.last_failure_at()) >= self.retention
});
if stale {
state.sessions.remove(key);
return CodexSessionTransportDecision::Default;
}
let Some(circuit) = state.sessions.get_mut(key) else {
return CodexSessionTransportDecision::Default;
};
if now.saturating_duration_since(*last_seen) >= self.ttl {
state.http_only.remove(key);
return false;
match circuit {
SessionCircuit::Open {
failure_count,
until,
..
} if now < *until => CodexSessionTransportDecision::HttpSse {
reason: SessionHttpFallbackReason::Cooldown,
failure_count: *failure_count,
retry_after: Some(until.saturating_duration_since(now)),
},
SessionCircuit::HalfOpen { failure_count, .. } => {
CodexSessionTransportDecision::HttpSse {
reason: SessionHttpFallbackReason::ProbeInFlight,
failure_count: *failure_count,
retry_after: None,
}
}
SessionCircuit::FreshPending {
failure_count,
last_failure_at,
}
| SessionCircuit::Open {
failure_count,
last_failure_at,
..
} => {
let failure_count = *failure_count;
let last_failure_at = *last_failure_at;
let probe_id = Uuid::new_v4();
*circuit = SessionCircuit::HalfOpen {
failure_count,
last_failure_at,
probe_id,
};
CodexSessionTransportDecision::FreshWebSocket {
failure_count,
probe: CodexSessionWebSocketProbe::new(self.clone(), key.clone(), probe_id),
}
}
}
*last_seen = now;
true
}
fn disable_websocket_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
fn record_post_send_failure_at(
&self,
key: &ProviderSessionAffinityKey,
now: Instant,
) -> CodexSessionRecoveryTransition {
let mut state = self.lock();
let already_disabled = state
.http_only
.get(key)
.is_some_and(|last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
if !already_disabled && !state.http_only.contains_key(key) {
let failure_count = self.next_failure_count(state.sessions.get(key), now);
if !state.sessions.contains_key(key) {
self.make_room(&mut state, now);
}
state.http_only.insert(key.clone(), now);
!already_disabled
if failure_count == 1 {
state.sessions.insert(
key.clone(),
SessionCircuit::FreshPending {
failure_count,
last_failure_at: now,
},
);
return CodexSessionRecoveryTransition {
failure_count,
cooldown: None,
};
}
let cooldown = self.cooldown_for(failure_count);
state.sessions.insert(
key.clone(),
SessionCircuit::Open {
failure_count,
last_failure_at: now,
until: now + cooldown,
},
);
CodexSessionRecoveryTransition {
failure_count,
cooldown: Some(cooldown),
}
}
fn make_room(&self, state: &mut SessionTransportFallbackState, now: Instant) {
if state.http_only.len() < self.capacity {
fn record_websocket_fallback_at(
&self,
key: &ProviderSessionAffinityKey,
fallback: SessionWebSocketFallback,
now: Instant,
) -> CodexSessionRecoveryTransition {
let mut state = self.lock();
let failure_count = self
.next_failure_count(state.sessions.get(key), now)
.max(fallback.minimum_failure_count());
if !state.sessions.contains_key(key) {
self.make_room(&mut state, now);
}
let cooldown = self.cooldown_for(failure_count);
state.sessions.insert(
key.clone(),
SessionCircuit::Open {
failure_count,
last_failure_at: now,
until: now + cooldown,
},
);
CodexSessionRecoveryTransition {
failure_count,
cooldown: Some(cooldown),
}
}
fn next_failure_count(&self, circuit: Option<&SessionCircuit>, now: Instant) -> u8 {
circuit.map_or(1, |circuit| {
if now.saturating_duration_since(circuit.last_failure_at()) < self.retention {
circuit.failure_count().saturating_add(1)
} else {
1
}
})
}
fn cooldown_for(&self, failure_count: u8) -> Duration {
let index = usize::from(failure_count.saturating_sub(2)).min(self.http_cooldowns.len() - 1);
self.http_cooldowns[index]
}
fn record_probe_success(&self, key: &ProviderSessionAffinityKey, probe_id: Uuid) -> bool {
let mut state = self.lock();
let owns_probe = matches!(
state.sessions.get(key),
Some(SessionCircuit::HalfOpen {
probe_id: active_probe_id,
..
}) if *active_probe_id == probe_id
);
if owns_probe {
state.sessions.remove(key);
}
owns_probe
}
fn cancel_probe(&self, key: &ProviderSessionAffinityKey, probe_id: Uuid) {
let mut state = self.lock();
let Some(SessionCircuit::HalfOpen {
failure_count,
last_failure_at,
probe_id: active_probe_id,
}) = state.sessions.get(key)
else {
return;
};
if *active_probe_id != probe_id {
return;
}
state
.http_only
.retain(|_, last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
if state.http_only.len() < self.capacity {
let failure_count = *failure_count;
let last_failure_at = *last_failure_at;
state.sessions.insert(
key.clone(),
SessionCircuit::FreshPending {
failure_count,
last_failure_at,
},
);
}
fn make_room(&self, state: &mut SessionTransportRecoveryState, now: Instant) {
if state.sessions.len() < self.capacity {
return;
}
state.sessions.retain(|_, circuit| {
now.saturating_duration_since(circuit.last_failure_at()) < self.retention
});
if state.sessions.len() < self.capacity {
return;
}
let oldest = state
.http_only
.sessions
.iter()
.min_by_key(|(_, last_seen)| **last_seen)
.min_by_key(|(_, circuit)| circuit.last_failure_at())
.map(|(key, _)| key.clone());
if let Some(oldest) = oldest {
state.http_only.remove(&oldest);
state.sessions.remove(&oldest);
}
}
fn lock(&self) -> MutexGuard<'_, SessionTransportFallbackState> {
fn lock(&self) -> MutexGuard<'_, SessionTransportRecoveryState> {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
@@ -24,7 +24,9 @@ use serde_json::{Value, map::Map};
use thiserror::Error;
use uuid::Uuid;
use crate::transport::protocol::responses::{CodexResponsesRequest, TransportRequirement};
use crate::transport::protocol::responses::{
CodexResponsesRequest, TransportRequirement, X_CODEX_TURN_STATE_CLIENT_METADATA_KEY,
};
use super::diagnostics::{CodexUpstreamDiagnostics, CodexUpstreamFailure, CodexUpstreamSendPhase};
use super::response_meta::CodexResponseMetadata;
@@ -546,7 +548,7 @@ pub struct CodexBackendStreamingResponse {
pub transport: CodexBackendTransport,
/// WebSocket 响应所绑定的连接;HTTP transport 为 `None`。
pub websocket_connection_id: Option<Uuid>,
/// 响应头里的最新 turn state。
/// 响应头或 metadata 事件里最先确认的 turn state。
pub turn_state: Option<String>,
/// 上游透传的 `set-cookie` 列表。
pub set_cookie_headers: Vec<String>,
@@ -760,32 +762,33 @@ pub(super) async fn read_error_response_body(
pub(super) fn websocket_upstream_request(request: &CodexResponsesRequest) -> CodexResponsesRequest {
let mut request = request.clone();
project_responses_lite_to_ws_metadata(&mut request);
stamp_ws_stream_request_start_ms(&mut request);
project_websocket_client_metadata(&mut request);
request
}
fn project_responses_lite_to_ws_metadata(request: &mut CodexResponsesRequest) {
let Some(responses_lite) = request.responses_lite.clone() else {
return;
};
let mut metadata = match request.client_metadata() {
Some(Value::Object(metadata)) => metadata.clone(),
None => Map::new(),
Some(_) => return,
};
metadata
.entry(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY.to_string())
.or_insert(Value::String(responses_lite));
request.set_client_metadata(Some(Value::Object(metadata)));
}
fn stamp_ws_stream_request_start_ms(request: &mut CodexResponsesRequest) {
fn project_websocket_client_metadata(request: &mut CodexResponsesRequest) {
let mut metadata = match request.client_metadata() {
Some(Value::Object(metadata)) => metadata.clone(),
None => Map::new(),
Some(_) => return,
};
if let Some(responses_lite) = request.responses_lite.clone() {
metadata
.entry(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY.to_owned())
.or_insert(Value::String(responses_lite));
}
let turn_state = request.turn_state.clone();
match turn_state {
Some(turn_state) => {
metadata.insert(
X_CODEX_TURN_STATE_CLIENT_METADATA_KEY.to_owned(),
Value::String(turn_state),
);
}
None => {
metadata.remove(X_CODEX_TURN_STATE_CLIENT_METADATA_KEY);
}
}
metadata.insert(
X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY.to_string(),
Value::String(now_unix_timestamp_millis().to_string()),
@@ -854,9 +857,6 @@ pub(super) fn http_fallback_decision(
error: &CodexWebSocketExchangeError,
) -> CodexTransportDecision {
match error.classified() {
CodexWebSocketExchangeError::FastPathTimeout { .. } => {
CodexTransportDecision::Http2WebSocketBudgetExhausted
}
CodexWebSocketExchangeError::OriginCircuitOpen
| CodexWebSocketExchangeError::OriginHalfOpenBusy => {
CodexTransportDecision::Http2BreakerOpen
@@ -37,8 +37,9 @@ use crate::transport::{
response_meta,
websocket::{
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
CodexWebSocketPoolKey, CodexWebSocketStreamingExchange, DEFAULT_INITIAL_EVENT_TIMEOUT,
WEBSOCKET_FAST_PATH_BUDGET, WebSocketOriginBreaker, WebSocketPoolDecision,
CodexWebSocketPoolKey, CodexWebSocketRequest, CodexWebSocketStreamingExchange,
DEFAULT_INITIAL_EVENT_TIMEOUT, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
WebSocketOriginBreaker, WebSocketPoolDecision,
execute_prepared_response_create_request_stream, post_send_ambiguous,
prepare_response_create_request_with_pool, websocket_audit_dir,
write_websocket_audit_artifact_from_env,
@@ -47,6 +48,16 @@ use crate::transport::{
use super::client::*;
/// 单次 Responses WebSocket opening 是否可以使用会话连接池。
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum WebSocketConnectionPreference {
/// 优先复用匹配的空闲连接,否则建立并登记一条新连接。
#[default]
ReuseOrConnect,
/// 绕过连接池建立独立连接,用于根会话恢复探针。
Fresh,
}
impl CodexBackendClient {
/// 构造客户端。
pub fn new(
@@ -94,11 +105,26 @@ impl CodexBackendClient {
upstream_body.insert("stream".to_owned(), serde_json::Value::Bool(true));
let body =
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
let endpoint = endpoint_url(&self.base_url, CODEX_RESPONSES_PATH);
tracing::info!(
target: "request_dump",
provider = "openai",
request_id = %context.request_id,
account_id = context.account_id.unwrap_or_default(),
request_direction = "upstream",
request_transport = "http_sse",
request_method = "POST",
request_url = %endpoint,
request_headers = ?headers,
request_body = %String::from_utf8_lossy(&body),
contains_sensitive_data = true,
"request dump (unredacted)"
);
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
.map_err(CodexClientError::RequestCompression)?;
let response = self
.client
.post(endpoint_url(&self.base_url, CODEX_RESPONSES_PATH))
.post(endpoint)
.headers(headers)
.header(CONTENT_ENCODING, HeaderValue::from_static("zstd"))
.body(body)
@@ -188,9 +214,16 @@ impl CodexBackendClient {
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
connection_preference: WebSocketConnectionPreference,
) -> CodexClientResult<CodexBackendStreamingResponse> {
let prepared = self
.prepare_response_transport(request, context, pool_account_id, false, true)
.prepare_response_transport(
request,
context,
pool_account_id,
connection_preference,
true,
)
.await?;
self.create_response_stream_with_prepared(request, context, prepared)
.await
@@ -204,18 +237,24 @@ impl CodexBackendClient {
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
) -> CodexClientResult<PreparedResponseTransport> {
self.prepare_response_transport(request, context, pool_account_id, false, false)
.await
self.prepare_response_transport(
request,
context,
pool_account_id,
WebSocketConnectionPreference::ReuseOrConnect,
false,
)
.await
}
/// 与 [`Self::prepare_response_transport_with_pool_account`] 相同;
/// `force_fresh` 时绕过连接池,为连接寿命限制重试强制新建连接。
/// fresh 偏好会绕过连接池,为受控恢复探针强制新建连接。
async fn prepare_response_transport(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
pool_account_id: Option<&str>,
force_fresh: bool,
connection_preference: WebSocketConnectionPreference,
defer_websocket_recovery: bool,
) -> CodexClientResult<PreparedResponseTransport> {
let requirement = transport_requirement(request);
@@ -256,10 +295,11 @@ impl CodexBackendClient {
let pool_key =
self.websocket_pool_key(request, context, pool_account_id, &connection_profile);
let pool_log_context = pool_key.as_ref().map(WebSocketPoolLogContext::from_key);
let pool = if force_fresh {
None
} else {
self.websocket_pool.as_deref().zip(pool_key)
let pool = match connection_preference {
WebSocketConnectionPreference::ReuseOrConnect => {
self.websocket_pool.as_deref().zip(pool_key)
}
WebSocketConnectionPreference::Fresh => None,
};
let fast_path_budget = match requirement {
TransportRequirement::PersistedContinuation | TransportRequirement::NewChain => {
@@ -282,7 +322,31 @@ impl CodexBackendClient {
)
.await;
let prepared = match prepared {
Ok(prepared) => prepared,
Ok(WebSocketFastPath::Ready(prepared)) => prepared,
Ok(WebSocketFastPath::Missed) => {
let decision = CodexTransportDecision::Http2WebSocketBudgetExhausted;
let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed());
tracing::info!(
request_id = %context.request_id,
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
transport_requirement = requirement.as_str(),
transport_decision = decision.as_str(),
transport_decision_wait_ms = wait_ms,
websocket_preconnect_continues = pool_log_context.is_some(),
"WebSocket fast path missed; using same-account HTTP"
);
return Ok(PreparedResponseTransport {
requirement,
route: PreparedResponseRoute::Http,
metrics: CodexTransportMetrics {
decision: Some(decision),
ws_connect_ms: None,
transport_decision_wait_ms: Some(wait_ms),
..CodexTransportMetrics::default()
},
defer_websocket_recovery,
});
}
Err(error)
if (!defer_websocket_recovery || error.requires_immediate_pool_fallback())
&& requirement.allows_pre_send_http_fallback()
@@ -377,6 +441,7 @@ impl CodexBackendClient {
prepared,
} = *route;
let delivery_wait_started_at = Instant::now();
log_unredacted_websocket_request(&websocket_request, context);
let mut exchange = match execute_prepared_response_create_request_stream(
&websocket_request,
prepared,
@@ -434,13 +499,20 @@ impl CodexBackendClient {
);
drop(exchange);
match self
.prepare_response_transport(request, context, None, true, false)
.prepare_response_transport(
request,
context,
None,
WebSocketConnectionPreference::Fresh,
false,
)
.await
{
Ok(PreparedResponseTransport {
route: PreparedResponseRoute::WebSocket(route),
..
}) => {
log_unredacted_websocket_request(&route.request, context);
match execute_prepared_response_create_request_stream(
&route.request,
route.prepared,
@@ -628,10 +700,12 @@ impl CodexBackendClient {
.local_conversation_id
.as_deref()
.or(request.previous_response_id())?;
Some(
CodexWebSocketPoolKey::new(&self.base_url, account_id, conversation_id)
.with_connection_profile(connection_profile),
)
let mut key = CodexWebSocketPoolKey::new(&self.base_url, account_id, conversation_id)
.with_connection_profile(connection_profile);
if let Some(connection_id) = request.downstream_websocket_connection_id.as_deref() {
key = key.with_downstream_connection_id(connection_id);
}
Some(key)
}
/// 获取后端模型目录条目。
@@ -835,6 +909,26 @@ impl CodexBackendClient {
}
}
fn log_unredacted_websocket_request(
request: &CodexWebSocketRequest,
context: CodexRequestContext<'_>,
) {
tracing::info!(
target: "request_dump",
provider = "openai",
request_id = %context.request_id,
account_id = context.account_id.unwrap_or_default(),
request_direction = "upstream",
request_transport = "websocket",
request_method = "GET + response.create",
request_url = request.connection().endpoint(),
request_headers = ?request.connection().headers(),
request_payload = request.payload_text(),
contains_sensitive_data = true,
"request dump (unredacted)"
);
}
/// 首个可投递帧前的交付边界结果。
enum DeliveryBoundary {
/// 已越过边界,可开始向下游投递。
@@ -21,6 +21,7 @@ pub mod tls;
pub mod usage;
pub mod websocket;
pub(crate) use self::client_sse::WebSocketConnectionPreference;
pub use self::{
canonical::{CodexCanonicalDecoder, CodexCanonicalError},
catalog::{
@@ -10,6 +10,8 @@ use serde_json::{Map, Value};
/// 官方 Codex 客户端据此触发完整历史重放的稳定错误码。
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_CODE: &str = "previous_response_not_found";
/// Responses WebSocket 用于回传同一 turn 不透明状态的官方 client metadata 键。
pub(crate) const X_CODEX_TURN_STATE_CLIENT_METADATA_KEY: &str = "x-codex-turn-state";
/// 本地生成 history unavailable 错误时使用的官方提示文本。
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE: &str =
"Previous response was not found. Retrying the full request.";
@@ -82,6 +84,8 @@ pub struct CodexResponsesRequest {
pub memgen_request: Option<String>,
/// Codex window ID。
pub codex_window_id: Option<String>,
/// 代理分配的下游 WebSocket 连接 ID,仅用于隔离本地连接池通道。
pub downstream_websocket_connection_id: Option<String>,
/// 父线程 ID。
pub parent_thread_id: Option<String>,
/// 已知 previous response 的持久化范围,仅用于本地 transport 校验。
@@ -162,8 +166,8 @@ impl TransportRequirement {
matches!(self, Self::NewChain)
}
/// 会话级 sticky 状态是否可以把首选 WebSocket 改为 HTTP。
pub fn allows_sticky_http_fallback(self) -> bool {
/// 会话级恢复状态是否可以为请求选择 fresh WebSocket 或临时 HTTP。
pub fn allows_session_transport_recovery(self) -> bool {
matches!(self, Self::NewChain)
}
@@ -519,6 +523,7 @@ impl CodexResponsesRequest {
responses_lite: None,
memgen_request: None,
codex_window_id: None,
downstream_websocket_connection_id: None,
parent_thread_id: None,
previous_response_scope: None,
}
@@ -110,9 +110,9 @@ fn is_internal_websocket_event(event: &str) -> bool {
event == "codex.rate_limits"
}
/// 提取 `response.metadata` 帧中的字符串响应头。
/// 提取 Responses WebSocket metadata 帧中的字符串响应头。
pub fn websocket_metadata_headers(value: &Value) -> Vec<(String, String)> {
if value.get("type").and_then(Value::as_str) != Some("response.metadata") {
if !is_websocket_metadata_event(websocket_event_type(value)) {
return Vec::new();
}
value
@@ -124,24 +124,18 @@ pub fn websocket_metadata_headers(value: &Value) -> Vec<(String, String)> {
.collect()
}
/// 从 `response.metadata` 帧中提取 `x-codex-turn-state`。
/// 从 Responses WebSocket metadata 帧中提取 `x-codex-turn-state`。
pub fn websocket_metadata_turn_state(value: &Value) -> Option<String> {
if value.get("type").and_then(Value::as_str) != Some("response.metadata") {
return None;
}
websocket_metadata_headers(value)
.into_iter()
.find_map(|(name, value)| {
name.eq_ignore_ascii_case("x-codex-turn-state")
.then_some(value)
})
.or_else(|| {
value
.pointer("/metadata/turn_state")
.or_else(|| value.get("turn_state"))
.and_then(Value::as_str)
.map(ToString::to_string)
})
}
fn is_websocket_metadata_event(event: Option<&str>) -> bool {
matches!(event, Some("response.metadata" | "codex.response.metadata"))
}
/// 上游 WebSocket 连接寿命限制错误码。
@@ -7,9 +7,12 @@ use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde_json::{Map, Value};
use sha2::{Digest, Sha256};
use crate::transport::protocol::responses::CodexResponsesRequest;
use crate::transport::protocol::responses::{
CodexResponsesRequest, X_CODEX_TURN_STATE_CLIENT_METADATA_KEY,
};
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
const TURN_ID_CLIENT_METADATA_KEY: &str = "turn_id";
const THREAD_SPAWN_SUBAGENT_KIND: &str = "thread_spawn";
const THREAD_SPAWN_CONVERSATION_PREFIX: &str = "thread-spawn:";
const UNSUPPORTED_CODEX_RESPONSES_FIELDS: &[&str] = &["max_output_tokens", "temperature"];
@@ -144,8 +147,16 @@ struct ExtractedRequestContext {
impl ExtractedRequestContext {
fn from_body(body: &Map<String, Value>) -> Self {
let client_metadata = body.get("client_metadata").and_then(Value::as_object);
Self {
turn_state: body_string(body, "turnState"),
// 官方 downstream WebSocket 无法逐帧更新 HTTP header,因此把
// response.metadata 返回的 turn state 放回下一帧 client_metadata。
// Provider 仍会在账号/turn 归属确定后决定是否允许复用该状态。
turn_state: body_string(body, "turnState").or_else(|| {
client_metadata.and_then(|metadata| {
string_value(metadata.get(X_CODEX_TURN_STATE_CLIENT_METADATA_KEY))
})
}),
turn_metadata: body_string(body, "turnMetadata"),
beta_features: body_string(body, "betaFeatures"),
version: body_string(body, "version"),
@@ -156,15 +167,15 @@ impl ExtractedRequestContext {
session_id: body_string(body, "session_id"),
thread_id: body_string(body, "thread_id"),
client_request_id: body_string(body, "x-client-request-id"),
turn_id: body_string(body, "turn_id"),
turn_id: body_string(body, "turn_id").or_else(|| {
client_metadata
.and_then(|metadata| string_value(metadata.get(TURN_ID_CLIENT_METADATA_KEY)))
}),
// Responses Lite 在 WebSocket body 中的这个键是官方 header 投影,
// 不是普通上下文字段的 metadata 回退。
responses_lite: body
.get("client_metadata")
.and_then(Value::as_object)
.and_then(|metadata| {
string_value(metadata.get(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY))
}),
responses_lite: client_metadata.and_then(|metadata| {
string_value(metadata.get(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY))
}),
// Memory consolidation 只由官方请求头提供;body 中同名字段不参与
// transport 事实提取。
memgen_request: None,
@@ -511,6 +522,9 @@ fn apply_protocol_context(request: &mut CodexResponsesRequest, context: &Map<Str
.or_else(|| request.include_timing_metrics.take());
request.codex_window_id =
context_string(context, "codex_window_id").or_else(|| request.codex_window_id.take());
request.downstream_websocket_connection_id =
context_string(context, "downstream_websocket_connection_id")
.or_else(|| request.downstream_websocket_connection_id.take());
request.parent_thread_id =
context_string(context, "parent_thread_id").or_else(|| request.parent_thread_id.take());
let prompt_cache_key = request.prompt_cache_key().map(ToOwned::to_owned);
@@ -29,6 +29,14 @@ use super::{
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
/// WebSocket 快路径的控制流结果;预算未命中不代表连接失败。
pub(crate) enum WebSocketFastPath<T> {
/// 在前台预算内获得结果。
Ready(T),
/// 前台停止等待;池化 opening 可以继续在后台完成。
Missed,
}
/// 尚未发送 `response.create` 的 WebSocket。只有该类型可以安全切换到 HTTP。
pub(crate) struct PreparedWebSocket {
connection: PooledWebSocketConnection,
@@ -104,7 +112,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
fast_path_budget: Option<Duration>,
require_pool: bool,
fallback_initial_event_timeout: Option<Duration>,
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
) -> Result<WebSocketFastPath<PreparedWebSocket>, CodexWebSocketExchangeError> {
let decision_started_at = Instant::now();
let Some((pool, key)) = pool else {
if require_pool || !request.continuation().permits_fresh_connection() {
@@ -142,7 +150,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
PreviousResponseUnavailableReason::LatestResponseMismatch,
));
}
Ok(PreparedWebSocket {
Ok(WebSocketFastPath::Ready(PreparedWebSocket {
connection: *connection,
binding: PoolBinding::Pooled {
lease: Box::new(lease),
@@ -152,7 +160,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
connect_elapsed: None,
decision_wait_elapsed: decision_started_at.elapsed(),
initial_event_timeout: pool.initial_event_timeout(),
})
}))
}
WebSocketPoolAcquire::Connect(connect_lease) => {
if !request.continuation().permits_fresh_connection() {
@@ -178,14 +186,14 @@ pub(crate) async fn prepare_response_create_request_with_pool(
PreviousResponseUnavailableReason::ConnectionBusy,
));
}
let outcome = wait_for_shared_connect(waiter, fast_path_budget).await?;
match outcome {
WebSocketPoolConnectOutcome::Ready => Err(continuation_unavailable(
PreviousResponseUnavailableReason::ConnectionBusy,
)),
WebSocketPoolConnectOutcome::Failed | WebSocketPoolConnectOutcome::Pending => {
Err(CodexWebSocketExchangeError::SharedConnectFailed)
}
match wait_for_shared_connect(waiter, fast_path_budget).await {
WebSocketFastPath::Missed => Ok(WebSocketFastPath::Missed),
WebSocketFastPath::Ready(WebSocketPoolConnectOutcome::Ready) => Err(
continuation_unavailable(PreviousResponseUnavailableReason::ConnectionBusy),
),
WebSocketFastPath::Ready(
WebSocketPoolConnectOutcome::Failed | WebSocketPoolConnectOutcome::Pending,
) => Err(CodexWebSocketExchangeError::SharedConnectFailed),
}
}
WebSocketPoolAcquire::ContinuationLost(observation) => Err(continuation_unavailable(
@@ -203,7 +211,7 @@ async fn prepare_unpooled_websocket(
fast_path_budget: Option<Duration>,
initial_event_timeout: Option<Duration>,
decision_started_at: Instant,
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
) -> Result<WebSocketFastPath<PreparedWebSocket>, CodexWebSocketExchangeError> {
let permit = acquire_breaker_permit(breaker, origin_key)?;
let context = pump_log_context_from_connection(request.connection());
let connected = connect_with_budget(
@@ -213,14 +221,22 @@ async fn prepare_unpooled_websocket(
context,
)
.await;
let connected = match connected {
Ok(WebSocketFastPath::Ready(connection)) => Ok(connection),
Ok(WebSocketFastPath::Missed) => {
permit.fast_timeout();
return Ok(WebSocketFastPath::Missed);
}
Err(error) => Err(error),
};
let (connection, connect_elapsed) = finish_breaker_attempt(permit, connected)?;
Ok(PreparedWebSocket {
Ok(WebSocketFastPath::Ready(PreparedWebSocket {
connection,
binding: PoolBinding::Unpooled,
connect_elapsed: Some(connect_elapsed),
decision_wait_elapsed: decision_started_at.elapsed(),
initial_event_timeout,
})
}))
}
async fn prepare_pooled_websocket(
@@ -231,7 +247,7 @@ async fn prepare_pooled_websocket(
origin_key: &str,
fast_path_budget: Option<Duration>,
decision_started_at: Instant,
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
) -> Result<WebSocketFastPath<PreparedWebSocket>, CodexWebSocketExchangeError> {
let permit = match acquire_breaker_permit(breaker, origin_key) {
Ok(permit) => permit,
Err(error) => {
@@ -245,18 +261,20 @@ async fn prepare_pooled_websocket(
connect_lease,
permit,
);
let handoff = waiter.wait(fast_path_budget).await?;
Ok(PreparedWebSocket {
connection: *handoff.connection,
binding: PoolBinding::Pooled {
lease: Box::new(handoff.lease),
reused: false,
decision: WebSocketPoolDecision::new(),
},
connect_elapsed: Some(handoff.connect_elapsed),
decision_wait_elapsed: decision_started_at.elapsed(),
initial_event_timeout: pool.initial_event_timeout(),
})
match waiter.wait(fast_path_budget).await? {
WebSocketFastPath::Ready(handoff) => Ok(WebSocketFastPath::Ready(PreparedWebSocket {
connection: *handoff.connection,
binding: PoolBinding::Pooled {
lease: Box::new(handoff.lease),
reused: false,
decision: WebSocketPoolDecision::new(),
},
connect_elapsed: Some(handoff.connect_elapsed),
decision_wait_elapsed: decision_started_at.elapsed(),
initial_event_timeout: pool.initial_event_timeout(),
})),
WebSocketFastPath::Missed => Ok(WebSocketFastPath::Missed),
}
}
struct PooledWebSocketConnectWaiter {
@@ -275,7 +293,7 @@ impl PooledWebSocketConnectWaiter {
async fn wait(
&mut self,
fast_path_budget: Option<Duration>,
) -> Result<PooledWebSocketHandoff, CodexWebSocketExchangeError> {
) -> Result<WebSocketFastPath<PooledWebSocketHandoff>, CodexWebSocketExchangeError> {
let received = match fast_path_budget {
Some(budget) => {
let remaining = budget.saturating_sub(self.started_at.elapsed());
@@ -283,15 +301,15 @@ impl PooledWebSocketConnectWaiter {
Ok(received) => received,
Err(_) => {
self.fast_path_reporter.missed();
return Err(CodexWebSocketExchangeError::FastPathTimeout {
timeout: budget,
});
return Ok(WebSocketFastPath::Missed);
}
}
}
None => (&mut self.receiver).await,
};
received.map_err(|_| CodexWebSocketExchangeError::SharedConnectFailed)?
received
.map_err(|_| CodexWebSocketExchangeError::SharedConnectFailed)?
.map(WebSocketFastPath::Ready)
}
}
@@ -327,7 +345,7 @@ fn start_pooled_websocket_connect(
);
return;
}
result = connect_with_budget(&connection, keepalive, None, context) => result,
result = connect_websocket_connection(&connection, keepalive, context) => result,
};
match finish_breaker_attempt(permit, connected) {
Ok((connection, connect_elapsed)) => {
@@ -401,8 +419,7 @@ fn opening_failure_reason(error: &CodexWebSocketExchangeError) -> &'static str {
CodexWebSocketExchangeError::InvalidRequest(_) => "request_build_failure",
CodexWebSocketExchangeError::Connect(error)
| CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
CodexWebSocketExchangeError::ConnectTimeout { .. }
| CodexWebSocketExchangeError::FastPathTimeout { .. } => "connect_timeout",
CodexWebSocketExchangeError::ConnectTimeout { .. } => "connect_timeout",
CodexWebSocketExchangeError::Upstream(_) => "upgrade_rejected",
CodexWebSocketExchangeError::OriginCircuitOpen
| CodexWebSocketExchangeError::OriginHalfOpenBusy => "connect_suppressed",
@@ -432,18 +449,32 @@ async fn connect_with_budget(
keepalive: PumpKeepalive,
fast_path_budget: Option<Duration>,
context: PumpLogContext,
) -> Result<WebSocketFastPath<(PooledWebSocketConnection, Duration)>, CodexWebSocketExchangeError> {
match fast_path_budget {
Some(budget) => {
let Ok(connected) = timeout(
budget,
connect_websocket_connection(connection, keepalive, context),
)
.await
else {
return Ok(WebSocketFastPath::Missed);
};
connected.map(WebSocketFastPath::Ready)
}
None => connect_websocket_connection(connection, keepalive, context)
.await
.map(WebSocketFastPath::Ready),
}
}
async fn connect_websocket_connection(
connection: &CodexWebSocketConnection,
keepalive: PumpKeepalive,
context: PumpLogContext,
) -> Result<(PooledWebSocketConnection, Duration), CodexWebSocketExchangeError> {
let started_at = Instant::now();
let connected = match fast_path_budget {
Some(budget) => timeout(
budget,
connect_pumped_websocket(connection, keepalive, context),
)
.await
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })?,
None => connect_pumped_websocket(connection, keepalive, context).await,
}?;
let (websocket, response) = connected;
let (websocket, response) = connect_pumped_websocket(connection, keepalive, context).await?;
Ok((
PooledWebSocketConnection {
websocket,
@@ -477,10 +508,6 @@ fn finish_breaker_attempt(
permit.succeed();
Ok(connection)
}
Err(error @ CodexWebSocketExchangeError::FastPathTimeout { .. }) => {
permit.fast_timeout();
Err(error)
}
Err(CodexWebSocketExchangeError::Upstream(upstream)) if upstream.status_code < 500 => {
// 账号或请求级 opening 响应证明 origin 可达,不得污染 transport 熔断器。
permit.succeed();
@@ -496,15 +523,16 @@ fn finish_breaker_attempt(
async fn wait_for_shared_connect(
waiter: pool::WebSocketPoolConnectWaiter,
fast_path_budget: Option<Duration>,
) -> Result<WebSocketPoolConnectOutcome, CodexWebSocketExchangeError> {
) -> WebSocketFastPath<WebSocketPoolConnectOutcome> {
match fast_path_budget {
Some(budget) => {
let remaining = waiter.remaining_budget(budget);
timeout(remaining, waiter.wait())
.await
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })
match timeout(remaining, waiter.wait()).await {
Ok(outcome) => WebSocketFastPath::Ready(outcome),
Err(_) => WebSocketFastPath::Missed,
}
}
None => Ok(waiter.wait().await),
None => WebSocketFastPath::Ready(waiter.wait().await),
}
}
@@ -31,12 +31,6 @@ pub enum CodexWebSocketExchangeError {
/// 建连超时时长。
timeout: Duration,
},
/// 普通请求的 WebSocket 快路径预算耗尽,payload 尚未发送。
#[error("websocket fast-path connect budget exhausted after {timeout:?}")]
FastPathTimeout {
/// 快路径等待时长。
timeout: Duration,
},
/// origin WebSocket 冷建连熔断中。
#[error("websocket origin circuit is open")]
OriginCircuitOpen,
@@ -341,7 +335,6 @@ impl CodexWebSocketExchangeError {
self,
Self::Connect(_)
| Self::ConnectTimeout { .. }
| Self::FastPathTimeout { .. }
| Self::OriginCircuitOpen
| Self::OriginHalfOpenBusy
| Self::SharedConnectFailed
@@ -31,7 +31,7 @@ pub struct CodexWebSocketStreamingExchange {
pub(crate) websocket_connection_id: Uuid,
/// 由 WebSocket 事件转换出的 live SSE 字节流。
pub body: CodexWebSocketSseStream,
/// 上游 metadata 帧中的最新 turn state。
/// 上游为本次响应返回的首个 turn state。
pub turn_state: Option<String>,
/// 上游握手响应里的 `set-cookie` 列表。
pub set_cookie_headers: Vec<String>,
@@ -39,7 +39,7 @@ pub struct CodexWebSocketStreamingExchange {
pub rate_limit_headers: Vec<(String, String)>,
/// 上游内部 `codex.rate_limits` 事件里的结构化动态更新。
pub rate_limit_updates: CodexWebSocketRateLimitUpdates,
/// 上游内部 metadata 事件里的动态 turn state。
/// 上游内部 metadata 事件里的首个动态 turn state。
pub turn_state_update: CodexWebSocketTurnStateUpdate,
/// WebSocket 连接池决策。
pub pool_decision: Option<WebSocketPoolDecision>,
@@ -63,5 +63,10 @@ pub(super) fn reusable_websocket_metadata(
mut metadata: CodexWebSocketConnectionMetadata,
) -> CodexWebSocketConnectionMetadata {
metadata.rate_limit_headers.clear();
metadata.turn_state = None;
metadata
.response_metadata
.client_headers
.retain(|(name, _)| !name.eq_ignore_ascii_case("x-codex-turn-state"));
metadata
}
@@ -21,7 +21,6 @@ pub(in crate::transport::websocket) enum WebSocketTerminalKind {
pub(super) enum ExchangeAction {
RateLimits(events::ParsedRateLimits),
TurnState(String),
Forward {
frame: String,
terminal: Option<WebSocketTerminalKind>,
@@ -32,6 +31,7 @@ pub(super) enum ExchangeAction {
pub(super) struct ReducedWebSocketEvent {
pub(super) action: ExchangeAction,
pub(super) diagnostic_event_type: Option<String>,
pub(super) turn_state_update: Option<String>,
}
pub(super) fn reduce_websocket_event(
@@ -45,6 +45,7 @@ pub(super) fn reduce_websocket_event(
return Ok(ReducedWebSocketEvent {
action: ExchangeAction::Ignore,
diagnostic_event_type: None,
turn_state_update: None,
});
};
let diagnostic_event_type = diagnostic_event_type(websocket_event_type(&value));
@@ -54,6 +55,7 @@ pub(super) fn reduce_websocket_event(
return Ok(ReducedWebSocketEvent {
action: ExchangeAction::RateLimits(parsed),
diagnostic_event_type,
turn_state_update: None,
});
}
@@ -61,13 +63,13 @@ pub(super) fn reduce_websocket_event(
&mut metadata.response_metadata,
websocket_metadata_headers(&value),
);
if let Some(turn_state) = websocket_metadata_turn_state(&value) {
let turn_state_update = websocket_metadata_turn_state(&value).and_then(|turn_state| {
if metadata.turn_state.is_some() {
return None;
}
metadata.turn_state = Some(turn_state.clone());
return Ok(ReducedWebSocketEvent {
action: ExchangeAction::TurnState(turn_state),
diagnostic_event_type,
});
}
Some(turn_state)
});
let event = websocket_event_type(&value);
if event == Some("response.completed")
@@ -89,6 +91,7 @@ pub(super) fn reduce_websocket_event(
Ok(ReducedWebSocketEvent {
action,
diagnostic_event_type,
turn_state_update,
})
}
@@ -271,15 +271,17 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
if let Some(event_type) = reduced.diagnostic_event_type {
last_event_type = Some(event_type);
}
if let Some(turn_state) = reduced.turn_state_update {
let mut pending = turn_state_update.lock().await;
if pending.is_none() {
*pending = Some(turn_state);
}
}
let (frame, terminal) = match reduced.action {
ExchangeAction::RateLimits(rate_limits) => {
rate_limit_updates.lock().await.push(rate_limits);
continue;
}
ExchangeAction::TurnState(turn_state) => {
*turn_state_update.lock().await = Some(turn_state);
continue;
}
ExchangeAction::Forward { frame, terminal } => (frame, terminal),
ExchangeAction::Ignore => continue,
};
@@ -37,7 +37,7 @@ pub use self::{
};
pub(crate) use self::{
coordinator::{
PreparedWebSocket, WEBSOCKET_FAST_PATH_BUDGET,
PreparedWebSocket, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
execute_prepared_response_create_request_stream, post_send_ambiguous,
prepare_response_create_request_with_pool,
},
@@ -23,6 +23,7 @@ pub struct CodexWebSocketPoolKey {
account_id: String,
conversation_id: String,
connection_profile: String,
downstream_connection_id: String,
}
impl CodexWebSocketPoolKey {
@@ -37,6 +38,7 @@ impl CodexWebSocketPoolKey {
account_id: account_id.into(),
conversation_id: conversation_id.into(),
connection_profile: String::new(),
downstream_connection_id: String::new(),
}
}
@@ -46,6 +48,15 @@ impl CodexWebSocketPoolKey {
self
}
/// 隔离同一逻辑会话中由不同下游 WebSocket 驱动的并发响应链。
pub(crate) fn with_downstream_connection_id(
mut self,
connection_id: impl Into<String>,
) -> Self {
self.downstream_connection_id = connection_id.into();
self
}
pub(crate) fn account_id(&self) -> &str {
&self.account_id
}
@@ -60,6 +71,7 @@ impl CodexWebSocketPoolKey {
self.account_id.as_str(),
self.conversation_id.as_str(),
self.connection_profile.as_str(),
self.downstream_connection_id.as_str(),
])
}
@@ -24,7 +24,7 @@ use gateway_core::engine::{
use gateway_core::error::{
ContinuationFailure, ContinuationRecoveryDisposition, PreDeliveryRetry, ProviderErrorKind,
};
use gateway_core::event::GatewayEvent;
use gateway_core::event::{GatewayEvent, WebSocketPoolKind};
use gateway_core::lifecycle::CancellationToken;
use gateway_core::operation::{
CapabilityRequirements, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
@@ -321,6 +321,25 @@ fn generate_with_session_context(
)
}
fn generate_with_persisted_session_context(
account_id: &str,
conversation_id: &str,
session_id: &str,
thread_id: &str,
) -> GenerateRequest {
generate_with_session_context(session_id, Some(thread_id), None).with_provider_session_state(
ProviderSessionState::new(
"openai",
Map::from_iter([
("account_id".to_owned(), json!(account_id)),
("conversation_id".to_owned(), json!(conversation_id)),
("continuation_scope".to_owned(), json!("persisted")),
]),
)
.expect("provider session state"),
)
}
fn http_generate_operation() -> Operation {
let payload = ProtocolPayload::json_object(
"openai",
@@ -439,7 +458,10 @@ fn contract_account_scope() -> Arc<FrozenAccountScope> {
"acct_truncated_stream",
"acct_usage_limit_request_path",
"acct_websocket_close",
"acct_websocket_cooldown",
"acct_websocket_fast_path",
"acct_websocket_busy_replay",
"acct_websocket_metadata_close",
"acct_websocket_turn_state",
]
.into_iter()
@@ -643,6 +665,7 @@ async fn capture_turn_state_request(
previous_turn_id: Option<&str>,
current_turn_id: Option<&str>,
client_turn_state: Option<&str>,
retry_checkpoint: bool,
) -> wiremock::Request {
let account_id = "acct_session_affinity";
let store = Arc::new(MemoryAccountStore::default());
@@ -667,6 +690,9 @@ async fn capture_turn_state_request(
if let Some(previous_turn_id) = previous_turn_id {
session_state.insert("client_turn_id".to_owned(), json!(previous_turn_id));
}
if retry_checkpoint {
session_state.insert("retry_checkpoint".to_owned(), json!(true));
}
let mut protocol_context = Map::from_iter([("use_websocket".to_owned(), json!(false))]);
if let Some(current_turn_id) = current_turn_id {
protocol_context.insert("turn_id".to_owned(), json!(current_turn_id));
@@ -1295,9 +1321,13 @@ async fn selection_infrastructure_errors_have_a_distinct_classification() {
}
#[tokio::test]
async fn websocket_close_after_delivery_preserves_details_without_enabling_sticky_http() {
async fn websocket_close_after_delivery_preserves_details_and_forces_a_fresh_client_retry() {
const ACCOUNT_ID: &str = "acct_websocket_close";
const CONVERSATION_ID: &str = "conversation-websocket-close-after-delivery";
const SESSION_ID: &str = "committed-websocket-session";
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_websocket_close").await;
create_account(&store, ACCOUNT_ID).await;
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind WebSocket listener");
@@ -1316,6 +1346,20 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
.await
.expect("WebSocket request")
.expect("valid WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.created",
"response": {
"id": "resp_before_committed_close",
"model": "gpt-5.4"
}
})
.to_string()
.into(),
))
.await
.expect("start WebSocket response");
websocket
.send(Message::Text(
json!({
@@ -1345,6 +1389,20 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
.await
.expect("next-turn WebSocket request")
.expect("valid next-turn WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.created",
"response": {
"id": "resp_after_committed_close",
"model": "gpt-5.4"
}
})
.to_string()
.into(),
))
.await
.expect("start next-turn WebSocket response");
websocket
.send(Message::Text(
json!({
@@ -1369,10 +1427,11 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
});
let provider = provider_with_base_url(&store, base_url);
let first_operation = Operation::Generate(generate_with_session_context(
"committed-websocket-session",
Some("thread-first"),
None,
let first_operation = Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
CONVERSATION_ID,
SESSION_ID,
"thread-first",
));
let mut stream = provider
.execute(
@@ -1409,10 +1468,11 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
Some("websocket_close_1009")
);
let second_operation = Operation::Generate(generate_with_session_context(
"committed-websocket-session",
Some("thread-second"),
None,
let second_operation = Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
CONVERSATION_ID,
SESSION_ID,
"thread-second",
));
let mut next_stream = provider
.execute(
@@ -1420,18 +1480,173 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
context("req_after_committed_close", CancellationToken::new()),
)
.await
.expect("post-delivery close must not enable sticky HTTP fallback");
.expect("post-delivery close must allow a fresh WebSocket client retry");
assert_eq!(next_stream.metadata().transport().as_str(), "websocket");
let mut retry_pool_observation = None;
while let Some(event) = next_stream.next().await {
event.expect("next turn WebSocket response");
let event = event.expect("next turn WebSocket response");
if let Some(observation) = event.response_observation()
&& observation.transport().as_str() == "websocket"
{
retry_pool_observation = Some(observation.websocket_pool());
}
}
assert_eq!(retry_pool_observation, Some(None));
server.await.expect("WebSocket server");
}
#[tokio::test]
async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session() {
async fn websocket_fast_path_miss_uses_http_and_keeps_background_preconnect() {
const ACCOUNT_ID: &str = "acct_websocket_fast_path";
const CONVERSATION_ID: &str = "conversation-websocket-fast-path";
const SESSION_ID: &str = "websocket-fast-path-session";
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_websocket_close").await;
create_account(&store, ACCOUNT_ID).await;
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind upstream listener");
let base_url = format!(
"http://{}",
listener.local_addr().expect("listener address")
);
let (preconnect_ready_tx, preconnect_ready_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (websocket_stream, _) = listener
.accept()
.await
.expect("accept background WebSocket opening");
let websocket = tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(1)).await;
let mut websocket = accept_codex_test_websocket(websocket_stream).await;
assert!(
timeout(Duration::from_millis(100), websocket.next())
.await
.is_err(),
"background preconnect must not send the HTTP-fallback request payload"
);
preconnect_ready_tx
.send(())
.expect("signal background WebSocket readiness");
let request = websocket
.next()
.await
.expect("next request should reuse the background WebSocket")
.expect("valid reused WebSocket request");
let payload: Value =
serde_json::from_str(request.to_text().expect("reused WebSocket request text"))
.expect("reused WebSocket request JSON");
assert_eq!(payload.get("type"), Some(&json!("response.create")));
websocket
.send(Message::Text(
json!({
"type": "response.completed",
"response": {
"id": "resp_background_preconnect",
"model": "gpt-5.4",
"status": "completed",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2
}
}
})
.to_string()
.into(),
))
.await
.expect("complete reused WebSocket request");
});
let (mut http, _) = listener
.accept()
.await
.expect("accept HTTP fallback request");
let request = capture_http_request(&mut http).await;
assert!(String::from_utf8_lossy(&request).starts_with("POST /codex/responses"));
http.write_all(
format!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
CAPTURE_COMPLETED_SSE.len()
)
.as_bytes(),
)
.await
.expect("write HTTP fallback response");
websocket.await.expect("background WebSocket server");
});
let provider = provider_with_base_url(&store, base_url);
let operation = |thread_id| {
Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
CONVERSATION_ID,
SESSION_ID,
thread_id,
))
};
let mut first = provider
.execute(
planned_request("openai", operation("thread-http-fallback")),
context("req_fast_path_http", CancellationToken::new()),
)
.await
.expect("prepare first provider stream");
let mut first_transport = None;
let mut first_decision = None;
while let Some(event) = first.next().await {
let event = event.expect("fast-path miss must not surface as a provider error");
if let Some(observation) = event.response_observation() {
first_transport = Some(observation.transport().as_str().to_owned());
if let Some(provider_metadata) = observation.provider_metadata() {
let metadata: Value = serde_json::from_str(provider_metadata.as_json())
.expect("OpenAI provider metadata JSON");
first_decision = metadata
.get("transportDecision")
.and_then(Value::as_str)
.map(ToOwned::to_owned);
}
}
}
assert_eq!(first_transport.as_deref(), Some("http_sse"));
assert_eq!(first_decision.as_deref(), Some("http2_ws_budget_exhausted"));
preconnect_ready_rx
.await
.expect("background WebSocket should become ready");
tokio::time::sleep(Duration::from_millis(20)).await;
let mut second = provider
.execute(
planned_request("openai", operation("thread-websocket-reuse")),
context("req_background_ws_reuse", CancellationToken::new()),
)
.await
.expect("prepare second provider stream");
let mut second_transport = None;
let mut second_pool = None;
while let Some(event) = second.next().await {
let event = event.expect("background WebSocket reuse should complete");
if let Some(observation) = event.response_observation() {
second_transport = Some(observation.transport().as_str().to_owned());
second_pool = Some(observation.websocket_pool());
}
}
assert_eq!(second_transport.as_deref(), Some("websocket"));
assert_eq!(second_pool, Some(Some(WebSocketPoolKind::Reuse)));
server.await.expect("HTTP and WebSocket upstream server");
}
#[tokio::test]
async fn ambiguous_websocket_close_probes_a_fresh_websocket_for_the_next_new_chain() {
const ACCOUNT_ID: &str = "acct_websocket_close";
const CONVERSATION_ID: &str = "conversation-websocket-ambiguous-close";
const SESSION_ID: &str = "sticky-websocket-session";
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, ACCOUNT_ID).await;
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind WebSocket listener");
@@ -1511,19 +1726,31 @@ async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session
.await
.expect("close WebSocket before terminal event");
let (mut http, _) = listener.accept().await.expect("accept sticky HTTP request");
let request = capture_http_request(&mut http).await;
let request_head = String::from_utf8_lossy(&request).to_ascii_lowercase();
assert!(!request_head.contains("upgrade: websocket"));
http.write_all(
format!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
CAPTURE_COMPLETED_SSE.len()
)
.as_bytes(),
)
.await
.expect("write sticky HTTP response");
let (stream, _) = listener
.accept()
.await
.expect("accept fresh WebSocket probe");
let mut websocket = accept_codex_test_websocket(stream).await;
let _request = websocket
.next()
.await
.expect("fresh WebSocket request")
.expect("valid fresh WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.created",
"response": {"id": "resp_fresh_retry", "model": "gpt-5.4"}
})
.to_string()
.into(),
))
.await
.expect("start fresh WebSocket response");
websocket
.send(completed_response("resp_fresh_retry"))
.await
.expect("complete fresh WebSocket request");
let (stream, _) = listener
.accept()
@@ -1568,10 +1795,11 @@ async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session
let provider = provider_with_base_url_and_retry_budget(&store, base_url, 2);
let mut saw_websocket_observation = false;
let operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
Some("thread-first"),
None,
let operation = Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
CONVERSATION_ID,
SESSION_ID,
"thread-first",
));
let mut stream = provider
.execute(
@@ -1640,23 +1868,31 @@ async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session
assert_eq!(connection.exit_reason(), "normal_close");
assert!(connection.age_ms() >= connection.idle_ms());
let second_operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
Some("thread-second"),
None,
let second_operation = Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
CONVERSATION_ID,
SESSION_ID,
"thread-second",
));
let mut sticky_stream = provider
let mut fresh_stream = provider
.execute(
planned_request("openai", second_operation),
context("req_websocket_sticky_http", CancellationToken::new()),
context("req_websocket_fresh_retry", CancellationToken::new()),
)
.await
.expect("same session should prepare an HTTP stream");
assert_eq!(sticky_stream.metadata().transport().as_str(), "http_sse");
while let Some(event) = sticky_stream.next().await {
event.expect("sticky HTTP response");
.expect("same session should prepare a fresh WebSocket stream");
assert_eq!(fresh_stream.metadata().transport().as_str(), "websocket");
let mut fresh_pool_observation = None;
while let Some(event) = fresh_stream.next().await {
let event = event.expect("fresh WebSocket response");
if let Some(observation) = event.response_observation()
&& observation.transport().as_str() == "websocket"
{
fresh_pool_observation = Some(observation.websocket_pool());
}
}
drop(sticky_stream);
assert_eq!(fresh_pool_observation, Some(None));
drop(fresh_stream);
let continuation_operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
@@ -1669,7 +1905,7 @@ async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session
external_continuation_context("req_sticky_external_continuation"),
)
.await
.expect("sticky state must not force a continuation onto HTTP");
.expect("successful fresh probe must restore normal continuation routing");
assert_eq!(
continuation_stream.metadata().transport().as_str(),
"websocket"
@@ -1719,12 +1955,375 @@ async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session
),
)
.await
.expect("required warmup must bypass sticky HTTP fallback");
.expect("required warmup must remain on WebSocket");
assert_eq!(warmup_stream.metadata().transport().as_str(), "websocket");
drop(warmup_stream);
server.await.expect("WebSocket and HTTP server");
}
#[tokio::test]
async fn repeated_ambiguous_websocket_close_uses_steep_sse_cooldowns_and_one_half_open_probe() {
const ACCOUNT_ID: &str = "acct_websocket_cooldown";
const CONVERSATION_ID: &str = "conversation-websocket-cooldown";
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, ACCOUNT_ID).await;
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind WebSocket listener");
let base_url = format!(
"http://{}",
listener.local_addr().expect("listener address")
);
let server = tokio::spawn(async move {
let completed_response = |response_id: &str| {
Message::Text(
json!({
"type": "response.completed",
"response": {
"id": response_id,
"model": "gpt-5.4",
"status": "completed",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2
}
}
})
.to_string()
.into(),
)
};
for response_id in ["resp_first_reset", "resp_fresh_probe_reset"] {
let (stream, _) = listener
.accept()
.await
.expect("accept failing WebSocket connection");
let mut websocket = accept_codex_test_websocket(stream).await;
let _request = websocket
.next()
.await
.expect("WebSocket request")
.expect("valid WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.created",
"response": {"id": response_id, "model": "gpt-5.4"}
})
.to_string()
.into(),
))
.await
.expect("send structural WebSocket event");
websocket
.close(Some(CloseFrame {
code: CloseCode::Normal,
reason: "".into(),
}))
.await
.expect("close WebSocket before terminal event");
}
for response_id in [
"resp_third_reset",
"resp_fourth_reset",
"resp_fifth_reset",
"resp_sixth_reset",
] {
let (mut http, _) = listener
.accept()
.await
.expect("accept cooldown HTTP request");
let request = capture_http_request(&mut http).await;
let request_head = String::from_utf8_lossy(&request).to_ascii_lowercase();
assert!(!request_head.contains("upgrade: websocket"));
http.write_all(
format!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
CAPTURE_COMPLETED_SSE.len()
)
.as_bytes(),
)
.await
.expect("write cooldown HTTP response");
let (stream, _) = listener
.accept()
.await
.expect("accept failing half-open WebSocket connection");
let mut websocket = accept_codex_test_websocket(stream).await;
let _request = websocket
.next()
.await
.expect("half-open WebSocket request")
.expect("valid half-open WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.created",
"response": {"id": response_id, "model": "gpt-5.4"}
})
.to_string()
.into(),
))
.await
.expect("send half-open structural event");
websocket
.close(Some(CloseFrame {
code: CloseCode::Normal,
reason: "".into(),
}))
.await
.expect("close half-open WebSocket before terminal event");
}
for _ in 0..2 {
let (mut http, _) = listener
.accept()
.await
.expect("accept final cooldown HTTP request");
let request = capture_http_request(&mut http).await;
let request_head = String::from_utf8_lossy(&request).to_ascii_lowercase();
assert!(!request_head.contains("upgrade: websocket"));
http.write_all(
format!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
CAPTURE_COMPLETED_SSE.len()
)
.as_bytes(),
)
.await
.expect("write final cooldown HTTP response");
}
for response_id in ["resp_half_open", "resp_recovered"] {
let (stream, _) = listener
.accept()
.await
.expect("accept recovery WebSocket connection");
let mut websocket = accept_codex_test_websocket(stream).await;
let _request = websocket
.next()
.await
.expect("recovery WebSocket request")
.expect("valid recovery WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.created",
"response": {"id": response_id, "model": "gpt-5.4"}
})
.to_string()
.into(),
))
.await
.expect("start recovery WebSocket response");
websocket
.send(completed_response(response_id))
.await
.expect("complete recovery WebSocket request");
}
});
let provider = provider_with_base_url(&store, base_url);
let operation = |thread_id: &str| {
Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
CONVERSATION_ID,
"websocket-cooldown-session",
thread_id,
))
};
let mut first = provider
.execute(
planned_request("openai", operation("thread-first-reset")),
context("req_first_reset", CancellationToken::new()),
)
.await
.expect("prepare first WebSocket stream");
let first_error = loop {
match first.next().await {
Some(Ok(_)) => {}
Some(Err(error)) => break error,
None => panic!("first WebSocket reset must surface a provider error"),
}
};
assert_eq!(first_error.send_state(), UpstreamSendState::Ambiguous);
assert_eq!(first_error.pre_delivery_retry(), None);
drop(first);
let mut fresh = provider
.execute(
planned_request("openai", operation("thread-fresh-reset")),
context("req_fresh_reset", CancellationToken::new()),
)
.await
.expect("prepare fresh WebSocket probe");
assert_eq!(fresh.metadata().transport().as_str(), "websocket");
let mut fresh_pool_observation = None;
let second_error = loop {
match fresh.next().await {
Some(Ok(event)) => {
if let Some(observation) = event.response_observation()
&& observation.transport().as_str() == "websocket"
{
fresh_pool_observation = Some(observation.websocket_pool());
}
}
Some(Err(error)) => break error,
None => panic!("fresh WebSocket reset must surface a provider error"),
}
};
assert_eq!(fresh_pool_observation, Some(None));
assert_eq!(second_error.send_state(), UpstreamSendState::Ambiguous);
assert_eq!(second_error.pre_delivery_retry(), None);
drop(fresh);
tokio::time::pause();
for (index, cooldown) in [
Duration::from_secs(5 * 60),
Duration::from_secs(30 * 60),
Duration::from_secs(60 * 60),
Duration::from_secs(4 * 60 * 60),
]
.into_iter()
.enumerate()
{
tokio::time::advance(cooldown - Duration::from_secs(60)).await;
let mut active_cooldown = provider
.execute(
planned_request(
"openai",
operation(&format!("thread-active-cooldown-{index}")),
),
context(
&format!("req_active_cooldown_{index}"),
CancellationToken::new(),
),
)
.await
.expect("prepare active cooldown HTTP stream");
assert_eq!(active_cooldown.metadata().transport().as_str(), "http_sse");
while let Some(event) = active_cooldown.next().await {
event.expect("active cooldown HTTP response");
}
tokio::time::advance(Duration::from_secs(60)).await;
tokio::time::resume();
let mut failed_probe = provider
.execute(
planned_request("openai", operation(&format!("thread-failed-probe-{index}"))),
context(
&format!("req_failed_probe_{index}"),
CancellationToken::new(),
),
)
.await
.expect("prepare half-open WebSocket probe");
assert_eq!(failed_probe.metadata().transport().as_str(), "websocket");
let mut failed_probe_pool_observation = None;
let probe_error = loop {
match failed_probe.next().await {
Some(Ok(event)) => {
if let Some(observation) = event.response_observation()
&& observation.transport().as_str() == "websocket"
{
failed_probe_pool_observation = Some(observation.websocket_pool());
}
}
Some(Err(error)) => break error,
None => panic!("half-open WebSocket reset must surface a provider error"),
}
};
assert_eq!(failed_probe_pool_observation, Some(None));
assert_eq!(probe_error.send_state(), UpstreamSendState::Ambiguous);
assert_eq!(probe_error.pre_delivery_retry(), None);
drop(failed_probe);
tokio::time::pause();
}
tokio::time::advance(Duration::from_secs(4 * 60 * 60 - 60)).await;
let mut capped_cooldown = provider
.execute(
planned_request("openai", operation("thread-capped-cooldown")),
context("req_capped_cooldown", CancellationToken::new()),
)
.await
.expect("prepare capped cooldown HTTP stream");
assert_eq!(capped_cooldown.metadata().transport().as_str(), "http_sse");
while let Some(event) = capped_cooldown.next().await {
event.expect("capped cooldown HTTP response");
}
tokio::time::advance(Duration::from_secs(60)).await;
tokio::time::resume();
let mut half_open = provider
.execute(
planned_request("openai", operation("thread-half-open")),
context("req_half_open", CancellationToken::new()),
)
.await
.expect("prepare half-open WebSocket probe");
assert_eq!(half_open.metadata().transport().as_str(), "websocket");
let mut concurrent = provider
.execute(
planned_request("openai", operation("thread-probe-busy")),
context("req_probe_busy", CancellationToken::new()),
)
.await
.expect("prepare concurrent HTTP stream");
assert_eq!(concurrent.metadata().transport().as_str(), "http_sse");
while let Some(event) = concurrent.next().await {
event.expect("probe-busy HTTP response");
}
let mut half_open_pool_observation = None;
let mut half_open_completed = false;
while let Some(event) = half_open.next().await {
let event = event.expect("half-open WebSocket response");
half_open_completed |= event
.canonical_facts()
.iter()
.any(|event| matches!(event, GatewayEvent::Completed(_)));
if let Some(observation) = event.response_observation()
&& observation.transport().as_str() == "websocket"
{
half_open_pool_observation = Some(observation.websocket_pool());
}
}
assert!(half_open_completed);
assert_eq!(half_open_pool_observation, Some(None));
let mut recovered = provider
.execute(
planned_request("openai", operation("thread-recovered")),
context("req_recovered", CancellationToken::new()),
)
.await
.expect("prepare recovered WebSocket stream");
assert_eq!(recovered.metadata().transport().as_str(), "websocket");
let mut recovered_pool_observation = None;
while let Some(event) = recovered.next().await {
let event = event.expect("recovered WebSocket response");
if let Some(observation) = event.response_observation()
&& observation.transport().as_str() == "websocket"
{
recovered_pool_observation = Some(observation.websocket_pool());
}
}
assert_eq!(
recovered_pool_observation,
Some(Some(WebSocketPoolKind::New))
);
server.await.expect("WebSocket and HTTP recovery server");
}
#[tokio::test]
async fn websocket_upgrade_required_immediately_enables_session_http_fallback() {
let store = Arc::new(MemoryAccountStore::default());
@@ -1925,6 +2524,114 @@ async fn websocket_turn_state_metadata_is_exposed_through_response_observation()
assert!(observed_turn_state);
}
#[tokio::test]
async fn websocket_turn_state_metadata_close_requests_checkpointed_retry() {
const ACCOUNT_ID: &str = "acct_websocket_metadata_close";
const TURN_STATE: &str = "turn-state-before-close";
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, ACCOUNT_ID).await;
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind WebSocket listener");
let base_url = format!(
"http://{}",
listener.local_addr().expect("listener address")
);
let server = tokio::spawn(async move {
let (stream, _) = listener
.accept()
.await
.expect("accept WebSocket connection");
let mut websocket = accept_codex_test_websocket(stream).await;
let _request = websocket
.next()
.await
.expect("WebSocket request")
.expect("valid WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "codex.response.metadata",
"headers": {"x-codex-turn-state": TURN_STATE}
})
.to_string()
.into(),
))
.await
.expect("send response metadata");
websocket
.close(Some(CloseFrame {
code: CloseCode::Normal,
reason: "".into(),
}))
.await
.expect("close before terminal response");
});
let operation = Operation::Generate(generate_with_persisted_session_context(
ACCOUNT_ID,
"conversation-metadata-close",
"session-metadata-close",
"turn-metadata-close",
));
let mut stream = provider_with_base_url(&store, base_url)
.execute(
planned_request("openai", operation),
context("req_websocket_metadata_close", CancellationToken::new()),
)
.await
.expect("prepare WebSocket provider stream");
let mut observed_turn_state = false;
let mut replay_checkpoint = None;
let error = loop {
match stream.next().await {
Some(Ok(event)) => {
if let Some(observation) = event.response_observation() {
observed_turn_state |= observation.client_headers().iter().any(|header| {
header.name().eq_ignore_ascii_case("x-codex-turn-state")
&& header.value().as_ref() == TURN_STATE.as_bytes()
});
}
if let Some(update) = event.session_update() {
replay_checkpoint = Some(update.clone());
}
}
Some(Err(error)) => break error,
None => panic!("metadata close must surface a provider error"),
}
};
server.await.expect("WebSocket server");
assert!(observed_turn_state);
let replay_checkpoint = replay_checkpoint.expect("metadata close checkpoint");
assert_eq!(
replay_checkpoint.payload().get("turn_state"),
Some(&json!(TURN_STATE))
);
assert_eq!(
replay_checkpoint.payload().get("retry_checkpoint"),
Some(&json!(true))
);
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
assert!(error.replay_is_safe());
assert!(matches!(
error.pre_delivery_retry(),
Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, .. })
if retry_index.get() == 1
));
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
Some("websocket_close_1000")
);
assert_eq!(
error.diagnostic().map(|diagnostic| diagnostic.as_str()),
Some(
"OpenAI WebSocket closed before a terminal response (close code 1000); last event type: codex.response.metadata"
)
);
}
#[tokio::test]
async fn disabled_account_is_excluded_from_normal_scheduling() {
let store = Arc::new(MemoryAccountStore::default());
@@ -2351,6 +3058,7 @@ async fn matching_turn_id_should_restore_previous_turn_state() {
Some("turn-same"),
Some("turn-same"),
None,
false,
)
.await;
@@ -2361,30 +3069,58 @@ async fn matching_turn_id_should_restore_previous_turn_state() {
}
#[tokio::test]
async fn matching_turn_id_should_prefer_the_latest_provider_state_over_a_stale_client_echo() {
async fn retry_checkpoint_should_restore_turn_state_without_client_turn_id() {
let request =
capture_turn_state_request("req_restore_retry_checkpoint", None, None, None, true).await;
assert_eq!(
captured_header_values(&request, "x-codex-turn-state"),
vec![b"previous-turn-state".to_vec()]
);
}
#[tokio::test]
async fn matching_turn_id_should_prefer_an_explicit_client_echo_over_saved_provider_state() {
let request = capture_turn_state_request(
"req_replace_stale_client_turn_state",
"req_use_client_turn_state",
Some("turn-same"),
Some("turn-same"),
Some("stale-client-turn-state"),
Some("client-turn-state"),
false,
)
.await;
assert_eq!(
captured_header_values(&request, "x-codex-turn-state"),
vec![b"previous-turn-state".to_vec()]
vec![b"client-turn-state".to_vec()]
);
}
#[tokio::test]
async fn new_or_unidentified_turn_should_not_restore_previous_turn_state() {
for (request_id, previous_turn_id, current_turn_id) in [
("req_new_turn_state", Some("turn-old"), Some("turn-new")),
("req_unidentified_turn_state", None, Some("turn-new")),
("req_missing_current_turn_state", Some("turn-old"), None),
for (request_id, previous_turn_id, current_turn_id, client_turn_state) in [
(
"req_new_turn_state",
Some("turn-old"),
Some("turn-new"),
Some("stale-client-turn-state"),
),
("req_unidentified_turn_state", None, Some("turn-new"), None),
(
"req_missing_current_turn_state",
Some("turn-old"),
None,
None,
),
] {
let request =
capture_turn_state_request(request_id, previous_turn_id, current_turn_id, None).await;
let request = capture_turn_state_request(
request_id,
previous_turn_id,
current_turn_id,
client_turn_state,
false,
)
.await;
assert!(captured_header_values(&request, "x-codex-turn-state").is_empty());
}
}
@@ -295,8 +295,8 @@ fn new_chain_should_allow_pre_delivery_http_fallback() {
}
#[test]
fn sticky_http_fallback_should_only_apply_to_new_chains() {
assert!(TransportRequirement::NewChain.allows_sticky_http_fallback());
fn session_transport_recovery_should_only_apply_to_new_chains() {
assert!(TransportRequirement::NewChain.allows_session_transport_recovery());
for requirement in [
TransportRequirement::HttpRequired,
@@ -306,8 +306,8 @@ fn sticky_http_fallback_should_only_apply_to_new_chains() {
TransportRequirement::ExternalUnknown,
] {
assert!(
!requirement.allows_sticky_http_fallback(),
"sticky HTTP must not override {}",
!requirement.allows_session_transport_recovery(),
"session recovery must not override {}",
requirement.as_str()
);
}
@@ -155,6 +155,7 @@ fn websocket_event_to_sse_should_forward_public_events_and_strip_internal_events
);
assert!(websocket_event_to_sse_frame(r#"{"type":"codex.rate_limits"}"#).is_none());
assert!(websocket_event_to_sse_frame(r#"{"type":"response.metadata"}"#).is_none());
assert!(websocket_event_to_sse_frame(r#"{"type":"codex.response.metadata"}"#).is_some());
assert!(websocket_event_to_sse_frame(r#"{"response":{}}"#).is_none());
assert!(websocket_event_to_sse_frame("not-json").is_none());
}
@@ -178,15 +179,48 @@ fn websocket_event_to_sse_should_add_missing_previous_response_recovery_code() {
#[test]
fn websocket_metadata_turn_state_should_accept_case_insensitive_header() {
let event = json!({
"type": "response.metadata",
"headers": {"X-Codex-Turn-State": ["turn-from-metadata"]}
});
for (event, expected) in [
(
json!({
"type": "response.metadata",
"headers": {"X-Codex-Turn-State": ["turn-from-legacy-metadata"]}
}),
"turn-from-legacy-metadata",
),
(
json!({
"type": "codex.response.metadata",
"headers": {"X-Codex-Turn-State": "turn-from-codex-metadata"}
}),
"turn-from-codex-metadata",
),
] {
assert_eq!(
websocket_metadata_turn_state(&event).as_deref(),
Some(expected)
);
}
}
assert_eq!(
websocket_metadata_turn_state(&event).as_deref(),
Some("turn-from-metadata")
);
#[test]
fn websocket_metadata_turn_state_should_ignore_unverified_shapes() {
for event in [
json!({
"type": "codex.response.metadata",
"headers": {"x-codex-turn-state": {"future": "shape"}}
}),
json!({
"type": "codex.response.metadata",
"metadata": {"turn_state": "nested-value"},
"turn_state": "top-level-value"
}),
json!({
"type": "future.response.metadata",
"headers": {"x-codex-turn-state": "unverified-event"}
}),
] {
assert_eq!(websocket_metadata_turn_state(&event), None);
}
}
#[test]
@@ -227,6 +227,26 @@ fn encoder_should_ignore_legacy_context_aliases_and_metadata_fallbacks() {
);
}
#[test]
fn encoder_should_extract_official_websocket_context_projection() {
let request = request(Map::from_iter([
("model".to_owned(), json!("client-model")),
("input".to_owned(), json!("prompt")),
(
"client_metadata".to_owned(),
json!({
"x-codex-turn-state": "metadata-turn-state",
"turn_id": "metadata-turn-id"
}),
),
]));
let encoded = encode_generate_request(&request, "gpt-test").expect("encode");
assert_eq!(encoded.turn_state.as_deref(), Some("metadata-turn-state"));
assert_eq!(encoded.client_turn_id.as_deref(), Some("metadata-turn-id"));
}
#[test]
fn header_context_should_win_over_body_topline_aliases() {
let payload = ProtocolPayload::json_object(
@@ -236,6 +256,13 @@ fn header_context_should_win_over_body_topline_aliases() {
("input".to_owned(), json!("prompt")),
("turnState".to_owned(), json!("body-turn-state")),
("turnMetadata".to_owned(), json!("body-turn-metadata")),
(
"client_metadata".to_owned(),
json!({
"x-codex-turn-state": "metadata-turn-state",
"turn_id": "metadata-turn-id"
}),
),
]),
)
.expect("OpenAI payload")
@@ -248,6 +275,10 @@ fn header_context_should_win_over_body_topline_aliases() {
"turn_metadata".to_owned(),
Value::String("header-turn-metadata".to_owned()),
),
(
"turn_id".to_owned(),
Value::String("header-turn-id".to_owned()),
),
]));
let request = GenerateRequest::from_protocol_payload(payload);
@@ -258,6 +289,36 @@ fn header_context_should_win_over_body_topline_aliases() {
encoded.turn_metadata.as_deref(),
Some("header-turn-metadata")
);
assert_eq!(encoded.client_turn_id.as_deref(), Some("header-turn-id"));
}
#[test]
fn encoder_should_preserve_downstream_websocket_connection_identity_outside_wire_body() {
let payload = ProtocolPayload::json_object(
"openai",
Map::from_iter([
("model".to_owned(), json!("client-model")),
("input".to_owned(), json!("prompt")),
]),
)
.expect("OpenAI payload")
.with_context(Map::from_iter([(
"downstream_websocket_connection_id".to_owned(),
Value::String("ws_downstream_a".to_owned()),
)]));
let request = GenerateRequest::from_protocol_payload(payload);
let encoded = encode_generate_request(&request, "gpt-test").expect("encode");
assert_eq!(
encoded.downstream_websocket_connection_id.as_deref(),
Some("ws_downstream_a")
);
assert!(
!encoded
.body()
.contains_key("downstream_websocket_connection_id")
);
}
#[test]
@@ -914,6 +914,191 @@ async fn websocket_execute_response_create_request_should_capture_internal_metad
);
}
#[tokio::test]
async fn websocket_codex_metadata_should_capture_first_turn_state_without_changing_forwarding() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut websocket = accept_codex_test_websocket(stream).await;
let _message = websocket.next().await.unwrap().unwrap();
websocket
.send(Message::Text(
json!({
"type": "codex.response.metadata",
"headers": {"x-codex-turn-state": "turn-first"}
})
.to_string()
.into(),
))
.await
.unwrap();
websocket
.send(Message::Text(
json!({
"type": "response.metadata",
"headers": {"x-codex-turn-state": ["turn-second"]}
})
.to_string()
.into(),
))
.await
.unwrap();
websocket
.send(Message::Text(
completed_websocket_response("resp_metadata_alias", 1, 1).into(),
))
.await
.unwrap();
});
let prepared = prepared_websocket_request(&format!("http://{addr}"));
let response = execute_response_create_request(&prepared)
.await
.expect("metadata aliases should preserve a completed response");
server.await.unwrap();
assert_eq!(response.turn_state.as_deref(), Some("turn-first"));
assert!(response.body.contains("event: codex.response.metadata"));
assert!(!response.body.contains("event: response.metadata\n"));
assert!(response.body.contains("event: response.completed"));
}
#[tokio::test]
async fn reused_websocket_should_not_leak_turn_state_into_the_next_response() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut websocket = accept_codex_test_websocket(stream).await;
let _first = websocket.next().await.unwrap().unwrap();
websocket
.send(Message::Text(
json!({
"type": "codex.response.metadata",
"headers": {"x-codex-turn-state": "turn-from-first-response"}
})
.to_string()
.into(),
))
.await
.unwrap();
websocket
.send(Message::Text(
completed_websocket_response("resp_turn_state_first", 1, 1).into(),
))
.await
.unwrap();
let _second = websocket.next().await.unwrap().unwrap();
websocket
.send(Message::Text(
completed_websocket_response("resp_turn_state_second", 1, 1).into(),
))
.await
.unwrap();
});
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
)
.with_websocket_pool(pool);
let request = pooled_websocket_request("conversation-turn-state-reset");
let first = backend
.create_response(
&request,
request_context("req_turn_state_first", Some("chatgpt-account")),
)
.await
.expect("first response should complete");
let second = backend
.create_response(
&request,
request_context("req_turn_state_second", Some("chatgpt-account")),
)
.await
.expect("second response should reuse the WebSocket");
server.await.unwrap();
assert_eq!(
first.turn_state.as_deref(),
Some("turn-from-first-response")
);
assert!(
second
.websocket_pool_decision
.is_some_and(WebSocketPoolDecision::is_reuse)
);
assert_eq!(second.turn_state, None);
assert!(
second
.response_metadata
.client_headers
.iter()
.all(|(name, _)| { !name.eq_ignore_ascii_case("x-codex-turn-state") })
);
}
#[tokio::test]
async fn websocket_request_should_project_trusted_turn_state_and_remove_stale_metadata() {
for (case, turn_state) in [("trusted", Some("trusted-turn-state")), ("cleared", None)] {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut websocket = accept_codex_test_websocket(stream).await;
let message = websocket.next().await.unwrap().unwrap();
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
.expect("response.create payload");
websocket
.send(Message::Text(
completed_websocket_response("resp_turn_state_projection", 1, 1).into(),
))
.await
.unwrap();
payload
});
let mut body = codex_request_body("gpt-5.5", "be brief", Vec::new());
body.insert("generate".to_owned(), json!(false));
body.insert("store".to_owned(), json!(false));
let mut request = CodexResponsesRequest::from_body(body);
request.turn_state = turn_state.map(str::to_owned);
request.local_conversation_id = Some(format!("turn-state-projection-{case}"));
request.set_client_metadata(Some(json!({
"x-codex-turn-state": "stale-client-value",
"keep": "preserved"
})));
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
)
.with_websocket_pool(Arc::new(CodexWebSocketPool::new(Duration::from_mins(1))));
let mut context = request_context(case, Some("chatgpt-account"));
context.turn_state = turn_state;
let response = backend
.create_response(&request, context)
.await
.unwrap_or_else(|error| panic!("required WebSocket request should complete: {error}"));
let payload = server.await.unwrap();
assert_eq!(response.transport, CodexBackendTransport::WebSocket);
assert_eq!(
payload.pointer("/client_metadata/keep"),
Some(&json!("preserved"))
);
assert_eq!(
payload.pointer("/client_metadata/x-codex-turn-state"),
turn_state.map(|turn_state| json!(turn_state)).as_ref(),
"case {case}"
);
}
}
#[tokio::test]
async fn codex_backend_client_should_not_reuse_websocket_rate_limit_headers() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -71,6 +71,138 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn websocket_pool_should_isolate_concurrent_downstream_lanes_for_one_conversation() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let accepted_connections = Arc::new(AtomicUsize::new(0));
let accepted_connections_for_server = Arc::clone(&accepted_connections);
let (first_accepted_tx, first_accepted_rx) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
let (first_stream, _) = listener.accept().await.unwrap();
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
let mut first = accept_codex_test_websocket(first_stream).await;
first_accepted_tx.send(()).unwrap();
let (second_stream, _) = timeout(Duration::from_secs(2), listener.accept())
.await
.expect("the second downstream lane should open its own upstream websocket")
.unwrap();
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
let mut second = accept_codex_test_websocket(second_stream).await;
let (first_warmup, second_warmup) = tokio::join!(first.next(), second.next());
first_warmup.unwrap().unwrap();
second_warmup.unwrap().unwrap();
first
.send(Message::Text(
completed_websocket_response("resp_lane_a", 1, 0).into(),
))
.await
.unwrap();
second
.send(Message::Text(
completed_websocket_response("resp_lane_b", 1, 0).into(),
))
.await
.unwrap();
let _first_continuation = first.next().await.unwrap().unwrap();
first
.send(Message::Text(
completed_websocket_response("resp_lane_a_next", 2, 1).into(),
))
.await
.unwrap();
let _second_continuation = second.next().await.unwrap().unwrap();
second
.send(Message::Text(
completed_websocket_response("resp_lane_b_next", 2, 1).into(),
))
.await
.unwrap();
first.close(None).await.unwrap();
second.close(None).await.unwrap();
});
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
)
.with_websocket_pool(Arc::clone(&pool));
let warmup = |connection_id: &str| {
let mut body = Map::new();
body.insert("model".to_owned(), json!("gpt-5.5"));
body.insert("input".to_owned(), json!([]));
body.insert("generate".to_owned(), json!(false));
body.insert("store".to_owned(), json!(false));
let mut request = CodexResponsesRequest::from_body(body);
request.use_websocket = true;
request.local_conversation_id = Some("conversation-lanes".to_owned());
request.downstream_websocket_connection_id = Some(connection_id.to_owned());
request
};
let first_backend = backend.clone();
let first_request = warmup("ws_downstream_a");
let first = tokio::spawn(async move {
first_backend
.create_response(
&first_request,
request_context("req_lane_a_warmup", Some("chatgpt-account")),
)
.await
});
first_accepted_rx.await.unwrap();
let second = backend
.create_response(
&warmup("ws_downstream_b"),
request_context("req_lane_b_warmup", Some("chatgpt-account")),
)
.await
.expect("the second downstream lane should not contend with the first warmup");
let first = first.await.unwrap().unwrap();
let mut first_continuation = pooled_websocket_request("conversation-lanes");
first_continuation.downstream_websocket_connection_id = Some("ws_downstream_a".to_owned());
first_continuation.set_previous_response_id(Some("resp_lane_a".to_owned()));
first_continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let first_continuation = backend
.create_response(
&first_continuation,
request_context("req_lane_a_next", Some("chatgpt-account")),
)
.await
.expect("the first lane should retain its connection-local response owner");
let mut second_continuation = pooled_websocket_request("conversation-lanes");
second_continuation.downstream_websocket_connection_id = Some("ws_downstream_b".to_owned());
second_continuation.set_previous_response_id(Some("resp_lane_b".to_owned()));
second_continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let second_continuation = backend
.create_response(
&second_continuation,
request_context("req_lane_b_next", Some("chatgpt-account")),
)
.await
.expect("the second lane should retain its connection-local response owner");
server.await.unwrap();
assert_eq!(first.websocket_pool_decision.unwrap().kind(), "new");
assert_eq!(second.websocket_pool_decision.unwrap().kind(), "new");
assert_eq!(
first_continuation.websocket_pool_decision.unwrap().kind(),
"reuse"
);
assert_eq!(
second_continuation.websocket_pool_decision.unwrap().kind(),
"reuse"
);
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
pool.shutdown().await;
}
#[tokio::test]
async fn exact_continuation_reuses_owning_socket_after_connection_profile_updates() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+1
View File
@@ -3,3 +3,4 @@
1b6ffab434c754814aebb0eee3b650bd9770010059472bccf5bc856b319880ca 0003_client_min_versions.sql
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
3b4bfd5e3f5135fa2fad1088b0c2a287861e1250d66af31dbd40dc890751b567 0006_openai_session_transport_recovery.sql
@@ -0,0 +1,61 @@
create index model_requests_pending_ws_transport_recovery_idx
on model_requests (
client_api_key_ref,
continuation_affinity_hash,
completed_at desc,
id desc
)
where provider_kind = 'openai'
and outcome = 'failed'
and error_kind = 'upstream_unavailable'
and upstream_transport = 'websocket'
and upstream_send_state = 'ambiguous'
and downstream_committed_at is null
and recovered_at is null
and continuation_affinity_hash is not null;
with session_transport_recoveries as (
select
prior.id as failed_request_id,
recovered.id as recovery_request_id,
recovered.started_at as recovery_started_at,
recovered.completed_at as recovered_at
from model_requests prior
cross join lateral (
select current.id, current.started_at, current.completed_at
from model_requests current
where current.client_api_key_ref = prior.client_api_key_ref
and current.continuation_affinity_hash = prior.continuation_affinity_hash
and current.provider_kind = prior.provider_kind
and current.outcome = 'succeeded'
and current.upstream_transport = 'http_sse'
and current.completed_at is not null
and current.started_at >= prior.completed_at
and current.started_at <= prior.completed_at + interval '30 seconds'
order by current.started_at, current.id
limit 1
) recovered
where prior.provider_kind = 'openai'
and prior.outcome = 'failed'
and prior.error_kind = 'upstream_unavailable'
and prior.upstream_transport = 'websocket'
and prior.upstream_send_state = 'ambiguous'
and prior.downstream_committed_at is null
and prior.recovered_at is null
and prior.continuation_affinity_hash is not null
)
update model_requests prior
set recovery_attempt_count = prior.recovery_attempt_count + 1,
recovery_request_id = recovery.recovery_request_id,
recovered_at = recovery.recovered_at,
recovery_retry_delay_ms = greatest(
0,
floor(extract(epoch from (recovery.recovery_started_at - prior.completed_at)) * 1000)
)::bigint,
recovery_total_latency_ms = greatest(
0,
floor(extract(epoch from (recovery.recovered_at - prior.completed_at)) * 1000)
)::bigint
from session_transport_recoveries recovery
where prior.id = recovery.failed_request_id
and prior.recovered_at is null;
+2
View File
@@ -20,6 +20,8 @@ host:
logging:
level: 'info'
stdout: true
# 将完整请求头和正文写入独立文件;包含凭据,仅排障时开启。
request_dump: false
file:
enabled: true
# 相对路径以 deploy/config.yaml 所在目录为基准。
+2
View File
@@ -83,6 +83,8 @@ export interface AccountUsage {
outputTokensDisplay: string
cachedTokens: number | null
cachedTokensDisplay: string
reasoningTokens: number | null
reasoningTokensDisplay: string
imageInputTokens: number | null
imageInputTokensDisplay: string
imageOutputTokens: number | null
+2 -1
View File
@@ -45,7 +45,8 @@ const thumbClasses = computed(() => [
model.value ? 'left-[calc(100%-1.375rem)]' : 'left-0.5',
])
const inlineTextClasses = computed(() => [
'pointer-events-none absolute inset-0 flex min-w-0 select-none items-center justify-center overflow-hidden text-[10px] leading-none font-heavy text-cp-white transition-[padding] duration-180 motion-reduce:transition-none',
'pointer-events-none absolute inset-0 flex min-w-0 select-none items-center justify-center overflow-hidden text-[10px] leading-none font-heavy transition-[padding,color] duration-180 motion-reduce:transition-none',
model.value ? 'text-cp-white' : 'text-cp-text-secondary',
model.value ? 'pr-6 pl-1' : 'pr-1 pl-6',
])
const inactiveTextClasses = computed(() => [
@@ -50,7 +50,7 @@ const additionalEntryCount = computed(() => Math.max(summaryEntries.value.length
class="flex shrink-0 items-baseline gap-1 text-[9px] font-emphasis text-cp-text-quaternary"
title="最近使用额度的当前已用比例"
>
<span>消耗比例</span>
<span>使用率</span>
<strong class="font-mono font-heavy tabular-nums" :class="currentUsageTextClass">
{{ currentUsageDisplay }}
</strong>
@@ -35,33 +35,33 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
<div class="xl:flex xl:min-h-0 xl:flex-col">
<div class="mb-3 flex shrink-0 items-baseline justify-between gap-3">
<h3 class="m-0 text-cp-lg font-heavy text-cp-text">
Token 结构
Tokens 结构
</h3>
<span class="text-cp-xs font-emphasis text-cp-text-quaternary">当前额度窗口</span>
</div>
<div class="grid gap-2 xl:min-h-0 xl:flex-1 xl:grid-rows-5">
<div class="flex items-center justify-between rounded-lg bg-cp-green-bg px-3 py-2">
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入 Tokens</span>
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入</span>
<strong class="font-mono text-cp text-cp-text">
{{ account.usage.inputTokensDisplay }}
</strong>
</div>
<div class="flex items-center justify-between rounded-lg bg-cp-orange-bg px-3 py-2">
<span class="text-cp-sm font-bold text-cp-orange-text-on-bg">输出 Tokens</span>
<span class="text-cp-sm font-bold text-cp-orange-text-on-bg">输出</span>
<strong class="font-mono text-cp text-cp-text">
{{ account.usage.outputTokensDisplay }}
</strong>
</div>
<div class="flex items-center justify-between rounded-lg bg-cp-cyan-bg px-3 py-2">
<span class="text-cp-sm font-bold text-cp-cyan-text-on-bg">缓存 Tokens</span>
<span class="text-cp-sm font-bold text-cp-cyan-text-on-bg">缓存</span>
<strong class="font-mono text-cp text-cp-text">
{{ account.usage.cachedTokensDisplay }}
</strong>
</div>
<div class="flex items-center justify-between rounded-lg bg-cp-blue-bg px-3 py-2">
<span class="text-cp-sm font-bold text-cp-blue-text-on-bg">创建</span>
<span class="text-cp-sm font-bold text-cp-blue-text-on-bg">推理</span>
<strong class="font-mono text-cp text-cp-text">
{{ account.usage.createdTokensDisplay }}
{{ account.usage.reasoningTokensDisplay }}
</strong>
</div>
<div class="flex items-center justify-between rounded-lg bg-cp-blue-bg px-3 py-2">
@@ -8,6 +8,10 @@ import BaseModal from '@/components/base/BaseModal/index.vue'
import BaseScrollbar from '@/components/base/BaseScrollbar.vue'
import BaseSegmented from '@/components/base/BaseSegmented.vue'
import BaseSwitch from '@/components/base/BaseSwitch.vue'
import {
buildCodexConfigFiles,
CODEX_WEBSOCKET_ENABLED_BY_DEFAULT,
} from '../utils/codexConfig'
const props = defineProps<{
apiKey: {
@@ -24,7 +28,7 @@ const emit = defineEmits<{
const open = defineModel<boolean>({ default: false })
const activePlatform = shallowRef('unix')
const websocketEnabled = shallowRef(true)
const websocketEnabled = shallowRef(CODEX_WEBSOCKET_ENABLED_BY_DEFAULT)
const platformOptions = [
{ label: 'macOS / Linux', value: 'unix', icon: Apple },
@@ -40,31 +44,15 @@ const configPath = computed(() =>
const authPath = computed(() =>
activePlatform.value === 'windows' ? '%userprofile%\\.codex\\auth.json' : '~/.codex/auth.json',
)
const codexAuthJson = computed(() => JSON.stringify({ OPENAI_API_KEY: keyValue.value }, null, 2))
const defaultModel = 'gpt-5.6-terra'
const codexConfigToml = computed(
() => `model_provider = "OpenAI"
model = "${defaultModel}"
review_model = "${defaultModel}"
model_reasoning_effort = "max"
service_tier = "default"
disable_response_storage = true
network_access = "enabled"
[model_providers.OpenAI]
name = "OpenAI"
base_url = "${props.apiBaseUrl}"
wire_api = "responses"${websocketEnabled.value ? '\nsupports_websockets = true' : ''}
requires_openai_auth = true
[features]${websocketEnabled.value ? '\nresponses_websockets_v2 = true' : ''}
goals = true`,
)
const codexConfigFiles = computed(() => buildCodexConfigFiles({
apiKey: keyValue.value,
baseUrl: props.apiBaseUrl,
websocketEnabled: websocketEnabled.value,
}))
const visibleFiles = computed(() => [
{ path: configPath.value, content: codexConfigToml.value, scrollbarHeight: '360px' },
{ path: authPath.value, content: codexAuthJson.value, scrollbarHeight: undefined },
{ path: configPath.value, content: codexConfigFiles.value.configToml, scrollbarHeight: '360px' },
{ path: authPath.value, content: codexConfigFiles.value.authJson, scrollbarHeight: undefined },
])
</script>
@@ -1,4 +1,8 @@
const DEFAULT_CC_SWITCH_MODEL = 'gpt-5.5'
import {
buildCodexConfigFiles,
CODEX_DEFAULT_MODEL,
CODEX_WEBSOCKET_ENABLED_BY_DEFAULT,
} from './codexConfig'
export interface CodexCcSwitchImportInput {
apiKey: string
@@ -7,18 +11,37 @@ export interface CodexCcSwitchImportInput {
}
export function buildCodexCcSwitchImportDeeplink(input: CodexCcSwitchImportInput): string {
const baseUrl = input.baseUrl.replace(/\/+$/, '')
const configFiles = buildCodexConfigFiles({
apiKey: input.apiKey,
baseUrl: input.baseUrl,
websocketEnabled: CODEX_WEBSOCKET_ENABLED_BY_DEFAULT,
})
// Carry the complete files shown in the UI; the explicit fields below keep
// compatibility with CCSwitch versions that only extract common settings.
const config = encodeBase64(JSON.stringify({
auth: configFiles.auth,
config: configFiles.configToml,
}))
const entries: [string, string][] = [
['resource', 'provider'],
['app', 'codex'],
['model', DEFAULT_CC_SWITCH_MODEL],
['model', CODEX_DEFAULT_MODEL],
['name', input.providerName],
['homepage', baseUrl],
['endpoint', baseUrl],
['homepage', configFiles.baseUrl],
['endpoint', configFiles.baseUrl],
['apiKey', input.apiKey],
['configFormat', 'json'],
['config', config],
['usageEnabled', 'false'],
]
return `ccswitch://v1/import?${new URLSearchParams(entries).toString()}`
}
function encodeBase64(value: string) {
const bytes = new TextEncoder().encode(value)
let binary = ''
for (const byte of bytes)
binary += String.fromCharCode(byte)
return btoa(binary)
}
@@ -0,0 +1,37 @@
export const CODEX_DEFAULT_MODEL = 'gpt-5.6-terra'
export const CODEX_WEBSOCKET_ENABLED_BY_DEFAULT = false
export interface CodexConfigInput {
apiKey: string
baseUrl: string
websocketEnabled?: boolean
}
export function buildCodexConfigFiles(input: CodexConfigInput) {
const baseUrl = input.baseUrl.replace(/\/+$/, '')
const websocketEnabled = input.websocketEnabled ?? CODEX_WEBSOCKET_ENABLED_BY_DEFAULT
const auth = { OPENAI_API_KEY: input.apiKey }
const configToml = `model_provider = "OpenAI"
model = "${CODEX_DEFAULT_MODEL}"
review_model = "${CODEX_DEFAULT_MODEL}"
model_reasoning_effort = "max"
service_tier = "default"
disable_response_storage = true
network_access = "enabled"
[model_providers.OpenAI]
name = "OpenAI"
base_url = "${baseUrl}"
wire_api = "responses"${websocketEnabled ? '\nsupports_websockets = true' : ''}
requires_openai_auth = true
[features]${websocketEnabled ? '\nresponses_websockets_v2 = true' : ''}
goals = true`
return {
auth,
authJson: JSON.stringify(auth, null, 2),
baseUrl,
configToml,
}
}
@@ -31,6 +31,10 @@ const accountLabel = computed(() => props.record?.accountEmail
|| props.record?.accountId)
const errorFields = computed(() => visibleFields([
{
label: '处理状态',
value: props.record?.metadata.recoveredAt ? '已自动恢复' : null,
},
{ label: 'Provider 错误码', value: props.record?.providerErrorCode, mono: true },
{ label: '失败分类', value: failureClassText(props.record?.failureClass) },
{ label: '分类代码', value: props.record?.failureClass, mono: true },
@@ -124,9 +124,17 @@ function upstreamSendStateText(value: string | null | undefined) {
</template>
<template #message="{ row }">
<div class="min-w-0 py-0.5" :title="row.message || opsErrorSummary(row)">
<code class="block max-w-full truncate font-mono text-cp-sm font-bold text-cp-error-text">
{{ opsErrorSummary(row) }}
</code>
<div class="flex min-w-0 items-center gap-2">
<code class="block min-w-0 flex-1 truncate font-mono text-cp-sm font-bold text-cp-error-text">
{{ opsErrorSummary(row) }}
</code>
<span
v-if="row.metadata.recoveredAt"
class="inline-flex h-5 shrink-0 items-center rounded-full bg-cp-success-bg px-2 text-cp-xs leading-none font-heavy text-cp-success-text"
>
已自动恢复
</span>
</div>
<p
v-if="row.message"
class="mt-1 mb-0 line-clamp-1 text-cp-xs leading-[1.45] font-emphasis text-cp-text-secondary"
@@ -19,7 +19,7 @@ const search = defineModel<string>('search', { required: true })
<template>
<div class="flex w-full items-center gap-3" role="group" aria-label="使用记录筛选与操作">
<div class="min-w-0 flex-1 sm:w-96 sm:flex-none">
<BaseInput v-model="search" placeholder="请求、Key、账号或模型 ID 前缀" class="w-full">
<BaseInput v-model="search" placeholder="请求、API Key、账号或模型 ID 前缀" class="w-full">
<template #prefix>
<Search class="size-4.5 text-cp-text-tertiary" />
</template>
@@ -21,6 +21,8 @@ interface UseUsageRecordsTableOptions {
type UsageLoadScope = 'all' | 'table'
const CLIENT_API_KEY_VISIBLE_PREFIX_LENGTH = 10
interface UsageLoadOptions {
scope?: UsageLoadScope
background?: boolean
@@ -49,7 +51,7 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
...(providerQuery.value ? { provider: providerQuery.value } : {}),
})
const filterParams = () => ({
search: searchQuery.value || undefined,
search: usageSearchParam(searchQuery.value),
})
const usagePagination = computed(() => ({
currentPage: currentPage.value,
@@ -244,6 +246,15 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
}
}
function usageSearchParam(value: string) {
const search = value.trim()
if (!search)
return undefined
if (search.startsWith('sk_'))
return search.slice(0, CLIENT_API_KEY_VISIBLE_PREFIX_LENGTH)
return search
}
function emptySummary() {
const summary: Awaited<ReturnType<typeof getUsageRecordSummary>> = {
totalRequests: '0',
+1 -1
View File
@@ -1 +1 @@
version: "3.3.18"
version: "3.3.23"