feat: add request diagnostics and align Codex transport contracts

This commit is contained in:
Zyy
2026-09-06 16:24:14 +08:00
parent 697f1870a0
commit 353dccedc4
90 changed files with 2899 additions and 647 deletions
+2
View File
@@ -33,6 +33,8 @@ jobs:
timeout-minutes: 30
env:
CARGO_TERM_COLOR: always
# Rust 1.97 编译 AWS SDK 调试信息需要更大的 codegen 线程栈。
RUST_MIN_STACK: 16777216
CPR_TEST_DATABASE_URL: postgres://codex_proxy:codex_proxy@127.0.0.1:5432/codex_proxy
CPR_TEST_REDIS_URL: redis://127.0.0.1:6379
services:
+49 -44
View File
@@ -154,9 +154,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
version = "1.17.1"
version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
dependencies = [
"aws-lc-sys",
"untrusted 0.7.1",
@@ -165,15 +165,14 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.42.0"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
"pkg-config",
]
[[package]]
@@ -195,7 +194,7 @@ dependencies = [
"bytes-utils",
"fastrand",
"http 0.2.12",
"http 1.4.2",
"http 1.4.0",
"http-body 0.4.6",
"http-body 1.0.1",
"percent-encoding",
@@ -231,7 +230,7 @@ dependencies = [
"hex",
"hmac",
"http 0.2.12",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"lru",
"percent-encoding",
@@ -257,7 +256,7 @@ dependencies = [
"hex",
"hmac",
"http 0.2.12",
"http 1.4.2",
"http 1.4.0",
"percent-encoding",
"sha2 0.11.0",
"time",
@@ -286,7 +285,7 @@ dependencies = [
"bytes",
"crc-fast",
"hex",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"md-5",
@@ -320,7 +319,7 @@ dependencies = [
"bytes-utils",
"futures-core",
"futures-util",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"percent-encoding",
@@ -339,7 +338,7 @@ dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"h2",
"http 1.4.2",
"http 1.4.0",
"hyper",
"hyper-rustls",
"hyper-util",
@@ -389,7 +388,7 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"http 1.4.2",
"http 1.4.0",
"http-body 0.4.6",
"http-body 1.0.1",
"http-body-util",
@@ -410,7 +409,7 @@ dependencies = [
"aws-smithy-types",
"bytes",
"http 0.2.12",
"http 1.4.2",
"http 1.4.0",
"pin-project-lite",
"tokio",
"tracing",
@@ -436,7 +435,7 @@ checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.4.2",
"http 1.4.0",
]
[[package]]
@@ -450,7 +449,7 @@ dependencies = [
"bytes-utils",
"futures-core",
"http 0.2.12",
"http 1.4.2",
"http 1.4.0",
"http-body 0.4.6",
"http-body 1.0.1",
"http-body-util",
@@ -503,7 +502,7 @@ dependencies = [
"bytes",
"form_urlencoded",
"futures-util",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"hyper",
@@ -535,7 +534,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
dependencies = [
"bytes",
"futures-core",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"mime",
@@ -1478,12 +1477,14 @@ name = "gateway-core"
version = "0.1.0"
dependencies = [
"async-trait",
"base64 0.23.1",
"bytes",
"futures",
"futures-timer",
"semver",
"serde",
"serde_json",
"sha2 0.11.0",
"thiserror",
"tracing",
"uuid",
@@ -1639,7 +1640,7 @@ dependencies = [
"fnv",
"futures-core",
"futures-sink",
"http 1.4.2",
"http 1.4.0",
"indexmap",
"slab",
"tokio",
@@ -1687,7 +1688,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"headers-core",
"http 1.4.2",
"http 1.4.0",
"httpdate",
"mime",
"sha1 0.10.7",
@@ -1699,7 +1700,7 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
dependencies = [
"http 1.4.2",
"http 1.4.0",
]
[[package]]
@@ -1761,9 +1762,9 @@ dependencies = [
[[package]]
name = "http"
version = "1.4.2"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
dependencies = [
"bytes",
"itoa",
@@ -1787,7 +1788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http 1.4.2",
"http 1.4.0",
]
[[package]]
@@ -1798,7 +1799,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
dependencies = [
"bytes",
"futures-core",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"pin-project-lite",
]
@@ -1834,21 +1835,22 @@ dependencies = [
[[package]]
name = "hyper"
version = "1.10.1"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
dependencies = [
"atomic-waker",
"bytes",
"futures-channel",
"futures-core",
"h2",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"pin-utils",
"smallvec",
"tokio",
"want",
@@ -1860,7 +1862,7 @@ version = "0.27.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
dependencies = [
"http 1.4.2",
"http 1.4.0",
"hyper",
"hyper-util",
"log",
@@ -1882,7 +1884,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"hyper",
"ipnet",
@@ -2594,6 +2596,7 @@ version = "0.0.0"
dependencies = [
"async-stream",
"async-trait",
"aws-lc-rs",
"base64 0.23.1",
"bytes",
"chrono",
@@ -2605,6 +2608,7 @@ dependencies = [
"gateway-protocol",
"getrandom 0.4.3",
"hex",
"hyper",
"hyper-rustls",
"reqwest",
"roxmltree",
@@ -2620,6 +2624,7 @@ dependencies = [
"tempfile",
"thiserror",
"tokio",
"tokio-rustls",
"tokio-tungstenite 0.28.0",
"tokio-util",
"tracing",
@@ -2911,7 +2916,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"hyper",
@@ -3021,9 +3026,9 @@ dependencies = [
[[package]]
name = "rustls-native-certs"
version = "0.8.4"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
dependencies = [
"openssl-probe",
"rustls-pki-types",
@@ -3033,9 +3038,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.15.1"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
dependencies = [
"web-time",
"zeroize",
@@ -3699,7 +3704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -3800,9 +3805,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.53.1"
version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"bytes",
"libc",
@@ -3912,7 +3917,7 @@ dependencies = [
"bitflags",
"bytes",
"futures-util",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"pin-project-lite",
"tower",
@@ -3931,7 +3936,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-util",
"http 1.4.2",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"http-range-header",
@@ -4063,7 +4068,7 @@ dependencies = [
"data-encoding",
"flate2",
"headers",
"http 1.4.2",
"http 1.4.0",
"httparse",
"log",
"rand 0.9.4",
@@ -4082,7 +4087,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8"
dependencies = [
"bytes",
"data-encoding",
"http 1.4.2",
"http 1.4.0",
"httparse",
"log",
"rand 0.9.4",
@@ -4332,9 +4337,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
version = "1.0.8"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
dependencies = [
"rustls-pki-types",
]
@@ -4536,7 +4541,7 @@ dependencies = [
"base64 0.22.1",
"deadpool",
"futures",
"http 1.4.2",
"http 1.4.0",
"http-body-util",
"hyper",
"hyper-util",
+2 -5
View File
@@ -143,10 +143,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
#[test]
fn config_loader_should_reject_invalid_codex_cli_version() {
assert_rejected(valid_config().replace(
"codex_version: '0.151.0-alpha.7.2'",
"codex_version: 'latest'",
));
assert_rejected(valid_config().replace("codex_version: '0.153.4'", "codex_version: 'latest'"));
}
#[test]
@@ -165,7 +162,7 @@ fn config_loader_should_reject_zero_server_port() {
#[test]
fn config_loader_should_reject_invalid_desktop_profile_fields() {
assert_rejected(valid_config().replace("desktop_build: '7377'", "desktop_build: 'build'"));
assert_rejected(valid_config().replace("desktop_build: '8109'", "desktop_build: 'build'"));
}
fn assert_rejected(config: String) {
@@ -704,6 +704,8 @@ pub struct UsageAttempt {
/// 一条请求及其全部尝试。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageDetail {
pub trace: Option<serde_json::Value>,
pub related_requests: Vec<serde_json::Value>,
pub request: UsageRecord,
pub attempts: Vec<UsageAttempt>,
}
@@ -505,6 +505,8 @@ pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptV
pub(crate) fn usage_detail_view(detail: domain::UsageDetail) -> UsageRecordDetailView {
UsageRecordDetailView {
trace: detail.trace,
related_requests: detail.related_requests,
request: usage_record_view(detail.request),
attempts: detail
.attempts
@@ -247,6 +247,8 @@ pub struct UsageAttemptView {
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UsageRecordDetailView {
pub trace: Option<serde_json::Value>,
pub related_requests: Vec<serde_json::Value>,
#[serde(flatten)]
pub request: UsageRecordView,
pub attempts: Vec<UsageAttemptView>,
+8 -5
View File
@@ -180,13 +180,16 @@ pub fn initialize(
"request",
request_id = %request_id,
method = %request.method().as_str(),
uri = %request
.uri()
.path_and_query()
.map(axum::http::uri::PathAndQuery::as_str)
.unwrap_or_default(),
uri = %request.uri().path(),
)
}
})
.on_request(|_request: &Request<_>, _span: &tracing::Span| {
tracing::info!(target: "request_trace", stage = "http.received", "HTTP request received");
})
.on_response(|response: &axum::http::Response<_>, latency: Duration, _span: &tracing::Span| {
tracing::info!(target: "request_trace", stage = "http.response", status = response.status().as_u16(),
headers_ms = latency.as_millis(), "HTTP response headers ready");
});
let router = router
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
@@ -14,6 +14,7 @@ use axum::{
response::{IntoResponse, Response},
};
use futures::stream;
use gateway_core::diagnostics::TraceContext;
use gateway_core::engine::execution::{ClientTransport, ExecutionSession, StartedExecution};
use gateway_core::engine::{CommitRequirement, EngineError};
use gateway_core::error::{GatewayError, GatewayErrorKind};
@@ -39,6 +40,7 @@ use super::{
pub(crate) async fn responses(
State(state): State<ApiState>,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
ingress_id: Option<Extension<tower_http::request_id::RequestId>>,
headers: HeaderMap,
body: Bytes,
) -> Response {
@@ -83,28 +85,25 @@ pub(crate) async fn responses(
Ok(started) => started,
Err(error) => return gateway_error_response(&error),
};
tracing::info!(
target: "request_dump",
provider = "openai",
request_id = %started.request_id,
request_direction = "downstream",
request_transport = if streaming { "http_sse" } else { "http_json" },
request_method = "POST",
request_path = "/v1/responses",
request_headers = ?headers,
request_body = %String::from_utf8_lossy(&body),
contains_sensitive_data = true,
"request dump (unredacted)"
);
let trace = started.session.trace();
trace.headers("client.request", serde_json::json!({
"ingressRequestId": ingress_id.as_ref().and_then(|Extension(id)| id.header_value().to_str().ok()),
}), headers.iter().map(|(name, value)| (name.as_str(), value.as_bytes())));
trace.capture("client.request.body", &body);
let request_id_header = HeaderValue::from_str(started.request_id.as_str()).ok();
let StartedExecution {
stream, session, ..
} = started;
if stream {
let mut response = if stream {
stream_execution_response(session, connection_guard).await
} else {
drop(connection_guard);
collect_execution_response(session).await
};
if let Some(id) = request_id_header {
response.headers_mut().insert("x-gateway-request-id", id);
}
response
}
/// 从 socket 与标准转发头提取旧 Usage 页面使用的诊断事实。
@@ -182,6 +181,11 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
}
};
let response_headers = session.response_headers().to_vec();
session.trace().record(
"downstream.encoded",
serde_json::json!({"transport": "http_json", "bytes": encoded.len()}),
);
session.trace().dump("downstream.body", &encoded);
let response = json_body_response(encoded, &response_headers);
let Some(session) = execution.session_mut() else {
@@ -334,6 +338,8 @@ pub async fn stream_execution_response(
let body = Body::from_stream(stream::unfold(state, |mut state| async move {
loop {
if let Some(chunk) = state.pending.pop_front() {
state.trace.dump("downstream.chunk", &chunk);
state.handed_off_bytes += chunk.len() as u64;
return Some((Ok::<Bytes, Infallible>(chunk), state));
}
if state.output_finished {
@@ -416,12 +422,18 @@ impl Drop for PendingExecution {
if session.is_finalized() {
return;
}
session.trace().record(
"downstream.cancelled",
serde_json::json!({"reason": "response_guard_dropped"}),
);
session.cancel();
detach_finalize(session);
}
}
struct ResponsesStreamState {
trace: TraceContext,
handed_off_bytes: u64,
session: Option<Box<dyn ExecutionSession>>,
encoder: OpenAiResponsesEncoder,
pending: VecDeque<Bytes>,
@@ -438,6 +450,8 @@ impl ResponsesStreamState {
connection_guard: Option<Box<dyn ConnectionGuard>>,
) -> Self {
Self {
trace: session.trace(),
handed_off_bytes: 0,
session: Some(session),
encoder,
pending: initial_frames.into_iter().collect(),
@@ -593,6 +607,11 @@ impl ResponsesStreamState {
impl Drop for ResponsesStreamState {
fn drop(&mut self) {
self.trace.record("downstream.body.closed", serde_json::json!({
"handedOffBytes": self.handed_off_bytes,
"pendingChunks": self.pending.len(),
"outputFinished": self.output_finished, "executionTerminal": self.execution_terminal,
}));
if self.execution_terminal {
return;
}
@@ -338,7 +338,12 @@ async fn next_active_input(
// pump 继续接收有界业务帧和处理 Ping/Pong;退出通过独立通知取消本轮。
tokio::select! {
biased;
_ = connection.wait_for_exit() => ActiveInput::Disconnect,
_ = connection.wait_for_exit() => {
session.trace().record("downstream.cancelled", serde_json::json!({
"reason": "websocket_connection_exit", "connectionId": connection.id(),
}));
ActiveInput::Disconnect
},
event = session.next_event() => ActiveInput::Event(event),
}
}
@@ -451,6 +456,7 @@ fn log_terminal_write_success(
provider_terminal_to_write: std::time::Duration,
) {
tracing::info!(
target: "request_trace", stage = "downstream.terminal.written",
websocket_connection_id = connection.id(),
request_id = %request_id,
provider_terminal_to_terminal_write_ms = provider_terminal_to_write.as_millis(),
@@ -465,6 +471,7 @@ fn log_terminal_write_failure(
provider_terminal_to_write: std::time::Duration,
) {
tracing::info!(
target: "request_trace", stage = "downstream.terminal.write_failed",
websocket_connection_id = connection.id(),
request_id = %request_id,
provider_terminal_to_terminal_write_ms = provider_terminal_to_write.as_millis(),
@@ -19,6 +19,7 @@ use axum::{
response::{IntoResponse, Response},
};
use gateway_core::{
diagnostics::{TraceContext, diagnostic_json},
engine::execution::{AuthenticatedClient, ClientTransport},
lifecycle::{ConnectionGuard, ConnectionLifecycle},
};
@@ -98,17 +99,12 @@ impl ResponsesWebSocketAdapter {
Err(_) => return runtime_unavailable_response().into_response(),
};
let connection_id = self.service.next_request_id().replacen("req_", "ws_", 1);
tracing::info!(
target: "request_dump",
provider = "openai",
websocket_connection_id = %connection_id,
request_direction = "downstream",
request_transport = "websocket_handshake",
request_method = "GET",
request_path = "/v1/responses",
request_headers = ?raw_headers,
contains_sensitive_data = true,
"request dump (unredacted)"
TraceContext::new(&connection_id).headers(
"client.connection.headers",
serde_json::json!({"transport": "websocket", "method": "GET", "path": "/v1/responses"}),
raw_headers
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes())),
);
let session = ResponsesWebSocketSession {
service: self.service.clone(),
@@ -187,26 +183,15 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
}
request_count = request_count.saturating_add(1);
let correlation_id = Arc::<str>::from(service.next_request_id());
tracing::info!(
target: "request_dump",
provider = "openai",
websocket_connection_id = connection.id(),
correlation_id = %correlation_id,
request_index = request_count,
request_direction = "downstream",
request_transport = "websocket",
request_payload = %payload,
contains_sensitive_data = true,
"request dump (unredacted)"
);
let decoded = match decode_response_create_with_context(&payload, &request_headers) {
Ok(decoded) => decoded.with_client_context(client_ip, user_agent.clone()),
Err(error) => {
tracing::info!(
websocket_connection_id = connection.id(),
request_id = %correlation_id,
error = %error,
"Responses WebSocket request rejected"
trace_rejected_request(
&correlation_id,
connection.id(),
&payload,
"decode",
&error.to_string(),
);
if send_protocol_error(
&mut connection,
@@ -225,6 +210,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
let decoded = replay.prepare(decoded);
// deadline 与 Text 可能同时就绪;在任何上游执行开始前再次封住该竞争窗口。
if connection.is_expired() {
trace_rejected_request(
&correlation_id,
connection.id(),
&payload,
"connection_expired",
"",
);
expire_connection(&mut connection).await;
break;
}
@@ -239,6 +231,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
{
Ok(started) => started,
Err(error) => {
trace_rejected_request(
&correlation_id,
connection.id(),
&payload,
"start",
&error.to_string(),
);
if send_gateway_error(&mut connection, &error, &correlation_id).await
== ForwardOutcome::Disconnect
{
@@ -247,14 +246,14 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
continue;
}
};
tracing::info!(
target: "request_dump",
provider = "openai",
websocket_connection_id = connection.id(),
correlation_id = %correlation_id,
request_id = %started.request_id,
"request dump correlation"
);
started.session.trace().record("client.connection", serde_json::json!({
"transport": "websocket", "connectionId": connection.id(), "correlationId": correlation_id,
"requestIndex": request_count,
}));
started
.session
.trace()
.capture("client.request.body", payload.as_bytes());
if forward_execution(&mut connection, started, &mut replay).await
== ForwardOutcome::Disconnect
@@ -270,6 +269,28 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
connection.log_summary(request_count);
}
// 尚未建立模型请求的拒绝也保留原文入口,用返回给客户端的 correlation ID 检索。
fn trace_rejected_request(
correlation_id: &str,
connection_id: &str,
payload: &str,
phase: &'static str,
error: &str,
) {
let trace = TraceContext::new(correlation_id);
trace.record(
"client.connection",
serde_json::json!({"connectionId": connection_id}),
);
trace.capture("client.request.body", payload.as_bytes());
trace.record(
"client.request.rejected",
serde_json::json!({
"phase": phase, "detail": diagnostic_json(&serde_json::json!({"message": error})),
}),
);
}
async fn expire_connection(connection: &mut ResponsesWebSocketConnection) {
if connection
.send_text(
@@ -429,6 +429,8 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
.lock()
.expect("usage detail")
.replace(UsageDetail {
trace: None,
related_requests: Vec::new(),
request: usage_record_with_account(
"req_detail",
"acct_snap_a",
+2
View File
@@ -13,11 +13,13 @@ path = "tests/main.rs"
[dependencies]
async-trait = "0.1.89"
base64 = "0.23.0"
bytes = "1.11.1"
futures = "0.3.32"
futures-timer = "3.0.3"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
sha2 = "0.11.0"
semver = "1.0.27"
thiserror = "2.0.18"
tracing = "0.1.44"
@@ -0,0 +1,157 @@
//! 诊断内容的统一筛选边界。未知字符串只保留长度和摘要。
use serde_json::{Map, Value, json};
use sha2::{Digest, Sha256};
/// 原始字节的长度与 SHA-256,不复制或保留正文。
#[must_use]
pub fn body_fingerprint(bytes: &[u8]) -> Value {
let digest = Sha256::digest(bytes);
let hex: String = digest
.iter()
.flat_map(|byte| {
let alphabet = b"0123456789abcdef";
[
char::from(alphabet[usize::from(byte >> 4)]),
char::from(alphabet[usize::from(byte & 15)]),
]
})
.collect();
json!({"bytes": bytes.len(), "sha256": hex})
}
/// 保留可诊断的协议结构;凭据永不进入普通日志或数据库。
#[must_use]
pub fn diagnostic_json(value: &Value) -> Value {
capture(value, "", 0, &mut 96)
}
fn capture(value: &Value, key: &str, depth: usize, budget: &mut usize) -> Value {
if *budget == 0 || depth > 6 {
return json!({"omitted": true});
}
*budget -= 1;
let key = key.to_ascii_lowercase();
if secret_key(&key) {
return Value::String("<redacted>".to_owned());
}
match value {
Value::Object(object) => {
let mut result = Map::new();
for (name, value) in object.iter().take(48) {
if *budget == 0 {
break;
}
result.insert(bounded(name, 96), capture(value, name, depth + 1, budget));
}
if result.len() < object.len() {
result.insert(
"_omittedFields".to_owned(),
json!(object.len() - result.len()),
);
}
Value::Object(result)
}
Value::Array(items) => json!({
"length": items.len(),
"sample": items.iter().take(4)
.map(|item| capture(item, key.as_str(), depth + 1, budget))
.collect::<Vec<_>>()
}),
Value::String(text)
if safe_header(&key) || safe_value_key(&key) && protocol_label(text) =>
{
Value::String(bounded(text, 256))
}
Value::String(text) => body_fingerprint(text.as_bytes()),
other => other.clone(),
}
}
/// 请求/响应头保持多值;安全 trace ID 原样保存,未知值只保存摘要。
#[must_use]
pub fn diagnostic_headers<'a>(headers: impl IntoIterator<Item = (&'a str, &'a str)>) -> Value {
let mut result = Map::new();
for (name, value) in headers.into_iter().take(64) {
let name = bounded(&name.to_ascii_lowercase(), 96);
let value = if secret_key(&name) {
Value::String("<redacted>".to_owned())
} else if safe_header(&name) {
Value::String(bounded(value, 256))
} else {
body_fingerprint(value.as_bytes())
};
let values = result
.entry(name)
.or_insert_with(|| Value::Array(Vec::new()));
if let Value::Array(values) = values {
values.push(value);
}
}
Value::Object(result)
}
fn secret_key(key: &str) -> bool {
key.contains("authorization")
|| key.contains("cookie")
|| key.contains("api_key")
|| key.contains("api-key")
|| key.contains("token") && !key.ends_with("tokens")
|| key.contains("password")
|| key.contains("secret")
|| key.contains("attestation")
|| matches!(key, "x-oai-is" | "x-oai-is-update")
}
fn safe_header(key: &str) -> bool {
matches!(
key,
"x-oai-request-id"
| "x-client-request-id"
| "x-request-id"
| "request-id"
| "openai-request-id"
| "x-openai-request-id"
| "cf-ray"
| "traceparent"
| "tracestate"
| "content-type"
| "content-length"
| "content-encoding"
| "retry-after"
| "openai-processing-ms"
| "x-processing-ms"
| "date"
| "server"
| "sec-websocket-extensions"
| "x-codex-allowed"
| "x-codex-limit-reached"
| "x-codex-active-limit"
| "x-models-etag"
)
}
fn safe_value_key(key: &str) -> bool {
safe_header(key)
|| matches!(
key,
"type" | "event" | "status" | "code" | "model" | "role" | "object" | "service_tier"
)
}
pub(super) fn bounded(value: &str, limit: usize) -> String {
value
.chars()
.take(limit)
.map(|c| if c.is_control() { ' ' } else { c })
.collect()
}
/// Event names/codes are protocol labels, never arbitrary upstream text.
pub(super) fn protocol_label(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 128
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'))
}
@@ -0,0 +1,9 @@
//! 有界、请求局部的诊断证据;不参与路由、重试或客户端协议决策。
mod capture;
mod stream;
mod trace;
pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json};
pub use stream::{StreamCapture, StreamFormat};
pub use trace::TraceContext;
@@ -0,0 +1,165 @@
//! 仅用于观测的有界流分帧;超过限额时明确记缺口,不影响实际协议解析。
use serde_json::json;
use super::TraceContext;
const MAX_FRAME_BYTES: usize = 128 * 1024;
#[derive(Clone, Copy)]
pub enum StreamFormat {
Sse,
JsonLines,
}
/// 同一 exchange 持有一个 observer,跨 HTTP chunk 拼接完整事件。
pub struct StreamCapture {
trace: TraceContext,
format: StreamFormat,
line: Vec<u8>,
data: Vec<u8>,
event_name: Option<String>,
previous_cr: bool,
first_line: bool,
skipping_line: bool,
skipping_event: bool,
finished: bool,
}
impl StreamCapture {
#[must_use]
pub fn new(trace: TraceContext, format: StreamFormat) -> Self {
Self {
trace,
format,
line: Vec::new(),
data: Vec::new(),
event_name: None,
previous_cr: false,
first_line: true,
skipping_line: false,
skipping_event: false,
finished: false,
}
}
pub fn push(&mut self, bytes: &[u8]) {
self.trace.dump("upstream.chunk", bytes);
for byte in bytes {
if self.previous_cr && *byte == b'\n' {
self.previous_cr = false;
continue;
}
self.previous_cr = *byte == b'\r';
if matches!(*byte, b'\n' | b'\r') {
if !self.skipping_line {
self.line();
}
self.line.clear();
self.skipping_line = false;
} else if !self.skipping_line {
if self.line.len() < MAX_FRAME_BYTES {
self.line.push(*byte);
} else {
self.trace.record(
"capture.gap",
json!({"reason": "line_size_limit", "limitBytes": MAX_FRAME_BYTES}),
);
self.line.clear();
self.skipping_line = true;
self.skipping_event = matches!(self.format, StreamFormat::Sse);
self.data.clear();
}
}
}
}
pub fn finish(&mut self) {
if self.finished {
return;
}
let partial_frame = !self.line.is_empty()
|| !self.data.is_empty()
|| self.skipping_event
|| self.skipping_line;
if !self.line.is_empty() && !self.skipping_line {
self.line();
}
if !self.data.is_empty() && !self.skipping_event {
self.trace.capture_named_event(
"upstream.event",
&self.data,
self.event_name.as_deref(),
);
}
self.trace
.record("upstream.eof", json!({"partialFrame": partial_frame}));
self.finished = true;
}
fn line(&mut self) {
let line = if self.first_line {
self.first_line = false;
self.line
.strip_prefix(b"\xef\xbb\xbf")
.unwrap_or(&self.line)
} else {
&self.line
};
if matches!(self.format, StreamFormat::JsonLines) {
if !line.is_empty() {
self.trace.capture_event("upstream.event", line);
}
return;
}
if line.is_empty() {
if !self.data.is_empty() && !self.skipping_event {
self.trace.capture_named_event(
"upstream.event",
&self.data,
self.event_name.as_deref(),
);
}
self.data.clear();
self.event_name = None;
self.skipping_event = false;
} else if let Some(name) = line.strip_prefix(b"event:") {
self.event_name = Some(super::capture::bounded(
String::from_utf8_lossy(name).trim(),
128,
));
} else if let Some(data) = line.strip_prefix(b"data:") {
let data = data.strip_prefix(b" ").unwrap_or(data);
if self.skipping_event {
return;
}
if self.data.len() + data.len() + 1 > MAX_FRAME_BYTES {
self.trace.record(
"capture.gap",
json!({"reason": "event_size_limit", "limitBytes": MAX_FRAME_BYTES}),
);
self.data.clear();
self.skipping_event = true;
} else {
if !self.data.is_empty() {
self.data.push(b'\n');
}
self.data.extend_from_slice(data);
}
}
}
}
impl Drop for StreamCapture {
fn drop(&mut self) {
if !self.finished {
self.trace.record(
"upstream.reader.dropped",
json!({
"partialBytes": self.line.len() + self.data.len(),
"captureIncomplete": self.skipping_event || self.skipping_line,
}),
);
}
}
}
@@ -0,0 +1,338 @@
//! 首部与最近事件同时保留,流式 delta 合并,所有淘汰均有计数。
use std::{
collections::VecDeque,
sync::{Arc, Mutex},
time::{Instant, SystemTime, UNIX_EPOCH},
};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde::Serialize;
use serde_json::{Value, json};
use super::capture::bounded;
use super::{body_fingerprint, diagnostic_headers, diagnostic_json};
const MAX_EVENTS: usize = 128;
const HEAD_EVENTS: usize = 8;
const MAX_DATA_BYTES: usize = 4096;
const MAX_BUFFER_BYTES: usize = 64 * 1024;
/// 显式传给异步任务的关联上下文;默认值禁用捕获,避免全局请求映射。
#[derive(Clone, Default)]
pub struct TraceContext {
state: Option<Arc<Mutex<TraceState>>>,
attempt_index: u32,
exchange_id: Option<u64>,
}
impl std::fmt::Debug for TraceContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TraceContext")
.field("attempt_index", &self.attempt_index)
.field("exchange_id", &self.exchange_id)
.finish_non_exhaustive()
}
}
struct TraceState {
request_id: String,
started_at_ms: u128,
started: Instant,
sequence: u64,
exchanges: u64,
wire_frames: u64,
wire_bytes: u64,
dropped_events: u64,
bytes: usize,
events: VecDeque<TraceEvent>,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct TraceEvent {
sequence: u64,
last_sequence: u64,
elapsed_ms: u64,
last_elapsed_ms: u64,
attempt_index: u32,
exchange_id: Option<u64>,
stage: &'static str,
count: u64,
data: Value,
#[serde(skip)]
bytes: usize,
}
impl TraceContext {
/// 一个模型请求只创建一次,所有 attempt 共享同一有界时间线。
#[must_use]
pub fn new(request_id: &str) -> Self {
Self {
state: Some(Arc::new(Mutex::new(TraceState {
request_id: bounded(request_id, 128),
started_at_ms: SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis(),
started: Instant::now(),
sequence: 0,
exchanges: 0,
wire_frames: 0,
wire_bytes: 0,
dropped_events: 0,
bytes: 0,
events: VecDeque::new(),
}))),
..Self::default()
}
}
#[must_use]
pub fn attempt(&self, attempt_index: u32) -> Self {
Self {
state: self.state.clone(),
attempt_index,
exchange_id: None,
}
}
/// 一个 attempt 中的每次实际 transport exchange 也有独立编号。
#[must_use]
pub fn exchange(&self, transport: &'static str) -> Self {
let mut context = self.clone();
if let Some(state) = &self.state {
let mut state = state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.exchanges += 1;
context.exchange_id = Some(state.exchanges);
}
context.record("upstream.exchange.started", json!({"transport": transport}));
context
}
/// details 只允许调用方构造的诊断事实;不接收原始请求/响应正文。
pub fn record(&self, stage: &'static str, data: Value) {
self.push(stage, data, false);
}
/// 同一个头部边界只采集一次:默认脱敏,显式开启 dump 时另存完整字节。
/// facts 必须为调用方构造的安全对象;原始头值只进入独立 dump。
pub fn headers<'a>(
&self,
stage: &'static str,
mut facts: Value,
headers: impl IntoIterator<Item = (&'a str, &'a [u8])>,
) {
if self.state.is_none() {
return;
}
let Some(facts_object) = facts.as_object_mut() else {
return;
};
let headers: Vec<_> = headers.into_iter().collect();
let summary = diagnostic_headers(
headers
.iter()
.map(|(name, value)| (*name, std::str::from_utf8(value).unwrap_or("<binary>"))),
);
facts_object.insert("headers".to_owned(), summary);
self.record(stage, facts);
if tracing::enabled!(target: "request_dump", tracing::Level::INFO) {
let values: Vec<_> = headers
.iter()
.map(|(name, value)| json!({"name": name, "valueBase64": STANDARD.encode(value)}))
.collect();
if let Ok(bytes) = serde_json::to_vec(&json!({"headers": values})) {
self.dump(stage, &bytes);
}
}
}
/// 入站 JSON 先记录再解析业务语义,包括未知事件和 metadata。
/// 原文仅在现有 request_dump 开关打开时写入其独立文件。
pub fn capture(&self, stage: &'static str, bytes: &[u8]) {
self.dump(stage, bytes);
self.capture_event(stage, bytes);
}
/// StreamCapture 已转储原始 chunk 时,只提取完整事件的安全摘要。
pub fn capture_event(&self, stage: &'static str, bytes: &[u8]) {
self.capture_named_event(stage, bytes, None);
}
pub fn capture_named_event(&self, stage: &'static str, bytes: &[u8], name: Option<&str>) {
if self.state.is_none() {
return;
}
let fingerprint = body_fingerprint(bytes);
if bytes.len() > 128 * 1024 {
self.record("capture.gap", json!({"reason": "json_inspection_size_limit", "stage": stage, "body": fingerprint}));
return;
}
let parsed = serde_json::from_slice::<Value>(bytes).ok();
let event_type = parsed
.as_ref()
.and_then(|v| v.get("type"))
.and_then(Value::as_str)
.or(name)
.filter(|kind| super::capture::protocol_label(kind));
let delta = event_type.is_some_and(|kind| kind.ends_with(".delta"));
let data = json!({
"body": fingerprint,
"eventType": event_type.map(|kind| bounded(kind, 128)),
"jsonValid": parsed.is_some(),
"metadata": parsed.as_ref().filter(|_| !delta).map(diagnostic_json),
});
self.push(stage, data, delta);
}
/// 只记录业务解析边界;metadata 已由 transport 在解析前捕获。
pub fn wire_event(&self, protocol: &str, event_type: Option<&str>) {
let event_type = event_type.filter(|kind| super::capture::protocol_label(kind));
let delta = event_type.is_some_and(|kind| kind.ends_with(".delta"));
self.push(
"provider.event",
json!({
"protocol": protocol,
"eventType": event_type.map(|kind| bounded(kind, 128)),
}),
delta,
);
}
/// 完整报文捕获与摘要使用相同 request / attempt / exchange ID。
pub fn dump(&self, stage: &'static str, bytes: &[u8]) {
if !tracing::enabled!(target: "request_dump", tracing::Level::INFO) {
return;
}
if let Some(state) = &self.state {
let mut state = state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.wire_frames += 1;
state.wire_bytes = state.wire_bytes.saturating_add(bytes.len() as u64);
let wire_sequence = state.wire_frames;
let chunk_count = bytes.len().div_ceil(32 * 1024).max(1);
// Each log entry stays bounded even for images or very large WebSocket frames.
for chunk_index in 0..chunk_count {
let start = chunk_index * 32 * 1024;
let chunk = &bytes[start..bytes.len().min(start + 32 * 1024)];
tracing::info!(target: "request_dump", request_id = %state.request_id,
attempt_index = self.attempt_index, exchange_id = self.exchange_id,
stage, wire_sequence, chunk_index, chunk_count,
body_bytes = bytes.len(), body_base64 = %STANDARD.encode(chunk),
contains_sensitive_data = true, "diagnostic wire capture");
}
}
}
/// 与请求终态原子持久化;旧记录没有此字段,不伪造历史时间线。
#[must_use]
pub fn snapshot(&self) -> Option<Value> {
let state = self
.state
.as_ref()?
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Some(json!({
"schemaVersion": 1, "requestId": state.request_id,
"startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence,
"droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS,
"captureMode": "sanitized", "events": state.events,
"wireDumpEnabled": tracing::enabled!(target: "request_dump", tracing::Level::INFO),
"wireFrames": state.wire_frames, "wireBytes": state.wire_bytes,
"maxBufferBytes": MAX_BUFFER_BYTES, "headEvents": HEAD_EVENTS,
"snapshotBoundary": "execution_finalization",
}))
}
fn push(&self, stage: &'static str, mut data: Value, coalesce: bool) {
let Some(state) = &self.state else { return };
let mut encoded = serde_json::to_vec(&data).unwrap_or_default();
if encoded.len() > MAX_DATA_BYTES {
data = json!({"truncated": true, "summary": body_fingerprint(&encoded),
"eventType": data.get("eventType"), "body": data.get("body"),
"jsonValid": data.get("jsonValid"),
});
encoded = serde_json::to_vec(&data).unwrap_or_default();
}
let mut state = state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.sequence += 1;
let sequence = state.sequence;
let elapsed_ms = u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX);
if coalesce
&& let Some(last) = state.events.iter_mut().rev().find(|event| {
event.stage == stage
&& event.attempt_index == self.attempt_index
&& event.exchange_id == self.exchange_id
})
&& last.stage == stage
&& last.attempt_index == self.attempt_index
&& last.exchange_id == self.exchange_id
&& last.data.get("eventType") == data.get("eventType")
{
last.count += 1;
last.last_sequence = sequence;
last.last_elapsed_ms = elapsed_ms;
return;
}
// 普通日志只输出有界安全事实;进程意外退出时仍可按 ID 检索已发生阶段。
tracing::info!(target: "request_trace", request_id = %state.request_id,
attempt_index = self.attempt_index, exchange_id = self.exchange_id,
sequence, elapsed_ms, stage, data = %data, "request trace");
let bytes = encoded.len() + 256;
state.bytes += bytes;
state.events.push_back(TraceEvent {
sequence,
last_sequence: sequence,
elapsed_ms,
last_elapsed_ms: elapsed_ms,
attempt_index: self.attempt_index,
exchange_id: self.exchange_id,
stage,
count: 1,
data,
bytes,
});
while state.events.len() > MAX_EVENTS || state.bytes > MAX_BUFFER_BYTES {
// Preserve failures and retry decisions ahead of routine middle-of-stream events.
// Head/tail and important stages compete under the same hard memory limit.
let index = (HEAD_EVENTS..state.events.len().saturating_sub(8))
.find(|index| {
!matches!(
state.events[*index].stage,
"attempt.started"
| "account.selected"
| "attempt.failed"
| "retry.decided"
| "upstream.close"
| "upstream.read.failed"
| "capture.gap"
| "request.finished"
)
})
.unwrap_or(if state.events.len() > HEAD_EVENTS {
HEAD_EVENTS
} else {
0
});
if let Some(event) = state.events.remove(index) {
state.bytes = state.bytes.saturating_sub(event.bytes);
state.dropped_events += event.count;
}
}
}
}
impl Drop for TraceState {
fn drop(&mut self) {
tracing::info!(target: "request_trace", request_id = %self.request_id,
stage = "request.trace.closed", total_events = self.sequence,
dropped_events = self.dropped_events, wire_frames = self.wire_frames,
wire_bytes = self.wire_bytes, "request diagnostic capture closed");
}
}
@@ -1,5 +1,8 @@
//! 唯一的账号重试、发送与下游 commit barrier owner。
use crate::diagnostics::TraceContext;
use serde_json::json;
use std::collections::BTreeSet;
use std::future::Future;
use std::num::NonZeroU32;
@@ -144,12 +147,26 @@ where
.position(|candidate| candidate.provider() == pin.provider())
.ok_or(EngineError::ContinuationPinMismatch)
})?;
let trace = TraceContext::new(request_id.as_str());
trace.record(
"request.routed",
json!({
"endpoint": request.endpoint, "protocol": request.protocol,
"clientTransport": request.client_transport,
"model": request.requested_model.as_ref().map(|model| model.as_str()),
"configRevision": format!("{:?}", request.config_revision),
"admissionMs": request.admission_decision_ms,
"continuation": format!("{continuation_attempt:?}"),
"candidateCount": plan.candidates().len(),
}),
);
let image_generation_requested = operation.image_generation_requested();
let mut session = ResponseExecutionSession {
engine: Arc::clone(&self.engine),
request_id,
client_api_key_ref,
observation: ResponseObservation::new(timing_started_at),
trace,
deadline,
deadline_timer: Delay::new(
deadline
@@ -238,6 +255,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
request_id: ModelRequestId,
client_api_key_ref: crate::policy::ClientApiKeyId,
observation: ResponseObservation,
trace: TraceContext,
deadline: SystemTime,
/// 会话级 deadline 计时器;deadline 固定,帧循环内复用而非逐事件新建。
deadline_timer: Fuse<Delay>,
@@ -291,6 +309,11 @@ impl<S: ?Sized> ResponseExecutionSession<S>
where
S: ExecutionStore,
{
/// 当前请求共享的诊断上下文。
pub fn trace(&self) -> TraceContext {
self.trace.clone()
}
/// 读取下一条 canonical event;首条未提交事件会携带 commit 要求。
///
/// # Errors
@@ -398,6 +421,10 @@ where
return Err(EngineError::InvalidDeliveryState);
}
let committed_at = SystemTime::now();
self.trace.record(
"downstream.committed",
json!({"status": client_status_code}),
);
if self.request_persisted {
best_effort_store_write(
"mark_downstream_committed",
@@ -563,6 +590,11 @@ where
return Err(EngineError::Deadline);
}
PollBoundary::Item(Some(Ok(mut event))) => {
if let Some(wire) = event.wire_event() {
self.trace
.attempt(self.attempts)
.wire_event(wire.protocol(), wire.event_type());
}
if let Some(state) = event.session_update() {
self.observe_session_checkpoint(state);
}
@@ -682,7 +714,8 @@ where
}));
let context = AttemptContext::new(
RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone())
.with_timing_started_at(self.observation.timing_started_at),
.with_timing_started_at(self.observation.timing_started_at)
.with_trace(self.trace.clone()),
next_attempt,
self.deadline,
self.plan.account_selection_policy(),
@@ -705,6 +738,18 @@ where
.ok_or_else(|| EngineError::ProviderNotRegistered {
provider: candidate.provider().as_str().to_owned(),
})?;
let attempt_trace = self.trace.attempt(next_attempt.get());
attempt_trace.record(
"attempt.started",
json!({
"provider": candidate.provider().as_str(),
"model": candidate.upstream_model().map(|model| model.as_str()),
"transportPolicy": format!("{attempt_transport:?}"),
"continuation": format!("{:?}", self.continuation_attempt),
"pinnedAccount": pinned_account.as_ref().map(|id| id.as_str()),
"transportRecovery": is_transport_recovery,
}),
);
let provider_request = ProviderRequest::new(self.operation.clone(), candidate.clone());
let stream = match poll_provider(
provider,
@@ -729,6 +774,7 @@ where
ProviderBoundary::Result(result) => match *result {
Ok(stream) => stream,
Err(error) => {
record_trace_error(&attempt_trace, &error);
if self.prepare_unavailable_native_continuation_replay(&error) {
return Ok(Some(PullOutcome::AttemptDiscarded));
}
@@ -800,6 +846,11 @@ where
}
let metadata = stream.metadata().clone();
attempt_trace.record("account.selected", json!({
"provider": metadata.provider().as_str(), "accountId": metadata.provider_account_id().as_str(),
"transport": metadata.transport().as_str(),
"selectionMs": metadata.selection_observation().map(|o| o.account_selection_wait_ms()),
}));
let selection_observation = metadata.selection_observation();
let capacity = selection_observation.and_then(|observation| observation.capacity());
if !self.account_selection.is_diagnostic()
@@ -1006,6 +1057,7 @@ where
) -> Result<StreamErrorOutcome, EngineError> {
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
// 均只接触已剥离的稳定错误字段。
record_trace_error(&self.trace.attempt(self.attempts), &error);
let mut atomic_client_events = error.take_atomic_client_events();
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
@@ -1093,6 +1145,14 @@ where
|| account_rotation_retry
|| transport_recovery.is_some();
self.trace.attempt(current.index.get()).record("retry.decided", json!({
"retryable": retryable, "continuationRetry": continuation_retry,
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
"delayMs": transport_recovery.map(|(_, delay)| duration_ms(delay)),
"downstreamCommitted": self.downstream_committed_at.is_some(),
"sendState": format!("{attempt_send_state:?}"),
}));
if retryable {
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
// 所有权保留到下一次 attempt 成功,或最终空选路时返回客户端。
@@ -1303,6 +1363,10 @@ where
self.current_transport_observation();
let service_tier = self.current_service_tier();
let provider_metadata_json = self.current_provider_metadata_json();
self.trace.record(
"request.finished",
json!({"outcome": "succeeded", "attempts": self.attempts}),
);
if self.request_persisted {
best_effort_store_write(
"finalize_model_request",
@@ -1325,6 +1389,7 @@ where
websocket_pool,
service_tier,
provider_metadata_json,
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
error: None,
provider_error_code: None,
raw_upstream_error: None,
@@ -1443,6 +1508,14 @@ where
if self.finalized {
return Ok(());
}
self.trace.record(
"request.finished",
json!({
"outcome": format!("{:?}", finalization.outcome),
"errorKind": finalization.error.kind().as_str(),
"sendState": format!("{:?}", finalization.send_state), "attempts": self.attempts,
}),
);
if !self.request_persisted {
self.finalized = true;
return Ok(());
@@ -1497,6 +1570,7 @@ where
websocket_pool,
service_tier,
provider_metadata_json,
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
error: Some(finalization.error),
provider_error_code: finalization.provider_error_code,
raw_upstream_error: finalization.raw_upstream_error,
@@ -1787,3 +1861,15 @@ const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> Upst
fn duration_ms(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
fn record_trace_error(trace: &TraceContext, error: &ProviderError) {
trace.record("attempt.failed", json!({
"kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()),
"upstreamStatus": error.upstream_status(),
"upstreamCode": error.upstream_code().map(|code| code.as_str()),
"rawError": error.raw_upstream_error().map(|raw| {
let value = serde_json::from_str(raw.as_str()).unwrap_or_else(|_| json!(raw.as_str()));
crate::diagnostics::diagnostic_json(&value)
}),
}));
}
@@ -155,6 +155,9 @@ pub struct StartedExecution {
}
pub trait ExecutionSession: Send {
fn trace(&self) -> crate::diagnostics::TraceContext {
crate::diagnostics::TraceContext::default()
}
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>>;
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>>;
fn response_headers(&self) -> &[ProviderResponseHeader];
@@ -1035,6 +1038,12 @@ impl Drop for DefaultExecutionSession {
}
impl ExecutionSession for DefaultExecutionSession {
fn trace(&self) -> crate::diagnostics::TraceContext {
self.core
.as_ref()
.map(ResponseExecutionSession::trace)
.unwrap_or_default()
}
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
Box::pin(async move {
let result = self.core_mut()?.next_event().await;
@@ -316,6 +316,7 @@ pub struct RequestAttemptContext {
request_id: ModelRequestId,
client_api_key_ref: ClientApiKeyId,
timing_started_at: Instant,
trace: crate::diagnostics::TraceContext,
}
impl RequestAttemptContext {
@@ -325,9 +326,16 @@ impl RequestAttemptContext {
request_id,
client_api_key_ref,
timing_started_at: Instant::now(),
trace: crate::diagnostics::TraceContext::default(),
}
}
#[must_use]
pub fn with_trace(mut self, trace: crate::diagnostics::TraceContext) -> Self {
self.trace = trace;
self
}
/// 覆盖本次请求的单调计时原点。
#[must_use]
pub fn with_timing_started_at(mut self, timing_started_at: Instant) -> Self {
@@ -367,6 +375,12 @@ pub struct AttemptContext {
}
impl AttemptContext {
/// 当前 attempt 的诊断关联;克隆后可传给后台 transport 任务。
#[must_use]
pub fn trace(&self) -> crate::diagnostics::TraceContext {
self.request.trace.attempt(self.attempt_index.get())
}
#[must_use]
pub const fn new(
request: RequestAttemptContext,
@@ -612,6 +626,8 @@ pub struct ModelRequestFinalization {
pub service_tier: Option<String>,
/// Provider 已筛选的专有观测 JSON;Core 不解释字段。
pub provider_metadata_json: Option<String>,
/// 请求全程的有界诊断快照,跨 Provider 与重试保留。
pub diagnostic_trace_json: Option<String>,
pub error: Option<GatewayError>,
pub provider_error_code: Option<String>,
/// Provider 返回的原始错误正文或 WebSocket close/error frame。
+1
View File
@@ -4,6 +4,7 @@
//! 具体客户端协议和具体 Provider 都通过外层 adapter 接入。
pub mod account;
pub mod diagnostics;
pub mod engine;
pub mod error;
pub mod event;
@@ -0,0 +1,49 @@
use gateway_core::diagnostics::{diagnostic_headers, diagnostic_json};
use serde_json::json;
#[test]
fn credentials_and_content_never_enter_diagnostics_but_trace_ids_survive() {
let headers = diagnostic_headers([
("Authorization", "Bearer never-persist-this"),
("Set-Cookie", "session=never-persist-this"),
("X-OAI-Request-ID", "upstream-123"),
("x-codex-turn-state", "opaque-never-persist-this"),
("cf-ray", "ray-one"),
("cf-ray", "ray-two"),
]);
assert_eq!(headers["x-oai-request-id"][0], "upstream-123");
assert_eq!(headers["cf-ray"].as_array().unwrap().len(), 2);
assert!(!headers.to_string().contains("never-persist-this"));
let content = diagnostic_json(&json!({
"type": "codex.response.metadata", "headers": headers,
"access_token": "never-persist-this", "refresh_token": "never-persist-this",
"response": {"output": [{"text": "private model output"}]},
"input": "private user prompt", "unknown": "private extension",
}));
assert_eq!(content["type"], "codex.response.metadata");
let text = content.to_string();
for secret in [
"never-persist-this",
"private model output",
"private user prompt",
"private extension",
] {
assert!(!text.contains(secret), "leaked {secret}");
}
}
#[test]
fn arbitrary_event_type_text_is_fingerprinted_instead_of_logged() {
let trace = gateway_core::diagnostics::TraceContext::new("req_type");
trace.capture(
"upstream.event",
br#"{"type":"secret@example.com","message":"private-content"}"#,
);
let snapshot = trace.snapshot().unwrap();
assert_eq!(
snapshot["events"][0]["data"]["eventType"],
serde_json::Value::Null
);
assert!(!snapshot.to_string().contains("secret@example.com"));
assert!(!snapshot.to_string().contains("private-content"));
}
@@ -0,0 +1,3 @@
mod capture;
mod stream;
mod trace;
@@ -0,0 +1,70 @@
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext};
#[test]
fn sse_capture_reassembles_every_chunk_boundary_and_preserves_unknown_events() {
let bytes = "event: future.metadata\r\ndata: {\"headers\":{\"x-request-id\":\"up-1\"},\"unknown\":\"中文\"}\r\n\r\ndata: {\"type\":\"response.completed\"}\n\n".as_bytes();
for boundary in 0..=bytes.len() {
let trace = TraceContext::new("req_sse");
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
capture.push(&bytes[..boundary]);
capture.push(&bytes[boundary..]);
capture.finish();
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
assert_eq!(events.len(), 3, "boundary {boundary}");
assert_eq!(events[0]["data"]["eventType"], "future.metadata");
assert_eq!(
events[0]["data"]["metadata"]["headers"]["x-request-id"],
"up-1"
);
assert_eq!(events[1]["data"]["eventType"], "response.completed");
assert_eq!(events[2]["data"]["partialFrame"], false);
}
}
#[test]
fn oversize_frames_report_a_gap_and_resume_at_the_next_frame() {
let trace = TraceContext::new("req_sse");
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
capture.push(b"data: ");
capture.push(&vec![b'x'; 130 * 1024]);
capture.push(b"\n\ndata: {\"type\":\"response.completed\"}\n\n");
capture.finish();
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
assert_eq!(events[0]["stage"], "capture.gap");
assert_eq!(events[1]["data"]["eventType"], "response.completed");
}
#[test]
fn dropping_a_partial_stream_is_distinct_from_upstream_eof() {
let trace = TraceContext::new("req_partial");
{
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
capture.push(b"data: {\"type\":");
}
let snapshot = trace.snapshot().unwrap();
assert_eq!(snapshot["events"][0]["stage"], "upstream.reader.dropped");
assert!(
snapshot["events"][0]["data"]["partialBytes"]
.as_u64()
.unwrap()
> 0
);
}
#[test]
fn sse_accepts_bom_and_bare_carriage_return_even_when_split_into_single_bytes() {
let trace = TraceContext::new("req_cr");
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
for byte in b"\xef\xbb\xbfdata: {\"type\":\"codex.response.metadata\"}\r\r" {
capture.push(&[*byte]);
}
capture.finish();
let snapshot = trace.snapshot().unwrap();
assert_eq!(
snapshot["events"][0]["data"]["eventType"],
"codex.response.metadata"
);
assert_eq!(snapshot["events"][1]["data"]["partialFrame"], false);
}
@@ -0,0 +1,142 @@
use gateway_core::diagnostics::TraceContext;
use serde_json::json;
#[test]
fn bounded_history_preserves_start_failure_and_final_result() {
let trace = TraceContext::new("req_trace");
trace.record("request.started", json!({}));
for index in 0..400 {
trace.record(
"test.event",
json!({"index": index, "detail": "x".repeat(2000)}),
);
}
trace
.attempt(1)
.record("attempt.failed", json!({"code": "websocket_close_1000"}));
trace
.attempt(2)
.record("request.finished", json!({"outcome": "succeeded"}));
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
assert_eq!(events[0]["stage"], "request.started");
assert_eq!(events[events.len() - 2]["attemptIndex"], 1);
assert_eq!(events.last().unwrap()["attemptIndex"], 2);
assert!(snapshot["droppedEvents"].as_u64().unwrap() > 0);
assert!(events.len() <= 128);
assert!(snapshot.to_string().len() < 64 * 1024);
let kept: u64 = events
.iter()
.map(|event| event["count"].as_u64().unwrap())
.sum();
assert_eq!(
kept + snapshot["droppedEvents"].as_u64().unwrap(),
snapshot["totalEvents"]
);
}
#[test]
fn shared_attempts_and_exchanges_coalesce_delta_without_losing_metadata() {
let trace = TraceContext::new("req_trace");
let ws = trace.attempt(1).exchange("websocket");
ws.capture(
"upstream.event",
br#"{"type":"codex.response.metadata","headers":{"x-request-id":"up-1"}}"#,
);
trace
.attempt(1)
.wire_event("openai", Some("codex.response.metadata"));
for _ in 0..1000 {
ws.capture(
"upstream.event",
br#"{"type":"response.output_text.delta","delta":"secret"}"#,
);
trace
.attempt(1)
.wire_event("openai", Some("response.output_text.delta"));
}
let http = trace.attempt(1).exchange("http_sse");
http.capture("upstream.event", br#"{"type":"response.completed"}"#);
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
assert_eq!(snapshot["droppedEvents"], 0);
assert_eq!(
events
.iter()
.find(|e| e["data"]["eventType"] == "response.output_text.delta")
.unwrap()["count"],
1000
);
assert_eq!(events.last().unwrap()["exchangeId"], 2);
assert!(
events
.iter()
.any(|e| e["data"]["metadata"]["headers"]["x-request-id"] == "up-1")
);
assert!(!snapshot.to_string().contains("secret"));
assert!(
events
.iter()
.filter(|event| event["stage"] == "provider.event")
.all(|event| event["data"].get("metadata").is_none())
);
}
#[test]
fn header_capture_keeps_duplicates_but_excludes_credentials_from_timeline() {
let trace = TraceContext::new("req_headers");
trace.headers(
"upstream.response.headers",
json!({"status": 101}),
[
("authorization", b"Bearer SECRET".as_slice()),
("x-request-id", b"upstream-one".as_slice()),
("x-request-id", b"upstream-two".as_slice()),
("set-cookie", b"PRIVATE_COOKIE".as_slice()),
],
);
let snapshot = trace.snapshot().unwrap();
assert_eq!(
snapshot["events"][0]["data"]["headers"]["x-request-id"],
json!(["upstream-one", "upstream-two"])
);
assert!(!snapshot.to_string().contains("SECRET"));
assert!(!snapshot.to_string().contains("PRIVATE_COOKIE"));
}
#[test]
fn disabled_context_does_not_allocate_a_history() {
let trace = TraceContext::default().attempt(2).exchange("http");
trace.capture("upstream.event", b"not json");
assert!(trace.snapshot().is_none());
}
#[test]
fn a_long_successful_retry_keeps_the_earlier_failure_ahead_of_routine_events() {
let trace = TraceContext::new("req_retry");
for _ in 0..12 {
trace.record("setup", json!({}));
}
trace
.attempt(1)
.record("attempt.failed", json!({"kind": "unavailable"}));
trace
.attempt(1)
.record("retry.decided", json!({"retryable": true}));
for index in 0..500 {
trace
.attempt(2)
.record("upstream.event", json!({"index": index}));
}
trace.record("request.finished", json!({"outcome": "succeeded"}));
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
assert!(
events
.iter()
.any(|event| event["stage"] == "attempt.failed" && event["attemptIndex"] == 1)
);
assert!(events.iter().any(|event| event["stage"] == "retry.decided"));
assert_eq!(events.last().unwrap()["stage"], "request.finished");
assert!(events.len() <= 128);
}
@@ -49,6 +49,7 @@ use serde_json::{Map, Value, json};
#[derive(Debug, Clone, PartialEq, Eq)]
struct FinalState {
diagnostic_trace_json: Option<String>,
outcome: ExecutionOutcome,
send_state: UpstreamSendState,
attempt_count: u32,
@@ -207,6 +208,7 @@ impl ExecutionStore for FakeStore {
.expect("store lock")
.finalizations
.push(FinalState {
diagnostic_trace_json: finalization.diagnostic_trace_json,
outcome: finalization.outcome,
send_state: finalization.send_state,
attempt_count: finalization.attempt_count,
@@ -1314,6 +1316,21 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
let state = store.state.lock().expect("store lock");
let finalization = &state.finalizations[0];
let trace: Value =
serde_json::from_str(finalization.diagnostic_trace_json.as_deref().unwrap()).unwrap();
let events = trace["events"].as_array().unwrap();
assert!(
events
.iter()
.any(|event| event["stage"] == "attempt.failed" && event["attemptIndex"] == 1)
);
assert!(
events
.iter()
.any(|event| event["stage"] == "attempt.started" && event["attemptIndex"] == 2)
);
assert_eq!(events.last().unwrap()["stage"], "request.finished");
assert_eq!(finalization.upstream_transport.as_deref(), Some("http_sse"));
assert_eq!(finalization.http_version.as_deref(), Some("HTTP/2"));
assert_eq!(
@@ -1,4 +1,5 @@
mod account;
mod diagnostics;
mod engine;
mod error;
mod event;
@@ -283,6 +283,7 @@ pub struct ModelRequestFinalization {
pub websocket_pool: Option<String>,
pub service_tier: Option<String>,
pub provider_metadata_json: Option<Value>,
pub diagnostic_trace_json: Option<Value>,
pub error_kind: Option<String>,
pub provider_error_code: Option<String>,
pub error_message: Option<String>,
@@ -353,6 +354,15 @@ impl ModelRequestFinalization {
{
return Err(invalid("provider observation must be a JSON object"));
}
if self
.diagnostic_trace_json
.as_ref()
.is_some_and(|trace| !trace.is_object() || trace.to_string().len() > 64 * 1024)
{
return Err(invalid(
"diagnostic trace must be a JSON object within 64 KiB",
));
}
validate_optional_stable_reason(
self.continuation_unavailable_reason.as_deref(),
"continuation unavailable reason",
@@ -725,7 +735,7 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_connection_id = $44,
upstream_connection_exit_reason = $45,
upstream_connection_age_ms = $46,
upstream_connection_idle_ms = $47
upstream_connection_idle_ms = $47, diagnostic_trace_json = $48
where id = $1 and outcome = 'running'
returning id, client_api_key_ref, continuation_affinity_hash,
continuation_requested, provider_kind, upstream_transport,
@@ -916,6 +926,7 @@ impl ModelRequestRepository for PgExecutionStore {
finalization.upstream_connection_idle_ms,
"upstream_connection_idle_ms",
)?)
.bind(finalization.diagnostic_trace_json.map(sqlx::types::Json))
.fetch_one(&self.pool)
.await
.map_err(|_| postgres_unavailable("finalize model request"))?;
@@ -1215,6 +1226,12 @@ impl ExecutionStore for PgExecutionStore {
websocket_pool: finalization.websocket_pool,
service_tier: finalization.service_tier,
provider_metadata_json,
diagnostic_trace_json: finalization
.diagnostic_trace_json
.as_deref()
.map(serde_json::from_str)
.transpose()
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
error_kind,
provider_error_code: finalization.provider_error_code,
error_message,
@@ -583,6 +583,7 @@ impl ExecutionObservationWrite {
finalization.websocket_pool.as_deref(),
finalization.service_tier.as_deref(),
finalization.provider_metadata_json.as_deref(),
finalization.diagnostic_trace_json.as_deref(),
finalization.provider_error_code.as_deref(),
])
.saturating_add(error_bytes)
@@ -584,6 +584,8 @@ pub(crate) fn admin_usage_detail(
detail: UsageRecordDetail,
) -> AdminStoreResult<admin_observability::UsageDetail> {
Ok(admin_observability::UsageDetail {
trace: detail.trace,
related_requests: detail.related_requests,
request: admin_usage_record(detail.request)?,
attempts: detail
.attempts
@@ -645,6 +645,8 @@ pub struct UsageAttemptObservation {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UsageRecordDetail {
pub trace: Option<serde_json::Value>,
pub related_requests: Vec<serde_json::Value>,
pub request: UsageRecord,
pub attempts: Vec<UsageAttemptObservation>,
}
@@ -1,6 +1,7 @@
//! Usage 明细、诊断与过滤查询族。
use super::super::*;
use serde_json::Value;
pub(crate) fn push_usage_filter(
query: &mut QueryBuilder<Postgres>,
@@ -145,7 +146,7 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
mr.provider_account_authentication_kind_snapshot
as provider_account_authentication_kind,
mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool,
mr.service_tier, mr.provider_observation_json,
mr.service_tier, mr.provider_observation_json, mr.diagnostic_trace_json,
mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at,
mr.outcome, mr.client_status_code, mr.upstream_status_code,
mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id,
@@ -237,7 +238,6 @@ pub(crate) async fn usage_record_detail(
let mut statement = QueryBuilder::<Postgres>::new(USAGE_RECORD_DETAIL_SELECT);
statement.push(" where mr.id = ");
statement.push_bind(request_id.to_owned());
push_completed_usage_fact_filter(&mut statement, "mr");
let row = statement
.build()
.fetch_optional(pool)
@@ -272,7 +272,24 @@ pub(crate) async fn usage_record_detail(
if request.attempt_count > 0 {
attempts.push(final_attempt_from_request(&request));
}
Ok(UsageRecordDetail { request, attempts })
let trace: Option<Value> = row
.try_get("diagnostic_trace_json")
.map_err(|_| postgres_unavailable("decode request trace"))?;
let related_requests = sqlx::query_scalar::<_, Value>(
"select jsonb_build_object('requestId', related.id, 'outcome', related.outcome,
'relation', case when related.id = current.recovery_request_id then 'recovered_by' else 'recovers' end,
'completedAt', related.completed_at)
from model_requests current join model_requests related
on related.id = current.recovery_request_id or related.recovery_request_id = current.id
where current.id = $1 order by related.started_at limit 20"
).bind(request_id).fetch_all(pool).await
.map_err(|_| postgres_unavailable("load related recovery requests"))?;
Ok(UsageRecordDetail {
request,
attempts,
trace,
related_requests,
})
}
pub(crate) fn intermediate_attempt_from_row(
@@ -8,7 +8,8 @@ use gateway_core::error::{GatewayError, GatewayErrorKind, ProviderConnectionObse
use gateway_core::metering::{CalculatedCost, CostEstimate, Usage};
use gateway_core::upstream::UpstreamSendState;
use gateway_store::postgres::{
ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest, PgExecutionStore,
ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest, ObservabilityRepository,
PgExecutionStore,
};
use super::TestDatabase;
@@ -463,6 +464,7 @@ async fn expired_image_request_should_be_recovered_as_failed() {
fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
CoreModelRequestFinalization {
diagnostic_trace_json: None,
request_id: ModelRequestId::new(id).expect("request id"),
outcome: ExecutionOutcome::Succeeded,
send_state: UpstreamSendState::Sent,
@@ -804,6 +806,30 @@ async fn successful_http_downgrade_should_mark_pending_websocket_failures_recove
assert_eq!(row.4, Some(expected_delay_ms));
assert!(row.5.is_some_and(|latency| latency >= expected_delay_ms));
}
let repository = super::observability_repository(&database.pool);
let failed_detail = repository
.usage_record_detail("req_websocket_failed_first")
.await
.unwrap();
assert_eq!(
failed_detail.related_requests[0]["requestId"],
"req_http_fallback_succeeded"
);
assert_eq!(
failed_detail.related_requests[0]["relation"],
"recovered_by"
);
let recovered_detail = repository
.usage_record_detail("req_http_fallback_succeeded")
.await
.unwrap();
assert_eq!(recovered_detail.related_requests.len(), 2);
assert!(
recovered_detail
.related_requests
.iter()
.all(|related| related["relation"] == "recovers")
);
database.close().await;
}
@@ -1072,3 +1098,40 @@ async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx:
.await?;
Ok(())
}
#[tokio::test]
async fn diagnostic_trace_is_finalized_atomically_and_available_for_failed_request_details() {
let Some(database) = TestDatabase::create("diagnostic_trace").await else {
return;
};
seed_running_request(&database.pool, "req_diagnostic_failed")
.await
.unwrap();
let store = PgExecutionStore::new(database.pool.clone());
let mut finalization = successful_core_finalization("req_diagnostic_failed");
finalization.outcome = ExecutionOutcome::Failed;
finalization.client_status_code = Some(502);
finalization.error = Some(GatewayError::new(
GatewayErrorKind::UpstreamUnavailable,
"upstream closed",
));
let trace = serde_json::json!({"schemaVersion": 1, "requestId": "req_diagnostic_failed", "events": [
{"attemptIndex": 1, "stage": "upstream.close", "data": {"code": 1000}},
]});
finalization.diagnostic_trace_json = Some(trace.to_string());
ExecutionStore::finalize_model_request(&store, finalization)
.await
.unwrap();
let persisted: (String, serde_json::Value) = sqlx::query_as(
"select outcome, diagnostic_trace_json from model_requests where id = 'req_diagnostic_failed'"
).fetch_one(&database.pool).await.unwrap();
assert_eq!(persisted, ("failed".to_owned(), trace.clone()));
let repository = super::observability_repository(&database.pool);
let detail = repository
.usage_record_detail("req_diagnostic_failed")
.await
.unwrap();
assert_eq!(detail.trace, Some(trace));
assert!(detail.related_requests.is_empty());
database.close().await;
}
@@ -853,13 +853,13 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
store
.usage_record_detail("req_observe_failed")
.await
.is_err()
.is_ok()
);
assert!(
store
.usage_record_detail("req_observe_uncommitted")
.await
.is_err()
.is_ok()
);
let overview = store
@@ -1243,7 +1243,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
repository
.usage_record_detail("req_observe_failed")
.await
.is_err()
.is_ok()
);
let overview = repository
+12 -11
View File
@@ -21,27 +21,28 @@ gateway-core = { path = "../../gateway-core" }
gateway-protocol = { path = "../../gateway-protocol" }
getrandom = "0.4.3"
hex = "0.4.3"
# 与官方 Codex Desktop app-server 的 TLS 栈逐版本对齐(2026-08 核实:
# ChatGPT.app Resources/codex 二进制内为 hyper-rustls 0.27.7 + rustls 0.23.36 +
# reqwest 0.12.28 + ring)。rustls 0.23.37 起 ClientHello 扩展顺序变化,JA3
# 会偏离官方指纹;升级前必须重新抓握手对比扩展顺序。
# TLS 依赖按官方 Core 锁文件固定,包含用于约束传递依赖的直接声明。
# 升级前须重复采样 ClientHello,核对算法、扩展集合及排序约束、key_share 与 ALPN。
aws-lc-rs = "=1.16.2"
hyper = { version = "=1.8.1", default-features = false }
hyper-rustls = "=0.27.7"
# 与官方 Codex Desktop 一致:reqwest 不启用压缩特性,上游请求不带 accept-encoding。
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "http2"] }
roxmltree = "0.21.1"
semver = "1.0.27"
# 保持 ring provider:reqwest rustls-tls-native-roots 即 __rustls-ring,与官方一致;
# 不要给 TLS 客户端路径启用 aws_lc_rs,否则密钥共享/扩展集可能变化。
rustls = { version = "=0.23.36", default-features = false, features = ["ring", "std", "tls12"] }
rustls-native-certs = "0.8.4"
rustls-pki-types = "1.14.1"
# 与官方 rustls 路径一致:aws-lc 支持 P-521,优先提供 X25519MLKEM768。
# reqwest 的 roots feature 仍会编译 ring;实际 provider 由 TLS 初始化显式选择。
rustls = { version = "=0.23.36", default-features = false, features = ["aws_lc_rs", "prefer-post-quantum", "std", "tls12"] }
rustls-native-certs = "=0.8.3"
rustls-pki-types = "=1.14.0"
secrecy = "0.10.3"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
sha2 = "0.11.0"
subtle = "2.6.1"
thiserror = "2.0.18"
tokio = { version = "1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] }
tokio = { version = "=1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] }
tokio-rustls = { version = "=0.26.4", default-features = false }
tokio-util = { version = "0.7.18", features = ["rt"] }
# 官方 Codex WebSocket 需要上游尚未发布的 proxy 与 permessage-deflate 实现,固定到已审计提交。
tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "0e5b2d73aa18dd9f0a50ee9ff199d5aef7594186", features = ["proxy", "rustls-tls-native-roots"] }
@@ -51,7 +52,7 @@ url = "2.5.8"
uuid = { version = "1.23.3", features = ["v4", "v7"] }
# 官方 Codex app-server 对 /v1/responses 请求体默认做 zstd 压缩(level 3),
# 并携带 Content-Encoding: zstd;保持同版本同级别以对齐上游指纹。
zstd = "0.13"
zstd = "=0.13.3"
[dev-dependencies]
tempfile = "3.27.0"
+10 -5
View File
@@ -8,7 +8,7 @@ use serde::Deserialize;
use url::{Host, Url};
use crate::credential::CodexQuotaRefreshPolicy;
use crate::transport::profile::{CodexWireProfile, CodexWireProfileState};
use crate::transport::profile::{CodexResidency, CodexWireProfile, CodexWireProfileState};
use crate::transport::session::{CodexSessionIdentity, CodexSessionIdentityError};
use crate::transport::websocket::CodexWebSocketPoolConfig;
use crate::{
@@ -287,6 +287,8 @@ pub struct CodexWireProfileConfig {
pub os_version: String,
pub arch: String,
pub terminal: String,
#[serde(default)]
pub residency: Option<CodexResidency>,
pub verified_at: DateTime<Utc>,
}
@@ -294,14 +296,16 @@ impl Default for CodexWireProfileConfig {
fn default() -> Self {
Self {
originator: "Codex Desktop".to_owned(),
codex_version: "0.147.0-alpha.6.6".to_owned(),
desktop_version: "26.803.81509".to_owned(),
desktop_build: "6415".to_owned(),
codex_version: "0.153.4".to_owned(),
desktop_version: "26.901.51231".to_owned(),
desktop_build: "8109".to_owned(),
os_type: "Mac OS".to_owned(),
os_version: "15.7.1".to_owned(),
arch: "arm64".to_owned(),
terminal: "unknown".to_owned(),
verified_at: Utc::now(),
residency: None,
// 制品核验于 2026-09-06T03:26:12.084Z;进程启动不构成重新核验。
verified_at: DateTime::UNIX_EPOCH + chrono::Duration::milliseconds(1_788_665_172_084),
}
}
}
@@ -361,6 +365,7 @@ impl From<CodexWireProfileConfig> for CodexWireProfile {
os_version: value.os_version,
arch: value.arch,
terminal: value.terminal,
residency: value.residency,
verified_at: value.verified_at,
}
}
@@ -8,7 +8,7 @@ use std::{
use gateway_core::account::ProviderAccountId;
use reqwest::Client;
use secrecy::ExposeSecret as _;
use secrecy::{ExposeSecret as _, SecretString};
use thiserror::Error;
use uuid::Uuid;
@@ -133,6 +133,30 @@ impl CodexCredentialProfileService {
&self,
account_id: &ProviderAccountId,
) -> Result<CodexProfileStatistics, CodexProfileStatisticsError> {
let (authorization, upstream_account_id) = self.account_authentication(account_id).await?;
let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple());
let statistics = CodexBackendClient::new(
self.http.clone(),
self.base_url.clone(),
self.profile.clone(),
)
.fetch_profile_statistics(CodexRequestContext::auxiliary(
authorization.expose_secret(),
upstream_account_id.as_deref(),
&request_id,
None,
))
.await
.map_err(map_client_error)?;
self.remember_avatar_source(account_id, statistics.image_url.as_deref());
Ok(statistics)
}
/// URL 缓存不缓存认证;每次下载都重新读取所属账号当前的 token。
async fn account_authentication(
&self,
account_id: &ProviderAccountId,
) -> Result<(SecretString, Option<String>), CodexProfileStatisticsError> {
let account = self
.repository
.store()
@@ -160,22 +184,10 @@ impl CodexCredentialProfileService {
.authentication
.authorization_header()
.map_err(|_| CodexProfileStatisticsError::InvalidCredentialData)?;
let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple());
let statistics = CodexBackendClient::new(
self.http.clone(),
self.base_url.clone(),
self.profile.clone(),
)
.fetch_profile_statistics(CodexRequestContext::auxiliary(
authorization.expose_secret(),
account.upstream_account_id(),
&request_id,
None,
Ok((
authorization,
account.upstream_account_id().map(str::to_owned),
))
.await
.map_err(map_client_error)?;
self.remember_avatar_source(account_id, statistics.image_url.as_deref());
Ok(statistics)
}
pub async fn profile_avatar(
@@ -190,10 +202,21 @@ impl CodexCredentialProfileService {
.image_url
.ok_or(CodexProfileAvatarError::Missing)?,
};
let desktop_user_agent = self.profile.snapshot().desktop_user_agent();
fetch_profile_avatar(&self.http, &self.base_url, &desktop_user_agent, &source)
.await
.map_err(map_avatar_fetch_error)
let (authorization, upstream_account_id) = self.account_authentication(account_id).await?;
fetch_profile_avatar(
&self.http,
&self.base_url,
&self.profile.snapshot(),
&source,
CodexRequestContext::auxiliary(
authorization.expose_secret(),
upstream_account_id.as_deref(),
"profile_avatar",
None,
),
)
.await
.map_err(map_avatar_fetch_error)
}
fn cached_avatar_source(&self, account_id: &ProviderAccountId) -> Option<String> {
@@ -1,5 +1,10 @@
//! OpenAI token 续期 Reqwest 适配器。
use crate::transport::{
headers::build_codex_profile_headers,
profile::CodexWireProfileState,
tls::{build_reqwest_client_with_custom_ca, ensure_rustls_provider},
};
use async_trait::async_trait;
use reqwest::{Client, StatusCode, redirect::Policy};
use secrecy::{ExposeSecret, SecretString};
@@ -243,6 +248,7 @@ pub struct TokenClientConfig {
pub struct OpenAiTokenClient {
client: Client,
config: TokenClientConfig,
profile: CodexWireProfileState,
}
/// 官方 Codex token client 无法安全构建。
@@ -251,9 +257,13 @@ pub struct OpenAiTokenClient {
pub struct TokenClientBuildError;
impl OpenAiTokenClient {
/// 使用 Reqwest 客户端和静态配置构造 token 续期客户端。
pub fn new(client: Client, config: TokenClientConfig) -> Self {
Self { client, config }
/// 共享运行时画像;刷新时取快照,授权码交换仍使用 raw auth 请求。
pub fn new(client: Client, config: TokenClientConfig, profile: CodexWireProfileState) -> Self {
Self {
client,
config,
profile,
}
}
}
@@ -264,14 +274,17 @@ impl OpenAiTokenClient {
/// 本地 TLS/HTTP client 初始化失败时返回脱敏错误。
pub fn openai_token_client(
config: TokenClientConfig,
profile: CodexWireProfileState,
) -> Result<OpenAiTokenClient, TokenClientBuildError> {
let client = Client::builder()
ensure_rustls_provider();
let builder = Client::builder()
.use_rustls_tls()
.no_proxy()
.redirect(Policy::none())
.connect_timeout(TOKEN_CONNECT_TIMEOUT)
.timeout(TOKEN_REQUEST_TIMEOUT)
.build()
.map_err(|_| TokenClientBuildError)?;
Ok(OpenAiTokenClient::new(client, config))
.timeout(TOKEN_REQUEST_TIMEOUT);
let client = build_reqwest_client_with_custom_ca(builder).map_err(|_| TokenClientBuildError)?;
Ok(OpenAiTokenClient::new(client, config, profile))
}
#[derive(Deserialize)]
@@ -345,9 +358,16 @@ struct AuthorizationCodeResponse {
#[async_trait]
impl TokenRefresher for OpenAiTokenClient {
async fn refresh(&self, refresh_token: &str) -> Result<TokenPair, RefreshFailure> {
let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|_| {
RefreshFailure::Transport {
message: Some("OpenAI OAuth refresh profile is invalid".to_owned()),
upstream: None,
}
})?;
let response = self
.client
.post(&self.config.token_endpoint)
.headers(headers)
.json(&RefreshTokenRequest {
client_id: self.config.client_id.as_str(),
grant_type: "refresh_token",
+5 -2
View File
@@ -146,8 +146,11 @@ pub async fn initialize(
.with_session_identity(session_identity),
);
let token_client = Arc::new(
credential::token_client::openai_token_client(config.token_client_config())
.map_err(|_| OpenAiInitializeError::TokenClient)?,
credential::token_client::openai_token_client(
config.token_client_config(),
profile.clone(),
)
.map_err(|_| OpenAiInitializeError::TokenClient)?,
);
let refresher: Arc<dyn TokenRefresher> = token_client.clone();
let exchanger: Arc<dyn AuthorizationCodeExchanger> = token_client;
@@ -358,12 +358,14 @@ pub(super) async fn create_json_attempt(
return Err(CodexHandshakeAttemptError::Timeout);
};
let request_id = request.context.request_id().as_str();
let trace = request.context.trace();
let mut request_context = CodexRequestContext::auxiliary(
authorization.expose_secret(),
account.upstream_account_id(),
request_id,
Some(installation_id),
);
request_context.trace = Some(&trace);
request_context.cookie_header = cookie_header.map(ExposeSecret::expose_secret);
request_context.turn_metadata = request.turn_metadata.as_deref();
request_context.account_selection = account_selection;
@@ -539,6 +541,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
lease.account_switch(),
);
let request_transport_requirement = transport_requirement(&request);
let trace = context.trace();
let response = create_response_attempt(
&client,
&request,
@@ -550,7 +553,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
&authorization,
cookie_header.as_ref(),
account_selection,
),
).with_trace(&trace),
active_account.id().as_str(),
context.deadline(),
&cancellation,
@@ -687,6 +687,7 @@ pub(super) fn codex_request_context<'a>(
account_selection: CodexAccountSelectionTelemetry<'a>,
) -> CodexRequestContext<'a> {
CodexRequestContext {
trace: None,
authorization: authorization.expose_secret(),
account_id: account.upstream_account_id(),
request_id,
@@ -395,6 +395,8 @@ impl<'a> CodexAccountSelectionTelemetry<'a> {
/// 单次 Codex 上游请求的上下文。
#[derive(Clone, Copy)]
pub struct CodexRequestContext<'a> {
/// 显式传递的请求诊断上下文,跨连接任务共享。
pub trace: Option<&'a gateway_core::diagnostics::TraceContext>,
/// Provider 已构造并脱敏持有的完整 Authorization 值。
pub authorization: &'a str,
/// ChatGPT 账号 ID。
@@ -440,6 +442,7 @@ impl<'a> CodexRequestContext<'a> {
installation_id: Option<&'a str>,
) -> Self {
Self {
trace: None,
authorization,
account_id,
request_id,
@@ -459,6 +462,12 @@ impl<'a> CodexRequestContext<'a> {
account_selection: CodexAccountSelectionTelemetry::NONE,
}
}
#[must_use]
pub const fn with_trace(mut self, trace: &'a gateway_core::diagnostics::TraceContext) -> Self {
self.trace = Some(trace);
self
}
}
impl fmt::Debug for CodexRequestContext<'_> {
@@ -3,7 +3,7 @@
use std::time::Instant;
use bytes::Bytes;
use reqwest::header::{ACCEPT, CONTENT_TYPE, HeaderName, HeaderValue};
use reqwest::header::{CONTENT_TYPE, HeaderValue};
use super::{
client::{
@@ -14,7 +14,7 @@ use super::{
},
diagnostics::CodexUpstreamSendPhase,
endpoints::endpoint_url,
headers::{build_codex_base_headers, insert_optional_header, insert_optional_protocol_header},
headers::insert_optional_protocol_header,
response_meta,
};
@@ -28,27 +28,30 @@ impl CodexBackendClient {
context: CodexRequestContext<'_>,
) -> CodexClientResult<CodexBackendJsonResponse> {
let profile = self.profile.snapshot();
let mut headers =
build_codex_base_headers(&profile, context.authorization, context.account_id)?;
let mut headers = self.model_request_headers(&profile, context)?;
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
insert_optional_header(
&mut headers,
"x-codex-installation-id",
context.installation_id,
)?;
insert_optional_protocol_header(&mut headers, "x-codex-image-turn-id", image_turn_id);
insert_optional_protocol_header(
&mut headers,
"x-codex-turn-metadata",
context.turn_metadata,
);
headers.insert(
HeaderName::from_static("x-client-request-id"),
HeaderValue::from_str(context.request_id)?,
);
let trace = context
.trace
.cloned()
.unwrap_or_default()
.exchange("http_json");
trace.headers(
"upstream.request.headers",
serde_json::json!({
"method": "POST", "endpoint": endpoint_path,
}),
headers
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes())),
);
trace.capture("upstream.request.body", &body);
let headers_started_at = Instant::now();
let response = self
.client
@@ -61,6 +64,17 @@ impl CodexBackendClient {
let upstream_headers_ms = elapsed_duration_millis(headers_started_at.elapsed());
let http_version = http_version_name(response.version()).to_owned();
let status = response.status();
trace.headers(
"upstream.response.headers",
serde_json::json!({
"status": status.as_u16(), "httpVersion": http_version,
"headersMs": upstream_headers_ms,
}),
response
.headers()
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes())),
);
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
@@ -81,6 +95,7 @@ impl CodexBackendClient {
let raw_body = read_error_response_body(response)
.await
.map_err(CodexClientError::HttpJson)?;
trace.capture("upstream.error.body", &raw_body);
let error_body = String::from_utf8_lossy(&raw_body).into_owned();
return Err(CodexClientError::Upstream {
status,
@@ -102,6 +117,7 @@ impl CodexBackendClient {
}
let body = response.bytes().await.map_err(CodexClientError::HttpJson)?;
trace.capture("upstream.response.body", &body);
Ok(CodexBackendJsonResponse {
body,
set_cookie_headers,
@@ -1,17 +1,16 @@
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext, diagnostic_json};
use std::{sync::Arc, time::Instant};
use futures::{StreamExt, TryStreamExt};
use gateway_protocol::openai::{
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
X_OPENAI_MEMGEN_REQUEST_HEADER,
events::{self, retry_after_seconds_from_body},
sse::{SseEventDecoder, SseFrame},
};
use reqwest::{
Client, Response as ReqwestResponse,
header::{
ACCEPT, AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE, COOKIE, HeaderMap, HeaderName,
HeaderValue, USER_AGENT,
},
header::{CONTENT_ENCODING, CONTENT_TYPE, HeaderMap, HeaderValue},
};
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
@@ -22,11 +21,8 @@ use crate::transport::{
},
diagnostics::CodexUpstreamSendPhase,
endpoints::{CODEX_RESPONSES_PATH, endpoint_url},
headers::{
build_codex_base_headers, insert_optional_header, insert_optional_protocol_header,
websocket_header_pairs,
},
profile::{CodexWireProfile, CodexWireProfileState},
headers::websocket_header_pairs,
profile::CodexWireProfileState,
protocol::{
responses::{CodexResponsesRequest, TransportRequirement, transport_requirement},
websocket::{
@@ -37,9 +33,8 @@ use crate::transport::{
response_meta,
websocket::{
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
CodexWebSocketPoolKey, CodexWebSocketRequest, CodexWebSocketStreamingExchange,
DEFAULT_INITIAL_EVENT_TIMEOUT, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
WebSocketOriginBreaker, WebSocketPoolDecision,
CodexWebSocketPoolKey, CodexWebSocketStreamingExchange, DEFAULT_INITIAL_EVENT_TIMEOUT,
WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath, WebSocketOriginBreaker,
execute_prepared_response_create_request_stream, post_send_ambiguous,
prepare_response_create_request_with_pool, websocket_audit_dir,
write_websocket_audit_artifact_from_env,
@@ -106,20 +101,21 @@ impl CodexBackendClient {
let body =
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
let endpoint = endpoint_url(&self.base_url, CODEX_RESPONSES_PATH);
tracing::info!(
target: "request_dump",
provider = "openai",
request_id = %context.request_id,
account_id = context.account_id.unwrap_or_default(),
request_direction = "upstream",
request_transport = "http_sse",
request_method = "POST",
request_url = %endpoint,
request_headers = ?headers,
request_body = %String::from_utf8_lossy(&body),
contains_sensitive_data = true,
"request dump (unredacted)"
let trace = context
.trace
.cloned()
.unwrap_or_default()
.exchange("http_sse");
trace.headers(
"upstream.request.headers",
serde_json::json!({
"method": "POST", "endpoint": CODEX_RESPONSES_PATH,
}),
headers
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes())),
);
trace.capture("upstream.request.body", &body);
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
.map_err(CodexClientError::RequestCompression)?;
let response = self
@@ -133,6 +129,17 @@ impl CodexBackendClient {
let upstream_headers_ms = elapsed_duration_millis(headers_started_at.elapsed());
let http_version = http_version_name(response.version()).to_string();
let status = response.status();
trace.headers(
"upstream.response.headers",
serde_json::json!({
"status": status.as_u16(), "httpVersion": http_version,
"headersMs": upstream_headers_ms,
}),
response
.headers()
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes())),
);
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
let turn_state = response_meta::turn_state(response.headers());
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
@@ -147,6 +154,7 @@ impl CodexBackendClient {
.map(|value| value.as_bytes().to_vec());
let client_headers = response_meta::client_headers(response.headers());
let raw_body = read_error_response_body(response).await?;
trace.capture("upstream.error.body", &raw_body);
let body = String::from_utf8_lossy(&raw_body).into_owned();
let retry_after_seconds =
retry_after_seconds.or_else(|| retry_after_seconds_from_body(&body));
@@ -175,7 +183,7 @@ impl CodexBackendClient {
let rate_limit_updates = Arc::new(tokio::sync::Mutex::new(Vec::new()));
Ok(CodexBackendStreamingResponse {
body: http_sse_stream(response, Arc::clone(&rate_limit_updates)),
body: http_sse_stream(response, Arc::clone(&rate_limit_updates), trace),
transport: CodexBackendTransport::HttpSse,
websocket_connection_id: None,
turn_state,
@@ -258,6 +266,9 @@ impl CodexBackendClient {
defer_websocket_recovery: bool,
) -> CodexClientResult<PreparedResponseTransport> {
let requirement = transport_requirement(request);
context.trace.cloned().unwrap_or_default().record("transport.preparing", serde_json::json!({
"requirement": requirement.as_str(), "connectionPreference": format!("{connection_preference:?}"),
}));
if requirement == TransportRequirement::HttpRequired {
return Ok(PreparedResponseTransport {
requirement,
@@ -279,6 +290,15 @@ impl CodexBackendClient {
&websocket_request,
)
.map_err(CodexClientError::WebSocketEncode)?;
context.trace.cloned().unwrap_or_default().headers(
"upstream.request.headers",
serde_json::json!({"transport": "websocket", "phase": "prepared_opening"}),
websocket_create
.connection()
.headers()
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes())),
);
// 审计未启用时跳过 artifact 构造:payload 快照会深拷贝整个请求 body,
// 且位于首字节前的关键路径上。
if websocket_audit_dir().is_some() {
@@ -326,14 +346,13 @@ impl CodexBackendClient {
Ok(WebSocketFastPath::Missed) => {
let decision = CodexTransportDecision::Http2WebSocketBudgetExhausted;
let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed());
tracing::info!(
request_id = %context.request_id,
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
transport_requirement = requirement.as_str(),
transport_decision = decision.as_str(),
transport_decision_wait_ms = wait_ms,
websocket_preconnect_continues = pool_log_context.is_some(),
"WebSocket fast path missed; using same-account HTTP"
context.trace.cloned().unwrap_or_default().record(
"transport.fallback",
serde_json::json!({
"to": "http_sse", "reason": "websocket_fast_path_budget",
"requirement": requirement.as_str(), "decision": decision.as_str(),
"waitMs": wait_ms, "preconnectContinues": pool_log_context.is_some(),
}),
);
return Ok(PreparedResponseTransport {
requirement,
@@ -354,14 +373,13 @@ impl CodexBackendClient {
{
let decision = http_fallback_decision(&error);
let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed());
tracing::warn!(
request_id = %context.request_id,
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
transport_requirement = requirement.as_str(),
transport_decision = decision.as_str(),
transport_decision_wait_ms = wait_ms,
error = %error,
"WebSocket preparation failed before payload send; using same-account HTTP"
context.trace.cloned().unwrap_or_default().record(
"transport.fallback", serde_json::json!({
"to": "http_sse", "reason": "websocket_pre_send_failure",
"requirement": requirement.as_str(), "decision": decision.as_str(),
"waitMs": wait_ms,
"detail": diagnostic_json(&serde_json::json!({"message": error.to_string()})),
}),
);
return Ok(PreparedResponseTransport {
requirement,
@@ -378,6 +396,15 @@ impl CodexBackendClient {
Err(error) => return Err(websocket_exchange_error_to_client_error(error)),
};
let decision = websocket_success_decision(requirement, &prepared);
context.trace.cloned().unwrap_or_default().record(
"transport.prepared",
serde_json::json!({
"decision": decision.as_str(),
"requirement": requirement.as_str(),
"connectMs": prepared.connect_elapsed().map(elapsed_duration_millis),
"waitMs": elapsed_duration_millis(prepared.decision_wait_elapsed()),
}),
);
let metrics = CodexTransportMetrics {
decision: Some(decision),
ws_connect_ms: prepared.connect_elapsed().map(elapsed_duration_millis),
@@ -394,15 +421,6 @@ impl CodexBackendClient {
pool_log_context.as_ref(),
prepared.pool_decision(),
);
tracing::info!(
request_id = %context.request_id,
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
transport_requirement = requirement.as_str(),
transport_decision = decision.as_str(),
ws_connect_ms = ?metrics.ws_connect_ms,
transport_decision_wait_ms = ?metrics.transport_decision_wait_ms,
"Responses transport prepared"
);
Ok(PreparedResponseTransport {
requirement,
route: PreparedResponseRoute::WebSocket(Box::new(PreparedWebSocketRoute {
@@ -427,6 +445,13 @@ impl CodexBackendClient {
metrics,
defer_websocket_recovery,
} = prepared;
context.trace.cloned().unwrap_or_default().record(
"transport.selected",
serde_json::json!({
"decision": metrics.decision.map(|decision| decision.as_str()),
"requirement": requirement.as_str(), "waitMs": metrics.transport_decision_wait_ms,
}),
);
match route {
PreparedResponseRoute::Http => self
.create_response_stream_http_sse(request, context)
@@ -441,10 +466,14 @@ impl CodexBackendClient {
prepared,
} = *route;
let delivery_wait_started_at = Instant::now();
log_unredacted_websocket_request(&websocket_request, context);
let mut exchange = match execute_prepared_response_create_request_stream(
&websocket_request,
prepared,
context
.trace
.cloned()
.unwrap_or_default()
.exchange("websocket"),
)
.await
{
@@ -493,9 +522,12 @@ impl CodexBackendClient {
.await;
}
fresh_retry_used = true;
tracing::warn!(
request_id = %context.request_id,
"WebSocket connection limit reached; retrying on a fresh connection"
context.trace.cloned().unwrap_or_default().record(
"transport.retry",
serde_json::json!({
"reason": "websocket_connection_limit_reached",
"connectionPreference": "fresh",
}),
);
drop(exchange);
match self
@@ -512,10 +544,14 @@ impl CodexBackendClient {
route: PreparedResponseRoute::WebSocket(route),
..
}) => {
log_unredacted_websocket_request(&route.request, context);
match execute_prepared_response_create_request_stream(
&route.request,
route.prepared,
context
.trace
.cloned()
.unwrap_or_default()
.exchange("websocket"),
)
.await
{
@@ -575,8 +611,9 @@ impl CodexBackendClient {
}
}
}
Err(error) if !defer_websocket_recovery
&& error.allows_pre_delivery_http_fallback() =>
Err(error)
if !defer_websocket_recovery
&& error.allows_pre_delivery_http_fallback() =>
{
return self
.fallback_to_http_before_websocket_delivery(
@@ -597,12 +634,6 @@ impl CodexBackendClient {
}
}
}
tracing::info!(
request_id = %context.request_id,
websocket_connection_id = %exchange.websocket_connection_id,
ws_pool = exchange.pool_decision.map_or("unpooled", WebSocketPoolDecision::kind),
"WebSocket response stream established"
);
Ok(CodexBackendStreamingResponse {
body: Box::pin(
exchange
@@ -625,15 +656,6 @@ impl CodexBackendClient {
})
}
}
.inspect_err(|error| {
tracing::warn!(
request_id = %context.request_id,
transport_requirement = requirement.as_str(),
failure_phase = "post_send_or_explicit_response",
error = %error,
"Responses stream transport failed after preparation"
);
})
}
async fn fallback_to_http_before_websocket_delivery(
@@ -671,14 +693,15 @@ impl CodexBackendClient {
.saturating_add(elapsed_duration_millis(delivery_wait_started_at.elapsed()));
metrics.decision = Some(CodexTransportDecision::Http2PreDeliveryFailure);
metrics.transport_decision_wait_ms = Some(decision_wait_ms);
tracing::warn!(
request_id = %context.request_id,
account_id = context.account_id.unwrap_or_default(),
transport_requirement = requirement.as_str(),
transport_decision = CodexTransportDecision::Http2PreDeliveryFailure.as_str(),
transport_decision_wait_ms = decision_wait_ms,
error = %detail,
"WebSocket failed before first deliverable event; using same-account HTTP"
context.trace.cloned().unwrap_or_default().record(
"transport.fallback",
serde_json::json!({
"to": "http_sse", "reason": "websocket_pre_delivery_failure",
"requirement": requirement.as_str(),
"decision": CodexTransportDecision::Http2PreDeliveryFailure.as_str(),
"waitMs": decision_wait_ms,
"detail": diagnostic_json(&serde_json::json!({"message": detail})),
}),
);
self.create_response_stream_http_sse(request, context)
.await
@@ -715,7 +738,7 @@ impl CodexBackendClient {
) -> CodexClientResult<CodexModelCatalogSnapshot> {
let endpoint = endpoint_url(&self.base_url, "codex/models");
let profile = self.profile.snapshot();
let headers = self.auxiliary_request_headers(&profile, context)?;
let headers = self.model_request_headers(&profile, context)?;
let response = self
.client
.get(endpoint)
@@ -751,182 +774,6 @@ impl CodexBackendClient {
}
Ok(parse_codex_model_catalog(&body, etag.as_deref())?)
}
fn request_headers_for_http_response(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let profile = self.profile.snapshot();
let mut headers = self.request_headers(&profile, context)?;
if let Some(subagent) = openai_subagent_from_metadata(request.client_metadata()) {
insert_optional_protocol_header(&mut headers, "x-openai-subagent", Some(&subagent));
}
insert_optional_protocol_header(
&mut headers,
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER,
request.responses_lite.as_deref(),
);
insert_optional_protocol_header(
&mut headers,
X_OPENAI_MEMGEN_REQUEST_HEADER,
request.memgen_request.as_deref(),
);
// 客户端确实携带的普通协议头优先;画像身份头只能由 Codex profile 生成。
for name in request.passthrough_headers.keys() {
// 上游指纹必须由运行时画像统一生成:originator/User-Agent/version 即使
// 绕过 API 透传黑名单也不能覆盖画像,避免下游客户端暴露不一致指纹。
if matches!(
name.as_str(),
"openai-beta"
| "originator"
| "user-agent"
| "version"
| "x-oai-attestation"
| "x-oai-is"
| "x-oai-is-update"
| "x-openai-internal-codex-residency"
) {
continue;
}
headers.remove(name);
for value in request.passthrough_headers.get_all(name) {
headers.append(name.clone(), value.clone());
}
}
Ok(headers)
}
fn request_headers_for_websocket_response(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers = self.request_headers_for_http_response(request, context)?;
headers.remove(X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER);
headers.insert(
HeaderName::from_static("openai-beta"),
HeaderValue::from_static("responses_websockets=2026-02-06"),
);
Ok(headers)
}
fn request_headers(
&self,
profile: &CodexWireProfile,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers =
build_codex_base_headers(profile, context.authorization, context.account_id)?;
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream"));
headers.insert(
HeaderName::from_static("x-client-request-id"),
HeaderValue::from_str(context.request_id)?,
);
insert_optional_protocol_header(
&mut headers,
"x-client-request-id",
context.client_request_id,
);
insert_optional_header(
&mut headers,
"x-codex-installation-id",
context.installation_id,
)?;
insert_optional_protocol_header(&mut headers, "session-id", context.session_id);
insert_optional_protocol_header(&mut headers, "thread-id", context.thread_id);
insert_optional_protocol_header(&mut headers, "x-codex-turn-id", context.turn_id);
insert_optional_protocol_header(&mut headers, "x-codex-window-id", context.codex_window_id);
insert_optional_protocol_header(&mut headers, "x-codex-turn-state", context.turn_state);
insert_optional_protocol_header(
&mut headers,
"x-codex-turn-metadata",
context.turn_metadata,
);
insert_optional_protocol_header(
&mut headers,
"x-codex-beta-features",
context.beta_features,
);
insert_optional_protocol_header(
&mut headers,
"x-responsesapi-include-timing-metrics",
context.include_timing_metrics,
);
// 官方 Desktop 的版本画像来自 clientInfo.version。保留下游是否携带
// `version` 扩展头的协议形状,但上游值统一使用当前 Desktop 制品版本。
if context
.version
.is_some_and(|version| HeaderValue::from_str(version).is_ok())
{
headers.insert(
HeaderName::from_static("version"),
HeaderValue::from_str(&profile.desktop_version)?,
);
}
insert_optional_protocol_header(
&mut headers,
"x-codex-parent-thread-id",
context.parent_thread_id,
);
Ok(headers)
}
fn auxiliary_request_headers(
&self,
profile: &CodexWireProfile,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers =
build_codex_base_headers(profile, context.authorization, context.account_id)?;
if let Some(cookie_header) = context.cookie_header {
headers.insert(COOKIE, HeaderValue::from_str(cookie_header)?);
}
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
insert_optional_header(
&mut headers,
"x-codex-installation-id",
context.installation_id,
)?;
Ok(headers)
}
pub(in crate::transport) fn usage_request_headers(
&self,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let profile = self.profile.snapshot();
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
headers.insert(AUTHORIZATION, HeaderValue::from_str(context.authorization)?);
insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?;
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
Ok(headers)
}
}
fn log_unredacted_websocket_request(
request: &CodexWebSocketRequest,
context: CodexRequestContext<'_>,
) {
tracing::info!(
target: "request_dump",
provider = "openai",
request_id = %context.request_id,
account_id = context.account_id.unwrap_or_default(),
request_direction = "upstream",
request_transport = "websocket",
request_method = "GET + response.create",
request_url = request.connection().endpoint(),
request_headers = ?request.connection().headers(),
request_payload = request.payload_text(),
contains_sensitive_data = true,
"request dump (unredacted)"
);
}
/// 首个可投递帧前的交付边界结果。
@@ -1011,6 +858,7 @@ fn websocket_connection_profile(headers: &HeaderMap) -> String {
fn http_sse_stream(
response: ReqwestResponse,
rate_limit_updates: CodexRateLimitUpdates,
trace: TraceContext,
) -> CodexBackendSseStream {
let stream: CodexBackendSseStream =
Box::pin(response.bytes_stream().map_err(CodexClientError::Http));
@@ -1028,6 +876,20 @@ fn http_sse_stream(
)),
}
}));
let stream = Box::pin(async_stream::stream! {
let mut stream = stream;
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
while let Some(chunk) = stream.next().await {
match &chunk {
Ok(bytes) => capture.push(bytes),
Err(error) => trace.record("upstream.read.failed", diagnostic_json(&serde_json::json!({"error": error.to_string()}))),
}
let failed = chunk.is_err();
yield chunk;
if failed { return; }
}
capture.finish();
});
observe_http_sse_rate_limits(stream, rate_limit_updates)
}
@@ -29,12 +29,17 @@ pub fn endpoint_url(base_url: &str, endpoint_path: &str) -> String {
/// 返回与 base path 对应的唯一 usage endpoint。
pub fn usage_endpoint_url(base_url: &str) -> String {
let path = if has_backend_api_base_path(base_url) {
WHAM_USAGE_PATH
account_endpoint_url(base_url, "usage")
}
/// Core backend-client 的两个官方路径风格,供所有账号接口共用。
pub fn account_endpoint_url(base_url: &str, resource: &str) -> String {
let prefix = if has_backend_api_base_path(base_url) {
"wham"
} else {
CODEX_USAGE_API_PATH
"api/codex"
};
endpoint_url(base_url, path)
endpoint_url(base_url, &format!("{prefix}/{resource}"))
}
fn has_backend_api_base_path(base_url: &str) -> bool {
@@ -1,23 +1,37 @@
use crate::transport::profile::CodexWireProfile;
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue, USER_AGENT};
//! 各官方调用链的请求头契约;版本与平台信息统一来自运行时画像快照。
use super::client::CodexClientResult;
use gateway_protocol::openai::{
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
};
use reqwest::header::{
ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT,
};
use super::client::{
CodexBackendClient, CodexClientResult, CodexRequestContext, openai_subagent_from_metadata,
};
use super::profile::{CodexResidency, CodexWireProfile};
use super::protocol::responses::CodexResponsesRequest;
const CODEX_RESIDENCY_HEADER: &str = "x-openai-internal-codex-residency";
/// 构造 Codex Core 为模型请求设置的稳定身份请求头。
pub fn build_codex_base_headers(
pub fn build_codex_model_headers(
profile: &CodexWireProfile,
authorization: &str,
account_id: Option<&str>,
) -> CodexClientResult<HeaderMap> {
let mut headers = build_codex_profile_headers(profile)?;
headers.insert(
HeaderName::from_static("version"),
HeaderValue::from_str(&profile.codex_version)?,
);
headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?);
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
Ok(headers)
}
/// 构造不依赖访问凭据的 Codex Desktop 身份请求头。
/// Core 默认客户端身份,用于模型接口及 OAuth refresh;raw token exchange 不使用。
pub fn build_codex_profile_headers(profile: &CodexWireProfile) -> CodexClientResult<HeaderMap> {
let mut headers = HeaderMap::new();
headers.insert(
@@ -25,14 +39,182 @@ pub fn build_codex_profile_headers(profile: &CodexWireProfile) -> CodexClientRes
HeaderValue::from_str(&profile.originator)?,
);
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
// 此部署固定采用官方支持的 US residency requirement。
if let Some(CodexResidency::Us) = profile.residency {
headers.insert(
HeaderName::from_static(CODEX_RESIDENCY_HEADER),
HeaderValue::from_static("us"),
);
}
Ok(headers)
}
/// Core backend-client 的账号请求:不携带模型版本或 originator。
pub fn build_codex_account_headers(
profile: &CodexWireProfile,
authorization: &str,
account_id: Option<&str>,
) -> CodexClientResult<HeaderMap> {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?);
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
Ok(headers)
}
/// 桌面下载的账号身份;完整性凭据须由认证主体持有,不能借用下游客户端的值。
pub fn build_codex_download_headers(
profile: &CodexWireProfile,
authorization: &str,
account_id: Option<&str>,
) -> CodexClientResult<HeaderMap> {
let mut headers = HeaderMap::new();
headers.insert(
HeaderName::from_static(CODEX_RESIDENCY_HEADER),
HeaderValue::from_static("us"),
USER_AGENT,
HeaderValue::from_str(&profile.desktop_user_agent())?,
);
headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?);
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
headers.insert(
HeaderName::from_static("originator"),
HeaderValue::from_str(&profile.originator)?,
);
Ok(headers)
}
impl CodexBackendClient {
pub(super) fn model_request_headers(
&self,
profile: &CodexWireProfile,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers =
build_codex_model_headers(profile, context.authorization, context.account_id)?;
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
Ok(headers)
}
pub(super) fn account_request_headers(
&self,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers = build_codex_account_headers(
&self.profile.snapshot(),
context.authorization,
context.account_id,
)?;
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
Ok(headers)
}
pub(super) fn request_headers_for_http_response(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers = self.response_headers(request, context)?;
headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream"));
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
insert_optional_protocol_header(
&mut headers,
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER,
request.responses_lite.as_deref(),
);
Ok(headers)
}
pub(super) fn request_headers_for_websocket_response(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers = self.response_headers(request, context)?;
headers.insert(
HeaderName::from_static("openai-beta"),
HeaderValue::from_static("responses_websockets=2026-02-06"),
);
Ok(headers)
}
/// HTTP 和 WS 共用的 Core 会话字段;传输专属字段由各自入口生成。
fn response_headers(
&self,
request: &CodexResponsesRequest,
context: CodexRequestContext<'_>,
) -> CodexClientResult<HeaderMap> {
let mut headers = self.model_request_headers(&self.profile.snapshot(), context)?;
headers.insert(
HeaderName::from_static("x-client-request-id"),
HeaderValue::from_str(context.request_id)?,
);
insert_optional_protocol_header(
&mut headers,
"x-client-request-id",
context
.client_request_id
.or(context.thread_id)
.or(context.session_id),
);
for (name, value) in [
("session-id", context.session_id),
("thread-id", context.thread_id),
("x-codex-window-id", context.codex_window_id),
("x-codex-turn-state", context.turn_state),
("x-codex-turn-metadata", context.turn_metadata),
("x-codex-beta-features", context.beta_features),
(
"x-responsesapi-include-timing-metrics",
context.include_timing_metrics,
),
("x-codex-parent-thread-id", context.parent_thread_id),
(
X_OPENAI_MEMGEN_REQUEST_HEADER,
request.memgen_request.as_deref(),
),
] {
insert_optional_protocol_header(&mut headers, name, value);
}
if let Some(subagent) = openai_subagent_from_metadata(request.client_metadata()) {
insert_optional_protocol_header(&mut headers, "x-openai-subagent", Some(&subagent));
}
let routing_hint = match request.service_tier() {
Some(tier) => format!("model={};tier={tier}", request.model()),
None => format!("model={}", request.model()),
};
insert_optional_protocol_header(&mut headers, "x-codex-routing-hint", Some(&routing_hint));
for name in request.passthrough_headers.keys() {
// 身份与传输字段只由画像/正文生成;其余协议头保留原始多值字节。
if matches!(
name.as_str(),
"originator"
| "user-agent"
| "version"
| "authorization"
| "chatgpt-account-id"
| "cookie"
| "x-openai-internal-codex-residency"
| "openai-beta"
| "accept"
| "content-type"
| "content-encoding"
| "x-codex-routing-hint"
| "x-codex-installation-id"
| "x-codex-turn-id"
| "x-oai-attestation"
| "x-oai-is"
| "x-oai-is-update"
| X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER
) {
continue;
}
headers.remove(name);
for value in request.passthrough_headers.get_all(name) {
headers.append(name.clone(), value.clone());
}
}
Ok(headers)
}
}
pub(super) fn insert_optional_header(
headers: &mut HeaderMap,
name: &'static str,
@@ -42,7 +42,7 @@ pub use self::{
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url,
},
headers::build_codex_base_headers,
headers::build_codex_model_headers,
profile_avatar::{
CodexProfileAvatar, CodexProfileAvatarFetchError, CodexProfileAvatarStreamError,
fetch_profile_avatar,
@@ -32,6 +32,13 @@ const MAX_APPCAST_BYTES: usize = 1024 * 1024;
const ARTIFACT_PROFILE_CACHE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1;
/// 与官方 managed residency requirement 相同的可选请求约束。
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum CodexResidency {
Us,
}
/// Codex Desktop 上游请求身份。
///
/// 启动配置提供经源码审计的 Core、运行环境和 Desktop 启动版本。运行时只会使用
@@ -40,7 +47,7 @@ const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1;
pub struct CodexWireProfile {
/// `originator` 请求头及 User-Agent 产品名。
pub originator: String,
/// Desktop ZIP 内嵌 Core 版本;用于 `/codex/models?client_version=` 与 UA。
/// Desktop ZIP 内嵌 Core 版本;用于模型请求的 version、client_version 与 UA。
pub codex_version: String,
/// Desktop 应用版本,用于 app-server `clientInfo.version` 对应的 UA 后缀。
pub desktop_version: String,
@@ -54,7 +61,9 @@ pub struct CodexWireProfile {
pub arch: String,
/// Codex Core UA 中的终端标记。
pub terminal: String,
/// 此画像最后一次经制品与源码核验的时间。
/// 未配置时不发送 residency 头;不随制品版本更新而改变。
pub residency: Option<CodexResidency>,
/// 版本元组最后一次经制品核验的时间;不表示 TLS 传输已重新核验。
pub verified_at: DateTime<Utc>,
}
@@ -7,17 +7,18 @@ use bytes::Bytes;
use futures::{Stream, StreamExt as _};
use reqwest::{
Client, Url,
header::{ACCEPT, CONTENT_TYPE, ETAG, USER_AGENT},
header::{CONTENT_TYPE, ETAG},
};
use tokio::time::timeout;
use super::endpoints::endpoint_url;
use super::{
CodexRequestContext, headers::build_codex_download_headers, profile::CodexWireProfile,
};
const OFFICIAL_AVATAR_ORIGIN: &str = "https://chatgpt.com";
const OFFICIAL_AVATAR_PATH_PREFIX: &str = "/backend-api/estuary/public_content/enc/";
const PROVIDER_AVATAR_PATH_PREFIX: &str = "/estuary/public_content/enc/";
const PROFILE_AVATAR_ACCEPT: &str =
"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8";
const PROFILE_AVATAR_HEADERS_TIMEOUT: Duration = Duration::from_secs(15);
const PROFILE_AVATAR_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
@@ -68,19 +69,18 @@ pub enum CodexProfileAvatarFetchError {
pub async fn fetch_profile_avatar(
client: &Client,
base_url: &str,
desktop_user_agent: &str,
profile: &CodexWireProfile,
source: &str,
context: CodexRequestContext<'_>,
) -> Result<CodexProfileAvatar, CodexProfileAvatarFetchError> {
let source_path = official_avatar_source_path(source)?;
let target = Url::parse(&endpoint_url(base_url, &source_path))
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?;
let headers = build_codex_download_headers(profile, context.authorization, context.account_id)
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?;
let response = timeout(
PROFILE_AVATAR_HEADERS_TIMEOUT,
client
.get(target)
.header(ACCEPT, PROFILE_AVATAR_ACCEPT)
.header(USER_AGENT, desktop_user_agent)
.send(),
client.get(target).headers(headers).send(),
)
.await
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?
@@ -12,7 +12,7 @@ use super::{
retry_after_seconds,
},
diagnostics::CodexUpstreamSendPhase,
endpoints::{WHAM_PROFILE_STATISTICS_PATH, endpoint_url},
endpoints::account_endpoint_url,
response_meta,
};
@@ -167,8 +167,8 @@ impl CodexBackendClient {
) -> CodexClientResult<CodexProfileStatistics> {
let response = self
.client
.get(endpoint_url(&self.base_url, WHAM_PROFILE_STATISTICS_PATH))
.headers(self.usage_request_headers(context)?)
.get(account_endpoint_url(&self.base_url, "profiles/me"))
.headers(self.account_request_headers(context)?)
.send()
.await?;
let status = response.status();
@@ -320,8 +320,8 @@ fn normalize_conversation_anchor_text(text: &str) -> String {
/// 把客户端正文收敛到当前 lease 的账号身份边界。
///
/// 真实 account ID 与 installation ID 由随后构造的 `CodexRequestContext` 注入
/// 请求头;正文只替换客户端原本声明过的 installation 字段,绝不接受客户端
/// 真实 account ID 由随后构造的 `CodexRequestContext` 注入请求头;installation ID
/// 统一写入 Core client_metadata,并替换原有兼容字段,绝不接受客户端
/// 提供的 token、cookie 或账号身份。`input` 是 Responses 的可回放会话正文,
/// item ID、encrypted content 和 compaction 都必须原样保留。
pub(crate) fn scope_request_to_account(
@@ -371,59 +371,57 @@ pub(crate) fn scope_request_to_account(
replace_existing_body_string(request, key, scoped.as_deref());
}
if let Some(client_metadata) = request.client_metadata().cloned() {
let scoped = match client_metadata {
Value::Object(mut metadata) => {
let scoped_turn_metadata =
["turnMetadata", "turn_metadata", "x-codex-turn-metadata"].map(|key| {
(
key,
metadata.get(key).and_then(Value::as_str).and_then(|value| {
scope_turn_metadata(value, installation_id, reset_account_state)
}),
)
});
if reset_account_state {
for key in CROSS_ACCOUNT_IDENTITY_KEYS
.iter()
.chain(ACCOUNT_BOUND_STATE_KEYS)
{
metadata.remove(*key);
}
let client_metadata = request
.client_metadata()
.cloned()
.unwrap_or_else(|| Value::Object(Map::new()));
let scoped = match client_metadata {
Value::Object(mut metadata) => {
let scoped_turn_metadata = ["turnMetadata", "turn_metadata", "x-codex-turn-metadata"]
.map(|key| {
(
key,
metadata.get(key).and_then(Value::as_str).and_then(|value| {
scope_turn_metadata(value, installation_id, reset_account_state)
}),
)
});
if reset_account_state {
for key in CROSS_ACCOUNT_IDENTITY_KEYS
.iter()
.chain(ACCOUNT_BOUND_STATE_KEYS)
{
metadata.remove(*key);
}
metadata.insert(
"x-codex-installation-id".to_owned(),
Value::String(installation_id.to_owned()),
);
replace_existing_metadata_field(
&mut metadata,
"installation_id",
Some(installation_id),
);
replace_existing_metadata_field(
&mut metadata,
"installationId",
Some(installation_id),
);
replace_metadata_field(
&mut metadata,
"x-codex-turn-state",
client_metadata_turn_state.as_deref(),
);
replace_metadata_field(
&mut metadata,
"x-codex-turn-metadata",
client_metadata_turn_metadata.as_deref(),
);
for (key, value) in scoped_turn_metadata {
replace_existing_metadata_field(&mut metadata, key, value.as_deref());
}
(!metadata.is_empty()).then_some(Value::Object(metadata))
}
value => Some(value),
};
request.set_client_metadata(scoped);
}
metadata.insert(
"installation_id".to_owned(),
Value::String(installation_id.to_owned()),
);
replace_existing_metadata_field(
&mut metadata,
"x-codex-installation-id",
Some(installation_id),
);
replace_existing_metadata_field(&mut metadata, "installationId", Some(installation_id));
replace_metadata_field(
&mut metadata,
"x-codex-turn-state",
client_metadata_turn_state.as_deref(),
);
replace_metadata_field(
&mut metadata,
"x-codex-turn-metadata",
client_metadata_turn_metadata.as_deref(),
);
for (key, value) in scoped_turn_metadata {
replace_existing_metadata_field(&mut metadata, key, value.as_deref());
}
(!metadata.is_empty()).then_some(Value::Object(metadata))
}
value => Some(value),
};
request.set_client_metadata(scoped);
request.turn_state = turn_state;
request.turn_metadata = turn_metadata;
@@ -1,10 +1,7 @@
//! Codex Desktop 主动额度重置卡 HTTP contract。
use chrono::{DateTime, Utc};
use reqwest::{
StatusCode,
header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT},
};
use reqwest::StatusCode;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
@@ -15,11 +12,7 @@ use super::{
retry_after_seconds,
},
diagnostics::CodexUpstreamSendPhase,
endpoints::{
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
endpoint_url,
},
headers::insert_optional_header,
endpoints::account_endpoint_url,
response_meta,
};
@@ -94,11 +87,11 @@ impl CodexBackendClient {
) -> CodexClientResult<CodexRateLimitResetCredits> {
let response = self
.client
.get(endpoint_url(
.get(account_endpoint_url(
&self.base_url,
WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
"rate-limit-reset-credits",
))
.headers(self.reset_credits_headers(context, false)?)
.headers(self.account_request_headers(context)?)
.send()
.await
.map_err(CodexClientError::HttpJson)?;
@@ -131,11 +124,11 @@ impl CodexBackendClient {
) -> CodexClientResult<CodexRateLimitResetCreditsConsumeResult> {
let response = self
.client
.post(endpoint_url(
.post(account_endpoint_url(
&self.base_url,
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH,
"rate-limit-reset-credits/consume",
))
.headers(self.reset_credits_headers(context, true)?)
.headers(self.account_request_headers(context)?)
.json(&ConsumeRequest {
credit_id,
redeem_request_id: redeem_request_id.to_string(),
@@ -151,36 +144,6 @@ impl CodexBackendClient {
credit: wire.credit.map(parse_credit).transpose()?,
})
}
fn reset_credits_headers(
&self,
context: CodexRequestContext<'_>,
json_body: bool,
) -> CodexClientResult<HeaderMap> {
let profile = self.profile.snapshot();
let mut headers = HeaderMap::new();
headers.insert(AUTHORIZATION, HeaderValue::from_str(context.authorization)?);
insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?;
headers.insert(
HeaderName::from_static("originator"),
HeaderValue::from_str(&profile.originator)?,
);
headers.insert(
HeaderName::from_static("oai-language"),
HeaderValue::from_static("en"),
);
headers.insert(
USER_AGENT,
HeaderValue::from_str(&profile.desktop_user_agent())?,
);
// Electron `net.fetch` 为 renderer 请求附加的默认 Accept;该 endpoint 的
// renderer 并没有显式改写为 Core auxiliary 请求的 application/json 画像。
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
if json_body {
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
}
Ok(headers)
}
}
async fn read_reset_credits_response(response: reqwest::Response) -> CodexClientResult<String> {
@@ -15,13 +15,13 @@ const CA_CERT_HINT: &str = "If you set CODEX_CA_CERTIFICATE or SSL_CERT_FILE, en
type PemSection = (SectionKind, Vec<u8>);
/// rustls 0.23 在同时编译 `ring` 与 `aws-lc-rs` 时无法自动选择进程级
/// CryptoProvider;本工程固定使用 ring(与官方 Codex Desktop JA3 一致),
/// 因此在首次 TLS 使用前显式安装一次。重复安装会被 rustls 忽略。
/// 与官方 Codex 的 rustls 路径一致,安装支持 P-521 和混合后量子密钥交换的
/// aws-lc provider。依赖图也包含 ring,因此不能让 rustls 自动选择 provider。
/// 与官方嵌入场景一致,保留宿主在初始化之前已经安装的 provider。
pub fn ensure_rustls_provider() {
static INSTALL: OnceLock<()> = OnceLock::new();
INSTALL.get_or_init(|| {
let _ = rustls::crypto::ring::default_provider().install_default();
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
});
}
@@ -676,7 +676,7 @@ pub const MAX_CODEX_USAGE_BODY_BYTES: usize = 1024 * 1024;
impl CodexBackendClient {
/// 获取 Codex usage JSON。
pub async fn fetch_usage(&self, context: CodexRequestContext<'_>) -> CodexClientResult<Value> {
let headers = self.usage_request_headers(context)?;
let headers = self.account_request_headers(context)?;
let response = self
.client
.get(usage_endpoint_url(&self.base_url))
@@ -559,6 +559,7 @@ fn continuation_unavailable(
pub(crate) async fn execute_prepared_response_create_request_stream(
request: &CodexWebSocketRequest,
prepared: PreparedWebSocket,
trace: gateway_core::diagnostics::TraceContext,
) -> Result<CodexWebSocketStreamingExchange, CodexWebSocketExchangeError> {
let PreparedWebSocket {
connection,
@@ -573,7 +574,19 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
continuation,
created_at,
} = connection;
trace.record("upstream.connection", serde_json::json!({
"connectionId": websocket.connection_id().to_string(), "reused": reused,
"pool": pool_decision.map_or("unpooled", WebSocketPoolDecision::kind),
"status": metadata.diagnostics.status_code,
"upstreamRequestId": metadata.diagnostics.request_id,
"headers": gateway_core::diagnostics::diagnostic_headers(metadata.diagnostics.trace_headers.iter().map(|(name, value)| (name.as_str(), value.as_str()))),
}));
trace.capture("upstream.request.body", request.payload_text().as_bytes());
if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await {
trace.record(
"upstream.send.failed",
serde_json::json!({"phase": "websocket_payload", "sendState": "ambiguous"}),
);
let observation = websocket
.observation()
.with_exit_reason("outbound_transport_error");
@@ -582,6 +595,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
error.with_connection_observation(observation),
));
}
trace.record("upstream.payload.sent", serde_json::json!({}));
let connection_local_available = lease.is_some();
let pool_return = lease.map(|lease| WebSocketStreamPoolReturn {
lease,
@@ -594,6 +608,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
pool_return,
reused,
initial_event_timeout,
trace,
);
exchange.pool_decision = pool_decision;
exchange.connection_local_continuation = connection_local_available;
@@ -3,6 +3,8 @@
use std::{sync::Arc, time::Duration};
use bytes::Bytes;
use gateway_core::diagnostics::{TraceContext, diagnostic_json};
use serde_json::json;
use tokio::sync::{Mutex, mpsc};
use tokio_util::sync::CancellationToken;
@@ -61,6 +63,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
pool_return: Option<WebSocketStreamPoolReturn>,
reused_connection: bool,
initial_event_timeout: Option<Duration>,
trace: TraceContext,
) -> CodexWebSocketStreamingExchange {
let websocket_connection_id = websocket.connection_id();
let response_metadata = metadata.clone();
@@ -83,6 +86,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
pool_return,
reused_connection,
initial_event_timeout,
trace,
shutdown,
rate_limit_updates: rate_limit_updates_for_task,
turn_state_update: turn_state_update_for_task,
@@ -116,6 +120,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
}
struct WebSocketStreamForwardState {
trace: TraceContext,
websocket: PumpedWebSocket,
metadata: CodexWebSocketConnectionMetadata,
pool_return: Option<WebSocketStreamPoolReturn>,
@@ -129,6 +134,7 @@ struct WebSocketStreamForwardState {
async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
let WebSocketStreamForwardState {
trace,
mut websocket,
mut metadata,
pool_return,
@@ -152,6 +158,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
// 客户端断开下游 SSE 流:立即丢弃连接并释放池 slot,
// 不再傻等上游 idle 超时(否则同会话后续请求会一直 bypass/busy)。
() = tx.closed() => {
trace.record("upstream.cancelled", json!({"reason": "receiver_dropped"}));
discard_stream_websocket(
websocket,
pool_return,
@@ -160,6 +167,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
return;
}
() = shutdown.cancelled() => {
trace.record("upstream.cancelled", json!({"reason": "pool_shutdown"}));
discard_stream_websocket(
websocket,
pool_return,
@@ -183,6 +191,13 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
}
error => error,
};
trace.record(
"upstream.read.failed",
json!({
"lastEventType": last_event_type,
"error": diagnostic_json(&json!({"message": error.to_string()})),
}),
);
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
@@ -201,6 +216,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
}
};
let Some(message) = message else {
trace.record(
"upstream.eof",
json!({"lastEventType": last_event_type, "terminalSeen": false}),
);
break;
};
let raw = match message {
@@ -208,7 +227,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
saw_upstream_activity = true;
text.to_string()
}
tungstenite::Message::Binary(_) => {
tungstenite::Message::Binary(bytes) => {
trace.capture("upstream.binary", &bytes);
let error = CodexWebSocketExchangeError::UnexpectedBinaryEvent;
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
@@ -223,6 +243,15 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
return;
}
tungstenite::Message::Close(frame) => {
if let Some(frame) = &frame {
trace.dump("upstream.close.reason", frame.reason.as_bytes());
}
trace.record("upstream.close", json!({
"code": frame.as_ref().map(|frame| u16::from(frame.code)),
"reason": frame.as_ref().map(|frame| diagnostic_json(&json!({"message": frame.reason.as_str()}))),
"lastEventType": last_event_type, "terminalSeen": false,
"connectionId": websocket.connection_id().to_string(),
}));
let websocket_connection_id = websocket.connection_id();
let error = CodexWebSocketExchangeError::closed_before_terminal_on(
websocket_connection_id,
@@ -248,6 +277,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
}
_ => continue,
};
trace.capture("upstream.event", raw.as_bytes());
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
Ok(reduced) => reduced,
Err(error) => {
@@ -286,6 +316,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
ExchangeAction::Ignore => continue,
};
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
trace.record(
"upstream.forward.failed",
json!({"reason": "receiver_dropped"}),
);
discard_stream_websocket(
websocket,
pool_return,
@@ -295,6 +329,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
return;
}
if let Some(terminal) = terminal {
trace.record(
"upstream.terminal",
json!({"kind": format!("{terminal:?}")}),
);
match terminal {
WebSocketTerminalKind::Completed => {
finish_stream_websocket(websocket, metadata, continuation, pool_return.take())
@@ -261,7 +261,7 @@ async fn openai_admin_provider_exposes_live_wire_profile_and_validated_billing()
async fn reset_credit_success_with_invalid_body_should_remain_an_unknown_consume_result() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/wham/rate-limit-reset-credits/consume"))
.and(path("/api/codex/rate-limit-reset-credits/consume"))
.respond_with(ResponseTemplate::new(200).set_body_raw("{}", "application/json"))
.expect(1)
.mount(&server)
@@ -287,7 +287,7 @@ async fn reset_credit_success_with_invalid_body_should_remain_an_unknown_consume
async fn reset_credit_explicit_http_rejection_should_preserve_the_raw_body() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/wham/rate-limit-reset-credits/consume"))
.and(path("/api/codex/rate-limit-reset-credits/consume"))
.respond_with(ResponseTemplate::new(409).set_body_raw(
r#"{"code":"nothing_to_reset","detail":"window is fresh"}"#,
"application/json",
@@ -1152,6 +1152,7 @@ fn valid_config() -> TestOpenAiConfig {
os_version: "15.5.0".to_owned(),
arch: "arm64".to_owned(),
terminal: "xterm-256color".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 7, 19, 0, 0, 0)
.single()
@@ -127,7 +127,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
);
assert_eq!(
config.wire_profile_state().snapshot().user_agent(),
"Codex Desktop/0.147.0-alpha.6.6 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.803.81509)"
"Codex Desktop/0.153.4 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.901.51231)"
);
}
@@ -153,6 +153,7 @@ fn valid_config() -> OpenAiConfig {
os_version: "15.5.0".to_owned(),
arch: "arm64".to_owned(),
terminal: "xterm-256color".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 7, 19, 0, 0, 0)
.single()
@@ -60,6 +60,7 @@ fn wire_profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "catalog-contract".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
.single()
@@ -160,6 +160,7 @@ fn selector_with_runtime(
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "selector-contract".to_owned(),
residency: None,
verified_at: chrono::Utc::now(),
});
let http = reqwest::Client::builder().build().expect("HTTP client");
@@ -310,7 +310,7 @@ async fn authorize_url_matches_the_official_desktop_parameter_contract() {
);
assert_eq!(
parameters.get("codex_app_version").map(String::as_str),
Some("26.803.81509")
Some("26.901.51231")
);
let surface_stable_id = parameters
.get("source_surface_stable_id")
@@ -21,6 +21,7 @@ fn wire_profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "profile-statistics-contract".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 8, 28, 0, 0, 0)
.single()
@@ -59,7 +60,7 @@ async fn service(
async fn profile_statistics_uses_one_profile_request_and_preserves_official_fields() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/wham/profiles/me"))
.and(path("/api/codex/profiles/me"))
.and(header(
"chatgpt-account-id",
"chatgpt-acct_profile_statistics",
@@ -135,7 +136,7 @@ async fn profile_statistics_uses_one_profile_request_and_preserves_official_fiel
async fn profile_statistics_keeps_profile_when_stats_are_unavailable() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/wham/profiles/me"))
.and(path("/api/codex/profiles/me"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"profile": {"display_name": "Ada"},
"stats": {},
@@ -158,10 +159,10 @@ async fn profile_statistics_keeps_profile_when_stats_are_unavailable() {
}
#[tokio::test]
async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials() {
async fn profile_avatar_reuses_cached_source_with_the_accounts_rotated_token() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/wham/profiles/me"))
.and(path("/api/codex/profiles/me"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"profile": {
"display_name": "Ada",
@@ -192,6 +193,11 @@ async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials()
.profile_statistics(account.id())
.await
.expect("profile statistics");
store
.repository()
.rotate_refreshed_oauth_secret(&account, secret("rotated-avatar-token"), None, None)
.await
.expect("rotate avatar account token");
let avatar = service
.profile_avatar(account.id())
.await
@@ -211,7 +217,13 @@ async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials()
assert_eq!(content_type.as_deref(), Some("image/svg+xml"));
assert_eq!(body, b"<svg xmlns=\"http://www.w3.org/2000/svg\"/>");
assert!(!avatar_request.headers.contains_key("authorization"));
assert_eq!(
avatar_request.headers["authorization"],
"Bearer rotated-avatar-token"
);
assert!(!avatar_request.headers.contains_key("cookie"));
assert!(!avatar_request.headers.contains_key("chatgpt-account-id"));
assert_eq!(
avatar_request.headers["chatgpt-account-id"],
"chatgpt-acct_profile_avatar"
);
}
@@ -33,6 +33,7 @@ fn wire_profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "quota-contract".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
.single()
@@ -431,6 +431,7 @@ async fn scheduled_refresh_persists_retryable_message_inside_the_two_hour_window
client_id: "test-public-client".to_owned(),
token_endpoint: format!("{}/oauth/token", server.uri()),
},
provider_openai::OpenAiConfig::default().wire_profile_state(),
)),
Arc::new(RefreshLeases),
Arc::new(RefreshCredentialState),
@@ -6,6 +6,73 @@ use secrecy::{ExposeSecret, SecretString};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
#[tokio::test]
async fn refresh_tracks_the_online_profile_without_contaminating_code_exchange() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/oauth/token"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"access_token": "audit-access", "refresh_token": "audit-refresh", "id_token": "header.e30.signature"
})))
.expect(4)
.mount(&server)
.await;
let profile = provider_openai::OpenAiConfig::default().wire_profile_state();
let client = provider_openai::credential::token_client::openai_token_client(
TokenClientConfig {
client_id: "audit-client".to_owned(),
token_endpoint: format!("{}/oauth/token", server.uri()),
},
profile.clone(),
)
.expect("production auth transport");
for (core, desktop, build) in [
("1.2.3", "26.800.10000", "8000"),
("1.2.4", "26.901.10000", "8109"),
] {
profile.update_bundled_release(
&provider_openai::transport::profile::CodexBundledReleaseProfile {
codex_version: core.to_owned(),
desktop_version: desktop.to_owned(),
desktop_build: build.to_owned(),
verified_at: chrono::Utc::now(),
},
);
client.refresh("audit-refresh").await.expect("refresh");
client
.exchange_authorization_code(AuthorizationCodeGrant {
code: SecretString::from("audit-code"),
code_verifier: SecretString::from("audit-verifier"),
})
.await
.expect("code exchange");
let requests = server.received_requests().await.expect("requests");
let refresh = &requests[requests.len() - 2];
let exchange = &requests[requests.len() - 1];
assert_eq!(
refresh.headers["user-agent"],
profile.snapshot().user_agent()
);
assert_eq!(refresh.headers["originator"], "Codex Desktop");
assert_eq!(refresh.headers["content-type"], "application/json");
assert_eq!(
exchange.headers["content-type"],
"application/x-www-form-urlencoded"
);
for name in [
"originator",
"user-agent",
"version",
"x-openai-internal-codex-residency",
] {
assert!(!exchange.headers.contains_key(name), "raw exchange: {name}");
}
for name in ["authorization", "chatgpt-account-id", "version"] {
assert!(!refresh.headers.contains_key(name), "refresh: {name}");
}
}
}
fn client(server: &MockServer) -> OpenAiTokenClient {
OpenAiTokenClient::new(
reqwest::Client::builder()
@@ -16,6 +83,7 @@ fn client(server: &MockServer) -> OpenAiTokenClient {
client_id: "test-public-client".to_owned(),
token_endpoint: format!("{}/oauth/token", server.uri()),
},
provider_openai::OpenAiConfig::default().wire_profile_state(),
)
}
@@ -136,6 +136,7 @@ fn wire_profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "provider-contract".to_owned(),
residency: None,
verified_at: Utc::now(),
})
}
@@ -1004,6 +1005,8 @@ async fn image_endpoints_bypass_only_the_text_catalog_and_preserve_the_current_c
.and(path(*endpoint))
.and(header("originator", "codex_cli_rs"))
.and(header("x-codex-image-turn-id", "turn_image_contract"))
.and(header("version", "0.144.0"))
.and(header("accept", "*/*"))
.and(body_bytes(body.to_vec()))
.respond_with(
ResponseTemplate::new(200)
@@ -1143,6 +1146,8 @@ async fn standalone_search_preserves_wire_and_scopes_turn_metadata_to_the_select
Mock::given(method("POST"))
.and(path("/codex/alpha/search"))
.and(header("originator", "codex_cli_rs"))
.and(header("version", "0.144.0"))
.and(header("accept", "*/*"))
.and(body_bytes(request_body.to_vec()))
.respond_with(
ResponseTemplate::new(200)
@@ -2731,6 +2736,15 @@ async fn same_account_scope_preserves_future_protocol_shapes() {
body.get("installation_id"),
Some(&json!("client-installation"))
);
assert_eq!(
body.get("installation_id"),
body.pointer("/client_metadata/installation_id")
);
assert!(
body.pointer("/client_metadata/x-codex-installation-id")
.is_none()
);
assert!(captured_header_values(&request, "x-codex-installation-id").is_empty());
}
#[tokio::test]
@@ -223,9 +223,8 @@ async fn fetch_should_send_official_catalog_headers_and_query() {
"user-agent",
"codex_cli_rs/0.144.0 (linux 6.8; x86_64) xterm (codex_cli_rs; 1.0.0)",
))
.and(header("x-openai-internal-codex-residency", "us"))
.and(header("accept", "application/json"))
.and(header("x-codex-installation-id", "installation-123"))
.and(header("accept", "*/*"))
.and(header("version", "0.144.0"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("content-type", "application/json")
@@ -306,6 +305,7 @@ fn profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "xterm".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
.single()
@@ -315,6 +315,7 @@ fn profile() -> CodexWireProfileState {
fn context() -> CodexRequestContext<'static> {
CodexRequestContext {
trace: None,
authorization: "Bearer oauth-access",
account_id: Some("acct_123"),
request_id: "req_catalog",
@@ -25,6 +25,7 @@ fn upstream_error_formatting_should_redact_body() {
#[test]
fn request_context_debug_should_redact_all_identity_material() {
let context = CodexRequestContext {
trace: None,
authorization: "Bearer access-secret-marker",
account_id: Some("account-secret-marker"),
request_id: "req_safe",
@@ -257,3 +257,87 @@ fn openai_account_failure_matrix_is_preserved() {
);
}
}
#[test]
fn diagnostic_wire_dump_preserves_binary_bytes_and_numbers_every_fragment() {
use base64::{Engine as _, engine::general_purpose::STANDARD};
use gateway_core::diagnostics::TraceContext;
use std::sync::{Arc, Mutex};
#[derive(Clone)]
struct Writer(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for Writer {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().write_all(bytes)?;
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let output = Arc::new(Mutex::new(Vec::new()));
let writer = Writer(Arc::clone(&output));
let subscriber = tracing_subscriber::fmt()
.json()
.with_writer(move || writer.clone())
.with_ansi(false)
.finish();
let bytes: Vec<u8> = (0..100_000).map(|index| (index % 256) as u8).collect();
tracing::subscriber::with_default(subscriber, || {
let trace = TraceContext::new("req_binary_dump");
trace
.attempt(3)
.exchange("websocket")
.dump("upstream.binary", &bytes);
trace.headers(
"client.connection.headers",
serde_json::json!({}),
[
("authorization", b"Bearer secret".as_slice()),
("x-binary", b"\xff\x80".as_slice()),
("x-binary", b"\x00".as_slice()),
],
);
let snapshot = trace.snapshot().unwrap();
assert_eq!(snapshot["wireFrames"], 2);
assert!(!snapshot.to_string().contains("Bearer secret"));
});
let log = output.lock().unwrap();
let records: Vec<serde_json::Value> = std::str::from_utf8(&log)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.filter(|record: &serde_json::Value| record["target"] == "request_dump")
.collect();
assert_eq!(records.len(), 5);
let mut recovered = Vec::new();
for (index, record) in records.iter().take(4).enumerate() {
let fields = &record["fields"];
assert_eq!(fields["wire_sequence"], 1);
assert_eq!(fields["attempt_index"], 3);
assert_eq!(fields["exchange_id"], 1);
assert_eq!(fields["chunk_index"], index);
assert_eq!(fields["chunk_count"], 4);
recovered.extend(
STANDARD
.decode(fields["body_base64"].as_str().unwrap())
.unwrap(),
);
}
assert_eq!(recovered, bytes);
let header_record = &records[4]["fields"];
assert_eq!(header_record["wire_sequence"], 2);
let header_bytes = STANDARD
.decode(header_record["body_base64"].as_str().unwrap())
.unwrap();
let headers: serde_json::Value = serde_json::from_slice(&header_bytes).unwrap();
for (index, expected) in [b"Bearer secret".as_slice(), b"\xff\x80", b"\x00"]
.into_iter()
.enumerate()
{
let value = STANDARD
.decode(headers["headers"][index]["valueBase64"].as_str().unwrap())
.unwrap();
assert_eq!(value, expected);
}
}
@@ -277,6 +277,7 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id: Some("chatgpt-account"),
request_id: "req_ws_security",
@@ -317,12 +318,8 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
);
let headers = received_headers.lock().expect("headers lock");
for (name, expected) in [
("x-client-request-id", "req_ws_security"),
("content-type", "application/json"),
("accept", "text/event-stream"),
("x-codex-installation-id", "install-123"),
("x-client-request-id", "cp_derived"),
("openai-beta", "responses_websockets=2026-02-06"),
("x-openai-internal-codex-residency", "us"),
("x-codex-turn-state", "turn-state"),
("x-codex-turn-metadata", "{\"thread_source\":\"subagent\"}"),
("x-codex-beta-features", "feature-a"),
@@ -341,6 +338,11 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
);
}
for forbidden in [
"x-codex-installation-id",
"x-codex-turn-id",
"x-openai-internal-codex-residency",
"accept",
"content-type",
"session_id",
"thread-id",
"x-openai-internal-codex-responses-lite",
@@ -365,7 +367,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
let profile = test_wire_profile();
let profile_snapshot = profile.snapshot();
let expected_user_agent = profile_snapshot.user_agent();
let expected_desktop_version = profile_snapshot.desktop_version;
let expected_core_version = profile_snapshot.codex_version;
let client = CodexBackendClient::new(
reqwest::Client::builder()
.no_proxy()
@@ -379,6 +381,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer lease-token",
account_id: Some("lease-account"),
installation_id: Some("lease-installation"),
@@ -401,16 +404,11 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
vec![b"turn-ascii".to_vec(), b"turn-\x80".to_vec()]
);
for (name, value) in [
("accept", b"application/vnd.openai.future+json".as_slice()),
(
"content-type",
b"application/vnd.openai.request+json".as_slice(),
),
("x-openai-internal-codex-residency", b"us".as_slice()),
("accept", b"text/event-stream".as_slice()),
("content-type", b"application/json".as_slice()),
("x-still-valid", b"after-invalid".as_slice()),
("authorization", b"Bearer lease-token".as_slice()),
("chatgpt-account-id", b"lease-account".as_slice()),
("x-codex-installation-id", b"lease-installation".as_slice()),
] {
assert_eq!(raw_header_values(&raw, name), vec![value.to_vec()]);
}
@@ -425,7 +423,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
);
assert_eq!(
raw_header_values(&raw, "version"),
vec![expected_desktop_version.into_bytes()]
vec![expected_core_version.into_bytes()]
);
for dropped in [
"openai-beta",
@@ -501,6 +499,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer lease-token",
account_id: Some("lease-account"),
installation_id: Some("lease-installation"),
@@ -529,24 +528,15 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
values("openai-beta"),
vec![b"responses_websockets=2026-02-06".to_vec()]
);
assert_eq!(
values("accept"),
vec![b"application/vnd.openai.future+json".to_vec()]
);
assert_eq!(
values("content-type"),
vec![b"application/vnd.openai.request+json".to_vec()]
);
assert!(values("accept").is_empty());
assert!(values("content-type").is_empty());
assert_eq!(
values("user-agent"),
vec![test_wire_profile().snapshot().user_agent().into_bytes()]
);
assert_eq!(values("originator"), vec![b"codex_cli_rs".to_vec()]);
assert_eq!(values("version"), vec![b"1.2.3".to_vec()]);
assert_eq!(
values("x-openai-internal-codex-residency"),
vec![b"us".to_vec()]
);
assert!(values("x-openai-internal-codex-residency").is_empty());
for dropped in [
"x-openai-actor-authorization",
"x-oai-attestation",
@@ -563,10 +553,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
values("chatgpt-account-id"),
vec![b"lease-account".to_vec()]
);
assert_eq!(
values("x-codex-installation-id"),
vec![b"lease-installation".to_vec()]
);
assert!(values("x-codex-installation-id").is_empty());
assert_eq!(values("x-codex-turn-state"), vec![b"turn-ascii".to_vec()]);
}
@@ -593,7 +580,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
let profile = test_wire_profile();
let profile_snapshot = profile.snapshot();
let expected_user_agent = profile_snapshot.user_agent();
let expected_desktop_version = profile_snapshot.desktop_version;
let expected_core_version = profile_snapshot.codex_version;
let client = CodexBackendClient::new(
reqwest::Client::builder()
.no_proxy()
@@ -607,6 +594,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id: Some("chatgpt-account"),
request_id: "req_order",
@@ -648,14 +636,14 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
Some("codex_cli_rs"),
Some(expected_user_agent.as_str()),
Some("text/event-stream"),
Some("us"),
Some("req_order"),
None,
Some("session-1"),
Some("turn-state"),
)
);
assert_eq!(
read_header_value(&raw_request, "version"),
Some(expected_desktop_version.as_str())
Some(expected_core_version.as_str())
);
for required in [
"authorization",
@@ -665,9 +653,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
"content-type",
"cookie",
"accept",
"x-openai-internal-codex-residency",
"x-client-request-id",
"x-codex-installation-id",
"session-id",
"x-codex-window-id",
"x-codex-turn-state",
@@ -680,6 +666,9 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
assert!(header_names.iter().any(|name| name == required));
}
for forbidden in [
"x-codex-installation-id",
"x-codex-turn-id",
"x-openai-internal-codex-residency",
"sec-ch-ua",
"sec-ch-ua-mobile",
"sec-ch-ua-platform",
@@ -729,6 +718,7 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo
.create_response(
&request,
CodexRequestContext {
trace: None,
client_request_id: Some("opaque\nclient-request-id"),
session_id: Some("opaque\nsession"),
thread_id: Some("opaque\nthread"),
@@ -751,6 +741,7 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo
read_header_value(&raw_request, "x-client-request-id"),
Some("req_protocol_fallback")
);
assert_eq!(read_header_value(&raw_request, "version"), Some("1.2.3"));
for omitted in [
"session-id",
"thread-id",
@@ -760,7 +751,6 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo
"x-codex-turn-metadata",
"x-codex-beta-features",
"x-responsesapi-include-timing-metrics",
"version",
"x-codex-parent-thread-id",
"x-openai-subagent",
"x-openai-internal-codex-responses-lite",
@@ -897,7 +887,14 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
let server = tokio::spawn(async move {
let (first_stream, _) = listener.accept().await.expect("first websocket");
let mut first_user_agent = String::new();
let mut first_version = String::new();
let mut first = accept_codex_test_websocket_with(first_stream, |request, _response| {
first_version = request.headers()["version"]
.to_str()
.expect("Core version")
.to_owned();
assert!(!request.headers().contains_key("accept"));
assert!(!request.headers().contains_key("content-type"));
first_user_agent = request
.headers()
.get("user-agent")
@@ -935,7 +932,14 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
.expect("new chain should use a new wire profile connection")
.expect("second websocket");
let mut second_user_agent = String::new();
let mut second_version = String::new();
let mut second = accept_codex_test_websocket_with(second_stream, |request, _response| {
second_version = request.headers()["version"]
.to_str()
.expect("Core version")
.to_owned();
assert!(!request.headers().contains_key("accept"));
assert!(!request.headers().contains_key("content-type"));
second_user_agent = request
.headers()
.get("user-agent")
@@ -956,7 +960,12 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
.await
.expect("second response.completed");
(first_user_agent, second_user_agent)
(
first_user_agent,
second_user_agent,
first_version,
second_version,
)
});
let profile = test_wire_profile();
@@ -983,7 +992,7 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
.expect("first response");
profile.update_bundled_release(&CodexBundledReleaseProfile {
codex_version: "1.2.4".to_owned(),
desktop_version: "1.2.4".to_owned(),
desktop_version: "26.999.10000".to_owned(),
desktop_build: "124".to_owned(),
verified_at: Utc::now(),
});
@@ -1006,8 +1015,12 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
.await
.expect("second response");
let (first_user_agent, second_user_agent) = server.await.expect("profile server task");
let (first_user_agent, second_user_agent, first_version, second_version) =
server.await.expect("profile server task");
assert!(first_user_agent.contains("1.2.3"));
assert!(second_user_agent.contains("1.2.4"));
assert!(second_user_agent.contains("26.999.10000"));
assert_eq!(first_version, "1.2.3");
assert_eq!(second_version, "1.2.4");
assert_ne!(first_user_agent, second_user_agent);
}
@@ -137,6 +137,7 @@ async fn codex_backend_client_should_preserve_oversized_error_response() {
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id: Some("chatgpt-account"),
request_id: "req_large_error",
@@ -236,6 +237,7 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id: Some("chatgpt-account"),
request_id: "req_http_retry_after_body",
@@ -294,10 +296,11 @@ async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updat
let mut request = codex_request("gpt-5.5", "", Vec::new());
request.force_http_sse = true;
let trace = gateway_core::diagnostics::TraceContext::new("req_http_rate_limit_event");
let response = client
.create_response(
&request,
request_context("req_http_rate_limit_event", Some("acct")),
request_context("req_http_rate_limit_event", Some("acct")).with_trace(&trace),
)
.await
.expect("HTTP SSE response");
@@ -315,6 +318,18 @@ async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updat
("x-codex-limit-reached".to_owned(), "false".to_owned()),
]
);
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
assert!(
events
.iter()
.any(|event| event["data"]["eventType"] == "codex.rate_limits")
);
assert!(
events
.iter()
.any(|event| event["data"]["eventType"] == "response.completed")
);
}
#[tokio::test(start_paused = true)]
@@ -407,6 +422,7 @@ async fn codex_backend_client_should_capture_forwardable_response_metadata() {
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id: Some("chatgpt-account"),
request_id: "req_response_metadata",
@@ -55,9 +55,11 @@ mod http_client;
mod latency;
mod profile;
mod profile_avatar;
mod profile_contract;
mod protocol;
mod request;
mod reset_credits;
mod tls;
mod usage;
mod websocket;
mod websocket_pool;
@@ -126,6 +128,7 @@ fn test_wire_profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "transport-test".to_owned(),
residency: None,
verified_at: Utc::now(),
})
}
@@ -135,6 +138,7 @@ fn request_context<'a>(
account_id: Option<&'a str>,
) -> CodexRequestContext<'a> {
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id,
request_id,
@@ -141,6 +141,7 @@ fn wire_profile_should_generate_bundled_core_app_server_user_agent() {
os_version: "15.7.1".to_owned(),
arch: "arm64".to_owned(),
terminal: "unknown".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 8, 3, 0, 0, 0)
.single()
@@ -360,6 +361,7 @@ fn wire_profile() -> CodexWireProfile {
os_version: "15.7.1".to_owned(),
arch: "arm64".to_owned(),
terminal: "unknown".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 8, 3, 0, 0, 0)
.single()
@@ -1,5 +1,7 @@
use futures::TryStreamExt as _;
use provider_openai::transport::{CodexProfileAvatarFetchError, fetch_profile_avatar};
use provider_openai::transport::{
CodexProfileAvatarFetchError, CodexRequestContext, fetch_profile_avatar,
};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
@@ -30,8 +32,14 @@ async fn profile_avatar_streams_unrestricted_content_type_and_body_size() {
let avatar = fetch_profile_avatar(
&client,
&server.uri(),
"codex_cli_rs/1.0.0 (linux; x86_64)",
&super::test_wire_profile().snapshot(),
OFFICIAL_AVATAR_SOURCE,
CodexRequestContext::auxiliary(
"Bearer avatar-token",
Some("avatar-account"),
"avatar",
None,
),
)
.await
.expect("profile avatar");
@@ -52,14 +60,18 @@ async fn profile_avatar_streams_unrestricted_content_type_and_body_size() {
assert_eq!(streamed, body);
assert_eq!(
headers.get("accept").and_then(|value| value.to_str().ok()),
Some("image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
Some("*/*")
);
assert_eq!(
headers
.get("user-agent")
.and_then(|value| value.to_str().ok()),
Some("codex_cli_rs/1.0.0 (linux; x86_64)")
Some("codex_cli_rs/1.2.3 (linux; x86_64)")
);
assert_eq!(headers["authorization"], "Bearer avatar-token");
assert_eq!(headers["chatgpt-account-id"], "avatar-account");
assert_eq!(headers["originator"], "codex_cli_rs");
assert!(!headers.contains_key("version"));
}
#[tokio::test]
@@ -72,9 +84,20 @@ async fn profile_avatar_rejects_non_official_sources_before_request() {
"https://chatgpt.com/backend-api/estuary/public_content/enc/token?next=1",
"https://chatgpt.com/backend-api/estuary/public_content/enc/",
] {
let error = fetch_profile_avatar(&client, "http://127.0.0.1:9", "test-agent", source)
.await
.expect_err("invalid source");
let error = fetch_profile_avatar(
&client,
"http://127.0.0.1:9",
&super::test_wire_profile().snapshot(),
source,
CodexRequestContext::auxiliary(
"Bearer avatar-token",
Some("avatar-account"),
"avatar",
None,
),
)
.await
.expect_err("invalid source");
assert!(matches!(error, CodexProfileAvatarFetchError::InvalidSource));
}
}
@@ -0,0 +1,136 @@
use super::*;
use provider_openai::transport::build_reqwest_client;
use provider_openai::transport::profile::CodexResidency;
use uuid::Uuid;
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
/// 官方本机样本:模型请求带 Core version,backend-client 账号请求不带。
/// 通过真实 HTTP 请求验证画像更新后每条路径取到同一制品中的正确字段。
#[tokio::test]
async fn each_core_endpoint_uses_the_current_bundled_release() {
let server = MockServer::start().await;
Mock::given(any())
.respond_with(ResponseTemplate::new(400).set_body_json(json!({"error":"audit fixture"})))
.expect(12)
.mount(&server)
.await;
let profile = test_wire_profile();
let client = CodexBackendClient::new(
build_reqwest_client().expect("HTTP client"),
format!("{}/backend-api", server.uri()),
profile.clone(),
);
for (core, desktop, build) in [
("0.152.8", "26.800.10000", "8000"),
("0.153.4", "26.901.51231", "8109"),
] {
profile.update_bundled_release(&CodexBundledReleaseProfile {
codex_version: core.to_owned(),
desktop_version: desktop.to_owned(),
desktop_build: build.to_owned(),
verified_at: Utc::now(),
});
let context = request_context("audit", Some("audit-account"));
client
.fetch_models_with_context(context)
.await
.expect_err("fixture rejection");
client
.fetch_usage(context)
.await
.expect_err("fixture rejection");
client
.fetch_profile_statistics(context)
.await
.expect_err("fixture rejection");
client
.list_rate_limit_reset_credits(context)
.await
.expect_err("fixture rejection");
client
.consume_rate_limit_reset_credit(context, None, Uuid::now_v7())
.await
.expect_err("fixture rejection");
let mut body = codex_request_body("gpt-test", "", Vec::new());
body.insert("service_tier".to_owned(), json!("priority"));
let mut request = CodexResponsesRequest::from_body(body);
request.force_http_sse = true;
client
.create_response(&request, context)
.await
.expect_err("fixture rejection");
let captured = server.received_requests().await.expect("captured requests");
for request in &captured[captured.len() - 6..] {
let headers = &request.headers;
assert_eq!(headers["user-agent"], profile.snapshot().user_agent());
assert_eq!(headers["authorization"], "Bearer access-token");
assert_eq!(headers["chatgpt-account-id"], "audit-account");
for forbidden in [
"x-codex-installation-id",
"x-codex-turn-id",
"x-openai-internal-codex-residency",
"oai-language",
"openai-beta",
] {
assert!(
!headers.contains_key(forbidden),
"{}: {forbidden}",
request.url
);
}
if request.url.path().contains("/codex/") {
assert_eq!(headers["originator"], "codex_cli_rs");
assert_eq!(headers["version"], core);
assert_ne!(headers["version"], desktop);
} else {
assert!(!headers.contains_key("originator"));
assert!(!headers.contains_key("version"));
}
if request.url.path().ends_with("/responses") {
assert_eq!(headers["accept"], "text/event-stream");
assert_eq!(
headers["x-codex-routing-hint"],
"model=gpt-test;tier=priority"
);
} else {
assert_eq!(headers["accept"], "*/*");
}
if request.url.path().ends_with("/models") {
assert_eq!(
request.url.query(),
Some(format!("client_version={core}").as_str())
);
}
}
}
}
#[test]
fn residency_is_explicit_and_survives_artifact_updates() {
let mut initial = test_wire_profile().snapshot();
initial.residency = Some(CodexResidency::Us);
let profile = CodexWireProfileState::new(initial);
profile.update_bundled_release(&CodexBundledReleaseProfile {
codex_version: "2.0.0".to_owned(),
desktop_version: "26.999.10000".to_owned(),
desktop_build: "9999".to_owned(),
verified_at: Utc::now(),
});
let snapshot = profile.snapshot();
let model = provider_openai::transport::headers::build_codex_model_headers(
&snapshot,
"Bearer audit",
None,
)
.expect("model headers");
let account = provider_openai::transport::headers::build_codex_account_headers(
&snapshot,
"Bearer audit",
None,
)
.expect("account headers");
assert_eq!(model["x-openai-internal-codex-residency"], "us");
assert_eq!(model["version"], "2.0.0");
assert!(!account.contains_key("x-openai-internal-codex-residency"));
}
@@ -26,6 +26,7 @@ fn client(base_url: &str) -> CodexBackendClient {
os_version: "15.5.0".to_owned(),
arch: "arm64".to_owned(),
terminal: "xterm-256color".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 8, 20, 0, 0, 0)
.single()
@@ -44,10 +45,10 @@ fn context() -> CodexRequestContext<'static> {
}
#[tokio::test]
async fn list_should_match_official_desktop_surface_profile() {
async fn list_should_match_official_core_account_profile() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/wham/rate-limit-reset-credits"))
.and(path("/api/codex/rate-limit-reset-credits"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"available_count": 1,
"credits": [{
@@ -81,11 +82,13 @@ async fn list_should_match_official_desktop_surface_profile() {
header(request, "chatgpt-account-id"),
Some("acct_workspace")
);
assert_eq!(header(request, "originator"), Some("Codex Desktop"));
assert_eq!(header(request, "oai-language"), Some("en"));
assert_eq!(header(request, "originator"), None);
assert_eq!(header(request, "oai-language"), None);
assert_eq!(
header(request, "user-agent"),
Some("Codex Desktop/26.818.21641 (Mac OS; arm64)")
Some(
"Codex Desktop/0.115.0-alpha.11 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 26.818.21641)"
)
);
assert_eq!(header(request, "accept"), Some("*/*"));
for name in [
@@ -110,7 +113,7 @@ async fn list_should_match_official_desktop_surface_profile() {
async fn consume_should_send_exact_credit_and_redeem_request_id_without_retry() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/wham/rate-limit-reset-credits/consume"))
.and(path("/api/codex/rate-limit-reset-credits/consume"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"code": "reset",
"credit": {
@@ -147,7 +150,7 @@ async fn consume_should_send_exact_credit_and_redeem_request_id_without_retry()
async fn consume_should_preserve_raw_upstream_error_and_never_retry() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/wham/rate-limit-reset-credits/consume"))
.and(path("/api/codex/rate-limit-reset-credits/consume"))
.respond_with(
ResponseTemplate::new(429)
.insert_header("retry-after", "17")
@@ -188,7 +191,7 @@ async fn consume_should_preserve_raw_upstream_error_and_never_retry() {
async fn list_should_reject_oversized_success_body() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/wham/rate-limit-reset-credits"))
.and(path("/api/codex/rate-limit-reset-credits"))
.respond_with(ResponseTemplate::new(200).set_body_raw(
"x".repeat(MAX_CODEX_RESET_CREDITS_BODY_BYTES + 1),
"application/json",
@@ -0,0 +1,146 @@
use std::collections::BTreeMap;
use bytes::{Buf as _, Bytes};
use super::*;
// Captured from Desktop 26.901.51231 / Core 0.153.4 on 2026-09-06.
// Compare wire semantics: rustls randomizes extension order and ephemeral key bytes.
#[derive(Debug, PartialEq, Eq)]
struct ClientHello {
cipher_suites: Vec<u16>,
extensions: Vec<u16>,
groups: Vec<u16>,
signature_algorithms: Vec<u16>,
key_shares: Vec<(u16, usize)>,
alpn: Vec<String>,
}
#[tokio::test]
async fn http_client_hello_should_match_official_rustls_transport() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"https://localhost:{}/",
listener.local_addr().unwrap().port()
);
let client = provider_openai::transport::build_reqwest_client().unwrap();
let (hello, response) = timeout(Duration::from_secs(10), async {
tokio::join!(read_client_hello(listener), client.get(url).send())
})
.await
.expect("HTTP ClientHello within timeout");
assert!(
response.is_err(),
"capture endpoint rejects TLS after ClientHello"
);
let mut expected = official_websocket_hello();
expected.extensions.insert(5, 16);
expected.alpn = vec!["h2".to_owned(), "http/1.1".to_owned()];
assert_eq!(hello, expected);
}
#[tokio::test]
async fn websocket_client_hello_should_match_official_rustls_transport() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("wss://localhost:{}/", listener.local_addr().unwrap().port());
let connector =
provider_openai::transport::tls::maybe_build_rustls_client_config_with_custom_ca()
.unwrap()
.map(tokio_tungstenite::Connector::Rustls);
let (hello, response) = timeout(Duration::from_secs(10), async {
tokio::join!(
read_client_hello(listener),
tokio_tungstenite::connect_async_tls_with_config(url, None, false, connector)
)
})
.await
.expect("WebSocket ClientHello within timeout");
assert!(
response.is_err(),
"capture endpoint rejects TLS after ClientHello"
);
assert_eq!(hello, official_websocket_hello());
}
fn official_websocket_hello() -> ClientHello {
ClientHello {
cipher_suites: vec![
4866, 4865, 4867, 49196, 49195, 52393, 49200, 49199, 52392, 255,
],
extensions: vec![0, 5, 10, 11, 13, 23, 35, 43, 45, 51],
groups: vec![4588, 29, 23, 24],
signature_algorithms: vec![1283, 1027, 1539, 2055, 2054, 2053, 2052, 1537, 1281, 1025],
key_shares: vec![(4588, 1216), (29, 32)],
alpn: Vec::new(),
}
}
async fn read_client_hello(listener: TcpListener) -> ClientHello {
let (mut stream, _) = listener.accept().await.unwrap();
let mut header = [0; 5];
stream.read_exact(&mut header).await.unwrap();
assert_eq!(header[0], 22, "TLS handshake record");
let length = usize::from(u16::from_be_bytes([header[3], header[4]]));
assert!(length <= 16_384, "bounded ClientHello record");
let mut record = vec![0; length];
stream.read_exact(&mut record).await.unwrap();
stream.write_all(&[21, 3, 3, 0, 2, 2, 40]).await.unwrap();
let mut hello = Bytes::from(record);
assert_eq!(hello.get_u8(), 1, "ClientHello message");
hello.advance(3 + 2 + 32); // Message length, legacy version, random.
let session_id_len = usize::from(hello.get_u8());
hello.advance(session_id_len);
let cipher_suites = u16_values(take_vector(&mut hello));
let compression_len = usize::from(hello.get_u8());
hello.advance(compression_len);
let mut extensions = take_vector(&mut hello);
let mut values = BTreeMap::new();
while extensions.has_remaining() {
let kind = extensions.get_u16();
assert!(values.insert(kind, take_vector(&mut extensions)).is_none());
}
let mut groups = values[&10].clone();
let groups = u16_values(take_vector(&mut groups));
let mut signatures = values[&13].clone();
let signature_algorithms = u16_values(take_vector(&mut signatures));
let mut shares = values[&51].clone();
let mut shares = take_vector(&mut shares);
let mut key_shares = Vec::new();
while shares.has_remaining() {
let group = shares.get_u16();
key_shares.push((group, take_vector(&mut shares).len()));
}
let mut alpn = Vec::new();
if let Some(protocols) = values.get(&16) {
let mut protocols = protocols.clone();
let mut protocols = take_vector(&mut protocols);
while protocols.has_remaining() {
let length = usize::from(protocols.get_u8());
alpn.push(String::from_utf8(protocols.split_to(length).to_vec()).unwrap());
}
}
ClientHello {
cipher_suites,
extensions: values.into_keys().collect(),
groups,
signature_algorithms,
key_shares,
alpn,
}
}
fn take_vector(bytes: &mut Bytes) -> Bytes {
let length = usize::from(bytes.get_u16());
bytes.split_to(length)
}
fn u16_values(mut bytes: Bytes) -> Vec<u16> {
let mut values = Vec::new();
while bytes.has_remaining() {
values.push(bytes.get_u16());
}
values
}
@@ -253,6 +253,7 @@ async fn fetch_should_use_wham_usage_headers_only() {
let usage = client(&server.uri())
.fetch_usage(CodexRequestContext {
trace: None,
authorization: "Bearer oauth-access",
account_id: Some("acct_123"),
request_id: "req_usage_headers",
@@ -283,7 +284,7 @@ async fn fetch_should_use_wham_usage_headers_only() {
for (name, expected) in [
("authorization", "Bearer oauth-access"),
("chatgpt-account-id", "acct_123"),
("accept", "application/json"),
("accept", "*/*"),
("cookie", "session=old"),
] {
assert_eq!(
@@ -576,6 +577,7 @@ fn profile() -> CodexWireProfileState {
os_version: "6.8".to_owned(),
arch: "x86_64".to_owned(),
terminal: "xterm".to_owned(),
residency: None,
verified_at: Utc
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
.single()
@@ -585,6 +587,7 @@ fn profile() -> CodexWireProfileState {
fn context() -> CodexRequestContext<'static> {
CodexRequestContext {
trace: None,
authorization: "Bearer oauth-access",
account_id: Some("acct_123"),
request_id: "req_usage_limit",
@@ -2150,6 +2150,7 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
.create_response(
&request,
CodexRequestContext {
trace: None,
authorization: "Bearer access-token",
account_id: Some("chatgpt-account"),
request_id: "req_ws_client",
@@ -2190,3 +2191,90 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
.any(|(name, value)| name == "x-ratelimit-remaining-requests" && value == "17")
);
}
#[tokio::test]
async fn diagnostics_capture_metadata_and_unknown_events_before_normal_close() {
use gateway_core::diagnostics::TraceContext;
use tokio_tungstenite::tungstenite::protocol::{CloseFrame, frame::coding::CloseCode};
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut websocket = accept_codex_test_websocket(stream).await;
websocket.next().await.unwrap().unwrap();
for event in [
json!({"type": "future.metadata", "detail": "private-data"}),
json!({"type": "codex.response.metadata", "headers": {
"x-request-id": "upstream-metadata-request", "x-codex-turn-state": "opaque-turn-secret",
}}),
] {
websocket
.send(Message::Text(event.to_string().into()))
.await
.unwrap();
}
websocket
.close(Some(CloseFrame {
code: CloseCode::Normal,
reason: "".into(),
}))
.await
.unwrap();
});
let trace = TraceContext::new("req_metadata_close");
let attempt = trace.attempt(2);
let mut request = codex_request("gpt-5.5", "be brief", Vec::new());
request.set_previous_response_id(Some("resp_previous".to_owned()));
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
request.force_http_sse = false;
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
);
let result = backend
.create_response_stream(
&request,
request_context("req_metadata_close", Some("chatgpt-account")).with_trace(&attempt),
)
.await;
let mut failed = result.is_err();
if let Ok(response) = result {
let mut body = response.body;
while let Some(chunk) = body.next().await {
if chunk.is_err() {
failed = true;
break;
}
}
}
assert!(failed, "close before terminal must fail the exchange");
server.await.unwrap();
let snapshot = trace.snapshot().unwrap();
let events = snapshot["events"].as_array().unwrap();
let metadata = events
.iter()
.find(|event| event["data"]["eventType"] == "codex.response.metadata")
.unwrap();
assert_eq!(metadata["attemptIndex"], 2);
assert_eq!(metadata["exchangeId"], 1);
assert_eq!(
metadata["data"]["metadata"]["headers"]["x-request-id"],
"upstream-metadata-request"
);
assert!(
events
.iter()
.any(|event| event["data"]["eventType"] == "future.metadata")
);
let close = events
.iter()
.find(|event| event["stage"] == "upstream.close")
.unwrap();
assert_eq!(close["data"]["code"], 1000);
assert_eq!(close["data"]["lastEventType"], "codex.response.metadata");
assert_eq!(close["data"]["terminalSeen"], false);
assert!(!snapshot.to_string().contains("opaque-turn-secret"));
assert!(!snapshot.to_string().contains("private-data"));
}
@@ -115,7 +115,7 @@ pub(super) fn cold_compaction_http_sse_stream(
headers.clone(),
body,
session.binding().clone(),
);
).with_trace(context.trace());
let Some(handshake_deadline) = remaining(context.deadline()) else {
Err(mark_transient_compaction_failure(provider_error(
ProviderErrorKind::Timeout,
@@ -403,7 +403,7 @@ pub(super) fn cold_http_sse_stream(
headers.clone(),
body,
session.binding().clone(),
);
).with_trace(context.trace());
let Some(handshake_deadline) = remaining(context.deadline()) else {
Err(provider_error(
ProviderErrorKind::Timeout,
@@ -14,6 +14,7 @@ use super::{GrokHeader, GrokSessionBinding};
/// 交给注入的 HTTP SSE transport 的自持有请求。
pub struct GrokInferenceRequest {
trace: gateway_core::diagnostics::TraceContext,
endpoint: Url,
headers: Vec<GrokHeader>,
body: Zeroizing<Vec<u8>>,
@@ -28,6 +29,7 @@ impl GrokInferenceRequest {
binding: GrokSessionBinding,
) -> Self {
Self {
trace: gateway_core::diagnostics::TraceContext::default(),
endpoint,
headers,
body: Zeroizing::new(body),
@@ -35,6 +37,16 @@ impl GrokInferenceRequest {
}
}
#[must_use]
pub fn with_trace(mut self, trace: gateway_core::diagnostics::TraceContext) -> Self {
self.trace = trace;
self
}
pub(crate) fn trace(&self) -> &gateway_core::diagnostics::TraceContext {
&self.trace
}
/// 返回严格限定的官方 Responses 端点。
#[must_use]
pub const fn endpoint(&self) -> &Url {
@@ -1,5 +1,6 @@
//! 关闭 redirect、proxy 与业务重试的生产 reqwest transport。
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext};
use std::collections::{HashMap, VecDeque};
use std::fmt;
use std::io;
@@ -371,6 +372,18 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
UpstreamSendState::NotSent,
));
}
let trace = request.trace().exchange("http_sse");
trace.headers(
"upstream.request.headers",
serde_json::json!({
"method": "POST", "endpoint": request.endpoint().path(),
}),
request
.headers()
.iter()
.map(|header| (header.name(), header.value().expose().as_bytes())),
);
trace.capture("upstream.request.body", request.body());
let (client, client_cache_status) = self.client_for(request.binding())?;
let mut builder = client
.post(request.endpoint().clone())
@@ -393,8 +406,12 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
Some(elapsed_millis(headers_started_at.elapsed())),
&dns_observer,
);
trace.headers("upstream.response.headers", serde_json::json!({
"status": response.status().as_u16(), "httpVersion": format!("{:?}", response.version()),
"headersMs": elapsed_millis(headers_started_at.elapsed()),
}), response.headers().iter().map(|(name, value)| (name.as_str(), value.as_bytes())));
if !response.status().is_success() {
return Err(classify_inference_status(response)
return Err(classify_inference_status(response, &trace)
.await
.with_transport_metrics(transport_metrics));
}
@@ -424,6 +441,20 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
};
std::future::ready(Some(item))
});
let body = async_stream::stream! {
let mut body = Box::pin(body);
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
while let Some(chunk) = body.next().await {
match &chunk {
Ok(bytes) => capture.push(bytes),
Err(_) => trace.record("upstream.read.failed", serde_json::json!({"phase": "response_body"})),
}
let failed = chunk.is_err();
yield chunk;
if failed { return; }
}
capture.finish();
};
Ok(
GrokInferenceResponse::new(Box::pin(body), http_version, status_code, request_id)
.with_transport_metrics(transport_metrics),
@@ -981,7 +1012,10 @@ fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTrans
)
}
async fn classify_inference_status(response: Response) -> GrokInferenceTransportError {
async fn classify_inference_status(
response: Response,
trace: &TraceContext,
) -> GrokInferenceTransportError {
let status = response.status();
let retry_after = retry_after(&response);
let http_version = upstream_http_version(response.version());
@@ -989,8 +1023,15 @@ async fn classify_inference_status(response: Response) -> GrokInferenceTransport
let status_code = status.as_u16();
let body = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await {
Ok(BoundedBody::Body(body)) => body,
Ok(BoundedBody::TooLarge) | Err(_) => Vec::new(),
Ok(BoundedBody::TooLarge) | Err(_) => {
trace.record(
"capture.gap",
serde_json::json!({"reason": "error_body_unavailable_or_too_large"}),
);
Vec::new()
}
};
trace.capture("upstream.error.body", &body);
let metadata = inference_error_metadata(&body);
let body_failure = classify_grok_body_failure(&metadata, &body);
let kind = match status {
+1
View File
@@ -4,3 +4,4 @@
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
3b4bfd5e3f5135fa2fad1088b0c2a287861e1250d66af31dbd40dc890751b567 0006_openai_session_transport_recovery.sql
8c0ec7fab80d95dfd5a370c70706c36ac5907bb5afea0cf1886196f21fd8317d 0007_request_diagnostics.sql
@@ -0,0 +1,5 @@
-- Bounded request evidence is retained with its owning model request.
-- NULL identifies historical requests for which no trace was captured.
alter table model_requests add column diagnostic_trace_json jsonb;
alter table model_requests add constraint model_requests_diagnostic_trace_object
check (diagnostic_trace_json is null or jsonb_typeof(diagnostic_trace_json) = 'object');
+7 -5
View File
@@ -20,7 +20,7 @@ host:
logging:
level: 'info'
stdout: true
# 将完整请求头和正文写入独立文件;包含凭据,仅排障时开启。
# 将完整请求与响应头、正文写入独立文件;包含凭据,仅排障时开启。
request_dump: false
file:
enabled: true
@@ -65,14 +65,16 @@ openai:
# 版本元组来自同一个官方 macOS Desktop ZIP;Provider 每 24 小时检查 appcast,
# 仅在包内 Core 核验和 Redis 条件覆盖都成功后原子更新。
originator: 'Codex Desktop'
codex_version: '0.151.0-alpha.7.2'
desktop_version: '26.825.51511'
desktop_build: '7377'
codex_version: '0.153.4'
desktop_version: '26.901.51231'
desktop_build: '8109'
os_type: 'Mac OS'
os_version: '15.7.1'
arch: 'arm64'
terminal: 'unknown'
verified_at: '2026-08-31T00:00:00+08:00'
# 部署选择的 US residency;省略或 null 时使用官方无地域约束的默认分支。
residency: us
verified_at: '2026-09-06T11:26:12.084+08:00'
xai:
# OAuth 端点、额度与模型目录策略由 xAI Provider 固定管理;这里只声明请求画像启动基线。
+5
View File
@@ -567,6 +567,11 @@ request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可
汇总与洞察中的请求数与 outcome 分布覆盖筛选范围内全部请求;token、缓存、延迟与成本聚合仅统计
已完整交付客户端的成功响应。
详情接口按 `id` 可读取成功、失败或未完成请求。新增 `trace`(历史未采集记录为 `null`)和
`relatedRequests[]`(`requestId / relation / outcome / completedAt`);`relation` 为 `recovered_by` 或
`recovers`。`trace` 是执行终态时的有界脱敏时间线,包含 request、attempt 和 exchange 关联、阶段、
事件摘要及淘汰计数;普通用量列表不携带此字段。
错误记录中的“已自动恢复”表示系统关联到了后续成功请求,不会把原来的失败记录改为成功。
`upstreamSendState = ambiguous` 表示无法确认该次上游执行结果,不代表后续恢复请求失败;
恢复关联也不等于逐字节验证过两次请求正文。
+32
View File
@@ -186,7 +186,39 @@ export interface UsageAttempt {
completedAt: string | null
}
export interface RequestTraceEvent {
sequence: number
lastSequence: number
elapsedMs: number
lastElapsedMs: number
attemptIndex: number
exchangeId: number | null
stage: string
count: number
data: Record<string, unknown>
}
export interface RequestTrace {
schemaVersion: number
requestId: string
startedAtUnixMs: number
totalEvents: number
droppedEvents: number
maxEvents: number
captureMode: string
events: RequestTraceEvent[]
}
export interface RelatedRequest {
requestId: string
relation: 'recovered_by' | 'recovers'
outcome: string
completedAt: string | null
}
export type UsageRecordDetail = UsageRecord & {
trace: RequestTrace | null
relatedRequests: RelatedRequest[]
attempts: UsageAttempt[]
/** 尝试列表是否完整;best-effort 下恒为 false。 */
attemptsComplete: boolean
@@ -5,6 +5,7 @@ import { computed } from 'vue'
import BaseButton from '@/components/base/BaseButton.vue'
import BaseModal from '@/components/base/BaseModal/index.vue'
import { failureClassText } from '../utils/opsErrorPresentation'
import RequestDiagnosticsPanel from './RequestDiagnosticsPanel.vue'
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue'
@@ -130,6 +131,7 @@ function visibleFields(items: DetailField[]) {
size="xl"
>
<template v-if="record">
<RequestDiagnosticsPanel v-if="open && record.requestId" class="mb-3" :request-id="record.requestId" />
<section :class="panelClass">
<h3 :class="panelTitleClass">
错误
@@ -142,7 +144,7 @@ function visibleFields(items: DetailField[]) {
<section v-if="record.rawUpstreamError" class="mt-3" :class="panelClass">
<UsageDetailCodePanel
title="上游返回原文"
title="上游错误与关闭信息"
max-height="360px"
:content="record.rawUpstreamError"
/>
@@ -0,0 +1,110 @@
<script setup lang="ts">
import { computed } from 'vue'
import BaseButton from '@/components/base/BaseButton.vue'
import { useDownload } from '@/composables/useDownload'
import { useRequestDiagnostics } from '../composables/useRequestDiagnostics'
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
const props = defineProps<{ requestId: string }>()
const { downloadJson } = useDownload()
const { selectedId, detail, loading, error, refresh } = useRequestDiagnostics(() => props.requestId)
const trace = computed(() => detail.value?.trace)
const events = computed(() => (trace.value?.events ?? []).map(event => ({
...event,
content: JSON.stringify(event.data, null, 2),
})))
function download() {
if (!detail.value)
return
const record = detail.value
const bundle = {
schemaVersion: 1,
exportedAt: new Date().toISOString(),
request: {
requestId: record.requestId,
route: record.route,
model: record.model,
provider: record.provider,
accountId: record.accountId,
clientTransport: record.clientTransport,
upstreamTransport: record.upstreamTransport,
upstreamRequestId: record.upstreamRequestId,
responseId: record.responseId,
statusCode: record.statusCode,
attemptCount: record.attemptCount,
createdAt: record.createdAt,
},
trace: record.trace,
attempts: record.attempts,
attemptsComplete: record.attemptsComplete,
relatedRequests: record.relatedRequests,
}
return downloadJson(bundle, `diagnostics-${record.requestId.replace(/[^\w-]/g, '_')}.json`)
}
</script>
<template>
<section class="mt-3 min-w-0 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5" aria-label="请求诊断">
<div class="flex flex-wrap items-center justify-between gap-3">
<h3 class="m-0 text-cp-sm font-heavy text-cp-text-secondary">
请求诊断
</h3>
<div class="flex flex-wrap gap-2">
<BaseButton v-if="selectedId !== requestId" size="sm" @click="selectedId = requestId">
返回本次请求
</BaseButton>
<BaseButton size="sm" :disabled="loading" @click="refresh">
刷新
</BaseButton>
<BaseButton size="sm" :disabled="!detail" @click="download">
导出诊断包
</BaseButton>
</div>
</div>
<p class="break-all font-mono text-cp-xs text-cp-text-secondary">
{{ selectedId }}
</p>
<p v-if="loading" role="status" class="text-cp-sm text-cp-text-secondary">
正在加载诊断记录…
</p>
<p v-else-if="error" role="alert" class="text-cp-sm text-cp-error-text">
{{ error }}
</p>
<template v-else-if="detail">
<div v-if="detail.relatedRequests?.length" class="mb-3 flex flex-wrap gap-2">
<BaseButton v-for="related in detail.relatedRequests" :key="related.requestId" size="sm" @click="selectedId = related.requestId">
{{ related.relation === 'recovered_by' ? '查看恢复请求' : '查看先前失败' }} · {{ related.requestId }}
</BaseButton>
</div>
<p v-if="!trace" class="text-cp-sm text-cp-text-secondary">
这条记录没有保存诊断时间线。旧记录无法补回当时未采集的事件。
</p>
<template v-else>
<p class="text-cp-xs text-cp-text-secondary">
已观测 {{ trace.totalEvents }} 个阶段或事件,展示 {{ events.length }} 条摘要。连续增量合并计数;正文和敏感字段仅保留摘要。
</p>
<p v-if="trace.droppedEvents" role="status" class="text-cp-xs text-cp-warning-text">
达到保存上限,{{ trace.droppedEvents }} 个事件已被淘汰;保留请求开头与最近事件。
</p>
<ol class="m-0 grid max-h-128 list-none gap-2 overflow-auto p-0">
<li v-for="event in events" :key="event.sequence" class="min-w-0 rounded-cp bg-cp-bg-container px-3 py-2">
<details>
<summary class="cursor-pointer break-all text-cp-xs leading-relaxed">
<span class="font-mono text-cp-text-secondary">+{{ event.elapsedMs }} ms · #{{ event.sequence }}</span>
<span class="mx-2 font-mono">{{ event.stage }}</span>
<span v-if="event.attemptIndex">尝试 {{ event.attemptIndex }}</span>
<span v-if="event.exchangeId"> · 交换 {{ event.exchangeId }}</span>
<span v-if="event.count > 1"> · {{ event.count }} 次(至 +{{ event.lastElapsedMs }} ms)</span>
</summary>
<UsageDetailCodePanel class="mt-2" title="诊断事实" :content="event.content" max-height="280px" />
</details>
</li>
</ol>
<p class="mb-0 text-cp-xs text-cp-text-secondary">
时间线保存至执行结束。客户端最终写入、进程退出等后续事实,可用此请求 ID 检索服务器日志。
</p>
</template>
</template>
</section>
</template>
@@ -26,6 +26,7 @@ import {
visibleRequestText,
visibleResponseText,
} from '../utils/records'
import RequestDiagnosticsPanel from './RequestDiagnosticsPanel.vue'
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue'
import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue'
@@ -470,6 +471,8 @@ const tokenDonutOption = computed<EChartsOption>(() => {
</section>
</div>
<RequestDiagnosticsPanel v-if="open && record" :request-id="record.requestId" />
<template #footer>
<BaseButton variant="primary" @click="open = false">
关闭
@@ -0,0 +1,37 @@
import type { UsageRecordDetail } from '@/api'
import { shallowRef, watch } from 'vue'
import { getUsageRecordDetail } from '@/api'
import { errorMessage } from '@/utils/async'
/** Requests can be switched while a lookup is pending; stale results never replace the active trace. */
export function useRequestDiagnostics(requestId: () => string) {
const selectedId = shallowRef(requestId())
const revision = shallowRef(0)
const detail = shallowRef<UsageRecordDetail | null>(null)
const loading = shallowRef(false)
const error = shallowRef('')
watch(requestId, id => selectedId.value = id)
watch([selectedId, revision], async ([id], _previous, onCleanup) => {
let active = true
onCleanup(() => active = false)
detail.value = null
error.value = ''
loading.value = true
try {
const result = await getUsageRecordDetail({ id })
if (active)
detail.value = result
}
catch (cause: unknown) {
if (active)
error.value = errorMessage(cause, '诊断记录暂不可用,请稍后刷新')
}
finally {
if (active)
loading.value = false
}
}, { immediate: true })
return { selectedId, detail, loading, error, refresh: () => revision.value++ }
}