From 353dccedc48bcc1b9271e76a75d8e21cec8c7610 Mon Sep 17 00:00:00 2001 From: Zyy Date: Sun, 6 Sep 2026 16:16:44 +0800 Subject: [PATCH] feat: add request diagnostics and align Codex transport contracts --- .github/workflows/_quality.yml | 2 + backend/Cargo.lock | 93 +++-- backend/apps/gateway/tests/bootstrap.rs | 7 +- .../gateway-admin/src/model/observability.rs | 2 + .../src/admin/observability/presenter.rs | 2 + .../src/admin/observability/wire.rs | 2 + backend/crates/gateway-api/src/lib.rs | 13 +- .../gateway-api/src/openai/responses/http.rs | 47 ++- .../src/openai/responses/websocket/forward.rs | 9 +- .../src/openai/responses/websocket/mod.rs | 93 +++-- .../tests/admin/observability/response.rs | 2 + backend/crates/gateway-core/Cargo.toml | 2 + .../gateway-core/src/diagnostics/capture.rs | 157 ++++++++ .../gateway-core/src/diagnostics/mod.rs | 9 + .../gateway-core/src/diagnostics/stream.rs | 165 ++++++++ .../gateway-core/src/diagnostics/trace.rs | 338 ++++++++++++++++ .../gateway-core/src/engine/coordinator.rs | 88 +++- .../gateway-core/src/engine/execution.rs | 9 + backend/crates/gateway-core/src/engine/mod.rs | 16 + backend/crates/gateway-core/src/lib.rs | 1 + .../gateway-core/tests/diagnostics/capture.rs | 49 +++ .../gateway-core/tests/diagnostics/mod.rs | 3 + .../gateway-core/tests/diagnostics/stream.rs | 70 ++++ .../gateway-core/tests/diagnostics/trace.rs | 142 +++++++ .../gateway-core/tests/engine/coordinator.rs | 17 + backend/crates/gateway-core/tests/main.rs | 1 + .../gateway-store/src/postgres/execution.rs | 19 +- .../src/postgres/execution_buffer.rs | 1 + .../src/postgres/observability/mapping.rs | 2 + .../src/postgres/observability/model.rs | 2 + .../postgres/observability/queries/usage.rs | 23 +- .../gateway-store/tests/postgres/execution.rs | 65 ++- .../tests/postgres/observability/mod.rs | 6 +- backend/crates/providers/openai/Cargo.toml | 23 +- backend/crates/providers/openai/src/config.rs | 15 +- .../src/credential/profile_statistics.rs | 63 ++- .../openai/src/credential/token_client.rs | 36 +- backend/crates/providers/openai/src/lib.rs | 7 +- .../openai/src/provider/execution.rs | 5 +- .../openai/src/provider/observation.rs | 1 + .../providers/openai/src/transport/client.rs | 9 + .../openai/src/transport/client_json.rs | 46 ++- .../openai/src/transport/client_sse.rs | 380 ++++++------------ .../openai/src/transport/endpoints.rs | 13 +- .../providers/openai/src/transport/headers.rs | 198 ++++++++- .../providers/openai/src/transport/mod.rs | 2 +- .../openai/src/transport/profile/mod.rs | 13 +- .../openai/src/transport/profile_avatar.rs | 18 +- .../src/transport/profile_statistics.rs | 6 +- .../providers/openai/src/transport/request.rs | 104 +++-- .../openai/src/transport/reset_credits.rs | 53 +-- .../providers/openai/src/transport/tls.rs | 8 +- .../providers/openai/src/transport/usage.rs | 2 +- .../src/transport/websocket/coordinator.rs | 15 + .../transport/websocket/exchange/stream.rs | 40 +- .../crates/providers/openai/tests/admin.rs | 5 +- .../crates/providers/openai/tests/config.rs | 3 +- .../openai/tests/credential/catalog.rs | 1 + .../openai/tests/credential/contract.rs | 1 + .../openai/tests/credential/oauth.rs | 2 +- .../tests/credential/profile_statistics.rs | 24 +- .../openai/tests/credential/quota/mod.rs | 1 + .../openai/tests/credential/refresh.rs | 1 + .../openai/tests/credential/token_client.rs | 68 ++++ .../openai/tests/provider/contract.rs | 14 + .../openai/tests/transport/catalog.rs | 7 +- .../openai/tests/transport/client.rs | 1 + .../openai/tests/transport/diagnostics.rs | 84 ++++ .../openai/tests/transport/headers.rs | 93 +++-- .../openai/tests/transport/http_client.rs | 18 +- .../providers/openai/tests/transport/mod.rs | 4 + .../openai/tests/transport/profile/mod.rs | 2 + .../openai/tests/transport/profile_avatar.rs | 37 +- .../tests/transport/profile_contract.rs | 136 +++++++ .../openai/tests/transport/reset_credits.rs | 19 +- .../providers/openai/tests/transport/tls.rs | 146 +++++++ .../providers/openai/tests/transport/usage.rs | 5 +- .../openai/tests/transport/websocket/mod.rs | 88 ++++ .../providers/xai/src/provider/stream.rs | 4 +- .../providers/xai/src/transport/inference.rs | 12 + .../providers/xai/src/transport/network.rs | 47 ++- backend/migrations/.frozen-sha256 | 1 + .../migrations/0007_request_diagnostics.sql | 5 + deploy/config.example.yaml | 12 +- docs/api.md | 5 + frontend/src/api/modules/usage.ts | 32 ++ .../usage/components/OpsErrorDetailModal.vue | 4 +- .../components/RequestDiagnosticsPanel.vue | 110 +++++ .../components/UsageRecordDetailModal.vue | 3 + .../composables/useRequestDiagnostics.ts | 37 ++ 90 files changed, 2899 insertions(+), 647 deletions(-) create mode 100644 backend/crates/gateway-core/src/diagnostics/capture.rs create mode 100644 backend/crates/gateway-core/src/diagnostics/mod.rs create mode 100644 backend/crates/gateway-core/src/diagnostics/stream.rs create mode 100644 backend/crates/gateway-core/src/diagnostics/trace.rs create mode 100644 backend/crates/gateway-core/tests/diagnostics/capture.rs create mode 100644 backend/crates/gateway-core/tests/diagnostics/mod.rs create mode 100644 backend/crates/gateway-core/tests/diagnostics/stream.rs create mode 100644 backend/crates/gateway-core/tests/diagnostics/trace.rs create mode 100644 backend/crates/providers/openai/tests/transport/profile_contract.rs create mode 100644 backend/crates/providers/openai/tests/transport/tls.rs create mode 100644 backend/migrations/0007_request_diagnostics.sql create mode 100644 frontend/src/views/usage/components/RequestDiagnosticsPanel.vue create mode 100644 frontend/src/views/usage/composables/useRequestDiagnostics.ts diff --git a/.github/workflows/_quality.yml b/.github/workflows/_quality.yml index fdab824ee..185bd1a8b 100644 --- a/.github/workflows/_quality.yml +++ b/.github/workflows/_quality.yml @@ -33,6 +33,8 @@ jobs: timeout-minutes: 30 env: CARGO_TERM_COLOR: always + # Rust 1.97 编译 AWS SDK 调试信息需要更大的 codegen 线程栈。 + RUST_MIN_STACK: 16777216 CPR_TEST_DATABASE_URL: postgres://codex_proxy:codex_proxy@127.0.0.1:5432/codex_proxy CPR_TEST_REDIS_URL: redis://127.0.0.1:6379 services: diff --git a/backend/Cargo.lock b/backend/Cargo.lock index afffe8899..2adea85d1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -165,15 +165,14 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" dependencies = [ "cc", "cmake", "dunce", "fs_extra", - "pkg-config", ] [[package]] @@ -195,7 +194,7 @@ dependencies = [ "bytes-utils", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", "percent-encoding", @@ -231,7 +230,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "lru", "percent-encoding", @@ -257,7 +256,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.4.2", + "http 1.4.0", "percent-encoding", "sha2 0.11.0", "time", @@ -286,7 +285,7 @@ dependencies = [ "bytes", "crc-fast", "hex", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "md-5", @@ -320,7 +319,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "percent-encoding", @@ -339,7 +338,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.4.2", + "http 1.4.0", "hyper", "hyper-rustls", "hyper-util", @@ -389,7 +388,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -410,7 +409,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.2", + "http 1.4.0", "pin-project-lite", "tokio", "tracing", @@ -436,7 +435,7 @@ checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.4.2", + "http 1.4.0", ] [[package]] @@ -450,7 +449,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.2", + "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -503,7 +502,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "hyper", @@ -535,7 +534,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "mime", @@ -1478,12 +1477,14 @@ name = "gateway-core" version = "0.1.0" dependencies = [ "async-trait", + "base64 0.23.1", "bytes", "futures", "futures-timer", "semver", "serde", "serde_json", + "sha2 0.11.0", "thiserror", "tracing", "uuid", @@ -1639,7 +1640,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.4.0", "indexmap", "slab", "tokio", @@ -1687,7 +1688,7 @@ dependencies = [ "base64 0.22.1", "bytes", "headers-core", - "http 1.4.2", + "http 1.4.0", "httpdate", "mime", "sha1 0.10.7", @@ -1699,7 +1700,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.4.2", + "http 1.4.0", ] [[package]] @@ -1761,9 +1762,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", "itoa", @@ -1787,7 +1788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.2", + "http 1.4.0", ] [[package]] @@ -1798,7 +1799,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] @@ -1834,21 +1835,22 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -1860,7 +1862,7 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.4.2", + "http 1.4.0", "hyper", "hyper-util", "log", @@ -1882,7 +1884,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "hyper", "ipnet", @@ -2594,6 +2596,7 @@ version = "0.0.0" dependencies = [ "async-stream", "async-trait", + "aws-lc-rs", "base64 0.23.1", "bytes", "chrono", @@ -2605,6 +2608,7 @@ dependencies = [ "gateway-protocol", "getrandom 0.4.3", "hex", + "hyper", "hyper-rustls", "reqwest", "roxmltree", @@ -2620,6 +2624,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-rustls", "tokio-tungstenite 0.28.0", "tokio-util", "tracing", @@ -2911,7 +2916,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "hyper", @@ -3021,9 +3026,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3033,9 +3038,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", "zeroize", @@ -3699,7 +3704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3800,9 +3805,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3912,7 +3917,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", "tower", @@ -3931,7 +3936,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.4.2", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "http-range-header", @@ -4063,7 +4068,7 @@ dependencies = [ "data-encoding", "flate2", "headers", - "http 1.4.2", + "http 1.4.0", "httparse", "log", "rand 0.9.4", @@ -4082,7 +4087,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http 1.4.2", + "http 1.4.0", "httparse", "log", "rand 0.9.4", @@ -4332,9 +4337,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.8" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" dependencies = [ "rustls-pki-types", ] @@ -4536,7 +4541,7 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http 1.4.2", + "http 1.4.0", "http-body-util", "hyper", "hyper-util", diff --git a/backend/apps/gateway/tests/bootstrap.rs b/backend/apps/gateway/tests/bootstrap.rs index 604622118..1b57b9211 100644 --- a/backend/apps/gateway/tests/bootstrap.rs +++ b/backend/apps/gateway/tests/bootstrap.rs @@ -143,10 +143,7 @@ fn config_loader_should_reject_removed_fingerprint_section() { #[test] fn config_loader_should_reject_invalid_codex_cli_version() { - assert_rejected(valid_config().replace( - "codex_version: '0.151.0-alpha.7.2'", - "codex_version: 'latest'", - )); + assert_rejected(valid_config().replace("codex_version: '0.153.4'", "codex_version: 'latest'")); } #[test] @@ -165,7 +162,7 @@ fn config_loader_should_reject_zero_server_port() { #[test] fn config_loader_should_reject_invalid_desktop_profile_fields() { - assert_rejected(valid_config().replace("desktop_build: '7377'", "desktop_build: 'build'")); + assert_rejected(valid_config().replace("desktop_build: '8109'", "desktop_build: 'build'")); } fn assert_rejected(config: String) { diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index 70882281e..d36012d50 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -704,6 +704,8 @@ pub struct UsageAttempt { /// 一条请求及其全部尝试。 #[derive(Debug, Clone, PartialEq, Eq)] pub struct UsageDetail { + pub trace: Option, + pub related_requests: Vec, pub request: UsageRecord, pub attempts: Vec, } diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs index bfe83eca2..c7b750121 100644 --- a/backend/crates/gateway-api/src/admin/observability/presenter.rs +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -505,6 +505,8 @@ pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptV pub(crate) fn usage_detail_view(detail: domain::UsageDetail) -> UsageRecordDetailView { UsageRecordDetailView { + trace: detail.trace, + related_requests: detail.related_requests, request: usage_record_view(detail.request), attempts: detail .attempts diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs index abb9393ae..2514f8b41 100644 --- a/backend/crates/gateway-api/src/admin/observability/wire.rs +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -247,6 +247,8 @@ pub struct UsageAttemptView { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct UsageRecordDetailView { + pub trace: Option, + pub related_requests: Vec, #[serde(flatten)] pub request: UsageRecordView, pub attempts: Vec, diff --git a/backend/crates/gateway-api/src/lib.rs b/backend/crates/gateway-api/src/lib.rs index 8a389825c..eb8b58f45 100644 --- a/backend/crates/gateway-api/src/lib.rs +++ b/backend/crates/gateway-api/src/lib.rs @@ -180,13 +180,16 @@ pub fn initialize( "request", request_id = %request_id, method = %request.method().as_str(), - uri = %request - .uri() - .path_and_query() - .map(axum::http::uri::PathAndQuery::as_str) - .unwrap_or_default(), + uri = %request.uri().path(), ) } + }) + .on_request(|_request: &Request<_>, _span: &tracing::Span| { + tracing::info!(target: "request_trace", stage = "http.received", "HTTP request received"); + }) + .on_response(|response: &axum::http::Response<_>, latency: Duration, _span: &tracing::Span| { + tracing::info!(target: "request_trace", stage = "http.response", status = response.status().as_u16(), + headers_ms = latency.as_millis(), "HTTP response headers ready"); }); let router = router .layer(PropagateRequestIdLayer::new(request_id_header.clone())) diff --git a/backend/crates/gateway-api/src/openai/responses/http.rs b/backend/crates/gateway-api/src/openai/responses/http.rs index 735299eb2..d91344256 100644 --- a/backend/crates/gateway-api/src/openai/responses/http.rs +++ b/backend/crates/gateway-api/src/openai/responses/http.rs @@ -14,6 +14,7 @@ use axum::{ response::{IntoResponse, Response}, }; use futures::stream; +use gateway_core::diagnostics::TraceContext; use gateway_core::engine::execution::{ClientTransport, ExecutionSession, StartedExecution}; use gateway_core::engine::{CommitRequirement, EngineError}; use gateway_core::error::{GatewayError, GatewayErrorKind}; @@ -39,6 +40,7 @@ use super::{ pub(crate) async fn responses( State(state): State, connect_info: Option>>, + ingress_id: Option>, headers: HeaderMap, body: Bytes, ) -> Response { @@ -83,28 +85,25 @@ pub(crate) async fn responses( Ok(started) => started, Err(error) => return gateway_error_response(&error), }; - tracing::info!( - target: "request_dump", - provider = "openai", - request_id = %started.request_id, - request_direction = "downstream", - request_transport = if streaming { "http_sse" } else { "http_json" }, - request_method = "POST", - request_path = "/v1/responses", - request_headers = ?headers, - request_body = %String::from_utf8_lossy(&body), - contains_sensitive_data = true, - "request dump (unredacted)" - ); + let trace = started.session.trace(); + trace.headers("client.request", serde_json::json!({ + "ingressRequestId": ingress_id.as_ref().and_then(|Extension(id)| id.header_value().to_str().ok()), + }), headers.iter().map(|(name, value)| (name.as_str(), value.as_bytes()))); + trace.capture("client.request.body", &body); + let request_id_header = HeaderValue::from_str(started.request_id.as_str()).ok(); let StartedExecution { stream, session, .. } = started; - if stream { + let mut response = if stream { stream_execution_response(session, connection_guard).await } else { drop(connection_guard); collect_execution_response(session).await + }; + if let Some(id) = request_id_header { + response.headers_mut().insert("x-gateway-request-id", id); } + response } /// 从 socket 与标准转发头提取旧 Usage 页面使用的诊断事实。 @@ -182,6 +181,11 @@ pub async fn collect_execution_response(session: Box) -> R } }; let response_headers = session.response_headers().to_vec(); + session.trace().record( + "downstream.encoded", + serde_json::json!({"transport": "http_json", "bytes": encoded.len()}), + ); + session.trace().dump("downstream.body", &encoded); let response = json_body_response(encoded, &response_headers); let Some(session) = execution.session_mut() else { @@ -334,6 +338,8 @@ pub async fn stream_execution_response( let body = Body::from_stream(stream::unfold(state, |mut state| async move { loop { if let Some(chunk) = state.pending.pop_front() { + state.trace.dump("downstream.chunk", &chunk); + state.handed_off_bytes += chunk.len() as u64; return Some((Ok::(chunk), state)); } if state.output_finished { @@ -416,12 +422,18 @@ impl Drop for PendingExecution { if session.is_finalized() { return; } + session.trace().record( + "downstream.cancelled", + serde_json::json!({"reason": "response_guard_dropped"}), + ); session.cancel(); detach_finalize(session); } } struct ResponsesStreamState { + trace: TraceContext, + handed_off_bytes: u64, session: Option>, encoder: OpenAiResponsesEncoder, pending: VecDeque, @@ -438,6 +450,8 @@ impl ResponsesStreamState { connection_guard: Option>, ) -> Self { Self { + trace: session.trace(), + handed_off_bytes: 0, session: Some(session), encoder, pending: initial_frames.into_iter().collect(), @@ -593,6 +607,11 @@ impl ResponsesStreamState { impl Drop for ResponsesStreamState { fn drop(&mut self) { + self.trace.record("downstream.body.closed", serde_json::json!({ + "handedOffBytes": self.handed_off_bytes, + "pendingChunks": self.pending.len(), + "outputFinished": self.output_finished, "executionTerminal": self.execution_terminal, + })); if self.execution_terminal { return; } diff --git a/backend/crates/gateway-api/src/openai/responses/websocket/forward.rs b/backend/crates/gateway-api/src/openai/responses/websocket/forward.rs index 207e0f6c7..c3fad5a1a 100644 --- a/backend/crates/gateway-api/src/openai/responses/websocket/forward.rs +++ b/backend/crates/gateway-api/src/openai/responses/websocket/forward.rs @@ -338,7 +338,12 @@ async fn next_active_input( // pump 继续接收有界业务帧和处理 Ping/Pong；退出通过独立通知取消本轮。 tokio::select! { biased; - _ = connection.wait_for_exit() => ActiveInput::Disconnect, + _ = connection.wait_for_exit() => { + session.trace().record("downstream.cancelled", serde_json::json!({ + "reason": "websocket_connection_exit", "connectionId": connection.id(), + })); + ActiveInput::Disconnect + }, event = session.next_event() => ActiveInput::Event(event), } } @@ -451,6 +456,7 @@ fn log_terminal_write_success( provider_terminal_to_write: std::time::Duration, ) { tracing::info!( + target: "request_trace", stage = "downstream.terminal.written", websocket_connection_id = connection.id(), request_id = %request_id, provider_terminal_to_terminal_write_ms = provider_terminal_to_write.as_millis(), @@ -465,6 +471,7 @@ fn log_terminal_write_failure( provider_terminal_to_write: std::time::Duration, ) { tracing::info!( + target: "request_trace", stage = "downstream.terminal.write_failed", websocket_connection_id = connection.id(), request_id = %request_id, provider_terminal_to_terminal_write_ms = provider_terminal_to_write.as_millis(), diff --git a/backend/crates/gateway-api/src/openai/responses/websocket/mod.rs b/backend/crates/gateway-api/src/openai/responses/websocket/mod.rs index 83ad91c2c..c2a4fe6f7 100644 --- a/backend/crates/gateway-api/src/openai/responses/websocket/mod.rs +++ b/backend/crates/gateway-api/src/openai/responses/websocket/mod.rs @@ -19,6 +19,7 @@ use axum::{ response::{IntoResponse, Response}, }; use gateway_core::{ + diagnostics::{TraceContext, diagnostic_json}, engine::execution::{AuthenticatedClient, ClientTransport}, lifecycle::{ConnectionGuard, ConnectionLifecycle}, }; @@ -98,17 +99,12 @@ impl ResponsesWebSocketAdapter { Err(_) => return runtime_unavailable_response().into_response(), }; let connection_id = self.service.next_request_id().replacen("req_", "ws_", 1); - tracing::info!( - target: "request_dump", - provider = "openai", - websocket_connection_id = %connection_id, - request_direction = "downstream", - request_transport = "websocket_handshake", - request_method = "GET", - request_path = "/v1/responses", - request_headers = ?raw_headers, - contains_sensitive_data = true, - "request dump (unredacted)" + TraceContext::new(&connection_id).headers( + "client.connection.headers", + serde_json::json!({"transport": "websocket", "method": "GET", "path": "/v1/responses"}), + raw_headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())), ); let session = ResponsesWebSocketSession { service: self.service.clone(), @@ -187,26 +183,15 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke } request_count = request_count.saturating_add(1); let correlation_id = Arc::::from(service.next_request_id()); - tracing::info!( - target: "request_dump", - provider = "openai", - websocket_connection_id = connection.id(), - correlation_id = %correlation_id, - request_index = request_count, - request_direction = "downstream", - request_transport = "websocket", - request_payload = %payload, - contains_sensitive_data = true, - "request dump (unredacted)" - ); let decoded = match decode_response_create_with_context(&payload, &request_headers) { Ok(decoded) => decoded.with_client_context(client_ip, user_agent.clone()), Err(error) => { - tracing::info!( - websocket_connection_id = connection.id(), - request_id = %correlation_id, - error = %error, - "Responses WebSocket request rejected" + trace_rejected_request( + &correlation_id, + connection.id(), + &payload, + "decode", + &error.to_string(), ); if send_protocol_error( &mut connection, @@ -225,6 +210,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke let decoded = replay.prepare(decoded); // deadline 与 Text 可能同时就绪；在任何上游执行开始前再次封住该竞争窗口。 if connection.is_expired() { + trace_rejected_request( + &correlation_id, + connection.id(), + &payload, + "connection_expired", + "", + ); expire_connection(&mut connection).await; break; } @@ -239,6 +231,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke { Ok(started) => started, Err(error) => { + trace_rejected_request( + &correlation_id, + connection.id(), + &payload, + "start", + &error.to_string(), + ); if send_gateway_error(&mut connection, &error, &correlation_id).await == ForwardOutcome::Disconnect { @@ -247,14 +246,14 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke continue; } }; - tracing::info!( - target: "request_dump", - provider = "openai", - websocket_connection_id = connection.id(), - correlation_id = %correlation_id, - request_id = %started.request_id, - "request dump correlation" - ); + started.session.trace().record("client.connection", serde_json::json!({ + "transport": "websocket", "connectionId": connection.id(), "correlationId": correlation_id, + "requestIndex": request_count, + })); + started + .session + .trace() + .capture("client.request.body", payload.as_bytes()); if forward_execution(&mut connection, started, &mut replay).await == ForwardOutcome::Disconnect @@ -270,6 +269,28 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke connection.log_summary(request_count); } +// 尚未建立模型请求的拒绝也保留原文入口，用返回给客户端的 correlation ID 检索。 +fn trace_rejected_request( + correlation_id: &str, + connection_id: &str, + payload: &str, + phase: &'static str, + error: &str, +) { + let trace = TraceContext::new(correlation_id); + trace.record( + "client.connection", + serde_json::json!({"connectionId": connection_id}), + ); + trace.capture("client.request.body", payload.as_bytes()); + trace.record( + "client.request.rejected", + serde_json::json!({ + "phase": phase, "detail": diagnostic_json(&serde_json::json!({"message": error})), + }), + ); +} + async fn expire_connection(connection: &mut ResponsesWebSocketConnection) { if connection .send_text( diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index 366d2e15a..cc87c502d 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -429,6 +429,8 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() { .lock() .expect("usage detail") .replace(UsageDetail { + trace: None, + related_requests: Vec::new(), request: usage_record_with_account( "req_detail", "acct_snap_a", diff --git a/backend/crates/gateway-core/Cargo.toml b/backend/crates/gateway-core/Cargo.toml index 00478d81e..f0c41b8ec 100644 --- a/backend/crates/gateway-core/Cargo.toml +++ b/backend/crates/gateway-core/Cargo.toml @@ -13,11 +13,13 @@ path = "tests/main.rs" [dependencies] async-trait = "0.1.89" +base64 = "0.23.0" bytes = "1.11.1" futures = "0.3.32" futures-timer = "3.0.3" serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] } +sha2 = "0.11.0" semver = "1.0.27" thiserror = "2.0.18" tracing = "0.1.44" diff --git a/backend/crates/gateway-core/src/diagnostics/capture.rs b/backend/crates/gateway-core/src/diagnostics/capture.rs new file mode 100644 index 000000000..e4f3d3258 --- /dev/null +++ b/backend/crates/gateway-core/src/diagnostics/capture.rs @@ -0,0 +1,157 @@ +//! 诊断内容的统一筛选边界。未知字符串只保留长度和摘要。 + +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; + +/// 原始字节的长度与 SHA-256，不复制或保留正文。 +#[must_use] +pub fn body_fingerprint(bytes: &[u8]) -> Value { + let digest = Sha256::digest(bytes); + let hex: String = digest + .iter() + .flat_map(|byte| { + let alphabet = b"0123456789abcdef"; + [ + char::from(alphabet[usize::from(byte >> 4)]), + char::from(alphabet[usize::from(byte & 15)]), + ] + }) + .collect(); + json!({"bytes": bytes.len(), "sha256": hex}) +} + +/// 保留可诊断的协议结构；凭据永不进入普通日志或数据库。 +#[must_use] +pub fn diagnostic_json(value: &Value) -> Value { + capture(value, "", 0, &mut 96) +} + +fn capture(value: &Value, key: &str, depth: usize, budget: &mut usize) -> Value { + if *budget == 0 || depth > 6 { + return json!({"omitted": true}); + } + *budget -= 1; + let key = key.to_ascii_lowercase(); + if secret_key(&key) { + return Value::String("".to_owned()); + } + match value { + Value::Object(object) => { + let mut result = Map::new(); + for (name, value) in object.iter().take(48) { + if *budget == 0 { + break; + } + result.insert(bounded(name, 96), capture(value, name, depth + 1, budget)); + } + if result.len() < object.len() { + result.insert( + "_omittedFields".to_owned(), + json!(object.len() - result.len()), + ); + } + Value::Object(result) + } + Value::Array(items) => json!({ + "length": items.len(), + "sample": items.iter().take(4) + .map(|item| capture(item, key.as_str(), depth + 1, budget)) + .collect::>() + }), + Value::String(text) + if safe_header(&key) || safe_value_key(&key) && protocol_label(text) => + { + Value::String(bounded(text, 256)) + } + Value::String(text) => body_fingerprint(text.as_bytes()), + other => other.clone(), + } +} + +/// 请求/响应头保持多值；安全 trace ID 原样保存，未知值只保存摘要。 +#[must_use] +pub fn diagnostic_headers<'a>(headers: impl IntoIterator) -> Value { + let mut result = Map::new(); + for (name, value) in headers.into_iter().take(64) { + let name = bounded(&name.to_ascii_lowercase(), 96); + let value = if secret_key(&name) { + Value::String("".to_owned()) + } else if safe_header(&name) { + Value::String(bounded(value, 256)) + } else { + body_fingerprint(value.as_bytes()) + }; + let values = result + .entry(name) + .or_insert_with(|| Value::Array(Vec::new())); + if let Value::Array(values) = values { + values.push(value); + } + } + Value::Object(result) +} + +fn secret_key(key: &str) -> bool { + key.contains("authorization") + || key.contains("cookie") + || key.contains("api_key") + || key.contains("api-key") + || key.contains("token") && !key.ends_with("tokens") + || key.contains("password") + || key.contains("secret") + || key.contains("attestation") + || matches!(key, "x-oai-is" | "x-oai-is-update") +} + +fn safe_header(key: &str) -> bool { + matches!( + key, + "x-oai-request-id" + | "x-client-request-id" + | "x-request-id" + | "request-id" + | "openai-request-id" + | "x-openai-request-id" + | "cf-ray" + | "traceparent" + | "tracestate" + | "content-type" + | "content-length" + | "content-encoding" + | "retry-after" + | "openai-processing-ms" + | "x-processing-ms" + | "date" + | "server" + | "sec-websocket-extensions" + | "x-codex-allowed" + | "x-codex-limit-reached" + | "x-codex-active-limit" + | "x-models-etag" + ) +} + +fn safe_value_key(key: &str) -> bool { + safe_header(key) + || matches!( + key, + "type" | "event" | "status" | "code" | "model" | "role" | "object" | "service_tier" + ) +} + +pub(super) fn bounded(value: &str, limit: usize) -> String { + value + .chars() + .take(limit) + .map(|c| if c.is_control() { ' ' } else { c }) + .collect() +} + +/// Event names/codes are protocol labels, never arbitrary upstream text. +pub(super) fn protocol_label(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':')) +} diff --git a/backend/crates/gateway-core/src/diagnostics/mod.rs b/backend/crates/gateway-core/src/diagnostics/mod.rs new file mode 100644 index 000000000..b5de9a0ef --- /dev/null +++ b/backend/crates/gateway-core/src/diagnostics/mod.rs @@ -0,0 +1,9 @@ +//! 有界、请求局部的诊断证据；不参与路由、重试或客户端协议决策。 + +mod capture; +mod stream; +mod trace; + +pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json}; +pub use stream::{StreamCapture, StreamFormat}; +pub use trace::TraceContext; diff --git a/backend/crates/gateway-core/src/diagnostics/stream.rs b/backend/crates/gateway-core/src/diagnostics/stream.rs new file mode 100644 index 000000000..4e6dc434c --- /dev/null +++ b/backend/crates/gateway-core/src/diagnostics/stream.rs @@ -0,0 +1,165 @@ +//! 仅用于观测的有界流分帧；超过限额时明确记缺口，不影响实际协议解析。 + +use serde_json::json; + +use super::TraceContext; + +const MAX_FRAME_BYTES: usize = 128 * 1024; + +#[derive(Clone, Copy)] +pub enum StreamFormat { + Sse, + JsonLines, +} + +/// 同一 exchange 持有一个 observer，跨 HTTP chunk 拼接完整事件。 +pub struct StreamCapture { + trace: TraceContext, + format: StreamFormat, + line: Vec, + data: Vec, + event_name: Option, + previous_cr: bool, + first_line: bool, + skipping_line: bool, + skipping_event: bool, + finished: bool, +} + +impl StreamCapture { + #[must_use] + pub fn new(trace: TraceContext, format: StreamFormat) -> Self { + Self { + trace, + format, + line: Vec::new(), + data: Vec::new(), + event_name: None, + previous_cr: false, + first_line: true, + skipping_line: false, + skipping_event: false, + finished: false, + } + } + + pub fn push(&mut self, bytes: &[u8]) { + self.trace.dump("upstream.chunk", bytes); + for byte in bytes { + if self.previous_cr && *byte == b'\n' { + self.previous_cr = false; + continue; + } + self.previous_cr = *byte == b'\r'; + if matches!(*byte, b'\n' | b'\r') { + if !self.skipping_line { + self.line(); + } + self.line.clear(); + self.skipping_line = false; + } else if !self.skipping_line { + if self.line.len() < MAX_FRAME_BYTES { + self.line.push(*byte); + } else { + self.trace.record( + "capture.gap", + json!({"reason": "line_size_limit", "limitBytes": MAX_FRAME_BYTES}), + ); + self.line.clear(); + self.skipping_line = true; + self.skipping_event = matches!(self.format, StreamFormat::Sse); + self.data.clear(); + } + } + } + } + + pub fn finish(&mut self) { + if self.finished { + return; + } + let partial_frame = !self.line.is_empty() + || !self.data.is_empty() + || self.skipping_event + || self.skipping_line; + if !self.line.is_empty() && !self.skipping_line { + self.line(); + } + if !self.data.is_empty() && !self.skipping_event { + self.trace.capture_named_event( + "upstream.event", + &self.data, + self.event_name.as_deref(), + ); + } + self.trace + .record("upstream.eof", json!({"partialFrame": partial_frame})); + self.finished = true; + } + + fn line(&mut self) { + let line = if self.first_line { + self.first_line = false; + self.line + .strip_prefix(b"\xef\xbb\xbf") + .unwrap_or(&self.line) + } else { + &self.line + }; + if matches!(self.format, StreamFormat::JsonLines) { + if !line.is_empty() { + self.trace.capture_event("upstream.event", line); + } + return; + } + if line.is_empty() { + if !self.data.is_empty() && !self.skipping_event { + self.trace.capture_named_event( + "upstream.event", + &self.data, + self.event_name.as_deref(), + ); + } + self.data.clear(); + self.event_name = None; + self.skipping_event = false; + } else if let Some(name) = line.strip_prefix(b"event:") { + self.event_name = Some(super::capture::bounded( + String::from_utf8_lossy(name).trim(), + 128, + )); + } else if let Some(data) = line.strip_prefix(b"data:") { + let data = data.strip_prefix(b" ").unwrap_or(data); + if self.skipping_event { + return; + } + if self.data.len() + data.len() + 1 > MAX_FRAME_BYTES { + self.trace.record( + "capture.gap", + json!({"reason": "event_size_limit", "limitBytes": MAX_FRAME_BYTES}), + ); + self.data.clear(); + self.skipping_event = true; + } else { + if !self.data.is_empty() { + self.data.push(b'\n'); + } + self.data.extend_from_slice(data); + } + } + } +} + +impl Drop for StreamCapture { + fn drop(&mut self) { + if !self.finished { + self.trace.record( + "upstream.reader.dropped", + json!({ + "partialBytes": self.line.len() + self.data.len(), + "captureIncomplete": self.skipping_event || self.skipping_line, + }), + ); + } + } +} diff --git a/backend/crates/gateway-core/src/diagnostics/trace.rs b/backend/crates/gateway-core/src/diagnostics/trace.rs new file mode 100644 index 000000000..01a056b96 --- /dev/null +++ b/backend/crates/gateway-core/src/diagnostics/trace.rs @@ -0,0 +1,338 @@ +//! 首部与最近事件同时保留，流式 delta 合并，所有淘汰均有计数。 + +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use serde::Serialize; +use serde_json::{Value, json}; + +use super::capture::bounded; +use super::{body_fingerprint, diagnostic_headers, diagnostic_json}; + +const MAX_EVENTS: usize = 128; +const HEAD_EVENTS: usize = 8; +const MAX_DATA_BYTES: usize = 4096; +const MAX_BUFFER_BYTES: usize = 64 * 1024; + +/// 显式传给异步任务的关联上下文；默认值禁用捕获，避免全局请求映射。 +#[derive(Clone, Default)] +pub struct TraceContext { + state: Option>>, + attempt_index: u32, + exchange_id: Option, +} + +impl std::fmt::Debug for TraceContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TraceContext") + .field("attempt_index", &self.attempt_index) + .field("exchange_id", &self.exchange_id) + .finish_non_exhaustive() + } +} + +struct TraceState { + request_id: String, + started_at_ms: u128, + started: Instant, + sequence: u64, + exchanges: u64, + wire_frames: u64, + wire_bytes: u64, + dropped_events: u64, + bytes: usize, + events: VecDeque, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct TraceEvent { + sequence: u64, + last_sequence: u64, + elapsed_ms: u64, + last_elapsed_ms: u64, + attempt_index: u32, + exchange_id: Option, + stage: &'static str, + count: u64, + data: Value, + #[serde(skip)] + bytes: usize, +} + +impl TraceContext { + /// 一个模型请求只创建一次，所有 attempt 共享同一有界时间线。 + #[must_use] + pub fn new(request_id: &str) -> Self { + Self { + state: Some(Arc::new(Mutex::new(TraceState { + request_id: bounded(request_id, 128), + started_at_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + started: Instant::now(), + sequence: 0, + exchanges: 0, + wire_frames: 0, + wire_bytes: 0, + dropped_events: 0, + bytes: 0, + events: VecDeque::new(), + }))), + ..Self::default() + } + } + + #[must_use] + pub fn attempt(&self, attempt_index: u32) -> Self { + Self { + state: self.state.clone(), + attempt_index, + exchange_id: None, + } + } + + /// 一个 attempt 中的每次实际 transport exchange 也有独立编号。 + #[must_use] + pub fn exchange(&self, transport: &'static str) -> Self { + let mut context = self.clone(); + if let Some(state) = &self.state { + let mut state = state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.exchanges += 1; + context.exchange_id = Some(state.exchanges); + } + context.record("upstream.exchange.started", json!({"transport": transport})); + context + } + + /// details 只允许调用方构造的诊断事实；不接收原始请求/响应正文。 + pub fn record(&self, stage: &'static str, data: Value) { + self.push(stage, data, false); + } + + /// 同一个头部边界只采集一次：默认脱敏，显式开启 dump 时另存完整字节。 + /// facts 必须为调用方构造的安全对象；原始头值只进入独立 dump。 + pub fn headers<'a>( + &self, + stage: &'static str, + mut facts: Value, + headers: impl IntoIterator, + ) { + if self.state.is_none() { + return; + } + let Some(facts_object) = facts.as_object_mut() else { + return; + }; + let headers: Vec<_> = headers.into_iter().collect(); + let summary = diagnostic_headers( + headers + .iter() + .map(|(name, value)| (*name, std::str::from_utf8(value).unwrap_or(""))), + ); + facts_object.insert("headers".to_owned(), summary); + self.record(stage, facts); + if tracing::enabled!(target: "request_dump", tracing::Level::INFO) { + let values: Vec<_> = headers + .iter() + .map(|(name, value)| json!({"name": name, "valueBase64": STANDARD.encode(value)})) + .collect(); + if let Ok(bytes) = serde_json::to_vec(&json!({"headers": values})) { + self.dump(stage, &bytes); + } + } + } + + /// 入站 JSON 先记录再解析业务语义，包括未知事件和 metadata。 + /// 原文仅在现有 request_dump 开关打开时写入其独立文件。 + pub fn capture(&self, stage: &'static str, bytes: &[u8]) { + self.dump(stage, bytes); + self.capture_event(stage, bytes); + } + + /// StreamCapture 已转储原始 chunk 时，只提取完整事件的安全摘要。 + pub fn capture_event(&self, stage: &'static str, bytes: &[u8]) { + self.capture_named_event(stage, bytes, None); + } + + pub fn capture_named_event(&self, stage: &'static str, bytes: &[u8], name: Option<&str>) { + if self.state.is_none() { + return; + } + let fingerprint = body_fingerprint(bytes); + if bytes.len() > 128 * 1024 { + self.record("capture.gap", json!({"reason": "json_inspection_size_limit", "stage": stage, "body": fingerprint})); + return; + } + let parsed = serde_json::from_slice::(bytes).ok(); + let event_type = parsed + .as_ref() + .and_then(|v| v.get("type")) + .and_then(Value::as_str) + .or(name) + .filter(|kind| super::capture::protocol_label(kind)); + let delta = event_type.is_some_and(|kind| kind.ends_with(".delta")); + let data = json!({ + "body": fingerprint, + "eventType": event_type.map(|kind| bounded(kind, 128)), + "jsonValid": parsed.is_some(), + "metadata": parsed.as_ref().filter(|_| !delta).map(diagnostic_json), + }); + self.push(stage, data, delta); + } + + /// 只记录业务解析边界；metadata 已由 transport 在解析前捕获。 + pub fn wire_event(&self, protocol: &str, event_type: Option<&str>) { + let event_type = event_type.filter(|kind| super::capture::protocol_label(kind)); + let delta = event_type.is_some_and(|kind| kind.ends_with(".delta")); + self.push( + "provider.event", + json!({ + "protocol": protocol, + "eventType": event_type.map(|kind| bounded(kind, 128)), + }), + delta, + ); + } + + /// 完整报文捕获与摘要使用相同 request / attempt / exchange ID。 + pub fn dump(&self, stage: &'static str, bytes: &[u8]) { + if !tracing::enabled!(target: "request_dump", tracing::Level::INFO) { + return; + } + if let Some(state) = &self.state { + let mut state = state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.wire_frames += 1; + state.wire_bytes = state.wire_bytes.saturating_add(bytes.len() as u64); + let wire_sequence = state.wire_frames; + let chunk_count = bytes.len().div_ceil(32 * 1024).max(1); + // Each log entry stays bounded even for images or very large WebSocket frames. + for chunk_index in 0..chunk_count { + let start = chunk_index * 32 * 1024; + let chunk = &bytes[start..bytes.len().min(start + 32 * 1024)]; + tracing::info!(target: "request_dump", request_id = %state.request_id, + attempt_index = self.attempt_index, exchange_id = self.exchange_id, + stage, wire_sequence, chunk_index, chunk_count, + body_bytes = bytes.len(), body_base64 = %STANDARD.encode(chunk), + contains_sensitive_data = true, "diagnostic wire capture"); + } + } + } + + /// 与请求终态原子持久化；旧记录没有此字段，不伪造历史时间线。 + #[must_use] + pub fn snapshot(&self) -> Option { + let state = self + .state + .as_ref()? + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + Some(json!({ + "schemaVersion": 1, "requestId": state.request_id, + "startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence, + "droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS, + "captureMode": "sanitized", "events": state.events, + "wireDumpEnabled": tracing::enabled!(target: "request_dump", tracing::Level::INFO), + "wireFrames": state.wire_frames, "wireBytes": state.wire_bytes, + "maxBufferBytes": MAX_BUFFER_BYTES, "headEvents": HEAD_EVENTS, + "snapshotBoundary": "execution_finalization", + })) + } + + fn push(&self, stage: &'static str, mut data: Value, coalesce: bool) { + let Some(state) = &self.state else { return }; + let mut encoded = serde_json::to_vec(&data).unwrap_or_default(); + if encoded.len() > MAX_DATA_BYTES { + data = json!({"truncated": true, "summary": body_fingerprint(&encoded), + "eventType": data.get("eventType"), "body": data.get("body"), + "jsonValid": data.get("jsonValid"), + }); + encoded = serde_json::to_vec(&data).unwrap_or_default(); + } + let mut state = state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.sequence += 1; + let sequence = state.sequence; + let elapsed_ms = u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX); + if coalesce + && let Some(last) = state.events.iter_mut().rev().find(|event| { + event.stage == stage + && event.attempt_index == self.attempt_index + && event.exchange_id == self.exchange_id + }) + && last.stage == stage + && last.attempt_index == self.attempt_index + && last.exchange_id == self.exchange_id + && last.data.get("eventType") == data.get("eventType") + { + last.count += 1; + last.last_sequence = sequence; + last.last_elapsed_ms = elapsed_ms; + return; + } + // 普通日志只输出有界安全事实；进程意外退出时仍可按 ID 检索已发生阶段。 + tracing::info!(target: "request_trace", request_id = %state.request_id, + attempt_index = self.attempt_index, exchange_id = self.exchange_id, + sequence, elapsed_ms, stage, data = %data, "request trace"); + let bytes = encoded.len() + 256; + state.bytes += bytes; + state.events.push_back(TraceEvent { + sequence, + last_sequence: sequence, + elapsed_ms, + last_elapsed_ms: elapsed_ms, + attempt_index: self.attempt_index, + exchange_id: self.exchange_id, + stage, + count: 1, + data, + bytes, + }); + while state.events.len() > MAX_EVENTS || state.bytes > MAX_BUFFER_BYTES { + // Preserve failures and retry decisions ahead of routine middle-of-stream events. + // Head/tail and important stages compete under the same hard memory limit. + let index = (HEAD_EVENTS..state.events.len().saturating_sub(8)) + .find(|index| { + !matches!( + state.events[*index].stage, + "attempt.started" + | "account.selected" + | "attempt.failed" + | "retry.decided" + | "upstream.close" + | "upstream.read.failed" + | "capture.gap" + | "request.finished" + ) + }) + .unwrap_or(if state.events.len() > HEAD_EVENTS { + HEAD_EVENTS + } else { + 0 + }); + if let Some(event) = state.events.remove(index) { + state.bytes = state.bytes.saturating_sub(event.bytes); + state.dropped_events += event.count; + } + } + } +} + +impl Drop for TraceState { + fn drop(&mut self) { + tracing::info!(target: "request_trace", request_id = %self.request_id, + stage = "request.trace.closed", total_events = self.sequence, + dropped_events = self.dropped_events, wire_frames = self.wire_frames, + wire_bytes = self.wire_bytes, "request diagnostic capture closed"); + } +} diff --git a/backend/crates/gateway-core/src/engine/coordinator.rs b/backend/crates/gateway-core/src/engine/coordinator.rs index c6f319604..4bd5d9ad4 100644 --- a/backend/crates/gateway-core/src/engine/coordinator.rs +++ b/backend/crates/gateway-core/src/engine/coordinator.rs @@ -1,5 +1,8 @@ //! 唯一的账号重试、发送与下游 commit barrier owner。 +use crate::diagnostics::TraceContext; +use serde_json::json; + use std::collections::BTreeSet; use std::future::Future; use std::num::NonZeroU32; @@ -144,12 +147,26 @@ where .position(|candidate| candidate.provider() == pin.provider()) .ok_or(EngineError::ContinuationPinMismatch) })?; + let trace = TraceContext::new(request_id.as_str()); + trace.record( + "request.routed", + json!({ + "endpoint": request.endpoint, "protocol": request.protocol, + "clientTransport": request.client_transport, + "model": request.requested_model.as_ref().map(|model| model.as_str()), + "configRevision": format!("{:?}", request.config_revision), + "admissionMs": request.admission_decision_ms, + "continuation": format!("{continuation_attempt:?}"), + "candidateCount": plan.candidates().len(), + }), + ); let image_generation_requested = operation.image_generation_requested(); let mut session = ResponseExecutionSession { engine: Arc::clone(&self.engine), request_id, client_api_key_ref, observation: ResponseObservation::new(timing_started_at), + trace, deadline, deadline_timer: Delay::new( deadline @@ -238,6 +255,7 @@ pub struct ResponseExecutionSession { request_id: ModelRequestId, client_api_key_ref: crate::policy::ClientApiKeyId, observation: ResponseObservation, + trace: TraceContext, deadline: SystemTime, /// 会话级 deadline 计时器；deadline 固定，帧循环内复用而非逐事件新建。 deadline_timer: Fuse, @@ -291,6 +309,11 @@ impl ResponseExecutionSession where S: ExecutionStore, { + /// 当前请求共享的诊断上下文。 + pub fn trace(&self) -> TraceContext { + self.trace.clone() + } + /// 读取下一条 canonical event；首条未提交事件会携带 commit 要求。 /// /// # Errors @@ -398,6 +421,10 @@ where return Err(EngineError::InvalidDeliveryState); } let committed_at = SystemTime::now(); + self.trace.record( + "downstream.committed", + json!({"status": client_status_code}), + ); if self.request_persisted { best_effort_store_write( "mark_downstream_committed", @@ -563,6 +590,11 @@ where return Err(EngineError::Deadline); } PollBoundary::Item(Some(Ok(mut event))) => { + if let Some(wire) = event.wire_event() { + self.trace + .attempt(self.attempts) + .wire_event(wire.protocol(), wire.event_type()); + } if let Some(state) = event.session_update() { self.observe_session_checkpoint(state); } @@ -682,7 +714,8 @@ where })); let context = AttemptContext::new( RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone()) - .with_timing_started_at(self.observation.timing_started_at), + .with_timing_started_at(self.observation.timing_started_at) + .with_trace(self.trace.clone()), next_attempt, self.deadline, self.plan.account_selection_policy(), @@ -705,6 +738,18 @@ where .ok_or_else(|| EngineError::ProviderNotRegistered { provider: candidate.provider().as_str().to_owned(), })?; + let attempt_trace = self.trace.attempt(next_attempt.get()); + attempt_trace.record( + "attempt.started", + json!({ + "provider": candidate.provider().as_str(), + "model": candidate.upstream_model().map(|model| model.as_str()), + "transportPolicy": format!("{attempt_transport:?}"), + "continuation": format!("{:?}", self.continuation_attempt), + "pinnedAccount": pinned_account.as_ref().map(|id| id.as_str()), + "transportRecovery": is_transport_recovery, + }), + ); let provider_request = ProviderRequest::new(self.operation.clone(), candidate.clone()); let stream = match poll_provider( provider, @@ -729,6 +774,7 @@ where ProviderBoundary::Result(result) => match *result { Ok(stream) => stream, Err(error) => { + record_trace_error(&attempt_trace, &error); if self.prepare_unavailable_native_continuation_replay(&error) { return Ok(Some(PullOutcome::AttemptDiscarded)); } @@ -800,6 +846,11 @@ where } let metadata = stream.metadata().clone(); + attempt_trace.record("account.selected", json!({ + "provider": metadata.provider().as_str(), "accountId": metadata.provider_account_id().as_str(), + "transport": metadata.transport().as_str(), + "selectionMs": metadata.selection_observation().map(|o| o.account_selection_wait_ms()), + })); let selection_observation = metadata.selection_observation(); let capacity = selection_observation.and_then(|observation| observation.capacity()); if !self.account_selection.is_diagnostic() @@ -1006,6 +1057,7 @@ where ) -> Result { // 原始 wire 只活在 request-local 决策状态；clone、attempt 记录与持久化终态 // 均只接触已剥离的稳定错误字段。 + record_trace_error(&self.trace.attempt(self.attempts), &error); let mut atomic_client_events = error.take_atomic_client_events(); let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?; self.record_provider_failure(current.metadata.provider().clone(), error.kind()); @@ -1093,6 +1145,14 @@ where || account_rotation_retry || transport_recovery.is_some(); + self.trace.attempt(current.index.get()).record("retry.decided", json!({ + "retryable": retryable, "continuationRetry": continuation_retry, + "sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry, + "ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(), + "delayMs": transport_recovery.map(|(_, delay)| duration_ms(delay)), + "downstreamCommitted": self.downstream_committed_at.is_some(), + "sendState": format!("{attempt_send_state:?}"), + })); if retryable { // 普通 clone 只保留稳定事实；原始 wire/HTTP response 由 request-local // 所有权保留到下一次 attempt 成功，或最终空选路时返回客户端。 @@ -1303,6 +1363,10 @@ where self.current_transport_observation(); let service_tier = self.current_service_tier(); let provider_metadata_json = self.current_provider_metadata_json(); + self.trace.record( + "request.finished", + json!({"outcome": "succeeded", "attempts": self.attempts}), + ); if self.request_persisted { best_effort_store_write( "finalize_model_request", @@ -1325,6 +1389,7 @@ where websocket_pool, service_tier, provider_metadata_json, + diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()), error: None, provider_error_code: None, raw_upstream_error: None, @@ -1443,6 +1508,14 @@ where if self.finalized { return Ok(()); } + self.trace.record( + "request.finished", + json!({ + "outcome": format!("{:?}", finalization.outcome), + "errorKind": finalization.error.kind().as_str(), + "sendState": format!("{:?}", finalization.send_state), "attempts": self.attempts, + }), + ); if !self.request_persisted { self.finalized = true; return Ok(()); @@ -1497,6 +1570,7 @@ where websocket_pool, service_tier, provider_metadata_json, + diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()), error: Some(finalization.error), provider_error_code: finalization.provider_error_code, raw_upstream_error: finalization.raw_upstream_error, @@ -1787,3 +1861,15 @@ const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> Upst fn duration_ms(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) } + +fn record_trace_error(trace: &TraceContext, error: &ProviderError) { + trace.record("attempt.failed", json!({ + "kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()), + "upstreamStatus": error.upstream_status(), + "upstreamCode": error.upstream_code().map(|code| code.as_str()), + "rawError": error.raw_upstream_error().map(|raw| { + let value = serde_json::from_str(raw.as_str()).unwrap_or_else(|_| json!(raw.as_str())); + crate::diagnostics::diagnostic_json(&value) + }), + })); +} diff --git a/backend/crates/gateway-core/src/engine/execution.rs b/backend/crates/gateway-core/src/engine/execution.rs index 5e61789e7..0986c1c76 100644 --- a/backend/crates/gateway-core/src/engine/execution.rs +++ b/backend/crates/gateway-core/src/engine/execution.rs @@ -155,6 +155,9 @@ pub struct StartedExecution { } pub trait ExecutionSession: Send { + fn trace(&self) -> crate::diagnostics::TraceContext { + crate::diagnostics::TraceContext::default() + } fn next_event(&mut self) -> BoxFuture<'_, Result, EngineError>>; fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result, EngineError>>; fn response_headers(&self) -> &[ProviderResponseHeader]; @@ -1035,6 +1038,12 @@ impl Drop for DefaultExecutionSession { } impl ExecutionSession for DefaultExecutionSession { + fn trace(&self) -> crate::diagnostics::TraceContext { + self.core + .as_ref() + .map(ResponseExecutionSession::trace) + .unwrap_or_default() + } fn next_event(&mut self) -> BoxFuture<'_, Result, EngineError>> { Box::pin(async move { let result = self.core_mut()?.next_event().await; diff --git a/backend/crates/gateway-core/src/engine/mod.rs b/backend/crates/gateway-core/src/engine/mod.rs index 460509481..6cd4eb580 100644 --- a/backend/crates/gateway-core/src/engine/mod.rs +++ b/backend/crates/gateway-core/src/engine/mod.rs @@ -316,6 +316,7 @@ pub struct RequestAttemptContext { request_id: ModelRequestId, client_api_key_ref: ClientApiKeyId, timing_started_at: Instant, + trace: crate::diagnostics::TraceContext, } impl RequestAttemptContext { @@ -325,9 +326,16 @@ impl RequestAttemptContext { request_id, client_api_key_ref, timing_started_at: Instant::now(), + trace: crate::diagnostics::TraceContext::default(), } } + #[must_use] + pub fn with_trace(mut self, trace: crate::diagnostics::TraceContext) -> Self { + self.trace = trace; + self + } + /// 覆盖本次请求的单调计时原点。 #[must_use] pub fn with_timing_started_at(mut self, timing_started_at: Instant) -> Self { @@ -367,6 +375,12 @@ pub struct AttemptContext { } impl AttemptContext { + /// 当前 attempt 的诊断关联；克隆后可传给后台 transport 任务。 + #[must_use] + pub fn trace(&self) -> crate::diagnostics::TraceContext { + self.request.trace.attempt(self.attempt_index.get()) + } + #[must_use] pub const fn new( request: RequestAttemptContext, @@ -612,6 +626,8 @@ pub struct ModelRequestFinalization { pub service_tier: Option, /// Provider 已筛选的专有观测 JSON；Core 不解释字段。 pub provider_metadata_json: Option, + /// 请求全程的有界诊断快照，跨 Provider 与重试保留。 + pub diagnostic_trace_json: Option, pub error: Option, pub provider_error_code: Option, /// Provider 返回的原始错误正文或 WebSocket close/error frame。 diff --git a/backend/crates/gateway-core/src/lib.rs b/backend/crates/gateway-core/src/lib.rs index fba868f19..89e6c6196 100644 --- a/backend/crates/gateway-core/src/lib.rs +++ b/backend/crates/gateway-core/src/lib.rs @@ -4,6 +4,7 @@ //! 具体客户端协议和具体 Provider 都通过外层 adapter 接入。 pub mod account; +pub mod diagnostics; pub mod engine; pub mod error; pub mod event; diff --git a/backend/crates/gateway-core/tests/diagnostics/capture.rs b/backend/crates/gateway-core/tests/diagnostics/capture.rs new file mode 100644 index 000000000..0ea7cd6d4 --- /dev/null +++ b/backend/crates/gateway-core/tests/diagnostics/capture.rs @@ -0,0 +1,49 @@ +use gateway_core::diagnostics::{diagnostic_headers, diagnostic_json}; +use serde_json::json; + +#[test] +fn credentials_and_content_never_enter_diagnostics_but_trace_ids_survive() { + let headers = diagnostic_headers([ + ("Authorization", "Bearer never-persist-this"), + ("Set-Cookie", "session=never-persist-this"), + ("X-OAI-Request-ID", "upstream-123"), + ("x-codex-turn-state", "opaque-never-persist-this"), + ("cf-ray", "ray-one"), + ("cf-ray", "ray-two"), + ]); + assert_eq!(headers["x-oai-request-id"][0], "upstream-123"); + assert_eq!(headers["cf-ray"].as_array().unwrap().len(), 2); + assert!(!headers.to_string().contains("never-persist-this")); + let content = diagnostic_json(&json!({ + "type": "codex.response.metadata", "headers": headers, + "access_token": "never-persist-this", "refresh_token": "never-persist-this", + "response": {"output": [{"text": "private model output"}]}, + "input": "private user prompt", "unknown": "private extension", + })); + assert_eq!(content["type"], "codex.response.metadata"); + let text = content.to_string(); + for secret in [ + "never-persist-this", + "private model output", + "private user prompt", + "private extension", + ] { + assert!(!text.contains(secret), "leaked {secret}"); + } +} + +#[test] +fn arbitrary_event_type_text_is_fingerprinted_instead_of_logged() { + let trace = gateway_core::diagnostics::TraceContext::new("req_type"); + trace.capture( + "upstream.event", + br#"{"type":"secret@example.com","message":"private-content"}"#, + ); + let snapshot = trace.snapshot().unwrap(); + assert_eq!( + snapshot["events"][0]["data"]["eventType"], + serde_json::Value::Null + ); + assert!(!snapshot.to_string().contains("secret@example.com")); + assert!(!snapshot.to_string().contains("private-content")); +} diff --git a/backend/crates/gateway-core/tests/diagnostics/mod.rs b/backend/crates/gateway-core/tests/diagnostics/mod.rs new file mode 100644 index 000000000..6f8d1e95c --- /dev/null +++ b/backend/crates/gateway-core/tests/diagnostics/mod.rs @@ -0,0 +1,3 @@ +mod capture; +mod stream; +mod trace; diff --git a/backend/crates/gateway-core/tests/diagnostics/stream.rs b/backend/crates/gateway-core/tests/diagnostics/stream.rs new file mode 100644 index 000000000..6346fc9c4 --- /dev/null +++ b/backend/crates/gateway-core/tests/diagnostics/stream.rs @@ -0,0 +1,70 @@ +use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext}; + +#[test] +fn sse_capture_reassembles_every_chunk_boundary_and_preserves_unknown_events() { + let bytes = "event: future.metadata\r\ndata: {\"headers\":{\"x-request-id\":\"up-1\"},\"unknown\":\"中文\"}\r\n\r\ndata: {\"type\":\"response.completed\"}\n\n".as_bytes(); + for boundary in 0..=bytes.len() { + let trace = TraceContext::new("req_sse"); + let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse); + capture.push(&bytes[..boundary]); + capture.push(&bytes[boundary..]); + capture.finish(); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + assert_eq!(events.len(), 3, "boundary {boundary}"); + assert_eq!(events[0]["data"]["eventType"], "future.metadata"); + assert_eq!( + events[0]["data"]["metadata"]["headers"]["x-request-id"], + "up-1" + ); + assert_eq!(events[1]["data"]["eventType"], "response.completed"); + assert_eq!(events[2]["data"]["partialFrame"], false); + } +} + +#[test] +fn oversize_frames_report_a_gap_and_resume_at_the_next_frame() { + let trace = TraceContext::new("req_sse"); + let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse); + capture.push(b"data: "); + capture.push(&vec![b'x'; 130 * 1024]); + capture.push(b"\n\ndata: {\"type\":\"response.completed\"}\n\n"); + capture.finish(); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + assert_eq!(events[0]["stage"], "capture.gap"); + assert_eq!(events[1]["data"]["eventType"], "response.completed"); +} + +#[test] +fn dropping_a_partial_stream_is_distinct_from_upstream_eof() { + let trace = TraceContext::new("req_partial"); + { + let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse); + capture.push(b"data: {\"type\":"); + } + let snapshot = trace.snapshot().unwrap(); + assert_eq!(snapshot["events"][0]["stage"], "upstream.reader.dropped"); + assert!( + snapshot["events"][0]["data"]["partialBytes"] + .as_u64() + .unwrap() + > 0 + ); +} + +#[test] +fn sse_accepts_bom_and_bare_carriage_return_even_when_split_into_single_bytes() { + let trace = TraceContext::new("req_cr"); + let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse); + for byte in b"\xef\xbb\xbfdata: {\"type\":\"codex.response.metadata\"}\r\r" { + capture.push(&[*byte]); + } + capture.finish(); + let snapshot = trace.snapshot().unwrap(); + assert_eq!( + snapshot["events"][0]["data"]["eventType"], + "codex.response.metadata" + ); + assert_eq!(snapshot["events"][1]["data"]["partialFrame"], false); +} diff --git a/backend/crates/gateway-core/tests/diagnostics/trace.rs b/backend/crates/gateway-core/tests/diagnostics/trace.rs new file mode 100644 index 000000000..00d57432e --- /dev/null +++ b/backend/crates/gateway-core/tests/diagnostics/trace.rs @@ -0,0 +1,142 @@ +use gateway_core::diagnostics::TraceContext; +use serde_json::json; + +#[test] +fn bounded_history_preserves_start_failure_and_final_result() { + let trace = TraceContext::new("req_trace"); + trace.record("request.started", json!({})); + for index in 0..400 { + trace.record( + "test.event", + json!({"index": index, "detail": "x".repeat(2000)}), + ); + } + trace + .attempt(1) + .record("attempt.failed", json!({"code": "websocket_close_1000"})); + trace + .attempt(2) + .record("request.finished", json!({"outcome": "succeeded"})); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + assert_eq!(events[0]["stage"], "request.started"); + assert_eq!(events[events.len() - 2]["attemptIndex"], 1); + assert_eq!(events.last().unwrap()["attemptIndex"], 2); + assert!(snapshot["droppedEvents"].as_u64().unwrap() > 0); + assert!(events.len() <= 128); + assert!(snapshot.to_string().len() < 64 * 1024); + let kept: u64 = events + .iter() + .map(|event| event["count"].as_u64().unwrap()) + .sum(); + assert_eq!( + kept + snapshot["droppedEvents"].as_u64().unwrap(), + snapshot["totalEvents"] + ); +} + +#[test] +fn shared_attempts_and_exchanges_coalesce_delta_without_losing_metadata() { + let trace = TraceContext::new("req_trace"); + let ws = trace.attempt(1).exchange("websocket"); + ws.capture( + "upstream.event", + br#"{"type":"codex.response.metadata","headers":{"x-request-id":"up-1"}}"#, + ); + trace + .attempt(1) + .wire_event("openai", Some("codex.response.metadata")); + for _ in 0..1000 { + ws.capture( + "upstream.event", + br#"{"type":"response.output_text.delta","delta":"secret"}"#, + ); + trace + .attempt(1) + .wire_event("openai", Some("response.output_text.delta")); + } + let http = trace.attempt(1).exchange("http_sse"); + http.capture("upstream.event", br#"{"type":"response.completed"}"#); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + assert_eq!(snapshot["droppedEvents"], 0); + assert_eq!( + events + .iter() + .find(|e| e["data"]["eventType"] == "response.output_text.delta") + .unwrap()["count"], + 1000 + ); + assert_eq!(events.last().unwrap()["exchangeId"], 2); + assert!( + events + .iter() + .any(|e| e["data"]["metadata"]["headers"]["x-request-id"] == "up-1") + ); + assert!(!snapshot.to_string().contains("secret")); + assert!( + events + .iter() + .filter(|event| event["stage"] == "provider.event") + .all(|event| event["data"].get("metadata").is_none()) + ); +} + +#[test] +fn header_capture_keeps_duplicates_but_excludes_credentials_from_timeline() { + let trace = TraceContext::new("req_headers"); + trace.headers( + "upstream.response.headers", + json!({"status": 101}), + [ + ("authorization", b"Bearer SECRET".as_slice()), + ("x-request-id", b"upstream-one".as_slice()), + ("x-request-id", b"upstream-two".as_slice()), + ("set-cookie", b"PRIVATE_COOKIE".as_slice()), + ], + ); + let snapshot = trace.snapshot().unwrap(); + assert_eq!( + snapshot["events"][0]["data"]["headers"]["x-request-id"], + json!(["upstream-one", "upstream-two"]) + ); + assert!(!snapshot.to_string().contains("SECRET")); + assert!(!snapshot.to_string().contains("PRIVATE_COOKIE")); +} + +#[test] +fn disabled_context_does_not_allocate_a_history() { + let trace = TraceContext::default().attempt(2).exchange("http"); + trace.capture("upstream.event", b"not json"); + assert!(trace.snapshot().is_none()); +} + +#[test] +fn a_long_successful_retry_keeps_the_earlier_failure_ahead_of_routine_events() { + let trace = TraceContext::new("req_retry"); + for _ in 0..12 { + trace.record("setup", json!({})); + } + trace + .attempt(1) + .record("attempt.failed", json!({"kind": "unavailable"})); + trace + .attempt(1) + .record("retry.decided", json!({"retryable": true})); + for index in 0..500 { + trace + .attempt(2) + .record("upstream.event", json!({"index": index})); + } + trace.record("request.finished", json!({"outcome": "succeeded"})); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + assert!( + events + .iter() + .any(|event| event["stage"] == "attempt.failed" && event["attemptIndex"] == 1) + ); + assert!(events.iter().any(|event| event["stage"] == "retry.decided")); + assert_eq!(events.last().unwrap()["stage"], "request.finished"); + assert!(events.len() <= 128); +} diff --git a/backend/crates/gateway-core/tests/engine/coordinator.rs b/backend/crates/gateway-core/tests/engine/coordinator.rs index 79bcb379a..982c68f61 100644 --- a/backend/crates/gateway-core/tests/engine/coordinator.rs +++ b/backend/crates/gateway-core/tests/engine/coordinator.rs @@ -49,6 +49,7 @@ use serde_json::{Map, Value, json}; #[derive(Debug, Clone, PartialEq, Eq)] struct FinalState { + diagnostic_trace_json: Option, outcome: ExecutionOutcome, send_state: UpstreamSendState, attempt_count: u32, @@ -207,6 +208,7 @@ impl ExecutionStore for FakeStore { .expect("store lock") .finalizations .push(FinalState { + diagnostic_trace_json: finalization.diagnostic_trace_json, outcome: finalization.outcome, send_state: finalization.send_state, attempt_count: finalization.attempt_count, @@ -1314,6 +1316,21 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() { let state = store.state.lock().expect("store lock"); let finalization = &state.finalizations[0]; + let trace: Value = + serde_json::from_str(finalization.diagnostic_trace_json.as_deref().unwrap()).unwrap(); + let events = trace["events"].as_array().unwrap(); + assert!( + events + .iter() + .any(|event| event["stage"] == "attempt.failed" && event["attemptIndex"] == 1) + ); + assert!( + events + .iter() + .any(|event| event["stage"] == "attempt.started" && event["attemptIndex"] == 2) + ); + assert_eq!(events.last().unwrap()["stage"], "request.finished"); + assert_eq!(finalization.upstream_transport.as_deref(), Some("http_sse")); assert_eq!(finalization.http_version.as_deref(), Some("HTTP/2")); assert_eq!( diff --git a/backend/crates/gateway-core/tests/main.rs b/backend/crates/gateway-core/tests/main.rs index cbb9914ad..2b1b2a4fa 100644 --- a/backend/crates/gateway-core/tests/main.rs +++ b/backend/crates/gateway-core/tests/main.rs @@ -1,4 +1,5 @@ mod account; +mod diagnostics; mod engine; mod error; mod event; diff --git a/backend/crates/gateway-store/src/postgres/execution.rs b/backend/crates/gateway-store/src/postgres/execution.rs index 8853d671f..3908722ea 100644 --- a/backend/crates/gateway-store/src/postgres/execution.rs +++ b/backend/crates/gateway-store/src/postgres/execution.rs @@ -283,6 +283,7 @@ pub struct ModelRequestFinalization { pub websocket_pool: Option, pub service_tier: Option, pub provider_metadata_json: Option, + pub diagnostic_trace_json: Option, pub error_kind: Option, pub provider_error_code: Option, pub error_message: Option, @@ -353,6 +354,15 @@ impl ModelRequestFinalization { { return Err(invalid("provider observation must be a JSON object")); } + if self + .diagnostic_trace_json + .as_ref() + .is_some_and(|trace| !trace.is_object() || trace.to_string().len() > 64 * 1024) + { + return Err(invalid( + "diagnostic trace must be a JSON object within 64 KiB", + )); + } validate_optional_stable_reason( self.continuation_unavailable_reason.as_deref(), "continuation unavailable reason", @@ -725,7 +735,7 @@ impl ModelRequestRepository for PgExecutionStore { upstream_connection_id = $44, upstream_connection_exit_reason = $45, upstream_connection_age_ms = $46, - upstream_connection_idle_ms = $47 + upstream_connection_idle_ms = $47, diagnostic_trace_json = $48 where id = $1 and outcome = 'running' returning id, client_api_key_ref, continuation_affinity_hash, continuation_requested, provider_kind, upstream_transport, @@ -916,6 +926,7 @@ impl ModelRequestRepository for PgExecutionStore { finalization.upstream_connection_idle_ms, "upstream_connection_idle_ms", )?) + .bind(finalization.diagnostic_trace_json.map(sqlx::types::Json)) .fetch_one(&self.pool) .await .map_err(|_| postgres_unavailable("finalize model request"))?; @@ -1215,6 +1226,12 @@ impl ExecutionStore for PgExecutionStore { websocket_pool: finalization.websocket_pool, service_tier: finalization.service_tier, provider_metadata_json, + diagnostic_trace_json: finalization + .diagnostic_trace_json + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, error_kind, provider_error_code: finalization.provider_error_code, error_message, diff --git a/backend/crates/gateway-store/src/postgres/execution_buffer.rs b/backend/crates/gateway-store/src/postgres/execution_buffer.rs index 243d3a376..9c2874b0f 100644 --- a/backend/crates/gateway-store/src/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/src/postgres/execution_buffer.rs @@ -583,6 +583,7 @@ impl ExecutionObservationWrite { finalization.websocket_pool.as_deref(), finalization.service_tier.as_deref(), finalization.provider_metadata_json.as_deref(), + finalization.diagnostic_trace_json.as_deref(), finalization.provider_error_code.as_deref(), ]) .saturating_add(error_bytes) diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index 4626e4aac..b07b038d6 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -584,6 +584,8 @@ pub(crate) fn admin_usage_detail( detail: UsageRecordDetail, ) -> AdminStoreResult { Ok(admin_observability::UsageDetail { + trace: detail.trace, + related_requests: detail.related_requests, request: admin_usage_record(detail.request)?, attempts: detail .attempts diff --git a/backend/crates/gateway-store/src/postgres/observability/model.rs b/backend/crates/gateway-store/src/postgres/observability/model.rs index a611f3963..95bceacae 100644 --- a/backend/crates/gateway-store/src/postgres/observability/model.rs +++ b/backend/crates/gateway-store/src/postgres/observability/model.rs @@ -645,6 +645,8 @@ pub struct UsageAttemptObservation { #[derive(Debug, Clone, PartialEq, Eq)] pub struct UsageRecordDetail { + pub trace: Option, + pub related_requests: Vec, pub request: UsageRecord, pub attempts: Vec, } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs index 8eb62448b..70bf830e5 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -1,6 +1,7 @@ //! Usage 明细、诊断与过滤查询族。 use super::super::*; +use serde_json::Value; pub(crate) fn push_usage_filter( query: &mut QueryBuilder, @@ -145,7 +146,7 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str = mr.provider_account_authentication_kind_snapshot as provider_account_authentication_kind, mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool, - mr.service_tier, mr.provider_observation_json, + mr.service_tier, mr.provider_observation_json, mr.diagnostic_trace_json, mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at, mr.outcome, mr.client_status_code, mr.upstream_status_code, mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id, @@ -237,7 +238,6 @@ pub(crate) async fn usage_record_detail( let mut statement = QueryBuilder::::new(USAGE_RECORD_DETAIL_SELECT); statement.push(" where mr.id = "); statement.push_bind(request_id.to_owned()); - push_completed_usage_fact_filter(&mut statement, "mr"); let row = statement .build() .fetch_optional(pool) @@ -272,7 +272,24 @@ pub(crate) async fn usage_record_detail( if request.attempt_count > 0 { attempts.push(final_attempt_from_request(&request)); } - Ok(UsageRecordDetail { request, attempts }) + let trace: Option = row + .try_get("diagnostic_trace_json") + .map_err(|_| postgres_unavailable("decode request trace"))?; + let related_requests = sqlx::query_scalar::<_, Value>( + "select jsonb_build_object('requestId', related.id, 'outcome', related.outcome, + 'relation', case when related.id = current.recovery_request_id then 'recovered_by' else 'recovers' end, + 'completedAt', related.completed_at) + from model_requests current join model_requests related + on related.id = current.recovery_request_id or related.recovery_request_id = current.id + where current.id = $1 order by related.started_at limit 20" + ).bind(request_id).fetch_all(pool).await + .map_err(|_| postgres_unavailable("load related recovery requests"))?; + Ok(UsageRecordDetail { + request, + attempts, + trace, + related_requests, + }) } pub(crate) fn intermediate_attempt_from_row( diff --git a/backend/crates/gateway-store/tests/postgres/execution.rs b/backend/crates/gateway-store/tests/postgres/execution.rs index 5a5f2854e..f6c0267ee 100644 --- a/backend/crates/gateway-store/tests/postgres/execution.rs +++ b/backend/crates/gateway-store/tests/postgres/execution.rs @@ -8,7 +8,8 @@ use gateway_core::error::{GatewayError, GatewayErrorKind, ProviderConnectionObse use gateway_core::metering::{CalculatedCost, CostEstimate, Usage}; use gateway_core::upstream::UpstreamSendState; use gateway_store::postgres::{ - ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest, PgExecutionStore, + ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest, ObservabilityRepository, + PgExecutionStore, }; use super::TestDatabase; @@ -463,6 +464,7 @@ async fn expired_image_request_should_be_recovered_as_failed() { fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization { CoreModelRequestFinalization { + diagnostic_trace_json: None, request_id: ModelRequestId::new(id).expect("request id"), outcome: ExecutionOutcome::Succeeded, send_state: UpstreamSendState::Sent, @@ -804,6 +806,30 @@ async fn successful_http_downgrade_should_mark_pending_websocket_failures_recove assert_eq!(row.4, Some(expected_delay_ms)); assert!(row.5.is_some_and(|latency| latency >= expected_delay_ms)); } + let repository = super::observability_repository(&database.pool); + let failed_detail = repository + .usage_record_detail("req_websocket_failed_first") + .await + .unwrap(); + assert_eq!( + failed_detail.related_requests[0]["requestId"], + "req_http_fallback_succeeded" + ); + assert_eq!( + failed_detail.related_requests[0]["relation"], + "recovered_by" + ); + let recovered_detail = repository + .usage_record_detail("req_http_fallback_succeeded") + .await + .unwrap(); + assert_eq!(recovered_detail.related_requests.len(), 2); + assert!( + recovered_detail + .related_requests + .iter() + .all(|related| related["relation"] == "recovers") + ); database.close().await; } @@ -1072,3 +1098,40 @@ async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx: .await?; Ok(()) } + +#[tokio::test] +async fn diagnostic_trace_is_finalized_atomically_and_available_for_failed_request_details() { + let Some(database) = TestDatabase::create("diagnostic_trace").await else { + return; + }; + seed_running_request(&database.pool, "req_diagnostic_failed") + .await + .unwrap(); + let store = PgExecutionStore::new(database.pool.clone()); + let mut finalization = successful_core_finalization("req_diagnostic_failed"); + finalization.outcome = ExecutionOutcome::Failed; + finalization.client_status_code = Some(502); + finalization.error = Some(GatewayError::new( + GatewayErrorKind::UpstreamUnavailable, + "upstream closed", + )); + let trace = serde_json::json!({"schemaVersion": 1, "requestId": "req_diagnostic_failed", "events": [ + {"attemptIndex": 1, "stage": "upstream.close", "data": {"code": 1000}}, + ]}); + finalization.diagnostic_trace_json = Some(trace.to_string()); + ExecutionStore::finalize_model_request(&store, finalization) + .await + .unwrap(); + let persisted: (String, serde_json::Value) = sqlx::query_as( + "select outcome, diagnostic_trace_json from model_requests where id = 'req_diagnostic_failed'" + ).fetch_one(&database.pool).await.unwrap(); + assert_eq!(persisted, ("failed".to_owned(), trace.clone())); + let repository = super::observability_repository(&database.pool); + let detail = repository + .usage_record_detail("req_diagnostic_failed") + .await + .unwrap(); + assert_eq!(detail.trace, Some(trace)); + assert!(detail.related_requests.is_empty()); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/observability/mod.rs b/backend/crates/gateway-store/tests/postgres/observability/mod.rs index 1f26d95b7..2478e9654 100644 --- a/backend/crates/gateway-store/tests/postgres/observability/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/observability/mod.rs @@ -853,13 +853,13 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det store .usage_record_detail("req_observe_failed") .await - .is_err() + .is_ok() ); assert!( store .usage_record_detail("req_observe_uncommitted") .await - .is_err() + .is_ok() ); let overview = store @@ -1243,7 +1243,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact repository .usage_record_detail("req_observe_failed") .await - .is_err() + .is_ok() ); let overview = repository diff --git a/backend/crates/providers/openai/Cargo.toml b/backend/crates/providers/openai/Cargo.toml index 120c0ead1..a3d1a511e 100644 --- a/backend/crates/providers/openai/Cargo.toml +++ b/backend/crates/providers/openai/Cargo.toml @@ -21,27 +21,28 @@ gateway-core = { path = "../../gateway-core" } gateway-protocol = { path = "../../gateway-protocol" } getrandom = "0.4.3" hex = "0.4.3" -# 与官方 Codex Desktop app-server 的 TLS 栈逐版本对齐（2026-08 核实： -# ChatGPT.app Resources/codex 二进制内为 hyper-rustls 0.27.7 + rustls 0.23.36 + -# reqwest 0.12.28 + ring）。rustls 0.23.37 起 ClientHello 扩展顺序变化，JA3 -# 会偏离官方指纹；升级前必须重新抓握手对比扩展顺序。 +# TLS 依赖按官方 Core 锁文件固定，包含用于约束传递依赖的直接声明。 +# 升级前须重复采样 ClientHello，核对算法、扩展集合及排序约束、key_share 与 ALPN。 +aws-lc-rs = "=1.16.2" +hyper = { version = "=1.8.1", default-features = false } hyper-rustls = "=0.27.7" # 与官方 Codex Desktop 一致：reqwest 不启用压缩特性，上游请求不带 accept-encoding。 reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "http2"] } roxmltree = "0.21.1" semver = "1.0.27" -# 保持 ring provider：reqwest rustls-tls-native-roots 即 __rustls-ring，与官方一致； -# 不要给 TLS 客户端路径启用 aws_lc_rs，否则密钥共享/扩展集可能变化。 -rustls = { version = "=0.23.36", default-features = false, features = ["ring", "std", "tls12"] } -rustls-native-certs = "0.8.4" -rustls-pki-types = "1.14.1" +# 与官方 rustls 路径一致：aws-lc 支持 P-521，优先提供 X25519MLKEM768。 +# reqwest 的 roots feature 仍会编译 ring；实际 provider 由 TLS 初始化显式选择。 +rustls = { version = "=0.23.36", default-features = false, features = ["aws_lc_rs", "prefer-post-quantum", "std", "tls12"] } +rustls-native-certs = "=0.8.3" +rustls-pki-types = "=1.14.0" secrecy = "0.10.3" serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] } sha2 = "0.11.0" subtle = "2.6.1" thiserror = "2.0.18" -tokio = { version = "1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] } +tokio = { version = "=1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] } +tokio-rustls = { version = "=0.26.4", default-features = false } tokio-util = { version = "0.7.18", features = ["rt"] } # 官方 Codex WebSocket 需要上游尚未发布的 proxy 与 permessage-deflate 实现，固定到已审计提交。 tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "0e5b2d73aa18dd9f0a50ee9ff199d5aef7594186", features = ["proxy", "rustls-tls-native-roots"] } @@ -51,7 +52,7 @@ url = "2.5.8" uuid = { version = "1.23.3", features = ["v4", "v7"] } # 官方 Codex app-server 对 /v1/responses 请求体默认做 zstd 压缩（level 3）， # 并携带 Content-Encoding: zstd；保持同版本同级别以对齐上游指纹。 -zstd = "0.13" +zstd = "=0.13.3" [dev-dependencies] tempfile = "3.27.0" diff --git a/backend/crates/providers/openai/src/config.rs b/backend/crates/providers/openai/src/config.rs index 2aea60481..395f882ba 100644 --- a/backend/crates/providers/openai/src/config.rs +++ b/backend/crates/providers/openai/src/config.rs @@ -8,7 +8,7 @@ use serde::Deserialize; use url::{Host, Url}; use crate::credential::CodexQuotaRefreshPolicy; -use crate::transport::profile::{CodexWireProfile, CodexWireProfileState}; +use crate::transport::profile::{CodexResidency, CodexWireProfile, CodexWireProfileState}; use crate::transport::session::{CodexSessionIdentity, CodexSessionIdentityError}; use crate::transport::websocket::CodexWebSocketPoolConfig; use crate::{ @@ -287,6 +287,8 @@ pub struct CodexWireProfileConfig { pub os_version: String, pub arch: String, pub terminal: String, + #[serde(default)] + pub residency: Option, pub verified_at: DateTime, } @@ -294,14 +296,16 @@ impl Default for CodexWireProfileConfig { fn default() -> Self { Self { originator: "Codex Desktop".to_owned(), - codex_version: "0.147.0-alpha.6.6".to_owned(), - desktop_version: "26.803.81509".to_owned(), - desktop_build: "6415".to_owned(), + codex_version: "0.153.4".to_owned(), + desktop_version: "26.901.51231".to_owned(), + desktop_build: "8109".to_owned(), os_type: "Mac OS".to_owned(), os_version: "15.7.1".to_owned(), arch: "arm64".to_owned(), terminal: "unknown".to_owned(), - verified_at: Utc::now(), + residency: None, + // 制品核验于 2026-09-06T03:26:12.084Z；进程启动不构成重新核验。 + verified_at: DateTime::UNIX_EPOCH + chrono::Duration::milliseconds(1_788_665_172_084), } } } @@ -361,6 +365,7 @@ impl From for CodexWireProfile { os_version: value.os_version, arch: value.arch, terminal: value.terminal, + residency: value.residency, verified_at: value.verified_at, } } diff --git a/backend/crates/providers/openai/src/credential/profile_statistics.rs b/backend/crates/providers/openai/src/credential/profile_statistics.rs index 35b9a4d99..a5175e702 100644 --- a/backend/crates/providers/openai/src/credential/profile_statistics.rs +++ b/backend/crates/providers/openai/src/credential/profile_statistics.rs @@ -8,7 +8,7 @@ use std::{ use gateway_core::account::ProviderAccountId; use reqwest::Client; -use secrecy::ExposeSecret as _; +use secrecy::{ExposeSecret as _, SecretString}; use thiserror::Error; use uuid::Uuid; @@ -133,6 +133,30 @@ impl CodexCredentialProfileService { &self, account_id: &ProviderAccountId, ) -> Result { + let (authorization, upstream_account_id) = self.account_authentication(account_id).await?; + let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple()); + let statistics = CodexBackendClient::new( + self.http.clone(), + self.base_url.clone(), + self.profile.clone(), + ) + .fetch_profile_statistics(CodexRequestContext::auxiliary( + authorization.expose_secret(), + upstream_account_id.as_deref(), + &request_id, + None, + )) + .await + .map_err(map_client_error)?; + self.remember_avatar_source(account_id, statistics.image_url.as_deref()); + Ok(statistics) + } + + /// URL 缓存不缓存认证；每次下载都重新读取所属账号当前的 token。 + async fn account_authentication( + &self, + account_id: &ProviderAccountId, + ) -> Result<(SecretString, Option), CodexProfileStatisticsError> { let account = self .repository .store() @@ -160,22 +184,10 @@ impl CodexCredentialProfileService { .authentication .authorization_header() .map_err(|_| CodexProfileStatisticsError::InvalidCredentialData)?; - let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple()); - let statistics = CodexBackendClient::new( - self.http.clone(), - self.base_url.clone(), - self.profile.clone(), - ) - .fetch_profile_statistics(CodexRequestContext::auxiliary( - authorization.expose_secret(), - account.upstream_account_id(), - &request_id, - None, + Ok(( + authorization, + account.upstream_account_id().map(str::to_owned), )) - .await - .map_err(map_client_error)?; - self.remember_avatar_source(account_id, statistics.image_url.as_deref()); - Ok(statistics) } pub async fn profile_avatar( @@ -190,10 +202,21 @@ impl CodexCredentialProfileService { .image_url .ok_or(CodexProfileAvatarError::Missing)?, }; - let desktop_user_agent = self.profile.snapshot().desktop_user_agent(); - fetch_profile_avatar(&self.http, &self.base_url, &desktop_user_agent, &source) - .await - .map_err(map_avatar_fetch_error) + let (authorization, upstream_account_id) = self.account_authentication(account_id).await?; + fetch_profile_avatar( + &self.http, + &self.base_url, + &self.profile.snapshot(), + &source, + CodexRequestContext::auxiliary( + authorization.expose_secret(), + upstream_account_id.as_deref(), + "profile_avatar", + None, + ), + ) + .await + .map_err(map_avatar_fetch_error) } fn cached_avatar_source(&self, account_id: &ProviderAccountId) -> Option { diff --git a/backend/crates/providers/openai/src/credential/token_client.rs b/backend/crates/providers/openai/src/credential/token_client.rs index d6c0408bf..64010bb41 100644 --- a/backend/crates/providers/openai/src/credential/token_client.rs +++ b/backend/crates/providers/openai/src/credential/token_client.rs @@ -1,5 +1,10 @@ //! OpenAI token 续期 Reqwest 适配器。 +use crate::transport::{ + headers::build_codex_profile_headers, + profile::CodexWireProfileState, + tls::{build_reqwest_client_with_custom_ca, ensure_rustls_provider}, +}; use async_trait::async_trait; use reqwest::{Client, StatusCode, redirect::Policy}; use secrecy::{ExposeSecret, SecretString}; @@ -243,6 +248,7 @@ pub struct TokenClientConfig { pub struct OpenAiTokenClient { client: Client, config: TokenClientConfig, + profile: CodexWireProfileState, } /// 官方 Codex token client 无法安全构建。 @@ -251,9 +257,13 @@ pub struct OpenAiTokenClient { pub struct TokenClientBuildError; impl OpenAiTokenClient { - /// 使用 Reqwest 客户端和静态配置构造 token 续期客户端。 - pub fn new(client: Client, config: TokenClientConfig) -> Self { - Self { client, config } + /// 共享运行时画像；刷新时取快照，授权码交换仍使用 raw auth 请求。 + pub fn new(client: Client, config: TokenClientConfig, profile: CodexWireProfileState) -> Self { + Self { + client, + config, + profile, + } } } @@ -264,14 +274,17 @@ impl OpenAiTokenClient { /// 本地 TLS/HTTP client 初始化失败时返回脱敏错误。 pub fn openai_token_client( config: TokenClientConfig, + profile: CodexWireProfileState, ) -> Result { - let client = Client::builder() + ensure_rustls_provider(); + let builder = Client::builder() + .use_rustls_tls() + .no_proxy() .redirect(Policy::none()) .connect_timeout(TOKEN_CONNECT_TIMEOUT) - .timeout(TOKEN_REQUEST_TIMEOUT) - .build() - .map_err(|_| TokenClientBuildError)?; - Ok(OpenAiTokenClient::new(client, config)) + .timeout(TOKEN_REQUEST_TIMEOUT); + let client = build_reqwest_client_with_custom_ca(builder).map_err(|_| TokenClientBuildError)?; + Ok(OpenAiTokenClient::new(client, config, profile)) } #[derive(Deserialize)] @@ -345,9 +358,16 @@ struct AuthorizationCodeResponse { #[async_trait] impl TokenRefresher for OpenAiTokenClient { async fn refresh(&self, refresh_token: &str) -> Result { + let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|_| { + RefreshFailure::Transport { + message: Some("OpenAI OAuth refresh profile is invalid".to_owned()), + upstream: None, + } + })?; let response = self .client .post(&self.config.token_endpoint) + .headers(headers) .json(&RefreshTokenRequest { client_id: self.config.client_id.as_str(), grant_type: "refresh_token", diff --git a/backend/crates/providers/openai/src/lib.rs b/backend/crates/providers/openai/src/lib.rs index c74a94613..cf61575e7 100644 --- a/backend/crates/providers/openai/src/lib.rs +++ b/backend/crates/providers/openai/src/lib.rs @@ -146,8 +146,11 @@ pub async fn initialize( .with_session_identity(session_identity), ); let token_client = Arc::new( - credential::token_client::openai_token_client(config.token_client_config()) - .map_err(|_| OpenAiInitializeError::TokenClient)?, + credential::token_client::openai_token_client( + config.token_client_config(), + profile.clone(), + ) + .map_err(|_| OpenAiInitializeError::TokenClient)?, ); let refresher: Arc = token_client.clone(); let exchanger: Arc = token_client; diff --git a/backend/crates/providers/openai/src/provider/execution.rs b/backend/crates/providers/openai/src/provider/execution.rs index c22c30ad4..58a94c5e8 100644 --- a/backend/crates/providers/openai/src/provider/execution.rs +++ b/backend/crates/providers/openai/src/provider/execution.rs @@ -358,12 +358,14 @@ pub(super) async fn create_json_attempt( return Err(CodexHandshakeAttemptError::Timeout); }; let request_id = request.context.request_id().as_str(); + let trace = request.context.trace(); let mut request_context = CodexRequestContext::auxiliary( authorization.expose_secret(), account.upstream_account_id(), request_id, Some(installation_id), ); + request_context.trace = Some(&trace); request_context.cookie_header = cookie_header.map(ExposeSecret::expose_secret); request_context.turn_metadata = request.turn_metadata.as_deref(); request_context.account_selection = account_selection; @@ -539,6 +541,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream { lease.account_switch(), ); let request_transport_requirement = transport_requirement(&request); + let trace = context.trace(); let response = create_response_attempt( &client, &request, @@ -550,7 +553,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream { &authorization, cookie_header.as_ref(), account_selection, - ), + ).with_trace(&trace), active_account.id().as_str(), context.deadline(), &cancellation, diff --git a/backend/crates/providers/openai/src/provider/observation.rs b/backend/crates/providers/openai/src/provider/observation.rs index 32dbf9ed1..0ffc286c1 100644 --- a/backend/crates/providers/openai/src/provider/observation.rs +++ b/backend/crates/providers/openai/src/provider/observation.rs @@ -687,6 +687,7 @@ pub(super) fn codex_request_context<'a>( account_selection: CodexAccountSelectionTelemetry<'a>, ) -> CodexRequestContext<'a> { CodexRequestContext { + trace: None, authorization: authorization.expose_secret(), account_id: account.upstream_account_id(), request_id, diff --git a/backend/crates/providers/openai/src/transport/client.rs b/backend/crates/providers/openai/src/transport/client.rs index 3bc7bbe5f..ceffb71bb 100644 --- a/backend/crates/providers/openai/src/transport/client.rs +++ b/backend/crates/providers/openai/src/transport/client.rs @@ -395,6 +395,8 @@ impl<'a> CodexAccountSelectionTelemetry<'a> { /// 单次 Codex 上游请求的上下文。 #[derive(Clone, Copy)] pub struct CodexRequestContext<'a> { + /// 显式传递的请求诊断上下文，跨连接任务共享。 + pub trace: Option<&'a gateway_core::diagnostics::TraceContext>, /// Provider 已构造并脱敏持有的完整 Authorization 值。 pub authorization: &'a str, /// ChatGPT 账号 ID。 @@ -440,6 +442,7 @@ impl<'a> CodexRequestContext<'a> { installation_id: Option<&'a str>, ) -> Self { Self { + trace: None, authorization, account_id, request_id, @@ -459,6 +462,12 @@ impl<'a> CodexRequestContext<'a> { account_selection: CodexAccountSelectionTelemetry::NONE, } } + + #[must_use] + pub const fn with_trace(mut self, trace: &'a gateway_core::diagnostics::TraceContext) -> Self { + self.trace = Some(trace); + self + } } impl fmt::Debug for CodexRequestContext<'_> { diff --git a/backend/crates/providers/openai/src/transport/client_json.rs b/backend/crates/providers/openai/src/transport/client_json.rs index de7edc794..89be0b020 100644 --- a/backend/crates/providers/openai/src/transport/client_json.rs +++ b/backend/crates/providers/openai/src/transport/client_json.rs @@ -3,7 +3,7 @@ use std::time::Instant; use bytes::Bytes; -use reqwest::header::{ACCEPT, CONTENT_TYPE, HeaderName, HeaderValue}; +use reqwest::header::{CONTENT_TYPE, HeaderValue}; use super::{ client::{ @@ -14,7 +14,7 @@ use super::{ }, diagnostics::CodexUpstreamSendPhase, endpoints::endpoint_url, - headers::{build_codex_base_headers, insert_optional_header, insert_optional_protocol_header}, + headers::insert_optional_protocol_header, response_meta, }; @@ -28,27 +28,30 @@ impl CodexBackendClient { context: CodexRequestContext<'_>, ) -> CodexClientResult { let profile = self.profile.snapshot(); - let mut headers = - build_codex_base_headers(&profile, context.authorization, context.account_id)?; + let mut headers = self.model_request_headers(&profile, context)?; headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - headers.insert(ACCEPT, HeaderValue::from_static("application/json")); - insert_optional_header(&mut headers, "cookie", context.cookie_header)?; - insert_optional_header( - &mut headers, - "x-codex-installation-id", - context.installation_id, - )?; insert_optional_protocol_header(&mut headers, "x-codex-image-turn-id", image_turn_id); insert_optional_protocol_header( &mut headers, "x-codex-turn-metadata", context.turn_metadata, ); - headers.insert( - HeaderName::from_static("x-client-request-id"), - HeaderValue::from_str(context.request_id)?, - ); + let trace = context + .trace + .cloned() + .unwrap_or_default() + .exchange("http_json"); + trace.headers( + "upstream.request.headers", + serde_json::json!({ + "method": "POST", "endpoint": endpoint_path, + }), + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())), + ); + trace.capture("upstream.request.body", &body); let headers_started_at = Instant::now(); let response = self .client @@ -61,6 +64,17 @@ impl CodexBackendClient { let upstream_headers_ms = elapsed_duration_millis(headers_started_at.elapsed()); let http_version = http_version_name(response.version()).to_owned(); let status = response.status(); + trace.headers( + "upstream.response.headers", + serde_json::json!({ + "status": status.as_u16(), "httpVersion": http_version, + "headersMs": upstream_headers_ms, + }), + response + .headers() + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())), + ); let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers()); let set_cookie_headers = response_meta::set_cookie_headers(response.headers()); let rate_limit_headers = response_meta::rate_limit_headers(response.headers()); @@ -81,6 +95,7 @@ impl CodexBackendClient { let raw_body = read_error_response_body(response) .await .map_err(CodexClientError::HttpJson)?; + trace.capture("upstream.error.body", &raw_body); let error_body = String::from_utf8_lossy(&raw_body).into_owned(); return Err(CodexClientError::Upstream { status, @@ -102,6 +117,7 @@ impl CodexBackendClient { } let body = response.bytes().await.map_err(CodexClientError::HttpJson)?; + trace.capture("upstream.response.body", &body); Ok(CodexBackendJsonResponse { body, set_cookie_headers, diff --git a/backend/crates/providers/openai/src/transport/client_sse.rs b/backend/crates/providers/openai/src/transport/client_sse.rs index 0f4cfb9da..72f321ff0 100644 --- a/backend/crates/providers/openai/src/transport/client_sse.rs +++ b/backend/crates/providers/openai/src/transport/client_sse.rs @@ -1,17 +1,16 @@ +use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext, diagnostic_json}; + use std::{sync::Arc, time::Instant}; use futures::{StreamExt, TryStreamExt}; use gateway_protocol::openai::{ - X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER, + X_OPENAI_MEMGEN_REQUEST_HEADER, events::{self, retry_after_seconds_from_body}, sse::{SseEventDecoder, SseFrame}, }; use reqwest::{ Client, Response as ReqwestResponse, - header::{ - ACCEPT, AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE, COOKIE, HeaderMap, HeaderName, - HeaderValue, USER_AGENT, - }, + header::{CONTENT_ENCODING, CONTENT_TYPE, HeaderMap, HeaderValue}, }; use tokio_tungstenite::tungstenite::handshake::client::generate_key; @@ -22,11 +21,8 @@ use crate::transport::{ }, diagnostics::CodexUpstreamSendPhase, endpoints::{CODEX_RESPONSES_PATH, endpoint_url}, - headers::{ - build_codex_base_headers, insert_optional_header, insert_optional_protocol_header, - websocket_header_pairs, - }, - profile::{CodexWireProfile, CodexWireProfileState}, + headers::websocket_header_pairs, + profile::CodexWireProfileState, protocol::{ responses::{CodexResponsesRequest, TransportRequirement, transport_requirement}, websocket::{ @@ -37,9 +33,8 @@ use crate::transport::{ response_meta, websocket::{ CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool, - CodexWebSocketPoolKey, CodexWebSocketRequest, CodexWebSocketStreamingExchange, - DEFAULT_INITIAL_EVENT_TIMEOUT, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath, - WebSocketOriginBreaker, WebSocketPoolDecision, + CodexWebSocketPoolKey, CodexWebSocketStreamingExchange, DEFAULT_INITIAL_EVENT_TIMEOUT, + WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath, WebSocketOriginBreaker, execute_prepared_response_create_request_stream, post_send_ambiguous, prepare_response_create_request_with_pool, websocket_audit_dir, write_websocket_audit_artifact_from_env, @@ -106,20 +101,21 @@ impl CodexBackendClient { let body = serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?; let endpoint = endpoint_url(&self.base_url, CODEX_RESPONSES_PATH); - tracing::info!( - target: "request_dump", - provider = "openai", - request_id = %context.request_id, - account_id = context.account_id.unwrap_or_default(), - request_direction = "upstream", - request_transport = "http_sse", - request_method = "POST", - request_url = %endpoint, - request_headers = ?headers, - request_body = %String::from_utf8_lossy(&body), - contains_sensitive_data = true, - "request dump (unredacted)" + let trace = context + .trace + .cloned() + .unwrap_or_default() + .exchange("http_sse"); + trace.headers( + "upstream.request.headers", + serde_json::json!({ + "method": "POST", "endpoint": CODEX_RESPONSES_PATH, + }), + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())), ); + trace.capture("upstream.request.body", &body); let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3) .map_err(CodexClientError::RequestCompression)?; let response = self @@ -133,6 +129,17 @@ impl CodexBackendClient { let upstream_headers_ms = elapsed_duration_millis(headers_started_at.elapsed()); let http_version = http_version_name(response.version()).to_string(); let status = response.status(); + trace.headers( + "upstream.response.headers", + serde_json::json!({ + "status": status.as_u16(), "httpVersion": http_version, + "headersMs": upstream_headers_ms, + }), + response + .headers() + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())), + ); let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers()); let turn_state = response_meta::turn_state(response.headers()); let set_cookie_headers = response_meta::set_cookie_headers(response.headers()); @@ -147,6 +154,7 @@ impl CodexBackendClient { .map(|value| value.as_bytes().to_vec()); let client_headers = response_meta::client_headers(response.headers()); let raw_body = read_error_response_body(response).await?; + trace.capture("upstream.error.body", &raw_body); let body = String::from_utf8_lossy(&raw_body).into_owned(); let retry_after_seconds = retry_after_seconds.or_else(|| retry_after_seconds_from_body(&body)); @@ -175,7 +183,7 @@ impl CodexBackendClient { let rate_limit_updates = Arc::new(tokio::sync::Mutex::new(Vec::new())); Ok(CodexBackendStreamingResponse { - body: http_sse_stream(response, Arc::clone(&rate_limit_updates)), + body: http_sse_stream(response, Arc::clone(&rate_limit_updates), trace), transport: CodexBackendTransport::HttpSse, websocket_connection_id: None, turn_state, @@ -258,6 +266,9 @@ impl CodexBackendClient { defer_websocket_recovery: bool, ) -> CodexClientResult { let requirement = transport_requirement(request); + context.trace.cloned().unwrap_or_default().record("transport.preparing", serde_json::json!({ + "requirement": requirement.as_str(), "connectionPreference": format!("{connection_preference:?}"), + })); if requirement == TransportRequirement::HttpRequired { return Ok(PreparedResponseTransport { requirement, @@ -279,6 +290,15 @@ impl CodexBackendClient { &websocket_request, ) .map_err(CodexClientError::WebSocketEncode)?; + context.trace.cloned().unwrap_or_default().headers( + "upstream.request.headers", + serde_json::json!({"transport": "websocket", "phase": "prepared_opening"}), + websocket_create + .connection() + .headers() + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())), + ); // 审计未启用时跳过 artifact 构造：payload 快照会深拷贝整个请求 body， // 且位于首字节前的关键路径上。 if websocket_audit_dir().is_some() { @@ -326,14 +346,13 @@ impl CodexBackendClient { Ok(WebSocketFastPath::Missed) => { let decision = CodexTransportDecision::Http2WebSocketBudgetExhausted; let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed()); - tracing::info!( - request_id = %context.request_id, - account_id = pool_account_id.or(context.account_id).unwrap_or_default(), - transport_requirement = requirement.as_str(), - transport_decision = decision.as_str(), - transport_decision_wait_ms = wait_ms, - websocket_preconnect_continues = pool_log_context.is_some(), - "WebSocket fast path missed; using same-account HTTP" + context.trace.cloned().unwrap_or_default().record( + "transport.fallback", + serde_json::json!({ + "to": "http_sse", "reason": "websocket_fast_path_budget", + "requirement": requirement.as_str(), "decision": decision.as_str(), + "waitMs": wait_ms, "preconnectContinues": pool_log_context.is_some(), + }), ); return Ok(PreparedResponseTransport { requirement, @@ -354,14 +373,13 @@ impl CodexBackendClient { { let decision = http_fallback_decision(&error); let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed()); - tracing::warn!( - request_id = %context.request_id, - account_id = pool_account_id.or(context.account_id).unwrap_or_default(), - transport_requirement = requirement.as_str(), - transport_decision = decision.as_str(), - transport_decision_wait_ms = wait_ms, - error = %error, - "WebSocket preparation failed before payload send; using same-account HTTP" + context.trace.cloned().unwrap_or_default().record( + "transport.fallback", serde_json::json!({ + "to": "http_sse", "reason": "websocket_pre_send_failure", + "requirement": requirement.as_str(), "decision": decision.as_str(), + "waitMs": wait_ms, + "detail": diagnostic_json(&serde_json::json!({"message": error.to_string()})), + }), ); return Ok(PreparedResponseTransport { requirement, @@ -378,6 +396,15 @@ impl CodexBackendClient { Err(error) => return Err(websocket_exchange_error_to_client_error(error)), }; let decision = websocket_success_decision(requirement, &prepared); + context.trace.cloned().unwrap_or_default().record( + "transport.prepared", + serde_json::json!({ + "decision": decision.as_str(), + "requirement": requirement.as_str(), + "connectMs": prepared.connect_elapsed().map(elapsed_duration_millis), + "waitMs": elapsed_duration_millis(prepared.decision_wait_elapsed()), + }), + ); let metrics = CodexTransportMetrics { decision: Some(decision), ws_connect_ms: prepared.connect_elapsed().map(elapsed_duration_millis), @@ -394,15 +421,6 @@ impl CodexBackendClient { pool_log_context.as_ref(), prepared.pool_decision(), ); - tracing::info!( - request_id = %context.request_id, - account_id = pool_account_id.or(context.account_id).unwrap_or_default(), - transport_requirement = requirement.as_str(), - transport_decision = decision.as_str(), - ws_connect_ms = ?metrics.ws_connect_ms, - transport_decision_wait_ms = ?metrics.transport_decision_wait_ms, - "Responses transport prepared" - ); Ok(PreparedResponseTransport { requirement, route: PreparedResponseRoute::WebSocket(Box::new(PreparedWebSocketRoute { @@ -427,6 +445,13 @@ impl CodexBackendClient { metrics, defer_websocket_recovery, } = prepared; + context.trace.cloned().unwrap_or_default().record( + "transport.selected", + serde_json::json!({ + "decision": metrics.decision.map(|decision| decision.as_str()), + "requirement": requirement.as_str(), "waitMs": metrics.transport_decision_wait_ms, + }), + ); match route { PreparedResponseRoute::Http => self .create_response_stream_http_sse(request, context) @@ -441,10 +466,14 @@ impl CodexBackendClient { prepared, } = *route; let delivery_wait_started_at = Instant::now(); - log_unredacted_websocket_request(&websocket_request, context); let mut exchange = match execute_prepared_response_create_request_stream( &websocket_request, prepared, + context + .trace + .cloned() + .unwrap_or_default() + .exchange("websocket"), ) .await { @@ -493,9 +522,12 @@ impl CodexBackendClient { .await; } fresh_retry_used = true; - tracing::warn!( - request_id = %context.request_id, - "WebSocket connection limit reached; retrying on a fresh connection" + context.trace.cloned().unwrap_or_default().record( + "transport.retry", + serde_json::json!({ + "reason": "websocket_connection_limit_reached", + "connectionPreference": "fresh", + }), ); drop(exchange); match self @@ -512,10 +544,14 @@ impl CodexBackendClient { route: PreparedResponseRoute::WebSocket(route), .. }) => { - log_unredacted_websocket_request(&route.request, context); match execute_prepared_response_create_request_stream( &route.request, route.prepared, + context + .trace + .cloned() + .unwrap_or_default() + .exchange("websocket"), ) .await { @@ -575,8 +611,9 @@ impl CodexBackendClient { } } } - Err(error) if !defer_websocket_recovery - && error.allows_pre_delivery_http_fallback() => + Err(error) + if !defer_websocket_recovery + && error.allows_pre_delivery_http_fallback() => { return self .fallback_to_http_before_websocket_delivery( @@ -597,12 +634,6 @@ impl CodexBackendClient { } } } - tracing::info!( - request_id = %context.request_id, - websocket_connection_id = %exchange.websocket_connection_id, - ws_pool = exchange.pool_decision.map_or("unpooled", WebSocketPoolDecision::kind), - "WebSocket response stream established" - ); Ok(CodexBackendStreamingResponse { body: Box::pin( exchange @@ -625,15 +656,6 @@ impl CodexBackendClient { }) } } - .inspect_err(|error| { - tracing::warn!( - request_id = %context.request_id, - transport_requirement = requirement.as_str(), - failure_phase = "post_send_or_explicit_response", - error = %error, - "Responses stream transport failed after preparation" - ); - }) } async fn fallback_to_http_before_websocket_delivery( @@ -671,14 +693,15 @@ impl CodexBackendClient { .saturating_add(elapsed_duration_millis(delivery_wait_started_at.elapsed())); metrics.decision = Some(CodexTransportDecision::Http2PreDeliveryFailure); metrics.transport_decision_wait_ms = Some(decision_wait_ms); - tracing::warn!( - request_id = %context.request_id, - account_id = context.account_id.unwrap_or_default(), - transport_requirement = requirement.as_str(), - transport_decision = CodexTransportDecision::Http2PreDeliveryFailure.as_str(), - transport_decision_wait_ms = decision_wait_ms, - error = %detail, - "WebSocket failed before first deliverable event; using same-account HTTP" + context.trace.cloned().unwrap_or_default().record( + "transport.fallback", + serde_json::json!({ + "to": "http_sse", "reason": "websocket_pre_delivery_failure", + "requirement": requirement.as_str(), + "decision": CodexTransportDecision::Http2PreDeliveryFailure.as_str(), + "waitMs": decision_wait_ms, + "detail": diagnostic_json(&serde_json::json!({"message": detail})), + }), ); self.create_response_stream_http_sse(request, context) .await @@ -715,7 +738,7 @@ impl CodexBackendClient { ) -> CodexClientResult { let endpoint = endpoint_url(&self.base_url, "codex/models"); let profile = self.profile.snapshot(); - let headers = self.auxiliary_request_headers(&profile, context)?; + let headers = self.model_request_headers(&profile, context)?; let response = self .client .get(endpoint) @@ -751,182 +774,6 @@ impl CodexBackendClient { } Ok(parse_codex_model_catalog(&body, etag.as_deref())?) } - - fn request_headers_for_http_response( - &self, - request: &CodexResponsesRequest, - context: CodexRequestContext<'_>, - ) -> CodexClientResult { - let profile = self.profile.snapshot(); - let mut headers = self.request_headers(&profile, context)?; - if let Some(subagent) = openai_subagent_from_metadata(request.client_metadata()) { - insert_optional_protocol_header(&mut headers, "x-openai-subagent", Some(&subagent)); - } - insert_optional_protocol_header( - &mut headers, - X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, - request.responses_lite.as_deref(), - ); - insert_optional_protocol_header( - &mut headers, - X_OPENAI_MEMGEN_REQUEST_HEADER, - request.memgen_request.as_deref(), - ); - // 客户端确实携带的普通协议头优先；画像身份头只能由 Codex profile 生成。 - for name in request.passthrough_headers.keys() { - // 上游指纹必须由运行时画像统一生成：originator/User-Agent/version 即使 - // 绕过 API 透传黑名单也不能覆盖画像，避免下游客户端暴露不一致指纹。 - if matches!( - name.as_str(), - "openai-beta" - | "originator" - | "user-agent" - | "version" - | "x-oai-attestation" - | "x-oai-is" - | "x-oai-is-update" - | "x-openai-internal-codex-residency" - ) { - continue; - } - headers.remove(name); - for value in request.passthrough_headers.get_all(name) { - headers.append(name.clone(), value.clone()); - } - } - Ok(headers) - } - - fn request_headers_for_websocket_response( - &self, - request: &CodexResponsesRequest, - context: CodexRequestContext<'_>, - ) -> CodexClientResult { - let mut headers = self.request_headers_for_http_response(request, context)?; - headers.remove(X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER); - headers.insert( - HeaderName::from_static("openai-beta"), - HeaderValue::from_static("responses_websockets=2026-02-06"), - ); - Ok(headers) - } - - fn request_headers( - &self, - profile: &CodexWireProfile, - context: CodexRequestContext<'_>, - ) -> CodexClientResult { - let mut headers = - build_codex_base_headers(profile, context.authorization, context.account_id)?; - headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - insert_optional_header(&mut headers, "cookie", context.cookie_header)?; - headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream")); - headers.insert( - HeaderName::from_static("x-client-request-id"), - HeaderValue::from_str(context.request_id)?, - ); - insert_optional_protocol_header( - &mut headers, - "x-client-request-id", - context.client_request_id, - ); - insert_optional_header( - &mut headers, - "x-codex-installation-id", - context.installation_id, - )?; - insert_optional_protocol_header(&mut headers, "session-id", context.session_id); - insert_optional_protocol_header(&mut headers, "thread-id", context.thread_id); - insert_optional_protocol_header(&mut headers, "x-codex-turn-id", context.turn_id); - insert_optional_protocol_header(&mut headers, "x-codex-window-id", context.codex_window_id); - insert_optional_protocol_header(&mut headers, "x-codex-turn-state", context.turn_state); - insert_optional_protocol_header( - &mut headers, - "x-codex-turn-metadata", - context.turn_metadata, - ); - insert_optional_protocol_header( - &mut headers, - "x-codex-beta-features", - context.beta_features, - ); - insert_optional_protocol_header( - &mut headers, - "x-responsesapi-include-timing-metrics", - context.include_timing_metrics, - ); - // 官方 Desktop 的版本画像来自 clientInfo.version。保留下游是否携带 - // `version` 扩展头的协议形状，但上游值统一使用当前 Desktop 制品版本。 - if context - .version - .is_some_and(|version| HeaderValue::from_str(version).is_ok()) - { - headers.insert( - HeaderName::from_static("version"), - HeaderValue::from_str(&profile.desktop_version)?, - ); - } - insert_optional_protocol_header( - &mut headers, - "x-codex-parent-thread-id", - context.parent_thread_id, - ); - - Ok(headers) - } - - fn auxiliary_request_headers( - &self, - profile: &CodexWireProfile, - context: CodexRequestContext<'_>, - ) -> CodexClientResult { - let mut headers = - build_codex_base_headers(profile, context.authorization, context.account_id)?; - if let Some(cookie_header) = context.cookie_header { - headers.insert(COOKIE, HeaderValue::from_str(cookie_header)?); - } - headers.insert(ACCEPT, HeaderValue::from_static("application/json")); - insert_optional_header( - &mut headers, - "x-codex-installation-id", - context.installation_id, - )?; - Ok(headers) - } - - pub(in crate::transport) fn usage_request_headers( - &self, - context: CodexRequestContext<'_>, - ) -> CodexClientResult { - let profile = self.profile.snapshot(); - let mut headers = HeaderMap::new(); - headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?); - headers.insert(AUTHORIZATION, HeaderValue::from_str(context.authorization)?); - insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?; - insert_optional_header(&mut headers, "cookie", context.cookie_header)?; - headers.insert(ACCEPT, HeaderValue::from_static("application/json")); - Ok(headers) - } -} - -fn log_unredacted_websocket_request( - request: &CodexWebSocketRequest, - context: CodexRequestContext<'_>, -) { - tracing::info!( - target: "request_dump", - provider = "openai", - request_id = %context.request_id, - account_id = context.account_id.unwrap_or_default(), - request_direction = "upstream", - request_transport = "websocket", - request_method = "GET + response.create", - request_url = request.connection().endpoint(), - request_headers = ?request.connection().headers(), - request_payload = request.payload_text(), - contains_sensitive_data = true, - "request dump (unredacted)" - ); } /// 首个可投递帧前的交付边界结果。 @@ -1011,6 +858,7 @@ fn websocket_connection_profile(headers: &HeaderMap) -> String { fn http_sse_stream( response: ReqwestResponse, rate_limit_updates: CodexRateLimitUpdates, + trace: TraceContext, ) -> CodexBackendSseStream { let stream: CodexBackendSseStream = Box::pin(response.bytes_stream().map_err(CodexClientError::Http)); @@ -1028,6 +876,20 @@ fn http_sse_stream( )), } })); + let stream = Box::pin(async_stream::stream! { + let mut stream = stream; + let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse); + while let Some(chunk) = stream.next().await { + match &chunk { + Ok(bytes) => capture.push(bytes), + Err(error) => trace.record("upstream.read.failed", diagnostic_json(&serde_json::json!({"error": error.to_string()}))), + } + let failed = chunk.is_err(); + yield chunk; + if failed { return; } + } + capture.finish(); + }); observe_http_sse_rate_limits(stream, rate_limit_updates) } diff --git a/backend/crates/providers/openai/src/transport/endpoints.rs b/backend/crates/providers/openai/src/transport/endpoints.rs index f43771ead..b721ab001 100644 --- a/backend/crates/providers/openai/src/transport/endpoints.rs +++ b/backend/crates/providers/openai/src/transport/endpoints.rs @@ -29,12 +29,17 @@ pub fn endpoint_url(base_url: &str, endpoint_path: &str) -> String { /// 返回与 base path 对应的唯一 usage endpoint。 pub fn usage_endpoint_url(base_url: &str) -> String { - let path = if has_backend_api_base_path(base_url) { - WHAM_USAGE_PATH + account_endpoint_url(base_url, "usage") +} + +/// Core backend-client 的两个官方路径风格，供所有账号接口共用。 +pub fn account_endpoint_url(base_url: &str, resource: &str) -> String { + let prefix = if has_backend_api_base_path(base_url) { + "wham" } else { - CODEX_USAGE_API_PATH + "api/codex" }; - endpoint_url(base_url, path) + endpoint_url(base_url, &format!("{prefix}/{resource}")) } fn has_backend_api_base_path(base_url: &str) -> bool { diff --git a/backend/crates/providers/openai/src/transport/headers.rs b/backend/crates/providers/openai/src/transport/headers.rs index 20d98e29d..cdf78df30 100644 --- a/backend/crates/providers/openai/src/transport/headers.rs +++ b/backend/crates/providers/openai/src/transport/headers.rs @@ -1,23 +1,37 @@ -use crate::transport::profile::CodexWireProfile; -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue, USER_AGENT}; +//! 各官方调用链的请求头契约；版本与平台信息统一来自运行时画像快照。 -use super::client::CodexClientResult; +use gateway_protocol::openai::{ + X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER, +}; +use reqwest::header::{ + ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT, +}; + +use super::client::{ + CodexBackendClient, CodexClientResult, CodexRequestContext, openai_subagent_from_metadata, +}; +use super::profile::{CodexResidency, CodexWireProfile}; +use super::protocol::responses::CodexResponsesRequest; const CODEX_RESIDENCY_HEADER: &str = "x-openai-internal-codex-residency"; /// 构造 Codex Core 为模型请求设置的稳定身份请求头。 -pub fn build_codex_base_headers( +pub fn build_codex_model_headers( profile: &CodexWireProfile, authorization: &str, account_id: Option<&str>, ) -> CodexClientResult { let mut headers = build_codex_profile_headers(profile)?; + headers.insert( + HeaderName::from_static("version"), + HeaderValue::from_str(&profile.codex_version)?, + ); headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?); insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?; Ok(headers) } -/// 构造不依赖访问凭据的 Codex Desktop 身份请求头。 +/// Core 默认客户端身份，用于模型接口及 OAuth refresh；raw token exchange 不使用。 pub fn build_codex_profile_headers(profile: &CodexWireProfile) -> CodexClientResult { let mut headers = HeaderMap::new(); headers.insert( @@ -25,14 +39,182 @@ pub fn build_codex_profile_headers(profile: &CodexWireProfile) -> CodexClientRes HeaderValue::from_str(&profile.originator)?, ); headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?); - // 此部署固定采用官方支持的 US residency requirement。 + if let Some(CodexResidency::Us) = profile.residency { + headers.insert( + HeaderName::from_static(CODEX_RESIDENCY_HEADER), + HeaderValue::from_static("us"), + ); + } + Ok(headers) +} + +/// Core backend-client 的账号请求：不携带模型版本或 originator。 +pub fn build_codex_account_headers( + profile: &CodexWireProfile, + authorization: &str, + account_id: Option<&str>, +) -> CodexClientResult { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?); + headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?); + insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?; + Ok(headers) +} + +/// 桌面下载的账号身份；完整性凭据须由认证主体持有，不能借用下游客户端的值。 +pub fn build_codex_download_headers( + profile: &CodexWireProfile, + authorization: &str, + account_id: Option<&str>, +) -> CodexClientResult { + let mut headers = HeaderMap::new(); headers.insert( - HeaderName::from_static(CODEX_RESIDENCY_HEADER), - HeaderValue::from_static("us"), + USER_AGENT, + HeaderValue::from_str(&profile.desktop_user_agent())?, + ); + headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?); + insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?; + headers.insert( + HeaderName::from_static("originator"), + HeaderValue::from_str(&profile.originator)?, ); Ok(headers) } +impl CodexBackendClient { + pub(super) fn model_request_headers( + &self, + profile: &CodexWireProfile, + context: CodexRequestContext<'_>, + ) -> CodexClientResult { + let mut headers = + build_codex_model_headers(profile, context.authorization, context.account_id)?; + insert_optional_header(&mut headers, "cookie", context.cookie_header)?; + Ok(headers) + } + + pub(super) fn account_request_headers( + &self, + context: CodexRequestContext<'_>, + ) -> CodexClientResult { + let mut headers = build_codex_account_headers( + &self.profile.snapshot(), + context.authorization, + context.account_id, + )?; + insert_optional_header(&mut headers, "cookie", context.cookie_header)?; + Ok(headers) + } + + pub(super) fn request_headers_for_http_response( + &self, + request: &CodexResponsesRequest, + context: CodexRequestContext<'_>, + ) -> CodexClientResult { + let mut headers = self.response_headers(request, context)?; + headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream")); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + insert_optional_protocol_header( + &mut headers, + X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, + request.responses_lite.as_deref(), + ); + Ok(headers) + } + + pub(super) fn request_headers_for_websocket_response( + &self, + request: &CodexResponsesRequest, + context: CodexRequestContext<'_>, + ) -> CodexClientResult { + let mut headers = self.response_headers(request, context)?; + headers.insert( + HeaderName::from_static("openai-beta"), + HeaderValue::from_static("responses_websockets=2026-02-06"), + ); + Ok(headers) + } + + /// HTTP 和 WS 共用的 Core 会话字段；传输专属字段由各自入口生成。 + fn response_headers( + &self, + request: &CodexResponsesRequest, + context: CodexRequestContext<'_>, + ) -> CodexClientResult { + let mut headers = self.model_request_headers(&self.profile.snapshot(), context)?; + headers.insert( + HeaderName::from_static("x-client-request-id"), + HeaderValue::from_str(context.request_id)?, + ); + insert_optional_protocol_header( + &mut headers, + "x-client-request-id", + context + .client_request_id + .or(context.thread_id) + .or(context.session_id), + ); + for (name, value) in [ + ("session-id", context.session_id), + ("thread-id", context.thread_id), + ("x-codex-window-id", context.codex_window_id), + ("x-codex-turn-state", context.turn_state), + ("x-codex-turn-metadata", context.turn_metadata), + ("x-codex-beta-features", context.beta_features), + ( + "x-responsesapi-include-timing-metrics", + context.include_timing_metrics, + ), + ("x-codex-parent-thread-id", context.parent_thread_id), + ( + X_OPENAI_MEMGEN_REQUEST_HEADER, + request.memgen_request.as_deref(), + ), + ] { + insert_optional_protocol_header(&mut headers, name, value); + } + if let Some(subagent) = openai_subagent_from_metadata(request.client_metadata()) { + insert_optional_protocol_header(&mut headers, "x-openai-subagent", Some(&subagent)); + } + let routing_hint = match request.service_tier() { + Some(tier) => format!("model={};tier={tier}", request.model()), + None => format!("model={}", request.model()), + }; + insert_optional_protocol_header(&mut headers, "x-codex-routing-hint", Some(&routing_hint)); + for name in request.passthrough_headers.keys() { + // 身份与传输字段只由画像/正文生成；其余协议头保留原始多值字节。 + if matches!( + name.as_str(), + "originator" + | "user-agent" + | "version" + | "authorization" + | "chatgpt-account-id" + | "cookie" + | "x-openai-internal-codex-residency" + | "openai-beta" + | "accept" + | "content-type" + | "content-encoding" + | "x-codex-routing-hint" + | "x-codex-installation-id" + | "x-codex-turn-id" + | "x-oai-attestation" + | "x-oai-is" + | "x-oai-is-update" + | X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER + ) { + continue; + } + headers.remove(name); + for value in request.passthrough_headers.get_all(name) { + headers.append(name.clone(), value.clone()); + } + } + Ok(headers) + } +} + pub(super) fn insert_optional_header( headers: &mut HeaderMap, name: &'static str, diff --git a/backend/crates/providers/openai/src/transport/mod.rs b/backend/crates/providers/openai/src/transport/mod.rs index 8bd26f19d..1b9cd0c69 100644 --- a/backend/crates/providers/openai/src/transport/mod.rs +++ b/backend/crates/providers/openai/src/transport/mod.rs @@ -42,7 +42,7 @@ pub use self::{ WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH, WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url, }, - headers::build_codex_base_headers, + headers::build_codex_model_headers, profile_avatar::{ CodexProfileAvatar, CodexProfileAvatarFetchError, CodexProfileAvatarStreamError, fetch_profile_avatar, diff --git a/backend/crates/providers/openai/src/transport/profile/mod.rs b/backend/crates/providers/openai/src/transport/profile/mod.rs index 860155c59..f223debb2 100644 --- a/backend/crates/providers/openai/src/transport/profile/mod.rs +++ b/backend/crates/providers/openai/src/transport/profile/mod.rs @@ -32,6 +32,13 @@ const MAX_APPCAST_BYTES: usize = 1024 * 1024; const ARTIFACT_PROFILE_CACHE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60); const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1; +/// 与官方 managed residency requirement 相同的可选请求约束。 +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CodexResidency { + Us, +} + /// Codex Desktop 上游请求身份。 /// /// 启动配置提供经源码审计的 Core、运行环境和 Desktop 启动版本。运行时只会使用 @@ -40,7 +47,7 @@ const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1; pub struct CodexWireProfile { /// `originator` 请求头及 User-Agent 产品名。 pub originator: String, - /// Desktop ZIP 内嵌 Core 版本；用于 `/codex/models?client_version=` 与 UA。 + /// Desktop ZIP 内嵌 Core 版本；用于模型请求的 version、client_version 与 UA。 pub codex_version: String, /// Desktop 应用版本，用于 app-server `clientInfo.version` 对应的 UA 后缀。 pub desktop_version: String, @@ -54,7 +61,9 @@ pub struct CodexWireProfile { pub arch: String, /// Codex Core UA 中的终端标记。 pub terminal: String, - /// 此画像最后一次经制品与源码核验的时间。 + /// 未配置时不发送 residency 头；不随制品版本更新而改变。 + pub residency: Option, + /// 版本元组最后一次经制品核验的时间；不表示 TLS 传输已重新核验。 pub verified_at: DateTime, } diff --git a/backend/crates/providers/openai/src/transport/profile_avatar.rs b/backend/crates/providers/openai/src/transport/profile_avatar.rs index 368fa1f7b..22682985a 100644 --- a/backend/crates/providers/openai/src/transport/profile_avatar.rs +++ b/backend/crates/providers/openai/src/transport/profile_avatar.rs @@ -7,17 +7,18 @@ use bytes::Bytes; use futures::{Stream, StreamExt as _}; use reqwest::{ Client, Url, - header::{ACCEPT, CONTENT_TYPE, ETAG, USER_AGENT}, + header::{CONTENT_TYPE, ETAG}, }; use tokio::time::timeout; use super::endpoints::endpoint_url; +use super::{ + CodexRequestContext, headers::build_codex_download_headers, profile::CodexWireProfile, +}; const OFFICIAL_AVATAR_ORIGIN: &str = "https://chatgpt.com"; const OFFICIAL_AVATAR_PATH_PREFIX: &str = "/backend-api/estuary/public_content/enc/"; const PROVIDER_AVATAR_PATH_PREFIX: &str = "/estuary/public_content/enc/"; -const PROFILE_AVATAR_ACCEPT: &str = - "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"; const PROFILE_AVATAR_HEADERS_TIMEOUT: Duration = Duration::from_secs(15); const PROFILE_AVATAR_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(30); @@ -68,19 +69,18 @@ pub enum CodexProfileAvatarFetchError { pub async fn fetch_profile_avatar( client: &Client, base_url: &str, - desktop_user_agent: &str, + profile: &CodexWireProfile, source: &str, + context: CodexRequestContext<'_>, ) -> Result { let source_path = official_avatar_source_path(source)?; let target = Url::parse(&endpoint_url(base_url, &source_path)) .map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?; + let headers = build_codex_download_headers(profile, context.authorization, context.account_id) + .map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?; let response = timeout( PROFILE_AVATAR_HEADERS_TIMEOUT, - client - .get(target) - .header(ACCEPT, PROFILE_AVATAR_ACCEPT) - .header(USER_AGENT, desktop_user_agent) - .send(), + client.get(target).headers(headers).send(), ) .await .map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)? diff --git a/backend/crates/providers/openai/src/transport/profile_statistics.rs b/backend/crates/providers/openai/src/transport/profile_statistics.rs index 5321187b7..1b5dd21dc 100644 --- a/backend/crates/providers/openai/src/transport/profile_statistics.rs +++ b/backend/crates/providers/openai/src/transport/profile_statistics.rs @@ -12,7 +12,7 @@ use super::{ retry_after_seconds, }, diagnostics::CodexUpstreamSendPhase, - endpoints::{WHAM_PROFILE_STATISTICS_PATH, endpoint_url}, + endpoints::account_endpoint_url, response_meta, }; @@ -167,8 +167,8 @@ impl CodexBackendClient { ) -> CodexClientResult { let response = self .client - .get(endpoint_url(&self.base_url, WHAM_PROFILE_STATISTICS_PATH)) - .headers(self.usage_request_headers(context)?) + .get(account_endpoint_url(&self.base_url, "profiles/me")) + .headers(self.account_request_headers(context)?) .send() .await?; let status = response.status(); diff --git a/backend/crates/providers/openai/src/transport/request.rs b/backend/crates/providers/openai/src/transport/request.rs index 873b86bdd..55c57b66f 100644 --- a/backend/crates/providers/openai/src/transport/request.rs +++ b/backend/crates/providers/openai/src/transport/request.rs @@ -320,8 +320,8 @@ fn normalize_conversation_anchor_text(text: &str) -> String { /// 把客户端正文收敛到当前 lease 的账号身份边界。 /// -/// 真实 account ID 与 installation ID 由随后构造的 `CodexRequestContext` 注入 -/// 请求头；正文只替换客户端原本声明过的 installation 字段，绝不接受客户端 +/// 真实 account ID 由随后构造的 `CodexRequestContext` 注入请求头；installation ID +/// 统一写入 Core client_metadata，并替换原有兼容字段，绝不接受客户端 /// 提供的 token、cookie 或账号身份。`input` 是 Responses 的可回放会话正文， /// item ID、encrypted content 和 compaction 都必须原样保留。 pub(crate) fn scope_request_to_account( @@ -371,59 +371,57 @@ pub(crate) fn scope_request_to_account( replace_existing_body_string(request, key, scoped.as_deref()); } - if let Some(client_metadata) = request.client_metadata().cloned() { - let scoped = match client_metadata { - Value::Object(mut metadata) => { - let scoped_turn_metadata = - ["turnMetadata", "turn_metadata", "x-codex-turn-metadata"].map(|key| { - ( - key, - metadata.get(key).and_then(Value::as_str).and_then(|value| { - scope_turn_metadata(value, installation_id, reset_account_state) - }), - ) - }); - if reset_account_state { - for key in CROSS_ACCOUNT_IDENTITY_KEYS - .iter() - .chain(ACCOUNT_BOUND_STATE_KEYS) - { - metadata.remove(*key); - } + let client_metadata = request + .client_metadata() + .cloned() + .unwrap_or_else(|| Value::Object(Map::new())); + let scoped = match client_metadata { + Value::Object(mut metadata) => { + let scoped_turn_metadata = ["turnMetadata", "turn_metadata", "x-codex-turn-metadata"] + .map(|key| { + ( + key, + metadata.get(key).and_then(Value::as_str).and_then(|value| { + scope_turn_metadata(value, installation_id, reset_account_state) + }), + ) + }); + if reset_account_state { + for key in CROSS_ACCOUNT_IDENTITY_KEYS + .iter() + .chain(ACCOUNT_BOUND_STATE_KEYS) + { + metadata.remove(*key); } - metadata.insert( - "x-codex-installation-id".to_owned(), - Value::String(installation_id.to_owned()), - ); - replace_existing_metadata_field( - &mut metadata, - "installation_id", - Some(installation_id), - ); - replace_existing_metadata_field( - &mut metadata, - "installationId", - Some(installation_id), - ); - replace_metadata_field( - &mut metadata, - "x-codex-turn-state", - client_metadata_turn_state.as_deref(), - ); - replace_metadata_field( - &mut metadata, - "x-codex-turn-metadata", - client_metadata_turn_metadata.as_deref(), - ); - for (key, value) in scoped_turn_metadata { - replace_existing_metadata_field(&mut metadata, key, value.as_deref()); - } - (!metadata.is_empty()).then_some(Value::Object(metadata)) } - value => Some(value), - }; - request.set_client_metadata(scoped); - } + metadata.insert( + "installation_id".to_owned(), + Value::String(installation_id.to_owned()), + ); + replace_existing_metadata_field( + &mut metadata, + "x-codex-installation-id", + Some(installation_id), + ); + replace_existing_metadata_field(&mut metadata, "installationId", Some(installation_id)); + replace_metadata_field( + &mut metadata, + "x-codex-turn-state", + client_metadata_turn_state.as_deref(), + ); + replace_metadata_field( + &mut metadata, + "x-codex-turn-metadata", + client_metadata_turn_metadata.as_deref(), + ); + for (key, value) in scoped_turn_metadata { + replace_existing_metadata_field(&mut metadata, key, value.as_deref()); + } + (!metadata.is_empty()).then_some(Value::Object(metadata)) + } + value => Some(value), + }; + request.set_client_metadata(scoped); request.turn_state = turn_state; request.turn_metadata = turn_metadata; diff --git a/backend/crates/providers/openai/src/transport/reset_credits.rs b/backend/crates/providers/openai/src/transport/reset_credits.rs index c6b499703..2592dccef 100644 --- a/backend/crates/providers/openai/src/transport/reset_credits.rs +++ b/backend/crates/providers/openai/src/transport/reset_credits.rs @@ -1,10 +1,7 @@ //! Codex Desktop 主动额度重置卡 HTTP contract。 use chrono::{DateTime, Utc}; -use reqwest::{ - StatusCode, - header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT}, -}; +use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -15,11 +12,7 @@ use super::{ retry_after_seconds, }, diagnostics::CodexUpstreamSendPhase, - endpoints::{ - WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH, - endpoint_url, - }, - headers::insert_optional_header, + endpoints::account_endpoint_url, response_meta, }; @@ -94,11 +87,11 @@ impl CodexBackendClient { ) -> CodexClientResult { let response = self .client - .get(endpoint_url( + .get(account_endpoint_url( &self.base_url, - WHAM_RATE_LIMIT_RESET_CREDITS_PATH, + "rate-limit-reset-credits", )) - .headers(self.reset_credits_headers(context, false)?) + .headers(self.account_request_headers(context)?) .send() .await .map_err(CodexClientError::HttpJson)?; @@ -131,11 +124,11 @@ impl CodexBackendClient { ) -> CodexClientResult { let response = self .client - .post(endpoint_url( + .post(account_endpoint_url( &self.base_url, - WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, + "rate-limit-reset-credits/consume", )) - .headers(self.reset_credits_headers(context, true)?) + .headers(self.account_request_headers(context)?) .json(&ConsumeRequest { credit_id, redeem_request_id: redeem_request_id.to_string(), @@ -151,36 +144,6 @@ impl CodexBackendClient { credit: wire.credit.map(parse_credit).transpose()?, }) } - - fn reset_credits_headers( - &self, - context: CodexRequestContext<'_>, - json_body: bool, - ) -> CodexClientResult { - let profile = self.profile.snapshot(); - let mut headers = HeaderMap::new(); - headers.insert(AUTHORIZATION, HeaderValue::from_str(context.authorization)?); - insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?; - headers.insert( - HeaderName::from_static("originator"), - HeaderValue::from_str(&profile.originator)?, - ); - headers.insert( - HeaderName::from_static("oai-language"), - HeaderValue::from_static("en"), - ); - headers.insert( - USER_AGENT, - HeaderValue::from_str(&profile.desktop_user_agent())?, - ); - // Electron `net.fetch` 为 renderer 请求附加的默认 Accept；该 endpoint 的 - // renderer 并没有显式改写为 Core auxiliary 请求的 application/json 画像。 - headers.insert(ACCEPT, HeaderValue::from_static("*/*")); - if json_body { - headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - } - Ok(headers) - } } async fn read_reset_credits_response(response: reqwest::Response) -> CodexClientResult { diff --git a/backend/crates/providers/openai/src/transport/tls.rs b/backend/crates/providers/openai/src/transport/tls.rs index 43de4de61..4eadd5305 100644 --- a/backend/crates/providers/openai/src/transport/tls.rs +++ b/backend/crates/providers/openai/src/transport/tls.rs @@ -15,13 +15,13 @@ const CA_CERT_HINT: &str = "If you set CODEX_CA_CERTIFICATE or SSL_CERT_FILE, en type PemSection = (SectionKind, Vec); -/// rustls 0.23 在同时编译 `ring` 与 `aws-lc-rs` 时无法自动选择进程级 -/// CryptoProvider；本工程固定使用 ring（与官方 Codex Desktop JA3 一致）， -/// 因此在首次 TLS 使用前显式安装一次。重复安装会被 rustls 忽略。 +/// 与官方 Codex 的 rustls 路径一致，安装支持 P-521 和混合后量子密钥交换的 +/// aws-lc provider。依赖图也包含 ring，因此不能让 rustls 自动选择 provider。 +/// 与官方嵌入场景一致，保留宿主在初始化之前已经安装的 provider。 pub fn ensure_rustls_provider() { static INSTALL: OnceLock<()> = OnceLock::new(); INSTALL.get_or_init(|| { - let _ = rustls::crypto::ring::default_provider().install_default(); + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); }); } diff --git a/backend/crates/providers/openai/src/transport/usage.rs b/backend/crates/providers/openai/src/transport/usage.rs index 4c6e6a7db..ff50c40ae 100644 --- a/backend/crates/providers/openai/src/transport/usage.rs +++ b/backend/crates/providers/openai/src/transport/usage.rs @@ -676,7 +676,7 @@ pub const MAX_CODEX_USAGE_BODY_BYTES: usize = 1024 * 1024; impl CodexBackendClient { /// 获取 Codex usage JSON。 pub async fn fetch_usage(&self, context: CodexRequestContext<'_>) -> CodexClientResult { - let headers = self.usage_request_headers(context)?; + let headers = self.account_request_headers(context)?; let response = self .client .get(usage_endpoint_url(&self.base_url)) diff --git a/backend/crates/providers/openai/src/transport/websocket/coordinator.rs b/backend/crates/providers/openai/src/transport/websocket/coordinator.rs index 2fe933508..8dd6ab7a0 100644 --- a/backend/crates/providers/openai/src/transport/websocket/coordinator.rs +++ b/backend/crates/providers/openai/src/transport/websocket/coordinator.rs @@ -559,6 +559,7 @@ fn continuation_unavailable( pub(crate) async fn execute_prepared_response_create_request_stream( request: &CodexWebSocketRequest, prepared: PreparedWebSocket, + trace: gateway_core::diagnostics::TraceContext, ) -> Result { let PreparedWebSocket { connection, @@ -573,7 +574,19 @@ pub(crate) async fn execute_prepared_response_create_request_stream( continuation, created_at, } = connection; + trace.record("upstream.connection", serde_json::json!({ + "connectionId": websocket.connection_id().to_string(), "reused": reused, + "pool": pool_decision.map_or("unpooled", WebSocketPoolDecision::kind), + "status": metadata.diagnostics.status_code, + "upstreamRequestId": metadata.diagnostics.request_id, + "headers": gateway_core::diagnostics::diagnostic_headers(metadata.diagnostics.trace_headers.iter().map(|(name, value)| (name.as_str(), value.as_str()))), + })); + trace.capture("upstream.request.body", request.payload_text().as_bytes()); if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await { + trace.record( + "upstream.send.failed", + serde_json::json!({"phase": "websocket_payload", "sendState": "ambiguous"}), + ); let observation = websocket .observation() .with_exit_reason("outbound_transport_error"); @@ -582,6 +595,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream( error.with_connection_observation(observation), )); } + trace.record("upstream.payload.sent", serde_json::json!({})); let connection_local_available = lease.is_some(); let pool_return = lease.map(|lease| WebSocketStreamPoolReturn { lease, @@ -594,6 +608,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream( pool_return, reused, initial_event_timeout, + trace, ); exchange.pool_decision = pool_decision; exchange.connection_local_continuation = connection_local_available; diff --git a/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs b/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs index 7191eb2bd..7098e3c7d 100644 --- a/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs +++ b/backend/crates/providers/openai/src/transport/websocket/exchange/stream.rs @@ -3,6 +3,8 @@ use std::{sync::Arc, time::Duration}; use bytes::Bytes; +use gateway_core::diagnostics::{TraceContext, diagnostic_json}; +use serde_json::json; use tokio::sync::{Mutex, mpsc}; use tokio_util::sync::CancellationToken; @@ -61,6 +63,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response( pool_return: Option, reused_connection: bool, initial_event_timeout: Option, + trace: TraceContext, ) -> CodexWebSocketStreamingExchange { let websocket_connection_id = websocket.connection_id(); let response_metadata = metadata.clone(); @@ -83,6 +86,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response( pool_return, reused_connection, initial_event_timeout, + trace, shutdown, rate_limit_updates: rate_limit_updates_for_task, turn_state_update: turn_state_update_for_task, @@ -116,6 +120,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response( } struct WebSocketStreamForwardState { + trace: TraceContext, websocket: PumpedWebSocket, metadata: CodexWebSocketConnectionMetadata, pool_return: Option, @@ -129,6 +134,7 @@ struct WebSocketStreamForwardState { async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { let WebSocketStreamForwardState { + trace, mut websocket, mut metadata, pool_return, @@ -152,6 +158,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { // 客户端断开下游 SSE 流：立即丢弃连接并释放池 slot， // 不再傻等上游 idle 超时（否则同会话后续请求会一直 bypass/busy）。 () = tx.closed() => { + trace.record("upstream.cancelled", json!({"reason": "receiver_dropped"})); discard_stream_websocket( websocket, pool_return, @@ -160,6 +167,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { return; } () = shutdown.cancelled() => { + trace.record("upstream.cancelled", json!({"reason": "pool_shutdown"})); discard_stream_websocket( websocket, pool_return, @@ -183,6 +191,13 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { } error => error, }; + trace.record( + "upstream.read.failed", + json!({ + "lastEventType": last_event_type, + "error": diagnostic_json(&json!({"message": error.to_string()})), + }), + ); let observation = connection_observation(&websocket, &error); discard_stream_websocket( websocket, @@ -201,6 +216,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { } }; let Some(message) = message else { + trace.record( + "upstream.eof", + json!({"lastEventType": last_event_type, "terminalSeen": false}), + ); break; }; let raw = match message { @@ -208,7 +227,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { saw_upstream_activity = true; text.to_string() } - tungstenite::Message::Binary(_) => { + tungstenite::Message::Binary(bytes) => { + trace.capture("upstream.binary", &bytes); let error = CodexWebSocketExchangeError::UnexpectedBinaryEvent; let observation = connection_observation(&websocket, &error); discard_stream_websocket( @@ -223,6 +243,15 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { return; } tungstenite::Message::Close(frame) => { + if let Some(frame) = &frame { + trace.dump("upstream.close.reason", frame.reason.as_bytes()); + } + trace.record("upstream.close", json!({ + "code": frame.as_ref().map(|frame| u16::from(frame.code)), + "reason": frame.as_ref().map(|frame| diagnostic_json(&json!({"message": frame.reason.as_str()}))), + "lastEventType": last_event_type, "terminalSeen": false, + "connectionId": websocket.connection_id().to_string(), + })); let websocket_connection_id = websocket.connection_id(); let error = CodexWebSocketExchangeError::closed_before_terminal_on( websocket_connection_id, @@ -248,6 +277,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { } _ => continue, }; + trace.capture("upstream.event", raw.as_bytes()); let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) { Ok(reduced) => reduced, Err(error) => { @@ -286,6 +316,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { ExchangeAction::Ignore => continue, }; if tx.send(Ok(Bytes::from(frame))).await.is_err() { + trace.record( + "upstream.forward.failed", + json!({"reason": "receiver_dropped"}), + ); discard_stream_websocket( websocket, pool_return, @@ -295,6 +329,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { return; } if let Some(terminal) = terminal { + trace.record( + "upstream.terminal", + json!({"kind": format!("{terminal:?}")}), + ); match terminal { WebSocketTerminalKind::Completed => { finish_stream_websocket(websocket, metadata, continuation, pool_return.take()) diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index bb98ee187..30676f356 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -261,7 +261,7 @@ async fn openai_admin_provider_exposes_live_wire_profile_and_validated_billing() async fn reset_credit_success_with_invalid_body_should_remain_an_unknown_consume_result() { let server = MockServer::start().await; Mock::given(method("POST")) - .and(path("/wham/rate-limit-reset-credits/consume")) + .and(path("/api/codex/rate-limit-reset-credits/consume")) .respond_with(ResponseTemplate::new(200).set_body_raw("{}", "application/json")) .expect(1) .mount(&server) @@ -287,7 +287,7 @@ async fn reset_credit_success_with_invalid_body_should_remain_an_unknown_consume async fn reset_credit_explicit_http_rejection_should_preserve_the_raw_body() { let server = MockServer::start().await; Mock::given(method("POST")) - .and(path("/wham/rate-limit-reset-credits/consume")) + .and(path("/api/codex/rate-limit-reset-credits/consume")) .respond_with(ResponseTemplate::new(409).set_body_raw( r#"{"code":"nothing_to_reset","detail":"window is fresh"}"#, "application/json", @@ -1152,6 +1152,7 @@ fn valid_config() -> TestOpenAiConfig { os_version: "15.5.0".to_owned(), arch: "arm64".to_owned(), terminal: "xterm-256color".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 7, 19, 0, 0, 0) .single() diff --git a/backend/crates/providers/openai/tests/config.rs b/backend/crates/providers/openai/tests/config.rs index 7b7a85fa0..c07ce58f2 100644 --- a/backend/crates/providers/openai/tests/config.rs +++ b/backend/crates/providers/openai/tests/config.rs @@ -127,7 +127,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() { ); assert_eq!( config.wire_profile_state().snapshot().user_agent(), - "Codex Desktop/0.147.0-alpha.6.6 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.803.81509)" + "Codex Desktop/0.153.4 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.901.51231)" ); } @@ -153,6 +153,7 @@ fn valid_config() -> OpenAiConfig { os_version: "15.5.0".to_owned(), arch: "arm64".to_owned(), terminal: "xterm-256color".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 7, 19, 0, 0, 0) .single() diff --git a/backend/crates/providers/openai/tests/credential/catalog.rs b/backend/crates/providers/openai/tests/credential/catalog.rs index f9f346376..f87c80c77 100644 --- a/backend/crates/providers/openai/tests/credential/catalog.rs +++ b/backend/crates/providers/openai/tests/credential/catalog.rs @@ -60,6 +60,7 @@ fn wire_profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "catalog-contract".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 7, 18, 0, 0, 0) .single() diff --git a/backend/crates/providers/openai/tests/credential/contract.rs b/backend/crates/providers/openai/tests/credential/contract.rs index bf89fefe7..f401ed986 100644 --- a/backend/crates/providers/openai/tests/credential/contract.rs +++ b/backend/crates/providers/openai/tests/credential/contract.rs @@ -160,6 +160,7 @@ fn selector_with_runtime( os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "selector-contract".to_owned(), + residency: None, verified_at: chrono::Utc::now(), }); let http = reqwest::Client::builder().build().expect("HTTP client"); diff --git a/backend/crates/providers/openai/tests/credential/oauth.rs b/backend/crates/providers/openai/tests/credential/oauth.rs index 89ba463fd..40dc9d5b8 100644 --- a/backend/crates/providers/openai/tests/credential/oauth.rs +++ b/backend/crates/providers/openai/tests/credential/oauth.rs @@ -310,7 +310,7 @@ async fn authorize_url_matches_the_official_desktop_parameter_contract() { ); assert_eq!( parameters.get("codex_app_version").map(String::as_str), - Some("26.803.81509") + Some("26.901.51231") ); let surface_stable_id = parameters .get("source_surface_stable_id") diff --git a/backend/crates/providers/openai/tests/credential/profile_statistics.rs b/backend/crates/providers/openai/tests/credential/profile_statistics.rs index ee5ca43d9..91a70d697 100644 --- a/backend/crates/providers/openai/tests/credential/profile_statistics.rs +++ b/backend/crates/providers/openai/tests/credential/profile_statistics.rs @@ -21,6 +21,7 @@ fn wire_profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "profile-statistics-contract".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 8, 28, 0, 0, 0) .single() @@ -59,7 +60,7 @@ async fn service( async fn profile_statistics_uses_one_profile_request_and_preserves_official_fields() { let server = MockServer::start().await; Mock::given(method("GET")) - .and(path("/wham/profiles/me")) + .and(path("/api/codex/profiles/me")) .and(header( "chatgpt-account-id", "chatgpt-acct_profile_statistics", @@ -135,7 +136,7 @@ async fn profile_statistics_uses_one_profile_request_and_preserves_official_fiel async fn profile_statistics_keeps_profile_when_stats_are_unavailable() { let server = MockServer::start().await; Mock::given(method("GET")) - .and(path("/wham/profiles/me")) + .and(path("/api/codex/profiles/me")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "profile": {"display_name": "Ada"}, "stats": {}, @@ -158,10 +159,10 @@ async fn profile_statistics_keeps_profile_when_stats_are_unavailable() { } #[tokio::test] -async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials() { +async fn profile_avatar_reuses_cached_source_with_the_accounts_rotated_token() { let server = MockServer::start().await; Mock::given(method("GET")) - .and(path("/wham/profiles/me")) + .and(path("/api/codex/profiles/me")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "profile": { "display_name": "Ada", @@ -192,6 +193,11 @@ async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials() .profile_statistics(account.id()) .await .expect("profile statistics"); + store + .repository() + .rotate_refreshed_oauth_secret(&account, secret("rotated-avatar-token"), None, None) + .await + .expect("rotate avatar account token"); let avatar = service .profile_avatar(account.id()) .await @@ -211,7 +217,13 @@ async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials() assert_eq!(content_type.as_deref(), Some("image/svg+xml")); assert_eq!(body, b""); - assert!(!avatar_request.headers.contains_key("authorization")); + assert_eq!( + avatar_request.headers["authorization"], + "Bearer rotated-avatar-token" + ); assert!(!avatar_request.headers.contains_key("cookie")); - assert!(!avatar_request.headers.contains_key("chatgpt-account-id")); + assert_eq!( + avatar_request.headers["chatgpt-account-id"], + "chatgpt-acct_profile_avatar" + ); } diff --git a/backend/crates/providers/openai/tests/credential/quota/mod.rs b/backend/crates/providers/openai/tests/credential/quota/mod.rs index cfbc767e6..863a63ef4 100644 --- a/backend/crates/providers/openai/tests/credential/quota/mod.rs +++ b/backend/crates/providers/openai/tests/credential/quota/mod.rs @@ -33,6 +33,7 @@ fn wire_profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "quota-contract".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 7, 18, 0, 0, 0) .single() diff --git a/backend/crates/providers/openai/tests/credential/refresh.rs b/backend/crates/providers/openai/tests/credential/refresh.rs index cfd967062..e754b3f60 100644 --- a/backend/crates/providers/openai/tests/credential/refresh.rs +++ b/backend/crates/providers/openai/tests/credential/refresh.rs @@ -431,6 +431,7 @@ async fn scheduled_refresh_persists_retryable_message_inside_the_two_hour_window client_id: "test-public-client".to_owned(), token_endpoint: format!("{}/oauth/token", server.uri()), }, + provider_openai::OpenAiConfig::default().wire_profile_state(), )), Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), diff --git a/backend/crates/providers/openai/tests/credential/token_client.rs b/backend/crates/providers/openai/tests/credential/token_client.rs index ee814d8e6..b26f6e3d5 100644 --- a/backend/crates/providers/openai/tests/credential/token_client.rs +++ b/backend/crates/providers/openai/tests/credential/token_client.rs @@ -6,6 +6,73 @@ use secrecy::{ExposeSecret, SecretString}; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; +#[tokio::test] +async fn refresh_tracks_the_online_profile_without_contaminating_code_exchange() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/oauth/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "access_token": "audit-access", "refresh_token": "audit-refresh", "id_token": "header.e30.signature" + }))) + .expect(4) + .mount(&server) + .await; + let profile = provider_openai::OpenAiConfig::default().wire_profile_state(); + let client = provider_openai::credential::token_client::openai_token_client( + TokenClientConfig { + client_id: "audit-client".to_owned(), + token_endpoint: format!("{}/oauth/token", server.uri()), + }, + profile.clone(), + ) + .expect("production auth transport"); + for (core, desktop, build) in [ + ("1.2.3", "26.800.10000", "8000"), + ("1.2.4", "26.901.10000", "8109"), + ] { + profile.update_bundled_release( + &provider_openai::transport::profile::CodexBundledReleaseProfile { + codex_version: core.to_owned(), + desktop_version: desktop.to_owned(), + desktop_build: build.to_owned(), + verified_at: chrono::Utc::now(), + }, + ); + client.refresh("audit-refresh").await.expect("refresh"); + client + .exchange_authorization_code(AuthorizationCodeGrant { + code: SecretString::from("audit-code"), + code_verifier: SecretString::from("audit-verifier"), + }) + .await + .expect("code exchange"); + let requests = server.received_requests().await.expect("requests"); + let refresh = &requests[requests.len() - 2]; + let exchange = &requests[requests.len() - 1]; + assert_eq!( + refresh.headers["user-agent"], + profile.snapshot().user_agent() + ); + assert_eq!(refresh.headers["originator"], "Codex Desktop"); + assert_eq!(refresh.headers["content-type"], "application/json"); + assert_eq!( + exchange.headers["content-type"], + "application/x-www-form-urlencoded" + ); + for name in [ + "originator", + "user-agent", + "version", + "x-openai-internal-codex-residency", + ] { + assert!(!exchange.headers.contains_key(name), "raw exchange: {name}"); + } + for name in ["authorization", "chatgpt-account-id", "version"] { + assert!(!refresh.headers.contains_key(name), "refresh: {name}"); + } + } +} + fn client(server: &MockServer) -> OpenAiTokenClient { OpenAiTokenClient::new( reqwest::Client::builder() @@ -16,6 +83,7 @@ fn client(server: &MockServer) -> OpenAiTokenClient { client_id: "test-public-client".to_owned(), token_endpoint: format!("{}/oauth/token", server.uri()), }, + provider_openai::OpenAiConfig::default().wire_profile_state(), ) } diff --git a/backend/crates/providers/openai/tests/provider/contract.rs b/backend/crates/providers/openai/tests/provider/contract.rs index ecb2beeff..24cc2fe2b 100644 --- a/backend/crates/providers/openai/tests/provider/contract.rs +++ b/backend/crates/providers/openai/tests/provider/contract.rs @@ -136,6 +136,7 @@ fn wire_profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "provider-contract".to_owned(), + residency: None, verified_at: Utc::now(), }) } @@ -1004,6 +1005,8 @@ async fn image_endpoints_bypass_only_the_text_catalog_and_preserve_the_current_c .and(path(*endpoint)) .and(header("originator", "codex_cli_rs")) .and(header("x-codex-image-turn-id", "turn_image_contract")) + .and(header("version", "0.144.0")) + .and(header("accept", "*/*")) .and(body_bytes(body.to_vec())) .respond_with( ResponseTemplate::new(200) @@ -1143,6 +1146,8 @@ async fn standalone_search_preserves_wire_and_scopes_turn_metadata_to_the_select Mock::given(method("POST")) .and(path("/codex/alpha/search")) .and(header("originator", "codex_cli_rs")) + .and(header("version", "0.144.0")) + .and(header("accept", "*/*")) .and(body_bytes(request_body.to_vec())) .respond_with( ResponseTemplate::new(200) @@ -2731,6 +2736,15 @@ async fn same_account_scope_preserves_future_protocol_shapes() { body.get("installation_id"), Some(&json!("client-installation")) ); + assert_eq!( + body.get("installation_id"), + body.pointer("/client_metadata/installation_id") + ); + assert!( + body.pointer("/client_metadata/x-codex-installation-id") + .is_none() + ); + assert!(captured_header_values(&request, "x-codex-installation-id").is_empty()); } #[tokio::test] diff --git a/backend/crates/providers/openai/tests/transport/catalog.rs b/backend/crates/providers/openai/tests/transport/catalog.rs index 839a5de03..395c80afe 100644 --- a/backend/crates/providers/openai/tests/transport/catalog.rs +++ b/backend/crates/providers/openai/tests/transport/catalog.rs @@ -223,9 +223,8 @@ async fn fetch_should_send_official_catalog_headers_and_query() { "user-agent", "codex_cli_rs/0.144.0 (linux 6.8; x86_64) xterm (codex_cli_rs; 1.0.0)", )) - .and(header("x-openai-internal-codex-residency", "us")) - .and(header("accept", "application/json")) - .and(header("x-codex-installation-id", "installation-123")) + .and(header("accept", "*/*")) + .and(header("version", "0.144.0")) .respond_with( ResponseTemplate::new(200) .insert_header("content-type", "application/json") @@ -306,6 +305,7 @@ fn profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "xterm".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 7, 18, 0, 0, 0) .single() @@ -315,6 +315,7 @@ fn profile() -> CodexWireProfileState { fn context() -> CodexRequestContext<'static> { CodexRequestContext { + trace: None, authorization: "Bearer oauth-access", account_id: Some("acct_123"), request_id: "req_catalog", diff --git a/backend/crates/providers/openai/tests/transport/client.rs b/backend/crates/providers/openai/tests/transport/client.rs index 5c18f9281..efe3038f9 100644 --- a/backend/crates/providers/openai/tests/transport/client.rs +++ b/backend/crates/providers/openai/tests/transport/client.rs @@ -25,6 +25,7 @@ fn upstream_error_formatting_should_redact_body() { #[test] fn request_context_debug_should_redact_all_identity_material() { let context = CodexRequestContext { + trace: None, authorization: "Bearer access-secret-marker", account_id: Some("account-secret-marker"), request_id: "req_safe", diff --git a/backend/crates/providers/openai/tests/transport/diagnostics.rs b/backend/crates/providers/openai/tests/transport/diagnostics.rs index fac97b6fb..091032107 100644 --- a/backend/crates/providers/openai/tests/transport/diagnostics.rs +++ b/backend/crates/providers/openai/tests/transport/diagnostics.rs @@ -257,3 +257,87 @@ fn openai_account_failure_matrix_is_preserved() { ); } } + +#[test] +fn diagnostic_wire_dump_preserves_binary_bytes_and_numbers_every_fragment() { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + use gateway_core::diagnostics::TraceContext; + use std::sync::{Arc, Mutex}; + + #[derive(Clone)] + struct Writer(Arc>>); + impl std::io::Write for Writer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().write_all(bytes)?; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let output = Arc::new(Mutex::new(Vec::new())); + let writer = Writer(Arc::clone(&output)); + let subscriber = tracing_subscriber::fmt() + .json() + .with_writer(move || writer.clone()) + .with_ansi(false) + .finish(); + let bytes: Vec = (0..100_000).map(|index| (index % 256) as u8).collect(); + tracing::subscriber::with_default(subscriber, || { + let trace = TraceContext::new("req_binary_dump"); + trace + .attempt(3) + .exchange("websocket") + .dump("upstream.binary", &bytes); + trace.headers( + "client.connection.headers", + serde_json::json!({}), + [ + ("authorization", b"Bearer secret".as_slice()), + ("x-binary", b"\xff\x80".as_slice()), + ("x-binary", b"\x00".as_slice()), + ], + ); + let snapshot = trace.snapshot().unwrap(); + assert_eq!(snapshot["wireFrames"], 2); + assert!(!snapshot.to_string().contains("Bearer secret")); + }); + let log = output.lock().unwrap(); + let records: Vec = std::str::from_utf8(&log) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .filter(|record: &serde_json::Value| record["target"] == "request_dump") + .collect(); + assert_eq!(records.len(), 5); + let mut recovered = Vec::new(); + for (index, record) in records.iter().take(4).enumerate() { + let fields = &record["fields"]; + assert_eq!(fields["wire_sequence"], 1); + assert_eq!(fields["attempt_index"], 3); + assert_eq!(fields["exchange_id"], 1); + assert_eq!(fields["chunk_index"], index); + assert_eq!(fields["chunk_count"], 4); + recovered.extend( + STANDARD + .decode(fields["body_base64"].as_str().unwrap()) + .unwrap(), + ); + } + assert_eq!(recovered, bytes); + let header_record = &records[4]["fields"]; + assert_eq!(header_record["wire_sequence"], 2); + let header_bytes = STANDARD + .decode(header_record["body_base64"].as_str().unwrap()) + .unwrap(); + let headers: serde_json::Value = serde_json::from_slice(&header_bytes).unwrap(); + for (index, expected) in [b"Bearer secret".as_slice(), b"\xff\x80", b"\x00"] + .into_iter() + .enumerate() + { + let value = STANDARD + .decode(headers["headers"][index]["valueBase64"].as_str().unwrap()) + .unwrap(); + assert_eq!(value, expected); + } +} diff --git a/backend/crates/providers/openai/tests/transport/headers.rs b/backend/crates/providers/openai/tests/transport/headers.rs index e3fa759bf..9602d7c47 100644 --- a/backend/crates/providers/openai/tests/transport/headers.rs +++ b/backend/crates/providers/openai/tests/transport/headers.rs @@ -277,6 +277,7 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id: Some("chatgpt-account"), request_id: "req_ws_security", @@ -317,12 +318,8 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f ); let headers = received_headers.lock().expect("headers lock"); for (name, expected) in [ - ("x-client-request-id", "req_ws_security"), - ("content-type", "application/json"), - ("accept", "text/event-stream"), - ("x-codex-installation-id", "install-123"), + ("x-client-request-id", "cp_derived"), ("openai-beta", "responses_websockets=2026-02-06"), - ("x-openai-internal-codex-residency", "us"), ("x-codex-turn-state", "turn-state"), ("x-codex-turn-metadata", "{\"thread_source\":\"subagent\"}"), ("x-codex-beta-features", "feature-a"), @@ -341,6 +338,11 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f ); } for forbidden in [ + "x-codex-installation-id", + "x-codex-turn-id", + "x-openai-internal-codex-residency", + "accept", + "content-type", "session_id", "thread-id", "x-openai-internal-codex-responses-lite", @@ -365,7 +367,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id let profile = test_wire_profile(); let profile_snapshot = profile.snapshot(); let expected_user_agent = profile_snapshot.user_agent(); - let expected_desktop_version = profile_snapshot.desktop_version; + let expected_core_version = profile_snapshot.codex_version; let client = CodexBackendClient::new( reqwest::Client::builder() .no_proxy() @@ -379,6 +381,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer lease-token", account_id: Some("lease-account"), installation_id: Some("lease-installation"), @@ -401,16 +404,11 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id vec![b"turn-ascii".to_vec(), b"turn-\x80".to_vec()] ); for (name, value) in [ - ("accept", b"application/vnd.openai.future+json".as_slice()), - ( - "content-type", - b"application/vnd.openai.request+json".as_slice(), - ), - ("x-openai-internal-codex-residency", b"us".as_slice()), + ("accept", b"text/event-stream".as_slice()), + ("content-type", b"application/json".as_slice()), ("x-still-valid", b"after-invalid".as_slice()), ("authorization", b"Bearer lease-token".as_slice()), ("chatgpt-account-id", b"lease-account".as_slice()), - ("x-codex-installation-id", b"lease-installation".as_slice()), ] { assert_eq!(raw_header_values(&raw, name), vec![value.to_vec()]); } @@ -425,7 +423,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id ); assert_eq!( raw_header_values(&raw, "version"), - vec![expected_desktop_version.into_bytes()] + vec![expected_core_version.into_bytes()] ); for dropped in [ "openai-beta", @@ -501,6 +499,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer lease-token", account_id: Some("lease-account"), installation_id: Some("lease-installation"), @@ -529,24 +528,15 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values values("openai-beta"), vec![b"responses_websockets=2026-02-06".to_vec()] ); - assert_eq!( - values("accept"), - vec![b"application/vnd.openai.future+json".to_vec()] - ); - assert_eq!( - values("content-type"), - vec![b"application/vnd.openai.request+json".to_vec()] - ); + assert!(values("accept").is_empty()); + assert!(values("content-type").is_empty()); assert_eq!( values("user-agent"), vec![test_wire_profile().snapshot().user_agent().into_bytes()] ); assert_eq!(values("originator"), vec![b"codex_cli_rs".to_vec()]); assert_eq!(values("version"), vec![b"1.2.3".to_vec()]); - assert_eq!( - values("x-openai-internal-codex-residency"), - vec![b"us".to_vec()] - ); + assert!(values("x-openai-internal-codex-residency").is_empty()); for dropped in [ "x-openai-actor-authorization", "x-oai-attestation", @@ -563,10 +553,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values values("chatgpt-account-id"), vec![b"lease-account".to_vec()] ); - assert_eq!( - values("x-codex-installation-id"), - vec![b"lease-installation".to_vec()] - ); + assert!(values("x-codex-installation-id").is_empty()); assert_eq!(values("x-codex-turn-state"), vec![b"turn-ascii".to_vec()]); } @@ -593,7 +580,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() { let profile = test_wire_profile(); let profile_snapshot = profile.snapshot(); let expected_user_agent = profile_snapshot.user_agent(); - let expected_desktop_version = profile_snapshot.desktop_version; + let expected_core_version = profile_snapshot.codex_version; let client = CodexBackendClient::new( reqwest::Client::builder() .no_proxy() @@ -607,6 +594,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() { .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id: Some("chatgpt-account"), request_id: "req_order", @@ -648,14 +636,14 @@ async fn backend_http_should_send_codex_context_without_browser_headers() { Some("codex_cli_rs"), Some(expected_user_agent.as_str()), Some("text/event-stream"), - Some("us"), - Some("req_order"), + None, + Some("session-1"), Some("turn-state"), ) ); assert_eq!( read_header_value(&raw_request, "version"), - Some(expected_desktop_version.as_str()) + Some(expected_core_version.as_str()) ); for required in [ "authorization", @@ -665,9 +653,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() { "content-type", "cookie", "accept", - "x-openai-internal-codex-residency", "x-client-request-id", - "x-codex-installation-id", "session-id", "x-codex-window-id", "x-codex-turn-state", @@ -680,6 +666,9 @@ async fn backend_http_should_send_codex_context_without_browser_headers() { assert!(header_names.iter().any(|name| name == required)); } for forbidden in [ + "x-codex-installation-id", + "x-codex-turn-id", + "x-openai-internal-codex-residency", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", @@ -729,6 +718,7 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo .create_response( &request, CodexRequestContext { + trace: None, client_request_id: Some("opaque\nclient-request-id"), session_id: Some("opaque\nsession"), thread_id: Some("opaque\nthread"), @@ -751,6 +741,7 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo read_header_value(&raw_request, "x-client-request-id"), Some("req_protocol_fallback") ); + assert_eq!(read_header_value(&raw_request, "version"), Some("1.2.3")); for omitted in [ "session-id", "thread-id", @@ -760,7 +751,6 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo "x-codex-turn-metadata", "x-codex-beta-features", "x-responsesapi-include-timing-metrics", - "version", "x-codex-parent-thread-id", "x-openai-subagent", "x-openai-internal-codex-responses-lite", @@ -897,7 +887,14 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la let server = tokio::spawn(async move { let (first_stream, _) = listener.accept().await.expect("first websocket"); let mut first_user_agent = String::new(); + let mut first_version = String::new(); let mut first = accept_codex_test_websocket_with(first_stream, |request, _response| { + first_version = request.headers()["version"] + .to_str() + .expect("Core version") + .to_owned(); + assert!(!request.headers().contains_key("accept")); + assert!(!request.headers().contains_key("content-type")); first_user_agent = request .headers() .get("user-agent") @@ -935,7 +932,14 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la .expect("new chain should use a new wire profile connection") .expect("second websocket"); let mut second_user_agent = String::new(); + let mut second_version = String::new(); let mut second = accept_codex_test_websocket_with(second_stream, |request, _response| { + second_version = request.headers()["version"] + .to_str() + .expect("Core version") + .to_owned(); + assert!(!request.headers().contains_key("accept")); + assert!(!request.headers().contains_key("content-type")); second_user_agent = request .headers() .get("user-agent") @@ -956,7 +960,12 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la .await .expect("second response.completed"); - (first_user_agent, second_user_agent) + ( + first_user_agent, + second_user_agent, + first_version, + second_version, + ) }); let profile = test_wire_profile(); @@ -983,7 +992,7 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la .expect("first response"); profile.update_bundled_release(&CodexBundledReleaseProfile { codex_version: "1.2.4".to_owned(), - desktop_version: "1.2.4".to_owned(), + desktop_version: "26.999.10000".to_owned(), desktop_build: "124".to_owned(), verified_at: Utc::now(), }); @@ -1006,8 +1015,12 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la .await .expect("second response"); - let (first_user_agent, second_user_agent) = server.await.expect("profile server task"); + let (first_user_agent, second_user_agent, first_version, second_version) = + server.await.expect("profile server task"); assert!(first_user_agent.contains("1.2.3")); assert!(second_user_agent.contains("1.2.4")); + assert!(second_user_agent.contains("26.999.10000")); + assert_eq!(first_version, "1.2.3"); + assert_eq!(second_version, "1.2.4"); assert_ne!(first_user_agent, second_user_agent); } diff --git a/backend/crates/providers/openai/tests/transport/http_client.rs b/backend/crates/providers/openai/tests/transport/http_client.rs index 504d1a189..8550b75f2 100644 --- a/backend/crates/providers/openai/tests/transport/http_client.rs +++ b/backend/crates/providers/openai/tests/transport/http_client.rs @@ -137,6 +137,7 @@ async fn codex_backend_client_should_preserve_oversized_error_response() { .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id: Some("chatgpt-account"), request_id: "req_large_error", @@ -236,6 +237,7 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id: Some("chatgpt-account"), request_id: "req_http_retry_after_body", @@ -294,10 +296,11 @@ async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updat let mut request = codex_request("gpt-5.5", "", Vec::new()); request.force_http_sse = true; + let trace = gateway_core::diagnostics::TraceContext::new("req_http_rate_limit_event"); let response = client .create_response( &request, - request_context("req_http_rate_limit_event", Some("acct")), + request_context("req_http_rate_limit_event", Some("acct")).with_trace(&trace), ) .await .expect("HTTP SSE response"); @@ -315,6 +318,18 @@ async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updat ("x-codex-limit-reached".to_owned(), "false".to_owned()), ] ); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + assert!( + events + .iter() + .any(|event| event["data"]["eventType"] == "codex.rate_limits") + ); + assert!( + events + .iter() + .any(|event| event["data"]["eventType"] == "response.completed") + ); } #[tokio::test(start_paused = true)] @@ -407,6 +422,7 @@ async fn codex_backend_client_should_capture_forwardable_response_metadata() { .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id: Some("chatgpt-account"), request_id: "req_response_metadata", diff --git a/backend/crates/providers/openai/tests/transport/mod.rs b/backend/crates/providers/openai/tests/transport/mod.rs index 8cca61c6e..ca481211d 100644 --- a/backend/crates/providers/openai/tests/transport/mod.rs +++ b/backend/crates/providers/openai/tests/transport/mod.rs @@ -55,9 +55,11 @@ mod http_client; mod latency; mod profile; mod profile_avatar; +mod profile_contract; mod protocol; mod request; mod reset_credits; +mod tls; mod usage; mod websocket; mod websocket_pool; @@ -126,6 +128,7 @@ fn test_wire_profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "transport-test".to_owned(), + residency: None, verified_at: Utc::now(), }) } @@ -135,6 +138,7 @@ fn request_context<'a>( account_id: Option<&'a str>, ) -> CodexRequestContext<'a> { CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id, request_id, diff --git a/backend/crates/providers/openai/tests/transport/profile/mod.rs b/backend/crates/providers/openai/tests/transport/profile/mod.rs index cfc943a8b..7ca7e15a9 100644 --- a/backend/crates/providers/openai/tests/transport/profile/mod.rs +++ b/backend/crates/providers/openai/tests/transport/profile/mod.rs @@ -141,6 +141,7 @@ fn wire_profile_should_generate_bundled_core_app_server_user_agent() { os_version: "15.7.1".to_owned(), arch: "arm64".to_owned(), terminal: "unknown".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 8, 3, 0, 0, 0) .single() @@ -360,6 +361,7 @@ fn wire_profile() -> CodexWireProfile { os_version: "15.7.1".to_owned(), arch: "arm64".to_owned(), terminal: "unknown".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 8, 3, 0, 0, 0) .single() diff --git a/backend/crates/providers/openai/tests/transport/profile_avatar.rs b/backend/crates/providers/openai/tests/transport/profile_avatar.rs index 6bb0a7578..57ffebd61 100644 --- a/backend/crates/providers/openai/tests/transport/profile_avatar.rs +++ b/backend/crates/providers/openai/tests/transport/profile_avatar.rs @@ -1,5 +1,7 @@ use futures::TryStreamExt as _; -use provider_openai::transport::{CodexProfileAvatarFetchError, fetch_profile_avatar}; +use provider_openai::transport::{ + CodexProfileAvatarFetchError, CodexRequestContext, fetch_profile_avatar, +}; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -30,8 +32,14 @@ async fn profile_avatar_streams_unrestricted_content_type_and_body_size() { let avatar = fetch_profile_avatar( &client, &server.uri(), - "codex_cli_rs/1.0.0 (linux; x86_64)", + &super::test_wire_profile().snapshot(), OFFICIAL_AVATAR_SOURCE, + CodexRequestContext::auxiliary( + "Bearer avatar-token", + Some("avatar-account"), + "avatar", + None, + ), ) .await .expect("profile avatar"); @@ -52,14 +60,18 @@ async fn profile_avatar_streams_unrestricted_content_type_and_body_size() { assert_eq!(streamed, body); assert_eq!( headers.get("accept").and_then(|value| value.to_str().ok()), - Some("image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8") + Some("*/*") ); assert_eq!( headers .get("user-agent") .and_then(|value| value.to_str().ok()), - Some("codex_cli_rs/1.0.0 (linux; x86_64)") + Some("codex_cli_rs/1.2.3 (linux; x86_64)") ); + assert_eq!(headers["authorization"], "Bearer avatar-token"); + assert_eq!(headers["chatgpt-account-id"], "avatar-account"); + assert_eq!(headers["originator"], "codex_cli_rs"); + assert!(!headers.contains_key("version")); } #[tokio::test] @@ -72,9 +84,20 @@ async fn profile_avatar_rejects_non_official_sources_before_request() { "https://chatgpt.com/backend-api/estuary/public_content/enc/token?next=1", "https://chatgpt.com/backend-api/estuary/public_content/enc/", ] { - let error = fetch_profile_avatar(&client, "http://127.0.0.1:9", "test-agent", source) - .await - .expect_err("invalid source"); + let error = fetch_profile_avatar( + &client, + "http://127.0.0.1:9", + &super::test_wire_profile().snapshot(), + source, + CodexRequestContext::auxiliary( + "Bearer avatar-token", + Some("avatar-account"), + "avatar", + None, + ), + ) + .await + .expect_err("invalid source"); assert!(matches!(error, CodexProfileAvatarFetchError::InvalidSource)); } } diff --git a/backend/crates/providers/openai/tests/transport/profile_contract.rs b/backend/crates/providers/openai/tests/transport/profile_contract.rs new file mode 100644 index 000000000..4d5d60084 --- /dev/null +++ b/backend/crates/providers/openai/tests/transport/profile_contract.rs @@ -0,0 +1,136 @@ +use super::*; +use provider_openai::transport::build_reqwest_client; +use provider_openai::transport::profile::CodexResidency; +use uuid::Uuid; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; + +/// 官方本机样本：模型请求带 Core version，backend-client 账号请求不带。 +/// 通过真实 HTTP 请求验证画像更新后每条路径取到同一制品中的正确字段。 +#[tokio::test] +async fn each_core_endpoint_uses_the_current_bundled_release() { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({"error":"audit fixture"}))) + .expect(12) + .mount(&server) + .await; + let profile = test_wire_profile(); + let client = CodexBackendClient::new( + build_reqwest_client().expect("HTTP client"), + format!("{}/backend-api", server.uri()), + profile.clone(), + ); + for (core, desktop, build) in [ + ("0.152.8", "26.800.10000", "8000"), + ("0.153.4", "26.901.51231", "8109"), + ] { + profile.update_bundled_release(&CodexBundledReleaseProfile { + codex_version: core.to_owned(), + desktop_version: desktop.to_owned(), + desktop_build: build.to_owned(), + verified_at: Utc::now(), + }); + let context = request_context("audit", Some("audit-account")); + client + .fetch_models_with_context(context) + .await + .expect_err("fixture rejection"); + client + .fetch_usage(context) + .await + .expect_err("fixture rejection"); + client + .fetch_profile_statistics(context) + .await + .expect_err("fixture rejection"); + client + .list_rate_limit_reset_credits(context) + .await + .expect_err("fixture rejection"); + client + .consume_rate_limit_reset_credit(context, None, Uuid::now_v7()) + .await + .expect_err("fixture rejection"); + let mut body = codex_request_body("gpt-test", "", Vec::new()); + body.insert("service_tier".to_owned(), json!("priority")); + let mut request = CodexResponsesRequest::from_body(body); + request.force_http_sse = true; + client + .create_response(&request, context) + .await + .expect_err("fixture rejection"); + + let captured = server.received_requests().await.expect("captured requests"); + for request in &captured[captured.len() - 6..] { + let headers = &request.headers; + assert_eq!(headers["user-agent"], profile.snapshot().user_agent()); + assert_eq!(headers["authorization"], "Bearer access-token"); + assert_eq!(headers["chatgpt-account-id"], "audit-account"); + for forbidden in [ + "x-codex-installation-id", + "x-codex-turn-id", + "x-openai-internal-codex-residency", + "oai-language", + "openai-beta", + ] { + assert!( + !headers.contains_key(forbidden), + "{}: {forbidden}", + request.url + ); + } + if request.url.path().contains("/codex/") { + assert_eq!(headers["originator"], "codex_cli_rs"); + assert_eq!(headers["version"], core); + assert_ne!(headers["version"], desktop); + } else { + assert!(!headers.contains_key("originator")); + assert!(!headers.contains_key("version")); + } + if request.url.path().ends_with("/responses") { + assert_eq!(headers["accept"], "text/event-stream"); + assert_eq!( + headers["x-codex-routing-hint"], + "model=gpt-test;tier=priority" + ); + } else { + assert_eq!(headers["accept"], "*/*"); + } + if request.url.path().ends_with("/models") { + assert_eq!( + request.url.query(), + Some(format!("client_version={core}").as_str()) + ); + } + } + } +} + +#[test] +fn residency_is_explicit_and_survives_artifact_updates() { + let mut initial = test_wire_profile().snapshot(); + initial.residency = Some(CodexResidency::Us); + let profile = CodexWireProfileState::new(initial); + profile.update_bundled_release(&CodexBundledReleaseProfile { + codex_version: "2.0.0".to_owned(), + desktop_version: "26.999.10000".to_owned(), + desktop_build: "9999".to_owned(), + verified_at: Utc::now(), + }); + let snapshot = profile.snapshot(); + let model = provider_openai::transport::headers::build_codex_model_headers( + &snapshot, + "Bearer audit", + None, + ) + .expect("model headers"); + let account = provider_openai::transport::headers::build_codex_account_headers( + &snapshot, + "Bearer audit", + None, + ) + .expect("account headers"); + assert_eq!(model["x-openai-internal-codex-residency"], "us"); + assert_eq!(model["version"], "2.0.0"); + assert!(!account.contains_key("x-openai-internal-codex-residency")); +} diff --git a/backend/crates/providers/openai/tests/transport/reset_credits.rs b/backend/crates/providers/openai/tests/transport/reset_credits.rs index ac684089f..785c82b96 100644 --- a/backend/crates/providers/openai/tests/transport/reset_credits.rs +++ b/backend/crates/providers/openai/tests/transport/reset_credits.rs @@ -26,6 +26,7 @@ fn client(base_url: &str) -> CodexBackendClient { os_version: "15.5.0".to_owned(), arch: "arm64".to_owned(), terminal: "xterm-256color".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 8, 20, 0, 0, 0) .single() @@ -44,10 +45,10 @@ fn context() -> CodexRequestContext<'static> { } #[tokio::test] -async fn list_should_match_official_desktop_surface_profile() { +async fn list_should_match_official_core_account_profile() { let server = MockServer::start().await; Mock::given(method("GET")) - .and(path("/wham/rate-limit-reset-credits")) + .and(path("/api/codex/rate-limit-reset-credits")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "available_count": 1, "credits": [{ @@ -81,11 +82,13 @@ async fn list_should_match_official_desktop_surface_profile() { header(request, "chatgpt-account-id"), Some("acct_workspace") ); - assert_eq!(header(request, "originator"), Some("Codex Desktop")); - assert_eq!(header(request, "oai-language"), Some("en")); + assert_eq!(header(request, "originator"), None); + assert_eq!(header(request, "oai-language"), None); assert_eq!( header(request, "user-agent"), - Some("Codex Desktop/26.818.21641 (Mac OS; arm64)") + Some( + "Codex Desktop/0.115.0-alpha.11 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 26.818.21641)" + ) ); assert_eq!(header(request, "accept"), Some("*/*")); for name in [ @@ -110,7 +113,7 @@ async fn list_should_match_official_desktop_surface_profile() { async fn consume_should_send_exact_credit_and_redeem_request_id_without_retry() { let server = MockServer::start().await; Mock::given(method("POST")) - .and(path("/wham/rate-limit-reset-credits/consume")) + .and(path("/api/codex/rate-limit-reset-credits/consume")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "code": "reset", "credit": { @@ -147,7 +150,7 @@ async fn consume_should_send_exact_credit_and_redeem_request_id_without_retry() async fn consume_should_preserve_raw_upstream_error_and_never_retry() { let server = MockServer::start().await; Mock::given(method("POST")) - .and(path("/wham/rate-limit-reset-credits/consume")) + .and(path("/api/codex/rate-limit-reset-credits/consume")) .respond_with( ResponseTemplate::new(429) .insert_header("retry-after", "17") @@ -188,7 +191,7 @@ async fn consume_should_preserve_raw_upstream_error_and_never_retry() { async fn list_should_reject_oversized_success_body() { let server = MockServer::start().await; Mock::given(method("GET")) - .and(path("/wham/rate-limit-reset-credits")) + .and(path("/api/codex/rate-limit-reset-credits")) .respond_with(ResponseTemplate::new(200).set_body_raw( "x".repeat(MAX_CODEX_RESET_CREDITS_BODY_BYTES + 1), "application/json", diff --git a/backend/crates/providers/openai/tests/transport/tls.rs b/backend/crates/providers/openai/tests/transport/tls.rs new file mode 100644 index 000000000..df2a12bc4 --- /dev/null +++ b/backend/crates/providers/openai/tests/transport/tls.rs @@ -0,0 +1,146 @@ +use std::collections::BTreeMap; + +use bytes::{Buf as _, Bytes}; + +use super::*; + +// Captured from Desktop 26.901.51231 / Core 0.153.4 on 2026-09-06. +// Compare wire semantics: rustls randomizes extension order and ephemeral key bytes. +#[derive(Debug, PartialEq, Eq)] +struct ClientHello { + cipher_suites: Vec, + extensions: Vec, + groups: Vec, + signature_algorithms: Vec, + key_shares: Vec<(u16, usize)>, + alpn: Vec, +} + +#[tokio::test] +async fn http_client_hello_should_match_official_rustls_transport() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!( + "https://localhost:{}/", + listener.local_addr().unwrap().port() + ); + let client = provider_openai::transport::build_reqwest_client().unwrap(); + let (hello, response) = timeout(Duration::from_secs(10), async { + tokio::join!(read_client_hello(listener), client.get(url).send()) + }) + .await + .expect("HTTP ClientHello within timeout"); + assert!( + response.is_err(), + "capture endpoint rejects TLS after ClientHello" + ); + + let mut expected = official_websocket_hello(); + expected.extensions.insert(5, 16); + expected.alpn = vec!["h2".to_owned(), "http/1.1".to_owned()]; + assert_eq!(hello, expected); +} + +#[tokio::test] +async fn websocket_client_hello_should_match_official_rustls_transport() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("wss://localhost:{}/", listener.local_addr().unwrap().port()); + let connector = + provider_openai::transport::tls::maybe_build_rustls_client_config_with_custom_ca() + .unwrap() + .map(tokio_tungstenite::Connector::Rustls); + let (hello, response) = timeout(Duration::from_secs(10), async { + tokio::join!( + read_client_hello(listener), + tokio_tungstenite::connect_async_tls_with_config(url, None, false, connector) + ) + }) + .await + .expect("WebSocket ClientHello within timeout"); + assert!( + response.is_err(), + "capture endpoint rejects TLS after ClientHello" + ); + assert_eq!(hello, official_websocket_hello()); +} + +fn official_websocket_hello() -> ClientHello { + ClientHello { + cipher_suites: vec![ + 4866, 4865, 4867, 49196, 49195, 52393, 49200, 49199, 52392, 255, + ], + extensions: vec![0, 5, 10, 11, 13, 23, 35, 43, 45, 51], + groups: vec![4588, 29, 23, 24], + signature_algorithms: vec![1283, 1027, 1539, 2055, 2054, 2053, 2052, 1537, 1281, 1025], + key_shares: vec![(4588, 1216), (29, 32)], + alpn: Vec::new(), + } +} + +async fn read_client_hello(listener: TcpListener) -> ClientHello { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut header = [0; 5]; + stream.read_exact(&mut header).await.unwrap(); + assert_eq!(header[0], 22, "TLS handshake record"); + let length = usize::from(u16::from_be_bytes([header[3], header[4]])); + assert!(length <= 16_384, "bounded ClientHello record"); + let mut record = vec![0; length]; + stream.read_exact(&mut record).await.unwrap(); + stream.write_all(&[21, 3, 3, 0, 2, 2, 40]).await.unwrap(); + + let mut hello = Bytes::from(record); + assert_eq!(hello.get_u8(), 1, "ClientHello message"); + hello.advance(3 + 2 + 32); // Message length, legacy version, random. + let session_id_len = usize::from(hello.get_u8()); + hello.advance(session_id_len); + let cipher_suites = u16_values(take_vector(&mut hello)); + let compression_len = usize::from(hello.get_u8()); + hello.advance(compression_len); + let mut extensions = take_vector(&mut hello); + let mut values = BTreeMap::new(); + while extensions.has_remaining() { + let kind = extensions.get_u16(); + assert!(values.insert(kind, take_vector(&mut extensions)).is_none()); + } + + let mut groups = values[&10].clone(); + let groups = u16_values(take_vector(&mut groups)); + let mut signatures = values[&13].clone(); + let signature_algorithms = u16_values(take_vector(&mut signatures)); + let mut shares = values[&51].clone(); + let mut shares = take_vector(&mut shares); + let mut key_shares = Vec::new(); + while shares.has_remaining() { + let group = shares.get_u16(); + key_shares.push((group, take_vector(&mut shares).len())); + } + let mut alpn = Vec::new(); + if let Some(protocols) = values.get(&16) { + let mut protocols = protocols.clone(); + let mut protocols = take_vector(&mut protocols); + while protocols.has_remaining() { + let length = usize::from(protocols.get_u8()); + alpn.push(String::from_utf8(protocols.split_to(length).to_vec()).unwrap()); + } + } + ClientHello { + cipher_suites, + extensions: values.into_keys().collect(), + groups, + signature_algorithms, + key_shares, + alpn, + } +} + +fn take_vector(bytes: &mut Bytes) -> Bytes { + let length = usize::from(bytes.get_u16()); + bytes.split_to(length) +} + +fn u16_values(mut bytes: Bytes) -> Vec { + let mut values = Vec::new(); + while bytes.has_remaining() { + values.push(bytes.get_u16()); + } + values +} diff --git a/backend/crates/providers/openai/tests/transport/usage.rs b/backend/crates/providers/openai/tests/transport/usage.rs index 4efa17e89..f304bf782 100644 --- a/backend/crates/providers/openai/tests/transport/usage.rs +++ b/backend/crates/providers/openai/tests/transport/usage.rs @@ -253,6 +253,7 @@ async fn fetch_should_use_wham_usage_headers_only() { let usage = client(&server.uri()) .fetch_usage(CodexRequestContext { + trace: None, authorization: "Bearer oauth-access", account_id: Some("acct_123"), request_id: "req_usage_headers", @@ -283,7 +284,7 @@ async fn fetch_should_use_wham_usage_headers_only() { for (name, expected) in [ ("authorization", "Bearer oauth-access"), ("chatgpt-account-id", "acct_123"), - ("accept", "application/json"), + ("accept", "*/*"), ("cookie", "session=old"), ] { assert_eq!( @@ -576,6 +577,7 @@ fn profile() -> CodexWireProfileState { os_version: "6.8".to_owned(), arch: "x86_64".to_owned(), terminal: "xterm".to_owned(), + residency: None, verified_at: Utc .with_ymd_and_hms(2026, 7, 18, 0, 0, 0) .single() @@ -585,6 +587,7 @@ fn profile() -> CodexWireProfileState { fn context() -> CodexRequestContext<'static> { CodexRequestContext { + trace: None, authorization: "Bearer oauth-access", account_id: Some("acct_123"), request_id: "req_usage_limit", diff --git a/backend/crates/providers/openai/tests/transport/websocket/mod.rs b/backend/crates/providers/openai/tests/transport/websocket/mod.rs index ad93e99e7..39c8a558b 100644 --- a/backend/crates/providers/openai/tests/transport/websocket/mod.rs +++ b/backend/crates/providers/openai/tests/transport/websocket/mod.rs @@ -2150,6 +2150,7 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_ .create_response( &request, CodexRequestContext { + trace: None, authorization: "Bearer access-token", account_id: Some("chatgpt-account"), request_id: "req_ws_client", @@ -2190,3 +2191,90 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_ .any(|(name, value)| name == "x-ratelimit-remaining-requests" && value == "17") ); } + +#[tokio::test] +async fn diagnostics_capture_metadata_and_unknown_events_before_normal_close() { + use gateway_core::diagnostics::TraceContext; + use tokio_tungstenite::tungstenite::protocol::{CloseFrame, frame::coding::CloseCode}; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut websocket = accept_codex_test_websocket(stream).await; + websocket.next().await.unwrap().unwrap(); + for event in [ + json!({"type": "future.metadata", "detail": "private-data"}), + json!({"type": "codex.response.metadata", "headers": { + "x-request-id": "upstream-metadata-request", "x-codex-turn-state": "opaque-turn-secret", + }}), + ] { + websocket + .send(Message::Text(event.to_string().into())) + .await + .unwrap(); + } + websocket + .close(Some(CloseFrame { + code: CloseCode::Normal, + reason: "".into(), + })) + .await + .unwrap(); + }); + let trace = TraceContext::new("req_metadata_close"); + let attempt = trace.attempt(2); + let mut request = codex_request("gpt-5.5", "be brief", Vec::new()); + request.set_previous_response_id(Some("resp_previous".to_owned())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); + request.force_http_sse = false; + let backend = CodexBackendClient::new( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{addr}"), + test_wire_profile(), + ); + let result = backend + .create_response_stream( + &request, + request_context("req_metadata_close", Some("chatgpt-account")).with_trace(&attempt), + ) + .await; + let mut failed = result.is_err(); + if let Ok(response) = result { + let mut body = response.body; + while let Some(chunk) = body.next().await { + if chunk.is_err() { + failed = true; + break; + } + } + } + assert!(failed, "close before terminal must fail the exchange"); + server.await.unwrap(); + let snapshot = trace.snapshot().unwrap(); + let events = snapshot["events"].as_array().unwrap(); + let metadata = events + .iter() + .find(|event| event["data"]["eventType"] == "codex.response.metadata") + .unwrap(); + assert_eq!(metadata["attemptIndex"], 2); + assert_eq!(metadata["exchangeId"], 1); + assert_eq!( + metadata["data"]["metadata"]["headers"]["x-request-id"], + "upstream-metadata-request" + ); + assert!( + events + .iter() + .any(|event| event["data"]["eventType"] == "future.metadata") + ); + let close = events + .iter() + .find(|event| event["stage"] == "upstream.close") + .unwrap(); + assert_eq!(close["data"]["code"], 1000); + assert_eq!(close["data"]["lastEventType"], "codex.response.metadata"); + assert_eq!(close["data"]["terminalSeen"], false); + assert!(!snapshot.to_string().contains("opaque-turn-secret")); + assert!(!snapshot.to_string().contains("private-data")); +} diff --git a/backend/crates/providers/xai/src/provider/stream.rs b/backend/crates/providers/xai/src/provider/stream.rs index 308dae6ab..f359ddd62 100644 --- a/backend/crates/providers/xai/src/provider/stream.rs +++ b/backend/crates/providers/xai/src/provider/stream.rs @@ -115,7 +115,7 @@ pub(super) fn cold_compaction_http_sse_stream( headers.clone(), body, session.binding().clone(), - ); + ).with_trace(context.trace()); let Some(handshake_deadline) = remaining(context.deadline()) else { Err(mark_transient_compaction_failure(provider_error( ProviderErrorKind::Timeout, @@ -403,7 +403,7 @@ pub(super) fn cold_http_sse_stream( headers.clone(), body, session.binding().clone(), - ); + ).with_trace(context.trace()); let Some(handshake_deadline) = remaining(context.deadline()) else { Err(provider_error( ProviderErrorKind::Timeout, diff --git a/backend/crates/providers/xai/src/transport/inference.rs b/backend/crates/providers/xai/src/transport/inference.rs index 790a12303..fb7f83a1d 100644 --- a/backend/crates/providers/xai/src/transport/inference.rs +++ b/backend/crates/providers/xai/src/transport/inference.rs @@ -14,6 +14,7 @@ use super::{GrokHeader, GrokSessionBinding}; /// 交给注入的 HTTP SSE transport 的自持有请求。 pub struct GrokInferenceRequest { + trace: gateway_core::diagnostics::TraceContext, endpoint: Url, headers: Vec, body: Zeroizing>, @@ -28,6 +29,7 @@ impl GrokInferenceRequest { binding: GrokSessionBinding, ) -> Self { Self { + trace: gateway_core::diagnostics::TraceContext::default(), endpoint, headers, body: Zeroizing::new(body), @@ -35,6 +37,16 @@ impl GrokInferenceRequest { } } + #[must_use] + pub fn with_trace(mut self, trace: gateway_core::diagnostics::TraceContext) -> Self { + self.trace = trace; + self + } + + pub(crate) fn trace(&self) -> &gateway_core::diagnostics::TraceContext { + &self.trace + } + /// 返回严格限定的官方 Responses 端点。 #[must_use] pub const fn endpoint(&self) -> &Url { diff --git a/backend/crates/providers/xai/src/transport/network.rs b/backend/crates/providers/xai/src/transport/network.rs index ed0aa0c9f..1b37bf05b 100644 --- a/backend/crates/providers/xai/src/transport/network.rs +++ b/backend/crates/providers/xai/src/transport/network.rs @@ -1,5 +1,6 @@ //! 关闭 redirect、proxy 与业务重试的生产 reqwest transport。 +use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext}; use std::collections::{HashMap, VecDeque}; use std::fmt; use std::io; @@ -371,6 +372,18 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport { UpstreamSendState::NotSent, )); } + let trace = request.trace().exchange("http_sse"); + trace.headers( + "upstream.request.headers", + serde_json::json!({ + "method": "POST", "endpoint": request.endpoint().path(), + }), + request + .headers() + .iter() + .map(|header| (header.name(), header.value().expose().as_bytes())), + ); + trace.capture("upstream.request.body", request.body()); let (client, client_cache_status) = self.client_for(request.binding())?; let mut builder = client .post(request.endpoint().clone()) @@ -393,8 +406,12 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport { Some(elapsed_millis(headers_started_at.elapsed())), &dns_observer, ); + trace.headers("upstream.response.headers", serde_json::json!({ + "status": response.status().as_u16(), "httpVersion": format!("{:?}", response.version()), + "headersMs": elapsed_millis(headers_started_at.elapsed()), + }), response.headers().iter().map(|(name, value)| (name.as_str(), value.as_bytes()))); if !response.status().is_success() { - return Err(classify_inference_status(response) + return Err(classify_inference_status(response, &trace) .await .with_transport_metrics(transport_metrics)); } @@ -424,6 +441,20 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport { }; std::future::ready(Some(item)) }); + let body = async_stream::stream! { + let mut body = Box::pin(body); + let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse); + while let Some(chunk) = body.next().await { + match &chunk { + Ok(bytes) => capture.push(bytes), + Err(_) => trace.record("upstream.read.failed", serde_json::json!({"phase": "response_body"})), + } + let failed = chunk.is_err(); + yield chunk; + if failed { return; } + } + capture.finish(); + }; Ok( GrokInferenceResponse::new(Box::pin(body), http_version, status_code, request_id) .with_transport_metrics(transport_metrics), @@ -981,7 +1012,10 @@ fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTrans ) } -async fn classify_inference_status(response: Response) -> GrokInferenceTransportError { +async fn classify_inference_status( + response: Response, + trace: &TraceContext, +) -> GrokInferenceTransportError { let status = response.status(); let retry_after = retry_after(&response); let http_version = upstream_http_version(response.version()); @@ -989,8 +1023,15 @@ async fn classify_inference_status(response: Response) -> GrokInferenceTransport let status_code = status.as_u16(); let body = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await { Ok(BoundedBody::Body(body)) => body, - Ok(BoundedBody::TooLarge) | Err(_) => Vec::new(), + Ok(BoundedBody::TooLarge) | Err(_) => { + trace.record( + "capture.gap", + serde_json::json!({"reason": "error_body_unavailable_or_too_large"}), + ); + Vec::new() + } }; + trace.capture("upstream.error.body", &body); let metadata = inference_error_metadata(&body); let body_failure = classify_grok_body_failure(&metadata, &body); let kind = match status { diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 8cee48305..fdbd742db 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -4,3 +4,4 @@ 22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql 194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql 3b4bfd5e3f5135fa2fad1088b0c2a287861e1250d66af31dbd40dc890751b567 0006_openai_session_transport_recovery.sql +8c0ec7fab80d95dfd5a370c70706c36ac5907bb5afea0cf1886196f21fd8317d 0007_request_diagnostics.sql diff --git a/backend/migrations/0007_request_diagnostics.sql b/backend/migrations/0007_request_diagnostics.sql new file mode 100644 index 000000000..a926cae6a --- /dev/null +++ b/backend/migrations/0007_request_diagnostics.sql @@ -0,0 +1,5 @@ +-- Bounded request evidence is retained with its owning model request. +-- NULL identifies historical requests for which no trace was captured. +alter table model_requests add column diagnostic_trace_json jsonb; +alter table model_requests add constraint model_requests_diagnostic_trace_object + check (diagnostic_trace_json is null or jsonb_typeof(diagnostic_trace_json) = 'object'); diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 05a0ac37f..904599851 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -20,7 +20,7 @@ host: logging: level: 'info' stdout: true - # 将完整请求头和正文写入独立文件；包含凭据，仅排障时开启。 + # 将完整请求与响应头、正文写入独立文件；包含凭据，仅排障时开启。 request_dump: false file: enabled: true @@ -65,14 +65,16 @@ openai: # 版本元组来自同一个官方 macOS Desktop ZIP；Provider 每 24 小时检查 appcast， # 仅在包内 Core 核验和 Redis 条件覆盖都成功后原子更新。 originator: 'Codex Desktop' - codex_version: '0.151.0-alpha.7.2' - desktop_version: '26.825.51511' - desktop_build: '7377' + codex_version: '0.153.4' + desktop_version: '26.901.51231' + desktop_build: '8109' os_type: 'Mac OS' os_version: '15.7.1' arch: 'arm64' terminal: 'unknown' - verified_at: '2026-08-31T00:00:00+08:00' + # 部署选择的 US residency；省略或 null 时使用官方无地域约束的默认分支。 + residency: us + verified_at: '2026-09-06T11:26:12.084+08:00' xai: # OAuth 端点、额度与模型目录策略由 xAI Provider 固定管理；这里只声明请求画像启动基线。 diff --git a/docs/api.md b/docs/api.md index f90ede831..4ef66d672 100644 --- a/docs/api.md +++ b/docs/api.md @@ -567,6 +567,11 @@ request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可 汇总与洞察中的请求数与 outcome 分布覆盖筛选范围内全部请求；token、缓存、延迟与成本聚合仅统计 已完整交付客户端的成功响应。 +详情接口按 `id` 可读取成功、失败或未完成请求。新增 `trace`（历史未采集记录为 `null`）和 +`relatedRequests[]`（`requestId / relation / outcome / completedAt`）；`relation` 为 `recovered_by` 或 +`recovers`。`trace` 是执行终态时的有界脱敏时间线，包含 request、attempt 和 exchange 关联、阶段、 +事件摘要及淘汰计数；普通用量列表不携带此字段。 + 错误记录中的“已自动恢复”表示系统关联到了后续成功请求，不会把原来的失败记录改为成功。 `upstreamSendState = ambiguous` 表示无法确认该次上游执行结果，不代表后续恢复请求失败； 恢复关联也不等于逐字节验证过两次请求正文。 diff --git a/frontend/src/api/modules/usage.ts b/frontend/src/api/modules/usage.ts index 35d6f4d97..a63e2cdb4 100644 --- a/frontend/src/api/modules/usage.ts +++ b/frontend/src/api/modules/usage.ts @@ -186,7 +186,39 @@ export interface UsageAttempt { completedAt: string | null } +export interface RequestTraceEvent { + sequence: number + lastSequence: number + elapsedMs: number + lastElapsedMs: number + attemptIndex: number + exchangeId: number | null + stage: string + count: number + data: Record +} + +export interface RequestTrace { + schemaVersion: number + requestId: string + startedAtUnixMs: number + totalEvents: number + droppedEvents: number + maxEvents: number + captureMode: string + events: RequestTraceEvent[] +} + +export interface RelatedRequest { + requestId: string + relation: 'recovered_by' | 'recovers' + outcome: string + completedAt: string | null +} + export type UsageRecordDetail = UsageRecord & { + trace: RequestTrace | null + relatedRequests: RelatedRequest[] attempts: UsageAttempt[] /** 尝试列表是否完整；best-effort 下恒为 false。 */ attemptsComplete: boolean diff --git a/frontend/src/views/usage/components/OpsErrorDetailModal.vue b/frontend/src/views/usage/components/OpsErrorDetailModal.vue index e29f36201..95ec24ce5 100644 --- a/frontend/src/views/usage/components/OpsErrorDetailModal.vue +++ b/frontend/src/views/usage/components/OpsErrorDetailModal.vue @@ -5,6 +5,7 @@ import { computed } from 'vue' import BaseButton from '@/components/base/BaseButton.vue' import BaseModal from '@/components/base/BaseModal/index.vue' import { failureClassText } from '../utils/opsErrorPresentation' +import RequestDiagnosticsPanel from './RequestDiagnosticsPanel.vue' import UsageDetailCodePanel from './UsageDetailCodePanel.vue' import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue' @@ -130,6 +131,7 @@ function visibleFields(items: DetailField[]) { size="xl" >

错误 @@ -142,7 +144,7 @@ function visibleFields(items: DetailField[]) {
diff --git a/frontend/src/views/usage/components/RequestDiagnosticsPanel.vue b/frontend/src/views/usage/components/RequestDiagnosticsPanel.vue new file mode 100644 index 000000000..f3e0163dd --- /dev/null +++ b/frontend/src/views/usage/components/RequestDiagnosticsPanel.vue @@ -0,0 +1,110 @@ + + + diff --git a/frontend/src/views/usage/components/UsageRecordDetailModal.vue b/frontend/src/views/usage/components/UsageRecordDetailModal.vue index f04731a1a..04a2a63cb 100644 --- a/frontend/src/views/usage/components/UsageRecordDetailModal.vue +++ b/frontend/src/views/usage/components/UsageRecordDetailModal.vue @@ -26,6 +26,7 @@ import { visibleRequestText, visibleResponseText, } from '../utils/records' +import RequestDiagnosticsPanel from './RequestDiagnosticsPanel.vue' import UsageDetailCodePanel from './UsageDetailCodePanel.vue' import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue' import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue' @@ -470,6 +471,8 @@ const tokenDonutOption = computed(() => {
+ +