mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
feat: add request diagnostics and align Codex transport contracts
This commit is contained in:
@@ -33,6 +33,8 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Rust 1.97 编译 AWS SDK 调试信息需要更大的 codegen 线程栈。
|
||||
RUST_MIN_STACK: 16777216
|
||||
CPR_TEST_DATABASE_URL: postgres://codex_proxy:codex_proxy@127.0.0.1:5432/codex_proxy
|
||||
CPR_TEST_REDIS_URL: redis://127.0.0.1:6379
|
||||
services:
|
||||
|
||||
Generated
+49
-44
@@ -154,9 +154,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.17.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
|
||||
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"untrusted 0.7.1",
|
||||
@@ -165,15 +165,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.42.0"
|
||||
version = "0.39.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
|
||||
checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -195,7 +194,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.0.1",
|
||||
"percent-encoding",
|
||||
@@ -231,7 +230,7 @@ dependencies = [
|
||||
"hex",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"lru",
|
||||
"percent-encoding",
|
||||
@@ -257,7 +256,7 @@ dependencies = [
|
||||
"hex",
|
||||
"hmac",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"percent-encoding",
|
||||
"sha2 0.11.0",
|
||||
"time",
|
||||
@@ -286,7 +285,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"md-5",
|
||||
@@ -320,7 +319,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"percent-encoding",
|
||||
@@ -339,7 +338,7 @@ dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"h2",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
@@ -389,7 +388,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
@@ -410,7 +409,7 @@ dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -436,7 +435,7 @@ checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -450,7 +449,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
@@ -503,7 +502,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -535,7 +534,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
@@ -1478,12 +1477,14 @@ name = "gateway-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"thiserror",
|
||||
"tracing",
|
||||
"uuid",
|
||||
@@ -1639,7 +1640,7 @@ dependencies = [
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"indexmap",
|
||||
"slab",
|
||||
"tokio",
|
||||
@@ -1687,7 +1688,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"headers-core",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"httpdate",
|
||||
"mime",
|
||||
"sha1 0.10.7",
|
||||
@@ -1699,7 +1700,7 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
|
||||
dependencies = [
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1761,9 +1762,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.2"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
|
||||
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
@@ -1787,7 +1788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1798,7 +1799,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"pin-project-lite",
|
||||
]
|
||||
@@ -1834,21 +1835,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.10.1"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
|
||||
checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"want",
|
||||
@@ -1860,7 +1862,7 @@ version = "0.27.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
|
||||
dependencies = [
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"log",
|
||||
@@ -1882,7 +1884,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"hyper",
|
||||
"ipnet",
|
||||
@@ -2594,6 +2596,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
@@ -2605,6 +2608,7 @@ dependencies = [
|
||||
"gateway-protocol",
|
||||
"getrandom 0.4.3",
|
||||
"hex",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
@@ -2620,6 +2624,7 @@ dependencies = [
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-tungstenite 0.28.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
@@ -2911,7 +2916,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
@@ -3021,9 +3026,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.8.4"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
|
||||
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pki-types",
|
||||
@@ -3033,9 +3038,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.15.1"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
||||
dependencies = [
|
||||
"web-time",
|
||||
"zeroize",
|
||||
@@ -3699,7 +3704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -3800,9 +3805,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.1"
|
||||
version = "1.52.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
@@ -3912,7 +3917,7 @@ dependencies = [
|
||||
"bitflags",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"pin-project-lite",
|
||||
"tower",
|
||||
@@ -3931,7 +3936,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"http-range-header",
|
||||
@@ -4063,7 +4068,7 @@ dependencies = [
|
||||
"data-encoding",
|
||||
"flate2",
|
||||
"headers",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.4",
|
||||
@@ -4082,7 +4087,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.4",
|
||||
@@ -4332,9 +4337,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.8"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
|
||||
checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
@@ -4536,7 +4541,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"deadpool",
|
||||
"futures",
|
||||
"http 1.4.2",
|
||||
"http 1.4.0",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
|
||||
@@ -143,10 +143,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"codex_version: '0.151.0-alpha.7.2'",
|
||||
"codex_version: 'latest'",
|
||||
));
|
||||
assert_rejected(valid_config().replace("codex_version: '0.153.4'", "codex_version: 'latest'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -165,7 +162,7 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
assert_rejected(valid_config().replace("desktop_build: '7377'", "desktop_build: 'build'"));
|
||||
assert_rejected(valid_config().replace("desktop_build: '8109'", "desktop_build: 'build'"));
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -704,6 +704,8 @@ pub struct UsageAttempt {
|
||||
/// 一条请求及其全部尝试。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageDetail {
|
||||
pub trace: Option<serde_json::Value>,
|
||||
pub related_requests: Vec<serde_json::Value>,
|
||||
pub request: UsageRecord,
|
||||
pub attempts: Vec<UsageAttempt>,
|
||||
}
|
||||
|
||||
@@ -505,6 +505,8 @@ pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptV
|
||||
|
||||
pub(crate) fn usage_detail_view(detail: domain::UsageDetail) -> UsageRecordDetailView {
|
||||
UsageRecordDetailView {
|
||||
trace: detail.trace,
|
||||
related_requests: detail.related_requests,
|
||||
request: usage_record_view(detail.request),
|
||||
attempts: detail
|
||||
.attempts
|
||||
|
||||
@@ -247,6 +247,8 @@ pub struct UsageAttemptView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordDetailView {
|
||||
pub trace: Option<serde_json::Value>,
|
||||
pub related_requests: Vec<serde_json::Value>,
|
||||
#[serde(flatten)]
|
||||
pub request: UsageRecordView,
|
||||
pub attempts: Vec<UsageAttemptView>,
|
||||
|
||||
@@ -180,13 +180,16 @@ pub fn initialize(
|
||||
"request",
|
||||
request_id = %request_id,
|
||||
method = %request.method().as_str(),
|
||||
uri = %request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(axum::http::uri::PathAndQuery::as_str)
|
||||
.unwrap_or_default(),
|
||||
uri = %request.uri().path(),
|
||||
)
|
||||
}
|
||||
})
|
||||
.on_request(|_request: &Request<_>, _span: &tracing::Span| {
|
||||
tracing::info!(target: "request_trace", stage = "http.received", "HTTP request received");
|
||||
})
|
||||
.on_response(|response: &axum::http::Response<_>, latency: Duration, _span: &tracing::Span| {
|
||||
tracing::info!(target: "request_trace", stage = "http.response", status = response.status().as_u16(),
|
||||
headers_ms = latency.as_millis(), "HTTP response headers ready");
|
||||
});
|
||||
let router = router
|
||||
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
|
||||
|
||||
@@ -14,6 +14,7 @@ use axum::{
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::stream;
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
use gateway_core::engine::execution::{ClientTransport, ExecutionSession, StartedExecution};
|
||||
use gateway_core::engine::{CommitRequirement, EngineError};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
@@ -39,6 +40,7 @@ use super::{
|
||||
pub(crate) async fn responses(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
ingress_id: Option<Extension<tower_http::request_id::RequestId>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
@@ -83,28 +85,25 @@ pub(crate) async fn responses(
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
request_id = %started.request_id,
|
||||
request_direction = "downstream",
|
||||
request_transport = if streaming { "http_sse" } else { "http_json" },
|
||||
request_method = "POST",
|
||||
request_path = "/v1/responses",
|
||||
request_headers = ?headers,
|
||||
request_body = %String::from_utf8_lossy(&body),
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
let trace = started.session.trace();
|
||||
trace.headers("client.request", serde_json::json!({
|
||||
"ingressRequestId": ingress_id.as_ref().and_then(|Extension(id)| id.header_value().to_str().ok()),
|
||||
}), headers.iter().map(|(name, value)| (name.as_str(), value.as_bytes())));
|
||||
trace.capture("client.request.body", &body);
|
||||
let request_id_header = HeaderValue::from_str(started.request_id.as_str()).ok();
|
||||
let StartedExecution {
|
||||
stream, session, ..
|
||||
} = started;
|
||||
if stream {
|
||||
let mut response = if stream {
|
||||
stream_execution_response(session, connection_guard).await
|
||||
} else {
|
||||
drop(connection_guard);
|
||||
collect_execution_response(session).await
|
||||
};
|
||||
if let Some(id) = request_id_header {
|
||||
response.headers_mut().insert("x-gateway-request-id", id);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// 从 socket 与标准转发头提取旧 Usage 页面使用的诊断事实。
|
||||
@@ -182,6 +181,11 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
}
|
||||
};
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
session.trace().record(
|
||||
"downstream.encoded",
|
||||
serde_json::json!({"transport": "http_json", "bytes": encoded.len()}),
|
||||
);
|
||||
session.trace().dump("downstream.body", &encoded);
|
||||
let response = json_body_response(encoded, &response_headers);
|
||||
|
||||
let Some(session) = execution.session_mut() else {
|
||||
@@ -334,6 +338,8 @@ pub async fn stream_execution_response(
|
||||
let body = Body::from_stream(stream::unfold(state, |mut state| async move {
|
||||
loop {
|
||||
if let Some(chunk) = state.pending.pop_front() {
|
||||
state.trace.dump("downstream.chunk", &chunk);
|
||||
state.handed_off_bytes += chunk.len() as u64;
|
||||
return Some((Ok::<Bytes, Infallible>(chunk), state));
|
||||
}
|
||||
if state.output_finished {
|
||||
@@ -416,12 +422,18 @@ impl Drop for PendingExecution {
|
||||
if session.is_finalized() {
|
||||
return;
|
||||
}
|
||||
session.trace().record(
|
||||
"downstream.cancelled",
|
||||
serde_json::json!({"reason": "response_guard_dropped"}),
|
||||
);
|
||||
session.cancel();
|
||||
detach_finalize(session);
|
||||
}
|
||||
}
|
||||
|
||||
struct ResponsesStreamState {
|
||||
trace: TraceContext,
|
||||
handed_off_bytes: u64,
|
||||
session: Option<Box<dyn ExecutionSession>>,
|
||||
encoder: OpenAiResponsesEncoder,
|
||||
pending: VecDeque<Bytes>,
|
||||
@@ -438,6 +450,8 @@ impl ResponsesStreamState {
|
||||
connection_guard: Option<Box<dyn ConnectionGuard>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
trace: session.trace(),
|
||||
handed_off_bytes: 0,
|
||||
session: Some(session),
|
||||
encoder,
|
||||
pending: initial_frames.into_iter().collect(),
|
||||
@@ -593,6 +607,11 @@ impl ResponsesStreamState {
|
||||
|
||||
impl Drop for ResponsesStreamState {
|
||||
fn drop(&mut self) {
|
||||
self.trace.record("downstream.body.closed", serde_json::json!({
|
||||
"handedOffBytes": self.handed_off_bytes,
|
||||
"pendingChunks": self.pending.len(),
|
||||
"outputFinished": self.output_finished, "executionTerminal": self.execution_terminal,
|
||||
}));
|
||||
if self.execution_terminal {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -338,7 +338,12 @@ async fn next_active_input(
|
||||
// pump 继续接收有界业务帧和处理 Ping/Pong;退出通过独立通知取消本轮。
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = connection.wait_for_exit() => ActiveInput::Disconnect,
|
||||
_ = connection.wait_for_exit() => {
|
||||
session.trace().record("downstream.cancelled", serde_json::json!({
|
||||
"reason": "websocket_connection_exit", "connectionId": connection.id(),
|
||||
}));
|
||||
ActiveInput::Disconnect
|
||||
},
|
||||
event = session.next_event() => ActiveInput::Event(event),
|
||||
}
|
||||
}
|
||||
@@ -451,6 +456,7 @@ fn log_terminal_write_success(
|
||||
provider_terminal_to_write: std::time::Duration,
|
||||
) {
|
||||
tracing::info!(
|
||||
target: "request_trace", stage = "downstream.terminal.written",
|
||||
websocket_connection_id = connection.id(),
|
||||
request_id = %request_id,
|
||||
provider_terminal_to_terminal_write_ms = provider_terminal_to_write.as_millis(),
|
||||
@@ -465,6 +471,7 @@ fn log_terminal_write_failure(
|
||||
provider_terminal_to_write: std::time::Duration,
|
||||
) {
|
||||
tracing::info!(
|
||||
target: "request_trace", stage = "downstream.terminal.write_failed",
|
||||
websocket_connection_id = connection.id(),
|
||||
request_id = %request_id,
|
||||
provider_terminal_to_terminal_write_ms = provider_terminal_to_write.as_millis(),
|
||||
|
||||
@@ -19,6 +19,7 @@ use axum::{
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::{
|
||||
diagnostics::{TraceContext, diagnostic_json},
|
||||
engine::execution::{AuthenticatedClient, ClientTransport},
|
||||
lifecycle::{ConnectionGuard, ConnectionLifecycle},
|
||||
};
|
||||
@@ -98,17 +99,12 @@ impl ResponsesWebSocketAdapter {
|
||||
Err(_) => return runtime_unavailable_response().into_response(),
|
||||
};
|
||||
let connection_id = self.service.next_request_id().replacen("req_", "ws_", 1);
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
websocket_connection_id = %connection_id,
|
||||
request_direction = "downstream",
|
||||
request_transport = "websocket_handshake",
|
||||
request_method = "GET",
|
||||
request_path = "/v1/responses",
|
||||
request_headers = ?raw_headers,
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
TraceContext::new(&connection_id).headers(
|
||||
"client.connection.headers",
|
||||
serde_json::json!({"transport": "websocket", "method": "GET", "path": "/v1/responses"}),
|
||||
raw_headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
let session = ResponsesWebSocketSession {
|
||||
service: self.service.clone(),
|
||||
@@ -187,26 +183,15 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
}
|
||||
request_count = request_count.saturating_add(1);
|
||||
let correlation_id = Arc::<str>::from(service.next_request_id());
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
websocket_connection_id = connection.id(),
|
||||
correlation_id = %correlation_id,
|
||||
request_index = request_count,
|
||||
request_direction = "downstream",
|
||||
request_transport = "websocket",
|
||||
request_payload = %payload,
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
let decoded = match decode_response_create_with_context(&payload, &request_headers) {
|
||||
Ok(decoded) => decoded.with_client_context(client_ip, user_agent.clone()),
|
||||
Err(error) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = connection.id(),
|
||||
request_id = %correlation_id,
|
||||
error = %error,
|
||||
"Responses WebSocket request rejected"
|
||||
trace_rejected_request(
|
||||
&correlation_id,
|
||||
connection.id(),
|
||||
&payload,
|
||||
"decode",
|
||||
&error.to_string(),
|
||||
);
|
||||
if send_protocol_error(
|
||||
&mut connection,
|
||||
@@ -225,6 +210,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
let decoded = replay.prepare(decoded);
|
||||
// deadline 与 Text 可能同时就绪;在任何上游执行开始前再次封住该竞争窗口。
|
||||
if connection.is_expired() {
|
||||
trace_rejected_request(
|
||||
&correlation_id,
|
||||
connection.id(),
|
||||
&payload,
|
||||
"connection_expired",
|
||||
"",
|
||||
);
|
||||
expire_connection(&mut connection).await;
|
||||
break;
|
||||
}
|
||||
@@ -239,6 +231,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => {
|
||||
trace_rejected_request(
|
||||
&correlation_id,
|
||||
connection.id(),
|
||||
&payload,
|
||||
"start",
|
||||
&error.to_string(),
|
||||
);
|
||||
if send_gateway_error(&mut connection, &error, &correlation_id).await
|
||||
== ForwardOutcome::Disconnect
|
||||
{
|
||||
@@ -247,14 +246,14 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
continue;
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
websocket_connection_id = connection.id(),
|
||||
correlation_id = %correlation_id,
|
||||
request_id = %started.request_id,
|
||||
"request dump correlation"
|
||||
);
|
||||
started.session.trace().record("client.connection", serde_json::json!({
|
||||
"transport": "websocket", "connectionId": connection.id(), "correlationId": correlation_id,
|
||||
"requestIndex": request_count,
|
||||
}));
|
||||
started
|
||||
.session
|
||||
.trace()
|
||||
.capture("client.request.body", payload.as_bytes());
|
||||
|
||||
if forward_execution(&mut connection, started, &mut replay).await
|
||||
== ForwardOutcome::Disconnect
|
||||
@@ -270,6 +269,28 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
connection.log_summary(request_count);
|
||||
}
|
||||
|
||||
// 尚未建立模型请求的拒绝也保留原文入口,用返回给客户端的 correlation ID 检索。
|
||||
fn trace_rejected_request(
|
||||
correlation_id: &str,
|
||||
connection_id: &str,
|
||||
payload: &str,
|
||||
phase: &'static str,
|
||||
error: &str,
|
||||
) {
|
||||
let trace = TraceContext::new(correlation_id);
|
||||
trace.record(
|
||||
"client.connection",
|
||||
serde_json::json!({"connectionId": connection_id}),
|
||||
);
|
||||
trace.capture("client.request.body", payload.as_bytes());
|
||||
trace.record(
|
||||
"client.request.rejected",
|
||||
serde_json::json!({
|
||||
"phase": phase, "detail": diagnostic_json(&serde_json::json!({"message": error})),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async fn expire_connection(connection: &mut ResponsesWebSocketConnection) {
|
||||
if connection
|
||||
.send_text(
|
||||
|
||||
@@ -429,6 +429,8 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.replace(UsageDetail {
|
||||
trace: None,
|
||||
related_requests: Vec::new(),
|
||||
request: usage_record_with_account(
|
||||
"req_detail",
|
||||
"acct_snap_a",
|
||||
|
||||
@@ -13,11 +13,13 @@ path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1.89"
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
semver = "1.0.27"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
//! 诊断内容的统一筛选边界。未知字符串只保留长度和摘要。
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// 原始字节的长度与 SHA-256,不复制或保留正文。
|
||||
#[must_use]
|
||||
pub fn body_fingerprint(bytes: &[u8]) -> Value {
|
||||
let digest = Sha256::digest(bytes);
|
||||
let hex: String = digest
|
||||
.iter()
|
||||
.flat_map(|byte| {
|
||||
let alphabet = b"0123456789abcdef";
|
||||
[
|
||||
char::from(alphabet[usize::from(byte >> 4)]),
|
||||
char::from(alphabet[usize::from(byte & 15)]),
|
||||
]
|
||||
})
|
||||
.collect();
|
||||
json!({"bytes": bytes.len(), "sha256": hex})
|
||||
}
|
||||
|
||||
/// 保留可诊断的协议结构;凭据永不进入普通日志或数据库。
|
||||
#[must_use]
|
||||
pub fn diagnostic_json(value: &Value) -> Value {
|
||||
capture(value, "", 0, &mut 96)
|
||||
}
|
||||
|
||||
fn capture(value: &Value, key: &str, depth: usize, budget: &mut usize) -> Value {
|
||||
if *budget == 0 || depth > 6 {
|
||||
return json!({"omitted": true});
|
||||
}
|
||||
*budget -= 1;
|
||||
let key = key.to_ascii_lowercase();
|
||||
if secret_key(&key) {
|
||||
return Value::String("<redacted>".to_owned());
|
||||
}
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
let mut result = Map::new();
|
||||
for (name, value) in object.iter().take(48) {
|
||||
if *budget == 0 {
|
||||
break;
|
||||
}
|
||||
result.insert(bounded(name, 96), capture(value, name, depth + 1, budget));
|
||||
}
|
||||
if result.len() < object.len() {
|
||||
result.insert(
|
||||
"_omittedFields".to_owned(),
|
||||
json!(object.len() - result.len()),
|
||||
);
|
||||
}
|
||||
Value::Object(result)
|
||||
}
|
||||
Value::Array(items) => json!({
|
||||
"length": items.len(),
|
||||
"sample": items.iter().take(4)
|
||||
.map(|item| capture(item, key.as_str(), depth + 1, budget))
|
||||
.collect::<Vec<_>>()
|
||||
}),
|
||||
Value::String(text)
|
||||
if safe_header(&key) || safe_value_key(&key) && protocol_label(text) =>
|
||||
{
|
||||
Value::String(bounded(text, 256))
|
||||
}
|
||||
Value::String(text) => body_fingerprint(text.as_bytes()),
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求/响应头保持多值;安全 trace ID 原样保存,未知值只保存摘要。
|
||||
#[must_use]
|
||||
pub fn diagnostic_headers<'a>(headers: impl IntoIterator<Item = (&'a str, &'a str)>) -> Value {
|
||||
let mut result = Map::new();
|
||||
for (name, value) in headers.into_iter().take(64) {
|
||||
let name = bounded(&name.to_ascii_lowercase(), 96);
|
||||
let value = if secret_key(&name) {
|
||||
Value::String("<redacted>".to_owned())
|
||||
} else if safe_header(&name) {
|
||||
Value::String(bounded(value, 256))
|
||||
} else {
|
||||
body_fingerprint(value.as_bytes())
|
||||
};
|
||||
let values = result
|
||||
.entry(name)
|
||||
.or_insert_with(|| Value::Array(Vec::new()));
|
||||
if let Value::Array(values) = values {
|
||||
values.push(value);
|
||||
}
|
||||
}
|
||||
Value::Object(result)
|
||||
}
|
||||
|
||||
fn secret_key(key: &str) -> bool {
|
||||
key.contains("authorization")
|
||||
|| key.contains("cookie")
|
||||
|| key.contains("api_key")
|
||||
|| key.contains("api-key")
|
||||
|| key.contains("token") && !key.ends_with("tokens")
|
||||
|| key.contains("password")
|
||||
|| key.contains("secret")
|
||||
|| key.contains("attestation")
|
||||
|| matches!(key, "x-oai-is" | "x-oai-is-update")
|
||||
}
|
||||
|
||||
fn safe_header(key: &str) -> bool {
|
||||
matches!(
|
||||
key,
|
||||
"x-oai-request-id"
|
||||
| "x-client-request-id"
|
||||
| "x-request-id"
|
||||
| "request-id"
|
||||
| "openai-request-id"
|
||||
| "x-openai-request-id"
|
||||
| "cf-ray"
|
||||
| "traceparent"
|
||||
| "tracestate"
|
||||
| "content-type"
|
||||
| "content-length"
|
||||
| "content-encoding"
|
||||
| "retry-after"
|
||||
| "openai-processing-ms"
|
||||
| "x-processing-ms"
|
||||
| "date"
|
||||
| "server"
|
||||
| "sec-websocket-extensions"
|
||||
| "x-codex-allowed"
|
||||
| "x-codex-limit-reached"
|
||||
| "x-codex-active-limit"
|
||||
| "x-models-etag"
|
||||
)
|
||||
}
|
||||
|
||||
fn safe_value_key(key: &str) -> bool {
|
||||
safe_header(key)
|
||||
|| matches!(
|
||||
key,
|
||||
"type" | "event" | "status" | "code" | "model" | "role" | "object" | "service_tier"
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn bounded(value: &str, limit: usize) -> String {
|
||||
value
|
||||
.chars()
|
||||
.take(limit)
|
||||
.map(|c| if c.is_control() { ' ' } else { c })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Event names/codes are protocol labels, never arbitrary upstream text.
|
||||
pub(super) fn protocol_label(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 128
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'))
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! 有界、请求局部的诊断证据;不参与路由、重试或客户端协议决策。
|
||||
|
||||
mod capture;
|
||||
mod stream;
|
||||
mod trace;
|
||||
|
||||
pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json};
|
||||
pub use stream::{StreamCapture, StreamFormat};
|
||||
pub use trace::TraceContext;
|
||||
@@ -0,0 +1,165 @@
|
||||
//! 仅用于观测的有界流分帧;超过限额时明确记缺口,不影响实际协议解析。
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::TraceContext;
|
||||
|
||||
const MAX_FRAME_BYTES: usize = 128 * 1024;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum StreamFormat {
|
||||
Sse,
|
||||
JsonLines,
|
||||
}
|
||||
|
||||
/// 同一 exchange 持有一个 observer,跨 HTTP chunk 拼接完整事件。
|
||||
pub struct StreamCapture {
|
||||
trace: TraceContext,
|
||||
format: StreamFormat,
|
||||
line: Vec<u8>,
|
||||
data: Vec<u8>,
|
||||
event_name: Option<String>,
|
||||
previous_cr: bool,
|
||||
first_line: bool,
|
||||
skipping_line: bool,
|
||||
skipping_event: bool,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl StreamCapture {
|
||||
#[must_use]
|
||||
pub fn new(trace: TraceContext, format: StreamFormat) -> Self {
|
||||
Self {
|
||||
trace,
|
||||
format,
|
||||
line: Vec::new(),
|
||||
data: Vec::new(),
|
||||
event_name: None,
|
||||
previous_cr: false,
|
||||
first_line: true,
|
||||
skipping_line: false,
|
||||
skipping_event: false,
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push(&mut self, bytes: &[u8]) {
|
||||
self.trace.dump("upstream.chunk", bytes);
|
||||
for byte in bytes {
|
||||
if self.previous_cr && *byte == b'\n' {
|
||||
self.previous_cr = false;
|
||||
continue;
|
||||
}
|
||||
self.previous_cr = *byte == b'\r';
|
||||
if matches!(*byte, b'\n' | b'\r') {
|
||||
if !self.skipping_line {
|
||||
self.line();
|
||||
}
|
||||
self.line.clear();
|
||||
self.skipping_line = false;
|
||||
} else if !self.skipping_line {
|
||||
if self.line.len() < MAX_FRAME_BYTES {
|
||||
self.line.push(*byte);
|
||||
} else {
|
||||
self.trace.record(
|
||||
"capture.gap",
|
||||
json!({"reason": "line_size_limit", "limitBytes": MAX_FRAME_BYTES}),
|
||||
);
|
||||
self.line.clear();
|
||||
self.skipping_line = true;
|
||||
self.skipping_event = matches!(self.format, StreamFormat::Sse);
|
||||
self.data.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
let partial_frame = !self.line.is_empty()
|
||||
|| !self.data.is_empty()
|
||||
|| self.skipping_event
|
||||
|| self.skipping_line;
|
||||
if !self.line.is_empty() && !self.skipping_line {
|
||||
self.line();
|
||||
}
|
||||
if !self.data.is_empty() && !self.skipping_event {
|
||||
self.trace.capture_named_event(
|
||||
"upstream.event",
|
||||
&self.data,
|
||||
self.event_name.as_deref(),
|
||||
);
|
||||
}
|
||||
self.trace
|
||||
.record("upstream.eof", json!({"partialFrame": partial_frame}));
|
||||
self.finished = true;
|
||||
}
|
||||
|
||||
fn line(&mut self) {
|
||||
let line = if self.first_line {
|
||||
self.first_line = false;
|
||||
self.line
|
||||
.strip_prefix(b"\xef\xbb\xbf")
|
||||
.unwrap_or(&self.line)
|
||||
} else {
|
||||
&self.line
|
||||
};
|
||||
if matches!(self.format, StreamFormat::JsonLines) {
|
||||
if !line.is_empty() {
|
||||
self.trace.capture_event("upstream.event", line);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if line.is_empty() {
|
||||
if !self.data.is_empty() && !self.skipping_event {
|
||||
self.trace.capture_named_event(
|
||||
"upstream.event",
|
||||
&self.data,
|
||||
self.event_name.as_deref(),
|
||||
);
|
||||
}
|
||||
self.data.clear();
|
||||
self.event_name = None;
|
||||
self.skipping_event = false;
|
||||
} else if let Some(name) = line.strip_prefix(b"event:") {
|
||||
self.event_name = Some(super::capture::bounded(
|
||||
String::from_utf8_lossy(name).trim(),
|
||||
128,
|
||||
));
|
||||
} else if let Some(data) = line.strip_prefix(b"data:") {
|
||||
let data = data.strip_prefix(b" ").unwrap_or(data);
|
||||
if self.skipping_event {
|
||||
return;
|
||||
}
|
||||
if self.data.len() + data.len() + 1 > MAX_FRAME_BYTES {
|
||||
self.trace.record(
|
||||
"capture.gap",
|
||||
json!({"reason": "event_size_limit", "limitBytes": MAX_FRAME_BYTES}),
|
||||
);
|
||||
self.data.clear();
|
||||
self.skipping_event = true;
|
||||
} else {
|
||||
if !self.data.is_empty() {
|
||||
self.data.push(b'\n');
|
||||
}
|
||||
self.data.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StreamCapture {
|
||||
fn drop(&mut self) {
|
||||
if !self.finished {
|
||||
self.trace.record(
|
||||
"upstream.reader.dropped",
|
||||
json!({
|
||||
"partialBytes": self.line.len() + self.data.len(),
|
||||
"captureIncomplete": self.skipping_event || self.skipping_line,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
//! 首部与最近事件同时保留,流式 delta 合并,所有淘汰均有计数。
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::capture::bounded;
|
||||
use super::{body_fingerprint, diagnostic_headers, diagnostic_json};
|
||||
|
||||
const MAX_EVENTS: usize = 128;
|
||||
const HEAD_EVENTS: usize = 8;
|
||||
const MAX_DATA_BYTES: usize = 4096;
|
||||
const MAX_BUFFER_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// 显式传给异步任务的关联上下文;默认值禁用捕获,避免全局请求映射。
|
||||
#[derive(Clone, Default)]
|
||||
pub struct TraceContext {
|
||||
state: Option<Arc<Mutex<TraceState>>>,
|
||||
attempt_index: u32,
|
||||
exchange_id: Option<u64>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for TraceContext {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("TraceContext")
|
||||
.field("attempt_index", &self.attempt_index)
|
||||
.field("exchange_id", &self.exchange_id)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
struct TraceState {
|
||||
request_id: String,
|
||||
started_at_ms: u128,
|
||||
started: Instant,
|
||||
sequence: u64,
|
||||
exchanges: u64,
|
||||
wire_frames: u64,
|
||||
wire_bytes: u64,
|
||||
dropped_events: u64,
|
||||
bytes: usize,
|
||||
events: VecDeque<TraceEvent>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct TraceEvent {
|
||||
sequence: u64,
|
||||
last_sequence: u64,
|
||||
elapsed_ms: u64,
|
||||
last_elapsed_ms: u64,
|
||||
attempt_index: u32,
|
||||
exchange_id: Option<u64>,
|
||||
stage: &'static str,
|
||||
count: u64,
|
||||
data: Value,
|
||||
#[serde(skip)]
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl TraceContext {
|
||||
/// 一个模型请求只创建一次,所有 attempt 共享同一有界时间线。
|
||||
#[must_use]
|
||||
pub fn new(request_id: &str) -> Self {
|
||||
Self {
|
||||
state: Some(Arc::new(Mutex::new(TraceState {
|
||||
request_id: bounded(request_id, 128),
|
||||
started_at_ms: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis(),
|
||||
started: Instant::now(),
|
||||
sequence: 0,
|
||||
exchanges: 0,
|
||||
wire_frames: 0,
|
||||
wire_bytes: 0,
|
||||
dropped_events: 0,
|
||||
bytes: 0,
|
||||
events: VecDeque::new(),
|
||||
}))),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn attempt(&self, attempt_index: u32) -> Self {
|
||||
Self {
|
||||
state: self.state.clone(),
|
||||
attempt_index,
|
||||
exchange_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 一个 attempt 中的每次实际 transport exchange 也有独立编号。
|
||||
#[must_use]
|
||||
pub fn exchange(&self, transport: &'static str) -> Self {
|
||||
let mut context = self.clone();
|
||||
if let Some(state) = &self.state {
|
||||
let mut state = state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.exchanges += 1;
|
||||
context.exchange_id = Some(state.exchanges);
|
||||
}
|
||||
context.record("upstream.exchange.started", json!({"transport": transport}));
|
||||
context
|
||||
}
|
||||
|
||||
/// details 只允许调用方构造的诊断事实;不接收原始请求/响应正文。
|
||||
pub fn record(&self, stage: &'static str, data: Value) {
|
||||
self.push(stage, data, false);
|
||||
}
|
||||
|
||||
/// 同一个头部边界只采集一次:默认脱敏,显式开启 dump 时另存完整字节。
|
||||
/// facts 必须为调用方构造的安全对象;原始头值只进入独立 dump。
|
||||
pub fn headers<'a>(
|
||||
&self,
|
||||
stage: &'static str,
|
||||
mut facts: Value,
|
||||
headers: impl IntoIterator<Item = (&'a str, &'a [u8])>,
|
||||
) {
|
||||
if self.state.is_none() {
|
||||
return;
|
||||
}
|
||||
let Some(facts_object) = facts.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
let headers: Vec<_> = headers.into_iter().collect();
|
||||
let summary = diagnostic_headers(
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| (*name, std::str::from_utf8(value).unwrap_or("<binary>"))),
|
||||
);
|
||||
facts_object.insert("headers".to_owned(), summary);
|
||||
self.record(stage, facts);
|
||||
if tracing::enabled!(target: "request_dump", tracing::Level::INFO) {
|
||||
let values: Vec<_> = headers
|
||||
.iter()
|
||||
.map(|(name, value)| json!({"name": name, "valueBase64": STANDARD.encode(value)}))
|
||||
.collect();
|
||||
if let Ok(bytes) = serde_json::to_vec(&json!({"headers": values})) {
|
||||
self.dump(stage, &bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 入站 JSON 先记录再解析业务语义,包括未知事件和 metadata。
|
||||
/// 原文仅在现有 request_dump 开关打开时写入其独立文件。
|
||||
pub fn capture(&self, stage: &'static str, bytes: &[u8]) {
|
||||
self.dump(stage, bytes);
|
||||
self.capture_event(stage, bytes);
|
||||
}
|
||||
|
||||
/// StreamCapture 已转储原始 chunk 时,只提取完整事件的安全摘要。
|
||||
pub fn capture_event(&self, stage: &'static str, bytes: &[u8]) {
|
||||
self.capture_named_event(stage, bytes, None);
|
||||
}
|
||||
|
||||
pub fn capture_named_event(&self, stage: &'static str, bytes: &[u8], name: Option<&str>) {
|
||||
if self.state.is_none() {
|
||||
return;
|
||||
}
|
||||
let fingerprint = body_fingerprint(bytes);
|
||||
if bytes.len() > 128 * 1024 {
|
||||
self.record("capture.gap", json!({"reason": "json_inspection_size_limit", "stage": stage, "body": fingerprint}));
|
||||
return;
|
||||
}
|
||||
let parsed = serde_json::from_slice::<Value>(bytes).ok();
|
||||
let event_type = parsed
|
||||
.as_ref()
|
||||
.and_then(|v| v.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
.or(name)
|
||||
.filter(|kind| super::capture::protocol_label(kind));
|
||||
let delta = event_type.is_some_and(|kind| kind.ends_with(".delta"));
|
||||
let data = json!({
|
||||
"body": fingerprint,
|
||||
"eventType": event_type.map(|kind| bounded(kind, 128)),
|
||||
"jsonValid": parsed.is_some(),
|
||||
"metadata": parsed.as_ref().filter(|_| !delta).map(diagnostic_json),
|
||||
});
|
||||
self.push(stage, data, delta);
|
||||
}
|
||||
|
||||
/// 只记录业务解析边界;metadata 已由 transport 在解析前捕获。
|
||||
pub fn wire_event(&self, protocol: &str, event_type: Option<&str>) {
|
||||
let event_type = event_type.filter(|kind| super::capture::protocol_label(kind));
|
||||
let delta = event_type.is_some_and(|kind| kind.ends_with(".delta"));
|
||||
self.push(
|
||||
"provider.event",
|
||||
json!({
|
||||
"protocol": protocol,
|
||||
"eventType": event_type.map(|kind| bounded(kind, 128)),
|
||||
}),
|
||||
delta,
|
||||
);
|
||||
}
|
||||
|
||||
/// 完整报文捕获与摘要使用相同 request / attempt / exchange ID。
|
||||
pub fn dump(&self, stage: &'static str, bytes: &[u8]) {
|
||||
if !tracing::enabled!(target: "request_dump", tracing::Level::INFO) {
|
||||
return;
|
||||
}
|
||||
if let Some(state) = &self.state {
|
||||
let mut state = state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.wire_frames += 1;
|
||||
state.wire_bytes = state.wire_bytes.saturating_add(bytes.len() as u64);
|
||||
let wire_sequence = state.wire_frames;
|
||||
let chunk_count = bytes.len().div_ceil(32 * 1024).max(1);
|
||||
// Each log entry stays bounded even for images or very large WebSocket frames.
|
||||
for chunk_index in 0..chunk_count {
|
||||
let start = chunk_index * 32 * 1024;
|
||||
let chunk = &bytes[start..bytes.len().min(start + 32 * 1024)];
|
||||
tracing::info!(target: "request_dump", request_id = %state.request_id,
|
||||
attempt_index = self.attempt_index, exchange_id = self.exchange_id,
|
||||
stage, wire_sequence, chunk_index, chunk_count,
|
||||
body_bytes = bytes.len(), body_base64 = %STANDARD.encode(chunk),
|
||||
contains_sensitive_data = true, "diagnostic wire capture");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 与请求终态原子持久化;旧记录没有此字段,不伪造历史时间线。
|
||||
#[must_use]
|
||||
pub fn snapshot(&self) -> Option<Value> {
|
||||
let state = self
|
||||
.state
|
||||
.as_ref()?
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
Some(json!({
|
||||
"schemaVersion": 1, "requestId": state.request_id,
|
||||
"startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence,
|
||||
"droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS,
|
||||
"captureMode": "sanitized", "events": state.events,
|
||||
"wireDumpEnabled": tracing::enabled!(target: "request_dump", tracing::Level::INFO),
|
||||
"wireFrames": state.wire_frames, "wireBytes": state.wire_bytes,
|
||||
"maxBufferBytes": MAX_BUFFER_BYTES, "headEvents": HEAD_EVENTS,
|
||||
"snapshotBoundary": "execution_finalization",
|
||||
}))
|
||||
}
|
||||
|
||||
fn push(&self, stage: &'static str, mut data: Value, coalesce: bool) {
|
||||
let Some(state) = &self.state else { return };
|
||||
let mut encoded = serde_json::to_vec(&data).unwrap_or_default();
|
||||
if encoded.len() > MAX_DATA_BYTES {
|
||||
data = json!({"truncated": true, "summary": body_fingerprint(&encoded),
|
||||
"eventType": data.get("eventType"), "body": data.get("body"),
|
||||
"jsonValid": data.get("jsonValid"),
|
||||
});
|
||||
encoded = serde_json::to_vec(&data).unwrap_or_default();
|
||||
}
|
||||
let mut state = state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.sequence += 1;
|
||||
let sequence = state.sequence;
|
||||
let elapsed_ms = u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
if coalesce
|
||||
&& let Some(last) = state.events.iter_mut().rev().find(|event| {
|
||||
event.stage == stage
|
||||
&& event.attempt_index == self.attempt_index
|
||||
&& event.exchange_id == self.exchange_id
|
||||
})
|
||||
&& last.stage == stage
|
||||
&& last.attempt_index == self.attempt_index
|
||||
&& last.exchange_id == self.exchange_id
|
||||
&& last.data.get("eventType") == data.get("eventType")
|
||||
{
|
||||
last.count += 1;
|
||||
last.last_sequence = sequence;
|
||||
last.last_elapsed_ms = elapsed_ms;
|
||||
return;
|
||||
}
|
||||
// 普通日志只输出有界安全事实;进程意外退出时仍可按 ID 检索已发生阶段。
|
||||
tracing::info!(target: "request_trace", request_id = %state.request_id,
|
||||
attempt_index = self.attempt_index, exchange_id = self.exchange_id,
|
||||
sequence, elapsed_ms, stage, data = %data, "request trace");
|
||||
let bytes = encoded.len() + 256;
|
||||
state.bytes += bytes;
|
||||
state.events.push_back(TraceEvent {
|
||||
sequence,
|
||||
last_sequence: sequence,
|
||||
elapsed_ms,
|
||||
last_elapsed_ms: elapsed_ms,
|
||||
attempt_index: self.attempt_index,
|
||||
exchange_id: self.exchange_id,
|
||||
stage,
|
||||
count: 1,
|
||||
data,
|
||||
bytes,
|
||||
});
|
||||
while state.events.len() > MAX_EVENTS || state.bytes > MAX_BUFFER_BYTES {
|
||||
// Preserve failures and retry decisions ahead of routine middle-of-stream events.
|
||||
// Head/tail and important stages compete under the same hard memory limit.
|
||||
let index = (HEAD_EVENTS..state.events.len().saturating_sub(8))
|
||||
.find(|index| {
|
||||
!matches!(
|
||||
state.events[*index].stage,
|
||||
"attempt.started"
|
||||
| "account.selected"
|
||||
| "attempt.failed"
|
||||
| "retry.decided"
|
||||
| "upstream.close"
|
||||
| "upstream.read.failed"
|
||||
| "capture.gap"
|
||||
| "request.finished"
|
||||
)
|
||||
})
|
||||
.unwrap_or(if state.events.len() > HEAD_EVENTS {
|
||||
HEAD_EVENTS
|
||||
} else {
|
||||
0
|
||||
});
|
||||
if let Some(event) = state.events.remove(index) {
|
||||
state.bytes = state.bytes.saturating_sub(event.bytes);
|
||||
state.dropped_events += event.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TraceState {
|
||||
fn drop(&mut self) {
|
||||
tracing::info!(target: "request_trace", request_id = %self.request_id,
|
||||
stage = "request.trace.closed", total_events = self.sequence,
|
||||
dropped_events = self.dropped_events, wire_frames = self.wire_frames,
|
||||
wire_bytes = self.wire_bytes, "request diagnostic capture closed");
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
//! 唯一的账号重试、发送与下游 commit barrier owner。
|
||||
|
||||
use crate::diagnostics::TraceContext;
|
||||
use serde_json::json;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
use std::num::NonZeroU32;
|
||||
@@ -144,12 +147,26 @@ where
|
||||
.position(|candidate| candidate.provider() == pin.provider())
|
||||
.ok_or(EngineError::ContinuationPinMismatch)
|
||||
})?;
|
||||
let trace = TraceContext::new(request_id.as_str());
|
||||
trace.record(
|
||||
"request.routed",
|
||||
json!({
|
||||
"endpoint": request.endpoint, "protocol": request.protocol,
|
||||
"clientTransport": request.client_transport,
|
||||
"model": request.requested_model.as_ref().map(|model| model.as_str()),
|
||||
"configRevision": format!("{:?}", request.config_revision),
|
||||
"admissionMs": request.admission_decision_ms,
|
||||
"continuation": format!("{continuation_attempt:?}"),
|
||||
"candidateCount": plan.candidates().len(),
|
||||
}),
|
||||
);
|
||||
let image_generation_requested = operation.image_generation_requested();
|
||||
let mut session = ResponseExecutionSession {
|
||||
engine: Arc::clone(&self.engine),
|
||||
request_id,
|
||||
client_api_key_ref,
|
||||
observation: ResponseObservation::new(timing_started_at),
|
||||
trace,
|
||||
deadline,
|
||||
deadline_timer: Delay::new(
|
||||
deadline
|
||||
@@ -238,6 +255,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
request_id: ModelRequestId,
|
||||
client_api_key_ref: crate::policy::ClientApiKeyId,
|
||||
observation: ResponseObservation,
|
||||
trace: TraceContext,
|
||||
deadline: SystemTime,
|
||||
/// 会话级 deadline 计时器;deadline 固定,帧循环内复用而非逐事件新建。
|
||||
deadline_timer: Fuse<Delay>,
|
||||
@@ -291,6 +309,11 @@ impl<S: ?Sized> ResponseExecutionSession<S>
|
||||
where
|
||||
S: ExecutionStore,
|
||||
{
|
||||
/// 当前请求共享的诊断上下文。
|
||||
pub fn trace(&self) -> TraceContext {
|
||||
self.trace.clone()
|
||||
}
|
||||
|
||||
/// 读取下一条 canonical event;首条未提交事件会携带 commit 要求。
|
||||
///
|
||||
/// # Errors
|
||||
@@ -398,6 +421,10 @@ where
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
}
|
||||
let committed_at = SystemTime::now();
|
||||
self.trace.record(
|
||||
"downstream.committed",
|
||||
json!({"status": client_status_code}),
|
||||
);
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_downstream_committed",
|
||||
@@ -563,6 +590,11 @@ where
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
PollBoundary::Item(Some(Ok(mut event))) => {
|
||||
if let Some(wire) = event.wire_event() {
|
||||
self.trace
|
||||
.attempt(self.attempts)
|
||||
.wire_event(wire.protocol(), wire.event_type());
|
||||
}
|
||||
if let Some(state) = event.session_update() {
|
||||
self.observe_session_checkpoint(state);
|
||||
}
|
||||
@@ -682,7 +714,8 @@ where
|
||||
}));
|
||||
let context = AttemptContext::new(
|
||||
RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone())
|
||||
.with_timing_started_at(self.observation.timing_started_at),
|
||||
.with_timing_started_at(self.observation.timing_started_at)
|
||||
.with_trace(self.trace.clone()),
|
||||
next_attempt,
|
||||
self.deadline,
|
||||
self.plan.account_selection_policy(),
|
||||
@@ -705,6 +738,18 @@ where
|
||||
.ok_or_else(|| EngineError::ProviderNotRegistered {
|
||||
provider: candidate.provider().as_str().to_owned(),
|
||||
})?;
|
||||
let attempt_trace = self.trace.attempt(next_attempt.get());
|
||||
attempt_trace.record(
|
||||
"attempt.started",
|
||||
json!({
|
||||
"provider": candidate.provider().as_str(),
|
||||
"model": candidate.upstream_model().map(|model| model.as_str()),
|
||||
"transportPolicy": format!("{attempt_transport:?}"),
|
||||
"continuation": format!("{:?}", self.continuation_attempt),
|
||||
"pinnedAccount": pinned_account.as_ref().map(|id| id.as_str()),
|
||||
"transportRecovery": is_transport_recovery,
|
||||
}),
|
||||
);
|
||||
let provider_request = ProviderRequest::new(self.operation.clone(), candidate.clone());
|
||||
let stream = match poll_provider(
|
||||
provider,
|
||||
@@ -729,6 +774,7 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
record_trace_error(&attempt_trace, &error);
|
||||
if self.prepare_unavailable_native_continuation_replay(&error) {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
@@ -800,6 +846,11 @@ where
|
||||
}
|
||||
|
||||
let metadata = stream.metadata().clone();
|
||||
attempt_trace.record("account.selected", json!({
|
||||
"provider": metadata.provider().as_str(), "accountId": metadata.provider_account_id().as_str(),
|
||||
"transport": metadata.transport().as_str(),
|
||||
"selectionMs": metadata.selection_observation().map(|o| o.account_selection_wait_ms()),
|
||||
}));
|
||||
let selection_observation = metadata.selection_observation();
|
||||
let capacity = selection_observation.and_then(|observation| observation.capacity());
|
||||
if !self.account_selection.is_diagnostic()
|
||||
@@ -1006,6 +1057,7 @@ where
|
||||
) -> Result<StreamErrorOutcome, EngineError> {
|
||||
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
|
||||
// 均只接触已剥离的稳定错误字段。
|
||||
record_trace_error(&self.trace.attempt(self.attempts), &error);
|
||||
let mut atomic_client_events = error.take_atomic_client_events();
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
|
||||
@@ -1093,6 +1145,14 @@ where
|
||||
|| account_rotation_retry
|
||||
|| transport_recovery.is_some();
|
||||
|
||||
self.trace.attempt(current.index.get()).record("retry.decided", json!({
|
||||
"retryable": retryable, "continuationRetry": continuation_retry,
|
||||
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
|
||||
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
|
||||
"delayMs": transport_recovery.map(|(_, delay)| duration_ms(delay)),
|
||||
"downstreamCommitted": self.downstream_committed_at.is_some(),
|
||||
"sendState": format!("{attempt_send_state:?}"),
|
||||
}));
|
||||
if retryable {
|
||||
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
|
||||
// 所有权保留到下一次 attempt 成功,或最终空选路时返回客户端。
|
||||
@@ -1303,6 +1363,10 @@ where
|
||||
self.current_transport_observation();
|
||||
let service_tier = self.current_service_tier();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
self.trace.record(
|
||||
"request.finished",
|
||||
json!({"outcome": "succeeded", "attempts": self.attempts}),
|
||||
);
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"finalize_model_request",
|
||||
@@ -1325,6 +1389,7 @@ where
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
provider_metadata_json,
|
||||
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
@@ -1443,6 +1508,14 @@ where
|
||||
if self.finalized {
|
||||
return Ok(());
|
||||
}
|
||||
self.trace.record(
|
||||
"request.finished",
|
||||
json!({
|
||||
"outcome": format!("{:?}", finalization.outcome),
|
||||
"errorKind": finalization.error.kind().as_str(),
|
||||
"sendState": format!("{:?}", finalization.send_state), "attempts": self.attempts,
|
||||
}),
|
||||
);
|
||||
if !self.request_persisted {
|
||||
self.finalized = true;
|
||||
return Ok(());
|
||||
@@ -1497,6 +1570,7 @@ where
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
provider_metadata_json,
|
||||
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
raw_upstream_error: finalization.raw_upstream_error,
|
||||
@@ -1787,3 +1861,15 @@ const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> Upst
|
||||
fn duration_ms(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn record_trace_error(trace: &TraceContext, error: &ProviderError) {
|
||||
trace.record("attempt.failed", json!({
|
||||
"kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()),
|
||||
"upstreamStatus": error.upstream_status(),
|
||||
"upstreamCode": error.upstream_code().map(|code| code.as_str()),
|
||||
"rawError": error.raw_upstream_error().map(|raw| {
|
||||
let value = serde_json::from_str(raw.as_str()).unwrap_or_else(|_| json!(raw.as_str()));
|
||||
crate::diagnostics::diagnostic_json(&value)
|
||||
}),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -155,6 +155,9 @@ pub struct StartedExecution {
|
||||
}
|
||||
|
||||
pub trait ExecutionSession: Send {
|
||||
fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
crate::diagnostics::TraceContext::default()
|
||||
}
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>>;
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>>;
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader];
|
||||
@@ -1035,6 +1038,12 @@ impl Drop for DefaultExecutionSession {
|
||||
}
|
||||
|
||||
impl ExecutionSession for DefaultExecutionSession {
|
||||
fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
self.core
|
||||
.as_ref()
|
||||
.map(ResponseExecutionSession::trace)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
let result = self.core_mut()?.next_event().await;
|
||||
|
||||
@@ -316,6 +316,7 @@ pub struct RequestAttemptContext {
|
||||
request_id: ModelRequestId,
|
||||
client_api_key_ref: ClientApiKeyId,
|
||||
timing_started_at: Instant,
|
||||
trace: crate::diagnostics::TraceContext,
|
||||
}
|
||||
|
||||
impl RequestAttemptContext {
|
||||
@@ -325,9 +326,16 @@ impl RequestAttemptContext {
|
||||
request_id,
|
||||
client_api_key_ref,
|
||||
timing_started_at: Instant::now(),
|
||||
trace: crate::diagnostics::TraceContext::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_trace(mut self, trace: crate::diagnostics::TraceContext) -> Self {
|
||||
self.trace = trace;
|
||||
self
|
||||
}
|
||||
|
||||
/// 覆盖本次请求的单调计时原点。
|
||||
#[must_use]
|
||||
pub fn with_timing_started_at(mut self, timing_started_at: Instant) -> Self {
|
||||
@@ -367,6 +375,12 @@ pub struct AttemptContext {
|
||||
}
|
||||
|
||||
impl AttemptContext {
|
||||
/// 当前 attempt 的诊断关联;克隆后可传给后台 transport 任务。
|
||||
#[must_use]
|
||||
pub fn trace(&self) -> crate::diagnostics::TraceContext {
|
||||
self.request.trace.attempt(self.attempt_index.get())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
request: RequestAttemptContext,
|
||||
@@ -612,6 +626,8 @@ pub struct ModelRequestFinalization {
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;Core 不解释字段。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
/// 请求全程的有界诊断快照,跨 Provider 与重试保留。
|
||||
pub diagnostic_trace_json: Option<String>,
|
||||
pub error: Option<GatewayError>,
|
||||
pub provider_error_code: Option<String>,
|
||||
/// Provider 返回的原始错误正文或 WebSocket close/error frame。
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
//! 具体客户端协议和具体 Provider 都通过外层 adapter 接入。
|
||||
|
||||
pub mod account;
|
||||
pub mod diagnostics;
|
||||
pub mod engine;
|
||||
pub mod error;
|
||||
pub mod event;
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
use gateway_core::diagnostics::{diagnostic_headers, diagnostic_json};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn credentials_and_content_never_enter_diagnostics_but_trace_ids_survive() {
|
||||
let headers = diagnostic_headers([
|
||||
("Authorization", "Bearer never-persist-this"),
|
||||
("Set-Cookie", "session=never-persist-this"),
|
||||
("X-OAI-Request-ID", "upstream-123"),
|
||||
("x-codex-turn-state", "opaque-never-persist-this"),
|
||||
("cf-ray", "ray-one"),
|
||||
("cf-ray", "ray-two"),
|
||||
]);
|
||||
assert_eq!(headers["x-oai-request-id"][0], "upstream-123");
|
||||
assert_eq!(headers["cf-ray"].as_array().unwrap().len(), 2);
|
||||
assert!(!headers.to_string().contains("never-persist-this"));
|
||||
let content = diagnostic_json(&json!({
|
||||
"type": "codex.response.metadata", "headers": headers,
|
||||
"access_token": "never-persist-this", "refresh_token": "never-persist-this",
|
||||
"response": {"output": [{"text": "private model output"}]},
|
||||
"input": "private user prompt", "unknown": "private extension",
|
||||
}));
|
||||
assert_eq!(content["type"], "codex.response.metadata");
|
||||
let text = content.to_string();
|
||||
for secret in [
|
||||
"never-persist-this",
|
||||
"private model output",
|
||||
"private user prompt",
|
||||
"private extension",
|
||||
] {
|
||||
assert!(!text.contains(secret), "leaked {secret}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arbitrary_event_type_text_is_fingerprinted_instead_of_logged() {
|
||||
let trace = gateway_core::diagnostics::TraceContext::new("req_type");
|
||||
trace.capture(
|
||||
"upstream.event",
|
||||
br#"{"type":"secret@example.com","message":"private-content"}"#,
|
||||
);
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
assert_eq!(
|
||||
snapshot["events"][0]["data"]["eventType"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
assert!(!snapshot.to_string().contains("secret@example.com"));
|
||||
assert!(!snapshot.to_string().contains("private-content"));
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
mod capture;
|
||||
mod stream;
|
||||
mod trace;
|
||||
@@ -0,0 +1,70 @@
|
||||
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext};
|
||||
|
||||
#[test]
|
||||
fn sse_capture_reassembles_every_chunk_boundary_and_preserves_unknown_events() {
|
||||
let bytes = "event: future.metadata\r\ndata: {\"headers\":{\"x-request-id\":\"up-1\"},\"unknown\":\"中文\"}\r\n\r\ndata: {\"type\":\"response.completed\"}\n\n".as_bytes();
|
||||
for boundary in 0..=bytes.len() {
|
||||
let trace = TraceContext::new("req_sse");
|
||||
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
|
||||
capture.push(&bytes[..boundary]);
|
||||
capture.push(&bytes[boundary..]);
|
||||
capture.finish();
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
assert_eq!(events.len(), 3, "boundary {boundary}");
|
||||
assert_eq!(events[0]["data"]["eventType"], "future.metadata");
|
||||
assert_eq!(
|
||||
events[0]["data"]["metadata"]["headers"]["x-request-id"],
|
||||
"up-1"
|
||||
);
|
||||
assert_eq!(events[1]["data"]["eventType"], "response.completed");
|
||||
assert_eq!(events[2]["data"]["partialFrame"], false);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversize_frames_report_a_gap_and_resume_at_the_next_frame() {
|
||||
let trace = TraceContext::new("req_sse");
|
||||
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
|
||||
capture.push(b"data: ");
|
||||
capture.push(&vec![b'x'; 130 * 1024]);
|
||||
capture.push(b"\n\ndata: {\"type\":\"response.completed\"}\n\n");
|
||||
capture.finish();
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
assert_eq!(events[0]["stage"], "capture.gap");
|
||||
assert_eq!(events[1]["data"]["eventType"], "response.completed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_a_partial_stream_is_distinct_from_upstream_eof() {
|
||||
let trace = TraceContext::new("req_partial");
|
||||
{
|
||||
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
|
||||
capture.push(b"data: {\"type\":");
|
||||
}
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
assert_eq!(snapshot["events"][0]["stage"], "upstream.reader.dropped");
|
||||
assert!(
|
||||
snapshot["events"][0]["data"]["partialBytes"]
|
||||
.as_u64()
|
||||
.unwrap()
|
||||
> 0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sse_accepts_bom_and_bare_carriage_return_even_when_split_into_single_bytes() {
|
||||
let trace = TraceContext::new("req_cr");
|
||||
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
|
||||
for byte in b"\xef\xbb\xbfdata: {\"type\":\"codex.response.metadata\"}\r\r" {
|
||||
capture.push(&[*byte]);
|
||||
}
|
||||
capture.finish();
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
assert_eq!(
|
||||
snapshot["events"][0]["data"]["eventType"],
|
||||
"codex.response.metadata"
|
||||
);
|
||||
assert_eq!(snapshot["events"][1]["data"]["partialFrame"], false);
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn bounded_history_preserves_start_failure_and_final_result() {
|
||||
let trace = TraceContext::new("req_trace");
|
||||
trace.record("request.started", json!({}));
|
||||
for index in 0..400 {
|
||||
trace.record(
|
||||
"test.event",
|
||||
json!({"index": index, "detail": "x".repeat(2000)}),
|
||||
);
|
||||
}
|
||||
trace
|
||||
.attempt(1)
|
||||
.record("attempt.failed", json!({"code": "websocket_close_1000"}));
|
||||
trace
|
||||
.attempt(2)
|
||||
.record("request.finished", json!({"outcome": "succeeded"}));
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
assert_eq!(events[0]["stage"], "request.started");
|
||||
assert_eq!(events[events.len() - 2]["attemptIndex"], 1);
|
||||
assert_eq!(events.last().unwrap()["attemptIndex"], 2);
|
||||
assert!(snapshot["droppedEvents"].as_u64().unwrap() > 0);
|
||||
assert!(events.len() <= 128);
|
||||
assert!(snapshot.to_string().len() < 64 * 1024);
|
||||
let kept: u64 = events
|
||||
.iter()
|
||||
.map(|event| event["count"].as_u64().unwrap())
|
||||
.sum();
|
||||
assert_eq!(
|
||||
kept + snapshot["droppedEvents"].as_u64().unwrap(),
|
||||
snapshot["totalEvents"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_attempts_and_exchanges_coalesce_delta_without_losing_metadata() {
|
||||
let trace = TraceContext::new("req_trace");
|
||||
let ws = trace.attempt(1).exchange("websocket");
|
||||
ws.capture(
|
||||
"upstream.event",
|
||||
br#"{"type":"codex.response.metadata","headers":{"x-request-id":"up-1"}}"#,
|
||||
);
|
||||
trace
|
||||
.attempt(1)
|
||||
.wire_event("openai", Some("codex.response.metadata"));
|
||||
for _ in 0..1000 {
|
||||
ws.capture(
|
||||
"upstream.event",
|
||||
br#"{"type":"response.output_text.delta","delta":"secret"}"#,
|
||||
);
|
||||
trace
|
||||
.attempt(1)
|
||||
.wire_event("openai", Some("response.output_text.delta"));
|
||||
}
|
||||
let http = trace.attempt(1).exchange("http_sse");
|
||||
http.capture("upstream.event", br#"{"type":"response.completed"}"#);
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
assert_eq!(snapshot["droppedEvents"], 0);
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.find(|e| e["data"]["eventType"] == "response.output_text.delta")
|
||||
.unwrap()["count"],
|
||||
1000
|
||||
);
|
||||
assert_eq!(events.last().unwrap()["exchangeId"], 2);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|e| e["data"]["metadata"]["headers"]["x-request-id"] == "up-1")
|
||||
);
|
||||
assert!(!snapshot.to_string().contains("secret"));
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|event| event["stage"] == "provider.event")
|
||||
.all(|event| event["data"].get("metadata").is_none())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_capture_keeps_duplicates_but_excludes_credentials_from_timeline() {
|
||||
let trace = TraceContext::new("req_headers");
|
||||
trace.headers(
|
||||
"upstream.response.headers",
|
||||
json!({"status": 101}),
|
||||
[
|
||||
("authorization", b"Bearer SECRET".as_slice()),
|
||||
("x-request-id", b"upstream-one".as_slice()),
|
||||
("x-request-id", b"upstream-two".as_slice()),
|
||||
("set-cookie", b"PRIVATE_COOKIE".as_slice()),
|
||||
],
|
||||
);
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
assert_eq!(
|
||||
snapshot["events"][0]["data"]["headers"]["x-request-id"],
|
||||
json!(["upstream-one", "upstream-two"])
|
||||
);
|
||||
assert!(!snapshot.to_string().contains("SECRET"));
|
||||
assert!(!snapshot.to_string().contains("PRIVATE_COOKIE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_context_does_not_allocate_a_history() {
|
||||
let trace = TraceContext::default().attempt(2).exchange("http");
|
||||
trace.capture("upstream.event", b"not json");
|
||||
assert!(trace.snapshot().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_successful_retry_keeps_the_earlier_failure_ahead_of_routine_events() {
|
||||
let trace = TraceContext::new("req_retry");
|
||||
for _ in 0..12 {
|
||||
trace.record("setup", json!({}));
|
||||
}
|
||||
trace
|
||||
.attempt(1)
|
||||
.record("attempt.failed", json!({"kind": "unavailable"}));
|
||||
trace
|
||||
.attempt(1)
|
||||
.record("retry.decided", json!({"retryable": true}));
|
||||
for index in 0..500 {
|
||||
trace
|
||||
.attempt(2)
|
||||
.record("upstream.event", json!({"index": index}));
|
||||
}
|
||||
trace.record("request.finished", json!({"outcome": "succeeded"}));
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event["stage"] == "attempt.failed" && event["attemptIndex"] == 1)
|
||||
);
|
||||
assert!(events.iter().any(|event| event["stage"] == "retry.decided"));
|
||||
assert_eq!(events.last().unwrap()["stage"], "request.finished");
|
||||
assert!(events.len() <= 128);
|
||||
}
|
||||
@@ -49,6 +49,7 @@ use serde_json::{Map, Value, json};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct FinalState {
|
||||
diagnostic_trace_json: Option<String>,
|
||||
outcome: ExecutionOutcome,
|
||||
send_state: UpstreamSendState,
|
||||
attempt_count: u32,
|
||||
@@ -207,6 +208,7 @@ impl ExecutionStore for FakeStore {
|
||||
.expect("store lock")
|
||||
.finalizations
|
||||
.push(FinalState {
|
||||
diagnostic_trace_json: finalization.diagnostic_trace_json,
|
||||
outcome: finalization.outcome,
|
||||
send_state: finalization.send_state,
|
||||
attempt_count: finalization.attempt_count,
|
||||
@@ -1314,6 +1316,21 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
|
||||
let state = store.state.lock().expect("store lock");
|
||||
let finalization = &state.finalizations[0];
|
||||
let trace: Value =
|
||||
serde_json::from_str(finalization.diagnostic_trace_json.as_deref().unwrap()).unwrap();
|
||||
let events = trace["events"].as_array().unwrap();
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event["stage"] == "attempt.failed" && event["attemptIndex"] == 1)
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event["stage"] == "attempt.started" && event["attemptIndex"] == 2)
|
||||
);
|
||||
assert_eq!(events.last().unwrap()["stage"], "request.finished");
|
||||
|
||||
assert_eq!(finalization.upstream_transport.as_deref(), Some("http_sse"));
|
||||
assert_eq!(finalization.http_version.as_deref(), Some("HTTP/2"));
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod account;
|
||||
mod diagnostics;
|
||||
mod engine;
|
||||
mod error;
|
||||
mod event;
|
||||
|
||||
@@ -283,6 +283,7 @@ pub struct ModelRequestFinalization {
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub provider_metadata_json: Option<Value>,
|
||||
pub diagnostic_trace_json: Option<Value>,
|
||||
pub error_kind: Option<String>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
@@ -353,6 +354,15 @@ impl ModelRequestFinalization {
|
||||
{
|
||||
return Err(invalid("provider observation must be a JSON object"));
|
||||
}
|
||||
if self
|
||||
.diagnostic_trace_json
|
||||
.as_ref()
|
||||
.is_some_and(|trace| !trace.is_object() || trace.to_string().len() > 64 * 1024)
|
||||
{
|
||||
return Err(invalid(
|
||||
"diagnostic trace must be a JSON object within 64 KiB",
|
||||
));
|
||||
}
|
||||
validate_optional_stable_reason(
|
||||
self.continuation_unavailable_reason.as_deref(),
|
||||
"continuation unavailable reason",
|
||||
@@ -725,7 +735,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
upstream_connection_id = $44,
|
||||
upstream_connection_exit_reason = $45,
|
||||
upstream_connection_age_ms = $46,
|
||||
upstream_connection_idle_ms = $47
|
||||
upstream_connection_idle_ms = $47, diagnostic_trace_json = $48
|
||||
where id = $1 and outcome = 'running'
|
||||
returning id, client_api_key_ref, continuation_affinity_hash,
|
||||
continuation_requested, provider_kind, upstream_transport,
|
||||
@@ -916,6 +926,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
finalization.upstream_connection_idle_ms,
|
||||
"upstream_connection_idle_ms",
|
||||
)?)
|
||||
.bind(finalization.diagnostic_trace_json.map(sqlx::types::Json))
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("finalize model request"))?;
|
||||
@@ -1215,6 +1226,12 @@ impl ExecutionStore for PgExecutionStore {
|
||||
websocket_pool: finalization.websocket_pool,
|
||||
service_tier: finalization.service_tier,
|
||||
provider_metadata_json,
|
||||
diagnostic_trace_json: finalization
|
||||
.diagnostic_trace_json
|
||||
.as_deref()
|
||||
.map(serde_json::from_str)
|
||||
.transpose()
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
||||
error_kind,
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
error_message,
|
||||
|
||||
@@ -583,6 +583,7 @@ impl ExecutionObservationWrite {
|
||||
finalization.websocket_pool.as_deref(),
|
||||
finalization.service_tier.as_deref(),
|
||||
finalization.provider_metadata_json.as_deref(),
|
||||
finalization.diagnostic_trace_json.as_deref(),
|
||||
finalization.provider_error_code.as_deref(),
|
||||
])
|
||||
.saturating_add(error_bytes)
|
||||
|
||||
@@ -584,6 +584,8 @@ pub(crate) fn admin_usage_detail(
|
||||
detail: UsageRecordDetail,
|
||||
) -> AdminStoreResult<admin_observability::UsageDetail> {
|
||||
Ok(admin_observability::UsageDetail {
|
||||
trace: detail.trace,
|
||||
related_requests: detail.related_requests,
|
||||
request: admin_usage_record(detail.request)?,
|
||||
attempts: detail
|
||||
.attempts
|
||||
|
||||
@@ -645,6 +645,8 @@ pub struct UsageAttemptObservation {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageRecordDetail {
|
||||
pub trace: Option<serde_json::Value>,
|
||||
pub related_requests: Vec<serde_json::Value>,
|
||||
pub request: UsageRecord,
|
||||
pub attempts: Vec<UsageAttemptObservation>,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Usage 明细、诊断与过滤查询族。
|
||||
|
||||
use super::super::*;
|
||||
use serde_json::Value;
|
||||
|
||||
pub(crate) fn push_usage_filter(
|
||||
query: &mut QueryBuilder<Postgres>,
|
||||
@@ -145,7 +146,7 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool,
|
||||
mr.service_tier, mr.provider_observation_json,
|
||||
mr.service_tier, mr.provider_observation_json, mr.diagnostic_trace_json,
|
||||
mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at,
|
||||
mr.outcome, mr.client_status_code, mr.upstream_status_code,
|
||||
mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id,
|
||||
@@ -237,7 +238,6 @@ pub(crate) async fn usage_record_detail(
|
||||
let mut statement = QueryBuilder::<Postgres>::new(USAGE_RECORD_DETAIL_SELECT);
|
||||
statement.push(" where mr.id = ");
|
||||
statement.push_bind(request_id.to_owned());
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
let row = statement
|
||||
.build()
|
||||
.fetch_optional(pool)
|
||||
@@ -272,7 +272,24 @@ pub(crate) async fn usage_record_detail(
|
||||
if request.attempt_count > 0 {
|
||||
attempts.push(final_attempt_from_request(&request));
|
||||
}
|
||||
Ok(UsageRecordDetail { request, attempts })
|
||||
let trace: Option<Value> = row
|
||||
.try_get("diagnostic_trace_json")
|
||||
.map_err(|_| postgres_unavailable("decode request trace"))?;
|
||||
let related_requests = sqlx::query_scalar::<_, Value>(
|
||||
"select jsonb_build_object('requestId', related.id, 'outcome', related.outcome,
|
||||
'relation', case when related.id = current.recovery_request_id then 'recovered_by' else 'recovers' end,
|
||||
'completedAt', related.completed_at)
|
||||
from model_requests current join model_requests related
|
||||
on related.id = current.recovery_request_id or related.recovery_request_id = current.id
|
||||
where current.id = $1 order by related.started_at limit 20"
|
||||
).bind(request_id).fetch_all(pool).await
|
||||
.map_err(|_| postgres_unavailable("load related recovery requests"))?;
|
||||
Ok(UsageRecordDetail {
|
||||
request,
|
||||
attempts,
|
||||
trace,
|
||||
related_requests,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn intermediate_attempt_from_row(
|
||||
|
||||
@@ -8,7 +8,8 @@ use gateway_core::error::{GatewayError, GatewayErrorKind, ProviderConnectionObse
|
||||
use gateway_core::metering::{CalculatedCost, CostEstimate, Usage};
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
use gateway_store::postgres::{
|
||||
ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest, PgExecutionStore,
|
||||
ModelRequestAttemptStart, ModelRequestRepository, NewModelRequest, ObservabilityRepository,
|
||||
PgExecutionStore,
|
||||
};
|
||||
|
||||
use super::TestDatabase;
|
||||
@@ -463,6 +464,7 @@ async fn expired_image_request_should_be_recovered_as_failed() {
|
||||
|
||||
fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
|
||||
CoreModelRequestFinalization {
|
||||
diagnostic_trace_json: None,
|
||||
request_id: ModelRequestId::new(id).expect("request id"),
|
||||
outcome: ExecutionOutcome::Succeeded,
|
||||
send_state: UpstreamSendState::Sent,
|
||||
@@ -804,6 +806,30 @@ async fn successful_http_downgrade_should_mark_pending_websocket_failures_recove
|
||||
assert_eq!(row.4, Some(expected_delay_ms));
|
||||
assert!(row.5.is_some_and(|latency| latency >= expected_delay_ms));
|
||||
}
|
||||
let repository = super::observability_repository(&database.pool);
|
||||
let failed_detail = repository
|
||||
.usage_record_detail("req_websocket_failed_first")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
failed_detail.related_requests[0]["requestId"],
|
||||
"req_http_fallback_succeeded"
|
||||
);
|
||||
assert_eq!(
|
||||
failed_detail.related_requests[0]["relation"],
|
||||
"recovered_by"
|
||||
);
|
||||
let recovered_detail = repository
|
||||
.usage_record_detail("req_http_fallback_succeeded")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(recovered_detail.related_requests.len(), 2);
|
||||
assert!(
|
||||
recovered_detail
|
||||
.related_requests
|
||||
.iter()
|
||||
.all(|related| related["relation"] == "recovers")
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -1072,3 +1098,40 @@ async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx:
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostic_trace_is_finalized_atomically_and_available_for_failed_request_details() {
|
||||
let Some(database) = TestDatabase::create("diagnostic_trace").await else {
|
||||
return;
|
||||
};
|
||||
seed_running_request(&database.pool, "req_diagnostic_failed")
|
||||
.await
|
||||
.unwrap();
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
let mut finalization = successful_core_finalization("req_diagnostic_failed");
|
||||
finalization.outcome = ExecutionOutcome::Failed;
|
||||
finalization.client_status_code = Some(502);
|
||||
finalization.error = Some(GatewayError::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"upstream closed",
|
||||
));
|
||||
let trace = serde_json::json!({"schemaVersion": 1, "requestId": "req_diagnostic_failed", "events": [
|
||||
{"attemptIndex": 1, "stage": "upstream.close", "data": {"code": 1000}},
|
||||
]});
|
||||
finalization.diagnostic_trace_json = Some(trace.to_string());
|
||||
ExecutionStore::finalize_model_request(&store, finalization)
|
||||
.await
|
||||
.unwrap();
|
||||
let persisted: (String, serde_json::Value) = sqlx::query_as(
|
||||
"select outcome, diagnostic_trace_json from model_requests where id = 'req_diagnostic_failed'"
|
||||
).fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(persisted, ("failed".to_owned(), trace.clone()));
|
||||
let repository = super::observability_repository(&database.pool);
|
||||
let detail = repository
|
||||
.usage_record_detail("req_diagnostic_failed")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(detail.trace, Some(trace));
|
||||
assert!(detail.related_requests.is_empty());
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -853,13 +853,13 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
store
|
||||
.usage_record_detail("req_observe_failed")
|
||||
.await
|
||||
.is_err()
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.usage_record_detail("req_observe_uncommitted")
|
||||
.await
|
||||
.is_err()
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
let overview = store
|
||||
@@ -1243,7 +1243,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
repository
|
||||
.usage_record_detail("req_observe_failed")
|
||||
.await
|
||||
.is_err()
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
let overview = repository
|
||||
|
||||
@@ -21,27 +21,28 @@ gateway-core = { path = "../../gateway-core" }
|
||||
gateway-protocol = { path = "../../gateway-protocol" }
|
||||
getrandom = "0.4.3"
|
||||
hex = "0.4.3"
|
||||
# 与官方 Codex Desktop app-server 的 TLS 栈逐版本对齐(2026-08 核实:
|
||||
# ChatGPT.app Resources/codex 二进制内为 hyper-rustls 0.27.7 + rustls 0.23.36 +
|
||||
# reqwest 0.12.28 + ring)。rustls 0.23.37 起 ClientHello 扩展顺序变化,JA3
|
||||
# 会偏离官方指纹;升级前必须重新抓握手对比扩展顺序。
|
||||
# TLS 依赖按官方 Core 锁文件固定,包含用于约束传递依赖的直接声明。
|
||||
# 升级前须重复采样 ClientHello,核对算法、扩展集合及排序约束、key_share 与 ALPN。
|
||||
aws-lc-rs = "=1.16.2"
|
||||
hyper = { version = "=1.8.1", default-features = false }
|
||||
hyper-rustls = "=0.27.7"
|
||||
# 与官方 Codex Desktop 一致:reqwest 不启用压缩特性,上游请求不带 accept-encoding。
|
||||
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "http2"] }
|
||||
roxmltree = "0.21.1"
|
||||
semver = "1.0.27"
|
||||
# 保持 ring provider:reqwest rustls-tls-native-roots 即 __rustls-ring,与官方一致;
|
||||
# 不要给 TLS 客户端路径启用 aws_lc_rs,否则密钥共享/扩展集可能变化。
|
||||
rustls = { version = "=0.23.36", default-features = false, features = ["ring", "std", "tls12"] }
|
||||
rustls-native-certs = "0.8.4"
|
||||
rustls-pki-types = "1.14.1"
|
||||
# 与官方 rustls 路径一致:aws-lc 支持 P-521,优先提供 X25519MLKEM768。
|
||||
# reqwest 的 roots feature 仍会编译 ring;实际 provider 由 TLS 初始化显式选择。
|
||||
rustls = { version = "=0.23.36", default-features = false, features = ["aws_lc_rs", "prefer-post-quantum", "std", "tls12"] }
|
||||
rustls-native-certs = "=0.8.3"
|
||||
rustls-pki-types = "=1.14.0"
|
||||
secrecy = "0.10.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] }
|
||||
tokio = { version = "=1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] }
|
||||
tokio-rustls = { version = "=0.26.4", default-features = false }
|
||||
tokio-util = { version = "0.7.18", features = ["rt"] }
|
||||
# 官方 Codex WebSocket 需要上游尚未发布的 proxy 与 permessage-deflate 实现,固定到已审计提交。
|
||||
tokio-tungstenite = { git = "https://github.com/openai-oss-forks/tokio-tungstenite", rev = "0e5b2d73aa18dd9f0a50ee9ff199d5aef7594186", features = ["proxy", "rustls-tls-native-roots"] }
|
||||
@@ -51,7 +52,7 @@ url = "2.5.8"
|
||||
uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
# 官方 Codex app-server 对 /v1/responses 请求体默认做 zstd 压缩(level 3),
|
||||
# 并携带 Content-Encoding: zstd;保持同版本同级别以对齐上游指纹。
|
||||
zstd = "0.13"
|
||||
zstd = "=0.13.3"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.27.0"
|
||||
|
||||
@@ -8,7 +8,7 @@ use serde::Deserialize;
|
||||
use url::{Host, Url};
|
||||
|
||||
use crate::credential::CodexQuotaRefreshPolicy;
|
||||
use crate::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use crate::transport::profile::{CodexResidency, CodexWireProfile, CodexWireProfileState};
|
||||
use crate::transport::session::{CodexSessionIdentity, CodexSessionIdentityError};
|
||||
use crate::transport::websocket::CodexWebSocketPoolConfig;
|
||||
use crate::{
|
||||
@@ -287,6 +287,8 @@ pub struct CodexWireProfileConfig {
|
||||
pub os_version: String,
|
||||
pub arch: String,
|
||||
pub terminal: String,
|
||||
#[serde(default)]
|
||||
pub residency: Option<CodexResidency>,
|
||||
pub verified_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -294,14 +296,16 @@ impl Default for CodexWireProfileConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
originator: "Codex Desktop".to_owned(),
|
||||
codex_version: "0.147.0-alpha.6.6".to_owned(),
|
||||
desktop_version: "26.803.81509".to_owned(),
|
||||
desktop_build: "6415".to_owned(),
|
||||
codex_version: "0.153.4".to_owned(),
|
||||
desktop_version: "26.901.51231".to_owned(),
|
||||
desktop_build: "8109".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
verified_at: Utc::now(),
|
||||
residency: None,
|
||||
// 制品核验于 2026-09-06T03:26:12.084Z;进程启动不构成重新核验。
|
||||
verified_at: DateTime::UNIX_EPOCH + chrono::Duration::milliseconds(1_788_665_172_084),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -361,6 +365,7 @@ impl From<CodexWireProfileConfig> for CodexWireProfile {
|
||||
os_version: value.os_version,
|
||||
arch: value.arch,
|
||||
terminal: value.terminal,
|
||||
residency: value.residency,
|
||||
verified_at: value.verified_at,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::{
|
||||
|
||||
use gateway_core::account::ProviderAccountId;
|
||||
use reqwest::Client;
|
||||
use secrecy::ExposeSecret as _;
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -133,6 +133,30 @@ impl CodexCredentialProfileService {
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<CodexProfileStatistics, CodexProfileStatisticsError> {
|
||||
let (authorization, upstream_account_id) = self.account_authentication(account_id).await?;
|
||||
let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple());
|
||||
let statistics = CodexBackendClient::new(
|
||||
self.http.clone(),
|
||||
self.base_url.clone(),
|
||||
self.profile.clone(),
|
||||
)
|
||||
.fetch_profile_statistics(CodexRequestContext::auxiliary(
|
||||
authorization.expose_secret(),
|
||||
upstream_account_id.as_deref(),
|
||||
&request_id,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.map_err(map_client_error)?;
|
||||
self.remember_avatar_source(account_id, statistics.image_url.as_deref());
|
||||
Ok(statistics)
|
||||
}
|
||||
|
||||
/// URL 缓存不缓存认证;每次下载都重新读取所属账号当前的 token。
|
||||
async fn account_authentication(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<(SecretString, Option<String>), CodexProfileStatisticsError> {
|
||||
let account = self
|
||||
.repository
|
||||
.store()
|
||||
@@ -160,22 +184,10 @@ impl CodexCredentialProfileService {
|
||||
.authentication
|
||||
.authorization_header()
|
||||
.map_err(|_| CodexProfileStatisticsError::InvalidCredentialData)?;
|
||||
let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple());
|
||||
let statistics = CodexBackendClient::new(
|
||||
self.http.clone(),
|
||||
self.base_url.clone(),
|
||||
self.profile.clone(),
|
||||
)
|
||||
.fetch_profile_statistics(CodexRequestContext::auxiliary(
|
||||
authorization.expose_secret(),
|
||||
account.upstream_account_id(),
|
||||
&request_id,
|
||||
None,
|
||||
Ok((
|
||||
authorization,
|
||||
account.upstream_account_id().map(str::to_owned),
|
||||
))
|
||||
.await
|
||||
.map_err(map_client_error)?;
|
||||
self.remember_avatar_source(account_id, statistics.image_url.as_deref());
|
||||
Ok(statistics)
|
||||
}
|
||||
|
||||
pub async fn profile_avatar(
|
||||
@@ -190,10 +202,21 @@ impl CodexCredentialProfileService {
|
||||
.image_url
|
||||
.ok_or(CodexProfileAvatarError::Missing)?,
|
||||
};
|
||||
let desktop_user_agent = self.profile.snapshot().desktop_user_agent();
|
||||
fetch_profile_avatar(&self.http, &self.base_url, &desktop_user_agent, &source)
|
||||
.await
|
||||
.map_err(map_avatar_fetch_error)
|
||||
let (authorization, upstream_account_id) = self.account_authentication(account_id).await?;
|
||||
fetch_profile_avatar(
|
||||
&self.http,
|
||||
&self.base_url,
|
||||
&self.profile.snapshot(),
|
||||
&source,
|
||||
CodexRequestContext::auxiliary(
|
||||
authorization.expose_secret(),
|
||||
upstream_account_id.as_deref(),
|
||||
"profile_avatar",
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(map_avatar_fetch_error)
|
||||
}
|
||||
|
||||
fn cached_avatar_source(&self, account_id: &ProviderAccountId) -> Option<String> {
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
//! OpenAI token 续期 Reqwest 适配器。
|
||||
|
||||
use crate::transport::{
|
||||
headers::build_codex_profile_headers,
|
||||
profile::CodexWireProfileState,
|
||||
tls::{build_reqwest_client_with_custom_ca, ensure_rustls_provider},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use reqwest::{Client, StatusCode, redirect::Policy};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
@@ -243,6 +248,7 @@ pub struct TokenClientConfig {
|
||||
pub struct OpenAiTokenClient {
|
||||
client: Client,
|
||||
config: TokenClientConfig,
|
||||
profile: CodexWireProfileState,
|
||||
}
|
||||
|
||||
/// 官方 Codex token client 无法安全构建。
|
||||
@@ -251,9 +257,13 @@ pub struct OpenAiTokenClient {
|
||||
pub struct TokenClientBuildError;
|
||||
|
||||
impl OpenAiTokenClient {
|
||||
/// 使用 Reqwest 客户端和静态配置构造 token 续期客户端。
|
||||
pub fn new(client: Client, config: TokenClientConfig) -> Self {
|
||||
Self { client, config }
|
||||
/// 共享运行时画像;刷新时取快照,授权码交换仍使用 raw auth 请求。
|
||||
pub fn new(client: Client, config: TokenClientConfig, profile: CodexWireProfileState) -> Self {
|
||||
Self {
|
||||
client,
|
||||
config,
|
||||
profile,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,14 +274,17 @@ impl OpenAiTokenClient {
|
||||
/// 本地 TLS/HTTP client 初始化失败时返回脱敏错误。
|
||||
pub fn openai_token_client(
|
||||
config: TokenClientConfig,
|
||||
profile: CodexWireProfileState,
|
||||
) -> Result<OpenAiTokenClient, TokenClientBuildError> {
|
||||
let client = Client::builder()
|
||||
ensure_rustls_provider();
|
||||
let builder = Client::builder()
|
||||
.use_rustls_tls()
|
||||
.no_proxy()
|
||||
.redirect(Policy::none())
|
||||
.connect_timeout(TOKEN_CONNECT_TIMEOUT)
|
||||
.timeout(TOKEN_REQUEST_TIMEOUT)
|
||||
.build()
|
||||
.map_err(|_| TokenClientBuildError)?;
|
||||
Ok(OpenAiTokenClient::new(client, config))
|
||||
.timeout(TOKEN_REQUEST_TIMEOUT);
|
||||
let client = build_reqwest_client_with_custom_ca(builder).map_err(|_| TokenClientBuildError)?;
|
||||
Ok(OpenAiTokenClient::new(client, config, profile))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -345,9 +358,16 @@ struct AuthorizationCodeResponse {
|
||||
#[async_trait]
|
||||
impl TokenRefresher for OpenAiTokenClient {
|
||||
async fn refresh(&self, refresh_token: &str) -> Result<TokenPair, RefreshFailure> {
|
||||
let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|_| {
|
||||
RefreshFailure::Transport {
|
||||
message: Some("OpenAI OAuth refresh profile is invalid".to_owned()),
|
||||
upstream: None,
|
||||
}
|
||||
})?;
|
||||
let response = self
|
||||
.client
|
||||
.post(&self.config.token_endpoint)
|
||||
.headers(headers)
|
||||
.json(&RefreshTokenRequest {
|
||||
client_id: self.config.client_id.as_str(),
|
||||
grant_type: "refresh_token",
|
||||
|
||||
@@ -146,8 +146,11 @@ pub async fn initialize(
|
||||
.with_session_identity(session_identity),
|
||||
);
|
||||
let token_client = Arc::new(
|
||||
credential::token_client::openai_token_client(config.token_client_config())
|
||||
.map_err(|_| OpenAiInitializeError::TokenClient)?,
|
||||
credential::token_client::openai_token_client(
|
||||
config.token_client_config(),
|
||||
profile.clone(),
|
||||
)
|
||||
.map_err(|_| OpenAiInitializeError::TokenClient)?,
|
||||
);
|
||||
let refresher: Arc<dyn TokenRefresher> = token_client.clone();
|
||||
let exchanger: Arc<dyn AuthorizationCodeExchanger> = token_client;
|
||||
|
||||
@@ -358,12 +358,14 @@ pub(super) async fn create_json_attempt(
|
||||
return Err(CodexHandshakeAttemptError::Timeout);
|
||||
};
|
||||
let request_id = request.context.request_id().as_str();
|
||||
let trace = request.context.trace();
|
||||
let mut request_context = CodexRequestContext::auxiliary(
|
||||
authorization.expose_secret(),
|
||||
account.upstream_account_id(),
|
||||
request_id,
|
||||
Some(installation_id),
|
||||
);
|
||||
request_context.trace = Some(&trace);
|
||||
request_context.cookie_header = cookie_header.map(ExposeSecret::expose_secret);
|
||||
request_context.turn_metadata = request.turn_metadata.as_deref();
|
||||
request_context.account_selection = account_selection;
|
||||
@@ -539,6 +541,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
lease.account_switch(),
|
||||
);
|
||||
let request_transport_requirement = transport_requirement(&request);
|
||||
let trace = context.trace();
|
||||
let response = create_response_attempt(
|
||||
&client,
|
||||
&request,
|
||||
@@ -550,7 +553,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
&authorization,
|
||||
cookie_header.as_ref(),
|
||||
account_selection,
|
||||
),
|
||||
).with_trace(&trace),
|
||||
active_account.id().as_str(),
|
||||
context.deadline(),
|
||||
&cancellation,
|
||||
|
||||
@@ -687,6 +687,7 @@ pub(super) fn codex_request_context<'a>(
|
||||
account_selection: CodexAccountSelectionTelemetry<'a>,
|
||||
) -> CodexRequestContext<'a> {
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: authorization.expose_secret(),
|
||||
account_id: account.upstream_account_id(),
|
||||
request_id,
|
||||
|
||||
@@ -395,6 +395,8 @@ impl<'a> CodexAccountSelectionTelemetry<'a> {
|
||||
/// 单次 Codex 上游请求的上下文。
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct CodexRequestContext<'a> {
|
||||
/// 显式传递的请求诊断上下文,跨连接任务共享。
|
||||
pub trace: Option<&'a gateway_core::diagnostics::TraceContext>,
|
||||
/// Provider 已构造并脱敏持有的完整 Authorization 值。
|
||||
pub authorization: &'a str,
|
||||
/// ChatGPT 账号 ID。
|
||||
@@ -440,6 +442,7 @@ impl<'a> CodexRequestContext<'a> {
|
||||
installation_id: Option<&'a str>,
|
||||
) -> Self {
|
||||
Self {
|
||||
trace: None,
|
||||
authorization,
|
||||
account_id,
|
||||
request_id,
|
||||
@@ -459,6 +462,12 @@ impl<'a> CodexRequestContext<'a> {
|
||||
account_selection: CodexAccountSelectionTelemetry::NONE,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_trace(mut self, trace: &'a gateway_core::diagnostics::TraceContext) -> Self {
|
||||
self.trace = Some(trace);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexRequestContext<'_> {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use bytes::Bytes;
|
||||
use reqwest::header::{ACCEPT, CONTENT_TYPE, HeaderName, HeaderValue};
|
||||
use reqwest::header::{CONTENT_TYPE, HeaderValue};
|
||||
|
||||
use super::{
|
||||
client::{
|
||||
@@ -14,7 +14,7 @@ use super::{
|
||||
},
|
||||
diagnostics::CodexUpstreamSendPhase,
|
||||
endpoints::endpoint_url,
|
||||
headers::{build_codex_base_headers, insert_optional_header, insert_optional_protocol_header},
|
||||
headers::insert_optional_protocol_header,
|
||||
response_meta,
|
||||
};
|
||||
|
||||
@@ -28,27 +28,30 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<CodexBackendJsonResponse> {
|
||||
let profile = self.profile.snapshot();
|
||||
let mut headers =
|
||||
build_codex_base_headers(&profile, context.authorization, context.account_id)?;
|
||||
let mut headers = self.model_request_headers(&profile, context)?;
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
|
||||
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
|
||||
insert_optional_header(
|
||||
&mut headers,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
)?;
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-image-turn-id", image_turn_id);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-turn-metadata",
|
||||
context.turn_metadata,
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-client-request-id"),
|
||||
HeaderValue::from_str(context.request_id)?,
|
||||
);
|
||||
|
||||
let trace = context
|
||||
.trace
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.exchange("http_json");
|
||||
trace.headers(
|
||||
"upstream.request.headers",
|
||||
serde_json::json!({
|
||||
"method": "POST", "endpoint": endpoint_path,
|
||||
}),
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
trace.capture("upstream.request.body", &body);
|
||||
let headers_started_at = Instant::now();
|
||||
let response = self
|
||||
.client
|
||||
@@ -61,6 +64,17 @@ impl CodexBackendClient {
|
||||
let upstream_headers_ms = elapsed_duration_millis(headers_started_at.elapsed());
|
||||
let http_version = http_version_name(response.version()).to_owned();
|
||||
let status = response.status();
|
||||
trace.headers(
|
||||
"upstream.response.headers",
|
||||
serde_json::json!({
|
||||
"status": status.as_u16(), "httpVersion": http_version,
|
||||
"headersMs": upstream_headers_ms,
|
||||
}),
|
||||
response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
@@ -81,6 +95,7 @@ impl CodexBackendClient {
|
||||
let raw_body = read_error_response_body(response)
|
||||
.await
|
||||
.map_err(CodexClientError::HttpJson)?;
|
||||
trace.capture("upstream.error.body", &raw_body);
|
||||
let error_body = String::from_utf8_lossy(&raw_body).into_owned();
|
||||
return Err(CodexClientError::Upstream {
|
||||
status,
|
||||
@@ -102,6 +117,7 @@ impl CodexBackendClient {
|
||||
}
|
||||
|
||||
let body = response.bytes().await.map_err(CodexClientError::HttpJson)?;
|
||||
trace.capture("upstream.response.body", &body);
|
||||
Ok(CodexBackendJsonResponse {
|
||||
body,
|
||||
set_cookie_headers,
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext, diagnostic_json};
|
||||
|
||||
use std::{sync::Arc, time::Instant};
|
||||
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
events::{self, retry_after_seconds_from_body},
|
||||
sse::{SseEventDecoder, SseFrame},
|
||||
};
|
||||
use reqwest::{
|
||||
Client, Response as ReqwestResponse,
|
||||
header::{
|
||||
ACCEPT, AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE, COOKIE, HeaderMap, HeaderName,
|
||||
HeaderValue, USER_AGENT,
|
||||
},
|
||||
header::{CONTENT_ENCODING, CONTENT_TYPE, HeaderMap, HeaderValue},
|
||||
};
|
||||
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||
|
||||
@@ -22,11 +21,8 @@ use crate::transport::{
|
||||
},
|
||||
diagnostics::CodexUpstreamSendPhase,
|
||||
endpoints::{CODEX_RESPONSES_PATH, endpoint_url},
|
||||
headers::{
|
||||
build_codex_base_headers, insert_optional_header, insert_optional_protocol_header,
|
||||
websocket_header_pairs,
|
||||
},
|
||||
profile::{CodexWireProfile, CodexWireProfileState},
|
||||
headers::websocket_header_pairs,
|
||||
profile::CodexWireProfileState,
|
||||
protocol::{
|
||||
responses::{CodexResponsesRequest, TransportRequirement, transport_requirement},
|
||||
websocket::{
|
||||
@@ -37,9 +33,8 @@ use crate::transport::{
|
||||
response_meta,
|
||||
websocket::{
|
||||
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
|
||||
CodexWebSocketPoolKey, CodexWebSocketRequest, CodexWebSocketStreamingExchange,
|
||||
DEFAULT_INITIAL_EVENT_TIMEOUT, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
|
||||
WebSocketOriginBreaker, WebSocketPoolDecision,
|
||||
CodexWebSocketPoolKey, CodexWebSocketStreamingExchange, DEFAULT_INITIAL_EVENT_TIMEOUT,
|
||||
WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath, WebSocketOriginBreaker,
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool, websocket_audit_dir,
|
||||
write_websocket_audit_artifact_from_env,
|
||||
@@ -106,20 +101,21 @@ impl CodexBackendClient {
|
||||
let body =
|
||||
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
let endpoint = endpoint_url(&self.base_url, CODEX_RESPONSES_PATH);
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
request_direction = "upstream",
|
||||
request_transport = "http_sse",
|
||||
request_method = "POST",
|
||||
request_url = %endpoint,
|
||||
request_headers = ?headers,
|
||||
request_body = %String::from_utf8_lossy(&body),
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
let trace = context
|
||||
.trace
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.exchange("http_sse");
|
||||
trace.headers(
|
||||
"upstream.request.headers",
|
||||
serde_json::json!({
|
||||
"method": "POST", "endpoint": CODEX_RESPONSES_PATH,
|
||||
}),
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
trace.capture("upstream.request.body", &body);
|
||||
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
|
||||
.map_err(CodexClientError::RequestCompression)?;
|
||||
let response = self
|
||||
@@ -133,6 +129,17 @@ impl CodexBackendClient {
|
||||
let upstream_headers_ms = elapsed_duration_millis(headers_started_at.elapsed());
|
||||
let http_version = http_version_name(response.version()).to_string();
|
||||
let status = response.status();
|
||||
trace.headers(
|
||||
"upstream.response.headers",
|
||||
serde_json::json!({
|
||||
"status": status.as_u16(), "httpVersion": http_version,
|
||||
"headersMs": upstream_headers_ms,
|
||||
}),
|
||||
response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let turn_state = response_meta::turn_state(response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
@@ -147,6 +154,7 @@ impl CodexBackendClient {
|
||||
.map(|value| value.as_bytes().to_vec());
|
||||
let client_headers = response_meta::client_headers(response.headers());
|
||||
let raw_body = read_error_response_body(response).await?;
|
||||
trace.capture("upstream.error.body", &raw_body);
|
||||
let body = String::from_utf8_lossy(&raw_body).into_owned();
|
||||
let retry_after_seconds =
|
||||
retry_after_seconds.or_else(|| retry_after_seconds_from_body(&body));
|
||||
@@ -175,7 +183,7 @@ impl CodexBackendClient {
|
||||
|
||||
let rate_limit_updates = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
body: http_sse_stream(response, Arc::clone(&rate_limit_updates)),
|
||||
body: http_sse_stream(response, Arc::clone(&rate_limit_updates), trace),
|
||||
transport: CodexBackendTransport::HttpSse,
|
||||
websocket_connection_id: None,
|
||||
turn_state,
|
||||
@@ -258,6 +266,9 @@ impl CodexBackendClient {
|
||||
defer_websocket_recovery: bool,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
let requirement = transport_requirement(request);
|
||||
context.trace.cloned().unwrap_or_default().record("transport.preparing", serde_json::json!({
|
||||
"requirement": requirement.as_str(), "connectionPreference": format!("{connection_preference:?}"),
|
||||
}));
|
||||
if requirement == TransportRequirement::HttpRequired {
|
||||
return Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
@@ -279,6 +290,15 @@ impl CodexBackendClient {
|
||||
&websocket_request,
|
||||
)
|
||||
.map_err(CodexClientError::WebSocketEncode)?;
|
||||
context.trace.cloned().unwrap_or_default().headers(
|
||||
"upstream.request.headers",
|
||||
serde_json::json!({"transport": "websocket", "phase": "prepared_opening"}),
|
||||
websocket_create
|
||||
.connection()
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| (name.as_str(), value.as_bytes())),
|
||||
);
|
||||
// 审计未启用时跳过 artifact 构造:payload 快照会深拷贝整个请求 body,
|
||||
// 且位于首字节前的关键路径上。
|
||||
if websocket_audit_dir().is_some() {
|
||||
@@ -326,14 +346,13 @@ impl CodexBackendClient {
|
||||
Ok(WebSocketFastPath::Missed) => {
|
||||
let decision = CodexTransportDecision::Http2WebSocketBudgetExhausted;
|
||||
let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed());
|
||||
tracing::info!(
|
||||
request_id = %context.request_id,
|
||||
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
|
||||
transport_requirement = requirement.as_str(),
|
||||
transport_decision = decision.as_str(),
|
||||
transport_decision_wait_ms = wait_ms,
|
||||
websocket_preconnect_continues = pool_log_context.is_some(),
|
||||
"WebSocket fast path missed; using same-account HTTP"
|
||||
context.trace.cloned().unwrap_or_default().record(
|
||||
"transport.fallback",
|
||||
serde_json::json!({
|
||||
"to": "http_sse", "reason": "websocket_fast_path_budget",
|
||||
"requirement": requirement.as_str(), "decision": decision.as_str(),
|
||||
"waitMs": wait_ms, "preconnectContinues": pool_log_context.is_some(),
|
||||
}),
|
||||
);
|
||||
return Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
@@ -354,14 +373,13 @@ impl CodexBackendClient {
|
||||
{
|
||||
let decision = http_fallback_decision(&error);
|
||||
let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed());
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
|
||||
transport_requirement = requirement.as_str(),
|
||||
transport_decision = decision.as_str(),
|
||||
transport_decision_wait_ms = wait_ms,
|
||||
error = %error,
|
||||
"WebSocket preparation failed before payload send; using same-account HTTP"
|
||||
context.trace.cloned().unwrap_or_default().record(
|
||||
"transport.fallback", serde_json::json!({
|
||||
"to": "http_sse", "reason": "websocket_pre_send_failure",
|
||||
"requirement": requirement.as_str(), "decision": decision.as_str(),
|
||||
"waitMs": wait_ms,
|
||||
"detail": diagnostic_json(&serde_json::json!({"message": error.to_string()})),
|
||||
}),
|
||||
);
|
||||
return Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
@@ -378,6 +396,15 @@ impl CodexBackendClient {
|
||||
Err(error) => return Err(websocket_exchange_error_to_client_error(error)),
|
||||
};
|
||||
let decision = websocket_success_decision(requirement, &prepared);
|
||||
context.trace.cloned().unwrap_or_default().record(
|
||||
"transport.prepared",
|
||||
serde_json::json!({
|
||||
"decision": decision.as_str(),
|
||||
"requirement": requirement.as_str(),
|
||||
"connectMs": prepared.connect_elapsed().map(elapsed_duration_millis),
|
||||
"waitMs": elapsed_duration_millis(prepared.decision_wait_elapsed()),
|
||||
}),
|
||||
);
|
||||
let metrics = CodexTransportMetrics {
|
||||
decision: Some(decision),
|
||||
ws_connect_ms: prepared.connect_elapsed().map(elapsed_duration_millis),
|
||||
@@ -394,15 +421,6 @@ impl CodexBackendClient {
|
||||
pool_log_context.as_ref(),
|
||||
prepared.pool_decision(),
|
||||
);
|
||||
tracing::info!(
|
||||
request_id = %context.request_id,
|
||||
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
|
||||
transport_requirement = requirement.as_str(),
|
||||
transport_decision = decision.as_str(),
|
||||
ws_connect_ms = ?metrics.ws_connect_ms,
|
||||
transport_decision_wait_ms = ?metrics.transport_decision_wait_ms,
|
||||
"Responses transport prepared"
|
||||
);
|
||||
Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
route: PreparedResponseRoute::WebSocket(Box::new(PreparedWebSocketRoute {
|
||||
@@ -427,6 +445,13 @@ impl CodexBackendClient {
|
||||
metrics,
|
||||
defer_websocket_recovery,
|
||||
} = prepared;
|
||||
context.trace.cloned().unwrap_or_default().record(
|
||||
"transport.selected",
|
||||
serde_json::json!({
|
||||
"decision": metrics.decision.map(|decision| decision.as_str()),
|
||||
"requirement": requirement.as_str(), "waitMs": metrics.transport_decision_wait_ms,
|
||||
}),
|
||||
);
|
||||
match route {
|
||||
PreparedResponseRoute::Http => self
|
||||
.create_response_stream_http_sse(request, context)
|
||||
@@ -441,10 +466,14 @@ impl CodexBackendClient {
|
||||
prepared,
|
||||
} = *route;
|
||||
let delivery_wait_started_at = Instant::now();
|
||||
log_unredacted_websocket_request(&websocket_request, context);
|
||||
let mut exchange = match execute_prepared_response_create_request_stream(
|
||||
&websocket_request,
|
||||
prepared,
|
||||
context
|
||||
.trace
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.exchange("websocket"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -493,9 +522,12 @@ impl CodexBackendClient {
|
||||
.await;
|
||||
}
|
||||
fresh_retry_used = true;
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
"WebSocket connection limit reached; retrying on a fresh connection"
|
||||
context.trace.cloned().unwrap_or_default().record(
|
||||
"transport.retry",
|
||||
serde_json::json!({
|
||||
"reason": "websocket_connection_limit_reached",
|
||||
"connectionPreference": "fresh",
|
||||
}),
|
||||
);
|
||||
drop(exchange);
|
||||
match self
|
||||
@@ -512,10 +544,14 @@ impl CodexBackendClient {
|
||||
route: PreparedResponseRoute::WebSocket(route),
|
||||
..
|
||||
}) => {
|
||||
log_unredacted_websocket_request(&route.request, context);
|
||||
match execute_prepared_response_create_request_stream(
|
||||
&route.request,
|
||||
route.prepared,
|
||||
context
|
||||
.trace
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.exchange("websocket"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -575,8 +611,9 @@ impl CodexBackendClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) if !defer_websocket_recovery
|
||||
&& error.allows_pre_delivery_http_fallback() =>
|
||||
Err(error)
|
||||
if !defer_websocket_recovery
|
||||
&& error.allows_pre_delivery_http_fallback() =>
|
||||
{
|
||||
return self
|
||||
.fallback_to_http_before_websocket_delivery(
|
||||
@@ -597,12 +634,6 @@ impl CodexBackendClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::info!(
|
||||
request_id = %context.request_id,
|
||||
websocket_connection_id = %exchange.websocket_connection_id,
|
||||
ws_pool = exchange.pool_decision.map_or("unpooled", WebSocketPoolDecision::kind),
|
||||
"WebSocket response stream established"
|
||||
);
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
body: Box::pin(
|
||||
exchange
|
||||
@@ -625,15 +656,6 @@ impl CodexBackendClient {
|
||||
})
|
||||
}
|
||||
}
|
||||
.inspect_err(|error| {
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
transport_requirement = requirement.as_str(),
|
||||
failure_phase = "post_send_or_explicit_response",
|
||||
error = %error,
|
||||
"Responses stream transport failed after preparation"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
async fn fallback_to_http_before_websocket_delivery(
|
||||
@@ -671,14 +693,15 @@ impl CodexBackendClient {
|
||||
.saturating_add(elapsed_duration_millis(delivery_wait_started_at.elapsed()));
|
||||
metrics.decision = Some(CodexTransportDecision::Http2PreDeliveryFailure);
|
||||
metrics.transport_decision_wait_ms = Some(decision_wait_ms);
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
transport_requirement = requirement.as_str(),
|
||||
transport_decision = CodexTransportDecision::Http2PreDeliveryFailure.as_str(),
|
||||
transport_decision_wait_ms = decision_wait_ms,
|
||||
error = %detail,
|
||||
"WebSocket failed before first deliverable event; using same-account HTTP"
|
||||
context.trace.cloned().unwrap_or_default().record(
|
||||
"transport.fallback",
|
||||
serde_json::json!({
|
||||
"to": "http_sse", "reason": "websocket_pre_delivery_failure",
|
||||
"requirement": requirement.as_str(),
|
||||
"decision": CodexTransportDecision::Http2PreDeliveryFailure.as_str(),
|
||||
"waitMs": decision_wait_ms,
|
||||
"detail": diagnostic_json(&serde_json::json!({"message": detail})),
|
||||
}),
|
||||
);
|
||||
self.create_response_stream_http_sse(request, context)
|
||||
.await
|
||||
@@ -715,7 +738,7 @@ impl CodexBackendClient {
|
||||
) -> CodexClientResult<CodexModelCatalogSnapshot> {
|
||||
let endpoint = endpoint_url(&self.base_url, "codex/models");
|
||||
let profile = self.profile.snapshot();
|
||||
let headers = self.auxiliary_request_headers(&profile, context)?;
|
||||
let headers = self.model_request_headers(&profile, context)?;
|
||||
let response = self
|
||||
.client
|
||||
.get(endpoint)
|
||||
@@ -751,182 +774,6 @@ impl CodexBackendClient {
|
||||
}
|
||||
Ok(parse_codex_model_catalog(&body, etag.as_deref())?)
|
||||
}
|
||||
|
||||
fn request_headers_for_http_response(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let profile = self.profile.snapshot();
|
||||
let mut headers = self.request_headers(&profile, context)?;
|
||||
if let Some(subagent) = openai_subagent_from_metadata(request.client_metadata()) {
|
||||
insert_optional_protocol_header(&mut headers, "x-openai-subagent", Some(&subagent));
|
||||
}
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER,
|
||||
request.responses_lite.as_deref(),
|
||||
);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
request.memgen_request.as_deref(),
|
||||
);
|
||||
// 客户端确实携带的普通协议头优先;画像身份头只能由 Codex profile 生成。
|
||||
for name in request.passthrough_headers.keys() {
|
||||
// 上游指纹必须由运行时画像统一生成:originator/User-Agent/version 即使
|
||||
// 绕过 API 透传黑名单也不能覆盖画像,避免下游客户端暴露不一致指纹。
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"openai-beta"
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
| "x-openai-internal-codex-residency"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
headers.remove(name);
|
||||
for value in request.passthrough_headers.get_all(name) {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
fn request_headers_for_websocket_response(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = self.request_headers_for_http_response(request, context)?;
|
||||
headers.remove(X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER);
|
||||
headers.insert(
|
||||
HeaderName::from_static("openai-beta"),
|
||||
HeaderValue::from_static("responses_websockets=2026-02-06"),
|
||||
);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
fn request_headers(
|
||||
&self,
|
||||
profile: &CodexWireProfile,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers =
|
||||
build_codex_base_headers(profile, context.authorization, context.account_id)?;
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream"));
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-client-request-id"),
|
||||
HeaderValue::from_str(context.request_id)?,
|
||||
);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-client-request-id",
|
||||
context.client_request_id,
|
||||
);
|
||||
insert_optional_header(
|
||||
&mut headers,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
)?;
|
||||
insert_optional_protocol_header(&mut headers, "session-id", context.session_id);
|
||||
insert_optional_protocol_header(&mut headers, "thread-id", context.thread_id);
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-turn-id", context.turn_id);
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-window-id", context.codex_window_id);
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-turn-state", context.turn_state);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-turn-metadata",
|
||||
context.turn_metadata,
|
||||
);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-beta-features",
|
||||
context.beta_features,
|
||||
);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-responsesapi-include-timing-metrics",
|
||||
context.include_timing_metrics,
|
||||
);
|
||||
// 官方 Desktop 的版本画像来自 clientInfo.version。保留下游是否携带
|
||||
// `version` 扩展头的协议形状,但上游值统一使用当前 Desktop 制品版本。
|
||||
if context
|
||||
.version
|
||||
.is_some_and(|version| HeaderValue::from_str(version).is_ok())
|
||||
{
|
||||
headers.insert(
|
||||
HeaderName::from_static("version"),
|
||||
HeaderValue::from_str(&profile.desktop_version)?,
|
||||
);
|
||||
}
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-parent-thread-id",
|
||||
context.parent_thread_id,
|
||||
);
|
||||
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
fn auxiliary_request_headers(
|
||||
&self,
|
||||
profile: &CodexWireProfile,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers =
|
||||
build_codex_base_headers(profile, context.authorization, context.account_id)?;
|
||||
if let Some(cookie_header) = context.cookie_header {
|
||||
headers.insert(COOKIE, HeaderValue::from_str(cookie_header)?);
|
||||
}
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
|
||||
insert_optional_header(
|
||||
&mut headers,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
)?;
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
pub(in crate::transport) fn usage_request_headers(
|
||||
&self,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let profile = self.profile.snapshot();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
|
||||
headers.insert(AUTHORIZATION, HeaderValue::from_str(context.authorization)?);
|
||||
insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?;
|
||||
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
|
||||
Ok(headers)
|
||||
}
|
||||
}
|
||||
|
||||
fn log_unredacted_websocket_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) {
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
request_direction = "upstream",
|
||||
request_transport = "websocket",
|
||||
request_method = "GET + response.create",
|
||||
request_url = request.connection().endpoint(),
|
||||
request_headers = ?request.connection().headers(),
|
||||
request_payload = request.payload_text(),
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
}
|
||||
|
||||
/// 首个可投递帧前的交付边界结果。
|
||||
@@ -1011,6 +858,7 @@ fn websocket_connection_profile(headers: &HeaderMap) -> String {
|
||||
fn http_sse_stream(
|
||||
response: ReqwestResponse,
|
||||
rate_limit_updates: CodexRateLimitUpdates,
|
||||
trace: TraceContext,
|
||||
) -> CodexBackendSseStream {
|
||||
let stream: CodexBackendSseStream =
|
||||
Box::pin(response.bytes_stream().map_err(CodexClientError::Http));
|
||||
@@ -1028,6 +876,20 @@ fn http_sse_stream(
|
||||
)),
|
||||
}
|
||||
}));
|
||||
let stream = Box::pin(async_stream::stream! {
|
||||
let mut stream = stream;
|
||||
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
|
||||
while let Some(chunk) = stream.next().await {
|
||||
match &chunk {
|
||||
Ok(bytes) => capture.push(bytes),
|
||||
Err(error) => trace.record("upstream.read.failed", diagnostic_json(&serde_json::json!({"error": error.to_string()}))),
|
||||
}
|
||||
let failed = chunk.is_err();
|
||||
yield chunk;
|
||||
if failed { return; }
|
||||
}
|
||||
capture.finish();
|
||||
});
|
||||
observe_http_sse_rate_limits(stream, rate_limit_updates)
|
||||
}
|
||||
|
||||
|
||||
@@ -29,12 +29,17 @@ pub fn endpoint_url(base_url: &str, endpoint_path: &str) -> String {
|
||||
|
||||
/// 返回与 base path 对应的唯一 usage endpoint。
|
||||
pub fn usage_endpoint_url(base_url: &str) -> String {
|
||||
let path = if has_backend_api_base_path(base_url) {
|
||||
WHAM_USAGE_PATH
|
||||
account_endpoint_url(base_url, "usage")
|
||||
}
|
||||
|
||||
/// Core backend-client 的两个官方路径风格,供所有账号接口共用。
|
||||
pub fn account_endpoint_url(base_url: &str, resource: &str) -> String {
|
||||
let prefix = if has_backend_api_base_path(base_url) {
|
||||
"wham"
|
||||
} else {
|
||||
CODEX_USAGE_API_PATH
|
||||
"api/codex"
|
||||
};
|
||||
endpoint_url(base_url, path)
|
||||
endpoint_url(base_url, &format!("{prefix}/{resource}"))
|
||||
}
|
||||
|
||||
fn has_backend_api_base_path(base_url: &str) -> bool {
|
||||
|
||||
@@ -1,23 +1,37 @@
|
||||
use crate::transport::profile::CodexWireProfile;
|
||||
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue, USER_AGENT};
|
||||
//! 各官方调用链的请求头契约;版本与平台信息统一来自运行时画像快照。
|
||||
|
||||
use super::client::CodexClientResult;
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
};
|
||||
use reqwest::header::{
|
||||
ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT,
|
||||
};
|
||||
|
||||
use super::client::{
|
||||
CodexBackendClient, CodexClientResult, CodexRequestContext, openai_subagent_from_metadata,
|
||||
};
|
||||
use super::profile::{CodexResidency, CodexWireProfile};
|
||||
use super::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
const CODEX_RESIDENCY_HEADER: &str = "x-openai-internal-codex-residency";
|
||||
|
||||
/// 构造 Codex Core 为模型请求设置的稳定身份请求头。
|
||||
pub fn build_codex_base_headers(
|
||||
pub fn build_codex_model_headers(
|
||||
profile: &CodexWireProfile,
|
||||
authorization: &str,
|
||||
account_id: Option<&str>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = build_codex_profile_headers(profile)?;
|
||||
headers.insert(
|
||||
HeaderName::from_static("version"),
|
||||
HeaderValue::from_str(&profile.codex_version)?,
|
||||
);
|
||||
headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?);
|
||||
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// 构造不依赖访问凭据的 Codex Desktop 身份请求头。
|
||||
/// Core 默认客户端身份,用于模型接口及 OAuth refresh;raw token exchange 不使用。
|
||||
pub fn build_codex_profile_headers(profile: &CodexWireProfile) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
@@ -25,14 +39,182 @@ pub fn build_codex_profile_headers(profile: &CodexWireProfile) -> CodexClientRes
|
||||
HeaderValue::from_str(&profile.originator)?,
|
||||
);
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
|
||||
// 此部署固定采用官方支持的 US residency requirement。
|
||||
if let Some(CodexResidency::Us) = profile.residency {
|
||||
headers.insert(
|
||||
HeaderName::from_static(CODEX_RESIDENCY_HEADER),
|
||||
HeaderValue::from_static("us"),
|
||||
);
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Core backend-client 的账号请求:不携带模型版本或 originator。
|
||||
pub fn build_codex_account_headers(
|
||||
profile: &CodexWireProfile,
|
||||
authorization: &str,
|
||||
account_id: Option<&str>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&profile.user_agent())?);
|
||||
headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?);
|
||||
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// 桌面下载的账号身份;完整性凭据须由认证主体持有,不能借用下游客户端的值。
|
||||
pub fn build_codex_download_headers(
|
||||
profile: &CodexWireProfile,
|
||||
authorization: &str,
|
||||
account_id: Option<&str>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
HeaderName::from_static(CODEX_RESIDENCY_HEADER),
|
||||
HeaderValue::from_static("us"),
|
||||
USER_AGENT,
|
||||
HeaderValue::from_str(&profile.desktop_user_agent())?,
|
||||
);
|
||||
headers.insert(AUTHORIZATION, HeaderValue::from_str(authorization)?);
|
||||
insert_optional_header(&mut headers, "chatgpt-account-id", account_id)?;
|
||||
headers.insert(
|
||||
HeaderName::from_static("originator"),
|
||||
HeaderValue::from_str(&profile.originator)?,
|
||||
);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
impl CodexBackendClient {
|
||||
pub(super) fn model_request_headers(
|
||||
&self,
|
||||
profile: &CodexWireProfile,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers =
|
||||
build_codex_model_headers(profile, context.authorization, context.account_id)?;
|
||||
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
pub(super) fn account_request_headers(
|
||||
&self,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = build_codex_account_headers(
|
||||
&self.profile.snapshot(),
|
||||
context.authorization,
|
||||
context.account_id,
|
||||
)?;
|
||||
insert_optional_header(&mut headers, "cookie", context.cookie_header)?;
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
pub(super) fn request_headers_for_http_response(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = self.response_headers(request, context)?;
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("text/event-stream"));
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER,
|
||||
request.responses_lite.as_deref(),
|
||||
);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
pub(super) fn request_headers_for_websocket_response(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = self.response_headers(request, context)?;
|
||||
headers.insert(
|
||||
HeaderName::from_static("openai-beta"),
|
||||
HeaderValue::from_static("responses_websockets=2026-02-06"),
|
||||
);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// HTTP 和 WS 共用的 Core 会话字段;传输专属字段由各自入口生成。
|
||||
fn response_headers(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let mut headers = self.model_request_headers(&self.profile.snapshot(), context)?;
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-client-request-id"),
|
||||
HeaderValue::from_str(context.request_id)?,
|
||||
);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-client-request-id",
|
||||
context
|
||||
.client_request_id
|
||||
.or(context.thread_id)
|
||||
.or(context.session_id),
|
||||
);
|
||||
for (name, value) in [
|
||||
("session-id", context.session_id),
|
||||
("thread-id", context.thread_id),
|
||||
("x-codex-window-id", context.codex_window_id),
|
||||
("x-codex-turn-state", context.turn_state),
|
||||
("x-codex-turn-metadata", context.turn_metadata),
|
||||
("x-codex-beta-features", context.beta_features),
|
||||
(
|
||||
"x-responsesapi-include-timing-metrics",
|
||||
context.include_timing_metrics,
|
||||
),
|
||||
("x-codex-parent-thread-id", context.parent_thread_id),
|
||||
(
|
||||
X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
request.memgen_request.as_deref(),
|
||||
),
|
||||
] {
|
||||
insert_optional_protocol_header(&mut headers, name, value);
|
||||
}
|
||||
if let Some(subagent) = openai_subagent_from_metadata(request.client_metadata()) {
|
||||
insert_optional_protocol_header(&mut headers, "x-openai-subagent", Some(&subagent));
|
||||
}
|
||||
let routing_hint = match request.service_tier() {
|
||||
Some(tier) => format!("model={};tier={tier}", request.model()),
|
||||
None => format!("model={}", request.model()),
|
||||
};
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-routing-hint", Some(&routing_hint));
|
||||
for name in request.passthrough_headers.keys() {
|
||||
// 身份与传输字段只由画像/正文生成;其余协议头保留原始多值字节。
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
| "authorization"
|
||||
| "chatgpt-account-id"
|
||||
| "cookie"
|
||||
| "x-openai-internal-codex-residency"
|
||||
| "openai-beta"
|
||||
| "accept"
|
||||
| "content-type"
|
||||
| "content-encoding"
|
||||
| "x-codex-routing-hint"
|
||||
| "x-codex-installation-id"
|
||||
| "x-codex-turn-id"
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
| X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
headers.remove(name);
|
||||
for value in request.passthrough_headers.get_all(name) {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn insert_optional_header(
|
||||
headers: &mut HeaderMap,
|
||||
name: &'static str,
|
||||
|
||||
@@ -42,7 +42,7 @@ pub use self::{
|
||||
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
|
||||
WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url,
|
||||
},
|
||||
headers::build_codex_base_headers,
|
||||
headers::build_codex_model_headers,
|
||||
profile_avatar::{
|
||||
CodexProfileAvatar, CodexProfileAvatarFetchError, CodexProfileAvatarStreamError,
|
||||
fetch_profile_avatar,
|
||||
|
||||
@@ -32,6 +32,13 @@ const MAX_APPCAST_BYTES: usize = 1024 * 1024;
|
||||
const ARTIFACT_PROFILE_CACHE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
|
||||
const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1;
|
||||
|
||||
/// 与官方 managed residency requirement 相同的可选请求约束。
|
||||
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum CodexResidency {
|
||||
Us,
|
||||
}
|
||||
|
||||
/// Codex Desktop 上游请求身份。
|
||||
///
|
||||
/// 启动配置提供经源码审计的 Core、运行环境和 Desktop 启动版本。运行时只会使用
|
||||
@@ -40,7 +47,7 @@ const ARTIFACT_PROFILE_SCHEMA_VERSION: u64 = 1;
|
||||
pub struct CodexWireProfile {
|
||||
/// `originator` 请求头及 User-Agent 产品名。
|
||||
pub originator: String,
|
||||
/// Desktop ZIP 内嵌 Core 版本;用于 `/codex/models?client_version=` 与 UA。
|
||||
/// Desktop ZIP 内嵌 Core 版本;用于模型请求的 version、client_version 与 UA。
|
||||
pub codex_version: String,
|
||||
/// Desktop 应用版本,用于 app-server `clientInfo.version` 对应的 UA 后缀。
|
||||
pub desktop_version: String,
|
||||
@@ -54,7 +61,9 @@ pub struct CodexWireProfile {
|
||||
pub arch: String,
|
||||
/// Codex Core UA 中的终端标记。
|
||||
pub terminal: String,
|
||||
/// 此画像最后一次经制品与源码核验的时间。
|
||||
/// 未配置时不发送 residency 头;不随制品版本更新而改变。
|
||||
pub residency: Option<CodexResidency>,
|
||||
/// 版本元组最后一次经制品核验的时间;不表示 TLS 传输已重新核验。
|
||||
pub verified_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
|
||||
@@ -7,17 +7,18 @@ use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use reqwest::{
|
||||
Client, Url,
|
||||
header::{ACCEPT, CONTENT_TYPE, ETAG, USER_AGENT},
|
||||
header::{CONTENT_TYPE, ETAG},
|
||||
};
|
||||
use tokio::time::timeout;
|
||||
|
||||
use super::endpoints::endpoint_url;
|
||||
use super::{
|
||||
CodexRequestContext, headers::build_codex_download_headers, profile::CodexWireProfile,
|
||||
};
|
||||
|
||||
const OFFICIAL_AVATAR_ORIGIN: &str = "https://chatgpt.com";
|
||||
const OFFICIAL_AVATAR_PATH_PREFIX: &str = "/backend-api/estuary/public_content/enc/";
|
||||
const PROVIDER_AVATAR_PATH_PREFIX: &str = "/estuary/public_content/enc/";
|
||||
const PROFILE_AVATAR_ACCEPT: &str =
|
||||
"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8";
|
||||
const PROFILE_AVATAR_HEADERS_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const PROFILE_AVATAR_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
@@ -68,19 +69,18 @@ pub enum CodexProfileAvatarFetchError {
|
||||
pub async fn fetch_profile_avatar(
|
||||
client: &Client,
|
||||
base_url: &str,
|
||||
desktop_user_agent: &str,
|
||||
profile: &CodexWireProfile,
|
||||
source: &str,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> Result<CodexProfileAvatar, CodexProfileAvatarFetchError> {
|
||||
let source_path = official_avatar_source_path(source)?;
|
||||
let target = Url::parse(&endpoint_url(base_url, &source_path))
|
||||
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?;
|
||||
let headers = build_codex_download_headers(profile, context.authorization, context.account_id)
|
||||
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?;
|
||||
let response = timeout(
|
||||
PROFILE_AVATAR_HEADERS_TIMEOUT,
|
||||
client
|
||||
.get(target)
|
||||
.header(ACCEPT, PROFILE_AVATAR_ACCEPT)
|
||||
.header(USER_AGENT, desktop_user_agent)
|
||||
.send(),
|
||||
client.get(target).headers(headers).send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?
|
||||
|
||||
@@ -12,7 +12,7 @@ use super::{
|
||||
retry_after_seconds,
|
||||
},
|
||||
diagnostics::CodexUpstreamSendPhase,
|
||||
endpoints::{WHAM_PROFILE_STATISTICS_PATH, endpoint_url},
|
||||
endpoints::account_endpoint_url,
|
||||
response_meta,
|
||||
};
|
||||
|
||||
@@ -167,8 +167,8 @@ impl CodexBackendClient {
|
||||
) -> CodexClientResult<CodexProfileStatistics> {
|
||||
let response = self
|
||||
.client
|
||||
.get(endpoint_url(&self.base_url, WHAM_PROFILE_STATISTICS_PATH))
|
||||
.headers(self.usage_request_headers(context)?)
|
||||
.get(account_endpoint_url(&self.base_url, "profiles/me"))
|
||||
.headers(self.account_request_headers(context)?)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
|
||||
@@ -320,8 +320,8 @@ fn normalize_conversation_anchor_text(text: &str) -> String {
|
||||
|
||||
/// 把客户端正文收敛到当前 lease 的账号身份边界。
|
||||
///
|
||||
/// 真实 account ID 与 installation ID 由随后构造的 `CodexRequestContext` 注入
|
||||
/// 请求头;正文只替换客户端原本声明过的 installation 字段,绝不接受客户端
|
||||
/// 真实 account ID 由随后构造的 `CodexRequestContext` 注入请求头;installation ID
|
||||
/// 统一写入 Core client_metadata,并替换原有兼容字段,绝不接受客户端
|
||||
/// 提供的 token、cookie 或账号身份。`input` 是 Responses 的可回放会话正文,
|
||||
/// item ID、encrypted content 和 compaction 都必须原样保留。
|
||||
pub(crate) fn scope_request_to_account(
|
||||
@@ -371,59 +371,57 @@ pub(crate) fn scope_request_to_account(
|
||||
replace_existing_body_string(request, key, scoped.as_deref());
|
||||
}
|
||||
|
||||
if let Some(client_metadata) = request.client_metadata().cloned() {
|
||||
let scoped = match client_metadata {
|
||||
Value::Object(mut metadata) => {
|
||||
let scoped_turn_metadata =
|
||||
["turnMetadata", "turn_metadata", "x-codex-turn-metadata"].map(|key| {
|
||||
(
|
||||
key,
|
||||
metadata.get(key).and_then(Value::as_str).and_then(|value| {
|
||||
scope_turn_metadata(value, installation_id, reset_account_state)
|
||||
}),
|
||||
)
|
||||
});
|
||||
if reset_account_state {
|
||||
for key in CROSS_ACCOUNT_IDENTITY_KEYS
|
||||
.iter()
|
||||
.chain(ACCOUNT_BOUND_STATE_KEYS)
|
||||
{
|
||||
metadata.remove(*key);
|
||||
}
|
||||
let client_metadata = request
|
||||
.client_metadata()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Value::Object(Map::new()));
|
||||
let scoped = match client_metadata {
|
||||
Value::Object(mut metadata) => {
|
||||
let scoped_turn_metadata = ["turnMetadata", "turn_metadata", "x-codex-turn-metadata"]
|
||||
.map(|key| {
|
||||
(
|
||||
key,
|
||||
metadata.get(key).and_then(Value::as_str).and_then(|value| {
|
||||
scope_turn_metadata(value, installation_id, reset_account_state)
|
||||
}),
|
||||
)
|
||||
});
|
||||
if reset_account_state {
|
||||
for key in CROSS_ACCOUNT_IDENTITY_KEYS
|
||||
.iter()
|
||||
.chain(ACCOUNT_BOUND_STATE_KEYS)
|
||||
{
|
||||
metadata.remove(*key);
|
||||
}
|
||||
metadata.insert(
|
||||
"x-codex-installation-id".to_owned(),
|
||||
Value::String(installation_id.to_owned()),
|
||||
);
|
||||
replace_existing_metadata_field(
|
||||
&mut metadata,
|
||||
"installation_id",
|
||||
Some(installation_id),
|
||||
);
|
||||
replace_existing_metadata_field(
|
||||
&mut metadata,
|
||||
"installationId",
|
||||
Some(installation_id),
|
||||
);
|
||||
replace_metadata_field(
|
||||
&mut metadata,
|
||||
"x-codex-turn-state",
|
||||
client_metadata_turn_state.as_deref(),
|
||||
);
|
||||
replace_metadata_field(
|
||||
&mut metadata,
|
||||
"x-codex-turn-metadata",
|
||||
client_metadata_turn_metadata.as_deref(),
|
||||
);
|
||||
for (key, value) in scoped_turn_metadata {
|
||||
replace_existing_metadata_field(&mut metadata, key, value.as_deref());
|
||||
}
|
||||
(!metadata.is_empty()).then_some(Value::Object(metadata))
|
||||
}
|
||||
value => Some(value),
|
||||
};
|
||||
request.set_client_metadata(scoped);
|
||||
}
|
||||
metadata.insert(
|
||||
"installation_id".to_owned(),
|
||||
Value::String(installation_id.to_owned()),
|
||||
);
|
||||
replace_existing_metadata_field(
|
||||
&mut metadata,
|
||||
"x-codex-installation-id",
|
||||
Some(installation_id),
|
||||
);
|
||||
replace_existing_metadata_field(&mut metadata, "installationId", Some(installation_id));
|
||||
replace_metadata_field(
|
||||
&mut metadata,
|
||||
"x-codex-turn-state",
|
||||
client_metadata_turn_state.as_deref(),
|
||||
);
|
||||
replace_metadata_field(
|
||||
&mut metadata,
|
||||
"x-codex-turn-metadata",
|
||||
client_metadata_turn_metadata.as_deref(),
|
||||
);
|
||||
for (key, value) in scoped_turn_metadata {
|
||||
replace_existing_metadata_field(&mut metadata, key, value.as_deref());
|
||||
}
|
||||
(!metadata.is_empty()).then_some(Value::Object(metadata))
|
||||
}
|
||||
value => Some(value),
|
||||
};
|
||||
request.set_client_metadata(scoped);
|
||||
|
||||
request.turn_state = turn_state;
|
||||
request.turn_metadata = turn_metadata;
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
//! Codex Desktop 主动额度重置卡 HTTP contract。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::{
|
||||
StatusCode,
|
||||
header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT},
|
||||
};
|
||||
use reqwest::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -15,11 +12,7 @@ use super::{
|
||||
retry_after_seconds,
|
||||
},
|
||||
diagnostics::CodexUpstreamSendPhase,
|
||||
endpoints::{
|
||||
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
|
||||
endpoint_url,
|
||||
},
|
||||
headers::insert_optional_header,
|
||||
endpoints::account_endpoint_url,
|
||||
response_meta,
|
||||
};
|
||||
|
||||
@@ -94,11 +87,11 @@ impl CodexBackendClient {
|
||||
) -> CodexClientResult<CodexRateLimitResetCredits> {
|
||||
let response = self
|
||||
.client
|
||||
.get(endpoint_url(
|
||||
.get(account_endpoint_url(
|
||||
&self.base_url,
|
||||
WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
|
||||
"rate-limit-reset-credits",
|
||||
))
|
||||
.headers(self.reset_credits_headers(context, false)?)
|
||||
.headers(self.account_request_headers(context)?)
|
||||
.send()
|
||||
.await
|
||||
.map_err(CodexClientError::HttpJson)?;
|
||||
@@ -131,11 +124,11 @@ impl CodexBackendClient {
|
||||
) -> CodexClientResult<CodexRateLimitResetCreditsConsumeResult> {
|
||||
let response = self
|
||||
.client
|
||||
.post(endpoint_url(
|
||||
.post(account_endpoint_url(
|
||||
&self.base_url,
|
||||
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH,
|
||||
"rate-limit-reset-credits/consume",
|
||||
))
|
||||
.headers(self.reset_credits_headers(context, true)?)
|
||||
.headers(self.account_request_headers(context)?)
|
||||
.json(&ConsumeRequest {
|
||||
credit_id,
|
||||
redeem_request_id: redeem_request_id.to_string(),
|
||||
@@ -151,36 +144,6 @@ impl CodexBackendClient {
|
||||
credit: wire.credit.map(parse_credit).transpose()?,
|
||||
})
|
||||
}
|
||||
|
||||
fn reset_credits_headers(
|
||||
&self,
|
||||
context: CodexRequestContext<'_>,
|
||||
json_body: bool,
|
||||
) -> CodexClientResult<HeaderMap> {
|
||||
let profile = self.profile.snapshot();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(AUTHORIZATION, HeaderValue::from_str(context.authorization)?);
|
||||
insert_optional_header(&mut headers, "chatgpt-account-id", context.account_id)?;
|
||||
headers.insert(
|
||||
HeaderName::from_static("originator"),
|
||||
HeaderValue::from_str(&profile.originator)?,
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("oai-language"),
|
||||
HeaderValue::from_static("en"),
|
||||
);
|
||||
headers.insert(
|
||||
USER_AGENT,
|
||||
HeaderValue::from_str(&profile.desktop_user_agent())?,
|
||||
);
|
||||
// Electron `net.fetch` 为 renderer 请求附加的默认 Accept;该 endpoint 的
|
||||
// renderer 并没有显式改写为 Core auxiliary 请求的 application/json 画像。
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
|
||||
if json_body {
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_reset_credits_response(response: reqwest::Response) -> CodexClientResult<String> {
|
||||
|
||||
@@ -15,13 +15,13 @@ const CA_CERT_HINT: &str = "If you set CODEX_CA_CERTIFICATE or SSL_CERT_FILE, en
|
||||
|
||||
type PemSection = (SectionKind, Vec<u8>);
|
||||
|
||||
/// rustls 0.23 在同时编译 `ring` 与 `aws-lc-rs` 时无法自动选择进程级
|
||||
/// CryptoProvider;本工程固定使用 ring(与官方 Codex Desktop JA3 一致),
|
||||
/// 因此在首次 TLS 使用前显式安装一次。重复安装会被 rustls 忽略。
|
||||
/// 与官方 Codex 的 rustls 路径一致,安装支持 P-521 和混合后量子密钥交换的
|
||||
/// aws-lc provider。依赖图也包含 ring,因此不能让 rustls 自动选择 provider。
|
||||
/// 与官方嵌入场景一致,保留宿主在初始化之前已经安装的 provider。
|
||||
pub fn ensure_rustls_provider() {
|
||||
static INSTALL: OnceLock<()> = OnceLock::new();
|
||||
INSTALL.get_or_init(|| {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -676,7 +676,7 @@ pub const MAX_CODEX_USAGE_BODY_BYTES: usize = 1024 * 1024;
|
||||
impl CodexBackendClient {
|
||||
/// 获取 Codex usage JSON。
|
||||
pub async fn fetch_usage(&self, context: CodexRequestContext<'_>) -> CodexClientResult<Value> {
|
||||
let headers = self.usage_request_headers(context)?;
|
||||
let headers = self.account_request_headers(context)?;
|
||||
let response = self
|
||||
.client
|
||||
.get(usage_endpoint_url(&self.base_url))
|
||||
|
||||
@@ -559,6 +559,7 @@ fn continuation_unavailable(
|
||||
pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
request: &CodexWebSocketRequest,
|
||||
prepared: PreparedWebSocket,
|
||||
trace: gateway_core::diagnostics::TraceContext,
|
||||
) -> Result<CodexWebSocketStreamingExchange, CodexWebSocketExchangeError> {
|
||||
let PreparedWebSocket {
|
||||
connection,
|
||||
@@ -573,7 +574,19 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
continuation,
|
||||
created_at,
|
||||
} = connection;
|
||||
trace.record("upstream.connection", serde_json::json!({
|
||||
"connectionId": websocket.connection_id().to_string(), "reused": reused,
|
||||
"pool": pool_decision.map_or("unpooled", WebSocketPoolDecision::kind),
|
||||
"status": metadata.diagnostics.status_code,
|
||||
"upstreamRequestId": metadata.diagnostics.request_id,
|
||||
"headers": gateway_core::diagnostics::diagnostic_headers(metadata.diagnostics.trace_headers.iter().map(|(name, value)| (name.as_str(), value.as_str()))),
|
||||
}));
|
||||
trace.capture("upstream.request.body", request.payload_text().as_bytes());
|
||||
if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await {
|
||||
trace.record(
|
||||
"upstream.send.failed",
|
||||
serde_json::json!({"phase": "websocket_payload", "sendState": "ambiguous"}),
|
||||
);
|
||||
let observation = websocket
|
||||
.observation()
|
||||
.with_exit_reason("outbound_transport_error");
|
||||
@@ -582,6 +595,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
error.with_connection_observation(observation),
|
||||
));
|
||||
}
|
||||
trace.record("upstream.payload.sent", serde_json::json!({}));
|
||||
let connection_local_available = lease.is_some();
|
||||
let pool_return = lease.map(|lease| WebSocketStreamPoolReturn {
|
||||
lease,
|
||||
@@ -594,6 +608,7 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
pool_return,
|
||||
reused,
|
||||
initial_event_timeout,
|
||||
trace,
|
||||
);
|
||||
exchange.pool_decision = pool_decision;
|
||||
exchange.connection_local_continuation = connection_local_available;
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use bytes::Bytes;
|
||||
use gateway_core::diagnostics::{TraceContext, diagnostic_json};
|
||||
use serde_json::json;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -61,6 +63,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
|
||||
pool_return: Option<WebSocketStreamPoolReturn>,
|
||||
reused_connection: bool,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
trace: TraceContext,
|
||||
) -> CodexWebSocketStreamingExchange {
|
||||
let websocket_connection_id = websocket.connection_id();
|
||||
let response_metadata = metadata.clone();
|
||||
@@ -83,6 +86,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
|
||||
pool_return,
|
||||
reused_connection,
|
||||
initial_event_timeout,
|
||||
trace,
|
||||
shutdown,
|
||||
rate_limit_updates: rate_limit_updates_for_task,
|
||||
turn_state_update: turn_state_update_for_task,
|
||||
@@ -116,6 +120,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
|
||||
}
|
||||
|
||||
struct WebSocketStreamForwardState {
|
||||
trace: TraceContext,
|
||||
websocket: PumpedWebSocket,
|
||||
metadata: CodexWebSocketConnectionMetadata,
|
||||
pool_return: Option<WebSocketStreamPoolReturn>,
|
||||
@@ -129,6 +134,7 @@ struct WebSocketStreamForwardState {
|
||||
|
||||
async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
let WebSocketStreamForwardState {
|
||||
trace,
|
||||
mut websocket,
|
||||
mut metadata,
|
||||
pool_return,
|
||||
@@ -152,6 +158,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
// 客户端断开下游 SSE 流:立即丢弃连接并释放池 slot,
|
||||
// 不再傻等上游 idle 超时(否则同会话后续请求会一直 bypass/busy)。
|
||||
() = tx.closed() => {
|
||||
trace.record("upstream.cancelled", json!({"reason": "receiver_dropped"}));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
@@ -160,6 +167,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
return;
|
||||
}
|
||||
() = shutdown.cancelled() => {
|
||||
trace.record("upstream.cancelled", json!({"reason": "pool_shutdown"}));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
@@ -183,6 +191,13 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
}
|
||||
error => error,
|
||||
};
|
||||
trace.record(
|
||||
"upstream.read.failed",
|
||||
json!({
|
||||
"lastEventType": last_event_type,
|
||||
"error": diagnostic_json(&json!({"message": error.to_string()})),
|
||||
}),
|
||||
);
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -201,6 +216,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
}
|
||||
};
|
||||
let Some(message) = message else {
|
||||
trace.record(
|
||||
"upstream.eof",
|
||||
json!({"lastEventType": last_event_type, "terminalSeen": false}),
|
||||
);
|
||||
break;
|
||||
};
|
||||
let raw = match message {
|
||||
@@ -208,7 +227,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
saw_upstream_activity = true;
|
||||
text.to_string()
|
||||
}
|
||||
tungstenite::Message::Binary(_) => {
|
||||
tungstenite::Message::Binary(bytes) => {
|
||||
trace.capture("upstream.binary", &bytes);
|
||||
let error = CodexWebSocketExchangeError::UnexpectedBinaryEvent;
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
@@ -223,6 +243,15 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
return;
|
||||
}
|
||||
tungstenite::Message::Close(frame) => {
|
||||
if let Some(frame) = &frame {
|
||||
trace.dump("upstream.close.reason", frame.reason.as_bytes());
|
||||
}
|
||||
trace.record("upstream.close", json!({
|
||||
"code": frame.as_ref().map(|frame| u16::from(frame.code)),
|
||||
"reason": frame.as_ref().map(|frame| diagnostic_json(&json!({"message": frame.reason.as_str()}))),
|
||||
"lastEventType": last_event_type, "terminalSeen": false,
|
||||
"connectionId": websocket.connection_id().to_string(),
|
||||
}));
|
||||
let websocket_connection_id = websocket.connection_id();
|
||||
let error = CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
websocket_connection_id,
|
||||
@@ -248,6 +277,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
trace.capture("upstream.event", raw.as_bytes());
|
||||
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
|
||||
Ok(reduced) => reduced,
|
||||
Err(error) => {
|
||||
@@ -286,6 +316,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
ExchangeAction::Ignore => continue,
|
||||
};
|
||||
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
|
||||
trace.record(
|
||||
"upstream.forward.failed",
|
||||
json!({"reason": "receiver_dropped"}),
|
||||
);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
@@ -295,6 +329,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
return;
|
||||
}
|
||||
if let Some(terminal) = terminal {
|
||||
trace.record(
|
||||
"upstream.terminal",
|
||||
json!({"kind": format!("{terminal:?}")}),
|
||||
);
|
||||
match terminal {
|
||||
WebSocketTerminalKind::Completed => {
|
||||
finish_stream_websocket(websocket, metadata, continuation, pool_return.take())
|
||||
|
||||
@@ -261,7 +261,7 @@ async fn openai_admin_provider_exposes_live_wire_profile_and_validated_billing()
|
||||
async fn reset_credit_success_with_invalid_body_should_remain_an_unknown_consume_result() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/wham/rate-limit-reset-credits/consume"))
|
||||
.and(path("/api/codex/rate-limit-reset-credits/consume"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw("{}", "application/json"))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
@@ -287,7 +287,7 @@ async fn reset_credit_success_with_invalid_body_should_remain_an_unknown_consume
|
||||
async fn reset_credit_explicit_http_rejection_should_preserve_the_raw_body() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/wham/rate-limit-reset-credits/consume"))
|
||||
.and(path("/api/codex/rate-limit-reset-credits/consume"))
|
||||
.respond_with(ResponseTemplate::new(409).set_body_raw(
|
||||
r#"{"code":"nothing_to_reset","detail":"window is fresh"}"#,
|
||||
"application/json",
|
||||
@@ -1152,6 +1152,7 @@ fn valid_config() -> TestOpenAiConfig {
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 19, 0, 0, 0)
|
||||
.single()
|
||||
|
||||
@@ -127,7 +127,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
);
|
||||
assert_eq!(
|
||||
config.wire_profile_state().snapshot().user_agent(),
|
||||
"Codex Desktop/0.147.0-alpha.6.6 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.803.81509)"
|
||||
"Codex Desktop/0.153.4 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.901.51231)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -153,6 +153,7 @@ fn valid_config() -> OpenAiConfig {
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 19, 0, 0, 0)
|
||||
.single()
|
||||
|
||||
@@ -60,6 +60,7 @@ fn wire_profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "catalog-contract".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
|
||||
.single()
|
||||
|
||||
@@ -160,6 +160,7 @@ fn selector_with_runtime(
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "selector-contract".to_owned(),
|
||||
residency: None,
|
||||
verified_at: chrono::Utc::now(),
|
||||
});
|
||||
let http = reqwest::Client::builder().build().expect("HTTP client");
|
||||
|
||||
@@ -310,7 +310,7 @@ async fn authorize_url_matches_the_official_desktop_parameter_contract() {
|
||||
);
|
||||
assert_eq!(
|
||||
parameters.get("codex_app_version").map(String::as_str),
|
||||
Some("26.803.81509")
|
||||
Some("26.901.51231")
|
||||
);
|
||||
let surface_stable_id = parameters
|
||||
.get("source_surface_stable_id")
|
||||
|
||||
@@ -21,6 +21,7 @@ fn wire_profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "profile-statistics-contract".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 28, 0, 0, 0)
|
||||
.single()
|
||||
@@ -59,7 +60,7 @@ async fn service(
|
||||
async fn profile_statistics_uses_one_profile_request_and_preserves_official_fields() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/wham/profiles/me"))
|
||||
.and(path("/api/codex/profiles/me"))
|
||||
.and(header(
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-acct_profile_statistics",
|
||||
@@ -135,7 +136,7 @@ async fn profile_statistics_uses_one_profile_request_and_preserves_official_fiel
|
||||
async fn profile_statistics_keeps_profile_when_stats_are_unavailable() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/wham/profiles/me"))
|
||||
.and(path("/api/codex/profiles/me"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"profile": {"display_name": "Ada"},
|
||||
"stats": {},
|
||||
@@ -158,10 +159,10 @@ async fn profile_statistics_keeps_profile_when_stats_are_unavailable() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials() {
|
||||
async fn profile_avatar_reuses_cached_source_with_the_accounts_rotated_token() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/wham/profiles/me"))
|
||||
.and(path("/api/codex/profiles/me"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"profile": {
|
||||
"display_name": "Ada",
|
||||
@@ -192,6 +193,11 @@ async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials()
|
||||
.profile_statistics(account.id())
|
||||
.await
|
||||
.expect("profile statistics");
|
||||
store
|
||||
.repository()
|
||||
.rotate_refreshed_oauth_secret(&account, secret("rotated-avatar-token"), None, None)
|
||||
.await
|
||||
.expect("rotate avatar account token");
|
||||
let avatar = service
|
||||
.profile_avatar(account.id())
|
||||
.await
|
||||
@@ -211,7 +217,13 @@ async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials()
|
||||
|
||||
assert_eq!(content_type.as_deref(), Some("image/svg+xml"));
|
||||
assert_eq!(body, b"<svg xmlns=\"http://www.w3.org/2000/svg\"/>");
|
||||
assert!(!avatar_request.headers.contains_key("authorization"));
|
||||
assert_eq!(
|
||||
avatar_request.headers["authorization"],
|
||||
"Bearer rotated-avatar-token"
|
||||
);
|
||||
assert!(!avatar_request.headers.contains_key("cookie"));
|
||||
assert!(!avatar_request.headers.contains_key("chatgpt-account-id"));
|
||||
assert_eq!(
|
||||
avatar_request.headers["chatgpt-account-id"],
|
||||
"chatgpt-acct_profile_avatar"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ fn wire_profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "quota-contract".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
|
||||
.single()
|
||||
|
||||
@@ -431,6 +431,7 @@ async fn scheduled_refresh_persists_retryable_message_inside_the_two_hour_window
|
||||
client_id: "test-public-client".to_owned(),
|
||||
token_endpoint: format!("{}/oauth/token", server.uri()),
|
||||
},
|
||||
provider_openai::OpenAiConfig::default().wire_profile_state(),
|
||||
)),
|
||||
Arc::new(RefreshLeases),
|
||||
Arc::new(RefreshCredentialState),
|
||||
|
||||
@@ -6,6 +6,73 @@ use secrecy::{ExposeSecret, SecretString};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_tracks_the_online_profile_without_contaminating_code_exchange() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/oauth/token"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"access_token": "audit-access", "refresh_token": "audit-refresh", "id_token": "header.e30.signature"
|
||||
})))
|
||||
.expect(4)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let profile = provider_openai::OpenAiConfig::default().wire_profile_state();
|
||||
let client = provider_openai::credential::token_client::openai_token_client(
|
||||
TokenClientConfig {
|
||||
client_id: "audit-client".to_owned(),
|
||||
token_endpoint: format!("{}/oauth/token", server.uri()),
|
||||
},
|
||||
profile.clone(),
|
||||
)
|
||||
.expect("production auth transport");
|
||||
for (core, desktop, build) in [
|
||||
("1.2.3", "26.800.10000", "8000"),
|
||||
("1.2.4", "26.901.10000", "8109"),
|
||||
] {
|
||||
profile.update_bundled_release(
|
||||
&provider_openai::transport::profile::CodexBundledReleaseProfile {
|
||||
codex_version: core.to_owned(),
|
||||
desktop_version: desktop.to_owned(),
|
||||
desktop_build: build.to_owned(),
|
||||
verified_at: chrono::Utc::now(),
|
||||
},
|
||||
);
|
||||
client.refresh("audit-refresh").await.expect("refresh");
|
||||
client
|
||||
.exchange_authorization_code(AuthorizationCodeGrant {
|
||||
code: SecretString::from("audit-code"),
|
||||
code_verifier: SecretString::from("audit-verifier"),
|
||||
})
|
||||
.await
|
||||
.expect("code exchange");
|
||||
let requests = server.received_requests().await.expect("requests");
|
||||
let refresh = &requests[requests.len() - 2];
|
||||
let exchange = &requests[requests.len() - 1];
|
||||
assert_eq!(
|
||||
refresh.headers["user-agent"],
|
||||
profile.snapshot().user_agent()
|
||||
);
|
||||
assert_eq!(refresh.headers["originator"], "Codex Desktop");
|
||||
assert_eq!(refresh.headers["content-type"], "application/json");
|
||||
assert_eq!(
|
||||
exchange.headers["content-type"],
|
||||
"application/x-www-form-urlencoded"
|
||||
);
|
||||
for name in [
|
||||
"originator",
|
||||
"user-agent",
|
||||
"version",
|
||||
"x-openai-internal-codex-residency",
|
||||
] {
|
||||
assert!(!exchange.headers.contains_key(name), "raw exchange: {name}");
|
||||
}
|
||||
for name in ["authorization", "chatgpt-account-id", "version"] {
|
||||
assert!(!refresh.headers.contains_key(name), "refresh: {name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn client(server: &MockServer) -> OpenAiTokenClient {
|
||||
OpenAiTokenClient::new(
|
||||
reqwest::Client::builder()
|
||||
@@ -16,6 +83,7 @@ fn client(server: &MockServer) -> OpenAiTokenClient {
|
||||
client_id: "test-public-client".to_owned(),
|
||||
token_endpoint: format!("{}/oauth/token", server.uri()),
|
||||
},
|
||||
provider_openai::OpenAiConfig::default().wire_profile_state(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -136,6 +136,7 @@ fn wire_profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "provider-contract".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc::now(),
|
||||
})
|
||||
}
|
||||
@@ -1004,6 +1005,8 @@ async fn image_endpoints_bypass_only_the_text_catalog_and_preserve_the_current_c
|
||||
.and(path(*endpoint))
|
||||
.and(header("originator", "codex_cli_rs"))
|
||||
.and(header("x-codex-image-turn-id", "turn_image_contract"))
|
||||
.and(header("version", "0.144.0"))
|
||||
.and(header("accept", "*/*"))
|
||||
.and(body_bytes(body.to_vec()))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
@@ -1143,6 +1146,8 @@ async fn standalone_search_preserves_wire_and_scopes_turn_metadata_to_the_select
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/alpha/search"))
|
||||
.and(header("originator", "codex_cli_rs"))
|
||||
.and(header("version", "0.144.0"))
|
||||
.and(header("accept", "*/*"))
|
||||
.and(body_bytes(request_body.to_vec()))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
@@ -2731,6 +2736,15 @@ async fn same_account_scope_preserves_future_protocol_shapes() {
|
||||
body.get("installation_id"),
|
||||
Some(&json!("client-installation"))
|
||||
);
|
||||
assert_eq!(
|
||||
body.get("installation_id"),
|
||||
body.pointer("/client_metadata/installation_id")
|
||||
);
|
||||
assert!(
|
||||
body.pointer("/client_metadata/x-codex-installation-id")
|
||||
.is_none()
|
||||
);
|
||||
assert!(captured_header_values(&request, "x-codex-installation-id").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -223,9 +223,8 @@ async fn fetch_should_send_official_catalog_headers_and_query() {
|
||||
"user-agent",
|
||||
"codex_cli_rs/0.144.0 (linux 6.8; x86_64) xterm (codex_cli_rs; 1.0.0)",
|
||||
))
|
||||
.and(header("x-openai-internal-codex-residency", "us"))
|
||||
.and(header("accept", "application/json"))
|
||||
.and(header("x-codex-installation-id", "installation-123"))
|
||||
.and(header("accept", "*/*"))
|
||||
.and(header("version", "0.144.0"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "application/json")
|
||||
@@ -306,6 +305,7 @@ fn profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "xterm".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
|
||||
.single()
|
||||
@@ -315,6 +315,7 @@ fn profile() -> CodexWireProfileState {
|
||||
|
||||
fn context() -> CodexRequestContext<'static> {
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer oauth-access",
|
||||
account_id: Some("acct_123"),
|
||||
request_id: "req_catalog",
|
||||
|
||||
@@ -25,6 +25,7 @@ fn upstream_error_formatting_should_redact_body() {
|
||||
#[test]
|
||||
fn request_context_debug_should_redact_all_identity_material() {
|
||||
let context = CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-secret-marker",
|
||||
account_id: Some("account-secret-marker"),
|
||||
request_id: "req_safe",
|
||||
|
||||
@@ -257,3 +257,87 @@ fn openai_account_failure_matrix_is_preserved() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_wire_dump_preserves_binary_bytes_and_numbers_every_fragment() {
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Writer(Arc<Mutex<Vec<u8>>>);
|
||||
impl std::io::Write for Writer {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().unwrap().write_all(bytes)?;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let output = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = Writer(Arc::clone(&output));
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.with_writer(move || writer.clone())
|
||||
.with_ansi(false)
|
||||
.finish();
|
||||
let bytes: Vec<u8> = (0..100_000).map(|index| (index % 256) as u8).collect();
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
let trace = TraceContext::new("req_binary_dump");
|
||||
trace
|
||||
.attempt(3)
|
||||
.exchange("websocket")
|
||||
.dump("upstream.binary", &bytes);
|
||||
trace.headers(
|
||||
"client.connection.headers",
|
||||
serde_json::json!({}),
|
||||
[
|
||||
("authorization", b"Bearer secret".as_slice()),
|
||||
("x-binary", b"\xff\x80".as_slice()),
|
||||
("x-binary", b"\x00".as_slice()),
|
||||
],
|
||||
);
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
assert_eq!(snapshot["wireFrames"], 2);
|
||||
assert!(!snapshot.to_string().contains("Bearer secret"));
|
||||
});
|
||||
let log = output.lock().unwrap();
|
||||
let records: Vec<serde_json::Value> = std::str::from_utf8(&log)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str(line).unwrap())
|
||||
.filter(|record: &serde_json::Value| record["target"] == "request_dump")
|
||||
.collect();
|
||||
assert_eq!(records.len(), 5);
|
||||
let mut recovered = Vec::new();
|
||||
for (index, record) in records.iter().take(4).enumerate() {
|
||||
let fields = &record["fields"];
|
||||
assert_eq!(fields["wire_sequence"], 1);
|
||||
assert_eq!(fields["attempt_index"], 3);
|
||||
assert_eq!(fields["exchange_id"], 1);
|
||||
assert_eq!(fields["chunk_index"], index);
|
||||
assert_eq!(fields["chunk_count"], 4);
|
||||
recovered.extend(
|
||||
STANDARD
|
||||
.decode(fields["body_base64"].as_str().unwrap())
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
assert_eq!(recovered, bytes);
|
||||
let header_record = &records[4]["fields"];
|
||||
assert_eq!(header_record["wire_sequence"], 2);
|
||||
let header_bytes = STANDARD
|
||||
.decode(header_record["body_base64"].as_str().unwrap())
|
||||
.unwrap();
|
||||
let headers: serde_json::Value = serde_json::from_slice(&header_bytes).unwrap();
|
||||
for (index, expected) in [b"Bearer secret".as_slice(), b"\xff\x80", b"\x00"]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let value = STANDARD
|
||||
.decode(headers["headers"][index]["valueBase64"].as_str().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(value, expected);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,6 +277,7 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_ws_security",
|
||||
@@ -317,12 +318,8 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
);
|
||||
let headers = received_headers.lock().expect("headers lock");
|
||||
for (name, expected) in [
|
||||
("x-client-request-id", "req_ws_security"),
|
||||
("content-type", "application/json"),
|
||||
("accept", "text/event-stream"),
|
||||
("x-codex-installation-id", "install-123"),
|
||||
("x-client-request-id", "cp_derived"),
|
||||
("openai-beta", "responses_websockets=2026-02-06"),
|
||||
("x-openai-internal-codex-residency", "us"),
|
||||
("x-codex-turn-state", "turn-state"),
|
||||
("x-codex-turn-metadata", "{\"thread_source\":\"subagent\"}"),
|
||||
("x-codex-beta-features", "feature-a"),
|
||||
@@ -341,6 +338,11 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
);
|
||||
}
|
||||
for forbidden in [
|
||||
"x-codex-installation-id",
|
||||
"x-codex-turn-id",
|
||||
"x-openai-internal-codex-residency",
|
||||
"accept",
|
||||
"content-type",
|
||||
"session_id",
|
||||
"thread-id",
|
||||
"x-openai-internal-codex-responses-lite",
|
||||
@@ -365,7 +367,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
let profile = test_wire_profile();
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_desktop_version = profile_snapshot.desktop_version;
|
||||
let expected_core_version = profile_snapshot.codex_version;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -379,6 +381,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer lease-token",
|
||||
account_id: Some("lease-account"),
|
||||
installation_id: Some("lease-installation"),
|
||||
@@ -401,16 +404,11 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
vec![b"turn-ascii".to_vec(), b"turn-\x80".to_vec()]
|
||||
);
|
||||
for (name, value) in [
|
||||
("accept", b"application/vnd.openai.future+json".as_slice()),
|
||||
(
|
||||
"content-type",
|
||||
b"application/vnd.openai.request+json".as_slice(),
|
||||
),
|
||||
("x-openai-internal-codex-residency", b"us".as_slice()),
|
||||
("accept", b"text/event-stream".as_slice()),
|
||||
("content-type", b"application/json".as_slice()),
|
||||
("x-still-valid", b"after-invalid".as_slice()),
|
||||
("authorization", b"Bearer lease-token".as_slice()),
|
||||
("chatgpt-account-id", b"lease-account".as_slice()),
|
||||
("x-codex-installation-id", b"lease-installation".as_slice()),
|
||||
] {
|
||||
assert_eq!(raw_header_values(&raw, name), vec![value.to_vec()]);
|
||||
}
|
||||
@@ -425,7 +423,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
);
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, "version"),
|
||||
vec![expected_desktop_version.into_bytes()]
|
||||
vec![expected_core_version.into_bytes()]
|
||||
);
|
||||
for dropped in [
|
||||
"openai-beta",
|
||||
@@ -501,6 +499,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer lease-token",
|
||||
account_id: Some("lease-account"),
|
||||
installation_id: Some("lease-installation"),
|
||||
@@ -529,24 +528,15 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
values("openai-beta"),
|
||||
vec![b"responses_websockets=2026-02-06".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("accept"),
|
||||
vec![b"application/vnd.openai.future+json".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("content-type"),
|
||||
vec![b"application/vnd.openai.request+json".to_vec()]
|
||||
);
|
||||
assert!(values("accept").is_empty());
|
||||
assert!(values("content-type").is_empty());
|
||||
assert_eq!(
|
||||
values("user-agent"),
|
||||
vec![test_wire_profile().snapshot().user_agent().into_bytes()]
|
||||
);
|
||||
assert_eq!(values("originator"), vec![b"codex_cli_rs".to_vec()]);
|
||||
assert_eq!(values("version"), vec![b"1.2.3".to_vec()]);
|
||||
assert_eq!(
|
||||
values("x-openai-internal-codex-residency"),
|
||||
vec![b"us".to_vec()]
|
||||
);
|
||||
assert!(values("x-openai-internal-codex-residency").is_empty());
|
||||
for dropped in [
|
||||
"x-openai-actor-authorization",
|
||||
"x-oai-attestation",
|
||||
@@ -563,10 +553,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
values("chatgpt-account-id"),
|
||||
vec![b"lease-account".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("x-codex-installation-id"),
|
||||
vec![b"lease-installation".to_vec()]
|
||||
);
|
||||
assert!(values("x-codex-installation-id").is_empty());
|
||||
assert_eq!(values("x-codex-turn-state"), vec![b"turn-ascii".to_vec()]);
|
||||
}
|
||||
|
||||
@@ -593,7 +580,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
let profile = test_wire_profile();
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_desktop_version = profile_snapshot.desktop_version;
|
||||
let expected_core_version = profile_snapshot.codex_version;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -607,6 +594,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_order",
|
||||
@@ -648,14 +636,14 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
Some("codex_cli_rs"),
|
||||
Some(expected_user_agent.as_str()),
|
||||
Some("text/event-stream"),
|
||||
Some("us"),
|
||||
Some("req_order"),
|
||||
None,
|
||||
Some("session-1"),
|
||||
Some("turn-state"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
read_header_value(&raw_request, "version"),
|
||||
Some(expected_desktop_version.as_str())
|
||||
Some(expected_core_version.as_str())
|
||||
);
|
||||
for required in [
|
||||
"authorization",
|
||||
@@ -665,9 +653,7 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
"content-type",
|
||||
"cookie",
|
||||
"accept",
|
||||
"x-openai-internal-codex-residency",
|
||||
"x-client-request-id",
|
||||
"x-codex-installation-id",
|
||||
"session-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-turn-state",
|
||||
@@ -680,6 +666,9 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
assert!(header_names.iter().any(|name| name == required));
|
||||
}
|
||||
for forbidden in [
|
||||
"x-codex-installation-id",
|
||||
"x-codex-turn-id",
|
||||
"x-openai-internal-codex-residency",
|
||||
"sec-ch-ua",
|
||||
"sec-ch-ua-mobile",
|
||||
"sec-ch-ua-platform",
|
||||
@@ -729,6 +718,7 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
client_request_id: Some("opaque\nclient-request-id"),
|
||||
session_id: Some("opaque\nsession"),
|
||||
thread_id: Some("opaque\nthread"),
|
||||
@@ -751,6 +741,7 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo
|
||||
read_header_value(&raw_request, "x-client-request-id"),
|
||||
Some("req_protocol_fallback")
|
||||
);
|
||||
assert_eq!(read_header_value(&raw_request, "version"), Some("1.2.3"));
|
||||
for omitted in [
|
||||
"session-id",
|
||||
"thread-id",
|
||||
@@ -760,7 +751,6 @@ async fn backend_http_should_ignore_unrepresentable_protocol_headers_without_blo
|
||||
"x-codex-turn-metadata",
|
||||
"x-codex-beta-features",
|
||||
"x-responsesapi-include-timing-metrics",
|
||||
"version",
|
||||
"x-codex-parent-thread-id",
|
||||
"x-openai-subagent",
|
||||
"x-openai-internal-codex-responses-lite",
|
||||
@@ -897,7 +887,14 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
|
||||
let server = tokio::spawn(async move {
|
||||
let (first_stream, _) = listener.accept().await.expect("first websocket");
|
||||
let mut first_user_agent = String::new();
|
||||
let mut first_version = String::new();
|
||||
let mut first = accept_codex_test_websocket_with(first_stream, |request, _response| {
|
||||
first_version = request.headers()["version"]
|
||||
.to_str()
|
||||
.expect("Core version")
|
||||
.to_owned();
|
||||
assert!(!request.headers().contains_key("accept"));
|
||||
assert!(!request.headers().contains_key("content-type"));
|
||||
first_user_agent = request
|
||||
.headers()
|
||||
.get("user-agent")
|
||||
@@ -935,7 +932,14 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
|
||||
.expect("new chain should use a new wire profile connection")
|
||||
.expect("second websocket");
|
||||
let mut second_user_agent = String::new();
|
||||
let mut second_version = String::new();
|
||||
let mut second = accept_codex_test_websocket_with(second_stream, |request, _response| {
|
||||
second_version = request.headers()["version"]
|
||||
.to_str()
|
||||
.expect("Core version")
|
||||
.to_owned();
|
||||
assert!(!request.headers().contains_key("accept"));
|
||||
assert!(!request.headers().contains_key("content-type"));
|
||||
second_user_agent = request
|
||||
.headers()
|
||||
.get("user-agent")
|
||||
@@ -956,7 +960,12 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
|
||||
.await
|
||||
.expect("second response.completed");
|
||||
|
||||
(first_user_agent, second_user_agent)
|
||||
(
|
||||
first_user_agent,
|
||||
second_user_agent,
|
||||
first_version,
|
||||
second_version,
|
||||
)
|
||||
});
|
||||
|
||||
let profile = test_wire_profile();
|
||||
@@ -983,7 +992,7 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
|
||||
.expect("first response");
|
||||
profile.update_bundled_release(&CodexBundledReleaseProfile {
|
||||
codex_version: "1.2.4".to_owned(),
|
||||
desktop_version: "1.2.4".to_owned(),
|
||||
desktop_version: "26.999.10000".to_owned(),
|
||||
desktop_build: "124".to_owned(),
|
||||
verified_at: Utc::now(),
|
||||
});
|
||||
@@ -1006,8 +1015,12 @@ async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_la
|
||||
.await
|
||||
.expect("second response");
|
||||
|
||||
let (first_user_agent, second_user_agent) = server.await.expect("profile server task");
|
||||
let (first_user_agent, second_user_agent, first_version, second_version) =
|
||||
server.await.expect("profile server task");
|
||||
assert!(first_user_agent.contains("1.2.3"));
|
||||
assert!(second_user_agent.contains("1.2.4"));
|
||||
assert!(second_user_agent.contains("26.999.10000"));
|
||||
assert_eq!(first_version, "1.2.3");
|
||||
assert_eq!(second_version, "1.2.4");
|
||||
assert_ne!(first_user_agent, second_user_agent);
|
||||
}
|
||||
|
||||
@@ -137,6 +137,7 @@ async fn codex_backend_client_should_preserve_oversized_error_response() {
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_large_error",
|
||||
@@ -236,6 +237,7 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_http_retry_after_body",
|
||||
@@ -294,10 +296,11 @@ async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updat
|
||||
let mut request = codex_request("gpt-5.5", "", Vec::new());
|
||||
request.force_http_sse = true;
|
||||
|
||||
let trace = gateway_core::diagnostics::TraceContext::new("req_http_rate_limit_event");
|
||||
let response = client
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_http_rate_limit_event", Some("acct")),
|
||||
request_context("req_http_rate_limit_event", Some("acct")).with_trace(&trace),
|
||||
)
|
||||
.await
|
||||
.expect("HTTP SSE response");
|
||||
@@ -315,6 +318,18 @@ async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updat
|
||||
("x-codex-limit-reached".to_owned(), "false".to_owned()),
|
||||
]
|
||||
);
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event["data"]["eventType"] == "codex.rate_limits")
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event["data"]["eventType"] == "response.completed")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
@@ -407,6 +422,7 @@ async fn codex_backend_client_should_capture_forwardable_response_metadata() {
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_response_metadata",
|
||||
|
||||
@@ -55,9 +55,11 @@ mod http_client;
|
||||
mod latency;
|
||||
mod profile;
|
||||
mod profile_avatar;
|
||||
mod profile_contract;
|
||||
mod protocol;
|
||||
mod request;
|
||||
mod reset_credits;
|
||||
mod tls;
|
||||
mod usage;
|
||||
mod websocket;
|
||||
mod websocket_pool;
|
||||
@@ -126,6 +128,7 @@ fn test_wire_profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "transport-test".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc::now(),
|
||||
})
|
||||
}
|
||||
@@ -135,6 +138,7 @@ fn request_context<'a>(
|
||||
account_id: Option<&'a str>,
|
||||
) -> CodexRequestContext<'a> {
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id,
|
||||
request_id,
|
||||
|
||||
@@ -141,6 +141,7 @@ fn wire_profile_should_generate_bundled_core_app_server_user_agent() {
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 3, 0, 0, 0)
|
||||
.single()
|
||||
@@ -360,6 +361,7 @@ fn wire_profile() -> CodexWireProfile {
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 3, 0, 0, 0)
|
||||
.single()
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use futures::TryStreamExt as _;
|
||||
use provider_openai::transport::{CodexProfileAvatarFetchError, fetch_profile_avatar};
|
||||
use provider_openai::transport::{
|
||||
CodexProfileAvatarFetchError, CodexRequestContext, fetch_profile_avatar,
|
||||
};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
@@ -30,8 +32,14 @@ async fn profile_avatar_streams_unrestricted_content_type_and_body_size() {
|
||||
let avatar = fetch_profile_avatar(
|
||||
&client,
|
||||
&server.uri(),
|
||||
"codex_cli_rs/1.0.0 (linux; x86_64)",
|
||||
&super::test_wire_profile().snapshot(),
|
||||
OFFICIAL_AVATAR_SOURCE,
|
||||
CodexRequestContext::auxiliary(
|
||||
"Bearer avatar-token",
|
||||
Some("avatar-account"),
|
||||
"avatar",
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("profile avatar");
|
||||
@@ -52,14 +60,18 @@ async fn profile_avatar_streams_unrestricted_content_type_and_body_size() {
|
||||
assert_eq!(streamed, body);
|
||||
assert_eq!(
|
||||
headers.get("accept").and_then(|value| value.to_str().ok()),
|
||||
Some("image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
|
||||
Some("*/*")
|
||||
);
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("user-agent")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("codex_cli_rs/1.0.0 (linux; x86_64)")
|
||||
Some("codex_cli_rs/1.2.3 (linux; x86_64)")
|
||||
);
|
||||
assert_eq!(headers["authorization"], "Bearer avatar-token");
|
||||
assert_eq!(headers["chatgpt-account-id"], "avatar-account");
|
||||
assert_eq!(headers["originator"], "codex_cli_rs");
|
||||
assert!(!headers.contains_key("version"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -72,9 +84,20 @@ async fn profile_avatar_rejects_non_official_sources_before_request() {
|
||||
"https://chatgpt.com/backend-api/estuary/public_content/enc/token?next=1",
|
||||
"https://chatgpt.com/backend-api/estuary/public_content/enc/",
|
||||
] {
|
||||
let error = fetch_profile_avatar(&client, "http://127.0.0.1:9", "test-agent", source)
|
||||
.await
|
||||
.expect_err("invalid source");
|
||||
let error = fetch_profile_avatar(
|
||||
&client,
|
||||
"http://127.0.0.1:9",
|
||||
&super::test_wire_profile().snapshot(),
|
||||
source,
|
||||
CodexRequestContext::auxiliary(
|
||||
"Bearer avatar-token",
|
||||
Some("avatar-account"),
|
||||
"avatar",
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("invalid source");
|
||||
assert!(matches!(error, CodexProfileAvatarFetchError::InvalidSource));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
use super::*;
|
||||
use provider_openai::transport::build_reqwest_client;
|
||||
use provider_openai::transport::profile::CodexResidency;
|
||||
use uuid::Uuid;
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
|
||||
|
||||
/// 官方本机样本:模型请求带 Core version,backend-client 账号请求不带。
|
||||
/// 通过真实 HTTP 请求验证画像更新后每条路径取到同一制品中的正确字段。
|
||||
#[tokio::test]
|
||||
async fn each_core_endpoint_uses_the_current_bundled_release() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(any())
|
||||
.respond_with(ResponseTemplate::new(400).set_body_json(json!({"error":"audit fixture"})))
|
||||
.expect(12)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let profile = test_wire_profile();
|
||||
let client = CodexBackendClient::new(
|
||||
build_reqwest_client().expect("HTTP client"),
|
||||
format!("{}/backend-api", server.uri()),
|
||||
profile.clone(),
|
||||
);
|
||||
for (core, desktop, build) in [
|
||||
("0.152.8", "26.800.10000", "8000"),
|
||||
("0.153.4", "26.901.51231", "8109"),
|
||||
] {
|
||||
profile.update_bundled_release(&CodexBundledReleaseProfile {
|
||||
codex_version: core.to_owned(),
|
||||
desktop_version: desktop.to_owned(),
|
||||
desktop_build: build.to_owned(),
|
||||
verified_at: Utc::now(),
|
||||
});
|
||||
let context = request_context("audit", Some("audit-account"));
|
||||
client
|
||||
.fetch_models_with_context(context)
|
||||
.await
|
||||
.expect_err("fixture rejection");
|
||||
client
|
||||
.fetch_usage(context)
|
||||
.await
|
||||
.expect_err("fixture rejection");
|
||||
client
|
||||
.fetch_profile_statistics(context)
|
||||
.await
|
||||
.expect_err("fixture rejection");
|
||||
client
|
||||
.list_rate_limit_reset_credits(context)
|
||||
.await
|
||||
.expect_err("fixture rejection");
|
||||
client
|
||||
.consume_rate_limit_reset_credit(context, None, Uuid::now_v7())
|
||||
.await
|
||||
.expect_err("fixture rejection");
|
||||
let mut body = codex_request_body("gpt-test", "", Vec::new());
|
||||
body.insert("service_tier".to_owned(), json!("priority"));
|
||||
let mut request = CodexResponsesRequest::from_body(body);
|
||||
request.force_http_sse = true;
|
||||
client
|
||||
.create_response(&request, context)
|
||||
.await
|
||||
.expect_err("fixture rejection");
|
||||
|
||||
let captured = server.received_requests().await.expect("captured requests");
|
||||
for request in &captured[captured.len() - 6..] {
|
||||
let headers = &request.headers;
|
||||
assert_eq!(headers["user-agent"], profile.snapshot().user_agent());
|
||||
assert_eq!(headers["authorization"], "Bearer access-token");
|
||||
assert_eq!(headers["chatgpt-account-id"], "audit-account");
|
||||
for forbidden in [
|
||||
"x-codex-installation-id",
|
||||
"x-codex-turn-id",
|
||||
"x-openai-internal-codex-residency",
|
||||
"oai-language",
|
||||
"openai-beta",
|
||||
] {
|
||||
assert!(
|
||||
!headers.contains_key(forbidden),
|
||||
"{}: {forbidden}",
|
||||
request.url
|
||||
);
|
||||
}
|
||||
if request.url.path().contains("/codex/") {
|
||||
assert_eq!(headers["originator"], "codex_cli_rs");
|
||||
assert_eq!(headers["version"], core);
|
||||
assert_ne!(headers["version"], desktop);
|
||||
} else {
|
||||
assert!(!headers.contains_key("originator"));
|
||||
assert!(!headers.contains_key("version"));
|
||||
}
|
||||
if request.url.path().ends_with("/responses") {
|
||||
assert_eq!(headers["accept"], "text/event-stream");
|
||||
assert_eq!(
|
||||
headers["x-codex-routing-hint"],
|
||||
"model=gpt-test;tier=priority"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(headers["accept"], "*/*");
|
||||
}
|
||||
if request.url.path().ends_with("/models") {
|
||||
assert_eq!(
|
||||
request.url.query(),
|
||||
Some(format!("client_version={core}").as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn residency_is_explicit_and_survives_artifact_updates() {
|
||||
let mut initial = test_wire_profile().snapshot();
|
||||
initial.residency = Some(CodexResidency::Us);
|
||||
let profile = CodexWireProfileState::new(initial);
|
||||
profile.update_bundled_release(&CodexBundledReleaseProfile {
|
||||
codex_version: "2.0.0".to_owned(),
|
||||
desktop_version: "26.999.10000".to_owned(),
|
||||
desktop_build: "9999".to_owned(),
|
||||
verified_at: Utc::now(),
|
||||
});
|
||||
let snapshot = profile.snapshot();
|
||||
let model = provider_openai::transport::headers::build_codex_model_headers(
|
||||
&snapshot,
|
||||
"Bearer audit",
|
||||
None,
|
||||
)
|
||||
.expect("model headers");
|
||||
let account = provider_openai::transport::headers::build_codex_account_headers(
|
||||
&snapshot,
|
||||
"Bearer audit",
|
||||
None,
|
||||
)
|
||||
.expect("account headers");
|
||||
assert_eq!(model["x-openai-internal-codex-residency"], "us");
|
||||
assert_eq!(model["version"], "2.0.0");
|
||||
assert!(!account.contains_key("x-openai-internal-codex-residency"));
|
||||
}
|
||||
@@ -26,6 +26,7 @@ fn client(base_url: &str) -> CodexBackendClient {
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 20, 0, 0, 0)
|
||||
.single()
|
||||
@@ -44,10 +45,10 @@ fn context() -> CodexRequestContext<'static> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_should_match_official_desktop_surface_profile() {
|
||||
async fn list_should_match_official_core_account_profile() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/wham/rate-limit-reset-credits"))
|
||||
.and(path("/api/codex/rate-limit-reset-credits"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"available_count": 1,
|
||||
"credits": [{
|
||||
@@ -81,11 +82,13 @@ async fn list_should_match_official_desktop_surface_profile() {
|
||||
header(request, "chatgpt-account-id"),
|
||||
Some("acct_workspace")
|
||||
);
|
||||
assert_eq!(header(request, "originator"), Some("Codex Desktop"));
|
||||
assert_eq!(header(request, "oai-language"), Some("en"));
|
||||
assert_eq!(header(request, "originator"), None);
|
||||
assert_eq!(header(request, "oai-language"), None);
|
||||
assert_eq!(
|
||||
header(request, "user-agent"),
|
||||
Some("Codex Desktop/26.818.21641 (Mac OS; arm64)")
|
||||
Some(
|
||||
"Codex Desktop/0.115.0-alpha.11 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 26.818.21641)"
|
||||
)
|
||||
);
|
||||
assert_eq!(header(request, "accept"), Some("*/*"));
|
||||
for name in [
|
||||
@@ -110,7 +113,7 @@ async fn list_should_match_official_desktop_surface_profile() {
|
||||
async fn consume_should_send_exact_credit_and_redeem_request_id_without_retry() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/wham/rate-limit-reset-credits/consume"))
|
||||
.and(path("/api/codex/rate-limit-reset-credits/consume"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"code": "reset",
|
||||
"credit": {
|
||||
@@ -147,7 +150,7 @@ async fn consume_should_send_exact_credit_and_redeem_request_id_without_retry()
|
||||
async fn consume_should_preserve_raw_upstream_error_and_never_retry() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/wham/rate-limit-reset-credits/consume"))
|
||||
.and(path("/api/codex/rate-limit-reset-credits/consume"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(429)
|
||||
.insert_header("retry-after", "17")
|
||||
@@ -188,7 +191,7 @@ async fn consume_should_preserve_raw_upstream_error_and_never_retry() {
|
||||
async fn list_should_reject_oversized_success_body() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/wham/rate-limit-reset-credits"))
|
||||
.and(path("/api/codex/rate-limit-reset-credits"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(
|
||||
"x".repeat(MAX_CODEX_RESET_CREDITS_BODY_BYTES + 1),
|
||||
"application/json",
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use bytes::{Buf as _, Bytes};
|
||||
|
||||
use super::*;
|
||||
|
||||
// Captured from Desktop 26.901.51231 / Core 0.153.4 on 2026-09-06.
|
||||
// Compare wire semantics: rustls randomizes extension order and ephemeral key bytes.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct ClientHello {
|
||||
cipher_suites: Vec<u16>,
|
||||
extensions: Vec<u16>,
|
||||
groups: Vec<u16>,
|
||||
signature_algorithms: Vec<u16>,
|
||||
key_shares: Vec<(u16, usize)>,
|
||||
alpn: Vec<String>,
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_client_hello_should_match_official_rustls_transport() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!(
|
||||
"https://localhost:{}/",
|
||||
listener.local_addr().unwrap().port()
|
||||
);
|
||||
let client = provider_openai::transport::build_reqwest_client().unwrap();
|
||||
let (hello, response) = timeout(Duration::from_secs(10), async {
|
||||
tokio::join!(read_client_hello(listener), client.get(url).send())
|
||||
})
|
||||
.await
|
||||
.expect("HTTP ClientHello within timeout");
|
||||
assert!(
|
||||
response.is_err(),
|
||||
"capture endpoint rejects TLS after ClientHello"
|
||||
);
|
||||
|
||||
let mut expected = official_websocket_hello();
|
||||
expected.extensions.insert(5, 16);
|
||||
expected.alpn = vec!["h2".to_owned(), "http/1.1".to_owned()];
|
||||
assert_eq!(hello, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_client_hello_should_match_official_rustls_transport() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("wss://localhost:{}/", listener.local_addr().unwrap().port());
|
||||
let connector =
|
||||
provider_openai::transport::tls::maybe_build_rustls_client_config_with_custom_ca()
|
||||
.unwrap()
|
||||
.map(tokio_tungstenite::Connector::Rustls);
|
||||
let (hello, response) = timeout(Duration::from_secs(10), async {
|
||||
tokio::join!(
|
||||
read_client_hello(listener),
|
||||
tokio_tungstenite::connect_async_tls_with_config(url, None, false, connector)
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("WebSocket ClientHello within timeout");
|
||||
assert!(
|
||||
response.is_err(),
|
||||
"capture endpoint rejects TLS after ClientHello"
|
||||
);
|
||||
assert_eq!(hello, official_websocket_hello());
|
||||
}
|
||||
|
||||
fn official_websocket_hello() -> ClientHello {
|
||||
ClientHello {
|
||||
cipher_suites: vec![
|
||||
4866, 4865, 4867, 49196, 49195, 52393, 49200, 49199, 52392, 255,
|
||||
],
|
||||
extensions: vec![0, 5, 10, 11, 13, 23, 35, 43, 45, 51],
|
||||
groups: vec![4588, 29, 23, 24],
|
||||
signature_algorithms: vec![1283, 1027, 1539, 2055, 2054, 2053, 2052, 1537, 1281, 1025],
|
||||
key_shares: vec![(4588, 1216), (29, 32)],
|
||||
alpn: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_client_hello(listener: TcpListener) -> ClientHello {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut header = [0; 5];
|
||||
stream.read_exact(&mut header).await.unwrap();
|
||||
assert_eq!(header[0], 22, "TLS handshake record");
|
||||
let length = usize::from(u16::from_be_bytes([header[3], header[4]]));
|
||||
assert!(length <= 16_384, "bounded ClientHello record");
|
||||
let mut record = vec![0; length];
|
||||
stream.read_exact(&mut record).await.unwrap();
|
||||
stream.write_all(&[21, 3, 3, 0, 2, 2, 40]).await.unwrap();
|
||||
|
||||
let mut hello = Bytes::from(record);
|
||||
assert_eq!(hello.get_u8(), 1, "ClientHello message");
|
||||
hello.advance(3 + 2 + 32); // Message length, legacy version, random.
|
||||
let session_id_len = usize::from(hello.get_u8());
|
||||
hello.advance(session_id_len);
|
||||
let cipher_suites = u16_values(take_vector(&mut hello));
|
||||
let compression_len = usize::from(hello.get_u8());
|
||||
hello.advance(compression_len);
|
||||
let mut extensions = take_vector(&mut hello);
|
||||
let mut values = BTreeMap::new();
|
||||
while extensions.has_remaining() {
|
||||
let kind = extensions.get_u16();
|
||||
assert!(values.insert(kind, take_vector(&mut extensions)).is_none());
|
||||
}
|
||||
|
||||
let mut groups = values[&10].clone();
|
||||
let groups = u16_values(take_vector(&mut groups));
|
||||
let mut signatures = values[&13].clone();
|
||||
let signature_algorithms = u16_values(take_vector(&mut signatures));
|
||||
let mut shares = values[&51].clone();
|
||||
let mut shares = take_vector(&mut shares);
|
||||
let mut key_shares = Vec::new();
|
||||
while shares.has_remaining() {
|
||||
let group = shares.get_u16();
|
||||
key_shares.push((group, take_vector(&mut shares).len()));
|
||||
}
|
||||
let mut alpn = Vec::new();
|
||||
if let Some(protocols) = values.get(&16) {
|
||||
let mut protocols = protocols.clone();
|
||||
let mut protocols = take_vector(&mut protocols);
|
||||
while protocols.has_remaining() {
|
||||
let length = usize::from(protocols.get_u8());
|
||||
alpn.push(String::from_utf8(protocols.split_to(length).to_vec()).unwrap());
|
||||
}
|
||||
}
|
||||
ClientHello {
|
||||
cipher_suites,
|
||||
extensions: values.into_keys().collect(),
|
||||
groups,
|
||||
signature_algorithms,
|
||||
key_shares,
|
||||
alpn,
|
||||
}
|
||||
}
|
||||
|
||||
fn take_vector(bytes: &mut Bytes) -> Bytes {
|
||||
let length = usize::from(bytes.get_u16());
|
||||
bytes.split_to(length)
|
||||
}
|
||||
|
||||
fn u16_values(mut bytes: Bytes) -> Vec<u16> {
|
||||
let mut values = Vec::new();
|
||||
while bytes.has_remaining() {
|
||||
values.push(bytes.get_u16());
|
||||
}
|
||||
values
|
||||
}
|
||||
@@ -253,6 +253,7 @@ async fn fetch_should_use_wham_usage_headers_only() {
|
||||
|
||||
let usage = client(&server.uri())
|
||||
.fetch_usage(CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer oauth-access",
|
||||
account_id: Some("acct_123"),
|
||||
request_id: "req_usage_headers",
|
||||
@@ -283,7 +284,7 @@ async fn fetch_should_use_wham_usage_headers_only() {
|
||||
for (name, expected) in [
|
||||
("authorization", "Bearer oauth-access"),
|
||||
("chatgpt-account-id", "acct_123"),
|
||||
("accept", "application/json"),
|
||||
("accept", "*/*"),
|
||||
("cookie", "session=old"),
|
||||
] {
|
||||
assert_eq!(
|
||||
@@ -576,6 +577,7 @@ fn profile() -> CodexWireProfileState {
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "xterm".to_owned(),
|
||||
residency: None,
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 7, 18, 0, 0, 0)
|
||||
.single()
|
||||
@@ -585,6 +587,7 @@ fn profile() -> CodexWireProfileState {
|
||||
|
||||
fn context() -> CodexRequestContext<'static> {
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer oauth-access",
|
||||
account_id: Some("acct_123"),
|
||||
request_id: "req_usage_limit",
|
||||
|
||||
@@ -2150,6 +2150,7 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
trace: None,
|
||||
authorization: "Bearer access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_ws_client",
|
||||
@@ -2190,3 +2191,90 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
|
||||
.any(|(name, value)| name == "x-ratelimit-remaining-requests" && value == "17")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_capture_metadata_and_unknown_events_before_normal_close() {
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
use tokio_tungstenite::tungstenite::protocol::{CloseFrame, frame::coding::CloseCode};
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
for event in [
|
||||
json!({"type": "future.metadata", "detail": "private-data"}),
|
||||
json!({"type": "codex.response.metadata", "headers": {
|
||||
"x-request-id": "upstream-metadata-request", "x-codex-turn-state": "opaque-turn-secret",
|
||||
}}),
|
||||
] {
|
||||
websocket
|
||||
.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
websocket
|
||||
.close(Some(CloseFrame {
|
||||
code: CloseCode::Normal,
|
||||
reason: "".into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let trace = TraceContext::new("req_metadata_close");
|
||||
let attempt = trace.attempt(2);
|
||||
let mut request = codex_request("gpt-5.5", "be brief", Vec::new());
|
||||
request.set_previous_response_id(Some("resp_previous".to_owned()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.force_http_sse = false;
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
);
|
||||
let result = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_metadata_close", Some("chatgpt-account")).with_trace(&attempt),
|
||||
)
|
||||
.await;
|
||||
let mut failed = result.is_err();
|
||||
if let Ok(response) = result {
|
||||
let mut body = response.body;
|
||||
while let Some(chunk) = body.next().await {
|
||||
if chunk.is_err() {
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(failed, "close before terminal must fail the exchange");
|
||||
server.await.unwrap();
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
let metadata = events
|
||||
.iter()
|
||||
.find(|event| event["data"]["eventType"] == "codex.response.metadata")
|
||||
.unwrap();
|
||||
assert_eq!(metadata["attemptIndex"], 2);
|
||||
assert_eq!(metadata["exchangeId"], 1);
|
||||
assert_eq!(
|
||||
metadata["data"]["metadata"]["headers"]["x-request-id"],
|
||||
"upstream-metadata-request"
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event["data"]["eventType"] == "future.metadata")
|
||||
);
|
||||
let close = events
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "upstream.close")
|
||||
.unwrap();
|
||||
assert_eq!(close["data"]["code"], 1000);
|
||||
assert_eq!(close["data"]["lastEventType"], "codex.response.metadata");
|
||||
assert_eq!(close["data"]["terminalSeen"], false);
|
||||
assert!(!snapshot.to_string().contains("opaque-turn-secret"));
|
||||
assert!(!snapshot.to_string().contains("private-data"));
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ pub(super) fn cold_compaction_http_sse_stream(
|
||||
headers.clone(),
|
||||
body,
|
||||
session.binding().clone(),
|
||||
);
|
||||
).with_trace(context.trace());
|
||||
let Some(handshake_deadline) = remaining(context.deadline()) else {
|
||||
Err(mark_transient_compaction_failure(provider_error(
|
||||
ProviderErrorKind::Timeout,
|
||||
@@ -403,7 +403,7 @@ pub(super) fn cold_http_sse_stream(
|
||||
headers.clone(),
|
||||
body,
|
||||
session.binding().clone(),
|
||||
);
|
||||
).with_trace(context.trace());
|
||||
let Some(handshake_deadline) = remaining(context.deadline()) else {
|
||||
Err(provider_error(
|
||||
ProviderErrorKind::Timeout,
|
||||
|
||||
@@ -14,6 +14,7 @@ use super::{GrokHeader, GrokSessionBinding};
|
||||
|
||||
/// 交给注入的 HTTP SSE transport 的自持有请求。
|
||||
pub struct GrokInferenceRequest {
|
||||
trace: gateway_core::diagnostics::TraceContext,
|
||||
endpoint: Url,
|
||||
headers: Vec<GrokHeader>,
|
||||
body: Zeroizing<Vec<u8>>,
|
||||
@@ -28,6 +29,7 @@ impl GrokInferenceRequest {
|
||||
binding: GrokSessionBinding,
|
||||
) -> Self {
|
||||
Self {
|
||||
trace: gateway_core::diagnostics::TraceContext::default(),
|
||||
endpoint,
|
||||
headers,
|
||||
body: Zeroizing::new(body),
|
||||
@@ -35,6 +37,16 @@ impl GrokInferenceRequest {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_trace(mut self, trace: gateway_core::diagnostics::TraceContext) -> Self {
|
||||
self.trace = trace;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn trace(&self) -> &gateway_core::diagnostics::TraceContext {
|
||||
&self.trace
|
||||
}
|
||||
|
||||
/// 返回严格限定的官方 Responses 端点。
|
||||
#[must_use]
|
||||
pub const fn endpoint(&self) -> &Url {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 关闭 redirect、proxy 与业务重试的生产 reqwest transport。
|
||||
|
||||
use gateway_core::diagnostics::{StreamCapture, StreamFormat, TraceContext};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::fmt;
|
||||
use std::io;
|
||||
@@ -371,6 +372,18 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
}
|
||||
let trace = request.trace().exchange("http_sse");
|
||||
trace.headers(
|
||||
"upstream.request.headers",
|
||||
serde_json::json!({
|
||||
"method": "POST", "endpoint": request.endpoint().path(),
|
||||
}),
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|header| (header.name(), header.value().expose().as_bytes())),
|
||||
);
|
||||
trace.capture("upstream.request.body", request.body());
|
||||
let (client, client_cache_status) = self.client_for(request.binding())?;
|
||||
let mut builder = client
|
||||
.post(request.endpoint().clone())
|
||||
@@ -393,8 +406,12 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
|
||||
Some(elapsed_millis(headers_started_at.elapsed())),
|
||||
&dns_observer,
|
||||
);
|
||||
trace.headers("upstream.response.headers", serde_json::json!({
|
||||
"status": response.status().as_u16(), "httpVersion": format!("{:?}", response.version()),
|
||||
"headersMs": elapsed_millis(headers_started_at.elapsed()),
|
||||
}), response.headers().iter().map(|(name, value)| (name.as_str(), value.as_bytes())));
|
||||
if !response.status().is_success() {
|
||||
return Err(classify_inference_status(response)
|
||||
return Err(classify_inference_status(response, &trace)
|
||||
.await
|
||||
.with_transport_metrics(transport_metrics));
|
||||
}
|
||||
@@ -424,6 +441,20 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
|
||||
};
|
||||
std::future::ready(Some(item))
|
||||
});
|
||||
let body = async_stream::stream! {
|
||||
let mut body = Box::pin(body);
|
||||
let mut capture = StreamCapture::new(trace.clone(), StreamFormat::Sse);
|
||||
while let Some(chunk) = body.next().await {
|
||||
match &chunk {
|
||||
Ok(bytes) => capture.push(bytes),
|
||||
Err(_) => trace.record("upstream.read.failed", serde_json::json!({"phase": "response_body"})),
|
||||
}
|
||||
let failed = chunk.is_err();
|
||||
yield chunk;
|
||||
if failed { return; }
|
||||
}
|
||||
capture.finish();
|
||||
};
|
||||
Ok(
|
||||
GrokInferenceResponse::new(Box::pin(body), http_version, status_code, request_id)
|
||||
.with_transport_metrics(transport_metrics),
|
||||
@@ -981,7 +1012,10 @@ fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTrans
|
||||
)
|
||||
}
|
||||
|
||||
async fn classify_inference_status(response: Response) -> GrokInferenceTransportError {
|
||||
async fn classify_inference_status(
|
||||
response: Response,
|
||||
trace: &TraceContext,
|
||||
) -> GrokInferenceTransportError {
|
||||
let status = response.status();
|
||||
let retry_after = retry_after(&response);
|
||||
let http_version = upstream_http_version(response.version());
|
||||
@@ -989,8 +1023,15 @@ async fn classify_inference_status(response: Response) -> GrokInferenceTransport
|
||||
let status_code = status.as_u16();
|
||||
let body = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await {
|
||||
Ok(BoundedBody::Body(body)) => body,
|
||||
Ok(BoundedBody::TooLarge) | Err(_) => Vec::new(),
|
||||
Ok(BoundedBody::TooLarge) | Err(_) => {
|
||||
trace.record(
|
||||
"capture.gap",
|
||||
serde_json::json!({"reason": "error_body_unavailable_or_too_large"}),
|
||||
);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
trace.capture("upstream.error.body", &body);
|
||||
let metadata = inference_error_metadata(&body);
|
||||
let body_failure = classify_grok_body_failure(&metadata, &body);
|
||||
let kind = match status {
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
|
||||
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
|
||||
3b4bfd5e3f5135fa2fad1088b0c2a287861e1250d66af31dbd40dc890751b567 0006_openai_session_transport_recovery.sql
|
||||
8c0ec7fab80d95dfd5a370c70706c36ac5907bb5afea0cf1886196f21fd8317d 0007_request_diagnostics.sql
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Bounded request evidence is retained with its owning model request.
|
||||
-- NULL identifies historical requests for which no trace was captured.
|
||||
alter table model_requests add column diagnostic_trace_json jsonb;
|
||||
alter table model_requests add constraint model_requests_diagnostic_trace_object
|
||||
check (diagnostic_trace_json is null or jsonb_typeof(diagnostic_trace_json) = 'object');
|
||||
@@ -20,7 +20,7 @@ host:
|
||||
logging:
|
||||
level: 'info'
|
||||
stdout: true
|
||||
# 将完整请求头和正文写入独立文件;包含凭据,仅排障时开启。
|
||||
# 将完整请求与响应头、正文写入独立文件;包含凭据,仅排障时开启。
|
||||
request_dump: false
|
||||
file:
|
||||
enabled: true
|
||||
@@ -65,14 +65,16 @@ openai:
|
||||
# 版本元组来自同一个官方 macOS Desktop ZIP;Provider 每 24 小时检查 appcast,
|
||||
# 仅在包内 Core 核验和 Redis 条件覆盖都成功后原子更新。
|
||||
originator: 'Codex Desktop'
|
||||
codex_version: '0.151.0-alpha.7.2'
|
||||
desktop_version: '26.825.51511'
|
||||
desktop_build: '7377'
|
||||
codex_version: '0.153.4'
|
||||
desktop_version: '26.901.51231'
|
||||
desktop_build: '8109'
|
||||
os_type: 'Mac OS'
|
||||
os_version: '15.7.1'
|
||||
arch: 'arm64'
|
||||
terminal: 'unknown'
|
||||
verified_at: '2026-08-31T00:00:00+08:00'
|
||||
# 部署选择的 US residency;省略或 null 时使用官方无地域约束的默认分支。
|
||||
residency: us
|
||||
verified_at: '2026-09-06T11:26:12.084+08:00'
|
||||
|
||||
xai:
|
||||
# OAuth 端点、额度与模型目录策略由 xAI Provider 固定管理;这里只声明请求画像启动基线。
|
||||
|
||||
@@ -567,6 +567,11 @@ request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可
|
||||
汇总与洞察中的请求数与 outcome 分布覆盖筛选范围内全部请求;token、缓存、延迟与成本聚合仅统计
|
||||
已完整交付客户端的成功响应。
|
||||
|
||||
详情接口按 `id` 可读取成功、失败或未完成请求。新增 `trace`(历史未采集记录为 `null`)和
|
||||
`relatedRequests[]`(`requestId / relation / outcome / completedAt`);`relation` 为 `recovered_by` 或
|
||||
`recovers`。`trace` 是执行终态时的有界脱敏时间线,包含 request、attempt 和 exchange 关联、阶段、
|
||||
事件摘要及淘汰计数;普通用量列表不携带此字段。
|
||||
|
||||
错误记录中的“已自动恢复”表示系统关联到了后续成功请求,不会把原来的失败记录改为成功。
|
||||
`upstreamSendState = ambiguous` 表示无法确认该次上游执行结果,不代表后续恢复请求失败;
|
||||
恢复关联也不等于逐字节验证过两次请求正文。
|
||||
|
||||
@@ -186,7 +186,39 @@ export interface UsageAttempt {
|
||||
completedAt: string | null
|
||||
}
|
||||
|
||||
export interface RequestTraceEvent {
|
||||
sequence: number
|
||||
lastSequence: number
|
||||
elapsedMs: number
|
||||
lastElapsedMs: number
|
||||
attemptIndex: number
|
||||
exchangeId: number | null
|
||||
stage: string
|
||||
count: number
|
||||
data: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface RequestTrace {
|
||||
schemaVersion: number
|
||||
requestId: string
|
||||
startedAtUnixMs: number
|
||||
totalEvents: number
|
||||
droppedEvents: number
|
||||
maxEvents: number
|
||||
captureMode: string
|
||||
events: RequestTraceEvent[]
|
||||
}
|
||||
|
||||
export interface RelatedRequest {
|
||||
requestId: string
|
||||
relation: 'recovered_by' | 'recovers'
|
||||
outcome: string
|
||||
completedAt: string | null
|
||||
}
|
||||
|
||||
export type UsageRecordDetail = UsageRecord & {
|
||||
trace: RequestTrace | null
|
||||
relatedRequests: RelatedRequest[]
|
||||
attempts: UsageAttempt[]
|
||||
/** 尝试列表是否完整;best-effort 下恒为 false。 */
|
||||
attemptsComplete: boolean
|
||||
|
||||
@@ -5,6 +5,7 @@ import { computed } from 'vue'
|
||||
import BaseButton from '@/components/base/BaseButton.vue'
|
||||
import BaseModal from '@/components/base/BaseModal/index.vue'
|
||||
import { failureClassText } from '../utils/opsErrorPresentation'
|
||||
import RequestDiagnosticsPanel from './RequestDiagnosticsPanel.vue'
|
||||
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
|
||||
import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue'
|
||||
|
||||
@@ -130,6 +131,7 @@ function visibleFields(items: DetailField[]) {
|
||||
size="xl"
|
||||
>
|
||||
<template v-if="record">
|
||||
<RequestDiagnosticsPanel v-if="open && record.requestId" class="mb-3" :request-id="record.requestId" />
|
||||
<section :class="panelClass">
|
||||
<h3 :class="panelTitleClass">
|
||||
错误
|
||||
@@ -142,7 +144,7 @@ function visibleFields(items: DetailField[]) {
|
||||
|
||||
<section v-if="record.rawUpstreamError" class="mt-3" :class="panelClass">
|
||||
<UsageDetailCodePanel
|
||||
title="上游返回原文"
|
||||
title="上游错误与关闭信息"
|
||||
max-height="360px"
|
||||
:content="record.rawUpstreamError"
|
||||
/>
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
import BaseButton from '@/components/base/BaseButton.vue'
|
||||
import { useDownload } from '@/composables/useDownload'
|
||||
import { useRequestDiagnostics } from '../composables/useRequestDiagnostics'
|
||||
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
|
||||
|
||||
const props = defineProps<{ requestId: string }>()
|
||||
const { downloadJson } = useDownload()
|
||||
const { selectedId, detail, loading, error, refresh } = useRequestDiagnostics(() => props.requestId)
|
||||
const trace = computed(() => detail.value?.trace)
|
||||
const events = computed(() => (trace.value?.events ?? []).map(event => ({
|
||||
...event,
|
||||
content: JSON.stringify(event.data, null, 2),
|
||||
})))
|
||||
|
||||
function download() {
|
||||
if (!detail.value)
|
||||
return
|
||||
const record = detail.value
|
||||
const bundle = {
|
||||
schemaVersion: 1,
|
||||
exportedAt: new Date().toISOString(),
|
||||
request: {
|
||||
requestId: record.requestId,
|
||||
route: record.route,
|
||||
model: record.model,
|
||||
provider: record.provider,
|
||||
accountId: record.accountId,
|
||||
clientTransport: record.clientTransport,
|
||||
upstreamTransport: record.upstreamTransport,
|
||||
upstreamRequestId: record.upstreamRequestId,
|
||||
responseId: record.responseId,
|
||||
statusCode: record.statusCode,
|
||||
attemptCount: record.attemptCount,
|
||||
createdAt: record.createdAt,
|
||||
},
|
||||
trace: record.trace,
|
||||
attempts: record.attempts,
|
||||
attemptsComplete: record.attemptsComplete,
|
||||
relatedRequests: record.relatedRequests,
|
||||
}
|
||||
return downloadJson(bundle, `diagnostics-${record.requestId.replace(/[^\w-]/g, '_')}.json`)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section class="mt-3 min-w-0 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5" aria-label="请求诊断">
|
||||
<div class="flex flex-wrap items-center justify-between gap-3">
|
||||
<h3 class="m-0 text-cp-sm font-heavy text-cp-text-secondary">
|
||||
请求诊断
|
||||
</h3>
|
||||
<div class="flex flex-wrap gap-2">
|
||||
<BaseButton v-if="selectedId !== requestId" size="sm" @click="selectedId = requestId">
|
||||
返回本次请求
|
||||
</BaseButton>
|
||||
<BaseButton size="sm" :disabled="loading" @click="refresh">
|
||||
刷新
|
||||
</BaseButton>
|
||||
<BaseButton size="sm" :disabled="!detail" @click="download">
|
||||
导出诊断包
|
||||
</BaseButton>
|
||||
</div>
|
||||
</div>
|
||||
<p class="break-all font-mono text-cp-xs text-cp-text-secondary">
|
||||
{{ selectedId }}
|
||||
</p>
|
||||
<p v-if="loading" role="status" class="text-cp-sm text-cp-text-secondary">
|
||||
正在加载诊断记录…
|
||||
</p>
|
||||
<p v-else-if="error" role="alert" class="text-cp-sm text-cp-error-text">
|
||||
{{ error }}
|
||||
</p>
|
||||
<template v-else-if="detail">
|
||||
<div v-if="detail.relatedRequests?.length" class="mb-3 flex flex-wrap gap-2">
|
||||
<BaseButton v-for="related in detail.relatedRequests" :key="related.requestId" size="sm" @click="selectedId = related.requestId">
|
||||
{{ related.relation === 'recovered_by' ? '查看恢复请求' : '查看先前失败' }} · {{ related.requestId }}
|
||||
</BaseButton>
|
||||
</div>
|
||||
<p v-if="!trace" class="text-cp-sm text-cp-text-secondary">
|
||||
这条记录没有保存诊断时间线。旧记录无法补回当时未采集的事件。
|
||||
</p>
|
||||
<template v-else>
|
||||
<p class="text-cp-xs text-cp-text-secondary">
|
||||
已观测 {{ trace.totalEvents }} 个阶段或事件,展示 {{ events.length }} 条摘要。连续增量合并计数;正文和敏感字段仅保留摘要。
|
||||
</p>
|
||||
<p v-if="trace.droppedEvents" role="status" class="text-cp-xs text-cp-warning-text">
|
||||
达到保存上限,{{ trace.droppedEvents }} 个事件已被淘汰;保留请求开头与最近事件。
|
||||
</p>
|
||||
<ol class="m-0 grid max-h-128 list-none gap-2 overflow-auto p-0">
|
||||
<li v-for="event in events" :key="event.sequence" class="min-w-0 rounded-cp bg-cp-bg-container px-3 py-2">
|
||||
<details>
|
||||
<summary class="cursor-pointer break-all text-cp-xs leading-relaxed">
|
||||
<span class="font-mono text-cp-text-secondary">+{{ event.elapsedMs }} ms · #{{ event.sequence }}</span>
|
||||
<span class="mx-2 font-mono">{{ event.stage }}</span>
|
||||
<span v-if="event.attemptIndex">尝试 {{ event.attemptIndex }}</span>
|
||||
<span v-if="event.exchangeId"> · 交换 {{ event.exchangeId }}</span>
|
||||
<span v-if="event.count > 1"> · {{ event.count }} 次(至 +{{ event.lastElapsedMs }} ms)</span>
|
||||
</summary>
|
||||
<UsageDetailCodePanel class="mt-2" title="诊断事实" :content="event.content" max-height="280px" />
|
||||
</details>
|
||||
</li>
|
||||
</ol>
|
||||
<p class="mb-0 text-cp-xs text-cp-text-secondary">
|
||||
时间线保存至执行结束。客户端最终写入、进程退出等后续事实,可用此请求 ID 检索服务器日志。
|
||||
</p>
|
||||
</template>
|
||||
</template>
|
||||
</section>
|
||||
</template>
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
visibleRequestText,
|
||||
visibleResponseText,
|
||||
} from '../utils/records'
|
||||
import RequestDiagnosticsPanel from './RequestDiagnosticsPanel.vue'
|
||||
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
|
||||
import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue'
|
||||
import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue'
|
||||
@@ -470,6 +471,8 @@ const tokenDonutOption = computed<EChartsOption>(() => {
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<RequestDiagnosticsPanel v-if="open && record" :request-id="record.requestId" />
|
||||
|
||||
<template #footer>
|
||||
<BaseButton variant="primary" @click="open = false">
|
||||
关闭
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { UsageRecordDetail } from '@/api'
|
||||
import { shallowRef, watch } from 'vue'
|
||||
import { getUsageRecordDetail } from '@/api'
|
||||
import { errorMessage } from '@/utils/async'
|
||||
|
||||
/** Requests can be switched while a lookup is pending; stale results never replace the active trace. */
|
||||
export function useRequestDiagnostics(requestId: () => string) {
|
||||
const selectedId = shallowRef(requestId())
|
||||
const revision = shallowRef(0)
|
||||
const detail = shallowRef<UsageRecordDetail | null>(null)
|
||||
const loading = shallowRef(false)
|
||||
const error = shallowRef('')
|
||||
|
||||
watch(requestId, id => selectedId.value = id)
|
||||
watch([selectedId, revision], async ([id], _previous, onCleanup) => {
|
||||
let active = true
|
||||
onCleanup(() => active = false)
|
||||
detail.value = null
|
||||
error.value = ''
|
||||
loading.value = true
|
||||
try {
|
||||
const result = await getUsageRecordDetail({ id })
|
||||
if (active)
|
||||
detail.value = result
|
||||
}
|
||||
catch (cause: unknown) {
|
||||
if (active)
|
||||
error.value = errorMessage(cause, '诊断记录暂不可用,请稍后刷新')
|
||||
}
|
||||
finally {
|
||||
if (active)
|
||||
loading.value = false
|
||||
}
|
||||
}, { immediate: true })
|
||||
|
||||
return { selectedId, detail, loading, error, refresh: () => revision.value++ }
|
||||
}
|
||||
Reference in New Issue
Block a user