diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5e7f7fa18..744057316 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -363,6 +363,7 @@ dependencies = [ "flate2", "futures", "hex", + "hmac 0.12.1", "indexmap", "ipnet", "percent-encoding", @@ -1061,7 +1062,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac", + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -2394,7 +2404,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.13.0", "itoa", "log", "md-5", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5f69b3559..def90a7d6 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -17,6 +17,7 @@ dirs = "6.0.0" flate2 = "1.1.9" futures = "0.3.32" hex = "0.4.3" +hmac = "0.12.1" indexmap = { version = "2.0", features = ["serde"] } ipnet = "2.12.0" percent-encoding = "2.3.2" diff --git a/backend/src/api/client/errors.rs b/backend/src/api/client/errors.rs index c7fd49933..298014914 100644 --- a/backend/src/api/client/errors.rs +++ b/backend/src/api/client/errors.rs @@ -10,6 +10,7 @@ use serde_json::{json, Value}; use crate::{ dispatch::errors::{DispatchFailureClass, ResponseDispatchError}, upstream::openai::protocol::sse::response_failed_sse_event as encode_response_failed_sse_event, + upstream::openai::transport::CodexClientError, }; const NO_ACTIVE_UPSTREAM_ACCOUNT_MESSAGE: &str = "No active upstream account is available"; @@ -19,6 +20,8 @@ const UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE: &str = "Upstream Codex request fail const INVALID_UPSTREAM_CODEX_RESPONSE_MESSAGE: &str = "Invalid upstream Codex response"; const UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE: &str = "Upstream Codex response failed"; const HISTORY_UNAVAILABLE_MESSAGE: &str = "Previous response context is unavailable. Start a new conversation or resend the complete input without previous_response_id."; +const CONTINUATION_BUSY_MESSAGE: &str = + "The previous response connection is busy. Retry the request."; #[derive(Clone, Copy)] enum ResponseDispatchMessageStyle { @@ -72,6 +75,10 @@ const HISTORY_UNAVAILABLE_ERROR: OpenAiErrorKind = OpenAiErrorKind { error_type: "invalid_request_error", code: "previous_response_unavailable", }; +const CONTINUATION_BUSY_ERROR: OpenAiErrorKind = OpenAiErrorKind { + error_type: "server_error", + code: "continuation_connection_busy", +}; const INVALID_UPSTREAM_RESPONSE_ERROR: OpenAiErrorKind = OpenAiErrorKind { error_type: "server_error", code: "invalid_upstream_response", @@ -105,11 +112,24 @@ pub fn openai_error_response( } pub fn responses_dispatch_error_response(error: ResponseDispatchError) -> Response { + responses_dispatch_error_response_ref(&error) +} + +pub fn responses_dispatch_error_response_ref(error: &ResponseDispatchError) -> Response { + if let ResponseDispatchError::Upstream(CodexClientError::Upstream { status, body, .. }) = error + { + if status.is_client_error() { + if let Ok(body) = serde_json::from_str::(body) { + return (*status, Json(body)).into_response(); + } + } + } + match error { ResponseDispatchError::NoActiveAccount => { responses_no_available_accounts_response().into_response() } - error => response_dispatch_openai_error_response(&error), + error => response_dispatch_openai_error_response(error), } } @@ -273,6 +293,7 @@ fn dispatch_failure_message( } DispatchFailureClass::ResponseFailed => UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE.to_owned(), DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_MESSAGE.to_owned(), + DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_MESSAGE.to_owned(), _ => UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE.to_owned(), } } @@ -290,6 +311,7 @@ fn dispatch_failure_openai_error_kind( | DispatchFailureClass::Banned => AUTHENTICATION_ERROR, DispatchFailureClass::ModelUnsupported => MODEL_NOT_FOUND_ERROR, DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_ERROR, + DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_ERROR, DispatchFailureClass::InvalidSse | DispatchFailureClass::MissingCompleted | DispatchFailureClass::EmptyUpstreamResponse => INVALID_UPSTREAM_RESPONSE_ERROR, @@ -297,7 +319,9 @@ fn dispatch_failure_openai_error_kind( DispatchFailureClass::Upstream if status.is_client_error() => { responses_error_kind_for_status(status) } - DispatchFailureClass::Upstream => UPSTREAM_ERROR, + DispatchFailureClass::Upstream | DispatchFailureClass::UpstreamUnavailable => { + UPSTREAM_ERROR + } DispatchFailureClass::CloudflareChallenge | DispatchFailureClass::CloudflarePathBlocked => { UPSTREAM_ERROR } diff --git a/backend/src/api/client/responses.rs b/backend/src/api/client/responses.rs index e3e0f881e..3df05710a 100644 --- a/backend/src/api/client/responses.rs +++ b/backend/src/api/client/responses.rs @@ -24,7 +24,7 @@ use super::{ errors::{ invalid_responses_request_response, missing_client_api_key_response, model_not_found_response, responses_dispatch_error_response, - responses_stream_dispatch_failed_sse_event, + responses_dispatch_error_response_ref, responses_stream_dispatch_failed_sse_event, }, models::model_catalog_for_state, sse::{done_sse_frame, event_stream_response as sse_event_stream_response, SseResponseOptions}, @@ -101,10 +101,43 @@ pub fn build_codex_request( None => {} } apply_context_header_fields(&mut request, headers); + apply_identity_fields(&mut request, headers); request.client_user_agent = user_agent_from_headers(headers); request } +fn apply_identity_fields(request: &mut CodexResponsesRequest, headers: &HeaderMap) { + request.client_conversation_id = request.prompt_cache_key().map(ToString::to_string); + request.client_session_id = identity_context_string(request.body(), "session_id") + .or_else(|| header_string(headers, "session-id")); + request.client_thread_id = identity_context_string(request.body(), "thread_id") + .or_else(|| header_string(headers, "thread-id")); + request.client_request_id = identity_context_string(request.body(), "x-client-request-id") + .or_else(|| header_string(headers, "x-client-request-id")); + request.client_turn_id = identity_context_string(request.body(), "turn_id") + .or_else(|| header_string(headers, "x-codex-turn-id")); + request.codex_window_id = request.codex_window_id.take().or_else(|| { + identity_context_string(request.body(), "x-codex-window-id") + .or_else(|| header_string(headers, "x-codex-window-id")) + }); + request.parent_thread_id = request.parent_thread_id.take().or_else(|| { + identity_context_string(request.body(), "x-codex-parent-thread-id") + .or_else(|| header_string(headers, "x-codex-parent-thread-id")) + }); +} + +fn identity_context_string(body: &Map, key: &str) -> Option { + body_context_string(body, key).or_else(|| { + body.get("client_metadata") + .and_then(Value::as_object) + .and_then(|metadata| metadata.get(key)) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) + }) +} + /// 从客户端原始 Responses JSON body 构造 Codex compact 请求。 /// /// compact 端点只返回一次性 JSON（代理按非流式读取），故剥离 `stream` 这一 @@ -115,6 +148,18 @@ pub fn build_compact_request( mut body: Map, headers: &HeaderMap, ) -> CodexCompactRequest { + let client_session_id = identity_context_string(&body, "session_id") + .or_else(|| header_string(headers, "session-id")); + let client_thread_id = + identity_context_string(&body, "thread_id").or_else(|| header_string(headers, "thread-id")); + let client_request_id = identity_context_string(&body, "x-client-request-id") + .or_else(|| header_string(headers, "x-client-request-id")); + let client_turn_id = identity_context_string(&body, "turn_id") + .or_else(|| header_string(headers, "x-codex-turn-id")); + let client_window_id = identity_context_string(&body, "x-codex-window-id") + .or_else(|| header_string(headers, "x-codex-window-id")); + let client_parent_thread_id = identity_context_string(&body, "x-codex-parent-thread-id") + .or_else(|| header_string(headers, "x-codex-parent-thread-id")); body.remove(COMPACT_STREAM_KEY); for key in TRANSPORT_ONLY_KEYS { body.remove(key); @@ -124,6 +169,12 @@ pub fn build_compact_request( client_ip: None, client_user_agent: user_agent_from_headers(headers), client_api_key_id: None, + client_session_id, + client_thread_id, + client_request_id, + client_turn_id, + client_window_id, + client_parent_thread_id, } } @@ -312,7 +363,10 @@ async fn handle_responses( .complete(request_id.as_str(), route, codex_request, &model) .await { - Ok(body) => (StatusCode::OK, Json(body)).into_response(), + Ok(result) => apply_safe_response_headers( + (StatusCode::OK, Json(result.body)).into_response(), + result.response_headers, + ), Err(error) => responses_dispatch_error_response(error), } } @@ -352,7 +406,10 @@ async fn handle_compact_responses( .compact(request_id.as_str(), compact_request, &model) .await { - Ok(body) => (StatusCode::OK, Json(body)).into_response(), + Ok(result) => apply_safe_response_headers( + (StatusCode::OK, Json(result.body)).into_response(), + result.response_headers, + ), Err(error) => responses_dispatch_error_response(error), } } @@ -424,10 +481,37 @@ fn event_stream_response(mut body: String) -> Response { } fn live_event_stream_response(stream: ResponseDispatchStream) -> Response { - sse_event_stream_response(Body::from_stream(stream.body), SseResponseOptions::BASIC) + let response = + sse_event_stream_response(Body::from_stream(stream.body), SseResponseOptions::BASIC); + apply_safe_response_headers(response, stream.response_headers) +} + +fn apply_safe_response_headers(mut response: Response, headers: Vec<(String, String)>) -> Response { + for (name, value) in headers { + let name = match name.as_str() { + "x-request-id" => axum::http::header::HeaderName::from_static("x-request-id"), + "openai-model" => axum::http::header::HeaderName::from_static("openai-model"), + "x-models-etag" => axum::http::header::HeaderName::from_static("x-models-etag"), + "x-reasoning-included" => { + axum::http::header::HeaderName::from_static("x-reasoning-included") + } + "openai-processing-ms" => { + axum::http::header::HeaderName::from_static("openai-processing-ms") + } + _ => continue, + }; + let Ok(value) = axum::http::HeaderValue::from_str(&value) else { + continue; + }; + response.headers_mut().insert(name, value); + } + response } fn response_dispatch_stream_error_response(error: &ResponseDispatchError) -> Response { + if (400..=499).contains(&error.client_http_status_code()) { + return responses_dispatch_error_response_ref(error); + } event_stream_response(responses_stream_dispatch_failed_sse_event(error)) } diff --git a/backend/src/bootstrap/config.rs b/backend/src/bootstrap/config.rs index cc2d08f78..667507b7d 100644 --- a/backend/src/bootstrap/config.rs +++ b/backend/src/bootstrap/config.rs @@ -173,16 +173,16 @@ pub struct WebSocketPoolSettings { pub max_age_ms: u64, /// 单账号最大连接数。 pub max_per_account: usize, - /// 首个真实输出（首 token）到达前的绝对超时；`0` 表示禁用。 + /// 首个真实输出（首事件）到达前的绝对超时；`0` 表示禁用。 /// /// 从发出 `response.create` 起算，覆盖建连/发送/上游排队/思考的全程，直到收到 /// 首个真实内容帧（`response.created`/`response.in_progress` 等纯生命周期帧不计）。 /// 超时判定为连接落到病态上游后端，丢弃并换新连接重试。 - #[serde(default = "default_ws_first_token_timeout_ms")] - pub first_token_timeout_ms: u64, + #[serde(default = "default_ws_initial_event_timeout_ms")] + pub initial_event_timeout_ms: u64, } -fn default_ws_first_token_timeout_ms() -> u64 { +fn default_ws_initial_event_timeout_ms() -> u64 { 20_000 } @@ -192,7 +192,7 @@ impl Default for WebSocketPoolSettings { enabled: true, max_age_ms: 3_300_000, max_per_account: 8, - first_token_timeout_ms: default_ws_first_token_timeout_ms(), + initial_event_timeout_ms: default_ws_initial_event_timeout_ms(), } } } diff --git a/backend/src/bootstrap/services.rs b/backend/src/bootstrap/services.rs index d9cb02a6f..e3235095c 100644 --- a/backend/src/bootstrap/services.rs +++ b/backend/src/bootstrap/services.rs @@ -25,8 +25,9 @@ use crate::{ }, infra::{ database::connect, + identity::AccountPseudonymizer, logging::{init_tracing, LogError, LogGuard, RotationConfig, TracingConfig}, - paths::{ensure_data_dir, load_or_create_installation_id}, + paths::{ensure_data_dir, load_or_create_identity_secret}, redis::RedisConnection, }, keys::{manage::KeyManageService, service::KeyVerifier, store::PgClientKeyStore}, @@ -115,7 +116,7 @@ pub struct Services { pub codex: Arc, pub websocket_pool: Option>, pub fingerprint: RuntimeFingerprint, - pub installation_id: Option, + pub account_pseudonymizer: Arc, pub background_tasks: BackgroundTaskStores, pub(crate) account_pool_static: AccountPoolStaticSettings, } @@ -151,32 +152,42 @@ impl Services { stores: BackgroundTaskStores, fingerprint: RuntimeFingerprint, ) -> Result { - Self::try_with_installation_id(config, stores, fingerprint, None) - } - - pub(crate) fn try_with_installation_id( - config: &AppConfig, - stores: BackgroundTaskStores, - fingerprint: RuntimeFingerprint, - installation_id: Option, - ) -> Result { - Self::try_with_installation_id_and_usage_record_options( + Self::try_with_usage_record_options( config, stores, fingerprint, - installation_id, UsageRecordOptions::from_config(config), ) } - pub fn try_with_installation_id_and_usage_record_options( + pub fn try_with_usage_record_options( config: &AppConfig, stores: BackgroundTaskStores, fingerprint: RuntimeFingerprint, - installation_id: Option, usage_record_options: UsageRecordOptions, ) -> Result { - let installation_id = installation_id.filter(|id| !id.trim().is_empty()); + let mut identity_secret = [0u8; 32]; + rand::Rng::fill_bytes(&mut rand::rng(), &mut identity_secret); + Self::try_with_identity_secret_and_usage_record_options( + config, + stores, + fingerprint, + identity_secret, + usage_record_options, + ) + } + + fn try_with_identity_secret_and_usage_record_options( + config: &AppConfig, + stores: BackgroundTaskStores, + fingerprint: RuntimeFingerprint, + identity_secret: [u8; 32], + usage_record_options: UsageRecordOptions, + ) -> Result { + let account_pseudonymizer = Arc::new(AccountPseudonymizer::new(identity_secret)); + let account_identity = Arc::new(crate::dispatch::affinity::AccountIdentityService::new( + (*account_pseudonymizer).clone(), + )); let database = stores.accounts.pool().clone(); let redis = stores.redis.clone(); let account_store = Arc::new(stores.accounts.clone()) as Arc; @@ -187,21 +198,21 @@ impl Services { enabled: true, max_age: std::time::Duration::from_millis(config.ws_pool.max_age_ms), max_per_account: config.ws_pool.max_per_account, - first_token_timeout: (config.ws_pool.first_token_timeout_ms > 0).then(|| { - std::time::Duration::from_millis(config.ws_pool.first_token_timeout_ms) + initial_event_timeout: (config.ws_pool.initial_event_timeout_ms > 0).then(|| { + std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms) }), ..CodexWebSocketPoolConfig::default() })) }); let codex = { - let websocket_first_token_timeout = (config.ws_pool.first_token_timeout_ms > 0) - .then(|| std::time::Duration::from_millis(config.ws_pool.first_token_timeout_ms)); + let websocket_initial_event_timeout = (config.ws_pool.initial_event_timeout_ms > 0) + .then(|| std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms)); let client = CodexBackendClient::new( build_reqwest_client(config.tls.force_http11)?, config.api.base_url.clone(), fingerprint.clone(), ) - .with_websocket_first_token_timeout(websocket_first_token_timeout); + .with_websocket_initial_event_timeout(websocket_initial_event_timeout); if let Some(pool) = &websocket_pool { Arc::new(client.with_websocket_pool(pool.clone())) } else { @@ -275,7 +286,7 @@ impl Services { config.auth.oauth_token_endpoint.clone(), ), refresh_policy: refresh_policy.clone(), - installation_id: installation_id.clone(), + account_pseudonymizer: account_pseudonymizer.clone(), })); let usage = Arc::new(AccountUsageQueryService::new(stores.account_usage.clone())); let session_affinity = @@ -289,7 +300,7 @@ impl Services { codex: codex.clone(), session_affinity: session_affinity.clone(), recorder, - installation_id: installation_id.clone(), + account_identity, cloudflare: cloudflare_recovery, })); @@ -314,7 +325,7 @@ impl Services { codex, websocket_pool, fingerprint, - installation_id, + account_pseudonymizer, background_tasks: stores, account_pool_static, }) @@ -520,18 +531,19 @@ async fn build_router( config.admin.validate_default_password()?; let default_admin_password = config.admin.default_password.clone(); let data_dir = ensure_data_dir()?; - let installation_id = load_or_create_installation_id(Some(&data_dir))?; + let identity_secret = load_or_create_identity_secret(&data_dir)?; let default_fingerprint = fingerprint_from_config(&config.fingerprint); let runtime_fingerprint = fingerprint_store .ensure_current_seed(&default_fingerprint) .await?; let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint); let runtime_config = crate::bootstrap::state::RuntimeConfig::from(config.clone()); - let services = Services::try_with_installation_id( + let services = Services::try_with_identity_secret_and_usage_record_options( &config, stores, runtime_fingerprint, - Some(installation_id), + identity_secret, + UsageRecordOptions::from_config(&config), )?; services.initialize_hot_path_state().await?; diff --git a/backend/src/bootstrap/tasks/coordinator.rs b/backend/src/bootstrap/tasks/coordinator.rs index 754b2659e..566b2b18f 100644 --- a/backend/src/bootstrap/tasks/coordinator.rs +++ b/backend/src/bootstrap/tasks/coordinator.rs @@ -53,9 +53,23 @@ impl TaskCoordinator { ); coordinator.push( "model_refresh", - ModelRefreshTask::new(services.models.clone(), services.account_pool.clone()) - .with_installation_id(services.installation_id.clone()) - .start(), + ModelRefreshTask::new( + services.models.clone(), + services.account_pool.clone(), + services.account_pseudonymizer.clone(), + ) + .start(), + ); + coordinator.push( + "model_etag_refresh", + SchedulerHandle::spawn( + ModelRefreshTask::new( + services.models.clone(), + services.account_pool.clone(), + services.account_pseudonymizer.clone(), + ) + .run_etag_refreshes(), + ), ); if config.auth.refresh_enabled { coordinator.push( @@ -69,6 +83,7 @@ impl TaskCoordinator { stores.accounts.clone(), stores.account_usage.clone(), services.codex.clone(), + services.account_pseudonymizer.clone(), config .quota .refresh_interval_minutes @@ -76,7 +91,6 @@ impl TaskCoordinator { .max(1), 30 * 60, ) - .with_installation_id(services.installation_id.clone()) .with_cookie_store(stores.cookies.clone()) .with_account_pool(services.account_pool.clone()) .start(), diff --git a/backend/src/bootstrap/tasks/model_refresh.rs b/backend/src/bootstrap/tasks/model_refresh.rs index abeacbf03..94a005845 100644 --- a/backend/src/bootstrap/tasks/model_refresh.rs +++ b/backend/src/bootstrap/tasks/model_refresh.rs @@ -8,6 +8,7 @@ use tokio::time::sleep; use tracing::{info, warn}; use crate::fleet::pool::AccountPoolService; +use crate::infra::identity::AccountPseudonymizer; use crate::models::service::{ ModelRefreshPlanAccount, ModelRefreshResult, ModelService, ModelServiceError, }; @@ -25,7 +26,7 @@ pub struct ModelRefreshTask { initial_delay_ms: u64, retry_delay_ms: u64, max_retries: u32, - installation_id: Option, + account_pseudonymizer: Arc, } const DEFAULT_REFRESH_INTERVAL_SECS: u64 = 3600; @@ -35,7 +36,11 @@ const MAX_RETRIES: u32 = 12; impl ModelRefreshTask { /// 构造默认任务。 - pub fn new(model_service: Arc, account_pool: Arc) -> Self { + pub fn new( + model_service: Arc, + account_pool: Arc, + account_pseudonymizer: Arc, + ) -> Self { Self { model_service, account_pool, @@ -43,16 +48,10 @@ impl ModelRefreshTask { initial_delay_ms: INITIAL_DELAY_MS, retry_delay_ms: RETRY_DELAY_MS, max_retries: MAX_RETRIES, - installation_id: None, + account_pseudonymizer, } } - /// 设置 Codex installation id。 - pub fn with_installation_id(mut self, installation_id: Option) -> Self { - self.installation_id = installation_id.filter(|id| !id.trim().is_empty()); - self - } - /// 启动后台刷新任务。 pub fn start(self) -> SchedulerHandle { let config = PeriodicTaskConfig::new( @@ -64,6 +63,27 @@ impl ModelRefreshTask { spawn_periodic_task(self, config) } + /// 在 Responses 响应声明模型目录 ETag 变化时异步刷新。 + pub async fn run_etag_refreshes(self) { + let mut etag_changes = self.model_service.subscribe_models_etag_changes(); + while etag_changes.changed().await.is_ok() { + // 合并同一批并发响应产生的重复通知。 + sleep(Duration::from_millis(250)).await; + etag_changes.borrow_and_update(); + match self.refresh_once().await { + Ok(result) => info!( + refreshed_plans = result.refreshed_plans, + model_count = result.model_count, + "模型 ETag 变化刷新完成" + ), + Err(ModelServiceError::NoAccounts) => { + info!("模型 ETag 变化刷新跳过：没有可用账号"); + } + Err(error) => warn!(error = ?error, "模型 ETag 变化刷新失败"), + } + } + } + /// 执行一次模型刷新并同步账号池模型计划约束。 pub async fn refresh_once(&self) -> Result { let request_id = uuid::Uuid::new_v4().to_string(); @@ -74,16 +94,15 @@ impl ModelRefreshTask { plan_type: selected.plan_type.clone(), access_token: selected.account.access_token.clone(), account_id: selected.account.account_id.clone(), + installation_id: self + .account_pseudonymizer + .installation_id(&selected.account.id), }) .collect::>(); let result = self .model_service - .refresh_backend_models_with_installation_id( - &plan_accounts, - &request_id, - self.installation_id.as_deref(), - ) + .refresh_backend_models(&plan_accounts, &request_id) .await; for selected in selected_accounts { selected.release().await; diff --git a/backend/src/bootstrap/tasks/quota_refresh.rs b/backend/src/bootstrap/tasks/quota_refresh.rs index d4df05037..975c03b6f 100644 --- a/backend/src/bootstrap/tasks/quota_refresh.rs +++ b/backend/src/bootstrap/tasks/quota_refresh.rs @@ -12,6 +12,7 @@ use crate::{ quota::{QuotaRefreshService, QuotaRefreshSummary}, store::PgAccountStore, }, + infra::identity::AccountPseudonymizer, telemetry::account_usage::store::PgAccountUsageStore, upstream::openai::transport::CodexBackendClient, }; @@ -34,6 +35,7 @@ impl QuotaRefreshTask { store: PgAccountStore, usage_store: PgAccountUsageStore, codex: Arc, + account_pseudonymizer: Arc, interval_secs: u64, min_refresh_interval_secs: u64, ) -> Self { @@ -42,6 +44,7 @@ impl QuotaRefreshTask { store, usage_store, codex, + account_pseudonymizer, min_refresh_interval_secs, ), interval_secs, @@ -49,12 +52,6 @@ impl QuotaRefreshTask { } } - /// 设置 Codex installation id。 - pub fn with_installation_id(mut self, installation_id: Option) -> Self { - self.service = self.service.with_installation_id(installation_id); - self - } - /// 设置 usage 请求可复用的账号 Cookie 存储。 pub fn with_cookie_store(mut self, cookie_store: PgCookieStore) -> Self { self.service = self.service.with_cookie_store(cookie_store); diff --git a/backend/src/dispatch/affinity/identity.rs b/backend/src/dispatch/affinity/identity.rs new file mode 100644 index 000000000..e3a1dea68 --- /dev/null +++ b/backend/src/dispatch/affinity/identity.rs @@ -0,0 +1,171 @@ +//! 客户端身份、本地会话索引与账号作用域 wire 身份隔离。 + +use crate::{ + infra::identity::AccountPseudonymizer, + upstream::openai::protocol::responses::{CodexCompactRequest, CodexResponsesRequest}, +}; + +/// 持久化密钥驱动的账号身份隔离服务。 +#[derive(Clone)] +pub struct AccountIdentityService { + pseudonymizer: AccountPseudonymizer, +} + +impl std::fmt::Debug for AccountIdentityService { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("AccountIdentityService") + .finish_non_exhaustive() + } +} + +/// 单次账号 attempt 的 wire 身份。 +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AccountScopedIdentity { + pub prompt_cache_key: Option, + pub session_id: Option, + pub thread_id: Option, + pub client_request_id: String, + pub turn_id: Option, + pub window_id: Option, + pub parent_thread_id: Option, + pub installation_id: String, +} + +struct IdentityValues<'a> { + prompt_cache_key: Option<&'a str>, + session_id: Option<&'a str>, + thread_id: Option<&'a str>, + client_request_id: Option<&'a str>, + turn_id: Option<&'a str>, + window_id: Option<&'a str>, + parent_thread_id: Option<&'a str>, +} + +impl AccountIdentityService { + pub fn new(pseudonymizer: AccountPseudonymizer) -> Self { + Self { pseudonymizer } + } + + pub fn prepare_local_identity(&self, request: &mut CodexResponsesRequest) { + if request.local_conversation_id.is_some() { + return; + } + let anchor = request + .prompt_cache_key() + .or(request.client_session_id.as_deref()) + .or(request.client_thread_id.as_deref()) + .or(request.client_conversation_id.as_deref()) + .map(ToString::to_string) + .or_else(|| super::resolve::derive_stable_conversation_key(request)); + request.local_conversation_id = anchor + .as_deref() + .map(|anchor| format!("lc_{}", self.pseudonym("local-conversation", None, anchor))); + } + + pub fn scope( + &self, + request: &CodexResponsesRequest, + account_id: &str, + proxy_request_id: &str, + ) -> AccountScopedIdentity { + self.scope_values( + account_id, + proxy_request_id, + IdentityValues { + prompt_cache_key: request.prompt_cache_key(), + session_id: request.client_session_id.as_deref(), + thread_id: request.client_thread_id.as_deref(), + client_request_id: request.client_request_id.as_deref(), + turn_id: request.client_turn_id.as_deref(), + window_id: request.codex_window_id.as_deref(), + parent_thread_id: request.parent_thread_id.as_deref(), + }, + ) + } + + pub fn scope_compact( + &self, + request: &CodexCompactRequest, + account_id: &str, + proxy_request_id: &str, + ) -> AccountScopedIdentity { + self.scope_values( + account_id, + proxy_request_id, + IdentityValues { + prompt_cache_key: request.prompt_cache_key(), + session_id: request.client_session_id.as_deref(), + thread_id: request.client_thread_id.as_deref(), + client_request_id: request.client_request_id.as_deref(), + turn_id: request.client_turn_id.as_deref(), + window_id: request.client_window_id.as_deref(), + parent_thread_id: request.client_parent_thread_id.as_deref(), + }, + ) + } + + pub fn scope_auxiliary( + &self, + account_id: &str, + proxy_request_id: &str, + ) -> AccountScopedIdentity { + AccountScopedIdentity { + client_request_id: format!( + "cr_{}", + self.pseudonym("client-request-id", Some(account_id), proxy_request_id) + ), + installation_id: self.pseudonymizer.installation_id(account_id), + ..AccountScopedIdentity::default() + } + } + + fn scope_values( + &self, + account_id: &str, + proxy_request_id: &str, + values: IdentityValues<'_>, + ) -> AccountScopedIdentity { + AccountScopedIdentity { + prompt_cache_key: self.scope_optional( + "prompt-cache-key", + account_id, + values.prompt_cache_key, + ), + session_id: self.scope_optional("session-id", account_id, values.session_id), + thread_id: self.scope_optional("thread-id", account_id, values.thread_id), + client_request_id: format!( + "cr_{}", + self.pseudonym( + "client-request-id", + Some(account_id), + values.client_request_id.unwrap_or(proxy_request_id), + ) + ), + turn_id: self.scope_optional("turn-id", account_id, values.turn_id), + window_id: self.scope_optional("window-id", account_id, values.window_id), + parent_thread_id: self.scope_optional( + "parent-thread-id", + account_id, + values.parent_thread_id, + ), + installation_id: self.pseudonymizer.installation_id(account_id), + } + } + + fn scope_optional( + &self, + domain: &str, + account_id: &str, + value: Option<&str>, + ) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| format!("wi_{}", self.pseudonym(domain, Some(account_id), value))) + } + + fn pseudonym(&self, domain: &str, account_id: Option<&str>, value: &str) -> String { + self.pseudonymizer.scoped(domain, account_id, value) + } +} diff --git a/backend/src/dispatch/affinity/mod.rs b/backend/src/dispatch/affinity/mod.rs index de11bb80a..98f13be67 100644 --- a/backend/src/dispatch/affinity/mod.rs +++ b/backend/src/dispatch/affinity/mod.rs @@ -1,10 +1,12 @@ //! 会话亲和身份、存储、解析与运行时服务。 +pub mod identity; pub mod resolve; pub mod service; pub mod store; pub mod types; +pub use identity::{AccountIdentityService, AccountScopedIdentity}; pub use resolve::*; pub use service::*; pub use store::*; diff --git a/backend/src/dispatch/affinity/resolve.rs b/backend/src/dispatch/affinity/resolve.rs index 7ccbb8c95..6a98915f5 100644 --- a/backend/src/dispatch/affinity/resolve.rs +++ b/backend/src/dispatch/affinity/resolve.rs @@ -1,89 +1,33 @@ -use std::sync::Arc; - use chrono::{DateTime, Utc}; use serde_json::Value; use sha2::{Digest, Sha256}; -use tokio::sync::Mutex; -use uuid::Uuid; use crate::{ dispatch::{ - affinity::{types::ConversationIdentity, SessionAffinityEntry, SessionAffinityService}, - recovery::{ - implicit_resume::{restore_full_request_without_history, ImplicitResumeSnapshot}, - reasoning_replay::ReasoningReplayCache, - }, + affinity::{SessionAffinityEntry, SessionAffinityService}, + recovery::history::HistoryRecoveryPlan, service::ResponseDispatchService, }, - fleet::account::AccountStatus, upstream::openai::protocol::{ events::TokenUsage, responses::{completed_response_metadata, CodexResponsesRequest}, }, }; -#[derive(Debug, Clone, Default)] -pub(in crate::dispatch) struct AccountAffinityDecision { - preferred_account_id: Option, - request_history_owner: RequestHistoryOwner, -} - -#[derive(Debug, Clone, Default)] -enum RequestHistoryOwner { - #[default] - Absent, - Known(String), - Unknown, -} - -impl AccountAffinityDecision { - pub(in crate::dispatch) fn preferred_account_id(&self) -> Option<&str> { - self.preferred_account_id.as_deref() - } - - fn should_strip_request_history_for(&self, account_id: &str) -> bool { - match &self.request_history_owner { - RequestHistoryOwner::Absent => false, - RequestHistoryOwner::Known(history_account_id) => history_account_id != account_id, - RequestHistoryOwner::Unknown => true, - } - } -} - impl ResponseDispatchService { - pub(in crate::dispatch) async fn account_affinity_for_request( - &self, - request: &CodexResponsesRequest, - now: DateTime, - ) -> AccountAffinityDecision { - let has_previous_response_id = request.previous_response_id().is_some(); - if let Some(account_id) = self.previous_response_account_id(request, now).await { - return AccountAffinityDecision { - preferred_account_id: Some(account_id.clone()), - request_history_owner: RequestHistoryOwner::Known(account_id), - }; - } - - let conversation_account_id = self.conversation_account_id(request, now).await; - AccountAffinityDecision { - preferred_account_id: conversation_account_id, - request_history_owner: if has_previous_response_id { - RequestHistoryOwner::Unknown - } else { - RequestHistoryOwner::Absent - }, - } - } - - async fn previous_response_account_id( + pub(in crate::dispatch) async fn preferred_account_id_for_request( &self, request: &CodexResponsesRequest, + history: &HistoryRecoveryPlan, now: DateTime, ) -> Option { - let previous_response_id = request.previous_response_id()?; - self.session_affinity - .lookup_account(previous_response_id, now) - .await + if let Some(account_id) = history.preferred_account_id() { + return Some(account_id.to_string()); + } + if request.previous_response_id().is_some() { + return None; + } + self.conversation_account_id(request, now).await } async fn conversation_account_id( @@ -92,7 +36,8 @@ impl ResponseDispatchService { now: DateTime, ) -> Option { let conversation_id = request - .prompt_cache_key() + .local_conversation_id + .as_deref() .map(str::trim) .filter(|value| !value.is_empty())?; let variant_hash = compute_variant_hash(request); @@ -100,89 +45,17 @@ impl ResponseDispatchService { .lookup_latest_account_by_conversation(conversation_id, None, Some(&variant_hash), now) .await } - - pub(in crate::dispatch) fn strip_history_if_account_changed( - request: &mut CodexResponsesRequest, - implicit_resume: &mut Option, - account_affinity: &AccountAffinityDecision, - acquired_account_id: &str, - ) { - if request.previous_response_id().is_none() - || !account_affinity.should_strip_request_history_for(acquired_account_id) - { - return; - } - - let Some(snapshot) = implicit_resume.take() else { - return; - }; - restore_full_request_without_history(request, snapshot); - } - - pub(in crate::dispatch) async fn apply_cascading_ban_defense( - &self, - request: &mut CodexResponsesRequest, - implicit_resume: &mut Option, - preferred_account_id: Option<&str>, - acquired_account_id: &str, - ) -> bool { - let Some(preferred_account_id) = - preferred_account_id.filter(|account_id| *account_id != acquired_account_id) - else { - return false; - }; - let has_history = request.previous_response_id().is_some() - || request - .turn_state - .as_deref() - .is_some_and(|value| !value.is_empty()); - if !has_history { - return false; - } - let Some(preferred_account) = self - .account_pool - .account_snapshot(preferred_account_id) - .await - else { - return false; - }; - if !matches!( - preferred_account.status, - AccountStatus::Banned | AccountStatus::Disabled - ) { - return false; - } - - let response_id_to_forget = request.previous_response_id().map(str::to_string); - let restored_full_request = if let Some(snapshot) = implicit_resume.take() { - restore_full_request_without_history(request, snapshot); - true - } else { - false - }; - if let Some(response_id) = response_id_to_forget { - self.session_affinity.forget(&response_id).await; - } - restored_full_request - } } -pub(crate) async fn record_response_affinity( - session_affinity: &Arc, - reasoning_replay: &Arc>, - request: &CodexResponsesRequest, +pub(in crate::dispatch) async fn record_response_affinity( + session_affinity: &SessionAffinityService, + history: &HistoryRecoveryPlan, + original_request: &CodexResponsesRequest, account_id: &str, body: &str, turn_state: Option, usage: Option, ) { - let Some(conversation_id) = request - .prompt_cache_key() - .map(str::trim) - .filter(|value| !value.is_empty()) - else { - return; - }; let metadata = match completed_response_metadata(body) { Ok(Some(metadata)) => metadata, Ok(None) => return, @@ -195,17 +68,27 @@ pub(crate) async fn record_response_affinity( } }; - let variant_hash = compute_variant_hash(request); + let conversation_id = history + .conversation_id(original_request) + .unwrap_or(&metadata.response_id) + .to_string(); + let variant_hash = compute_variant_hash(original_request); let entry = SessionAffinityEntry { account_id: account_id.to_string(), - conversation_id: conversation_id.to_string(), + conversation_id, turn_state: turn_state .filter(|value| !value.trim().is_empty()) - .or_else(|| request.turn_state.clone()), - instructions_hash: Some(hash_instructions(Some(request.instructions()))), + .or_else(|| original_request.turn_state.clone()), + instructions_hash: Some(hash_instructions(Some(original_request.instructions()))), input_tokens: usage.map(|usage| usage.input_tokens), function_call_ids: metadata.function_call_ids, - variant_hash: Some(variant_hash.clone()), + variant_hash: Some(variant_hash), + continuation_scope: if original_request.store() { + crate::upstream::openai::protocol::responses::PreviousResponseScope::Persisted + } else { + crate::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal + }, + replay: history.completed_replay(original_request, &metadata.output), created_at: Utc::now(), }; if let Err(error) = session_affinity @@ -219,30 +102,6 @@ pub(crate) async fn record_response_affinity( "failed to record session affinity" ); } - - reasoning_replay.lock().await.record( - metadata.response_id, - account_id, - conversation_id, - &variant_hash, - &metadata.replay_items, - Utc::now(), - ); -} - -pub(crate) async fn evict_reasoning_replay( - reasoning_replay: &Arc>, - request: &CodexResponsesRequest, - account_id: &str, -) { - let variant_hash = compute_variant_hash(request); - let conversation_id = request.prompt_cache_key().unwrap_or("").to_string(); - reasoning_replay.lock().await.evict_by_identity( - account_id, - &conversation_id, - &variant_hash, - Utc::now(), - ); } // ==================================================================== @@ -309,58 +168,9 @@ pub fn hash_instructions(instructions: Option<&str>) -> String { hex::encode(hasher.finalize()) } -// ==================================================================== -// Conversation identity 构建器 -// ==================================================================== - const LEADING_SYSTEM_REMINDER_OPEN: &str = ""; const LEADING_SYSTEM_REMINDER_CLOSE: &str = ""; -/// 从 prompt_cache_key 和可选的 window_id 构建 conversation identity -pub fn build_conversation_identity( - prompt_cache_key: Option<&str>, - client_window_id: Option<&str>, - account_scope: &str, -) -> ConversationIdentity { - let conversation_id = prompt_cache_key - .filter(|s| !s.trim().is_empty()) - .map(|key| build_account_scoped_identity("conversation", account_scope, key)); - - let window_id = if let Some(client_win) = client_window_id.filter(|s| !s.trim().is_empty()) { - Some(build_account_scoped_identity( - "window", - account_scope, - client_win, - )) - } else { - conversation_id - .as_ref() - .map(|conv_id| format!("{}:0", conv_id)) - }; - - ConversationIdentity { - conversation_id, - window_id, - } -} - -/// 确保请求拥有上游可复用的 prompt cache key。 -pub fn ensure_prompt_cache_key(request: &mut CodexResponsesRequest) { - if let Some(existing) = request - .prompt_cache_key() - .map(str::trim) - .filter(|value| !value.is_empty()) - { - let existing = existing.to_string(); - request.set_prompt_cache_key(Some(existing)); - return; - } - - request.set_prompt_cache_key(Some( - derive_stable_conversation_key(request).unwrap_or_else(|| Uuid::new_v4().to_string()), - )); -} - /// 按原版 `stable-conversation-key.ts` 的规则派生稳定 conversation key。 pub fn derive_stable_conversation_key(request: &CodexResponsesRequest) -> Option { let instructions = request @@ -436,25 +246,3 @@ fn normalize_conversation_anchor_text(text: &str) -> String { } rest.to_string() } - -/// 构建账号作用域的身份哈希。 -fn build_account_scoped_identity(kind: &str, account_scope: &str, client_value: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(kind.as_bytes()); - hasher.update(b"\0"); - hasher.update(account_scope.as_bytes()); - hasher.update(b"\0"); - hasher.update(client_value.as_bytes()); - - let digest = hasher.finalize(); - let hex = hex::encode(digest); - let truncated = &hex[..32]; - - let prefix = match kind { - "conversation" => "cp", - "window" => "cw", - _ => "cx", - }; - - format!("{}_{}", prefix, truncated) -} diff --git a/backend/src/dispatch/affinity/store.rs b/backend/src/dispatch/affinity/store.rs index 2242600b8..7ee6aad26 100644 --- a/backend/src/dispatch/affinity/store.rs +++ b/backend/src/dispatch/affinity/store.rs @@ -181,14 +181,15 @@ impl RedisSessionAffinityStore { } fn response_key(&self, response_id: &str) -> String { - self.redis.key(&format!("affinity:resp:{response_id}")) + self.redis.key(&format!("affinity:v2:resp:{response_id}")) } fn conversation_key(&self, conversation_id: &str) -> String { - self.redis.key(&format!("affinity:conv:{conversation_id}")) + self.redis + .key(&format!("affinity:v2:conv:{conversation_id}")) } fn account_key(&self, account_id: &str) -> String { - self.redis.key(&format!("affinity:account:{account_id}")) + self.redis.key(&format!("affinity:v2:account:{account_id}")) } } diff --git a/backend/src/dispatch/affinity/types.rs b/backend/src/dispatch/affinity/types.rs index e5e1d8ac7..8c479f984 100644 --- a/backend/src/dispatch/affinity/types.rs +++ b/backend/src/dispatch/affinity/types.rs @@ -2,6 +2,9 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::upstream::openai::protocol::responses::PreviousResponseScope; /// 会话亲和性条目。 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -14,12 +17,14 @@ pub struct SessionAffinityEntry { pub input_tokens: Option, pub function_call_ids: Vec, pub variant_hash: Option, + pub continuation_scope: PreviousResponseScope, + pub replay: Option, pub created_at: DateTime, } -/// 从请求上下文派生的 conversation identity。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ConversationIdentity { - pub conversation_id: Option, - pub window_id: Option, +/// 截止某个 completed response 的完整、无凭据业务输入快照。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResponseReplaySnapshot { + pub full_input: Vec, } diff --git a/backend/src/dispatch/attempts.rs b/backend/src/dispatch/attempts.rs new file mode 100644 index 000000000..585ecd3a1 --- /dev/null +++ b/backend/src/dispatch/attempts.rs @@ -0,0 +1,82 @@ +//! 请求级账号候选快照与尝试账本。 + +use std::collections::{BTreeSet, VecDeque}; + +use chrono::Utc; + +use crate::fleet::pool::{ + AccountAcquireRequest, AccountCandidateLease, AccountLease, AccountPoolService, +}; + +/// 请求开始时冻结的完整候选集，以及候选的最终处理状态。 +pub(in crate::dispatch) struct AccountAttemptLedger { + model: String, + candidate_ids: Vec, + pending: VecDeque, + busy: VecDeque, + attempted: BTreeSet, + state_excluded: BTreeSet, +} + +impl AccountAttemptLedger { + pub(in crate::dispatch) async fn freeze( + account_pool: &AccountPoolService, + request: &AccountAcquireRequest, + ) -> Self { + let candidate_ids = account_pool.candidate_snapshot(request).await; + Self { + model: request.model.clone(), + pending: candidate_ids.iter().cloned().collect(), + candidate_ids, + busy: VecDeque::new(), + attempted: BTreeSet::new(), + state_excluded: BTreeSet::new(), + } + } + + pub(in crate::dispatch) async fn acquire_next( + &mut self, + account_pool: &AccountPoolService, + ) -> Option { + loop { + while let Some(account_id) = self.pending.pop_front() { + match account_pool + .acquire_candidate(&self.model, &account_id, Utc::now()) + .await + { + AccountCandidateLease::Acquired(lease) => { + self.attempted.insert(account_id); + return Some(*lease); + } + AccountCandidateLease::Busy => self.busy.push_back(account_id), + AccountCandidateLease::Unavailable => { + self.state_excluded.insert(account_id); + } + } + } + + if self.busy.is_empty() { + debug_assert_eq!( + self.candidate_ids.len(), + self.attempted.len() + self.state_excluded.len() + ); + return None; + } + + account_pool.wait_for_candidate_change().await; + std::mem::swap(&mut self.pending, &mut self.busy); + } + } + + pub(in crate::dispatch) fn candidate_count(&self) -> usize { + self.candidate_ids.len() + } + + pub(in crate::dispatch) fn attempted_count(&self) -> usize { + self.attempted.len() + } + + pub(in crate::dispatch) fn state_excluded_count(&self) -> usize { + self.state_excluded.len() + } +} diff --git a/backend/src/dispatch/errors.rs b/backend/src/dispatch/errors.rs index 054d6b9ee..6be0deb54 100644 --- a/backend/src/dispatch/errors.rs +++ b/backend/src/dispatch/errors.rs @@ -35,6 +35,8 @@ pub(crate) enum DispatchFailureClass { CloudflareChallenge, CloudflarePathBlocked, ModelUnsupported, + UpstreamUnavailable, + ContinuationBusy, HistoryUnavailable, Upstream, InvalidSse, @@ -55,6 +57,8 @@ impl DispatchFailureClass { Self::CloudflareChallenge => "cloudflare_challenge", Self::CloudflarePathBlocked => "cloudflare_path_blocked", Self::ModelUnsupported => "model_unsupported", + Self::UpstreamUnavailable => "upstream_unavailable", + Self::ContinuationBusy => "continuation_busy", Self::HistoryUnavailable => "history_unavailable", Self::Upstream => "upstream", Self::InvalidSse => "invalid_sse", @@ -121,6 +125,7 @@ impl From for DispatchFailureClass { ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge, ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked, ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported, + ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable, } } } @@ -193,12 +198,25 @@ pub(crate) fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool { pub(crate) fn is_retryable_upstream_5xx_error(error: &CodexClientError) -> bool { matches!( error, - CodexClientError::Upstream { status, body, .. } + CodexClientError::Upstream { status, .. } if status_code_is_transient_upstream(status.as_u16()) - && !is_history_recovery_signal(body) ) } +pub(crate) fn is_retryable_account_transport_error(error: &CodexClientError) -> bool { + match error { + CodexClientError::Http(error) => error.is_connect() || error.is_timeout(), + CodexClientError::WebSocket( + crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::Transport(_) + | crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ClosedBeforeTerminal + | crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReceiveIdleTimeout { .. } + | crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { .. } + | crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::InitialEventTimeout { .. }, + ) => true, + _ => is_retryable_upstream_5xx_error(error), + } +} + pub(crate) fn is_quota_exhausted_upstream_error(error: &CodexClientError) -> bool { matches!( error, @@ -217,9 +235,26 @@ pub(crate) fn is_model_unsupported_upstream_error(error: &CodexClientError) -> b } pub(crate) fn is_history_recovery_upstream_error(error: &CodexClientError) -> bool { + upstream_error_code(error).is_some_and(|code| is_history_recovery_code(&code)) + || matches!( + error, + CodexClientError::WebSocket( + crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable { reason } + ) if !matches!( + reason, + crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy + ) + ) +} + +pub(crate) fn is_continuation_busy_error(error: &CodexClientError) -> bool { matches!( error, - CodexClientError::Upstream { body, .. } if is_history_recovery_signal(body) + CodexClientError::WebSocket( + crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable { + reason: crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy, + } + ) ) } @@ -250,18 +285,27 @@ pub(crate) fn is_model_unsupported_signal(value: &str) -> bool { || value.contains("not_available"))) } -pub(crate) fn is_history_recovery_signal(value: &str) -> bool { - let value = value.to_ascii_lowercase(); - value.contains("previous_response_not_found") - || (value.contains("previous response") && value.contains("not found")) - || value.contains("no tool output found for function call") - || is_invalid_encrypted_content_signal(&value) +pub(crate) fn is_history_recovery_code(code: &str) -> bool { + matches!( + code, + "previous_response_not_found" + | "invalid_encrypted_content" + | "missing_tool_output" + | "no_tool_output" + ) } -pub(crate) fn is_invalid_encrypted_content_signal(value: &str) -> bool { - let value = value.to_ascii_lowercase(); - value.contains("invalid_encrypted_content") - || (value.contains("invalid") && value.contains("encrypted") && value.contains("content")) +pub(crate) fn upstream_error_code(error: &CodexClientError) -> Option { + let CodexClientError::Upstream { body, .. } = error else { + return None; + }; + let value = serde_json::from_str::(body).ok()?; + value + .pointer("/response/error/code") + .or_else(|| value.pointer("/error/code")) + .or_else(|| value.get("code")) + .and_then(Value::as_str) + .map(ToString::to_string) } pub(crate) fn rate_limit_cooldown_until( @@ -349,6 +393,13 @@ pub enum ResponseDispatchError { count: usize, upstream_error: String, }, + #[error("all account candidates failed with transient upstream errors")] + UpstreamUnavailable { + count: usize, + upstream_error: String, + }, + #[error("the previous response connection is busy; retry the request")] + ContinuationBusy, #[error("previous response context is unavailable: {upstream_error}")] HistoryUnavailable { upstream_error: String }, #[error("upstream request failed: {0}")] @@ -399,6 +450,10 @@ impl ResponseDispatchError { count: exhausted.count, upstream_error: exhausted.upstream_error, }, + ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, } } @@ -414,6 +469,8 @@ impl ResponseDispatchError { | Self::InvalidSse(_) | Self::MissingCompleted | Self::EmptyUpstreamResponse => 502, + Self::UpstreamUnavailable { .. } => 503, + Self::ContinuationBusy => 503, Self::Failed(failure) => stream_failure_http_status(failure), Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400, Self::Upstream(error) => upstream_error_http_status(error), @@ -481,6 +538,17 @@ impl ResponseDispatchError { *count, upstream_error, ), + Self::UpstreamUnavailable { + count, + upstream_error, + } => Self::exhausted_metadata( + ExhaustedAccountKind::UpstreamUnavailable, + *count, + upstream_error, + ), + Self::ContinuationBusy => { + DispatchErrorMetadata::simple(DispatchFailureClass::ContinuationBusy) + } Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata { failure_class: DispatchFailureClass::HistoryUnavailable, exhausted_count: None, @@ -573,6 +641,14 @@ impl ResponseDispatchError { count: *count, upstream_error, }), + Self::UpstreamUnavailable { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::UpstreamUnavailable, + count: *count, + upstream_error, + }), _ => None, } } diff --git a/backend/src/dispatch/mod.rs b/backend/src/dispatch/mod.rs index 5979d966e..73a5e94d0 100644 --- a/backend/src/dispatch/mod.rs +++ b/backend/src/dispatch/mod.rs @@ -1,6 +1,7 @@ //! 请求调度模块（账号选择、回退、恢复、会话亲和性）。 pub mod affinity; +mod attempts; pub mod errors; mod recording; pub mod recovery; diff --git a/backend/src/dispatch/recording.rs b/backend/src/dispatch/recording.rs index d9fd5c44f..13c97e064 100644 --- a/backend/src/dispatch/recording.rs +++ b/backend/src/dispatch/recording.rs @@ -479,7 +479,30 @@ pub(super) async fn record_live_response_stream_event( rate_limit_headers: &[(String, String)], body: &str, ) { + let effective_model = context + .response_metadata + .effective_model + .as_deref() + .unwrap_or(&context.display_model); ensure_stream_metadata_flag(&mut metadata); + if let Some(object) = metadata.as_object_mut() { + object.insert( + "effectiveModel".to_string(), + Value::String(effective_model.to_string()), + ); + object.insert( + "modelsEtag".to_string(), + context + .response_metadata + .models_etag + .as_ref() + .map_or(Value::Null, |etag| Value::String(etag.clone())), + ); + object.insert( + "reasoningIncluded".to_string(), + Value::Bool(context.response_metadata.reasoning_included), + ); + } enrich_event_route_metadata(&mut metadata, &context.route); enrich_live_response_stream_metadata( context, @@ -492,7 +515,7 @@ pub(super) async fn record_live_response_stream_event( enrich_usage_record_identity( &mut metadata, Some(&context.requested_model), - &context.display_model, + effective_model, context.client_ip.as_deref(), context.request.client_user_agent.as_deref(), reasoning_effort_from_request(&context.request), @@ -507,7 +530,7 @@ pub(super) async fn record_live_response_stream_event( client_api_key_id: context.request.client_api_key_id.as_deref(), account_id: &context.account_id, route: &context.route, - model: &context.display_model, + model: effective_model, requested_model: Some(&context.requested_model), client_ip: context.client_ip.as_deref(), client_user_agent: context.request.client_user_agent.as_deref(), @@ -528,7 +551,7 @@ pub(super) async fn record_live_response_stream_event( event.provider = Some("openai".to_string()); event.account_id = Some(context.account_id.clone()); event.route = Some(context.route.clone()); - event.model = Some(context.display_model.clone()); + event.model = Some(effective_model.to_string()); event.status_code = Some(status_code); event.latency_ms = Some(elapsed_millis_i64(context.started_at)); event.metadata = metadata; diff --git a/backend/src/dispatch/recovery/exhaustion.rs b/backend/src/dispatch/recovery/exhaustion.rs index 69f52c852..dea0491a3 100644 --- a/backend/src/dispatch/recovery/exhaustion.rs +++ b/backend/src/dispatch/recovery/exhaustion.rs @@ -12,6 +12,7 @@ pub(crate) enum ExhaustedAccountKind { CloudflareChallenge, CloudflarePathBlocked, ModelUnsupported, + UpstreamUnavailable, } impl ExhaustedAccountKind { @@ -25,6 +26,7 @@ impl ExhaustedAccountKind { Self::CloudflareChallenge => "cloudflare-challenge", Self::CloudflarePathBlocked => "cloudflare-path-block", Self::ModelUnsupported => "model-unsupported", + Self::UpstreamUnavailable => "upstream-unavailable", } } } @@ -56,7 +58,7 @@ pub(crate) struct AccountExhaustionTracker { cloudflare_challenge: ExhaustedCounter, cloudflare_path_blocked: ExhaustedCounter, model_unsupported: ExhaustedCounter, - model_unsupported_retry_used: bool, + upstream_unavailable: ExhaustedCounter, } #[derive(Default)] @@ -143,7 +145,6 @@ impl AccountExhaustionTracker { account_id: Option<&str>, upstream_error: impl Into, ) { - self.model_unsupported_retry_used = true; self.record( account_id, ExhaustedAccountKind::ModelUnsupported, @@ -152,16 +153,17 @@ impl AccountExhaustionTracker { ); } - pub(crate) fn model_unsupported_retry_exhausted( - &self, + pub(crate) fn record_upstream_unavailable( + &mut self, + account_id: Option<&str>, upstream_error: impl Into, - ) -> Option { - self.model_unsupported_retry_used.then(|| ExhaustedAccount { - kind: ExhaustedAccountKind::ModelUnsupported, - count: self.model_unsupported.count + 1, - upstream_error: upstream_error.into(), - status_code: None, - }) + ) { + self.record( + account_id, + ExhaustedAccountKind::UpstreamUnavailable, + upstream_error, + None, + ); } pub(crate) fn last_exhausted(&self) -> Option { @@ -204,6 +206,7 @@ impl AccountExhaustionTracker { ExhaustedAccountKind::CloudflareChallenge => &self.cloudflare_challenge, ExhaustedAccountKind::CloudflarePathBlocked => &self.cloudflare_path_blocked, ExhaustedAccountKind::ModelUnsupported => &self.model_unsupported, + ExhaustedAccountKind::UpstreamUnavailable => &self.upstream_unavailable, } } @@ -217,6 +220,7 @@ impl AccountExhaustionTracker { ExhaustedAccountKind::CloudflareChallenge => &mut self.cloudflare_challenge, ExhaustedAccountKind::CloudflarePathBlocked => &mut self.cloudflare_path_blocked, ExhaustedAccountKind::ModelUnsupported => &mut self.model_unsupported, + ExhaustedAccountKind::UpstreamUnavailable => &mut self.upstream_unavailable, } } } diff --git a/backend/src/dispatch/recovery/history.rs b/backend/src/dispatch/recovery/history.rs new file mode 100644 index 000000000..c2abfd7e5 --- /dev/null +++ b/backend/src/dispatch/recovery/history.rs @@ -0,0 +1,202 @@ +//! previous response 所有权与完整上下文重放。 + +use chrono::Utc; +use serde_json::Value; + +use crate::{ + dispatch::affinity::{ + types::ResponseReplaySnapshot, SessionAffinityEntry, SessionAffinityService, + }, + upstream::openai::protocol::responses::{CodexResponsesRequest, StreamCommitPolicy}, +}; + +/// 当前请求的 previous response 来源。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::dispatch) enum HistorySource { + None, + Managed, + ExternalUnknown, +} + +/// 请求级 previous response 恢复计划。 +pub(in crate::dispatch) struct HistoryRecoveryPlan { + source: HistorySource, + managed_entry: Option, + external_attempted_account_id: Option, + replay_owner_account: bool, +} + +impl HistoryRecoveryPlan { + pub(in crate::dispatch) async fn load( + session_affinity: &SessionAffinityService, + request: &CodexResponsesRequest, + ) -> Self { + let Some(previous_response_id) = request.previous_response_id() else { + return Self { + source: HistorySource::None, + managed_entry: None, + external_attempted_account_id: None, + replay_owner_account: false, + }; + }; + let managed_entry = session_affinity + .lookup(previous_response_id, Utc::now()) + .await; + Self { + source: if managed_entry.is_some() { + HistorySource::Managed + } else { + HistorySource::ExternalUnknown + }, + managed_entry, + external_attempted_account_id: None, + replay_owner_account: false, + } + } + + pub(in crate::dispatch) fn preferred_account_id(&self) -> Option<&str> { + self.managed_entry + .as_ref() + .map(|entry| entry.account_id.as_str()) + } + + pub(in crate::dispatch) fn prepare_attempt( + &mut self, + original: &CodexResponsesRequest, + account_id: &str, + ) -> Option { + match self.source { + HistorySource::None => Some(original.clone()), + HistorySource::ExternalUnknown => match self.external_attempted_account_id.as_deref() { + None => { + self.external_attempted_account_id = Some(account_id.to_string()); + let mut request = original.clone(); + request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal; + request.previous_response_scope = Some( + crate::upstream::openai::protocol::responses::PreviousResponseScope::ExternalUnknown, + ); + Some(request) + } + Some(first_account_id) if first_account_id == account_id => { + let mut request = original.clone(); + request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal; + request.previous_response_scope = Some( + crate::upstream::openai::protocol::responses::PreviousResponseScope::ExternalUnknown, + ); + Some(request) + } + Some(_) => None, + }, + HistorySource::Managed => { + let entry = self.managed_entry.as_ref()?; + if entry.account_id == account_id && !self.replay_owner_account { + let mut request = original.clone(); + request.local_conversation_id = Some(entry.conversation_id.clone()); + request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal; + request.previous_response_scope = Some(entry.continuation_scope); + if request.turn_state.as_deref().is_none_or(str::is_empty) { + request.turn_state.clone_from(&entry.turn_state); + } + Some(request) + } else { + entry.replay.as_ref().map(|snapshot| { + let mut request = full_replay_request(original, snapshot); + request.local_conversation_id = Some(entry.conversation_id.clone()); + request + }) + } + } + } + } + + /// previous ID 在原连接不可用时，切换为同账号完整重放。 + pub(in crate::dispatch) fn recover_managed_history(&mut self, account_id: &str) -> bool { + let Some(entry) = self.managed_entry.as_ref() else { + return false; + }; + if entry.account_id != account_id || entry.replay.is_none() || self.replay_owner_account { + return false; + } + self.replay_owner_account = true; + true + } + + pub(in crate::dispatch) fn can_failover(&self) -> bool { + !matches!(self.source, HistorySource::ExternalUnknown) + && self + .managed_entry + .as_ref() + .is_none_or(|entry| entry.replay.is_some()) + } + + pub(in crate::dispatch) fn is_external_unknown(&self) -> bool { + self.source == HistorySource::ExternalUnknown + } + + pub(in crate::dispatch) fn completed_replay( + &self, + original: &CodexResponsesRequest, + output: &[Value], + ) -> Option { + let mut full_input = match self.source { + HistorySource::None => original.input().to_vec(), + HistorySource::Managed => self + .managed_entry + .as_ref()? + .replay + .as_ref()? + .full_input + .clone(), + HistorySource::ExternalUnknown => return None, + }; + if matches!(self.source, HistorySource::Managed) { + full_input.extend_from_slice(original.input()); + } + full_input.extend_from_slice(output); + Some(ResponseReplaySnapshot { full_input }) + } + + pub(in crate::dispatch) fn conversation_id<'a>( + &'a self, + original: &'a CodexResponsesRequest, + ) -> Option<&'a str> { + self.managed_entry + .as_ref() + .map(|entry| entry.conversation_id.as_str()) + .or_else(|| non_empty(original.local_conversation_id.as_deref())) + } +} + +fn full_replay_request( + original: &CodexResponsesRequest, + snapshot: &ResponseReplaySnapshot, +) -> CodexResponsesRequest { + let mut request = original.clone(); + let mut input = snapshot.full_input.clone(); + input.extend_from_slice(original.input()); + strip_account_bound_encrypted_content(&mut input); + request.set_input(input); + request.set_previous_response_id(None); + request.turn_state = None; + request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal; + request +} + +fn strip_account_bound_encrypted_content(values: &mut [Value]) { + for value in values { + match value { + Value::Array(values) => strip_account_bound_encrypted_content(values), + Value::Object(object) => { + object.remove("encrypted_content"); + for value in object.values_mut() { + strip_account_bound_encrypted_content(std::slice::from_mut(value)); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {} + } + } +} + +fn non_empty(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} diff --git a/backend/src/dispatch/recovery/implicit_resume.rs b/backend/src/dispatch/recovery/implicit_resume.rs deleted file mode 100644 index 4dd1a0c1c..000000000 --- a/backend/src/dispatch/recovery/implicit_resume.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! Responses 隐式续接策略。 - -use std::collections::HashSet; - -use chrono::{Duration, Utc}; -use serde_json::Value; - -use crate::{ - dispatch::{ - affinity::{ - compute_variant_hash, ensure_prompt_cache_key, hash_instructions, - prepare_variant_identity, - }, - service::ResponseDispatchService, - }, - upstream::openai::protocol::responses::CodexResponsesRequest, -}; - -const IMPLICIT_RESUME_MAX_AGE_SECS: i64 = 55 * 60; - -/// 隐式续接前的可恢复请求状态。 -#[derive(Debug, Clone, PartialEq)] -pub struct ImplicitResumeSnapshot { - pub input: Vec, - pub previous_response_id: Option, - pub turn_state: Option, - pub use_websocket: bool, - pub force_http_sse: bool, -} - -impl ImplicitResumeSnapshot { - /// 捕获当前请求中隐式续接会改写的字段。 - pub fn capture(request: &CodexResponsesRequest) -> Self { - Self { - input: request.input().to_vec(), - previous_response_id: request.previous_response_id().map(ToString::to_string), - turn_state: request.turn_state.clone(), - use_websocket: request.use_websocket, - force_http_sse: request.force_http_sse, - } - } - - /// 恢复隐式续接前的请求字段。 - pub fn restore(self, request: &mut CodexResponsesRequest) { - request.set_input(self.input); - request.set_previous_response_id(self.previous_response_id); - request.turn_state = self.turn_state; - request.use_websocket = self.use_websocket; - request.force_http_sse = self.force_http_sse; - } -} - -impl ResponseDispatchService { - pub(in crate::dispatch) async fn prepare_response_session( - &self, - request: &mut CodexResponsesRequest, - ) -> Option { - prepare_variant_identity(request); - if let Some(previous_response_id) = request.previous_response_id().map(ToString::to_string) - { - if request.prompt_cache_key().is_none() { - let conversation_id = self - .session_affinity - .lookup_conversation_id(&previous_response_id, Utc::now()) - .await; - request.set_prompt_cache_key(conversation_id); - } - if request.turn_state.as_deref().is_none_or(str::is_empty) { - request.turn_state = self - .session_affinity - .lookup_turn_state(&previous_response_id, Utc::now()) - .await; - } - ensure_prompt_cache_key(request); - return None; - } - - ensure_prompt_cache_key(request); - self.apply_implicit_resume(request).await - } - - async fn apply_implicit_resume( - &self, - request: &mut CodexResponsesRequest, - ) -> Option { - let continuation_start = continuation_input_start(request.input()); - if continuation_start == 0 || continuation_start >= request.input().len() { - return None; - } - let conversation_id = request - .prompt_cache_key() - .map(str::trim) - .filter(|value| !value.is_empty())? - .to_string(); - let snapshot = ImplicitResumeSnapshot::capture(request); - let variant_hash = compute_variant_hash(request); - let now = Utc::now(); - let previous_response_id = self - .session_affinity - .lookup_latest_response_by_conversation( - &conversation_id, - Some(Duration::seconds(IMPLICIT_RESUME_MAX_AGE_SECS)), - Some(&variant_hash), - now, - ) - .await?; - let current_instructions_hash = hash_instructions(Some(request.instructions())); - if self - .session_affinity - .lookup_instructions_hash(&previous_response_id, now) - .await - .as_deref() - != Some(current_instructions_hash.as_str()) - { - return None; - } - let stored_function_call_ids = self - .session_affinity - .lookup_function_call_ids(&previous_response_id, now) - .await; - if !implicit_resume_allowed( - &request.input()[continuation_start..], - request.input(), - &stored_function_call_ids, - ) { - return None; - } - let account_id = self - .session_affinity - .lookup_account(&previous_response_id, now) - .await?; - let replay_items = self.reasoning_replay.lock().await.lookup( - &previous_response_id, - &account_id, - &conversation_id, - &variant_hash, - now, - ); - let continuation = request.input()[continuation_start..].to_vec(); - let mut input = replay_items; - input.extend(continuation); - - request.set_previous_response_id(Some(previous_response_id.clone())); - request.use_websocket = true; - request.force_http_sse = false; - request.set_input(input); - if let Some(turn_state) = self - .session_affinity - .lookup_turn_state(&previous_response_id, now) - .await - { - request.turn_state = Some(turn_state); - } - - Some(snapshot) - } - - pub(in crate::dispatch) async fn try_recover_implicit_resume( - &self, - request: &mut CodexResponsesRequest, - implicit_resume: &mut Option, - account_id: &str, - evict_reasoning_replay: bool, - ) -> bool { - let Some(snapshot) = implicit_resume.take() else { - return false; - }; - if evict_reasoning_replay { - self.evict_reasoning_replay(request, account_id).await; - } - if let Some(previous_response_id) = request.previous_response_id() { - self.session_affinity.forget(previous_response_id).await; - } - restore_full_request_without_history(request, snapshot); - true - } -} - -/// 返回续接输入在完整输入中的起始位置。 -pub fn continuation_input_start(input: &[Value]) -> usize { - let mut last_model_output_index = None; - for (index, item) in input.iter().enumerate() { - if item.get("role").is_some() { - if item.get("role").and_then(Value::as_str) == Some("assistant") { - last_model_output_index = Some(index); - } - continue; - } - if item.get("type").and_then(Value::as_str) == Some("function_call") { - last_model_output_index = Some(index); - } - } - last_model_output_index.map_or(0, |index| index.saturating_add(1)) -} - -/// 判断完整输入历史是否可以隐式续接到已记录响应。 -pub fn implicit_resume_allowed( - continuation_input: &[Value], - full_input: &[Value], - stored_function_call_ids: &[String], -) -> bool { - let required_call_ids = function_call_output_ids(continuation_input); - if required_call_ids.is_empty() { - return stored_function_call_ids.is_empty(); - } - - let inline_call_ids = inline_function_call_ids(full_input); - if required_call_ids - .iter() - .all(|call_id| inline_call_ids.contains(call_id)) - { - return false; - } - - let stored_call_ids = stored_function_call_ids.iter().collect::>(); - let required_call_ids = required_call_ids.iter().collect::>(); - required_call_ids - .iter() - .all(|call_id| stored_call_ids.contains(*call_id)) - && stored_call_ids - .iter() - .all(|call_id| required_call_ids.contains(*call_id)) -} - -fn strip_request_history(request: &mut CodexResponsesRequest) { - request.set_previous_response_id(None); - request.turn_state = None; -} - -pub(in crate::dispatch) fn restore_full_request_without_history( - request: &mut CodexResponsesRequest, - snapshot: ImplicitResumeSnapshot, -) { - snapshot.restore(request); - strip_request_history(request); -} - -fn function_call_output_ids(input: &[Value]) -> Vec { - input - .iter() - .filter(|item| item.get("role").is_none()) - .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call_output")) - .filter_map(|item| item.get("call_id").and_then(Value::as_str)) - .filter(|call_id| !call_id.trim().is_empty()) - .map(ToString::to_string) - .collect() -} - -fn inline_function_call_ids(input: &[Value]) -> HashSet { - input - .iter() - .filter(|item| item.get("role").is_none()) - .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) - .filter_map(|item| item.get("call_id").and_then(Value::as_str)) - .filter(|call_id| !call_id.trim().is_empty()) - .map(ToString::to_string) - .collect() -} diff --git a/backend/src/dispatch/recovery/mod.rs b/backend/src/dispatch/recovery/mod.rs index 0d9d76bc0..a71720469 100644 --- a/backend/src/dispatch/recovery/mod.rs +++ b/backend/src/dispatch/recovery/mod.rs @@ -3,5 +3,4 @@ pub(crate) mod auth; pub mod cloudflare; pub(crate) mod exhaustion; -pub mod implicit_resume; -pub mod reasoning_replay; +pub(crate) mod history; diff --git a/backend/src/dispatch/recovery/reasoning_replay.rs b/backend/src/dispatch/recovery/reasoning_replay.rs deleted file mode 100644 index c5bc25acd..000000000 --- a/backend/src/dispatch/recovery/reasoning_replay.rs +++ /dev/null @@ -1,274 +0,0 @@ -//! Responses reasoning replay 缓存策略。 - -use std::collections::{BTreeMap, HashSet}; - -use chrono::{DateTime, Duration, Utc}; -use serde_json::{json, Map, Value}; - -const MAX_ENTRIES: usize = 512; -const MAX_ENTRY_BYTES: usize = 256 * 1024; -const MAX_TOTAL_BYTES: usize = 4 * 1024 * 1024; - -/// 单条 reasoning replay 缓存记录。 -#[derive(Debug, Clone, PartialEq)] -struct ReasoningReplayEntry { - account_id: String, - conversation_id: String, - variant_hash: String, - items: Vec, - byte_size: usize, - created_at: DateTime, - sequence: u64, -} - -/// 纯内存 reasoning replay 缓存。 -#[derive(Debug, Clone)] -pub struct ReasoningReplayCache { - entries: BTreeMap, - total_bytes: usize, - next_sequence: u64, - ttl: Duration, -} - -impl ReasoningReplayCache { - pub fn new(ttl: Duration) -> Self { - Self { - entries: BTreeMap::new(), - total_bytes: 0, - next_sequence: 0, - ttl, - } - } - - pub fn record( - &mut self, - response_id: String, - account_id: &str, - conversation_id: &str, - variant_hash: &str, - items: &[Value], - now: DateTime, - ) -> usize { - let sanitized = sanitize_reasoning_replay_items(items); - self.cleanup_expired(now); - self.delete_entry(&response_id); - - if sanitized.is_empty() { - return 0; - } - let byte_size = estimate_items_bytes(&sanitized); - if byte_size > MAX_ENTRY_BYTES { - return 0; - } - - let sequence = self.next_sequence; - self.next_sequence = self.next_sequence.saturating_add(1); - self.total_bytes = self.total_bytes.saturating_add(byte_size); - self.entries.insert( - response_id, - ReasoningReplayEntry { - account_id: account_id.to_string(), - conversation_id: conversation_id.to_string(), - variant_hash: variant_hash.to_string(), - items: sanitized, - byte_size, - created_at: now, - sequence, - }, - ); - self.evict_overflow(); - self.entries - .values() - .max_by_key(|entry| entry.sequence) - .map_or(0, |entry| entry.items.len()) - } - - pub fn lookup( - &mut self, - response_id: &str, - account_id: &str, - conversation_id: &str, - variant_hash: &str, - now: DateTime, - ) -> Vec { - self.cleanup_expired(now); - let Some(entry) = self.entries.get(response_id) else { - return Vec::new(); - }; - if entry.account_id != account_id - || entry.conversation_id != conversation_id - || entry.variant_hash != variant_hash - { - return Vec::new(); - } - entry.items.clone() - } - - pub fn evict_by_identity( - &mut self, - account_id: &str, - conversation_id: &str, - variant_hash: &str, - now: DateTime, - ) -> usize { - self.cleanup_expired(now); - let response_ids = self - .entries - .iter() - .filter(|(_, entry)| { - entry.account_id == account_id - && entry.conversation_id == conversation_id - && entry.variant_hash == variant_hash - }) - .map(|(response_id, _)| response_id.clone()) - .collect::>(); - let evicted = response_ids.len(); - for response_id in response_ids { - self.delete_entry(&response_id); - } - evicted - } - - fn cleanup_expired(&mut self, now: DateTime) { - let ttl = self.ttl; - let response_ids = self - .entries - .iter() - .filter(|(_, entry)| entry_expired(entry.created_at, ttl, now)) - .map(|(response_id, _)| response_id.clone()) - .collect::>(); - for response_id in response_ids { - self.delete_entry(&response_id); - } - } - - fn evict_overflow(&mut self) { - while self.entries.len() > MAX_ENTRIES || self.total_bytes > MAX_TOTAL_BYTES { - let Some(response_id) = self.oldest_response_id() else { - return; - }; - self.delete_entry(&response_id); - } - } - - fn oldest_response_id(&self) -> Option { - self.entries - .iter() - .min_by_key(|(_, entry)| (entry.created_at, entry.sequence)) - .map(|(response_id, _)| response_id.clone()) - } - - fn delete_entry(&mut self, response_id: &str) { - let Some(entry) = self.entries.remove(response_id) else { - return; - }; - self.total_bytes = self.total_bytes.saturating_sub(entry.byte_size); - } -} - -fn sanitize_reasoning_replay_items(items: &[Value]) -> Vec { - let mut seen = HashSet::new(); - let mut sanitized = Vec::new(); - for item in items { - let Some(replay_item) = - sanitize_reasoning_item(item).or_else(|| sanitize_function_call(item)) - else { - continue; - }; - let Some(key) = replay_item_key(&replay_item) else { - continue; - }; - if !seen.insert(key) { - continue; - } - sanitized.push(replay_item); - } - sanitized -} - -fn sanitize_reasoning_item(value: &Value) -> Option { - let object = value.as_object()?; - if object.get("type").and_then(Value::as_str) != Some("reasoning") { - return None; - } - let id = non_empty_str(object.get("id")?)?; - let summary = sanitize_summary(object.get("summary")?)?; - let encrypted_content = non_empty_str(object.get("encrypted_content")?)?; - - let mut sanitized = Map::new(); - sanitized.insert("type".to_string(), json!("reasoning")); - sanitized.insert("id".to_string(), json!(id)); - if let Some(status) = object - .get("status") - .and_then(Value::as_str) - .filter(|status| matches!(*status, "in_progress" | "completed" | "incomplete")) - { - sanitized.insert("status".to_string(), json!(status)); - } - sanitized.insert("summary".to_string(), Value::Array(summary)); - sanitized.insert("encrypted_content".to_string(), json!(encrypted_content)); - Some(Value::Object(sanitized)) -} - -fn sanitize_function_call(value: &Value) -> Option { - let object = value.as_object()?; - if object.get("type").and_then(Value::as_str) != Some("function_call") { - return None; - } - let call_id = non_empty_str(object.get("call_id")?)?; - let name = non_empty_str(object.get("name")?)?; - let arguments = object.get("arguments")?.as_str()?; - - let mut sanitized = Map::new(); - sanitized.insert("type".to_string(), json!("function_call")); - if let Some(id) = object.get("id").and_then(non_empty_str) { - sanitized.insert("id".to_string(), json!(id)); - } - sanitized.insert("call_id".to_string(), json!(call_id)); - sanitized.insert("name".to_string(), json!(name)); - sanitized.insert("arguments".to_string(), json!(arguments)); - Some(Value::Object(sanitized)) -} - -fn sanitize_summary(value: &Value) -> Option> { - Some( - value - .as_array()? - .iter() - .filter_map(|part| { - let object = part.as_object()?; - if object.get("type").and_then(Value::as_str) != Some("summary_text") { - return None; - } - let text = object.get("text")?.as_str()?; - Some(json!({"type": "summary_text", "text": text})) - }) - .collect(), - ) -} - -fn replay_item_key(item: &Value) -> Option { - match item.get("type").and_then(Value::as_str)? { - "reasoning" => item - .get("id") - .and_then(Value::as_str) - .map(|id| format!("reasoning:{id}")), - "function_call" => item - .get("call_id") - .and_then(Value::as_str) - .map(|call_id| format!("function_call:{call_id}")), - _ => None, - } -} - -fn non_empty_str(value: &Value) -> Option<&str> { - value.as_str().filter(|value| !value.trim().is_empty()) -} - -fn estimate_items_bytes(items: &[Value]) -> usize { - serde_json::to_vec(items).map_or(usize::MAX, |bytes| bytes.len()) -} - -fn entry_expired(created_at: DateTime, ttl: Duration, now: DateTime) -> bool { - now.signed_duration_since(created_at) >= ttl -} diff --git a/backend/src/dispatch/service.rs b/backend/src/dispatch/service.rs index d4bce1e14..fd13f155f 100644 --- a/backend/src/dispatch/service.rs +++ b/backend/src/dispatch/service.rs @@ -10,20 +10,21 @@ use std::{pin::Pin, sync::Arc, time::Instant}; use bytes::Bytes; -use chrono::{Duration, Utc}; +use chrono::Utc; use futures::stream::Stream; use serde_json::{json, Value}; -use tokio::sync::Mutex; use crate::{ dispatch::{ - affinity::resolve::{evict_reasoning_replay, record_response_affinity}, - affinity::SessionAffinityService, + affinity::resolve::{prepare_variant_identity, record_response_affinity}, + affinity::{AccountIdentityService, SessionAffinityService}, + attempts::AccountAttemptLedger, errors::{ - backend_transport_name, is_history_recovery_upstream_error, + backend_transport_name, is_continuation_busy_error, is_history_recovery_upstream_error, is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error, - is_rate_limit_upstream_error, rate_limit_cooldown_until, upstream_error_body, - upstream_error_http_status, upstream_error_set_cookie_headers, + is_rate_limit_upstream_error, is_retryable_account_transport_error, + rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status, + upstream_error_set_cookie_headers, }, recovery::{ auth::{auth_failure_account_status, is_auth_upstream_error}, @@ -33,11 +34,11 @@ use crate::{ CloudflareRecovery, }, exhaustion::AccountExhaustionTracker, - reasoning_replay::ReasoningReplayCache, + history::HistoryRecoveryPlan, }, upstream_call::{ create_response_with_account_retrying_5xx, verify_acquired_quota_if_required, - QuotaVerificationContext, QuotaVerificationDecision, + AccountUpstreamContext, QuotaVerificationContext, QuotaVerificationDecision, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE, }, }, @@ -73,10 +74,9 @@ use super::{ }, stream::{ sse_failure::{ - auth_sse_failure_account_status, client_error_invalid_reasoning_replay, - is_auth_sse_failure, is_history_recovery_sse_failure, is_model_unsupported_sse_failure, - is_quota_exhausted_sse_failure, sse_failure_error_body, - sse_failure_invalid_reasoning_replay, stream_failure_http_status, + auth_sse_failure_account_status, is_auth_sse_failure, is_history_recovery_sse_failure, + is_model_unsupported_sse_failure, is_quota_exhausted_sse_failure, + sse_failure_error_body, stream_failure_http_status, }, trace::{ResponseDispatchAttempt, ResponseDispatchTrace}, }, @@ -89,9 +89,8 @@ pub struct ResponseDispatchService { pub(in crate::dispatch) models: Arc, pub(in crate::dispatch) codex: Arc, pub(in crate::dispatch) session_affinity: Arc, - pub(in crate::dispatch) reasoning_replay: Arc>, + pub(in crate::dispatch) account_identity: Arc, pub(in crate::dispatch) recorder: Arc, - pub(in crate::dispatch) installation_id: Option, pub(in crate::dispatch) cloudflare: CloudflareRecovery, } @@ -100,13 +99,11 @@ pub(crate) struct ResponseDispatchServiceParts { pub models: Arc, pub codex: Arc, pub session_affinity: Arc, + pub account_identity: Arc, pub recorder: Arc, - pub installation_id: Option, pub cloudflare: CloudflareRecovery, } -/// 默认 reasoning replay TTL 秒数。 -const DEFAULT_REASONING_REPLAY_TTL_SECS: i64 = 55 * 60; /// Responses live SSE 响应体流。 pub type ResponseBodyStream = Pin> + Send + 'static>>; @@ -114,6 +111,13 @@ pub type ResponseBodyStream = /// Responses live SSE 调度结果。 pub struct ResponseDispatchStream { pub body: ResponseBodyStream, + pub response_headers: Vec<(String, String)>, +} + +/// Responses 非流式调度结果。 +pub struct ResponseDispatchResponse { + pub body: Value, + pub response_headers: Vec<(String, String)>, } impl ResponseDispatchService { @@ -123,26 +127,16 @@ impl ResponseDispatchService { models: parts.models, codex: parts.codex, session_affinity: parts.session_affinity, - reasoning_replay: Arc::new(Mutex::new(ReasoningReplayCache::new(Duration::seconds( - DEFAULT_REASONING_REPLAY_TTL_SECS, - )))), + account_identity: parts.account_identity, recorder: parts.recorder, - installation_id: parts.installation_id, cloudflare: parts.cloudflare, } } - pub(in crate::dispatch) async fn evict_reasoning_replay( - &self, - request: &CodexResponsesRequest, - account_id: &str, - ) { - evict_reasoning_replay(&self.reasoning_replay, request, account_id).await; - } - async fn record_response_affinity( &self, - request: &CodexResponsesRequest, + history: &HistoryRecoveryPlan, + original_request: &CodexResponsesRequest, account_id: &str, body: &str, turn_state: Option, @@ -150,8 +144,8 @@ impl ResponseDispatchService { ) { record_response_affinity( &self.session_affinity, - &self.reasoning_replay, - request, + history, + original_request, account_id, body, turn_state, @@ -169,7 +163,7 @@ impl ResponseDispatchService { route: &str, mut request: CodexResponsesRequest, requested_model: &str, - ) -> Result { + ) -> Result { let started_at = Instant::now(); let catalog = self.models.catalog().await; let display_model = catalog.resolve_model_id(requested_model); @@ -177,40 +171,44 @@ impl ResponseDispatchService { let tuple_schema = request.tuple_schema.clone(); let image_generation_requested = request.expects_image_generation(); let now = Utc::now(); - let mut implicit_resume = self.prepare_response_session(&mut request).await; - let account_affinity = self.account_affinity_for_request(&request, now).await; + prepare_variant_identity(&mut request); + self.account_identity.prepare_local_identity(&mut request); + let mut history = HistoryRecoveryPlan::load(&self.session_affinity, &request).await; + let preferred_account_id = self + .preferred_account_id_for_request(&request, &history, now) + .await; let mut acquire_request = AccountAcquireRequest::new(request.model(), now); - if let Some(preferred_account_id) = account_affinity.preferred_account_id() { + if let Some(preferred_account_id) = preferred_account_id { acquire_request = acquire_request.with_preferred_account_id(preferred_account_id); } - let mut excluded_account_ids = Vec::new(); + let mut candidates = + AccountAttemptLedger::freeze(&self.account_pool, &acquire_request).await; let mut exhausted_accounts = AccountExhaustionTracker::default(); let mut next_required_account_id: Option = None; - let mut empty_response_retries = 0u8; - let mut quota_verify_attempts = 0usize; let mut trace = ResponseDispatchTrace::default(); - const MAX_EMPTY_RESPONSE_RETRIES: u8 = 2; let (account, response, collected_response, attempt): ( Account, CodexBackendResponse, CollectedResponse, ResponseDispatchAttempt, ) = loop { - let mut attempt_acquire_request = acquire_request - .clone() - .with_exclude_account_ids(excluded_account_ids.iter().cloned()); - if let Some(account_id) = next_required_account_id.take() { - attempt_acquire_request = - attempt_acquire_request.with_required_account_id(account_id); - } - let allow_quota_verify_account_retry = - attempt_acquire_request.required_account_id.is_none(); - attempt_acquire_request.now = Utc::now(); - let Some(acquired) = self - .account_pool - .acquire_with(&attempt_acquire_request) - .await - else { + let acquired = if let Some(account_id) = next_required_account_id.take() { + self.account_pool + .acquire_with( + &AccountAcquireRequest::new(request.model(), Utc::now()) + .with_required_account_id(account_id), + ) + .await + } else { + candidates.acquire_next(&self.account_pool).await + }; + let Some(acquired) = acquired else { + tracing::info!( + candidate_count = candidates.candidate_count(), + attempted = candidates.attempted_count(), + state_excluded = candidates.state_excluded_count(), + "Responses account candidate ledger exhausted" + ); let error = exhausted_accounts .last_exhausted() .map(ResponseDispatchError::from_exhausted_account) @@ -235,18 +233,14 @@ impl ResponseDispatchService { return Err(error); }; let acquired_account_id = acquired.account.id.clone(); - // 配额验证 let acquired = match verify_acquired_quota_if_required( QuotaVerificationContext { account_pool: self.account_pool.as_ref(), codex: self.codex.as_ref(), cloudflare: &self.cloudflare, - installation_id: self.installation_id.as_deref(), + account_identity: &self.account_identity, request_id, - excluded_account_ids: &mut excluded_account_ids, - verify_attempts: &mut quota_verify_attempts, - allow_retry_with_another_account: allow_quota_verify_account_retry, }, acquired, ) @@ -254,65 +248,35 @@ impl ResponseDispatchService { { QuotaVerificationDecision::Ready(acquired) => *acquired, QuotaVerificationDecision::RetryWithAnotherAccount => { - exhausted_accounts - .record_rate_limited(None, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE); - continue; - } - QuotaVerificationDecision::MaxAttemptsReached - | QuotaVerificationDecision::RequiredAccountUnavailable => { exhausted_accounts.record_rate_limited( Some(&acquired_account_id), QUOTA_VERIFY_LIMIT_REACHED_MESSAGE, ); - let error = exhausted_accounts - .last_exhausted() - .map(ResponseDispatchError::from_exhausted_account) - .unwrap_or(ResponseDispatchError::NoActiveAccount); - self.record_response_dispatch_error( - request_id, - route, - requested_model, - started_at, - ResponseDispatchErrorDetails { - client_api_key_id: request.client_api_key_id.as_deref(), - account_id: Some(&acquired_account_id), - stream: false, - compact: false, - transport: Some(backend_transport_name( - backend_transport_for_response_request(&request), - )), - }, - &error, - ) - .await; - return Err(error); + continue; } }; - self.apply_cascading_ban_defense( - &mut request, - &mut implicit_resume, - account_affinity.preferred_account_id(), - &acquired.account.id, - ) - .await; - Self::strip_history_if_account_changed( - &mut request, - &mut implicit_resume, - &account_affinity, - &acquired.account.id, - ); + let Some(attempt_request) = history.prepare_attempt(&request, &acquired.account.id) + else { + acquired.release_without_usage().await; + return Err(ResponseDispatchError::HistoryUnavailable { + upstream_error: "previous response history cannot be sent to another account" + .to_string(), + }); + }; self.account_pool.wait_for_request_interval(&acquired).await; let account = acquired.account.clone(); let release_account_id = account.id.clone(); let attempt = trace.start_attempt(&release_account_id); let response_result = create_response_with_account_retrying_5xx( - &self.codex, - self.installation_id.as_deref(), - &self.cloudflare, - &request, - request_id, - &account, + AccountUpstreamContext { + codex: &self.codex, + account_identity: &self.account_identity, + cloudflare: &self.cloudflare, + request_id, + account: &account, + }, + &attempt_request, started_at, ) .await; @@ -368,54 +332,31 @@ impl ResponseDispatchService { image_generation_requested, ) .await; - empty_response_retries += 1; - if empty_response_retries <= MAX_EMPTY_RESPONSE_RETRIES { - continue; + exhausted_accounts.record_upstream_unavailable( + Some(&release_account_id), + "upstream response did not include visible output", + ); + if !history.can_failover() { + break (account, response, collected_response, attempt); } + continue; } if let CollectedResponse::Failed(failure) = &collected_response { if is_history_recovery_sse_failure(failure) - && self - .try_recover_implicit_resume( - &mut request, - &mut implicit_resume, - &release_account_id, - sse_failure_invalid_reasoning_replay(failure), - ) - .await + && history.recover_managed_history(&release_account_id) { next_required_account_id = Some(release_account_id); continue; } if is_model_unsupported_sse_failure(failure) { let upstream_error = sse_failure_error_body(failure); - if let Some(exhausted) = exhausted_accounts - .model_unsupported_retry_exhausted(upstream_error.clone()) - { - let error = - ResponseDispatchError::from_exhausted_account(exhausted); - self.record_response_dispatch_error( - request_id, - route, - requested_model, - started_at, - ResponseDispatchErrorDetails { - client_api_key_id: request.client_api_key_id.as_deref(), - account_id: Some(&release_account_id), - stream: false, - compact: false, - transport: Some(backend_transport_name(response.transport)), - }, - &error, - ) - .await; - return Err(error); - } exhausted_accounts.record_model_unsupported( Some(&release_account_id), upstream_error, ); - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + break (account, response, collected_response, attempt); + } continue; } if is_quota_exhausted_sse_failure(failure) { @@ -426,7 +367,9 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::QuotaExhausted) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + break (account, response, collected_response, attempt); + } continue; } if is_auth_sse_failure(failure) { @@ -441,7 +384,9 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, account_status) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + break (account, response, collected_response, attempt); + } continue; } } @@ -456,7 +401,9 @@ impl ResponseDispatchService { self.account_pool .mark_quota_limited_until(&release_account_id, cooldown_until) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) if is_quota_exhausted_upstream_error(&error) => { exhausted_accounts.record_quota_exhausted( @@ -466,7 +413,9 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::QuotaExhausted) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) if is_auth_upstream_error(&error) => { let upstream_error = upstream_error_body(&error); @@ -480,7 +429,9 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, account_status) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) if is_cloudflare_challenge_upstream_error(&error) => { exhausted_accounts.record_cloudflare_challenge( @@ -490,7 +441,9 @@ impl ResponseDispatchService { self.cloudflare .apply_challenge(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) if is_cloudflare_path_block_upstream_error(&error) => { exhausted_accounts.record_cloudflare_path_blocked( @@ -500,36 +453,17 @@ impl ResponseDispatchService { self.cloudflare .apply_path_block(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) if is_model_unsupported_upstream_error(&error) => { let upstream_error = upstream_error_body(&error); - if let Some(exhausted) = - exhausted_accounts.model_unsupported_retry_exhausted(upstream_error.clone()) - { - let error = ResponseDispatchError::from_exhausted_account(exhausted); - self.record_response_dispatch_error( - request_id, - route, - requested_model, - started_at, - ResponseDispatchErrorDetails { - client_api_key_id: request.client_api_key_id.as_deref(), - account_id: Some(&release_account_id), - stream: false, - compact: false, - transport: Some(backend_transport_name( - backend_transport_for_response_request(&request), - )), - }, - &error, - ) - .await; - return Err(error); - } exhausted_accounts .record_model_unsupported(Some(&release_account_id), upstream_error); - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) if is_banned_upstream_error(&error) => { exhausted_accounts.record_auth_failure( @@ -541,20 +475,16 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::Banned) .await; - excluded_account_ids.push(release_account_id); + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } } Err(error) => { + if is_continuation_busy_error(&error) { + return Err(ResponseDispatchError::ContinuationBusy); + } let history_unavailable = is_history_recovery_upstream_error(&error); - if history_unavailable - && self - .try_recover_implicit_resume( - &mut request, - &mut implicit_resume, - &release_account_id, - client_error_invalid_reasoning_replay(&error), - ) - .await - { + if history_unavailable && history.recover_managed_history(&release_account_id) { next_required_account_id = Some(release_account_id); continue; } @@ -567,14 +497,27 @@ impl ResponseDispatchService { model: requested_model, started_at, stream: false, - transport: backend_transport_for_response_request(&request), - request: &request, + transport: backend_transport_for_response_request(&attempt_request), + request: &attempt_request, error: &error, trace: &trace, attempt: Some(&attempt), }) .await; + if is_retryable_account_transport_error(&error) { + if !history.can_failover() { + return Err(ResponseDispatchError::Upstream(error)); + } + exhausted_accounts.record_upstream_unavailable( + Some(&release_account_id), + upstream_error_body(&error), + ); + continue; + } if history_unavailable { + if history.is_external_unknown() { + return Err(ResponseDispatchError::Upstream(error)); + } return Err(ResponseDispatchError::HistoryUnavailable { upstream_error: upstream_error_body(&error), }); @@ -584,8 +527,9 @@ impl ResponseDispatchService { } }; + let completed = matches!(&collected_response, CollectedResponse::Completed(_)); match collected_response { - CollectedResponse::Completed(body) => { + CollectedResponse::Completed(body) | CollectedResponse::Incomplete(body) => { let response_id = body.get("id").and_then(Value::as_str); self.cloudflare.reset_account_recovery(&account.id).await; if let Some(usage) = response.usage { @@ -598,20 +542,33 @@ impl ResponseDispatchService { ) .await; } - self.record_response_affinity( - &request, - &account.id, - &response.body, - response.turn_state.clone(), - response.usage, - ) - .await; + if completed { + self.record_response_affinity( + &history, + &request, + &account.id, + &response.body, + response.turn_state.clone(), + response.usage, + ) + .await; + } + let effective_model = response + .response_metadata + .effective_model + .as_deref() + .unwrap_or(&display_model); let mut metadata = json!({ "responseId": response_id, "stream": false, + "completed": completed, + "incomplete": !completed, "transport": backend_transport_name(response.transport), "firstTokenMs": response.first_token_ms, "usage": response.usage, + "effectiveModel": effective_model, + "modelsEtag": response.response_metadata.models_etag.as_deref(), + "reasoningIncluded": response.response_metadata.reasoning_included, }); insert_response_status_metadata( &mut metadata, @@ -628,7 +585,7 @@ impl ResponseDispatchService { client_api_key_id: request.client_api_key_id.as_deref(), account_id: &account.id, route, - model: &display_model, + model: effective_model, requested_model: Some(requested_model), client_ip: request.client_ip.as_deref(), client_user_agent: request.client_user_agent.as_deref(), @@ -636,21 +593,24 @@ impl ResponseDispatchService { service_tier: request.service_tier(), started_at, status_code: 200, - message: "v1 responses completed", + message: if completed { + "v1 responses completed" + } else { + "v1 responses incomplete" + }, metadata, rate_limit_headers: &response.rate_limit_headers, }) .await; - Ok(body) + self.models + .observe_models_etag(response.response_metadata.models_etag.as_deref()); + Ok(ResponseDispatchResponse { + body, + response_headers: response.response_metadata.client_headers, + }) } CollectedResponse::Failed(failure) => { - let error = if is_history_recovery_sse_failure(&failure) { - ResponseDispatchError::HistoryUnavailable { - upstream_error: sse_failure_error_body(&failure), - } - } else { - ResponseDispatchError::Failed(failure) - }; + let error = ResponseDispatchError::Failed(failure); self.record_response_dispatch_error( request_id, route, diff --git a/backend/src/dispatch/stream/lifecycle.rs b/backend/src/dispatch/stream/lifecycle.rs index 99ac373a9..61b9fa5e7 100644 --- a/backend/src/dispatch/stream/lifecycle.rs +++ b/backend/src/dispatch/stream/lifecycle.rs @@ -4,11 +4,14 @@ use chrono::Utc; use crate::{ dispatch::{ + affinity::prepare_variant_identity, + attempts::AccountAttemptLedger, errors::{ - backend_transport_name, is_history_recovery_upstream_error, + backend_transport_name, is_continuation_busy_error, is_history_recovery_upstream_error, is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error, - is_rate_limit_upstream_error, rate_limit_cooldown_until, upstream_error_body, - upstream_error_http_status, upstream_error_set_cookie_headers, ResponseDispatchError, + is_rate_limit_upstream_error, is_retryable_account_transport_error, + rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status, + upstream_error_set_cookie_headers, ResponseDispatchError, }, recording::{ record_prefetched_response_stream_failure_event, record_response_upstream_error_event, @@ -22,23 +25,22 @@ use crate::{ is_cloudflare_challenge_upstream_error, is_cloudflare_path_block_upstream_error, }, exhaustion::AccountExhaustionTracker, + history::HistoryRecoveryPlan, }, service::{ResponseDispatchService, ResponseDispatchStream}, stream::{ live::{spawn_live_response_stream, LiveResponseStreamContext}, prefetch::prefetch_first_sse_chunk, sse_failure::{ - auth_sse_failure_account_status, client_error_invalid_reasoning_replay, - first_sse_failure, is_auth_sse_failure, is_history_recovery_sse_failure, - is_model_unsupported_sse_failure, is_quota_exhausted_sse_failure, - sse_failure_error_body, sse_failure_invalid_reasoning_replay, - stream_failure_http_status, + auth_sse_failure_account_status, first_sse_failure, is_auth_sse_failure, + is_history_recovery_sse_failure, is_model_unsupported_sse_failure, + is_quota_exhausted_sse_failure, sse_failure_error_body, stream_failure_http_status, }, trace::ResponseDispatchTrace, }, upstream_call::{ create_response_stream_with_account_retrying_5xx, verify_acquired_quota_if_required, - QuotaVerificationContext, QuotaVerificationDecision, + AccountUpstreamContext, QuotaVerificationContext, QuotaVerificationDecision, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE, }, }, @@ -65,16 +67,20 @@ impl ResponseDispatchService { request.set_stream(true); let tuple_schema = request.tuple_schema.clone(); let now = Utc::now(); - let mut implicit_resume = self.prepare_response_session(&mut request).await; - let account_affinity = self.account_affinity_for_request(&request, now).await; + prepare_variant_identity(&mut request); + self.account_identity.prepare_local_identity(&mut request); + let mut history = HistoryRecoveryPlan::load(&self.session_affinity, &request).await; + let preferred_account_id = self + .preferred_account_id_for_request(&request, &history, now) + .await; let mut acquire_request = AccountAcquireRequest::new(request.model(), now); - if let Some(preferred_account_id) = account_affinity.preferred_account_id() { + if let Some(preferred_account_id) = preferred_account_id { acquire_request = acquire_request.with_preferred_account_id(preferred_account_id); } - let mut excluded_account_ids = Vec::new(); + let mut candidates = + AccountAttemptLedger::freeze(&self.account_pool, &acquire_request).await; let mut exhausted_accounts = AccountExhaustionTracker::default(); let mut next_required_account_id: Option = None; - let mut quota_verify_attempts = 0usize; let mut trace = ResponseDispatchTrace::default(); macro_rules! return_stream_dispatch_error { ($error:expr) => {{ @@ -119,21 +125,23 @@ impl ResponseDispatchService { }}; } loop { - let mut attempt_acquire_request = acquire_request - .clone() - .with_exclude_account_ids(excluded_account_ids.iter().cloned()); - if let Some(account_id) = next_required_account_id.take() { - attempt_acquire_request = - attempt_acquire_request.with_required_account_id(account_id); - } - let allow_quota_verify_account_retry = - attempt_acquire_request.required_account_id.is_none(); - attempt_acquire_request.now = Utc::now(); - let Some(acquired) = self - .account_pool - .acquire_with(&attempt_acquire_request) - .await - else { + let acquired = if let Some(account_id) = next_required_account_id.take() { + self.account_pool + .acquire_with( + &AccountAcquireRequest::new(request.model(), Utc::now()) + .with_required_account_id(account_id), + ) + .await + } else { + candidates.acquire_next(&self.account_pool).await + }; + let Some(acquired) = acquired else { + tracing::info!( + candidate_count = candidates.candidate_count(), + attempted = candidates.attempted_count(), + state_excluded = candidates.state_excluded_count(), + "Responses stream account candidate ledger exhausted" + ); let error = exhausted_accounts .last_exhausted() .map(ResponseDispatchError::from_exhausted_account) @@ -146,11 +154,8 @@ impl ResponseDispatchService { account_pool: self.account_pool.as_ref(), codex: self.codex.as_ref(), cloudflare: &self.cloudflare, - installation_id: self.installation_id.as_deref(), + account_identity: &self.account_identity, request_id, - excluded_account_ids: &mut excluded_account_ids, - verify_attempts: &mut quota_verify_attempts, - allow_retry_with_another_account: allow_quota_verify_account_retry, }, acquired, ) @@ -158,55 +163,36 @@ impl ResponseDispatchService { { QuotaVerificationDecision::Ready(acquired) => *acquired, QuotaVerificationDecision::RetryWithAnotherAccount => { - exhausted_accounts - .record_rate_limited(None, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE); - continue; - } - QuotaVerificationDecision::MaxAttemptsReached - | QuotaVerificationDecision::RequiredAccountUnavailable => { exhausted_accounts.record_rate_limited( Some(&acquired_account_id), QUOTA_VERIFY_LIMIT_REACHED_MESSAGE, ); - let error = exhausted_accounts - .last_exhausted() - .map(ResponseDispatchError::from_exhausted_account) - .unwrap_or(ResponseDispatchError::NoActiveAccount); - return_stream_dispatch_error!( - error, - account_id: Some(&acquired_account_id), - transport: Some(backend_transport_name(backend_transport_for_response_request( - &request - ))) - ); + continue; } }; - self.apply_cascading_ban_defense( - &mut request, - &mut implicit_resume, - account_affinity.preferred_account_id(), - &acquired.account.id, - ) - .await; - Self::strip_history_if_account_changed( - &mut request, - &mut implicit_resume, - &account_affinity, - &acquired.account.id, - ); + let Some(attempt_request) = history.prepare_attempt(&request, &acquired.account.id) + else { + acquired.release_without_usage().await; + return_stream_dispatch_error!(ResponseDispatchError::HistoryUnavailable { + upstream_error: "previous response history cannot be sent to another account" + .to_string(), + }); + }; self.account_pool.wait_for_request_interval(&acquired).await; let account = acquired.account.clone(); let release_account_id = account.id.clone(); let attempt = trace.start_attempt(&release_account_id); let response_result = create_response_stream_with_account_retrying_5xx( - &self.codex, - self.installation_id.as_deref(), - &self.cloudflare, - &request, - request_id, - &account, + AccountUpstreamContext { + codex: &self.codex, + account_identity: &self.account_identity, + cloudflare: &self.cloudflare, + request_id, + account: &account, + }, + &attempt_request, ) .await; if let Err(error) = &response_result { @@ -228,6 +214,9 @@ impl ResponseDispatchService { let websocket_pool_decision = response.websocket_pool_decision; let turn_state = response.turn_state; let diagnostics = response.diagnostics; + let response_metadata = response.response_metadata; + self.models + .observe_models_etag(response_metadata.models_etag.as_deref()); self.cloudflare .capture_set_cookie_headers(&release_account_id, &set_cookie_headers) .await; @@ -248,7 +237,13 @@ impl ResponseDispatchService { self.account_pool .mark_quota_limited_until(&release_account_id, cooldown_until) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(ResponseDispatchError::Upstream(error)) @@ -262,7 +257,13 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::QuotaExhausted) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(ResponseDispatchError::Upstream(error)) @@ -280,7 +281,13 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, account_status) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(ResponseDispatchError::Upstream(error)) @@ -294,7 +301,13 @@ impl ResponseDispatchService { self.cloudflare .apply_challenge(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(ResponseDispatchError::Upstream(error)) @@ -308,7 +321,13 @@ impl ResponseDispatchService { self.cloudflare .apply_path_block(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(ResponseDispatchError::Upstream(error)) @@ -316,20 +335,17 @@ impl ResponseDispatchService { { acquired.complete().await; let upstream_error = upstream_error_body(&error); - if let Some(exhausted) = exhausted_accounts - .model_unsupported_retry_exhausted(upstream_error.clone()) - { - return_stream_dispatch_error!( - ResponseDispatchError::from_exhausted_account(exhausted), - account_id: Some(&release_account_id), - transport: Some(backend_transport_name(transport)) - ); - } exhausted_accounts.record_model_unsupported( Some(&release_account_id), upstream_error, ); - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(ResponseDispatchError::Upstream(error)) @@ -345,23 +361,29 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::Banned) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } Err(error) => { acquired.complete().await; if let ResponseDispatchError::Upstream(upstream_error) = &error { + if is_continuation_busy_error(upstream_error) { + return_stream_dispatch_error!( + ResponseDispatchError::ContinuationBusy, + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } let history_unavailable = is_history_recovery_upstream_error(upstream_error); if history_unavailable - && self - .try_recover_implicit_resume( - &mut request, - &mut implicit_resume, - &release_account_id, - client_error_invalid_reasoning_replay(upstream_error), - ) - .await + && history.recover_managed_history(&release_account_id) { next_required_account_id = Some(release_account_id); continue; @@ -377,14 +399,27 @@ impl ResponseDispatchService { started_at, stream: true, transport, - request: &request, + request: &attempt_request, error: upstream_error, trace: &trace, attempt: Some(&attempt), }, ) .await; + if is_retryable_account_transport_error(upstream_error) { + exhausted_accounts.record_upstream_unavailable( + Some(&release_account_id), + upstream_error_body(upstream_error), + ); + if history.is_external_unknown() { + return Err(error); + } + continue; + } if history_unavailable { + if history.is_external_unknown() { + return Err(error); + } return Err(ResponseDispatchError::HistoryUnavailable { upstream_error: upstream_error_body(upstream_error), }); @@ -411,14 +446,7 @@ impl ResponseDispatchService { }; if let Some(failure) = first_failure { if is_history_recovery_sse_failure(&failure) - && self - .try_recover_implicit_resume( - &mut request, - &mut implicit_resume, - &release_account_id, - sse_failure_invalid_reasoning_replay(&failure), - ) - .await + && history.recover_managed_history(&release_account_id) { acquired.complete().await; next_required_account_id = Some(release_account_id); @@ -426,22 +454,18 @@ impl ResponseDispatchService { } if is_model_unsupported_sse_failure(&failure) { let upstream_error = sse_failure_error_body(&failure); - if let Some(exhausted) = exhausted_accounts - .model_unsupported_retry_exhausted(upstream_error.clone()) - { - acquired.complete().await; - return_stream_dispatch_error!( - ResponseDispatchError::from_exhausted_account(exhausted), - account_id: Some(&release_account_id), - transport: Some(backend_transport_name(transport)) - ); - } exhausted_accounts.record_model_unsupported( Some(&release_account_id), upstream_error, ); - excluded_account_ids.push(release_account_id); acquired.complete().await; + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Failed(failure), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } if is_quota_exhausted_sse_failure(&failure) { @@ -452,8 +476,14 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::QuotaExhausted) .await; - excluded_account_ids.push(release_account_id); acquired.complete().await; + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Failed(failure), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } if is_auth_sse_failure(&failure) { @@ -468,8 +498,14 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, account_status) .await; - excluded_account_ids.push(release_account_id); acquired.complete().await; + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Failed(failure), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name(transport)) + ); + } continue; } acquired.complete().await; @@ -483,7 +519,7 @@ impl ResponseDispatchService { requested_model, started_at, transport, - request: &request, + request: &attempt_request, failure: &failure, diagnostics: &diagnostics, rate_limit_headers: &rate_limit_headers, @@ -493,7 +529,9 @@ impl ResponseDispatchService { }, ) .await; - let error = if is_history_recovery_sse_failure(&failure) { + let error = if is_history_recovery_sse_failure(&failure) + && !history.is_external_unknown() + { ResponseDispatchError::HistoryUnavailable { upstream_error: sse_failure_error_body(&failure), } @@ -507,7 +545,7 @@ impl ResponseDispatchService { account_pool: Arc::clone(&self.account_pool), account_lease: acquired, session_affinity: Arc::clone(&self.session_affinity), - reasoning_replay: Arc::clone(&self.reasoning_replay), + history, recorder: Arc::clone(&self.recorder), cloudflare: self.cloudflare.clone(), account_id: account.id, @@ -527,6 +565,7 @@ impl ResponseDispatchService { websocket_pool_decision, turn_state, diagnostics, + response_metadata, attempt: attempt.clone(), attempts: trace.attempts().to_vec(), started_at, @@ -543,7 +582,15 @@ impl ResponseDispatchService { self.account_pool .mark_quota_limited_until(&release_account_id, cooldown_until) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) if is_quota_exhausted_upstream_error(&error) => { acquired.complete().await; @@ -554,7 +601,15 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::QuotaExhausted) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) if is_auth_upstream_error(&error) => { acquired.complete().await; @@ -569,7 +624,15 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, account_status) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) if is_cloudflare_challenge_upstream_error(&error) => { acquired.complete().await; @@ -580,7 +643,15 @@ impl ResponseDispatchService { self.cloudflare .apply_challenge(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) if is_cloudflare_path_block_upstream_error(&error) => { acquired.complete().await; @@ -591,25 +662,30 @@ impl ResponseDispatchService { self.cloudflare .apply_path_block(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) if is_model_unsupported_upstream_error(&error) => { acquired.complete().await; let upstream_error = upstream_error_body(&error); - if let Some(exhausted) = - exhausted_accounts.model_unsupported_retry_exhausted(upstream_error.clone()) - { - return_stream_dispatch_error!( - ResponseDispatchError::from_exhausted_account(exhausted), - account_id: Some(&release_account_id), - transport: Some(backend_transport_name(backend_transport_for_response_request( - &request - ))) - ); - } exhausted_accounts .record_model_unsupported(Some(&release_account_id), upstream_error); - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) if is_banned_upstream_error(&error) => { acquired.complete().await; @@ -622,21 +698,29 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::Banned) .await; - excluded_account_ids.push(release_account_id); + if history.is_external_unknown() { + return_stream_dispatch_error!( + ResponseDispatchError::Upstream(error), + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } } Err(error) => { acquired.complete().await; + if is_continuation_busy_error(&error) { + return_stream_dispatch_error!( + ResponseDispatchError::ContinuationBusy, + account_id: Some(&release_account_id), + transport: Some(backend_transport_name( + backend_transport_for_response_request(&attempt_request) + )) + ); + } let history_unavailable = is_history_recovery_upstream_error(&error); - if history_unavailable - && self - .try_recover_implicit_resume( - &mut request, - &mut implicit_resume, - &release_account_id, - client_error_invalid_reasoning_replay(&error), - ) - .await - { + if history_unavailable && history.recover_managed_history(&release_account_id) { next_required_account_id = Some(release_account_id); continue; } @@ -649,14 +733,27 @@ impl ResponseDispatchService { model: requested_model, started_at, stream: true, - transport: backend_transport_for_response_request(&request), - request: &request, + transport: backend_transport_for_response_request(&attempt_request), + request: &attempt_request, error: &error, trace: &trace, attempt: Some(&attempt), }) .await; + if is_retryable_account_transport_error(&error) { + exhausted_accounts.record_upstream_unavailable( + Some(&release_account_id), + upstream_error_body(&error), + ); + if history.is_external_unknown() { + return Err(ResponseDispatchError::Upstream(error)); + } + continue; + } if history_unavailable { + if history.is_external_unknown() { + return Err(ResponseDispatchError::Upstream(error)); + } return Err(ResponseDispatchError::HistoryUnavailable { upstream_error: upstream_error_body(&error), }); diff --git a/backend/src/dispatch/stream/live.rs b/backend/src/dispatch/stream/live.rs index 91d35b108..87f4d5f7d 100644 --- a/backend/src/dispatch/stream/live.rs +++ b/backend/src/dispatch/stream/live.rs @@ -8,20 +8,17 @@ use std::{ use bytes::Bytes; use futures::{stream::Stream, StreamExt}; use serde_json::Value; -use tokio::sync::{mpsc, oneshot, Mutex}; +use tokio::sync::{mpsc, oneshot}; use crate::{ dispatch::{ - affinity::{ - resolve::{evict_reasoning_replay, record_response_affinity}, - SessionAffinityService, - }, + affinity::{resolve::record_response_affinity, SessionAffinityService}, errors::{backend_transport_name, ResponseDispatchStreamError}, recording::{ insert_first_token_ms, live_response_rate_limit_headers, live_response_turn_state, record_live_response_stream_event, }, - recovery::{cloudflare::CloudflareRecovery, reasoning_replay::ReasoningReplayCache}, + recovery::{cloudflare::CloudflareRecovery, history::HistoryRecoveryPlan}, service::ResponseDispatchStream, }, fleet::pool::{AccountLease, AccountPoolService}, @@ -42,16 +39,17 @@ use crate::{ }, transport::{ CodexBackendSseStream, CodexBackendTransport, CodexRateLimitHeaderUpdates, - CodexTurnStateUpdate, CodexUpstreamDiagnostics, WebSocketPoolDecision, + CodexResponseMetadata, CodexTurnStateUpdate, CodexUpstreamDiagnostics, + WebSocketPoolDecision, }, }, }; use super::{ sse_failure::{ - sse_failure_invalid_reasoning_replay, status_code_for_stream_failure, - stream_failure_metadata, stream_failure_source, synthetic_stream_disconnected_detail, - STREAM_DISCONNECTED_CODE, STREAM_DISCONNECTED_MESSAGE, + status_code_for_stream_failure, stream_failure_metadata, stream_failure_source, + synthetic_stream_disconnected_detail, STREAM_DISCONNECTED_CODE, + STREAM_DISCONNECTED_MESSAGE, }, trace::ResponseDispatchAttempt, }; @@ -82,6 +80,7 @@ pub(in crate::dispatch) fn spawn_live_response_stream( prefetched: Bytes, mut body: CodexBackendSseStream, ) -> ResponseDispatchStream { + let response_headers = context.response_metadata.client_headers.clone(); let (sender, receiver) = mpsc::channel(8); let (cancel_sender, mut cancel_receiver) = oneshot::channel(); tokio::spawn(async move { @@ -183,6 +182,7 @@ pub(in crate::dispatch) fn spawn_live_response_stream( receiver, cancel: Some(cancel_sender), }), + response_headers, } } @@ -342,7 +342,7 @@ pub(in crate::dispatch) struct LiveResponseStreamContext { pub(in crate::dispatch) account_pool: Arc, pub(in crate::dispatch) account_lease: AccountLease, pub(in crate::dispatch) session_affinity: Arc, - pub(in crate::dispatch) reasoning_replay: Arc>, + pub(in crate::dispatch) history: HistoryRecoveryPlan, pub(in crate::dispatch) recorder: Arc, pub(in crate::dispatch) cloudflare: CloudflareRecovery, pub(in crate::dispatch) account_id: String, @@ -362,6 +362,7 @@ pub(in crate::dispatch) struct LiveResponseStreamContext { pub(in crate::dispatch) websocket_pool_decision: Option, pub(in crate::dispatch) turn_state: Option, pub(in crate::dispatch) diagnostics: CodexUpstreamDiagnostics, + pub(in crate::dispatch) response_metadata: CodexResponseMetadata, pub(in crate::dispatch) attempt: ResponseDispatchAttempt, pub(in crate::dispatch) attempts: Vec, pub(in crate::dispatch) started_at: Instant, @@ -428,31 +429,40 @@ pub(in crate::dispatch) async fn finalize_live_response_stream( } }; - match response_from_codex_sse(&body, context.tuple_schema.as_ref()) { - Ok(CollectedResponse::Completed(completed)) => { + let collected_response = response_from_codex_sse(&body, context.tuple_schema.as_ref()); + let completed_terminal = matches!(&collected_response, Ok(CollectedResponse::Completed(_))); + match collected_response { + Ok(CollectedResponse::Completed(terminal) | CollectedResponse::Incomplete(terminal)) => { context .cloudflare .reset_account_recovery(&context.account_id) .await; - let response_id = completed.get("id").and_then(Value::as_str); - record_response_affinity( - &context.session_affinity, - &context.reasoning_replay, - &context.request, - &context.account_id, - &body, - turn_state, - usage, - ) - .await; + let response_id = terminal.get("id").and_then(Value::as_str); + if completed_terminal { + record_response_affinity( + &context.session_affinity, + &context.history, + &context.request, + &context.account_id, + &body, + turn_state, + usage, + ) + .await; + } record_live_response_stream_event( &context, 200, UsageRecordLevel::Info, - "v1 responses stream completed", + if completed_terminal { + "v1 responses stream completed" + } else { + "v1 responses stream incomplete" + }, serde_json::json!({ "stream": true, - "completed": true, + "completed": completed_terminal, + "incomplete": !completed_terminal, "responseId": response_id, "firstTokenMs": first_token_ms, "usage": usage, @@ -463,14 +473,6 @@ pub(in crate::dispatch) async fn finalize_live_response_stream( .await; } Ok(CollectedResponse::Failed(failure)) => { - if sse_failure_invalid_reasoning_replay(&failure) { - evict_reasoning_replay( - &context.reasoning_replay, - &context.request, - &context.account_id, - ) - .await; - } let response_id = latest_response_id(&body); let latency_ms = elapsed_millis_i64(context.started_at); let failure_source = stream_failure_source(&failure); diff --git a/backend/src/dispatch/stream/sse_failure.rs b/backend/src/dispatch/stream/sse_failure.rs index cd99b7c18..6167231a5 100644 --- a/backend/src/dispatch/stream/sse_failure.rs +++ b/backend/src/dispatch/stream/sse_failure.rs @@ -1,10 +1,7 @@ use serde_json::Value; use crate::{ - dispatch::errors::{ - is_history_recovery_signal, is_invalid_encrypted_content_signal, - is_model_unsupported_signal, - }, + dispatch::errors::{is_history_recovery_code, is_model_unsupported_signal}, fleet::account::AccountStatus, upstream::openai::{ protocol::{ @@ -12,7 +9,7 @@ use crate::{ responses::{response_from_codex_sse, CollectedResponse, ResponsesSseFailure}, sse::SseError, }, - transport::{is_banned_auth_signal, CodexClientError}, + transport::is_banned_auth_signal, }, }; @@ -93,20 +90,10 @@ pub(in crate::dispatch) fn is_model_unsupported_sse_failure(failure: &ResponsesS } pub(in crate::dispatch) fn is_history_recovery_sse_failure(failure: &ResponsesSseFailure) -> bool { - sse_failure_matches(failure, is_history_recovery_signal) -} - -pub(in crate::dispatch) fn sse_failure_invalid_reasoning_replay( - failure: &ResponsesSseFailure, -) -> bool { - sse_failure_matches(failure, is_invalid_encrypted_content_signal) -} - -pub(in crate::dispatch) fn client_error_invalid_reasoning_replay(error: &CodexClientError) -> bool { - matches!( - error, - CodexClientError::Upstream { body, .. } if is_invalid_encrypted_content_signal(body) - ) + failure + .upstream_code + .as_deref() + .is_some_and(is_history_recovery_code) } pub(crate) fn auth_sse_failure_account_status(failure: &ResponsesSseFailure) -> AccountStatus { @@ -124,6 +111,7 @@ pub(in crate::dispatch) fn first_sse_failure( match response_from_codex_sse(&body, None)? { CollectedResponse::Failed(failure) => Ok(Some(failure)), CollectedResponse::Completed(_) + | CollectedResponse::Incomplete(_) | CollectedResponse::MissingCompleted | CollectedResponse::Empty => Ok(None), } diff --git a/backend/src/dispatch/upstream_call.rs b/backend/src/dispatch/upstream_call.rs index f2d202146..cd6b68aff 100644 --- a/backend/src/dispatch/upstream_call.rs +++ b/backend/src/dispatch/upstream_call.rs @@ -7,12 +7,13 @@ use serde_json::{json, Value}; use crate::{ dispatch::{ - affinity::build_conversation_identity, + affinity::{AccountIdentityService, AccountScopedIdentity}, + attempts::AccountAttemptLedger, errors::{ is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error, - is_rate_limit_upstream_error, is_retryable_upstream_5xx_error, - rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status, - upstream_error_set_cookie_headers, ResponseDispatchError, + is_rate_limit_upstream_error, is_retryable_account_transport_error, + is_retryable_upstream_5xx_error, rate_limit_cooldown_until, upstream_error_body, + upstream_error_http_status, upstream_error_set_cookie_headers, ResponseDispatchError, }, recording::{ insert_response_status_metadata, insert_response_trace_metadata, @@ -28,7 +29,7 @@ use crate::{ }, exhaustion::AccountExhaustionTracker, }, - service::ResponseDispatchService, + service::{ResponseDispatchResponse, ResponseDispatchService}, stream::trace::ResponseDispatchTrace, }, fleet::{ @@ -56,27 +57,29 @@ use crate::{ }, }; -const MAX_QUOTA_VERIFY_ATTEMPTS: usize = 5; - pub(crate) const QUOTA_VERIFY_LIMIT_REACHED_MESSAGE: &str = "Upstream usage quota still reports limit_reached"; pub(crate) enum QuotaVerificationDecision { Ready(Box), RetryWithAnotherAccount, - RequiredAccountUnavailable, - MaxAttemptsReached, } pub(crate) struct QuotaVerificationContext<'a> { pub account_pool: &'a AccountPoolService, pub codex: &'a CodexBackendClient, pub cloudflare: &'a CloudflareRecovery, - pub installation_id: Option<&'a str>, + pub account_identity: &'a AccountIdentityService, pub request_id: &'a str, - pub excluded_account_ids: &'a mut Vec, - pub verify_attempts: &'a mut usize, - pub allow_retry_with_another_account: bool, +} + +#[derive(Clone, Copy)] +pub(crate) struct AccountUpstreamContext<'a> { + pub codex: &'a CodexBackendClient, + pub account_identity: &'a AccountIdentityService, + pub cloudflare: &'a CloudflareRecovery, + pub request_id: &'a str, + pub account: &'a Account, } pub(crate) async fn verify_acquired_quota_if_required( @@ -88,6 +91,9 @@ pub(crate) async fn verify_acquired_quota_if_required( } let account_id = acquired.account.id.clone(); + let identity = context + .account_identity + .scope_auxiliary(&account_id, context.request_id); let cookie_header = context .cloudflare .cookie_header_for_request(&account_id, "/codex/usage") @@ -106,8 +112,12 @@ pub(crate) async fn verify_acquired_quota_if_required( codex_window_id: None, parent_thread_id: None, cookie_header: cookie_header.as_deref(), - installation_id: context.installation_id, + installation_id: Some(&identity.installation_id), session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: Some(&identity.client_request_id), + turn_id: None, }) .await; @@ -134,24 +144,14 @@ pub(crate) async fn verify_acquired_quota_if_required( .await; if quota_snapshot_limit_reached("a) { acquired.release_without_usage().await; - context.excluded_account_ids.push(account_id.clone()); - *context.verify_attempts += 1; - let max_attempts_reached = *context.verify_attempts >= MAX_QUOTA_VERIFY_ATTEMPTS; tracing::info!( request_id = %context.request_id, account_id = %account_id, quota_verify_required = true, quota_verify_result = "limit_reached", - verify_attempts = *context.verify_attempts, - retry_with_another_account = context.allow_retry_with_another_account && !max_attempts_reached, + retry_with_another_account = true, "quota verification reported exhausted account before upstream request" ); - if !context.allow_retry_with_another_account { - return QuotaVerificationDecision::RequiredAccountUnavailable; - } - if max_attempts_reached { - return QuotaVerificationDecision::MaxAttemptsReached; - } return QuotaVerificationDecision::RetryWithAnotherAccount; } @@ -175,118 +175,212 @@ fn acquired_with_verified_quota(mut acquired: AccountLease, quota: &Value) -> Ac } pub(crate) async fn create_response_with_account( - codex: &CodexBackendClient, - installation_id: Option<&str>, - cloudflare: &CloudflareRecovery, + context: AccountUpstreamContext<'_>, request: &CodexResponsesRequest, - request_id: &str, - account: &Account, started_at: Instant, ) -> Result { - let cookie_header = cloudflare - .cookie_header_for_request(&account.id, "/codex/responses") + let cookie_header = context + .cloudflare + .cookie_header_for_request(&context.account.id, "/codex/responses") .await; - let identity = build_conversation_identity( - request.prompt_cache_key(), - request.codex_window_id.as_deref(), - &account.id, - ); - codex + let identity = context + .account_identity + .scope(request, &context.account.id, context.request_id); + context + .codex .create_response_with_pool_account_started_at( request, CodexRequestContext { - access_token: &account.access_token, - account_id: account.account_id.as_deref(), - request_id, + access_token: &context.account.access_token, + account_id: context.account.account_id.as_deref(), + request_id: context.request_id, turn_state: request.turn_state.as_deref(), turn_metadata: request.turn_metadata.as_deref(), beta_features: request.beta_features.as_deref(), include_timing_metrics: request.include_timing_metrics.as_deref(), version: request.version.as_deref(), codex_window_id: identity.window_id.as_deref(), - parent_thread_id: request.parent_thread_id.as_deref(), + parent_thread_id: identity.parent_thread_id.as_deref(), cookie_header: cookie_header.as_deref(), - installation_id, - session_id: identity.conversation_id.as_deref(), + installation_id: Some(&identity.installation_id), + session_id: identity.session_id.as_deref(), + thread_id: identity.thread_id.as_deref(), + prompt_cache_key: identity.prompt_cache_key.as_deref(), + client_request_id: Some(&identity.client_request_id), + turn_id: identity.turn_id.as_deref(), }, - Some(&account.id), + Some(&context.account.id), started_at, ) .await } async fn create_response_stream_with_account( - codex: &CodexBackendClient, - installation_id: Option<&str>, - cloudflare: &CloudflareRecovery, + context: AccountUpstreamContext<'_>, request: &CodexResponsesRequest, - request_id: &str, - account: &Account, ) -> Result { - let cookie_header = cloudflare - .cookie_header_for_request(&account.id, "/codex/responses") + let cookie_header = context + .cloudflare + .cookie_header_for_request(&context.account.id, "/codex/responses") .await; - let identity = build_conversation_identity( - request.prompt_cache_key(), - request.codex_window_id.as_deref(), - &account.id, - ); - codex + let identity = context + .account_identity + .scope(request, &context.account.id, context.request_id); + context + .codex .create_response_stream_with_pool_account( request, CodexRequestContext { - access_token: &account.access_token, - account_id: account.account_id.as_deref(), - request_id, + access_token: &context.account.access_token, + account_id: context.account.account_id.as_deref(), + request_id: context.request_id, turn_state: request.turn_state.as_deref(), turn_metadata: request.turn_metadata.as_deref(), beta_features: request.beta_features.as_deref(), include_timing_metrics: request.include_timing_metrics.as_deref(), version: request.version.as_deref(), codex_window_id: identity.window_id.as_deref(), - parent_thread_id: request.parent_thread_id.as_deref(), + parent_thread_id: identity.parent_thread_id.as_deref(), cookie_header: cookie_header.as_deref(), - installation_id, - session_id: identity.conversation_id.as_deref(), + installation_id: Some(&identity.installation_id), + session_id: identity.session_id.as_deref(), + thread_id: identity.thread_id.as_deref(), + prompt_cache_key: identity.prompt_cache_key.as_deref(), + client_request_id: Some(&identity.client_request_id), + turn_id: identity.turn_id.as_deref(), }, - Some(&account.id), + Some(&context.account.id), ) .await } async fn create_compact_response_with_account( - codex: &CodexBackendClient, - installation_id: Option<&str>, - cloudflare: &CloudflareRecovery, + context: AccountUpstreamContext<'_>, request: &CodexCompactRequest, - request_id: &str, - account: &Account, ) -> Result { - let cookie_header = cloudflare - .cookie_header_for_request(&account.id, "/codex/responses/compact") + let cookie_header = context + .cloudflare + .cookie_header_for_request(&context.account.id, "/codex/responses/compact") .await; - codex + let identity = + context + .account_identity + .scope_compact(request, &context.account.id, context.request_id); + let upstream_request = compact_upstream_request(request, &identity); + context + .codex .create_compact_response( - request, + &upstream_request, CodexRequestContext { - access_token: &account.access_token, - account_id: account.account_id.as_deref(), - request_id, + access_token: &context.account.access_token, + account_id: context.account.account_id.as_deref(), + request_id: context.request_id, turn_state: None, turn_metadata: None, beta_features: None, include_timing_metrics: None, version: None, - codex_window_id: None, - parent_thread_id: None, + codex_window_id: identity.window_id.as_deref(), + parent_thread_id: identity.parent_thread_id.as_deref(), cookie_header: cookie_header.as_deref(), - installation_id, - session_id: None, + installation_id: Some(&identity.installation_id), + session_id: identity.session_id.as_deref(), + thread_id: identity.thread_id.as_deref(), + prompt_cache_key: identity.prompt_cache_key.as_deref(), + client_request_id: Some(&identity.client_request_id), + turn_id: identity.turn_id.as_deref(), }, ) .await } +fn compact_upstream_request( + request: &CodexCompactRequest, + identity: &AccountScopedIdentity, +) -> CodexCompactRequest { + let mut upstream = request.clone(); + for (key, value) in [ + ("prompt_cache_key", identity.prompt_cache_key.as_deref()), + ("session_id", identity.session_id.as_deref()), + ("thread_id", identity.thread_id.as_deref()), + ("turn_id", identity.turn_id.as_deref()), + ( + "x-client-request-id", + Some(identity.client_request_id.as_str()), + ), + ("x-codex-window-id", identity.window_id.as_deref()), + ( + "x-codex-parent-thread-id", + identity.parent_thread_id.as_deref(), + ), + ( + "x-codex-installation-id", + Some(identity.installation_id.as_str()), + ), + ] { + replace_existing_identity_field(&mut upstream.body, key, value); + } + + let metadata = match upstream.body.get("client_metadata") { + Some(Value::Object(metadata)) => Some(metadata.clone()), + None => Some(serde_json::Map::new()), + Some(_) => None, + }; + if let Some(mut metadata) = metadata { + for (key, value) in [ + ( + "x-codex-installation-id", + Some(identity.installation_id.as_str()), + ), + ("session_id", identity.session_id.as_deref()), + ("thread_id", identity.thread_id.as_deref()), + ( + "x-client-request-id", + Some(identity.client_request_id.as_str()), + ), + ("turn_id", identity.turn_id.as_deref()), + ("x-codex-window-id", identity.window_id.as_deref()), + ( + "x-codex-parent-thread-id", + identity.parent_thread_id.as_deref(), + ), + ] { + match value.filter(|value| !value.trim().is_empty()) { + Some(value) => { + metadata.insert(key.to_string(), Value::String(value.to_string())); + } + None => { + metadata.remove(key); + } + } + } + if !metadata.is_empty() { + upstream + .body + .insert("client_metadata".to_string(), Value::Object(metadata)); + } + } + upstream +} + +fn replace_existing_identity_field( + body: &mut serde_json::Map, + key: &str, + value: Option<&str>, +) { + if !body.get(key).is_some_and(Value::is_string) { + return; + } + match value.filter(|value| !value.trim().is_empty()) { + Some(value) => { + body.insert(key.to_string(), Value::String(value.to_string())); + } + None => { + body.remove(key); + } + } +} + const MAX_UPSTREAM_5XX_RETRIES_PER_ACCOUNT: usize = 2; async fn retry_upstream_5xx( @@ -392,87 +486,47 @@ fn log_upstream_failure( } pub(crate) async fn create_response_with_account_retrying_5xx( - codex: &CodexBackendClient, - installation_id: Option<&str>, - cloudflare: &CloudflareRecovery, + context: AccountUpstreamContext<'_>, request: &CodexResponsesRequest, - request_id: &str, - account: &Account, started_at: Instant, ) -> Result { retry_upstream_5xx( - request_id, - &account.id, + context.request_id, + &context.account.id, None, "upstream response request failed with retryable 5xx", "upstream response request failed", - || { - create_response_with_account( - codex, - installation_id, - cloudflare, - request, - request_id, - account, - started_at, - ) - }, + || create_response_with_account(context, request, started_at), ) .await } pub(crate) async fn create_response_stream_with_account_retrying_5xx( - codex: &CodexBackendClient, - installation_id: Option<&str>, - cloudflare: &CloudflareRecovery, + context: AccountUpstreamContext<'_>, request: &CodexResponsesRequest, - request_id: &str, - account: &Account, ) -> Result { retry_upstream_5xx( - request_id, - &account.id, + context.request_id, + &context.account.id, None, "upstream response stream request failed with retryable 5xx", "upstream response stream request failed", - || { - create_response_stream_with_account( - codex, - installation_id, - cloudflare, - request, - request_id, - account, - ) - }, + || create_response_stream_with_account(context, request), ) .await } pub(crate) async fn create_compact_response_with_account_retrying_5xx( - codex: &CodexBackendClient, - installation_id: Option<&str>, - cloudflare: &CloudflareRecovery, + context: AccountUpstreamContext<'_>, request: &CodexCompactRequest, - request_id: &str, - account: &Account, ) -> Result { retry_upstream_5xx( - request_id, - &account.id, + context.request_id, + &context.account.id, Some("compact"), "upstream compact request failed with retryable 5xx", "upstream compact request failed", - || { - create_compact_response_with_account( - codex, - installation_id, - cloudflare, - request, - request_id, - account, - ) - }, + || create_compact_response_with_account(context, request), ) .await } @@ -484,23 +538,28 @@ impl ResponseDispatchService { request_id: &str, mut request: CodexCompactRequest, requested_model: &str, - ) -> Result { + ) -> Result { let started_at = Instant::now(); let client_api_key_id = request.client_api_key_id.clone(); let catalog = self.models.catalog().await; let display_model = catalog.resolve_model_id(requested_model); request.set_model(display_model.clone()); - let mut excluded_account_ids = Vec::new(); let mut exhausted_accounts = AccountExhaustionTracker::default(); - let mut quota_verify_attempts = 0usize; let mut trace = ResponseDispatchTrace::default(); + let acquire_request = AccountAcquireRequest::new(request.model(), Utc::now()); + let mut candidates = + AccountAttemptLedger::freeze(&self.account_pool, &acquire_request).await; loop { - let acquire_request = AccountAcquireRequest::new(request.model(), Utc::now()) - .with_exclude_account_ids(excluded_account_ids.iter().cloned()); - let acquired = match self.account_pool.acquire_with(&acquire_request).await { + let acquired = match candidates.acquire_next(&self.account_pool).await { Some(acquired) => acquired, None => { + tracing::info!( + candidate_count = candidates.candidate_count(), + attempted = candidates.attempted_count(), + state_excluded = candidates.state_excluded_count(), + "Responses compact account candidate ledger exhausted" + ); let error = exhausted_accounts .last_exhausted() .map(ResponseDispatchError::from_exhausted_account) @@ -523,11 +582,8 @@ impl ResponseDispatchService { account_pool: self.account_pool.as_ref(), codex: self.codex.as_ref(), cloudflare: &self.cloudflare, - installation_id: self.installation_id.as_deref(), + account_identity: &self.account_identity, request_id, - excluded_account_ids: &mut excluded_account_ids, - verify_attempts: &mut quota_verify_attempts, - allow_retry_with_another_account: true, }, acquired, ) @@ -535,42 +591,25 @@ impl ResponseDispatchService { { QuotaVerificationDecision::Ready(acquired) => *acquired, QuotaVerificationDecision::RetryWithAnotherAccount => { - exhausted_accounts - .record_rate_limited(None, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE); - continue; - } - QuotaVerificationDecision::MaxAttemptsReached - | QuotaVerificationDecision::RequiredAccountUnavailable => { exhausted_accounts.record_rate_limited( Some(&acquired_account_id), QUOTA_VERIFY_LIMIT_REACHED_MESSAGE, ); - let error = exhausted_accounts - .last_exhausted() - .map(ResponseDispatchError::from_exhausted_account) - .unwrap_or(ResponseDispatchError::NoActiveAccount); - self.record_compact_dispatch_error( - request_id, - client_api_key_id.as_deref(), - requested_model, - started_at, - Some(&acquired_account_id), - &error, - ) - .await; - return Err(error); + continue; } }; let account = acquired.account.clone(); let release_account_id = account.id.clone(); let attempt = trace.start_attempt(&release_account_id); let response_result = create_compact_response_with_account_retrying_5xx( - &self.codex, - self.installation_id.as_deref(), - &self.cloudflare, + AccountUpstreamContext { + codex: &self.codex, + account_identity: &self.account_identity, + cloudflare: &self.cloudflare, + request_id, + account: &account, + }, &request, - request_id, - &account, ) .await; acquired.complete().await; @@ -605,6 +644,9 @@ impl ResponseDispatchService { "stream": false, "compact": true, "usage": usage, + "effectiveModel": response.response_metadata.effective_model.as_deref(), + "modelsEtag": response.response_metadata.models_etag.as_deref(), + "reasoningIncluded": response.response_metadata.reasoning_included, }); insert_response_status_metadata( &mut metadata, @@ -620,7 +662,11 @@ impl ResponseDispatchService { client_api_key_id: request.client_api_key_id.as_deref(), account_id: &account.id, route: "/v1/responses/compact", - model: &display_model, + model: response + .response_metadata + .effective_model + .as_deref() + .unwrap_or(&display_model), requested_model: Some(requested_model), client_ip: request.client_ip.as_deref(), client_user_agent: request.client_user_agent.as_deref(), @@ -633,7 +679,12 @@ impl ResponseDispatchService { rate_limit_headers: &response.rate_limit_headers, }) .await; - return Ok(response.body); + self.models + .observe_models_etag(response.response_metadata.models_etag.as_deref()); + return Ok(ResponseDispatchResponse { + body: response.body, + response_headers: response.response_metadata.client_headers, + }); } Err(error) if is_rate_limit_upstream_error(&error) => { exhausted_accounts.record_rate_limited( @@ -644,7 +695,6 @@ impl ResponseDispatchService { self.account_pool .mark_quota_limited_until(&release_account_id, cooldown_until) .await; - excluded_account_ids.push(release_account_id); } Err(error) if is_quota_exhausted_upstream_error(&error) => { exhausted_accounts.record_quota_exhausted( @@ -654,7 +704,6 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::QuotaExhausted) .await; - excluded_account_ids.push(release_account_id); } Err(error) if is_auth_upstream_error(&error) => { let upstream_error = upstream_error_body(&error); @@ -668,7 +717,6 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, account_status) .await; - excluded_account_ids.push(release_account_id); } Err(error) if is_cloudflare_challenge_upstream_error(&error) => { exhausted_accounts.record_cloudflare_challenge( @@ -678,7 +726,6 @@ impl ResponseDispatchService { self.cloudflare .apply_challenge(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); } Err(error) if is_cloudflare_path_block_upstream_error(&error) => { exhausted_accounts.record_cloudflare_path_blocked( @@ -688,28 +735,11 @@ impl ResponseDispatchService { self.cloudflare .apply_path_block(self.account_pool.as_ref(), &release_account_id) .await; - excluded_account_ids.push(release_account_id); } Err(error) if is_model_unsupported_upstream_error(&error) => { let upstream_error = upstream_error_body(&error); - if let Some(exhausted) = - exhausted_accounts.model_unsupported_retry_exhausted(upstream_error.clone()) - { - let error = ResponseDispatchError::from_exhausted_account(exhausted); - self.record_compact_dispatch_error( - request_id, - client_api_key_id.as_deref(), - requested_model, - started_at, - Some(&release_account_id), - &error, - ) - .await; - return Err(error); - } exhausted_accounts .record_model_unsupported(Some(&release_account_id), upstream_error); - excluded_account_ids.push(release_account_id); } Err(error) if is_banned_upstream_error(&error) => { exhausted_accounts.record_auth_failure( @@ -721,9 +751,15 @@ impl ResponseDispatchService { self.account_pool .set_status(&release_account_id, AccountStatus::Banned) .await; - excluded_account_ids.push(release_account_id); } Err(error) => { + if is_retryable_account_transport_error(&error) { + exhausted_accounts.record_upstream_unavailable( + Some(&release_account_id), + upstream_error_body(&error), + ); + continue; + } let error = ResponseDispatchError::Upstream(error); self.record_compact_dispatch_error( request_id, diff --git a/backend/src/fleet/manage/import.rs b/backend/src/fleet/manage/import.rs index 23eeefb18..ee9571e70 100644 --- a/backend/src/fleet/manage/import.rs +++ b/backend/src/fleet/manage/import.rs @@ -226,6 +226,8 @@ impl AccountManageService { } let request_id = uuid::Uuid::new_v4().to_string(); + let installation_id = + account_id.map(|account_id| self.account_pseudonymizer.installation_id(account_id)); let context = crate::upstream::openai::transport::CodexRequestContext { access_token, account_id, @@ -238,8 +240,12 @@ impl AccountManageService { codex_window_id: None, parent_thread_id: None, cookie_header: None, - installation_id: self.installation_id.as_deref(), + installation_id: installation_id.as_deref(), session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }; match self.codex.fetch_usage(context).await { diff --git a/backend/src/fleet/manage/probe.rs b/backend/src/fleet/manage/probe.rs index ffae6c480..b133998f7 100644 --- a/backend/src/fleet/manage/probe.rs +++ b/backend/src/fleet/manage/probe.rs @@ -82,14 +82,11 @@ impl AccountManageService { plan_type: plan_type.to_string(), access_token: account.access_token.expose_secret().to_string(), account_id: account.account_id.clone(), + installation_id: self.account_pseudonymizer.installation_id(&account.id), }; if let Err(error) = self .models - .refresh_backend_models_with_installation_id( - &[plan_account], - &request_id, - self.installation_id.as_deref(), - ) + .refresh_backend_models(&[plan_account], &request_id) .await { tracing::warn!( @@ -121,7 +118,7 @@ impl AccountManageService { .await .ok() .flatten(); - let installation_id = self.installation_id.clone(); + let installation_id = self.account_pseudonymizer.installation_id(&account.id); let codex = self.codex.clone(); let service = self.clone(); let stored_account_id = account.id.clone(); @@ -161,8 +158,12 @@ impl AccountManageService { codex_window_id: None, parent_thread_id: None, cookie_header: cookie_header.as_deref(), - installation_id: installation_id.as_deref(), + installation_id: Some(&installation_id), session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }; let outcome = match codex.create_response_stream(&request, context).await { @@ -265,6 +266,8 @@ fn test_responses_request(model: String) -> CodexResponsesRequest { ] })], ); + request.set_stream(true); + request.set_store(false); request.force_http_sse = true; request } diff --git a/backend/src/fleet/manage/quota.rs b/backend/src/fleet/manage/quota.rs index 777b85794..498078198 100644 --- a/backend/src/fleet/manage/quota.rs +++ b/backend/src/fleet/manage/quota.rs @@ -34,6 +34,7 @@ impl AccountManageService { let request_id = uuid::Uuid::new_v4().to_string(); let token = stored.access_token.expose_secret().to_string(); let cookie_header = self.usage_cookie_header(account_id).await; + let installation_id = self.account_pseudonymizer.installation_id(account_id); let context = crate::upstream::openai::transport::CodexRequestContext { access_token: &token, account_id: stored.account_id.as_deref(), @@ -46,8 +47,12 @@ impl AccountManageService { codex_window_id: None, parent_thread_id: None, cookie_header: cookie_header.as_deref(), - installation_id: self.installation_id.as_deref(), + installation_id: Some(&installation_id), session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }; let raw = self .codex diff --git a/backend/src/fleet/manage/service.rs b/backend/src/fleet/manage/service.rs index 9fa3857b0..3ffaf514f 100644 --- a/backend/src/fleet/manage/service.rs +++ b/backend/src/fleet/manage/service.rs @@ -14,6 +14,7 @@ use crate::{ refresh::{RedisRefreshLeaseStore, RuntimeRefreshPolicy}, store::PgAccountStore, }, + infra::identity::AccountPseudonymizer, models::service::ModelService, upstream::openai::{token_client::TokenRefresher, transport::CodexBackendClient}, }; @@ -32,7 +33,7 @@ pub struct AccountManageService { pub(crate) refresh_lease_owner_prefix: String, pub(crate) oauth: oauth::AccountOAuthService, pub(crate) refresh_policy: RuntimeRefreshPolicy, - pub(crate) installation_id: Option, + pub(crate) account_pseudonymizer: Arc, } pub struct AccountManageServiceParts { @@ -45,7 +46,7 @@ pub struct AccountManageServiceParts { pub refresh_leases: RedisRefreshLeaseStore, pub oauth: oauth::AccountOAuthService, pub refresh_policy: RuntimeRefreshPolicy, - pub installation_id: Option, + pub account_pseudonymizer: Arc, } impl AccountManageService { @@ -64,7 +65,7 @@ impl AccountManageService { ), oauth: parts.oauth, refresh_policy: parts.refresh_policy, - installation_id: parts.installation_id, + account_pseudonymizer: parts.account_pseudonymizer, } } diff --git a/backend/src/fleet/pool/mod.rs b/backend/src/fleet/pool/mod.rs index cbb3d5300..d15c319ae 100644 --- a/backend/src/fleet/pool/mod.rs +++ b/backend/src/fleet/pool/mod.rs @@ -71,6 +71,7 @@ pub struct AccountPoolService { store: Arc, usage_store: Arc, request_interval: Arc>, + candidate_changed: Arc, } /// 一次请求对账号池 slot 的唯一所有权凭据。 @@ -79,6 +80,14 @@ pub struct AccountLease { slot: AccountSlotLease, } +/// 冻结候选后的单账号实时租用结果。 +#[derive(Debug)] +pub enum AccountCandidateLease { + Acquired(Box), + Busy, + Unavailable, +} + impl AccountLease { fn new(service: AccountPoolService, acquired: AcquiredAccount) -> Self { let slot = AccountSlotLease::new(service, acquired.account.id.clone(), acquired.slot_id); @@ -190,6 +199,7 @@ impl AccountPoolService { request_interval: Arc::new(std::sync::RwLock::new(StdDuration::from_millis( request_interval_ms, ))), + candidate_changed: Arc::new(tokio::sync::Notify::new()), } } @@ -300,6 +310,7 @@ impl AccountPoolService { let Some(released) = released else { return; }; + self.candidate_changed.notify_one(); if !record_request_usage { return; } @@ -387,6 +398,47 @@ impl AccountPoolService { lease } + /// 冻结请求开始时跨全部订阅层级的有序候选账号 ID。 + pub async fn candidate_snapshot(&self, request: &AccountAcquireRequest) -> Vec { + let refresh = self + .pool + .lock() + .await + .candidate_snapshot_with_status_refresh(request); + self.persist_runtime_account_states(refresh.refreshed_accounts) + .await; + refresh.account_ids + } + + /// 按候选 ID 重新读取实时状态并租用账号。 + pub async fn acquire_candidate( + &self, + model: &str, + account_id: &str, + now: DateTime, + ) -> AccountCandidateLease { + let refresh = self + .pool + .lock() + .await + .acquire_candidate_with_status_refresh(model, account_id, now); + let decision = match refresh.decision { + AccountCandidateAcquireDecision::Acquired(acquired) => AccountCandidateLease::Acquired( + Box::new(AccountLease::new(self.clone(), *acquired)), + ), + AccountCandidateAcquireDecision::Busy => AccountCandidateLease::Busy, + AccountCandidateAcquireDecision::Unavailable => AccountCandidateLease::Unavailable, + }; + self.persist_runtime_account_states(refresh.refreshed_accounts) + .await; + decision + } + + /// 等待任一账号 slot 或运行状态发生变化后重试 Busy 候选。 + pub async fn wait_for_candidate_change(&self) { + self.candidate_changed.notified().await; + } + /// 按订阅计划各选一个可用账号，用于刷新模型列表。 pub async fn distinct_plan_accounts( &self, diff --git a/backend/src/fleet/pool/state.rs b/backend/src/fleet/pool/state.rs index d9781894e..a10b49bd9 100644 --- a/backend/src/fleet/pool/state.rs +++ b/backend/src/fleet/pool/state.rs @@ -8,6 +8,25 @@ pub(super) struct AccountAcquireWithStatusRefresh { pub(super) refreshed_accounts: Vec, } +#[derive(Debug)] +pub(super) struct AccountCandidateSnapshotWithStatusRefresh { + pub(super) account_ids: Vec, + pub(super) refreshed_accounts: Vec, +} + +#[derive(Debug)] +pub(super) struct AccountCandidateAcquireWithStatusRefresh { + pub(super) decision: AccountCandidateAcquireDecision, + pub(super) refreshed_accounts: Vec, +} + +#[derive(Debug)] +pub(super) enum AccountCandidateAcquireDecision { + Acquired(Box), + Busy, + Unavailable, +} + #[derive(Debug)] pub(super) struct DistinctPlanAccountsWithStatusRefresh { pub(super) accounts: Vec, @@ -422,6 +441,65 @@ impl AccountPool { } } + pub(super) fn candidate_snapshot_with_status_refresh( + &mut self, + request: &AccountAcquireRequest, + ) -> AccountCandidateSnapshotWithStatusRefresh { + self.cleanup_stale_slots(request.now); + let refreshed_accounts = self.refresh_account_statuses(request.now); + let candidates = self.snapshot_candidates(request); + let mut ordered = self.scheduler.order( + self.options.rotation_strategy, + &candidates, + &|account_id| self.slot_count(account_id), + request.now, + ); + ordered.sort_by_key(|account| { + candidates::tier_rank(account.plan_type.as_deref(), &self.options.tier_priority) + }); + + let mut account_ids = Vec::with_capacity(ordered.len()); + if let Some(preferred_account_id) = request.preferred_account_id.as_deref() { + if let Some(index) = ordered + .iter() + .position(|account| account.id == preferred_account_id) + { + account_ids.push(ordered.remove(index).id); + } + } + account_ids.extend(ordered.into_iter().map(|account| account.id)); + AccountCandidateSnapshotWithStatusRefresh { + account_ids, + refreshed_accounts, + } + } + + pub(super) fn acquire_candidate_with_status_refresh( + &mut self, + model: &str, + account_id: &str, + now: DateTime, + ) -> AccountCandidateAcquireWithStatusRefresh { + let request = AccountAcquireRequest::new(model, now).with_required_account_id(account_id); + self.cleanup_stale_slots(now); + let refreshed_accounts = self.refresh_account_statuses(now); + let snapshot_eligible = !self.snapshot_candidates(&request).is_empty(); + let decision = if !snapshot_eligible { + AccountCandidateAcquireDecision::Unavailable + } else if self.slot_count(account_id) >= self.options.max_concurrent_per_account { + AccountCandidateAcquireDecision::Busy + } else { + match self.acquire_with_status_refresh(&request).acquired { + Some(acquired) => AccountCandidateAcquireDecision::Acquired(Box::new(acquired)), + None => AccountCandidateAcquireDecision::Unavailable, + } + }; + AccountCandidateAcquireWithStatusRefresh { + decision, + refreshed_accounts, + } + } + fn refresh_account_statuses(&mut self, now: DateTime) -> Vec { let mut refreshed_accounts = Vec::new(); let mut expired_account_ids = Vec::new(); @@ -707,6 +785,27 @@ impl AccountPool { ) } + fn snapshot_candidates(&self, request: &AccountAcquireRequest) -> Vec<&Account> { + let filter = CandidateFilter { + max_concurrent_per_account: self.options.max_concurrent_per_account, + skip_quota_limited: self.options.skip_quota_limited, + tier_priority: &self.options.tier_priority, + model_plan_allowlist: &self.options.model_plan_allowlist, + fetched_model_plan_types: &self.options.fetched_model_plan_types, + slot_count: &|account_id| self.slot_count(account_id), + }; + candidates::snapshot( + self.accounts.values(), + &filter, + &CandidateRequest { + model: &request.model, + exclude_account_ids: &request.exclude_account_ids, + required_account_id: request.required_account_id.as_deref(), + now: request.now, + }, + ) + } + /// 按当前策略从候选集合中选择一个账号（委托 [`AccountScheduler::select`]）。 fn select_candidate(&self, candidates: &[&Account], now: DateTime) -> Option { let slot_count = |account_id: &str| self.slots.get(account_id).map_or(0, VecDeque::len); diff --git a/backend/src/fleet/quota/runtime.rs b/backend/src/fleet/quota/runtime.rs index befe23493..e28c43c89 100644 --- a/backend/src/fleet/quota/runtime.rs +++ b/backend/src/fleet/quota/runtime.rs @@ -35,7 +35,7 @@ pub struct QuotaRefreshService { codex: Arc, min_refresh_interval_secs: u64, request_spacing: Duration, - installation_id: Option, + account_pseudonymizer: Arc, cookie_store: Option, account_pool: Option>, } @@ -46,6 +46,7 @@ impl QuotaRefreshService { store: PgAccountStore, usage_store: PgAccountUsageStore, codex: Arc, + account_pseudonymizer: Arc, ) -> Self { Self { store, @@ -53,7 +54,7 @@ impl QuotaRefreshService { codex, min_refresh_interval_secs: MIN_REFRESH_INTERVAL_SECS, request_spacing: Self::default_request_spacing(), - installation_id: None, + account_pseudonymizer, cookie_store: None, account_pool: None, } @@ -68,6 +69,7 @@ impl QuotaRefreshService { store: PgAccountStore, usage_store: PgAccountUsageStore, codex: Arc, + account_pseudonymizer: Arc, min_refresh_interval_secs: u64, ) -> Self { Self { @@ -76,7 +78,7 @@ impl QuotaRefreshService { codex, min_refresh_interval_secs, request_spacing: Self::default_request_spacing(), - installation_id: None, + account_pseudonymizer, cookie_store: None, account_pool: None, } @@ -88,12 +90,6 @@ impl QuotaRefreshService { self } - /// 设置 Codex installation id。 - pub fn with_installation_id(mut self, installation_id: Option) -> Self { - self.installation_id = installation_id.filter(|id| !id.trim().is_empty()); - self - } - /// 设置 usage 请求可复用的账号 Cookie 存储。 pub fn with_cookie_store(mut self, cookie_store: PgCookieStore) -> Self { self.cookie_store = Some(cookie_store); @@ -155,6 +151,7 @@ impl QuotaRefreshService { let request_id = uuid::Uuid::new_v4().to_string(); let cookie_header = self.usage_cookie_header(&account.id).await; + let installation_id = self.account_pseudonymizer.installation_id(&account.id); match self .codex .fetch_usage(CodexRequestContext { @@ -169,8 +166,12 @@ impl QuotaRefreshService { codex_window_id: None, parent_thread_id: None, cookie_header: cookie_header.as_deref(), - installation_id: self.installation_id.as_deref(), + installation_id: Some(&installation_id), session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }) .await { diff --git a/backend/src/fleet/scheduler/candidates.rs b/backend/src/fleet/scheduler/candidates.rs index a431b36e6..5bcf118ed 100644 --- a/backend/src/fleet/scheduler/candidates.rs +++ b/backend/src/fleet/scheduler/candidates.rs @@ -68,14 +68,40 @@ pub fn filter<'a>( candidates } +/// 冻结一次客户端请求可遍历的完整候选集。 +/// +/// 与 [`filter`] 不同，这里不按最高订阅层级收窄，也不因瞬时 slot 已满而删除账号。 +/// 层级和 slot 只影响后续顺序与租用时机，不能缩小请求级 failover 边界。 +pub fn snapshot<'a>( + accounts: impl Iterator, + filter: &CandidateFilter<'_>, + request: &CandidateRequest<'_>, +) -> Vec<&'a Account> { + let mut candidates = accounts + .filter(|account| is_snapshot_eligible(account, filter, request)) + .collect::>(); + if let Some(required_account_id) = request.required_account_id { + candidates.retain(|account| account.id == required_account_id); + } + candidates +} + /// 判断账号是否满足全部基础可用条件。 fn is_base_available( account: &Account, filter: &CandidateFilter<'_>, request: &CandidateRequest<'_>, ) -> bool { - account.status == AccountStatus::Active + is_snapshot_eligible(account, filter, request) && (filter.slot_count)(&account.id) < filter.max_concurrent_per_account +} + +fn is_snapshot_eligible( + account: &Account, + filter: &CandidateFilter<'_>, + request: &CandidateRequest<'_>, +) -> bool { + account.status == AccountStatus::Active && quota_available_at(account, request.now, filter.skip_quota_limited) && is_model_allowed( account, @@ -90,6 +116,12 @@ fn is_base_available( && cloudflare_available_at(account, request.now) } +pub(crate) fn tier_rank(plan_type: Option<&str>, tier_priority: &[String]) -> usize { + plan_type + .and_then(|plan_type| tier_priority.iter().position(|tier| tier == plan_type)) + .unwrap_or(tier_priority.len()) +} + /// 判断模型是否允许该账号的订阅计划使用。 fn is_model_allowed( account: &Account, diff --git a/backend/src/fleet/scheduler/feedback.rs b/backend/src/fleet/scheduler/feedback.rs index ac29989c1..8ab440c9f 100644 --- a/backend/src/fleet/scheduler/feedback.rs +++ b/backend/src/fleet/scheduler/feedback.rs @@ -1,6 +1,6 @@ //! 账号运行时反馈:错误率与 TTFT 的 EWMA 统计。 //! -//! 每个账号维护两个指数加权移动平均(EWMA):错误率与首 token 延迟(TTFT， +//! 每个账号维护两个指数加权移动平均(EWMA):错误率与首个有效输出事件延迟(TTFT， //! 毫秒)。请求结束后经 [`FeedbackStats::report`] 回灌，直接进 [`super::score`] //! 打分并可驱动 sticky escape。统计以 `AtomicU64`(存 `f64` 位模式)+ CAS 无锁更新, //! 对齐 sub2api 的设计,避免选择热路径上的锁与 DB 往返。 @@ -127,7 +127,7 @@ impl FeedbackStats { } } - /// 回灌一次请求结果:成功/失败 + 首 token 延迟(毫秒)。 + /// 回灌一次请求结果:成功/失败 + 首个有效输出事件延迟(毫秒)。 pub fn report(&self, account_id: &str, success: bool, first_token_ms: Option) { // 快路径:账号已存在时只需读锁 + 原子更新。 { diff --git a/backend/src/fleet/scheduler/mod.rs b/backend/src/fleet/scheduler/mod.rs index 24af3fd28..9abbf69e0 100644 --- a/backend/src/fleet/scheduler/mod.rs +++ b/backend/src/fleet/scheduler/mod.rs @@ -129,6 +129,32 @@ impl AccountScheduler { select_with_strategy(strategy, &input, &mut cursor) } + /// 按当前策略为完整候选集排序，供单个请求依次执行账号 failover。 + pub fn order( + &self, + strategy: RotationStrategy, + candidates: &[&Account], + slot_count: &dyn Fn(&str) -> usize, + now: DateTime, + ) -> Vec { + let input = SelectionInput { + candidates, + slot_count, + feedback: &self.feedback, + weights: &self.weights, + now, + }; + let mut cursor = self.lock_cursor(); + match strategy { + RotationStrategy::Smart => strategy::smart::order(&input, &mut cursor), + RotationStrategy::QuotaResetPriority => { + strategy::quota_reset::order(&input, &mut cursor) + } + RotationStrategy::RoundRobin => strategy::round_robin::order(&input, &mut cursor), + RotationStrategy::Sticky => strategy::sticky::order(&input), + } + } + /// 回灌一次请求结果到运行时 EWMA 反馈（错误率 / TTFT），供 Smart 打分。 pub fn report_feedback(&self, account_id: &str, success: bool, first_token_ms: Option) { self.feedback.report(account_id, success, first_token_ms); @@ -181,6 +207,28 @@ pub(crate) fn select_by( Some((*sorted[index]).clone()) } +pub(crate) fn order_by( + candidates: &[&Account], + cursor: &mut usize, + compare: impl Fn(&Account, &Account) -> Ordering, +) -> Vec { + let mut sorted = candidates.to_vec(); + sorted.sort_by(|a, b| compare(a, b)); + let rotation = *cursor; + *cursor = cursor.wrapping_add(1); + + let mut start = 0; + while start < sorted.len() { + let mut end = start + 1; + while end < sorted.len() && compare(sorted[start], sorted[end]) == Ordering::Equal { + end += 1; + } + sorted[start..end].rotate_left(rotation % (end - start)); + start = end; + } + sorted.into_iter().cloned().collect() +} + pub(crate) fn account_window_token_count(account: &Account) -> u64 { account .window_input_tokens diff --git a/backend/src/fleet/scheduler/strategy/quota_reset.rs b/backend/src/fleet/scheduler/strategy/quota_reset.rs index 69abec846..02777b670 100644 --- a/backend/src/fleet/scheduler/strategy/quota_reset.rs +++ b/backend/src/fleet/scheduler/strategy/quota_reset.rs @@ -11,13 +11,20 @@ use std::cmp::Ordering; use crate::fleet::account::Account; -use crate::fleet::scheduler::{compare_last_used, compare_window_reset, select_by, SelectionInput}; +use crate::fleet::scheduler::{ + compare_last_used, compare_window_reset, order_by, select_by, SelectionInput, +}; /// QuotaResetPriority 策略选择。 pub fn select(input: &SelectionInput<'_>, cursor: &mut usize) -> Option { select_by(input.candidates, cursor, compare_quota_reset_priority) } +/// 按额度窗口重置优先级为完整候选集排序。 +pub fn order(input: &SelectionInput<'_>, cursor: &mut usize) -> Vec { + order_by(input.candidates, cursor, compare_quota_reset_priority) +} + /// 配额重置优先级比较：升序，最优（最应先用）排在最前。 fn compare_quota_reset_priority(a: &Account, b: &Account) -> Ordering { a.quota_limit_reached diff --git a/backend/src/fleet/scheduler/strategy/round_robin.rs b/backend/src/fleet/scheduler/strategy/round_robin.rs index faa403926..ae0b0403d 100644 --- a/backend/src/fleet/scheduler/strategy/round_robin.rs +++ b/backend/src/fleet/scheduler/strategy/round_robin.rs @@ -17,3 +17,19 @@ pub fn select(input: &SelectionInput<'_>, cursor: &mut usize) -> Option *cursor = cursor.wrapping_add(1); Some((*candidates[index]).clone()) } + +/// 从当前轮转位置开始返回完整候选顺序。 +pub fn order(input: &SelectionInput<'_>, cursor: &mut usize) -> Vec { + let mut ordered = input + .candidates + .iter() + .map(|account| (**account).clone()) + .collect::>(); + if ordered.is_empty() { + return ordered; + } + let candidate_count = ordered.len(); + ordered.rotate_left(*cursor % candidate_count); + *cursor = cursor.wrapping_add(1); + ordered +} diff --git a/backend/src/fleet/scheduler/strategy/smart.rs b/backend/src/fleet/scheduler/strategy/smart.rs index 6be1df89b..c43ed77cb 100644 --- a/backend/src/fleet/scheduler/strategy/smart.rs +++ b/backend/src/fleet/scheduler/strategy/smart.rs @@ -30,7 +30,7 @@ pub struct ScoreWeights { pub window_requests: f64, /// 错误率 EWMA（越健康越优先）。 pub error_rate: f64, - /// 首 token 延迟 EWMA（越快越优先）。 + /// 首个有效输出事件延迟 EWMA（越快越优先）。 pub ttft: f64, /// 窗口重置临近度（越快重置越优先，use-it-or-lose-it，默认关闭）。 pub reset: f64, @@ -97,6 +97,35 @@ pub fn select(input: &SelectionInput<'_>, cursor: &mut usize) -> Option Some((*input.candidates[account_index]).clone()) } +/// 按 Smart 得分为完整候选集排序；同分账号沿用调度器轮转游标破并列。 +pub fn order(input: &SelectionInput<'_>, cursor: &mut usize) -> Vec { + let mut ranked = rank_candidate_refs( + input.candidates, + input.weights, + input.now.timestamp(), + input.slot_count, + input.feedback, + ); + let rotation = *cursor; + *cursor = cursor.wrapping_add(1); + + let mut start = 0; + while start < ranked.len() { + let score = ranked[start].1.total.to_bits(); + let mut end = start + 1; + while end < ranked.len() && ranked[end].1.total.to_bits() == score { + end += 1; + } + ranked[start..end].rotate_left(rotation % (end - start)); + start = end; + } + + ranked + .into_iter() + .map(|(index, _)| (*input.candidates[index]).clone()) + .collect() +} + /// 对候选集合按加权得分排序（降序），返回 `(账号索引, 打分明细)` 列表。 /// /// 只做打分与排序，最终的并列破并列交给 [`select`]。公开供测试直接断言排序结果。 diff --git a/backend/src/fleet/scheduler/strategy/sticky.rs b/backend/src/fleet/scheduler/strategy/sticky.rs index 9e1ef3225..e6638c1a3 100644 --- a/backend/src/fleet/scheduler/strategy/sticky.rs +++ b/backend/src/fleet/scheduler/strategy/sticky.rs @@ -25,3 +25,14 @@ pub fn select(input: &SelectionInput<'_>, _cursor: &mut usize) -> Option) -> Vec { + let mut ordered = input + .candidates + .iter() + .map(|account| (**account).clone()) + .collect::>(); + ordered.sort_by(|a, b| compare_last_used(b.last_used_at.as_deref(), a.last_used_at.as_deref())); + ordered +} diff --git a/backend/src/infra/identity.rs b/backend/src/infra/identity.rs index fdd05a88b..aa6b37e8a 100644 --- a/backend/src/infra/identity.rs +++ b/backend/src/infra/identity.rs @@ -5,9 +5,63 @@ use argon2::{ Argon2, }; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use hmac::{Hmac, Mac}; use rand::Rng; use sha2::{Digest, Sha256}; use thiserror::Error; +use uuid::Uuid; + +type HmacSha256 = Hmac; + +/// 持久密钥驱动的账号作用域伪名原语。 +#[derive(Clone)] +pub struct AccountPseudonymizer { + template: HmacSha256, +} + +impl std::fmt::Debug for AccountPseudonymizer { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("AccountPseudonymizer") + .finish_non_exhaustive() + } +} + +impl AccountPseudonymizer { + /// 使用服务端持久密钥构造伪名器。 + pub fn new(secret: [u8; 32]) -> Self { + let template = + HmacSha256::new_from_slice(&secret).expect("HMAC-SHA256 accepts every 32-byte key"); + Self { template } + } + + /// 按字段域和账号派生稳定伪名。 + pub fn scoped(&self, domain: &str, account_id: Option<&str>, value: &str) -> String { + URL_SAFE_NO_PAD.encode(self.digest(domain, account_id, value)) + } + + /// 按账号派生 UUID 形态的 installation ID。 + pub fn installation_id(&self, account_id: &str) -> String { + let digest = self.digest("installation-id", Some(account_id), ""); + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Uuid::from_bytes(bytes).to_string() + } + + fn digest(&self, domain: &str, account_id: Option<&str>, value: &str) -> Vec { + let mut mac = self.template.clone(); + mac.update(domain.as_bytes()); + mac.update(b"\0"); + if let Some(account_id) = account_id { + mac.update(account_id.as_bytes()); + } + mac.update(b"\0"); + mac.update(value.as_bytes()); + mac.finalize().into_bytes().to_vec() + } +} // --------------------------------------------------------------------------- // AuthError / AuthResult diff --git a/backend/src/infra/paths.rs b/backend/src/infra/paths.rs index f77b325a1..4fcdbbab7 100644 --- a/backend/src/infra/paths.rs +++ b/backend/src/infra/paths.rs @@ -1,13 +1,14 @@ -//! 数据目录、installation ID 等路径辅助。 +//! 数据目录与持久身份密钥路径辅助。 use std::{ fs, io, + io::Write, path::{Path, PathBuf}, }; -use uuid::Uuid; +use rand::Rng; -const INSTALLATION_ID_FILE_NAME: &str = "installation_id"; +const IDENTITY_SECRET_FILE_NAME: &str = "identity_hmac_secret"; fn data_dir() -> PathBuf { dirs::data_local_dir().unwrap_or_else(|| PathBuf::from(".runtime/data")) @@ -20,73 +21,49 @@ pub fn ensure_data_dir() -> std::io::Result { Ok(dir) } -fn codex_desktop_installation_id_path() -> Option { - dirs::home_dir().map(|home| home.join(".codex").join(INSTALLATION_ID_FILE_NAME)) -} +/// 读取或创建账号身份隔离使用的 256-bit HMAC 密钥。 +pub fn load_or_create_identity_secret(data_dir: &Path) -> io::Result<[u8; 32]> { + let path = data_dir.join(IDENTITY_SECRET_FILE_NAME); + match read_identity_secret(&path) { + Ok(Some(secret)) => return Ok(secret), + Ok(None) => {} + Err(error) => return Err(error), + } -fn data_installation_id_path(data_dir: &Path) -> PathBuf { - data_dir.join(INSTALLATION_ID_FILE_NAME) -} - -/// 按读取优先级读取或生成 installation ID。 -/// -/// 顺序为本应用数据目录文件、真实 Codex Desktop 文件、生成并写入本应用数据目录。 -/// 当 `data_dir` 为 `None` 且没有可读文件时，返回一个不会持久化的新 UUID。 -pub fn load_or_create_installation_id(data_dir: Option<&Path>) -> io::Result { - let codex_path = codex_desktop_installation_id_path(); - let data_path = data_dir.map(data_installation_id_path); - load_or_create_installation_id_from_paths(codex_path.as_deref(), data_path.as_deref()) -} - -fn load_or_create_installation_id_from_paths( - codex_desktop_path: Option<&Path>, - data_path: Option<&Path>, -) -> io::Result { - if let Some(path) = data_path { - if let Some(id) = read_installation_id(path)? { - return Ok(id); + let mut secret = [0u8; 32]; + rand::rng().fill_bytes(&mut secret); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(&path) { + Ok(mut file) => { + file.write_all(hex::encode(secret).as_bytes())?; + file.sync_all()?; + Ok(secret) } - } - - if let Some(path) = codex_desktop_path { - if let Some(id) = read_installation_id(path)? { - if let Some(data_path) = data_path { - persist_installation_id(data_path, &id)?; - } - return Ok(id); - } - } - - if let Some(path) = data_path { - let generated = generate_installation_id(); - persist_installation_id(path, &generated)?; - return Ok(generated); - } - - Ok(generate_installation_id()) -} - -fn read_installation_id(path: &Path) -> io::Result> { - match fs::read_to_string(path) { - Ok(content) => Ok(parse_installation_id(&content)), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => read_identity_secret(&path)? + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid identity secret")), Err(error) => Err(error), } } -fn persist_installation_id(path: &Path, installation_id: &str) -> io::Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(path, installation_id) -} - -fn parse_installation_id(raw: &str) -> Option { - let trimmed = raw.trim(); - Uuid::parse_str(trimmed).ok()?; - Some(trimmed.to_string()) -} - -fn generate_installation_id() -> String { - Uuid::new_v4().to_string() +fn read_identity_secret(path: &Path) -> io::Result> { + let raw = match fs::read_to_string(path) { + Ok(raw) => raw, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let decoded = hex::decode(raw.trim()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let secret = decoded.try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "identity secret must be 32 bytes", + ) + })?; + Ok(Some(secret)) } diff --git a/backend/src/models/service.rs b/backend/src/models/service.rs index 99ad4bb70..386ec91cf 100644 --- a/backend/src/models/service.rs +++ b/backend/src/models/service.rs @@ -80,6 +80,8 @@ pub struct ModelRefreshPlanAccount { pub access_token: String, /// 上游账号标识。 pub account_id: Option, + /// 按账号作用域派生的 installation ID。 + pub installation_id: String, } /// 模型目录服务。 @@ -90,6 +92,9 @@ pub struct ModelService { catalog: Arc>, store: Option>, upstream_client: Option>, + models_etag: Arc>>, + etag_change_tx: watch::Sender, + refresh_lock: Arc>, } impl ModelService { @@ -100,15 +105,44 @@ impl ModelService { upstream_client: Option>, ) -> Self { let catalog = ModelCatalog::from_config(&config); + let (etag_change_tx, _) = watch::channel(0); Self { config: Arc::new(RwLock::new(config)), snapshots: Arc::new(RwLock::new(Vec::new())), catalog: Arc::new(RwLock::new(catalog)), store, upstream_client, + models_etag: Arc::new(RwLock::new(None)), + etag_change_tx, + refresh_lock: Arc::new(tokio::sync::Mutex::new(())), } } + /// 记录 Responses 响应中观察到的模型目录 ETag。 + /// + /// 仅在值真正变化时通知后台刷新任务。 + pub fn observe_models_etag(&self, etag: Option<&str>) -> bool { + let Some(etag) = etag.map(str::trim).filter(|etag| !etag.is_empty()) else { + return false; + }; + let mut current = self + .models_etag + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.as_deref() == Some(etag) { + return false; + } + *current = Some(etag.to_string()); + drop(current); + self.etag_change_tx.send_modify(|revision| *revision += 1); + true + } + + /// 订阅模型目录 ETag 变化。 + pub fn subscribe_models_etag_changes(&self) -> watch::Receiver { + self.etag_change_tx.subscribe() + } + /// 更新模型服务配置。 pub fn update_config(&self, config: ModelConfig) { *self @@ -150,13 +184,13 @@ impl ModelService { self.cached_catalog() } - /// 使用运行时 installation id 刷新活跃账号对应的后端模型目录。 - pub async fn refresh_backend_models_with_installation_id( + /// 刷新活跃账号对应的后端模型目录。 + pub async fn refresh_backend_models( &self, plan_accounts: &[ModelRefreshPlanAccount], request_id: &str, - installation_id: Option<&str>, ) -> Result { + let _refresh_guard = self.refresh_lock.lock().await; let store = self .store .as_ref() @@ -181,7 +215,7 @@ impl ModelService { access_token: &plan_account.access_token, account_id: plan_account.account_id.as_deref(), request_id, - installation_id, + installation_id: Some(&plan_account.installation_id), plan_type: &plan_account.plan_type, }; diff --git a/backend/src/telemetry/usage/types.rs b/backend/src/telemetry/usage/types.rs index 2877e8281..8eb6e1a7a 100644 --- a/backend/src/telemetry/usage/types.rs +++ b/backend/src/telemetry/usage/types.rs @@ -57,7 +57,7 @@ pub struct UsageRecord { pub upstream_request_id: Option, /// 请求耗时毫秒。 pub latency_ms: Option, - /// 首 token 延迟毫秒。 + /// 首个有效输出事件延迟毫秒。 pub first_token_ms: Option, /// 输入 token。 pub input_tokens: Option, diff --git a/backend/src/upstream/openai/protocol/events.rs b/backend/src/upstream/openai/protocol/events.rs index ff68282cb..3aa90a19c 100644 --- a/backend/src/upstream/openai/protocol/events.rs +++ b/backend/src/upstream/openai/protocol/events.rs @@ -166,9 +166,13 @@ pub struct RateLimitWindow { pub reset_at: Option, } -/// 复合限流信息，通常用于 code review 配额。 -#[derive(Debug, Clone, Copy, PartialEq)] +/// 单个计量项的限流信息。 +#[derive(Debug, Clone, PartialEq)] pub struct RateLimitDetails { + /// 稳定的计量项 ID。 + pub limit_id: String, + /// 上游提供的可读名称。 + pub limit_name: Option, /// 当前请求是否被允许。 pub allowed: Option, /// 当前窗口是否已经触顶。 @@ -179,15 +183,23 @@ pub struct RateLimitDetails { pub secondary: Option, } +/// 上游账户的 credits 快照。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreditsSnapshot { + pub has_credits: bool, + pub unlimited: bool, + pub balance: Option, +} + /// 从 header 或内部事件中解析出的完整限流状态。 -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, PartialEq)] pub struct ParsedRateLimits { - /// 主限流窗口。 - pub primary: Option, - /// 次级限流窗口。 - pub secondary: Option, - /// code review 限流窗口。 - pub code_review: Option, + /// 以标准化 limit ID 为键的全部计量项。 + pub limits: BTreeMap, + pub credits: Option, + pub plan_type: Option, + pub promo_message: Option, + pub rate_limit_reached_type: Option, } /// 从响应头对中解析限流信息。 @@ -197,23 +209,60 @@ pub fn parse_rate_limit_headers(headers: &[(String, String)]) -> Option>(); + limit_ids.insert("codex".to_string()); + let limits = limit_ids + .into_iter() + .filter_map(|limit_id| { + let prefix = format!("x-{}", limit_id.replace('_', "-")); + parse_details_from_lookup(&normalized, &prefix, &limit_id) + .map(|details| (limit_id, details)) + }) + .collect::>(); + let credits = parse_credits_from_lookup(&normalized); + let plan_type = lookup_non_empty(&normalized, "x-codex-plan-type"); + let promo_message = lookup_non_empty(&normalized, "x-codex-promo-message"); + let rate_limit_reached_type = lookup_non_empty(&normalized, "x-codex-rate-limit-reached-type"); - if primary.is_none() && secondary.is_none() && code_review.is_none() { + if limits.is_empty() + && credits.is_none() + && plan_type.is_none() + && promo_message.is_none() + && rate_limit_reached_type.is_none() + { return None; } Some(ParsedRateLimits { - primary, - secondary, - code_review, + limits, + credits, + plan_type, + promo_message, + rate_limit_reached_type, }) } +/// 判断响应头是否属于可被动同步的限流领域。 +pub(crate) fn is_rate_limit_header_name(name: &str) -> bool { + let normalized = name.to_ascii_lowercase(); + normalized == "retry-after" + || normalized.contains("ratelimit") + || normalized.contains("rate-limit") + || matches!( + normalized.as_str(), + "x-codex-credits-has-credits" + | "x-codex-credits-unlimited" + | "x-codex-credits-balance" + | "x-codex-plan-type" + | "x-codex-promo-message" + | "x-codex-rate-limit-reached-type" + ) + || rate_limit_id_from_header_name(&normalized).is_some() +} + /// 从内部 `codex.rate_limits` 事件中解析限流信息。 pub fn parse_rate_limits_event(value: &Value) -> Option { if value @@ -224,33 +273,52 @@ pub fn parse_rate_limits_event(value: &Value) -> Option { return None; } - let details = value.get("rate_limits").and_then(parse_details_from_object); - let explicit_code_review = value - .get("code_review_rate_limits") - .and_then(parse_details_from_object) - .or_else(|| { - value - .get("code_review_rate_limit") - .and_then(parse_details_from_object) - }); + let limit_id = rate_limit_name(value) + .map(normalize_limit_id) + .unwrap_or_else(|| "codex".to_string()); + let limit_name = value + .get("metered_limit_name") + .and_then(Value::as_str) + .and_then(|_| value.get("limit_name")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(ToString::to_string); + let details = value + .get("rate_limits") + .and_then(|details| parse_details_from_object(details, &limit_id, limit_name)); + let credits = value.get("credits").and_then(parse_credits_from_object); + let plan_type = value + .get("plan_type") + .and_then(Value::as_str) + .map(str::trim) + .filter(|plan| !plan.is_empty()) + .map(ToString::to_string); - let mut primary = details.and_then(|details| details.primary); - let mut secondary = details.and_then(|details| details.secondary); - let mut code_review = explicit_code_review; - if details.is_some() && is_review_limit_name(rate_limit_name(value)) { - code_review = code_review.or(details); - primary = None; - secondary = None; - } - - if primary.is_none() && secondary.is_none() && code_review.is_none() { + if details.is_none() && credits.is_none() && plan_type.is_none() { return None; } + let mut limits = BTreeMap::new(); + if let Some(details) = details { + limits.insert(limit_id, details); + } Some(ParsedRateLimits { - primary, - secondary, - code_review, + limits, + credits, + plan_type, + promo_message: value + .get("promo_message") + .and_then(Value::as_str) + .map(str::trim) + .filter(|message| !message.is_empty()) + .map(ToString::to_string), + rate_limit_reached_type: value + .get("rate_limit_reached_type") + .and_then(Value::as_str) + .map(str::trim) + .filter(|kind| !kind.is_empty()) + .map(ToString::to_string), }) } @@ -267,41 +335,35 @@ pub fn rate_limit_quota( plan_type: Option<&str>, existing_quota: Option<&Value>, ) -> Value { - let mut snapshots = Vec::new(); - if let Some(snapshot) = quota_snapshot_from_windows( - "core", - None, - None, - rate_limits.primary, - rate_limits.secondary, - None, - None, - ) { - snapshots.push(snapshot); - } - - let has_code_review_update = rate_limits.code_review.is_some(); - if let Some(details) = rate_limits.code_review { - if let Some(snapshot) = quota_snapshot_from_windows( - "code_review", - Some("code_review"), - None, - details.primary, - details.secondary, - details.allowed, - details.limit_reached, - ) { - snapshots.push(snapshot); - } - } + let mut snapshots = rate_limits + .limits + .values() + .filter_map(|details| { + let (source, metered_feature) = quota_identity(&details.limit_id); + let limit_name = details + .limit_name + .as_deref() + .or_else(|| (source != "core").then_some(details.limit_id.as_str())); + quota_snapshot_from_windows( + source, + limit_name, + metered_feature, + details.primary, + details.secondary, + details.allowed, + details.limit_reached, + ) + }) + .collect::>(); if let Some(existing_quota) = existing_quota { if let Some(existing_snapshots) = existing_quota.get("snapshots").and_then(Value::as_array) { for snapshot in existing_snapshots { - let source = snapshot.get("source").and_then(Value::as_str); - if source == Some("additional") - || (!has_code_review_update && source == Some("code_review")) + if !rate_limits + .limits + .keys() + .any(|limit_id| quota_snapshot_matches_limit(snapshot, limit_id)) { snapshots.push(snapshot.clone()); } @@ -312,42 +374,96 @@ pub fn rate_limit_quota( let monthly_limit = monthly_limit_from_snapshots(&snapshots) .or_else(|| existing_quota.and_then(|quota| quota.get("monthly_limit").cloned())) .unwrap_or(Value::Null); - let credits = existing_quota - .and_then(|quota| quota.get("credits").cloned()) + let credits = rate_limits + .credits + .as_ref() + .map(|credits| { + json!({ + "has_credits": credits.has_credits, + "unlimited": credits.unlimited, + "balance": credits.balance, + }) + }) + .or_else(|| existing_quota.and_then(|quota| quota.get("credits").cloned())) .unwrap_or(Value::Null); let spend_control = existing_quota .and_then(|quota| quota.get("spend_control").cloned()) .unwrap_or(Value::Null); json!({ - "plan_type": plan_type.unwrap_or("unknown"), + "plan_type": rate_limits.plan_type.as_deref().or(plan_type).unwrap_or("unknown"), "snapshots": snapshots, "monthly_limit": monthly_limit, "credits": credits, "spend_control": spend_control, + "promo_message": rate_limits.promo_message, + "rate_limit_reached_type": rate_limits.rate_limit_reached_type, }) } -/// 将限流状态转换回 HTTP 头键值对。 +/// 将限流状态转换回内部传输头键值对。 pub fn rate_limits_to_header_pairs(rate_limits: &ParsedRateLimits) -> Vec<(String, String)> { let mut headers = Vec::new(); - push_window_headers(&mut headers, "x-codex-primary", rate_limits.primary); - push_window_headers(&mut headers, "x-codex-secondary", rate_limits.secondary); - if let Some(code_review) = rate_limits.code_review { + for details in rate_limits.limits.values() { + let prefix = format!("x-{}", details.limit_id.replace('_', "-")); + push_window_headers(&mut headers, &format!("{prefix}-primary"), details.primary); push_window_headers( &mut headers, - "x-codex-code-review-primary", - code_review.primary, - ); - push_window_headers( - &mut headers, - "x-codex-code-review-secondary", - code_review.secondary, + &format!("{prefix}-secondary"), + details.secondary, ); + if let Some(limit_name) = &details.limit_name { + headers.push((format!("{prefix}-limit-name"), limit_name.clone())); + } + } + if let Some(credits) = &rate_limits.credits { + headers.push(( + "x-codex-credits-has-credits".to_string(), + credits.has_credits.to_string(), + )); + headers.push(( + "x-codex-credits-unlimited".to_string(), + credits.unlimited.to_string(), + )); + if let Some(balance) = &credits.balance { + headers.push(("x-codex-credits-balance".to_string(), balance.clone())); + } + } + if let Some(plan_type) = &rate_limits.plan_type { + headers.push(("x-codex-plan-type".to_string(), plan_type.clone())); + } + if let Some(promo_message) = &rate_limits.promo_message { + headers.push(("x-codex-promo-message".to_string(), promo_message.clone())); + } + if let Some(reached_type) = &rate_limits.rate_limit_reached_type { + headers.push(( + "x-codex-rate-limit-reached-type".to_string(), + reached_type.clone(), + )); } headers } +fn quota_identity(limit_id: &str) -> (&'static str, Option<&str>) { + if limit_id == "codex" { + ("core", None) + } else if is_review_limit_name(Some(limit_id)) { + ("code_review", Some(limit_id)) + } else { + ("additional", Some(limit_id)) + } +} + +fn quota_snapshot_matches_limit(snapshot: &Value, limit_id: &str) -> bool { + let (source, metered_feature) = quota_identity(limit_id); + if snapshot.get("source").and_then(Value::as_str) != Some(source) { + return false; + } + source == "core" + || snapshot.get("metered_feature").and_then(Value::as_str) == metered_feature + || snapshot.get("limit_name").and_then(Value::as_str) == Some(limit_id) +} + fn number_field(value: &Value, field: &str) -> Option { value.get(field)?.as_u64() } @@ -464,20 +580,88 @@ fn push_window_headers( fn parse_details_from_lookup( headers: &BTreeMap, prefix: &str, + limit_id: &str, ) -> Option { let primary = parse_window_from_lookup(headers, &format!("{prefix}-primary")); let secondary = parse_window_from_lookup(headers, &format!("{prefix}-secondary")); - if primary.is_none() && secondary.is_none() { + let limit_name = lookup_non_empty(headers, &format!("{prefix}-limit-name")); + let allowed = lookup_bool(headers, &format!("{prefix}-allowed")); + let limit_reached = lookup_bool(headers, &format!("{prefix}-limit-reached")); + if primary.is_none() + && secondary.is_none() + && limit_name.is_none() + && allowed.is_none() + && limit_reached.is_none() + { return None; } Some(RateLimitDetails { - allowed: None, - limit_reached: None, + limit_id: limit_id.to_string(), + limit_name, + allowed, + limit_reached, primary, secondary, }) } +fn rate_limit_id_from_header_name(name: &str) -> Option { + const SUFFIXES: [&str; 9] = [ + "-primary-used-percent", + "-primary-window-minutes", + "-primary-reset-at", + "-secondary-used-percent", + "-secondary-window-minutes", + "-secondary-reset-at", + "-limit-name", + "-allowed", + "-limit-reached", + ]; + let name = name.strip_prefix("x-")?; + let raw_id = SUFFIXES + .iter() + .find_map(|suffix| name.strip_suffix(suffix))?; + (!raw_id.is_empty()).then(|| normalize_limit_id(raw_id)) +} + +fn parse_credits_from_lookup(headers: &BTreeMap) -> Option { + Some(CreditsSnapshot { + has_credits: lookup_bool(headers, "x-codex-credits-has-credits")?, + unlimited: lookup_bool(headers, "x-codex-credits-unlimited")?, + balance: lookup_non_empty(headers, "x-codex-credits-balance"), + }) +} + +fn parse_credits_from_object(value: &Value) -> Option { + Some(CreditsSnapshot { + has_credits: value.get("has_credits")?.as_bool()?, + unlimited: value.get("unlimited")?.as_bool()?, + balance: value.get("balance").and_then(|balance| match balance { + Value::String(balance) => Some(balance.clone()), + Value::Number(balance) => Some(balance.to_string()), + _ => None, + }), + }) +} + +fn lookup_non_empty(headers: &BTreeMap, name: &str) -> Option { + headers + .get(name) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) +} + +fn lookup_bool(headers: &BTreeMap, name: &str) -> Option { + match headers.get(name)?.trim() { + "1" => Some(true), + "0" => Some(false), + value if value.eq_ignore_ascii_case("true") => Some(true), + value if value.eq_ignore_ascii_case("false") => Some(false), + _ => None, + } +} + fn parse_window_from_lookup( headers: &BTreeMap, prefix: &str, @@ -499,15 +683,23 @@ fn parse_window_from_lookup( }) } -fn parse_details_from_object(value: &Value) -> Option { +fn parse_details_from_object( + value: &Value, + limit_id: &str, + limit_name: Option, +) -> Option { let primary = value.get("primary").and_then(parse_window_from_object); let secondary = value.get("secondary").and_then(parse_window_from_object); - if primary.is_none() && secondary.is_none() { + let allowed = value.get("allowed").and_then(Value::as_bool); + let limit_reached = value.get("limit_reached").and_then(Value::as_bool); + if primary.is_none() && secondary.is_none() && allowed.is_none() && limit_reached.is_none() { return None; } Some(RateLimitDetails { - allowed: value.get("allowed").and_then(Value::as_bool), - limit_reached: value.get("limit_reached").and_then(Value::as_bool), + limit_id: limit_id.to_string(), + limit_name, + allowed, + limit_reached, primary, secondary, }) @@ -645,6 +837,10 @@ fn rate_limit_name(value: &Value) -> Option<&str> { .and_then(Value::as_str) } +fn normalize_limit_id(value: impl AsRef) -> String { + value.as_ref().trim().to_ascii_lowercase().replace('-', "_") +} + fn is_review_limit_name(value: Option<&str>) -> bool { let Some(value) = value else { return false; diff --git a/backend/src/upstream/openai/protocol/responses.rs b/backend/src/upstream/openai/protocol/responses.rs index 692f852ad..45aacde85 100644 --- a/backend/src/upstream/openai/protocol/responses.rs +++ b/backend/src/upstream/openai/protocol/responses.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeSet, time::Instant}; use serde::Serialize; -use serde_json::{json, Map, Value}; +use serde_json::{Map, Value}; use crate::{ infra::time::elapsed_millis_i64, @@ -16,8 +16,8 @@ use crate::{ /// 发往上游的 Responses 请求。`body` 持有客户端原始 JSON object，逐字段（含顺序、 /// 含未知字段）透传上游，是上游请求体的唯一来源；`use_websocket`/`force_http_sse` /// 仅用于本地传输选择，不写入 body。常用字段通过访问器方法读写。 -/// 代理只做最小 patch（`stream`/`store` 由传输层控制，`input` 字符串做官方兼容转换， -/// implicit resume 时替换 `input`，模型后缀路由时写 `model`/`reasoning`/`service_tier`）。 +/// 普通客户端请求不修改 body；仅在代理持有完整历史并换号重放时替换历史字段， +/// 或在模型后缀路由时写入 `model`/`reasoning`/`service_tier`。 /// /// 其余字段是代理控制状态，不进上游 body（原 `#[serde(skip)]` 字段）。 #[derive(Debug, Clone)] @@ -30,6 +30,16 @@ pub struct CodexResponsesRequest { pub explicit_prompt_cache_key: bool, /// 客户端会话 ID。 pub client_conversation_id: Option, + /// 客户端 session ID，仅保留在受控本地上下文。 + pub client_session_id: Option, + /// 客户端 thread ID，仅保留在受控本地上下文。 + pub client_thread_id: Option, + /// 客户端 request ID，仅保留在受控本地上下文。 + pub client_request_id: Option, + /// 客户端 turn ID，仅保留在受控本地上下文。 + pub client_turn_id: Option, + /// 连接池和 affinity 使用的本地会话身份，不发送上游。 + pub local_conversation_id: Option, /// 变体身份键。 pub variant_identity: Option, /// 代理侧识别的客户端 IP，仅用于管理端使用记录展示。 @@ -56,6 +66,26 @@ pub struct CodexResponsesRequest { pub codex_window_id: Option, /// 父线程 ID。 pub parent_thread_id: Option, + /// 已知 previous response 的持久化范围，仅用于本地 transport 校验。 + pub previous_response_scope: Option, + /// 流式响应在交给下游前的本地提交策略。 + pub stream_commit_policy: StreamCommitPolicy, +} + +/// previous response 在上游的可续接范围。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PreviousResponseScope { + Persisted, + ConnectionLocal, + ExternalUnknown, +} + +/// 流式请求的可逆提交边界。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamCommitPolicy { + FirstForwardableEvent, + UntilOutputOrTerminal, } impl Serialize for CodexResponsesRequest { @@ -114,7 +144,7 @@ pub fn response_body_has_first_output(body_bytes: &[u8]) -> bool { .is_ok_and(|events| events.iter().any(response_sse_event_has_first_output)) } -/// 已收到首个有效 Responses SSE 输出事件时记录首 token 耗时。 +/// 已收到首个有效 Responses SSE 输出事件时记录耗时。 pub fn update_first_response_output_ms( started_at: Instant, body_bytes: &[u8], @@ -180,8 +210,10 @@ fn complete_sse_body_prefix(body: &str) -> Option<&str> { /// 从 Codex SSE 收集出的非流式 Responses 结果。 #[derive(Debug, Clone, PartialEq)] pub enum CollectedResponse { - /// 收集到 `response.completed` 或 `response.incomplete`。 + /// 收集到 `response.completed`。 Completed(Value), + /// 收集到 `response.incomplete`。 + Incomplete(Value), /// 收集到 `response.failed` 或 `error`。 Failed(ResponsesSseFailure), /// SSE 未包含成功终止响应。 @@ -218,8 +250,8 @@ pub struct CompletedResponseMetadata { pub response_id: String, /// 完成响应中的 function call ids。 pub function_call_ids: Vec, - /// 可用于 reasoning replay 的 output items。 - pub replay_items: Vec, + /// 完成响应的完整 output items。 + pub output: Vec, } /// 将 Codex Responses SSE 完成响应收集为非流式 Responses JSON。 @@ -231,6 +263,7 @@ pub fn response_from_codex_sse( let mut output_text = String::new(); let mut output_items = Vec::new(); let mut terminal_response = None; + let mut terminal_incomplete = false; let mut failed_response = None; for event in events { @@ -248,8 +281,13 @@ pub fn response_from_codex_sse( output_items.push(item.clone()); } } - Some("response.completed" | "response.incomplete") => { + Some("response.completed") => { terminal_response = value.get("response").cloned(); + terminal_incomplete = false; + } + Some("response.incomplete") => { + terminal_response = value.get("response").cloned(); + terminal_incomplete = true; } Some(event_name @ ("error" | "response.failed")) if failed_response.is_none() => { failed_response = Some(ResponsesSseFailure::from_event(event_name, &value)); @@ -264,13 +302,15 @@ pub fn response_from_codex_sse( let Some(mut response) = terminal_response else { return Ok(CollectedResponse::MissingCompleted); }; + if terminal_incomplete { + reconvert_completed_response_tuple_values(&mut response, tuple_schema); + return Ok(CollectedResponse::Incomplete(response)); + } if is_empty_response(&response, &output_text, &output_items) { return Ok(CollectedResponse::Empty); } - ensure_completed_response_output(&mut response, &output_items, &output_text); reconvert_completed_response_tuple_values(&mut response, tuple_schema); - sync_output_text_from_output(&mut response); Ok(CollectedResponse::Completed(response)) } @@ -281,7 +321,8 @@ pub fn completed_response_metadata( let events = parse_sse_events(body)?; let mut response_id = None; let mut function_call_ids = BTreeSet::new(); - let mut replay_items = Vec::new(); + let mut output = Vec::new(); + let mut completed_items = Vec::new(); for event in events { let Ok(value) = serde_json::from_str::(&event.data) else { @@ -290,22 +331,25 @@ pub fn completed_response_metadata( match event.event.as_deref() { Some("response.output_item.done") => { if let Some(item) = value.get("item") { - collect_response_replay_items(item, &mut replay_items); + completed_items.push(item.clone()); } if let Some(call_id) = value.pointer("/item/call_id").and_then(Value::as_str) { function_call_ids.insert(call_id.to_string()); } } - Some("response.completed" | "response.incomplete") => { + Some("response.completed") => { response_id = value .pointer("/response/id") .and_then(Value::as_str) .map(ToString::to_string); collect_response_function_call_ids(&value, &mut function_call_ids); - if let Some(output) = value.pointer("/response/output").and_then(Value::as_array) { - for item in output { - collect_response_replay_items(item, &mut replay_items); - } + if let Some(completed_output) = + value.pointer("/response/output").and_then(Value::as_array) + { + completed_output.clone_into(&mut output); + } + if output.is_empty() { + output.clone_from(&completed_items); } } _ => {} @@ -315,7 +359,7 @@ pub fn completed_response_metadata( Ok(response_id.map(|response_id| CompletedResponseMetadata { response_id, function_call_ids: function_call_ids.into_iter().collect(), - replay_items, + output, })) } @@ -337,7 +381,6 @@ pub fn reconvert_responses_sse_event_tuple_values( Some("response.completed" | "response.incomplete") => { if let Some(response) = data.get_mut("response") { reconvert_completed_response_tuple_values(response, Some(tuple_schema)); - sync_output_text_from_output(response); } } _ => {} @@ -390,43 +433,6 @@ fn is_empty_response(response: &Value, output_text: &str, output_items: &[Value] == 0 } -fn ensure_completed_response_output( - response: &mut Value, - output_items: &[Value], - output_text: &str, -) { - let output_is_empty = response - .get("output") - .and_then(Value::as_array) - .is_none_or(Vec::is_empty); - if !output_is_empty { - return; - } - - if !output_items.is_empty() { - response["output"] = Value::Array(output_items.to_vec()); - return; - } - if output_text.is_empty() { - return; - } - - let item_status = response - .get("status") - .and_then(Value::as_str) - .unwrap_or("completed"); - response["output"] = json!([{ - "type": "message", - "status": item_status, - "role": "assistant", - "content": [{ - "type": "output_text", - "text": output_text, - "annotations": [] - }] - }]); -} - fn reconvert_completed_response_tuple_values(response: &mut Value, tuple_schema: Option<&Value>) { let Some(tuple_schema) = tuple_schema else { return; @@ -459,20 +465,6 @@ fn reconvert_output_item_tuple_values(item: &mut Value, tuple_schema: &Value) { } } -fn sync_output_text_from_output(response: &mut Value) { - let Some(items) = response.get("output").and_then(Value::as_array) else { - return; - }; - let texts = items - .iter() - .filter_map(output_text_from_item) - .collect::>(); - if texts.is_empty() { - return; - } - response["output_text"] = Value::String(texts.join("\n\n")); -} - fn collect_response_function_call_ids(value: &Value, function_call_ids: &mut BTreeSet) { let Some(output) = value.pointer("/response/output").and_then(Value::as_array) else { return; @@ -484,30 +476,6 @@ fn collect_response_function_call_ids(value: &Value, function_call_ids: &mut BTr } } -fn collect_response_replay_items(item: &Value, replay_items: &mut Vec) { - if matches!( - item.get("type").and_then(Value::as_str), - Some("reasoning" | "function_call") - ) { - replay_items.push(item.clone()); - } -} - -fn output_text_from_item(item: &Value) -> Option { - let content = item.get("content")?.as_array()?; - let text = content - .iter() - .filter_map(|part| { - let part_type = part.get("type")?.as_str()?; - if part_type != "output_text" && part_type != "text" { - return None; - } - part.get("text")?.as_str() - }) - .collect::(); - (!text.is_empty()).then_some(text) -} - fn failure_message(value: &Value) -> Option { value .pointer("/response/error/message") @@ -531,18 +499,19 @@ fn failure_code(value: &Value) -> Option { impl CodexResponsesRequest { /// 从客户端原始 Responses JSON object 构造上游请求。 /// - /// 只做最小规范化：`input` 缺省为 `[]`、`stream` 缺省 `true`、`store` 缺省 `false`。 - /// 其余字段（含未知字段）原样保留在 `body` 中透传上游。 - pub fn from_body(mut body: Map) -> Self { - body.entry("input") - .or_insert_with(|| Value::Array(Vec::new())); - body.entry("stream").or_insert(Value::Bool(true)); - body.entry("store").or_insert(Value::Bool(false)); + /// 客户端提供的字段（含未知字段）原样保留在 `body` 中透传上游。 + /// 协议默认值仅由类型化访问器在本地解释，不写回上游正文。 + pub fn from_body(body: Map) -> Self { Self { body, tuple_schema: None, explicit_prompt_cache_key: false, client_conversation_id: None, + client_session_id: None, + client_thread_id: None, + client_request_id: None, + client_turn_id: None, + local_conversation_id: None, variant_identity: None, client_ip: None, client_user_agent: None, @@ -556,6 +525,8 @@ impl CodexResponsesRequest { include_timing_metrics: None, codex_window_id: None, parent_thread_id: None, + previous_response_scope: None, + stream_commit_policy: StreamCommitPolicy::FirstForwardableEvent, } } @@ -612,7 +583,7 @@ impl CodexResponsesRequest { .map_or(&[], Vec::as_slice) } - /// 替换输入条目（implicit resume / 字符串输入兼容）。 + /// 替换输入条目（仅用于代理持有完整历史时的换号重放）。 pub fn set_input(&mut self, input: Vec) { self.body.insert("input".to_string(), Value::Array(input)); } @@ -638,6 +609,11 @@ impl CodexResponsesRequest { .unwrap_or(false) } + /// 设置上游存储标志。 + pub fn set_store(&mut self, store: bool) { + self.body.insert("store".to_string(), Value::Bool(store)); + } + /// reasoning 配置（透传，不规整）。 pub fn reasoning(&self) -> Option<&Value> { self.body.get("reasoning") @@ -691,6 +667,7 @@ impl CodexResponsesRequest { } None => { self.body.remove("previous_response_id"); + self.previous_response_scope = None; } } } @@ -730,6 +707,22 @@ impl CodexResponsesRequest { } } + /// 仅在客户端原本提供字符串字段时替换其账号作用域 wire 值。 + pub(crate) fn replace_existing_identity_field(&mut self, key: &str, value: Option<&str>) { + if !self.body.get(key).is_some_and(Value::is_string) { + return; + } + match value.filter(|value| !value.trim().is_empty()) { + Some(value) => { + self.body + .insert(key.to_string(), Value::String(value.to_string())); + } + None => { + self.body.remove(key); + } + } + } + /// 判断请求是否声明了图片生成工具。 pub fn expects_image_generation(&self) -> bool { self.tools().is_some_and(|tools| { @@ -755,6 +748,18 @@ pub struct CodexCompactRequest { pub client_user_agent: Option, /// 已鉴权客户端 API key 的稳定 ID，仅用于事实归因。 pub client_api_key_id: Option, + /// 客户端 session ID，仅保留在本地身份上下文。 + pub client_session_id: Option, + /// 客户端 thread ID，仅保留在本地身份上下文。 + pub client_thread_id: Option, + /// 客户端 request ID，仅保留在本地身份上下文。 + pub client_request_id: Option, + /// 客户端 turn ID，仅保留在本地身份上下文。 + pub client_turn_id: Option, + /// 客户端 window ID，仅保留在本地身份上下文。 + pub client_window_id: Option, + /// 客户端 parent thread ID，仅保留在本地身份上下文。 + pub client_parent_thread_id: Option, } impl Serialize for CodexCompactRequest { @@ -785,4 +790,8 @@ impl CodexCompactRequest { pub fn reasoning(&self) -> Option<&Value> { self.body.get("reasoning") } + + pub fn prompt_cache_key(&self) -> Option<&str> { + self.body.get("prompt_cache_key").and_then(Value::as_str) + } } diff --git a/backend/src/upstream/openai/protocol/websocket.rs b/backend/src/upstream/openai/protocol/websocket.rs index 922b4eadb..1b7e2e951 100644 --- a/backend/src/upstream/openai/protocol/websocket.rs +++ b/backend/src/upstream/openai/protocol/websocket.rs @@ -51,16 +51,26 @@ pub struct WebSocketAuditArtifact { } /// 从 WebSocket 错误帧推导出的上游错误分类。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ClassifiedWebSocketError { /// 应映射给 HTTP 层的状态码。 pub status_code: u16, + /// 上游错误类型。 + pub error_type: Option, + /// 上游错误码。 + pub code: Option, + /// 上游错误消息。 + pub message: Option, + /// 上游错误参数。 + pub param: Option, + /// 错误帧携带的安全响应头。 + pub headers: Vec<(String, String)>, } #[derive(Debug, Deserialize)] struct ResponseCompleted { #[serde(rename = "id")] - _id: String, + id: String, #[serde(default, rename = "usage")] _usage: Option, #[serde(default, rename = "end_turn")] @@ -148,26 +158,34 @@ fn is_internal_websocket_event(event: &str) -> bool { /// 会话状态，不下发客户端。业务事件一律原样转发，不做 schema 校验。 /// （`codex.rate_limits` 内部事件由 `is_internal_websocket_event` 单独剥离。） fn websocket_event_should_skip(raw: &str) -> bool { - websocket_metadata_turn_state(raw).is_some() + websocket_event_type(raw).as_deref() == Some("response.metadata") +} + +/// 提取 `response.metadata` 帧中的字符串响应头。 +pub fn websocket_metadata_headers(raw: &str) -> Vec<(String, String)> { + let Ok(value) = serde_json::from_str::(raw) else { + return Vec::new(); + }; + if value.get("type").and_then(Value::as_str) != Some("response.metadata") { + return Vec::new(); + } + value + .get("headers") + .and_then(Value::as_object) + .into_iter() + .flat_map(|headers| headers.iter()) + .filter_map(|(name, value)| json_value_as_string(value).map(|value| (name.clone(), value))) + .collect() } /// 从 `response.metadata` 帧中提取 `x-codex-turn-state`。 pub fn websocket_metadata_turn_state(raw: &str) -> Option { let value = serde_json::from_str::(raw).ok()?; - if value.get("type").and_then(Value::as_str) != Some("response.metadata") { - return None; - } - value - .get("headers") - .and_then(Value::as_object) - .and_then(|headers| { - headers.iter().find_map(|(name, value)| { - if name.eq_ignore_ascii_case("x-codex-turn-state") { - json_value_as_string(value) - } else { - None - } - }) + websocket_metadata_headers(raw) + .into_iter() + .find_map(|(name, value)| { + name.eq_ignore_ascii_case("x-codex-turn-state") + .then_some(value) }) .or_else(|| { value @@ -186,34 +204,22 @@ fn json_value_as_string(value: &Value) -> Option { } } -/// 校验 `response.completed` 的官方响应形状。 -pub fn websocket_response_completed_parse_error(raw: &str) -> Option { - let value = serde_json::from_str::(raw).ok()?; +/// 校验 `response.completed` 的官方响应形状并提取 response ID。 +pub fn websocket_response_completed_id(raw: &str) -> Result, String> { + let value = serde_json::from_str::(raw).map_err(|error| error.to_string())?; if value.get("type").and_then(Value::as_str) != Some("response.completed") { - return None; + return Ok(None); } - let response = value.get("response")?; - if response.is_null() { - return None; + let response = value + .get("response") + .filter(|response| !response.is_null()) + .ok_or_else(|| "response.completed is missing response".to_string())?; + let completed = serde_json::from_value::(response.clone()) + .map_err(|error| format!("failed to parse ResponseCompleted: {error}"))?; + if completed.id.trim().is_empty() { + return Err("response.completed contains an empty response id".to_string()); } - match serde_json::from_value::(response.clone()) { - Ok(_) => None, - Err(error) => Some(format!("failed to parse ResponseCompleted: {error}")), - } -} - -/// 从 `response.incomplete` 中提取 incomplete reason。 -pub fn websocket_incomplete_response_reason(raw: &str) -> Option { - let value = serde_json::from_str::(raw).ok()?; - if value.get("type").and_then(Value::as_str) != Some("response.incomplete") { - return None; - } - value - .pointer("/response/incomplete_details/reason") - .or_else(|| value.pointer("/incomplete_details/reason")) - .or_else(|| value.get("reason")) - .and_then(Value::as_str) - .map(ToString::to_string) + Ok(Some(completed.id)) } #[path = "websocket_errors.rs"] @@ -222,10 +228,13 @@ pub use errors::*; /// 判断 WebSocket 事件是否会结束当前响应流。 pub fn is_terminal_websocket_event(event: &str) -> bool { - matches!(event, "response.completed" | "response.failed" | "error") + matches!( + event, + "response.completed" | "response.incomplete" | "response.failed" | "error" + ) } -/// 将 WebSocket 错误帧映射为上游 HTTP 状态分类。 +/// 将 WebSocket 错误帧映射为结构化上游错误。 pub fn classify_websocket_error_frame(raw: &str) -> Option { let value = serde_json::from_str::(raw).ok()?; let event_type = value.get("type").and_then(Value::as_str)?; @@ -233,37 +242,49 @@ pub fn classify_websocket_error_frame(raw: &str) -> Option Option { +fn websocket_error_value(value: &Value) -> Option<&Map> { value - .pointer("/response/error/code") - .or_else(|| value.pointer("/response/error/type")) - .or_else(|| value.pointer("/error/code")) - .or_else(|| value.pointer("/error/type")) - .and_then(Value::as_str) - .map(str::to_ascii_lowercase) + .pointer("/response/error") + .or_else(|| value.get("error")) + .and_then(Value::as_object) } fn explicit_error_status_code(value: &Value) -> Option { @@ -274,17 +295,42 @@ fn explicit_error_status_code(value: &Value) -> Option { .and_then(|status| u16::try_from(status).ok()) } -fn rotatable_error_status_code(code: &str) -> Option { +fn error_status_code(code: &str) -> Option { match code { "usage_limit_reached" | "rate_limit_exceeded" | "rate_limit_reached" => Some(429), "quota_exhausted" | "payment_required" => Some(402), "unauthorized" | "token_invalid" | "token_expired" | "account_deactivated" => Some(401), "forbidden" | "account_banned" | "banned" => Some(403), - "previous_response_not_found" => Some(400), + "previous_response_not_found" + | "invalid_encrypted_content" + | "missing_tool_output" + | "no_tool_output" => Some(400), + "websocket_connection_limit_reached" => Some(503), _ => None, } } +fn error_type_status_code(error_type: &str) -> Option { + match error_type { + "invalid_request_error" => Some(400), + "authentication_error" => Some(401), + "permission_error" => Some(403), + "rate_limit_error" => Some(429), + "server_error" | "api_error" => Some(500), + _ => None, + } +} + +fn websocket_error_headers(value: &Value) -> Vec<(String, String)> { + value + .get("headers") + .and_then(Value::as_object) + .into_iter() + .flatten() + .filter_map(|(name, value)| json_value_as_string(value).map(|value| (name.clone(), value))) + .collect() +} + /// 从包裹在 WebSocket `error.headers` 中的 retry-after 值提取秒数。 pub fn retry_after_seconds_from_wrapped_error_headers(raw: &str) -> Option { let value = serde_json::from_str::(raw).ok()?; diff --git a/backend/src/upstream/openai/transport/client.rs b/backend/src/upstream/openai/transport/client.rs index b1df57eb4..604715e2d 100644 --- a/backend/src/upstream/openai/transport/client.rs +++ b/backend/src/upstream/openai/transport/client.rs @@ -20,7 +20,6 @@ use reqwest::{ use serde_json::{map::Map, Value}; use thiserror::Error; use tokio_tungstenite::tungstenite::handshake::client::generate_key; -use uuid::Uuid; use crate::upstream::openai::fingerprint::{Fingerprint, RuntimeFingerprint}; use crate::upstream::openai::protocol::events::{extract_sse_usage, retry_after_seconds_from_body}; @@ -47,7 +46,7 @@ use super::websocket::{ CodexWebSocketRateLimitHeaderUpdates, CodexWebSocketTurnStateUpdate, }; use super::websocket_pool::{ - CodexWebSocketPool, CodexWebSocketPoolKey, WebSocketPoolDecision, DEFAULT_FIRST_TOKEN_TIMEOUT, + CodexWebSocketPool, CodexWebSocketPoolKey, WebSocketPoolDecision, DEFAULT_INITIAL_EVENT_TIMEOUT, }; // --------------------------------------------------------------------------- @@ -238,6 +237,27 @@ pub struct CodexRequestContext<'a> { pub installation_id: Option<&'a str>, /// session_id。 pub session_id: Option<&'a str>, + /// thread_id。 + pub thread_id: Option<&'a str>, + /// 账号作用域 prompt cache key。 + pub prompt_cache_key: Option<&'a str>, + /// 账号作用域 client request ID。 + pub client_request_id: Option<&'a str>, + /// 账号作用域 turn ID。 + pub turn_id: Option<&'a str>, +} + +/// Codex Responses 上游响应。 +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CodexResponseMetadata { + /// 上游实际选用的模型。 + pub effective_model: Option, + /// 模型目录版本。 + pub models_etag: Option, + /// 上游是否声明响应包含 reasoning。 + pub reasoning_included: bool, + /// 允许透传给客户端的安全响应头。 + pub client_headers: Vec<(String, String)>, } /// Codex Responses 上游响应。 @@ -261,6 +281,8 @@ pub struct CodexBackendResponse { pub websocket_pool_decision: Option, /// 上游诊断元数据。 pub diagnostics: CodexUpstreamDiagnostics, + /// 安全响应元数据。 + pub response_metadata: CodexResponseMetadata, } /// Codex Responses 实际使用的上游传输。 @@ -310,6 +332,8 @@ pub struct CodexBackendStreamingResponse { pub websocket_pool_decision: Option, /// 上游诊断元数据。 pub diagnostics: CodexUpstreamDiagnostics, + /// 安全响应元数据。 + pub response_metadata: CodexResponseMetadata, } /// Codex compact 端点响应。 @@ -323,6 +347,8 @@ pub struct CodexCompactResponse { pub rate_limit_headers: Vec<(String, String)>, /// 上游诊断元数据。 pub diagnostics: CodexUpstreamDiagnostics, + /// 安全响应元数据。 + pub response_metadata: CodexResponseMetadata, } // --------------------------------------------------------------------------- @@ -336,7 +362,7 @@ pub struct CodexBackendClient { pub(super) base_url: String, fingerprint: RuntimeFingerprint, websocket_pool: Option>, - websocket_first_token_timeout: Option, + websocket_initial_event_timeout: Option, } #[path = "client_sse.rs"] @@ -362,6 +388,10 @@ impl CodexModelCatalogClient for CodexBackendClient { cookie_header: None, installation_id: request.installation_id, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }) .await .map_err(|error| CodexModelCatalogClientError::RequestFailed { @@ -490,8 +520,19 @@ fn response_upstream_request( context: CodexRequestContext<'_>, ) -> CodexResponsesRequest { let mut upstream = request.clone(); - if let Some(session_id) = context.session_id { - upstream.set_prompt_cache_key(Some(session_id.to_string())); + if request.prompt_cache_key().is_some() { + upstream.set_prompt_cache_key(context.prompt_cache_key.map(ToString::to_string)); + } + for (key, value) in [ + ("session_id", context.session_id), + ("thread_id", context.thread_id), + ("turn_id", context.turn_id), + ("x-client-request-id", context.client_request_id), + ("x-codex-window-id", context.codex_window_id), + ("x-codex-parent-thread-id", context.parent_thread_id), + ("x-codex-installation-id", context.installation_id), + ] { + upstream.replace_existing_identity_field(key, value); } upstream.set_client_metadata(response_client_metadata(request.client_metadata(), context)); upstream @@ -506,7 +547,8 @@ fn websocket_upstream_request(request: &CodexResponsesRequest) -> CodexResponses fn stamp_ws_stream_request_start_ms(request: &mut CodexResponsesRequest) { let mut metadata = match request.client_metadata() { Some(Value::Object(metadata)) => metadata.clone(), - _ => Map::new(), + None => Map::new(), + Some(_) => return, }; metadata.insert( X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY.to_string(), @@ -528,25 +570,31 @@ fn response_client_metadata( ) -> Option { // 以客户端原始 client_metadata 为基础（保留 number/bool/object 等所有值类型）， // 在其上追加代理自身的上下文字段。 - let mut metadata = match client_metadata { - Some(Value::Object(input)) => input.clone(), - _ => Map::new(), + let mut metadata = match client_metadata? { + Value::Object(input) => input.clone(), + value => return Some(value.clone()), }; - insert_metadata_string( + replace_metadata_string( &mut metadata, "x-codex-installation-id", context.installation_id, ); - insert_metadata_string(&mut metadata, "session_id", context.session_id); - insert_metadata_string(&mut metadata, "thread_id", context.session_id); - insert_metadata_string(&mut metadata, "x-codex-window-id", context.codex_window_id); + replace_metadata_string(&mut metadata, "session_id", context.session_id); + replace_metadata_string(&mut metadata, "thread_id", context.thread_id); + replace_metadata_string( + &mut metadata, + "x-client-request-id", + context.client_request_id, + ); + replace_metadata_string(&mut metadata, "turn_id", context.turn_id); + replace_metadata_string(&mut metadata, "x-codex-window-id", context.codex_window_id); insert_metadata_string( &mut metadata, "x-codex-turn-metadata", context.turn_metadata, ); - insert_metadata_string( + replace_metadata_string( &mut metadata, "x-codex-parent-thread-id", context.parent_thread_id, @@ -565,6 +613,17 @@ fn insert_metadata_string(metadata: &mut Map, key: &str, value: O } } +fn replace_metadata_string(metadata: &mut Map, key: &str, value: Option<&str>) { + match value.filter(|value| !value.trim().is_empty()) { + Some(value) => { + metadata.insert(key.to_string(), Value::String(value.to_string())); + } + None => { + metadata.remove(key); + } + } +} + fn openai_subagent_from_metadata(client_metadata: Option<&Value>) -> Option { let value = client_metadata? .as_object()? diff --git a/backend/src/upstream/openai/transport/client_sse.rs b/backend/src/upstream/openai/transport/client_sse.rs index 12e5a0154..24a42312f 100644 --- a/backend/src/upstream/openai/transport/client_sse.rs +++ b/backend/src/upstream/openai/transport/client_sse.rs @@ -12,12 +12,12 @@ impl CodexBackendClient { base_url: base_url.into().trim_end_matches('/').to_string(), fingerprint, websocket_pool: None, - websocket_first_token_timeout: Some(DEFAULT_FIRST_TOKEN_TIMEOUT), + websocket_initial_event_timeout: Some(DEFAULT_INITIAL_EVENT_TIMEOUT), } } - pub fn with_websocket_first_token_timeout(mut self, timeout: Option) -> Self { - self.websocket_first_token_timeout = timeout.filter(|timeout| !timeout.is_zero()); + pub fn with_websocket_initial_event_timeout(mut self, timeout: Option) -> Self { + self.websocket_initial_event_timeout = timeout.filter(|timeout| !timeout.is_zero()); self } @@ -55,6 +55,7 @@ impl CodexBackendClient { let turn_state = response_meta::turn_state(response.headers()); let set_cookie_headers = response_meta::set_cookie_headers(response.headers()); let rate_limit_headers = response_meta::rate_limit_headers(response.headers()); + let response_metadata = response_meta::response_metadata(response.headers()); let retry_after_seconds = retry_after_seconds(response.headers(), None); if !status.is_success() { @@ -88,6 +89,7 @@ impl CodexBackendClient { first_token_ms, websocket_pool_decision: None, diagnostics, + response_metadata, }) } @@ -110,6 +112,7 @@ impl CodexBackendClient { let turn_state = response_meta::turn_state(response.headers()); let set_cookie_headers = response_meta::set_cookie_headers(response.headers()); let rate_limit_headers = response_meta::rate_limit_headers(response.headers()); + let response_metadata = response_meta::response_metadata(response.headers()); let retry_after_seconds = retry_after_seconds(response.headers(), None); if !status.is_success() { @@ -134,6 +137,7 @@ impl CodexBackendClient { turn_state_update: None, websocket_pool_decision: None, diagnostics, + response_metadata, }) } @@ -284,7 +288,7 @@ impl CodexBackendClient { &prepared, Some((pool, key)), started_at, - self.websocket_first_token_timeout, + self.websocket_initial_event_timeout, ) .await } @@ -293,7 +297,7 @@ impl CodexBackendClient { &prepared, None, started_at, - self.websocket_first_token_timeout, + self.websocket_initial_event_timeout, ) .await } @@ -316,6 +320,7 @@ impl CodexBackendClient { first_token_ms: exchange.first_token_ms, websocket_pool_decision: exchange.pool_decision, diagnostics: exchange.diagnostics, + response_metadata: exchange.response_metadata, }) } @@ -349,7 +354,7 @@ impl CodexBackendClient { execute_response_create_request_stream_with_pool( &prepared, Some((pool, key)), - self.websocket_first_token_timeout, + self.websocket_initial_event_timeout, ) .await } @@ -357,7 +362,7 @@ impl CodexBackendClient { execute_response_create_request_stream_with_pool( &prepared, None, - self.websocket_first_token_timeout, + self.websocket_initial_event_timeout, ) .await } @@ -384,6 +389,7 @@ impl CodexBackendClient { turn_state_update: Some(exchange.turn_state_update), websocket_pool_decision: exchange.pool_decision, diagnostics: exchange.diagnostics, + response_metadata: exchange.response_metadata, }) } @@ -395,8 +401,8 @@ impl CodexBackendClient { ) -> Option { let account_id = pool_account_id.or(context.account_id)?; let conversation_id = request - .prompt_cache_key() - .or(request.client_conversation_id.as_deref()) + .local_conversation_id + .as_deref() .or(request.previous_response_id())?; Some(CodexWebSocketPoolKey::new( &self.base_url, @@ -424,6 +430,7 @@ impl CodexBackendClient { let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers()); let set_cookie_headers = response_meta::set_cookie_headers(response.headers()); let rate_limit_headers = response_meta::rate_limit_headers(response.headers()); + let response_metadata = response_meta::response_metadata(response.headers()); let retry_after_seconds = retry_after_seconds(response.headers(), None); let body = response.text().await?; @@ -456,6 +463,7 @@ impl CodexBackendClient { set_cookie_headers, rate_limit_headers, diagnostics, + response_metadata, }) } @@ -513,7 +521,7 @@ impl CodexBackendClient { fn request_headers(&self, context: CodexRequestContext<'_>) -> CodexClientResult { let fingerprint = self.fingerprint.current(); - let request_id = context.session_id.unwrap_or(context.request_id); + let request_id = context.client_request_id.unwrap_or(context.request_id); let ordered_headers = build_ordered_codex_base_headers( &fingerprint, context.access_token, @@ -543,7 +551,8 @@ impl CodexBackendClient { context.installation_id, )?; insert_optional_header(&mut headers, "session-id", context.session_id)?; - insert_optional_header(&mut headers, "thread-id", context.session_id)?; + insert_optional_header(&mut headers, "thread-id", context.thread_id)?; + insert_optional_header(&mut headers, "x-codex-turn-id", context.turn_id)?; insert_optional_header(&mut headers, "x-codex-window-id", context.codex_window_id)?; insert_optional_header(&mut headers, "x-codex-turn-state", context.turn_state)?; insert_optional_header(&mut headers, "x-codex-turn-metadata", context.turn_metadata)?; @@ -635,13 +644,22 @@ impl CodexBackendClient { ); headers.insert( HeaderName::from_static("x-client-request-id"), - HeaderValue::from_str(&Uuid::new_v4().to_string())?, + HeaderValue::from_str(context.client_request_id.unwrap_or(context.request_id))?, ); insert_optional_header( &mut headers, "x-codex-installation-id", context.installation_id, )?; + insert_optional_header(&mut headers, "session-id", context.session_id)?; + insert_optional_header(&mut headers, "thread-id", context.thread_id)?; + insert_optional_header(&mut headers, "x-codex-turn-id", context.turn_id)?; + insert_optional_header(&mut headers, "x-codex-window-id", context.codex_window_id)?; + insert_optional_header( + &mut headers, + "x-codex-parent-thread-id", + context.parent_thread_id, + )?; Ok(headers) } diff --git a/backend/src/upstream/openai/transport/diagnostics.rs b/backend/src/upstream/openai/transport/diagnostics.rs index 8a755a961..b057b894c 100644 --- a/backend/src/upstream/openai/transport/diagnostics.rs +++ b/backend/src/upstream/openai/transport/diagnostics.rs @@ -40,6 +40,21 @@ impl CodexUpstreamDiagnostics { } } + pub fn from_pairs(status_code: Option, headers: &[(String, String)]) -> Self { + let request_id = UPSTREAM_REQUEST_ID_HEADERS + .iter() + .find_map(|name| pair_value(headers, name)); + let trace_headers = UPSTREAM_TRACE_HEADERS + .iter() + .filter_map(|name| pair_value(headers, name).map(|value| ((*name).to_string(), value))) + .collect(); + Self { + status_code, + request_id, + trace_headers, + } + } + pub fn is_empty(&self) -> bool { self.status_code.is_none() && self.request_id.is_none() && self.trace_headers.is_empty() } @@ -52,6 +67,15 @@ impl CodexUpstreamDiagnostics { } } +fn pair_value(headers: &[(String, String)], name: &str) -> Option { + headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.trim()) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) +} + fn first_header(headers: &HeaderMap, names: &[&str]) -> Option { names.iter().find_map(|name| header_value(headers, name)) } diff --git a/backend/src/upstream/openai/transport/mod.rs b/backend/src/upstream/openai/transport/mod.rs index d9e4fb9cb..b91719a8e 100644 --- a/backend/src/upstream/openai/transport/mod.rs +++ b/backend/src/upstream/openai/transport/mod.rs @@ -18,7 +18,7 @@ pub use self::{ CodexBackendStreamingResponse, CodexBackendTransport, CodexClientError, CodexClientResult, CodexCompactResponse, CodexModelCatalogClient, CodexModelCatalogClientError, CodexModelCatalogRequest, CodexRateLimitHeaderUpdates, CodexRequestContext, - CodexTurnStateUpdate, + CodexResponseMetadata, CodexTurnStateUpdate, }, diagnostics::CodexUpstreamDiagnostics, endpoints::{ diff --git a/backend/src/upstream/openai/transport/response_meta.rs b/backend/src/upstream/openai/transport/response_meta.rs index f2f511cbd..765ea7e34 100644 --- a/backend/src/upstream/openai/transport/response_meta.rs +++ b/backend/src/upstream/openai/transport/response_meta.rs @@ -4,9 +4,19 @@ use std::time::Instant; use reqwest::header::{HeaderMap, SET_COOKIE}; -use crate::upstream::openai::protocol::responses::update_first_response_output_ms; +use crate::upstream::openai::protocol::{ + events::is_rate_limit_header_name, responses::update_first_response_output_ms, +}; -use super::diagnostics::CodexUpstreamDiagnostics; +use super::{client::CodexResponseMetadata, diagnostics::CodexUpstreamDiagnostics}; + +const CLIENT_RESPONSE_HEADERS: [&str; 5] = [ + "x-request-id", + "openai-model", + "x-models-etag", + "x-reasoning-included", + "openai-processing-ms", +]; pub(super) fn diagnostics( status_code: Option, @@ -33,7 +43,7 @@ pub(super) fn set_cookie_headers(headers: &HeaderMap) -> Vec { pub(super) fn rate_limit_headers(headers: &HeaderMap) -> Vec<(String, String)> { headers .iter() - .filter(|(name, _)| is_rate_limit_header(name.as_str())) + .filter(|(name, _)| is_rate_limit_header_name(name.as_str())) .filter_map(|(name, value)| { value .to_str() @@ -43,6 +53,59 @@ pub(super) fn rate_limit_headers(headers: &HeaderMap) -> Vec<(String, String)> { .collect() } +pub(super) fn response_metadata(headers: &HeaderMap) -> CodexResponseMetadata { + response_metadata_from_pairs( + headers + .iter() + .filter_map(|(name, value)| value.to_str().ok().map(|value| (name.as_str(), value))), + ) +} + +pub(super) fn merge_response_metadata( + metadata: &mut CodexResponseMetadata, + headers: impl IntoIterator, +) { + for (name, value) in headers { + apply_response_header(metadata, &name, &value); + } +} + +fn response_metadata_from_pairs<'a>( + headers: impl IntoIterator, +) -> CodexResponseMetadata { + let mut metadata = CodexResponseMetadata::default(); + for (name, value) in headers { + apply_response_header(&mut metadata, name, value); + } + metadata +} + +fn apply_response_header(metadata: &mut CodexResponseMetadata, name: &str, value: &str) { + let name = name.to_ascii_lowercase(); + let value = value.trim(); + if value.is_empty() || !CLIENT_RESPONSE_HEADERS.contains(&name.as_str()) { + return; + } + match name.as_str() { + "openai-model" => metadata.effective_model = Some(value.to_string()), + "x-models-etag" => metadata.models_etag = Some(value.to_string()), + "x-reasoning-included" => metadata.reasoning_included = true, + _ => {} + } + upsert_client_header(&mut metadata.client_headers, (name, value.to_string())); +} + +fn upsert_client_header(headers: &mut Vec<(String, String)>, header: (String, String)) { + if let Some(existing) = headers + .iter_mut() + .find(|(name, _)| name.eq_ignore_ascii_case(&header.0)) + { + *existing = header; + } else { + headers.push(header); + } +} + pub(super) fn update_first_token_ms( started_at: Instant, body_bytes: &[u8], @@ -50,15 +113,3 @@ pub(super) fn update_first_token_ms( ) { update_first_response_output_ms(started_at, body_bytes, first_token_ms); } - -fn is_rate_limit_header(name: &str) -> bool { - let name = name.to_ascii_lowercase(); - name == "retry-after" - || name.contains("ratelimit") - || name.contains("rate-limit") - || name.starts_with("x-codex-primary-") - || name.starts_with("x-codex-secondary-") - || name.starts_with("x-codex-code-review-") - || name.starts_with("x-codex-review-") - || name.starts_with("x-code-review-") -} diff --git a/backend/src/upstream/openai/transport/websocket.rs b/backend/src/upstream/openai/transport/websocket.rs index b88253393..0532cc182 100644 --- a/backend/src/upstream/openai/transport/websocket.rs +++ b/backend/src/upstream/openai/transport/websocket.rs @@ -28,23 +28,25 @@ use tungstenite::{ use crate::infra::time::china_filename_timestamp_millis; use crate::upstream::openai::protocol::events::{self, TokenUsage}; use crate::upstream::openai::protocol::responses::{ - response_body_has_first_output, CodexResponsesRequest, + response_body_has_first_output, CodexResponsesRequest, PreviousResponseScope, + StreamCommitPolicy, }; use crate::upstream::openai::protocol::sse::SseError; use crate::upstream::openai::protocol::websocket::{ classify_websocket_error_frame, is_terminal_websocket_event, retry_after_seconds_from_wrapped_error_headers, websocket_event_to_sse_frame, - websocket_event_type, websocket_incomplete_response_reason, websocket_metadata_turn_state, - websocket_response_completed_parse_error, websocket_response_create_payload_text, - OpeningAuditHeader, OpeningAuditSnapshot, WebSocketAuditArtifact, + websocket_event_type, websocket_metadata_headers, websocket_metadata_turn_state, + websocket_response_completed_id, websocket_response_create_payload_text, + ClassifiedWebSocketError, OpeningAuditHeader, OpeningAuditSnapshot, WebSocketAuditArtifact, }; use super::websocket_pool::{ CodexWebSocketConnectionMetadata, CodexWebSocketPool, CodexWebSocketPoolKey, - PooledWebSocketConnection, WebSocketPoolAcquire, WebSocketPoolDecision, WebSocketPoolLease, + PooledWebSocketConnection, WebSocketContinuationState, WebSocketPoolAcquire, + WebSocketPoolDecision, WebSocketPoolLease, }; use super::websocket_pump::{PumpKeepalive, PumpedWebSocket, RawWsStream}; -use super::{diagnostics::CodexUpstreamDiagnostics, response_meta}; +use super::{client::CodexResponseMetadata, diagnostics::CodexUpstreamDiagnostics, response_meta}; use uuid::Uuid; const REDACTED_HEADER_VALUE: &str = ""; @@ -102,16 +104,89 @@ pub async fn write_websocket_audit_artifact_from_env( pub struct CodexWebSocketRequest { pub(super) connection: CodexWebSocketConnection, pub(super) payload_text: String, + pub(super) continuation: WebSocketContinuationRequirement, + pub(super) stream_commit_policy: StreamCommitPolicy, +} + +/// 当前 WebSocket 请求对 previous response 状态的要求。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WebSocketContinuationRequirement { + /// 不依赖任何已有响应状态。 + NewChain, + /// 上游已持久化 response，可在新连接 hydration。 + Persisted { response_id: String }, + /// 代理没有所有权信息，只允许 dispatch 选定的单个账号原样尝试。 + ExternalUnknown { response_id: String }, + /// `store=false` response，只能在拥有该 ID 的原连接续接。 + ConnectionLocal { response_id: String }, +} + +impl WebSocketContinuationRequirement { + fn from_request(request: &CodexResponsesRequest) -> Self { + match request.previous_response_id() { + None => Self::NewChain, + Some(response_id) + if request.previous_response_scope == Some(PreviousResponseScope::Persisted) + || (request.previous_response_scope.is_none() && request.store()) => + { + Self::Persisted { + response_id: response_id.to_string(), + } + } + Some(response_id) + if request.previous_response_scope + == Some(PreviousResponseScope::ExternalUnknown) => + { + Self::ExternalUnknown { + response_id: response_id.to_string(), + } + } + Some(response_id) => Self::ConnectionLocal { + response_id: response_id.to_string(), + }, + } + } + + fn permits_fresh_connection(&self) -> bool { + matches!( + self, + Self::NewChain | Self::Persisted { .. } | Self::ExternalUnknown { .. } + ) + } +} + +/// 连接本地 previous response 无法满足的原因。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PreviousResponseUnavailableReason { + PoolUnavailable, + FreshConnectionRequired, + ConnectionBusy, + LatestResponseMismatch, + ReusedConnectionLost, + UpstreamRejected, +} + +impl fmt::Display for PreviousResponseUnavailableReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::PoolUnavailable => "pool_unavailable", + Self::FreshConnectionRequired => "fresh_connection_required", + Self::ConnectionBusy => "connection_busy", + Self::LatestResponseMismatch => "latest_response_mismatch", + Self::ReusedConnectionLost => "reused_connection_lost", + Self::UpstreamRejected => "upstream_rejected", + }) + } } #[path = "websocket_frames.rs"] mod frames; use frames::{ - audit_header_value, collect_websocket_response, is_first_token_timeout, + audit_header_value, collect_websocket_response, is_initial_event_timeout, prefetch_stream_frames_until_output_or_terminal, reusable_websocket_metadata, reused_stream_prefetch_error, stream_websocket_response, websocket_audit_file_name, - websocket_connection_metadata, WebSocketStreamPoolReturn, + websocket_connection_metadata, WebSocketStreamPoolReturn, WebSocketTerminalKind, }; pub use frames::{ execute_response_create_request, CodexWebSocketExchange, CodexWebSocketExchangeError, @@ -129,6 +204,15 @@ impl CodexWebSocketRequest { pub fn payload_text(&self) -> &str { &self.payload_text } + + /// 返回连接续接要求。 + pub fn continuation(&self) -> &WebSocketContinuationRequirement { + &self.continuation + } + + pub fn stream_commit_policy(&self) -> StreamCommitPolicy { + self.stream_commit_policy + } } impl CodexWebSocketConnection { @@ -175,6 +259,8 @@ impl CodexWebSocketConnection { Ok(CodexWebSocketRequest { connection: Self::responses(base_url, websocket_key, business_headers), payload_text: websocket_response_create_payload_text(request)?, + continuation: WebSocketContinuationRequirement::from_request(request), + stream_commit_policy: request.stream_commit_policy, }) } @@ -330,74 +416,79 @@ fn request_line_for_endpoint(endpoint: &str) -> String { async fn execute_fresh_response_create_request( request: &CodexWebSocketRequest, started_at: Instant, - first_token_timeout: Option, + initial_event_timeout: Option, ) -> Result { let (websocket, response) = connect_pumped_websocket(request.connection(), PumpKeepalive::disabled()).await?; - let first_token_started_at = Instant::now(); websocket .send(Message::Text(request.payload_text().to_string().into())) .await?; let metadata = websocket_connection_metadata(&response); - let (exchange, _websocket, _metadata) = collect_websocket_response( - websocket, - metadata, - false, - started_at, - first_token_started_at, - first_token_timeout, - ) - .await?; + let (exchange, mut websocket, _metadata, _continuation, _terminal) = + collect_websocket_response( + websocket, + metadata, + WebSocketContinuationState::default(), + false, + started_at, + initial_event_timeout, + ) + .await?; + websocket.close().await; Ok(exchange) } async fn execute_fresh_response_create_request_stream( request: &CodexWebSocketRequest, - first_token_timeout: Option, + initial_event_timeout: Option, ) -> Result { let (mut websocket, response) = connect_pumped_websocket(request.connection(), PumpKeepalive::disabled()).await?; - let first_token_started_at = Instant::now(); websocket .send(Message::Text(request.payload_text().to_string().into())) .await?; let mut metadata = websocket_connection_metadata(&response); - let prefetched_frames = match prefetch_stream_frames_until_output_or_terminal( - &mut websocket, - &mut metadata, - first_token_started_at, - first_token_timeout, - ) - .await - { - Ok(prefetched_frames) => prefetched_frames, - Err(error) => { - websocket.close().await; - return Err(error); + let prefetched_frames = if should_prefetch_until_output_or_terminal(request) { + match prefetch_stream_frames_until_output_or_terminal( + &mut websocket, + &mut metadata, + initial_event_timeout, + ) + .await + { + Ok(prefetched_frames) => prefetched_frames, + Err(error) => { + websocket.close().await; + return Err(error); + } } + } else { + Vec::new() }; Ok(stream_websocket_response( websocket, metadata, None, prefetched_frames, + initial_event_timeout, )) } async fn execute_fresh_response_create_request_with_retries( request: &CodexWebSocketRequest, started_at: Instant, - first_token_timeout: Option, + initial_event_timeout: Option, attempts: usize, ) -> Result { let attempts = attempts.max(1); for attempt in 1..=attempts { - match execute_fresh_response_create_request(request, started_at, first_token_timeout).await + match execute_fresh_response_create_request(request, started_at, initial_event_timeout) + .await { Ok(exchange) => return Ok(exchange), - Err(error) if attempt < attempts && is_first_token_timeout(&error) => continue, + Err(error) if attempt < attempts && is_initial_event_timeout(&error) => continue, Err(error) => return Err(error), } } @@ -406,14 +497,14 @@ async fn execute_fresh_response_create_request_with_retries( async fn execute_fresh_response_create_request_stream_with_retries( request: &CodexWebSocketRequest, - first_token_timeout: Option, + initial_event_timeout: Option, attempts: usize, ) -> Result { let attempts = attempts.max(1); for attempt in 1..=attempts { - match execute_fresh_response_create_request_stream(request, first_token_timeout).await { + match execute_fresh_response_create_request_stream(request, initial_event_timeout).await { Ok(exchange) => return Ok(exchange), - Err(error) if attempt < attempts && is_first_token_timeout(&error) => continue, + Err(error) if attempt < attempts && is_initial_event_timeout(&error) => continue, Err(error) => return Err(error), } } @@ -428,13 +519,18 @@ pub(crate) async fn execute_response_create_request_with_pool( request: &CodexWebSocketRequest, pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>, started_at: Instant, - fallback_first_token_timeout: Option, + fallback_initial_event_timeout: Option, ) -> Result { let Some((pool, key)) = pool else { + if !request.continuation().permits_fresh_connection() { + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::PoolUnavailable, + )); + } return execute_fresh_response_create_request_with_retries( request, started_at, - fallback_first_token_timeout, + fallback_initial_event_timeout, WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS, ) .await; @@ -442,6 +538,16 @@ pub(crate) async fn execute_response_create_request_with_pool( match pool.acquire(&key).await { WebSocketPoolAcquire::Reused { connection, lease } => { + if let WebSocketContinuationRequirement::ConnectionLocal { response_id } = + request.continuation() + { + if connection.continuation.latest_response_id() != Some(response_id.as_str()) { + lease.put(*connection).await; + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::LatestResponseMismatch, + )); + } + } let result = execute_pooled_response_create_request( request, pool, @@ -454,11 +560,16 @@ pub(crate) async fn execute_response_create_request_with_pool( Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { .. }) - | Err(CodexWebSocketExchangeError::FirstTokenTimeout { .. }) => { + | Err(CodexWebSocketExchangeError::InitialEventTimeout { .. }) => { + if !request.continuation().permits_fresh_connection() { + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::ReusedConnectionLost, + )); + } let mut exchange = execute_fresh_response_create_request_with_retries( request, started_at, - pool.first_token_timeout(), + pool.initial_event_timeout(), WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS, ) .await?; @@ -473,6 +584,12 @@ pub(crate) async fn execute_response_create_request_with_pool( } } WebSocketPoolAcquire::FreshReserved(lease) => { + if !request.continuation().permits_fresh_connection() { + lease.discard().await; + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::FreshConnectionRequired, + )); + } let result = execute_fresh_pooled_response_create_request(request, pool, lease, started_at) .await; @@ -481,11 +598,11 @@ pub(crate) async fn execute_response_create_request_with_pool( exchange.pool_decision = Some(WebSocketPoolDecision::new()); Ok(exchange) } - Err(error) if is_first_token_timeout(&error) => { + Err(error) if is_initial_event_timeout(&error) => { let mut exchange = execute_fresh_response_create_request_with_retries( request, started_at, - pool.first_token_timeout(), + pool.initial_event_timeout(), 1, ) .await?; @@ -496,10 +613,21 @@ pub(crate) async fn execute_response_create_request_with_pool( } } WebSocketPoolAcquire::Bypass(reason) => { + if !request.continuation().permits_fresh_connection() { + return Err(continuation_unavailable(match reason { + super::websocket_pool::WebSocketPoolBypassReason::Busy => { + PreviousResponseUnavailableReason::ConnectionBusy + } + super::websocket_pool::WebSocketPoolBypassReason::Disabled + | super::websocket_pool::WebSocketPoolBypassReason::Cap => { + PreviousResponseUnavailableReason::PoolUnavailable + } + })); + } let mut exchange = execute_fresh_response_create_request_with_retries( request, started_at, - pool.first_token_timeout(), + pool.initial_event_timeout(), WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS, ) .await?; @@ -512,12 +640,17 @@ pub(crate) async fn execute_response_create_request_with_pool( pub(crate) async fn execute_response_create_request_stream_with_pool( request: &CodexWebSocketRequest, pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>, - fallback_first_token_timeout: Option, + fallback_initial_event_timeout: Option, ) -> Result { let Some((pool, key)) = pool else { + if !request.continuation().permits_fresh_connection() { + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::PoolUnavailable, + )); + } return execute_fresh_response_create_request_stream_with_retries( request, - fallback_first_token_timeout, + fallback_initial_event_timeout, WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS, ) .await; @@ -525,6 +658,16 @@ pub(crate) async fn execute_response_create_request_stream_with_pool( match pool.acquire(&key).await { WebSocketPoolAcquire::Reused { connection, lease } => { + if let WebSocketContinuationRequirement::ConnectionLocal { response_id } = + request.continuation() + { + if connection.continuation.latest_response_id() != Some(response_id.as_str()) { + lease.put(*connection).await; + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::LatestResponseMismatch, + )); + } + } let result = execute_pooled_response_create_request_stream(request, pool, lease, *connection) .await; @@ -532,10 +675,15 @@ pub(crate) async fn execute_response_create_request_stream_with_pool( Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { .. }) - | Err(CodexWebSocketExchangeError::FirstTokenTimeout { .. }) => { + | Err(CodexWebSocketExchangeError::InitialEventTimeout { .. }) => { + if !request.continuation().permits_fresh_connection() { + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::ReusedConnectionLost, + )); + } let mut exchange = execute_fresh_response_create_request_stream_with_retries( request, - pool.first_token_timeout(), + pool.initial_event_timeout(), WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS, ) .await?; @@ -550,6 +698,12 @@ pub(crate) async fn execute_response_create_request_stream_with_pool( } } WebSocketPoolAcquire::FreshReserved(lease) => { + if !request.continuation().permits_fresh_connection() { + lease.discard().await; + return Err(continuation_unavailable( + PreviousResponseUnavailableReason::FreshConnectionRequired, + )); + } let result = execute_fresh_pooled_response_create_request_stream(request, pool, lease).await; match result { @@ -557,10 +711,10 @@ pub(crate) async fn execute_response_create_request_stream_with_pool( exchange.pool_decision = Some(WebSocketPoolDecision::new()); Ok(exchange) } - Err(error) if is_first_token_timeout(&error) => { + Err(error) if is_initial_event_timeout(&error) => { let mut exchange = execute_fresh_response_create_request_stream_with_retries( request, - pool.first_token_timeout(), + pool.initial_event_timeout(), 1, ) .await?; @@ -571,9 +725,20 @@ pub(crate) async fn execute_response_create_request_stream_with_pool( } } WebSocketPoolAcquire::Bypass(reason) => { + if !request.continuation().permits_fresh_connection() { + return Err(continuation_unavailable(match reason { + super::websocket_pool::WebSocketPoolBypassReason::Busy => { + PreviousResponseUnavailableReason::ConnectionBusy + } + super::websocket_pool::WebSocketPoolBypassReason::Disabled + | super::websocket_pool::WebSocketPoolBypassReason::Cap => { + PreviousResponseUnavailableReason::PoolUnavailable + } + })); + } let mut exchange = execute_fresh_response_create_request_stream_with_retries( request, - pool.first_token_timeout(), + pool.initial_event_timeout(), WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS, ) .await?; @@ -583,6 +748,12 @@ pub(crate) async fn execute_response_create_request_stream_with_pool( } } +fn continuation_unavailable( + reason: PreviousResponseUnavailableReason, +) -> CodexWebSocketExchangeError { + CodexWebSocketExchangeError::ContinuationUnavailable { reason } +} + async fn execute_fresh_pooled_response_create_request( request: &CodexWebSocketRequest, pool: &CodexWebSocketPool, @@ -597,7 +768,6 @@ async fn execute_fresh_pooled_response_create_request( return Err(error); } }; - let first_token_started_at = Instant::now(); if let Err(error) = websocket .send(Message::Text(request.payload_text().to_string().into())) .await @@ -608,29 +778,39 @@ async fn execute_fresh_pooled_response_create_request( let now = Instant::now(); let metadata = websocket_connection_metadata(&response); - let (exchange, websocket, metadata) = match collect_websocket_response( - websocket, - metadata, - false, - started_at, - first_token_started_at, - pool.first_token_timeout(), - ) - .await - { - Ok(exchange) => exchange, - Err(error) => { - lease.discard().await; - return Err(error); - } - }; - lease - .put(PooledWebSocketConnection { + let (exchange, mut websocket, metadata, continuation, terminal) = + match collect_websocket_response( websocket, - metadata: reusable_websocket_metadata(metadata), - created_at: now, - }) - .await; + metadata, + WebSocketContinuationState::default(), + false, + started_at, + pool.initial_event_timeout(), + ) + .await + { + Ok(exchange) => exchange, + Err(error) => { + lease.discard().await; + return Err(error); + } + }; + match terminal { + WebSocketTerminalKind::Completed => { + lease + .put(PooledWebSocketConnection { + websocket, + metadata: reusable_websocket_metadata(metadata), + continuation, + created_at: now, + }) + .await; + } + WebSocketTerminalKind::Incomplete | WebSocketTerminalKind::Failed => { + lease.discard().await; + websocket.close().await; + } + } Ok(exchange) } @@ -647,7 +827,6 @@ async fn execute_fresh_pooled_response_create_request_stream( return Err(error); } }; - let first_token_started_at = Instant::now(); if let Err(error) = websocket .send(Message::Text(request.payload_text().to_string().into())) .await @@ -658,20 +837,23 @@ async fn execute_fresh_pooled_response_create_request_stream( let now = Instant::now(); let mut metadata = websocket_connection_metadata(&response); - let prefetched_frames = match prefetch_stream_frames_until_output_or_terminal( - &mut websocket, - &mut metadata, - first_token_started_at, - pool.first_token_timeout(), - ) - .await - { - Ok(prefetched_frames) => prefetched_frames, - Err(error) => { - websocket.close().await; - lease.discard().await; - return Err(error); + let prefetched_frames = if should_prefetch_until_output_or_terminal(request) { + match prefetch_stream_frames_until_output_or_terminal( + &mut websocket, + &mut metadata, + pool.initial_event_timeout(), + ) + .await + { + Ok(prefetched_frames) => prefetched_frames, + Err(error) => { + websocket.close().await; + lease.discard().await; + return Err(error); + } } + } else { + Vec::new() }; Ok(stream_websocket_response( websocket, @@ -679,8 +861,10 @@ async fn execute_fresh_pooled_response_create_request_stream( Some(WebSocketStreamPoolReturn { lease, created_at: now, + continuation: WebSocketContinuationState::default(), }), prefetched_frames, + pool.initial_event_timeout(), )) } @@ -691,7 +875,6 @@ async fn execute_pooled_response_create_request( connection: PooledWebSocketConnection, started_at: Instant, ) -> Result { - let first_token_started_at = Instant::now(); if let Err(error) = connection .websocket .send(Message::Text(request.payload_text().to_string().into())) @@ -709,21 +892,30 @@ async fn execute_pooled_response_create_request( match collect_websocket_response( connection.websocket, reusable_websocket_metadata(connection.metadata), + connection.continuation, true, started_at, - first_token_started_at, - pool.first_token_timeout(), + pool.initial_event_timeout(), ) .await { - Ok((exchange, websocket, metadata)) => { - lease - .put(PooledWebSocketConnection { - websocket, - metadata: reusable_websocket_metadata(metadata), - created_at, - }) - .await; + Ok((exchange, mut websocket, metadata, continuation, terminal)) => { + match terminal { + WebSocketTerminalKind::Completed => { + lease + .put(PooledWebSocketConnection { + websocket, + metadata: reusable_websocket_metadata(metadata), + continuation, + created_at, + }) + .await; + } + WebSocketTerminalKind::Incomplete | WebSocketTerminalKind::Failed => { + lease.discard().await; + websocket.close().await; + } + } Ok(exchange) } Err(error) => { @@ -742,7 +934,7 @@ async fn execute_pooled_response_create_request_stream( let mut websocket = connection.websocket; let mut metadata = reusable_websocket_metadata(connection.metadata); let created_at = connection.created_at; - let first_token_started_at = Instant::now(); + let continuation = connection.continuation; if let Err(error) = websocket .send(Message::Text(request.payload_text().to_string().into())) .await @@ -755,26 +947,41 @@ async fn execute_pooled_response_create_request_stream( ); } - let prefetched_frames = match prefetch_stream_frames_until_output_or_terminal( - &mut websocket, - &mut metadata, - first_token_started_at, - pool.first_token_timeout(), - ) - .await - { - Ok(prefetched_frames) => prefetched_frames, - Err(error) => { - lease.discard().await; - websocket.close().await; - return Err(reused_stream_prefetch_error(error)); + let prefetched_frames = if should_prefetch_until_output_or_terminal(request) { + match prefetch_stream_frames_until_output_or_terminal( + &mut websocket, + &mut metadata, + pool.initial_event_timeout(), + ) + .await + { + Ok(prefetched_frames) => prefetched_frames, + Err(error) => { + lease.discard().await; + websocket.close().await; + return Err(reused_stream_prefetch_error(error)); + } } + } else { + Vec::new() }; Ok(stream_websocket_response( websocket, metadata, - Some(WebSocketStreamPoolReturn { lease, created_at }), + Some(WebSocketStreamPoolReturn { + lease, + created_at, + continuation, + }), prefetched_frames, + pool.initial_event_timeout(), )) } + +fn should_prefetch_until_output_or_terminal(request: &CodexWebSocketRequest) -> bool { + matches!( + request.stream_commit_policy(), + StreamCommitPolicy::UntilOutputOrTerminal + ) +} diff --git a/backend/src/upstream/openai/transport/websocket_frames.rs b/backend/src/upstream/openai/transport/websocket_frames.rs index 2cb95b06c..e44aecc5d 100644 --- a/backend/src/upstream/openai/transport/websocket_frames.rs +++ b/backend/src/upstream/openai/transport/websocket_frames.rs @@ -21,6 +21,8 @@ pub struct CodexWebSocketExchange { pub pool_decision: Option, /// 上游诊断元数据。 pub diagnostics: CodexUpstreamDiagnostics, + /// 安全响应元数据。 + pub response_metadata: CodexResponseMetadata, } /// Responses WebSocket live SSE exchange result. @@ -41,6 +43,8 @@ pub struct CodexWebSocketStreamingExchange { pub pool_decision: Option, /// 上游诊断元数据。 pub diagnostics: CodexUpstreamDiagnostics, + /// 安全响应元数据。 + pub response_metadata: CodexResponseMetadata, } /// Responses WebSocket live SSE byte stream. @@ -51,6 +55,13 @@ pub type CodexWebSocketRateLimitHeaderUpdates = Arc> /// WebSocket live stream turn-state update. pub type CodexWebSocketTurnStateUpdate = Arc>>; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum WebSocketTerminalKind { + Completed, + Incomplete, + Failed, +} + /// Responses WebSocket exchange error. #[derive(Debug, Error)] pub enum CodexWebSocketExchangeError { @@ -66,11 +77,10 @@ pub enum CodexWebSocketExchangeError { /// 上游 WebSocket 错误帧。 #[error("{0}")] Upstream(Box), - /// 上游返回 `response.incomplete`。 - #[error("Incomplete response returned, reason: {reason}")] - IncompleteResponse { - /// incomplete_details.reason。 - reason: String, + /// 请求依赖的连接本地 previous response 无法在当前连接满足。 + #[error("websocket continuation unavailable: {reason}")] + ContinuationUnavailable { + reason: PreviousResponseUnavailableReason, }, /// 上游返回无法按官方形状解析的 `response.completed`。 #[error("{message}")] @@ -98,8 +108,8 @@ pub enum CodexWebSocketExchangeError { }, /// 首个真实输出在配置的绝对超时内未到达（连接落到病态上游后端）。 #[error("websocket first output not received within {timeout:?}")] - FirstTokenTimeout { - /// 首 token 超时时长。 + InitialEventTimeout { + /// 首个上游事件超时时长。 timeout: Duration, }, } @@ -113,6 +123,16 @@ pub struct CodexWebSocketUpstreamError { pub retry_after_seconds: Option, /// 原始错误帧。 pub body: String, + /// 上游错误类型。 + pub error_type: Option, + /// 上游错误码。 + pub code: Option, + /// 上游错误消息。 + pub message: Option, + /// 上游错误参数。 + pub param: Option, + /// 错误事件携带的响应头。 + pub headers: Vec<(String, String)>, /// 上游透传的 `set-cookie` 列表。 pub set_cookie_headers: Vec, /// 上游诊断元数据。 @@ -141,10 +161,36 @@ impl CodexWebSocketExchangeError { status_code, retry_after_seconds, body, + error_type: None, + code: None, + message: None, + param: None, + headers: Vec::new(), set_cookie_headers, diagnostics, })) } + + pub(super) fn upstream_frame( + classified: ClassifiedWebSocketError, + retry_after_seconds: Option, + body: String, + ) -> Self { + let diagnostics = + CodexUpstreamDiagnostics::from_pairs(Some(classified.status_code), &classified.headers); + Self::Upstream(Box::new(CodexWebSocketUpstreamError { + status_code: classified.status_code, + retry_after_seconds, + body, + error_type: classified.error_type, + code: classified.code, + message: classified.message, + param: classified.param, + headers: classified.headers, + set_cookie_headers: Vec::new(), + diagnostics, + })) + } } /// 执行一次 prepared Responses WebSocket 请求并聚合为 SSE。 @@ -161,40 +207,37 @@ pub async fn execute_response_create_request( pub(super) async fn collect_websocket_response( mut websocket: PumpedWebSocket, mut metadata: CodexWebSocketConnectionMetadata, + mut continuation: WebSocketContinuationState, reused_connection: bool, started_at: Instant, - first_token_started_at: Instant, - first_token_timeout: Option, + initial_event_timeout: Option, ) -> Result< ( CodexWebSocketExchange, PumpedWebSocket, CodexWebSocketConnectionMetadata, + WebSocketContinuationState, + WebSocketTerminalKind, ), CodexWebSocketExchangeError, > { let mut body = String::new(); - let mut saw_first_output = false; + let mut saw_upstream_activity = false; let mut first_token_ms = None; loop { - let receive_timeout = receive_idle_timeout_with_first_token_deadline( - saw_first_output, - first_token_started_at, - first_token_timeout, - )?; + let receive_timeout = receive_idle_timeout(saw_upstream_activity, initial_event_timeout); let message = match next_websocket_message(&mut websocket, receive_timeout).await { Ok(message) => message, - Err(error) if !saw_first_output => { - if let Some(timeout) = - elapsed_first_token_timeout(first_token_started_at, first_token_timeout) - { - return Err(CodexWebSocketExchangeError::FirstTokenTimeout { timeout }); - } + Err(CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout }) + if !saw_upstream_activity => + { if reused_connection { - return Err(reused_connection_died_before_first_output(&error)); + return Err(reused_connection_died_before_first_output( + &CodexWebSocketExchangeError::InitialEventTimeout { timeout }, + )); } - return Err(error); + return Err(CodexWebSocketExchangeError::InitialEventTimeout { timeout }); } Err(error) => return Err(error), }; @@ -202,9 +245,12 @@ pub(super) async fn collect_websocket_response( break; }; let text = match message { - Message::Text(text) => text, + Message::Text(text) => { + saw_upstream_activity = true; + text + } Message::Binary(_) => return Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent), - Message::Close(_) if reused_connection && !saw_first_output => { + Message::Close(_) if reused_connection && !saw_upstream_activity => { return Err( CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { message: "websocket closed".to_string(), @@ -218,34 +264,42 @@ pub(super) async fn collect_websocket_response( if let Some(classified) = classify_websocket_error_frame(&raw) { let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw) .or_else(|| events::retry_after_seconds_from_body(&raw)); - return Err(CodexWebSocketExchangeError::upstream( - classified.status_code, + return Err(CodexWebSocketExchangeError::upstream_frame( + classified, retry_after_seconds, raw, - Vec::new(), - CodexUpstreamDiagnostics::with_status(classified.status_code), )); } - if let Some(reason) = websocket_incomplete_response_reason(&raw) { - return Err(CodexWebSocketExchangeError::IncompleteResponse { reason }); - } - if let Some(message) = websocket_response_completed_parse_error(&raw) { - return Err(CodexWebSocketExchangeError::InvalidCompletedResponse { message }); - } if let Some(headers) = websocket_rate_limit_event_headers(&raw) { metadata.rate_limit_headers.extend(headers); continue; } + update_websocket_response_metadata(&mut metadata, &raw); if let Some(metadata_turn_state) = websocket_metadata_turn_state(&raw) { metadata.turn_state = Some(metadata_turn_state); continue; } let event = websocket_event_type(&raw); + if event.as_deref() == Some("response.completed") { + let response_id = websocket_response_completed_id(&raw) + .map_err( + |message| CodexWebSocketExchangeError::InvalidCompletedResponse { message }, + )? + .ok_or_else(|| CodexWebSocketExchangeError::InvalidCompletedResponse { + message: "response.completed is missing response id".to_string(), + })?; + continuation.record_completed(response_id); + } + let terminal = match event.as_deref() { + Some("response.completed") => Some(WebSocketTerminalKind::Completed), + Some("response.incomplete") => Some(WebSocketTerminalKind::Incomplete), + Some("response.failed" | "error") => Some(WebSocketTerminalKind::Failed), + _ => None, + }; let forwarded = if let Some(frame) = websocket_event_to_sse_frame(&raw) { let has_first_output = response_body_has_first_output(frame.as_bytes()); body.push_str(&frame); if has_first_output { - saw_first_output = true; response_meta::update_first_token_ms( started_at, body.as_bytes(), @@ -256,7 +310,7 @@ pub(super) async fn collect_websocket_response( } else { false }; - if forwarded && event.as_deref().is_some_and(is_terminal_websocket_event) { + if let (true, Some(terminal)) = (forwarded, terminal) { let usage = events::extract_sse_usage(&body)?; let exchange = CodexWebSocketExchange { body, @@ -267,12 +321,13 @@ pub(super) async fn collect_websocket_response( first_token_ms, pool_decision: None, diagnostics: metadata.diagnostics.clone(), + response_metadata: metadata.response_metadata.clone(), }; - return Ok((exchange, websocket, metadata)); + return Ok((exchange, websocket, metadata, continuation, terminal)); } } - if reused_connection && !saw_first_output { + if reused_connection && !saw_upstream_activity { return Err( CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { message: "websocket closed before terminal event".to_string(), @@ -286,28 +341,28 @@ pub(super) async fn collect_websocket_response( pub(super) async fn prefetch_stream_frames_until_output_or_terminal( websocket: &mut PumpedWebSocket, metadata: &mut CodexWebSocketConnectionMetadata, - first_token_started_at: Instant, - first_token_timeout: Option, + initial_event_timeout: Option, ) -> Result, CodexWebSocketExchangeError> { let mut prefetched_frames = Vec::new(); + let mut saw_upstream_activity = false; loop { - let receive_timeout = - receive_timeout_before_first_output(first_token_started_at, first_token_timeout)?; + let receive_timeout = receive_idle_timeout(saw_upstream_activity, initial_event_timeout); let message = match next_websocket_message(websocket, receive_timeout).await { Ok(Some(message)) => message, Ok(None) => return Err(CodexWebSocketExchangeError::ClosedBeforeTerminal), - Err(error) => { - if let Some(timeout) = - elapsed_first_token_timeout(first_token_started_at, first_token_timeout) - { - return Err(CodexWebSocketExchangeError::FirstTokenTimeout { timeout }); - } - return Err(error); + Err(CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout }) + if !saw_upstream_activity => + { + return Err(CodexWebSocketExchangeError::InitialEventTimeout { timeout }); } + Err(error) => return Err(error), }; let raw = match message { - Message::Text(text) => text.to_string(), + Message::Text(text) => { + saw_upstream_activity = true; + text.to_string() + } Message::Binary(_) => return Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent), Message::Close(_) => return Err(CodexWebSocketExchangeError::ClosedBeforeTerminal), _ => continue, @@ -316,24 +371,22 @@ pub(super) async fn prefetch_stream_frames_until_output_or_terminal( if let Some(classified) = classify_websocket_error_frame(&raw) { let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw) .or_else(|| events::retry_after_seconds_from_body(&raw)); - return Err(CodexWebSocketExchangeError::upstream( - classified.status_code, + return Err(CodexWebSocketExchangeError::upstream_frame( + classified, retry_after_seconds, raw, - Vec::new(), - CodexUpstreamDiagnostics::with_status(classified.status_code), )); } - if let Some(reason) = websocket_incomplete_response_reason(&raw) { - return Err(CodexWebSocketExchangeError::IncompleteResponse { reason }); - } - if let Some(message) = websocket_response_completed_parse_error(&raw) { - return Err(CodexWebSocketExchangeError::InvalidCompletedResponse { message }); + if websocket_event_type(&raw).as_deref() == Some("response.completed") { + websocket_response_completed_id(&raw).map_err(|message| { + CodexWebSocketExchangeError::InvalidCompletedResponse { message } + })?; } if let Some(headers) = websocket_rate_limit_event_headers(&raw) { metadata.rate_limit_headers.extend(headers); continue; } + update_websocket_response_metadata(metadata, &raw); if let Some(metadata_turn_state) = websocket_metadata_turn_state(&raw) { metadata.turn_state = Some(metadata_turn_state); continue; @@ -380,6 +433,7 @@ pub(super) fn websocket_connection_metadata( turn_state: response_meta::turn_state(response.headers()), set_cookie_headers: response_meta::set_cookie_headers(response.headers()), rate_limit_headers: response_meta::rate_limit_headers(response.headers()), + response_metadata: response_meta::response_metadata(response.headers()), diagnostics: response_meta::diagnostics( Some(response.status().as_u16()), response.headers(), @@ -401,6 +455,7 @@ pub(super) fn reusable_websocket_metadata( pub(super) struct WebSocketStreamPoolReturn { pub(super) lease: WebSocketPoolLease, pub(super) created_at: Instant, + pub(super) continuation: WebSocketContinuationState, } #[derive(Debug, Clone, Copy)] @@ -408,6 +463,7 @@ enum StreamWebSocketDiscardReason { ClientCancelled, DownstreamSendFailed, IncompleteResponse, + FailedResponse, InvalidCompletedResponse, UnexpectedBinaryEvent, UpstreamClosed, @@ -421,6 +477,7 @@ impl StreamWebSocketDiscardReason { Self::ClientCancelled => "client_cancelled", Self::DownstreamSendFailed => "downstream_send_failed", Self::IncompleteResponse => "incomplete_response", + Self::FailedResponse => "failed_response", Self::InvalidCompletedResponse => "invalid_completed_response", Self::UnexpectedBinaryEvent => "unexpected_binary_event", Self::UpstreamClosed => "upstream_closed", @@ -435,6 +492,7 @@ pub(super) fn stream_websocket_response( metadata: CodexWebSocketConnectionMetadata, pool_return: Option, prefetched_frames: Vec, + initial_event_timeout: Option, ) -> CodexWebSocketStreamingExchange { let response_metadata = metadata.clone(); let rate_limit_header_updates = Arc::new(Mutex::new(Vec::new())); @@ -443,15 +501,16 @@ pub(super) fn stream_websocket_response( let turn_state_update_for_task = Arc::clone(&turn_state_update); let (tx, rx) = mpsc::channel(WEBSOCKET_STREAM_BUFFER); tokio::spawn(async move { - forward_websocket_response_stream( + forward_websocket_response_stream(WebSocketStreamForwardState { websocket, metadata, pool_return, prefetched_frames, - rate_limit_header_updates_for_task, - turn_state_update_for_task, + initial_event_timeout, + rate_limit_header_updates: rate_limit_header_updates_for_task, + turn_state_update: turn_state_update_for_task, tx, - ) + }) .await; }); @@ -468,21 +527,39 @@ pub(super) fn stream_websocket_response( turn_state_update, pool_decision: None, diagnostics: response_metadata.diagnostics, + response_metadata: response_metadata.response_metadata, } } -async fn forward_websocket_response_stream( - mut websocket: PumpedWebSocket, - mut metadata: CodexWebSocketConnectionMetadata, +struct WebSocketStreamForwardState { + websocket: PumpedWebSocket, + metadata: CodexWebSocketConnectionMetadata, pool_return: Option, prefetched_frames: Vec, + initial_event_timeout: Option, rate_limit_header_updates: CodexWebSocketRateLimitHeaderUpdates, turn_state_update: CodexWebSocketTurnStateUpdate, tx: mpsc::Sender>, -) { +} + +async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) { + let WebSocketStreamForwardState { + mut websocket, + mut metadata, + pool_return, + prefetched_frames, + initial_event_timeout, + rate_limit_header_updates, + turn_state_update, + tx, + } = state; let mut pool_return = pool_return; + let mut continuation = pool_return + .as_mut() + .map(|pool_return| std::mem::take(&mut pool_return.continuation)) + .unwrap_or_default(); + let mut saw_upstream_activity = !prefetched_frames.is_empty(); let mut prefetched_frames: VecDeque = prefetched_frames.into(); - let mut saw_first_output = false; loop { let raw = if let Some(text) = prefetched_frames.pop_front() { text @@ -501,7 +578,7 @@ async fn forward_websocket_response_stream( } message = next_websocket_message( &mut websocket, - receive_idle_timeout(saw_first_output), + receive_idle_timeout(saw_upstream_activity, initial_event_timeout), ) => message, }; let message = match message { @@ -521,7 +598,10 @@ async fn forward_websocket_response_stream( break; }; match message { - Message::Text(text) => text.to_string(), + Message::Text(text) => { + saw_upstream_activity = true; + text.to_string() + } Message::Binary(_) => { discard_stream_websocket( websocket, @@ -559,60 +639,50 @@ async fn forward_websocket_response_stream( let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw) .or_else(|| events::retry_after_seconds_from_body(&raw)); let _ = tx - .send(Err(CodexWebSocketExchangeError::upstream( - classified.status_code, + .send(Err(CodexWebSocketExchangeError::upstream_frame( + classified, retry_after_seconds, raw, - Vec::new(), - CodexUpstreamDiagnostics::with_status(classified.status_code), ))) .await; return; } - if let Some(reason) = websocket_incomplete_response_reason(&raw) { - discard_stream_websocket( - websocket, - pool_return, - StreamWebSocketDiscardReason::IncompleteResponse, - ) - .await; - let _ = tx - .send(Err(CodexWebSocketExchangeError::IncompleteResponse { - reason, - })) - .await; - return; - } - if let Some(message) = websocket_response_completed_parse_error(&raw) { - discard_stream_websocket( - websocket, - pool_return, - StreamWebSocketDiscardReason::InvalidCompletedResponse, - ) - .await; - let _ = tx - .send(Err(CodexWebSocketExchangeError::InvalidCompletedResponse { - message, - })) - .await; - return; - } if let Some(headers) = websocket_rate_limit_event_headers(&raw) { metadata.rate_limit_headers.extend(headers.iter().cloned()); rate_limit_header_updates.lock().await.extend(headers); continue; } + update_websocket_response_metadata(&mut metadata, &raw); if let Some(metadata_turn_state) = websocket_metadata_turn_state(&raw) { metadata.turn_state = Some(metadata_turn_state); *turn_state_update.lock().await = metadata.turn_state.clone(); continue; } let event = websocket_event_type(&raw); + if event.as_deref() == Some("response.completed") { + match websocket_response_completed_id(&raw) { + Ok(Some(response_id)) => continuation.record_completed(response_id), + Ok(None) => unreachable!("event type was checked"), + Err(message) => { + discard_stream_websocket( + websocket, + pool_return, + StreamWebSocketDiscardReason::InvalidCompletedResponse, + ) + .await; + let _ = tx + .send(Err(CodexWebSocketExchangeError::InvalidCompletedResponse { + message, + })) + .await; + return; + } + } + } let Some(frame) = websocket_event_to_sse_frame(&raw) else { continue; }; let terminal = event.as_deref().is_some_and(is_terminal_websocket_event); - let has_first_output = response_body_has_first_output(frame.as_bytes()); if tx.send(Ok(Bytes::from(frame))).await.is_err() { discard_stream_websocket( websocket, @@ -622,11 +692,30 @@ async fn forward_websocket_response_stream( .await; return; } - if has_first_output { - saw_first_output = true; - } if terminal { - finish_stream_websocket(websocket, metadata, pool_return.take()).await; + match event.as_deref() { + Some("response.completed") => { + finish_stream_websocket(websocket, metadata, continuation, pool_return.take()) + .await; + } + Some("response.incomplete") => { + discard_stream_websocket( + websocket, + pool_return, + StreamWebSocketDiscardReason::IncompleteResponse, + ) + .await; + } + Some("response.failed" | "error") => { + discard_stream_websocket( + websocket, + pool_return, + StreamWebSocketDiscardReason::FailedResponse, + ) + .await; + } + _ => unreachable!("terminal websocket event was matched above"), + } return; } } @@ -642,55 +731,24 @@ async fn forward_websocket_response_stream( .await; } -fn receive_idle_timeout(saw_response_frame: bool) -> Duration { - if saw_response_frame { +fn receive_idle_timeout( + saw_upstream_activity: bool, + initial_event_timeout: Option, +) -> Duration { + if saw_upstream_activity { WEBSOCKET_ACTIVE_STREAM_IDLE_TIMEOUT } else { - WEBSOCKET_RECEIVE_IDLE_TIMEOUT + initial_event_timeout + .filter(|timeout| !timeout.is_zero()) + .unwrap_or(WEBSOCKET_RECEIVE_IDLE_TIMEOUT) } } -fn receive_idle_timeout_with_first_token_deadline( - saw_first_output: bool, - first_token_started_at: Instant, - first_token_timeout: Option, -) -> Result { - if saw_first_output { - return Ok(WEBSOCKET_ACTIVE_STREAM_IDLE_TIMEOUT); - } - receive_timeout_before_first_output(first_token_started_at, first_token_timeout) -} - -fn receive_timeout_before_first_output( - first_token_started_at: Instant, - first_token_timeout: Option, -) -> Result { - let Some(first_token_timeout) = first_token_timeout.filter(|timeout| !timeout.is_zero()) else { - return Ok(WEBSOCKET_RECEIVE_IDLE_TIMEOUT); - }; - let Some(remaining) = first_token_timeout.checked_sub(first_token_started_at.elapsed()) else { - return Err(CodexWebSocketExchangeError::FirstTokenTimeout { - timeout: first_token_timeout, - }); - }; - if remaining.is_zero() { - return Err(CodexWebSocketExchangeError::FirstTokenTimeout { - timeout: first_token_timeout, - }); - } - Ok(WEBSOCKET_RECEIVE_IDLE_TIMEOUT.min(remaining)) -} - -fn elapsed_first_token_timeout( - first_token_started_at: Instant, - first_token_timeout: Option, -) -> Option { - let first_token_timeout = first_token_timeout.filter(|timeout| !timeout.is_zero())?; - (first_token_started_at.elapsed() >= first_token_timeout).then_some(first_token_timeout) -} - -pub(super) fn is_first_token_timeout(error: &CodexWebSocketExchangeError) -> bool { - matches!(error, CodexWebSocketExchangeError::FirstTokenTimeout { .. }) +pub(super) fn is_initial_event_timeout(error: &CodexWebSocketExchangeError) -> bool { + matches!( + error, + CodexWebSocketExchangeError::InitialEventTimeout { .. } + ) } async fn next_websocket_message( @@ -708,6 +766,7 @@ async fn next_websocket_message( async fn finish_stream_websocket( mut websocket: PumpedWebSocket, metadata: CodexWebSocketConnectionMetadata, + continuation: WebSocketContinuationState, pool_return: Option, ) { let Some(pool_return) = pool_return else { @@ -719,6 +778,7 @@ async fn finish_stream_websocket( .put(PooledWebSocketConnection { websocket, metadata: reusable_websocket_metadata(metadata), + continuation, created_at: pool_return.created_at, }) .await; @@ -749,6 +809,13 @@ fn websocket_rate_limit_event_headers(raw: &str) -> Option .map(|parsed| events::rate_limits_to_header_pairs(&parsed)) } +fn update_websocket_response_metadata(metadata: &mut CodexWebSocketConnectionMetadata, raw: &str) { + response_meta::merge_response_metadata( + &mut metadata.response_metadata, + websocket_metadata_headers(raw), + ); +} + pub(super) fn audit_header_value(name: &str, value: &str) -> String { if is_sensitive_opening_header(name) { REDACTED_HEADER_VALUE.to_string() diff --git a/backend/src/upstream/openai/transport/websocket_pool.rs b/backend/src/upstream/openai/transport/websocket_pool.rs index 1036da43f..76926dead 100644 --- a/backend/src/upstream/openai/transport/websocket_pool.rs +++ b/backend/src/upstream/openai/transport/websocket_pool.rs @@ -18,7 +18,7 @@ const DEFAULT_MAX_AGE: Duration = Duration::from_mins(55); const DEFAULT_MAINTENANCE_INTERVAL: Duration = Duration::from_secs(25); const DEFAULT_PING_INTERVAL: Duration = Duration::from_secs(25); const DEFAULT_PING_TIMEOUT: Duration = Duration::from_secs(5); -pub(crate) const DEFAULT_FIRST_TOKEN_TIMEOUT: Duration = Duration::from_secs(20); +pub(crate) const DEFAULT_INITIAL_EVENT_TIMEOUT: Duration = Duration::from_secs(20); /// WebSocket 连接池 key。 #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -89,8 +89,8 @@ pub struct CodexWebSocketPoolConfig { pub ping_timeout: Duration, /// idle socket 无活动多久后视为失活。 pub liveness_timeout: Option, - /// 首个真实输出到达前的绝对超时；`None` 表示禁用首 token 熔断。 - pub first_token_timeout: Option, + /// 首个上游事件到达前的超时；`None` 表示禁用首事件超时。 + pub initial_event_timeout: Option, } impl Default for CodexWebSocketPoolConfig { @@ -105,7 +105,7 @@ impl Default for CodexWebSocketPoolConfig { // idle 连接不设失活截断：靠 ping/pong 保活，只在 max_age（55 分钟） // 或 ping 失败时关闭，最大化跨轮复用（对齐 Codex CLI 的长连接策略）。 liveness_timeout: None, - first_token_timeout: Some(DEFAULT_FIRST_TOKEN_TIMEOUT), + initial_event_timeout: Some(DEFAULT_INITIAL_EVENT_TIMEOUT), } } } @@ -148,9 +148,9 @@ impl CodexWebSocketPool { self.config.keepalive() } - /// 首个真实输出到达前的绝对超时；`None` 表示禁用首 token 熔断。 - pub(crate) fn first_token_timeout(&self) -> Option { - self.config.first_token_timeout + /// 首个上游事件到达前的超时；`None` 表示禁用首事件超时。 + pub(crate) fn initial_event_timeout(&self) -> Option { + self.config.initial_event_timeout } pub(crate) async fn acquire(&self, key: &CodexWebSocketPoolKey) -> WebSocketPoolAcquire { @@ -376,15 +376,33 @@ pub(crate) struct CodexWebSocketConnectionMetadata { pub(crate) turn_state: Option, pub(crate) set_cookie_headers: Vec, pub(crate) rate_limit_headers: Vec<(String, String)>, + pub(crate) response_metadata: super::client::CodexResponseMetadata, pub(crate) diagnostics: super::diagnostics::CodexUpstreamDiagnostics, } pub(crate) struct PooledWebSocketConnection { pub(crate) websocket: PumpedWebSocket, pub(crate) metadata: CodexWebSocketConnectionMetadata, + pub(crate) continuation: WebSocketContinuationState, pub(crate) created_at: Instant, } +/// 只随具体 WebSocket 生命周期存在的续接状态。 +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct WebSocketContinuationState { + latest_response_id: Option, +} + +impl WebSocketContinuationState { + pub(crate) fn latest_response_id(&self) -> Option<&str> { + self.latest_response_id.as_deref() + } + + pub(crate) fn record_completed(&mut self, response_id: String) { + self.latest_response_id = Some(response_id); + } +} + enum WebSocketPoolSlot { Idle(Box), Busy(WebSocketPoolReservation), diff --git a/backend/tests/api/admin/accounts_routes/lifecycle_routes.rs b/backend/tests/api/admin/accounts_routes/lifecycle_routes.rs index 68b4259d3..f5c4d59af 100644 --- a/backend/tests/api/admin/accounts_routes/lifecycle_routes.rs +++ b/backend/tests/api/admin/accounts_routes/lifecycle_routes.rs @@ -31,6 +31,8 @@ async fn admin_accounts_lifecycle_should_update_and_delete_accounts() { input_tokens: None, function_call_ids: Vec::new(), variant_hash: None, + continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal, + replay: None, created_at: chrono::Utc::now(), }, chrono::Duration::hours(1), diff --git a/backend/tests/bootstrap/tasks/coordinator.rs b/backend/tests/bootstrap/tasks/coordinator.rs index 69d0df54f..e8444b1fe 100644 --- a/backend/tests/bootstrap/tasks/coordinator.rs +++ b/backend/tests/bootstrap/tasks/coordinator.rs @@ -21,7 +21,7 @@ async fn scheduler_handle_should_shutdown_websocket_pool() { max_per_account: 1, enabled: true, ping_timeout: Duration::ZERO, - first_token_timeout: None, + initial_event_timeout: None, })); SchedulerHandle::from_websocket_pool(pool.clone()) diff --git a/backend/tests/bootstrap/tasks/model_refresh.rs b/backend/tests/bootstrap/tasks/model_refresh.rs index b60a3d639..592587987 100644 --- a/backend/tests/bootstrap/tasks/model_refresh.rs +++ b/backend/tests/bootstrap/tasks/model_refresh.rs @@ -7,6 +7,7 @@ use std::{ use codex_proxy_rs::{ fleet::pool::{AccountAcquireRequest, AccountPoolOptions, AccountPoolService}, + infra::identity::AccountPseudonymizer, models::{ store::{ModelSnapshotStore, ModelSnapshotStoreResult}, types::{BackendModelEntry, ModelPlanSnapshot}, @@ -30,6 +31,7 @@ async fn model_refresh_task_should_start_and_shutdown() { let handle = codex_proxy_rs::bootstrap::tasks::model_refresh::ModelRefreshTask::new( model_service, account_pool, + Arc::new(AccountPseudonymizer::new([7; 32])), ) .start(); @@ -78,6 +80,7 @@ async fn model_refresh_task_should_sync_model_plan_allowlist_to_account_pool() { codex_proxy_rs::bootstrap::tasks::model_refresh::ModelRefreshTask::new( model_service, account_pool.clone(), + Arc::new(AccountPseudonymizer::new([7; 32])), ) .refresh_once() .await diff --git a/backend/tests/dispatch/affinity/mod.rs b/backend/tests/dispatch/affinity/mod.rs index 2f3324eca..985962faf 100644 --- a/backend/tests/dispatch/affinity/mod.rs +++ b/backend/tests/dispatch/affinity/mod.rs @@ -120,6 +120,8 @@ fn affinity( input_tokens: Some(7), function_call_ids: vec!["call_1".to_string()], variant_hash: variant_hash.map(ToString::to_string), + continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal, + replay: None, created_at, } } diff --git a/backend/tests/dispatch/service.rs b/backend/tests/dispatch/service.rs index ebf38c624..09304be49 100644 --- a/backend/tests/dispatch/service.rs +++ b/backend/tests/dispatch/service.rs @@ -16,18 +16,10 @@ use codex_proxy_rs::{ api::AppState, bootstrap::config::AppConfig, bootstrap::services::{Services, UsageRecordOptions}, - dispatch::affinity::{ - build_conversation_identity, compute_variant_hash, prepare_variant_identity, - SessionAffinityEntry, - }, - fleet::{ - account::AccountStatus, - cookies::PgCookieStore, - pool::{AccountAcquireRequest, AccountLease}, - }, + dispatch::affinity::{ResponseReplaySnapshot, SessionAffinityEntry}, + fleet::{account::AccountStatus, cookies::PgCookieStore}, infra::identity::hash_credential, telemetry::usage::store::{PgUsageRecordStore, UsageRecordFilter}, - upstream::openai::protocol::responses::CodexResponsesRequest, }; use futures::{SinkExt, StreamExt}; use serde_json::{json, Value}; @@ -63,8 +55,6 @@ use crate::support::{ storage::{background_task_stores, create_test_redis, init_test_db, test_database_url}, }; -const TEST_INSTALLATION_ID: &str = "b4f9d503-07b1-457b-a0da-87e6836b1c43"; - fn websocket_accept_config() -> WebSocketConfig { let mut extensions = ExtensionsConfig::default(); extensions.permessage_deflate = Some(DeflateConfig::default()); @@ -210,8 +200,6 @@ const RESPONSES_FAILED_MODEL_UNSUPPORTED_SSE: &str = concat!( "\n" ); -const WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY: &str = - include_str!("../fixtures/responses/websocket/completed_with_reasoning_replay.json"); const WEBSOCKET_HISTORY_RATE_LIMITED: &str = include_str!("../fixtures/responses/websocket/history_rate_limited.json"); const WEBSOCKET_RATE_LIMITED: &str = @@ -222,57 +210,42 @@ const WEBSOCKET_FIRST_ACCOUNT_LIMITED: &str = include_str!("../fixtures/responses/websocket/first_account_limited.json"); const WEBSOCKET_SECOND_ACCOUNT_LIMITED: &str = include_str!("../fixtures/responses/websocket/second_account_limited.json"); -const WEBSOCKET_INVALID_ENCRYPTED_CONTENT: &str = - include_str!("../fixtures/responses/websocket/invalid_encrypted_content.json"); -const WEBSOCKET_PREVIOUS_RESPONSE_NOT_FOUND: &str = - include_str!("../fixtures/responses/websocket/previous_response_not_found.json"); -const REASONING_REPLAY_REQUEST_GOLDEN: &str = - include_str!("../fixtures/responses/golden/reasoning_replay_request.json"); - mod compact_routes; mod responses_http; mod responses_recovery; mod responses_websocket; mod usage_logging; -async fn test_app_state_with_pool_and_installation_id( - config: &AppConfig, - pool: PgPool, - installation_id: String, -) -> AppState { - test_app_state_with_pool_installation_id_and_usage_record_options( +async fn test_app_state_with_pool(config: &AppConfig, pool: PgPool) -> AppState { + test_app_state_with_pool_and_usage_record_options( config.clone(), pool, - installation_id, UsageRecordOptions::from_config(config), ) .await } -async fn test_app_state_with_pool_installation_id_and_usage_record_options( +async fn test_app_state_with_pool_and_usage_record_options( config: AppConfig, pool: PgPool, - installation_id: String, usage_record_options: UsageRecordOptions, ) -> AppState { let redis = create_test_redis("chat-upstream").await; - test_app_state_with_storage(config, pool, redis, installation_id, usage_record_options).await + test_app_state_with_storage(config, pool, redis, usage_record_options).await } async fn test_app_state_with_storage( config: AppConfig, pool: PgPool, redis: codex_proxy_rs::infra::redis::RedisConnection, - installation_id: String, usage_record_options: UsageRecordOptions, ) -> AppState { insert_model_snapshot(&redis).await; let stores = background_task_stores(pool, redis); - let services = Services::try_with_installation_id_and_usage_record_options( + let services = Services::try_with_usage_record_options( &config, stores, crate::support::fingerprint::runtime_test_fingerprint(), - Some(installation_id), usage_record_options, ) .expect("failed to build runtime services with configured TLS transport"); @@ -289,27 +262,11 @@ async fn test_app_with_account( axum::Router, String, crate::support::storage::TestDatabaseGuard, -) { - test_app_with_account_and_installation_id(base_url, TEST_INSTALLATION_ID.to_string()).await -} - -async fn test_app_with_account_and_installation_id( - base_url: String, - installation_id: String, -) -> ( - axum::Router, - String, - crate::support::storage::TestDatabaseGuard, ) { let (pool, dir) = init_test_db("openai-chat-upstream").await; let api_key = insert_client_api_key(&pool).await; insert_account(&pool).await; - let state = test_app_state_with_pool_and_installation_id( - &test_config(test_database_url(), base_url), - pool, - installation_id, - ) - .await; + let state = test_app_state_with_pool(&test_config(test_database_url(), base_url), pool).await; state .services .account_pool @@ -352,12 +309,8 @@ async fn test_app_without_accounts( ) { let (pool, dir) = init_test_db("openai-responses-no-accounts").await; let api_key = insert_client_api_key(&pool).await; - let state = test_app_state_with_pool_and_installation_id( - &test_config(test_database_url(), base_url), - pool.clone(), - TEST_INSTALLATION_ID.to_string(), - ) - .await; + let state = + test_app_state_with_pool(&test_config(test_database_url(), base_url), pool.clone()).await; state .services .account_pool @@ -392,12 +345,8 @@ async fn test_app_with_account_pool_and_disabled_telemetry( let (pool, dir) = init_test_db("openai-record-disabled-logging").await; let api_key = insert_client_api_key(&pool).await; insert_account(&pool).await; - let state = test_app_state_with_pool_and_installation_id( - &test_config(test_database_url(), base_url), - pool.clone(), - TEST_INSTALLATION_ID.to_string(), - ) - .await; + let state = + test_app_state_with_pool(&test_config(test_database_url(), base_url), pool.clone()).await; state .services .account_pool @@ -424,10 +373,9 @@ async fn test_app_with_account_pool_and_telemetry_capture_body( enabled: true, capture_body: true, }; - let state = test_app_state_with_pool_installation_id_and_usage_record_options( + let state = test_app_state_with_pool_and_usage_record_options( config, pool.clone(), - TEST_INSTALLATION_ID.to_string(), usage_record_options, ) .await; @@ -467,12 +415,7 @@ async fn test_app_with_account_pool_config_and_affinity( insert_account(&pool).await; let mut config = test_config(test_database_url(), base_url); configure(&mut config); - let state = test_app_state_with_pool_and_installation_id( - &config, - pool.clone(), - TEST_INSTALLATION_ID.to_string(), - ) - .await; + let state = test_app_state_with_pool(&config, pool.clone()).await; if let Some(response_id) = affinity_response_id { insert_session_affinity(&state.services.session_affinity, response_id, "acct_chat").await; } @@ -497,12 +440,7 @@ async fn test_app_with_two_accounts_and_affinity( insert_named_account(&pool, "acct_a", "access-default", "chatgpt-default").await; insert_named_account(&pool, "acct_z", "access-affinity", "chatgpt-affinity").await; let now = Utc::now(); - let state = test_app_state_with_pool_and_installation_id( - &test_config(test_database_url(), base_url), - pool, - TEST_INSTALLATION_ID.to_string(), - ) - .await; + let state = test_app_state_with_pool(&test_config(test_database_url(), base_url), pool).await; state .services .session_affinity @@ -516,6 +454,13 @@ async fn test_app_with_two_accounts_and_affinity( input_tokens: None, function_call_ids: Vec::new(), variant_hash: None, + continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted, + replay: Some(ResponseReplaySnapshot { + full_input: vec![json!({ + "role": "user", + "content": "Original managed history" + })], + }), created_at: now, }, ) @@ -560,12 +505,8 @@ async fn test_app_with_two_accounts_and_affinity_status( .await .unwrap(); let now = Utc::now(); - let state = test_app_state_with_pool_and_installation_id( - &test_config(test_database_url(), base_url), - pool.clone(), - TEST_INSTALLATION_ID.to_string(), - ) - .await; + let state = + test_app_state_with_pool(&test_config(test_database_url(), base_url), pool.clone()).await; state .services .session_affinity @@ -579,6 +520,13 @@ async fn test_app_with_two_accounts_and_affinity_status( input_tokens: None, function_call_ids: Vec::new(), variant_hash: None, + continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted, + replay: Some(ResponseReplaySnapshot { + full_input: vec![json!({ + "role": "user", + "content": "Original affinity history" + })], + }), created_at: now, }, ) @@ -661,12 +609,7 @@ async fn test_app_with_two_accounts_and_state_config( .await; let mut config = test_config(test_database_url(), base_url); configure(&mut config); - let state = test_app_state_with_pool_and_installation_id( - &config, - pool.clone(), - TEST_INSTALLATION_ID.to_string(), - ) - .await; + let state = test_app_state_with_pool(&config, pool.clone()).await; state .services .account_pool @@ -682,6 +625,46 @@ async fn test_app_with_two_accounts_and_state_config( ) } +async fn test_app_with_ranked_accounts( + base_url: String, + account_count: usize, +) -> ( + axum::Router, + String, + crate::support::storage::TestDatabaseGuard, +) { + let (pool, dir) = init_test_db("openai-responses-full-candidate-ledger").await; + let api_key = insert_client_api_key(&pool).await; + for index in 0..account_count { + let account_id = format!("acct-{index}"); + insert_named_account( + &pool, + &account_id, + &format!("access-{index}"), + &format!("chatgpt-{index}"), + ) + .await; + sqlx::query( + "insert into account_usage (account_id, request_count, window_request_count) values ($1, $2, $2)", + ) + .bind(&account_id) + .bind(i64::try_from(index).unwrap()) + .execute(&pool) + .await + .unwrap(); + } + let mut config = test_config(test_database_url(), base_url); + config.auth.rotation_strategy = "quota_reset_priority".to_string(); + let state = test_app_state_with_pool(&config, pool).await; + state + .services + .account_pool + .restore_from_store() + .await + .expect("account pool should restore"); + (router::router().with_state(state), api_key, dir) +} + async fn seed_openai_admin_key(pool: &PgPool, key: &str) { sqlx::query( "insert into runtime_settings ( @@ -784,6 +767,8 @@ async fn insert_session_affinity( input_tokens: None, function_call_ids: Vec::new(), variant_hash: None, + continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted, + replay: None, created_at: Utc::now(), }, ) @@ -1145,66 +1130,6 @@ async fn accept_websocket_response_with_message( payload } -async fn accept_followup_websocket_response( - listener: &TcpListener, - websocket: &mut tokio_tungstenite::WebSocketStream, - response_message: String, -) -> Value { - tokio::select! { - message = websocket.next() => { - match message { - Some(Ok(message)) if message.is_text() => { - let payload = serde_json::from_str::(&message.into_text().unwrap()) - .expect("websocket payload should be json"); - websocket - .send(Message::Text(response_message.into())) - .await - .unwrap(); - payload - } - _ => accept_websocket_response_with_message(listener, response_message).await, - } - } - accepted = listener.accept() => { - let (stream, _) = accepted.unwrap(); - let mut followup = accept_async(stream).await.unwrap(); - let payload = - send_websocket_response_and_capture_payload(&mut followup, response_message).await; - followup.close(None).await.unwrap(); - payload - } - } -} - -async fn accept_followup_websocket_response_sequence( - listener: &TcpListener, - websocket: &mut tokio_tungstenite::WebSocketStream, - response_messages: Vec, -) -> Value { - let followup = tokio::select! { - message = websocket.next() => { - match message { - Some(Ok(message)) if message.is_text() => { - let payload = serde_json::from_str::(&message.into_text().unwrap()) - .expect("websocket payload should be json"); - send_websocket_messages(websocket, &response_messages).await; - return payload; - } - _ => None, - } - } - accepted = listener.accept() => Some(accepted.unwrap().0), - }; - - let stream = followup.expect("follow-up websocket connection should be available"); - let mut followup = accept_async(stream).await.unwrap(); - let message = followup.next().await.unwrap().unwrap(); - let payload = serde_json::from_str::(&message.into_text().unwrap()) - .expect("websocket payload should be json"); - send_websocket_messages(&mut followup, &response_messages).await; - payload -} - async fn send_websocket_messages( websocket: &mut tokio_tungstenite::WebSocketStream, messages: &[String], @@ -1270,13 +1195,12 @@ async fn read_http_upgrade_request(stream: &mut TcpStream) -> String { String::from_utf8(request).unwrap() } -fn assert_response_failed_stream( +fn assert_openai_error_body( body: &str, expected_error_type: &str, expected_code: &str, expected_fragments: &[&str], ) { - assert!(body.contains("event: response.failed")); assert!( body.contains(&format!("\"type\":\"{expected_error_type}\"")) || body.contains(&format!("\"type\": \"{expected_error_type}\"")), @@ -1380,30 +1304,6 @@ fn websocket_completed_response( .to_string() } -fn websocket_completed_function_call_response(response_id: &str, call_id: &str) -> String { - json!({ - "type": "response.completed", - "response": { - "id": response_id, - "object": "response", - "status": "completed", - "output": [{ - "type": "function_call", - "id": format!("fc_{call_id}"), - "call_id": call_id, - "name": "lookup", - "arguments": "{}" - }], - "usage": { - "input_tokens": 6, - "output_tokens": 2, - "total_tokens": 8 - } - } - }) - .to_string() -} - fn responses_previous_stream_request(api_key: &str, content: &str) -> Request { Request::builder() .method("POST") @@ -1516,16 +1416,29 @@ async fn latest_response_usage_record(pool: &PgPool) -> ResponseUsageRecordSnaps } async fn latest_response_ops_error_log(pool: &PgPool) -> ResponseUsageRecordSnapshot { + latest_response_ops_error_log_with_message(pool, None).await +} + +async fn latest_response_upstream_ops_error_log(pool: &PgPool) -> ResponseUsageRecordSnapshot { + latest_response_ops_error_log_with_message(pool, Some("v1 responses upstream request failed")) + .await +} + +async fn latest_response_ops_error_log_with_message( + pool: &PgPool, + message: Option<&str>, +) -> ResponseUsageRecordSnapshot { let row = sqlx::query_as::<_, OpsErrorLogSnapshot>( "select request_id, account_id, route, model, transport, status_code, client_status_code, upstream_status_code, failure_class, attempt_index, response_id, metadata_json::text as metadata_json from ops_error_logs - where kind = $1 + where kind = $1 and ($2::text is null or message = $2) order by created_at desc, id desc limit 1", ) .bind("v1.response") + .bind(message) .fetch_optional(pool) .await .unwrap(); diff --git a/backend/tests/dispatch/service/compact_routes.rs b/backend/tests/dispatch/service/compact_routes.rs index 9d5b08aa7..75399d27b 100644 --- a/backend/tests/dispatch/service/compact_routes.rs +++ b/backend/tests/dispatch/service/compact_routes.rs @@ -203,10 +203,12 @@ async fn responses_compact_should_post_json_to_codex_compact_upstream() { json!([{"type": "function", "name": "lookup"}]) ); assert_eq!(upstream_body["text"]["format"]["type"], "json_schema"); - // compact 只剥离 stream（端点不支持流式）；store/prompt_cache_key 等业务字段原样透传。 + // compact 只剥离 stream；prompt_cache_key 按账号作用域伪名化。 assert!(upstream_body.get("stream").is_none()); assert_eq!(upstream_body["store"], true); - assert_eq!(upstream_body["prompt_cache_key"], "session-seed"); + assert!(upstream_body["prompt_cache_key"] + .as_str() + .is_some_and(|value| value.starts_with("wi_") && value != "session-seed")); assert_eq!(upstream_body["input"].as_array().unwrap().len(), 4); assert_eq!(upstream_body["input"][1]["ignored"], "drop"); assert_eq!( @@ -333,8 +335,11 @@ async fn responses_compact_should_preserve_upstream_client_error_status() { let body = response_json(response).await; assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["code"], "codex_client_error"); + assert_eq!(body["error"]["code"], "invalid_encrypted_content"); + assert_eq!( + body["error"]["message"], + "The encrypted content could not be verified." + ); let event = latest_response_ops_error_log(&pool).await; let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); assert_eq!( @@ -354,7 +359,7 @@ async fn responses_compact_should_preserve_upstream_client_error_status() { } #[tokio::test] -async fn responses_compact_should_map_upstream_internal_error_to_bad_gateway() { +async fn responses_compact_should_report_upstream_unavailable_after_all_5xx_retries() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses/compact")) @@ -387,7 +392,7 @@ async fn responses_compact_should_map_upstream_internal_error_to_bad_gateway() { let status = response.status(); let body = response_json(response).await; - assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); assert_eq!(body["error"]["type"], "server_error"); assert_eq!(body["error"]["code"], "upstream_error"); } diff --git a/backend/tests/dispatch/service/responses_http.rs b/backend/tests/dispatch/service/responses_http.rs index fff4f1aa0..89c02f444 100644 --- a/backend/tests/dispatch/service/responses_http.rs +++ b/backend/tests/dispatch/service/responses_http.rs @@ -60,6 +60,11 @@ async fn responses_should_honor_explicit_http_sse_transport() { .respond_with( ResponseTemplate::new(200) .insert_header("content-type", "text/event-stream") + .insert_header("openai-model", "gpt-5.5-effective") + .insert_header("x-models-etag", "models-etag-2") + .insert_header("x-reasoning-included", "true") + .insert_header("set-cookie", "account-secret=value") + .insert_header("x-codex-primary-used-percent", "12") .set_body_string(RESPONSES_STREAM_USAGE_SSE), ) .expect(1) @@ -87,6 +92,26 @@ async fn responses_should_honor_explicit_http_sse_transport() { .await .unwrap(); let status = response.status(); + let effective_model = response + .headers() + .get("openai-model") + .and_then(|value| value.to_str().ok()) + .map(ToString::to_string); + let models_etag = response + .headers() + .get("x-models-etag") + .and_then(|value| value.to_str().ok()) + .map(ToString::to_string); + let reasoning_included = response + .headers() + .get("x-reasoning-included") + .and_then(|value| value.to_str().ok()) + .map(ToString::to_string); + assert!(response.headers().get("set-cookie").is_none()); + assert!(response + .headers() + .get("x-codex-primary-used-percent") + .is_none()); let body = response_text(response).await; let requests = server.received_requests().await.unwrap(); let upstream_body: Value = serde_json::from_slice(&requests[0].body).unwrap(); @@ -96,6 +121,9 @@ async fn responses_should_honor_explicit_http_sse_transport() { assert_eq!(requests.len(), 1); assert_eq!(requests[0].method.as_str(), "POST"); assert_eq!(upstream_body["model"], "gpt-5.5"); + assert_eq!(effective_model.as_deref(), Some("gpt-5.5-effective")); + assert_eq!(models_etag.as_deref(), Some("models-etag-2")); + assert_eq!(reasoning_included.as_deref(), Some("true")); assert!(upstream_body.get("use_websocket").is_none()); } @@ -254,13 +282,14 @@ async fn responses_should_use_websocket_upstream_by_default_while_serving_sse() // 透明代理：`generate` 原样透传上游，不再被剥离。 assert_eq!(captured.payload["generate"], false); assert!(captured.payload.get("previous_response_id").is_none()); - assert!(captured.payload["prompt_cache_key"] + assert!(captured.payload.get("prompt_cache_key").is_none()); + let metadata = captured.payload["client_metadata"] + .as_object() + .expect("official websocket timing metadata should be present"); + assert_eq!(metadata.len(), 1); + assert!(metadata["x-codex-ws-stream-request-start-ms"] .as_str() - .is_some_and(|value| value.starts_with("cp_"))); - assert!(captured.payload["client_metadata"]["x-codex-installation-id"].is_string()); - assert!(captured.payload["client_metadata"]["x-codex-window-id"] - .as_str() - .is_some_and(|value| value.starts_with("cp_") && value.ends_with(":0"))); + .is_some_and(|value| value.parse::().is_ok())); } #[tokio::test] @@ -299,7 +328,7 @@ async fn responses_non_stream_should_record_websocket_transport_metadata() { let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); assert_eq!(metadata["stream"], false); assert_eq!(event.transport.as_deref(), Some("websocket")); - assert_eq!(metadata["websocketPool"]["kind"], "new"); + assert!(metadata.get("websocketPool").is_none()); } #[tokio::test] @@ -388,8 +417,7 @@ async fn responses_should_forward_text_schema_verbatim_to_upstream() { #[tokio::test] async fn responses_should_pass_through_tuple_schema_output_verbatim() { - // 透明代理：不再重写请求 schema，因此也不再对响应做 tuple 回转换—— - // 上游返回的 object 形态 `{"point":{"0":1,"1":2}}` 原样透传给客户端。 + // 透明代理：completed 事件中 output_text 和 output 的原始值都不用 delta 覆盖。 let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses")) @@ -420,7 +448,7 @@ async fn responses_should_pass_through_tuple_schema_output_verbatim() { assert_eq!(status, StatusCode::OK); assert_eq!(body["id"], "resp_tuple"); - assert_eq!(body["output_text"], "{\"point\":{\"0\":1,\"1\":2}}"); + assert_eq!(body["output_text"], "{\"point\":{\"0\":0,\"1\":0}}"); assert_eq!( body["output"][0]["content"][0]["text"], "{\"point\":{\"0\":1,\"1\":2}}" @@ -531,17 +559,6 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop .find(|request| request.url.path() == "/codex/responses") .expect("responses upstream request should be sent"); let upstream_body: Value = serde_json::from_slice(&upstream.body).unwrap(); - // 透明代理：window-id 不再从 client_metadata 回退读取，客户端未通过 body - // `codexWindowId` 或 `x-codex-window-id` 请求头提供时，window-id 由会话派生（`:0` 后缀）。 - let identity = build_conversation_identity(Some("pcache"), None, "acct_chat"); - let conversation_id = identity - .conversation_id - .as_deref() - .expect("conversation identity should be scoped"); - let window_id = identity - .window_id - .as_deref() - .expect("window identity should be scoped"); let upstream_header = |name: &str| { upstream .headers @@ -554,38 +571,36 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop assert_eq!(upstream_body["service_tier"], "priority"); assert_eq!(upstream_body["reasoning"], json!({"effort": "high"})); assert!(upstream_body.get("include").is_none()); - assert_eq!(upstream_body["prompt_cache_key"], conversation_id); + let prompt_cache_key = upstream_body["prompt_cache_key"] + .as_str() + .expect("prompt cache key should be account scoped"); + assert_ne!(prompt_cache_key, "pcache"); assert_eq!(upstream_body["client_metadata"]["safe"], "yes"); + assert_eq!(upstream_body["client_metadata"]["drop"], 42); assert_eq!( upstream_body["client_metadata"]["x-openai-subagent"], "review" ); - assert_eq!( - upstream_body["client_metadata"]["x-codex-installation-id"], - TEST_INSTALLATION_ID - ); - assert_eq!( - upstream_body["client_metadata"]["session_id"], - conversation_id - ); - assert_eq!( - upstream_body["client_metadata"]["thread_id"], - conversation_id - ); - assert_eq!( - upstream_body["client_metadata"]["x-codex-window-id"], - window_id - ); + let installation_id = upstream_body["client_metadata"]["x-codex-installation-id"] + .as_str() + .expect("installation id should be account scoped"); + assert!(uuid::Uuid::parse_str(installation_id).is_ok()); + assert!(upstream_body["client_metadata"].get("session_id").is_none()); + assert!(upstream_body["client_metadata"].get("thread_id").is_none()); + let window_id = upstream_body["client_metadata"]["x-codex-window-id"] + .as_str() + .expect("window id should be account scoped"); + assert_ne!(window_id, "window-metadata"); assert_eq!( upstream_body["client_metadata"]["x-codex-turn-metadata"], "meta-metadata" ); - assert_eq!( - upstream_body["client_metadata"]["x-codex-parent-thread-id"], - "parent-metadata" - ); - assert_eq!(upstream_header("session-id"), Some(conversation_id)); - assert_eq!(upstream_header("thread-id"), Some(conversation_id)); + let parent_thread_id = upstream_body["client_metadata"]["x-codex-parent-thread-id"] + .as_str() + .expect("parent thread id should be account scoped"); + assert_ne!(parent_thread_id, "parent-metadata"); + assert_eq!(upstream_header("session-id"), None); + assert_eq!(upstream_header("thread-id"), None); assert_eq!(upstream_header("session_id"), None); assert_eq!(upstream_header("x-codex-window-id"), Some(window_id)); assert_eq!(upstream_header("x-codex-turn-state"), Some("turn-body")); @@ -598,7 +613,10 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop Some("true") ); assert_eq!(upstream_header("version"), Some("2026-06-12")); - assert_eq!(upstream_header("x-codex-parent-thread-id"), None); + assert_eq!( + upstream_header("x-codex-parent-thread-id"), + Some(parent_thread_id) + ); assert_eq!(upstream_header("x-openai-subagent"), Some("review")); // 透明代理：上下文透传字段提取到代理控制状态用于加请求头，body 中同时原样保留。 for context_field in [ @@ -766,7 +784,7 @@ async fn responses_should_preserve_input_items_before_upstream() { } #[tokio::test] -async fn responses_should_reconstruct_non_stream_output_text_from_sse_deltas() { +async fn responses_should_not_synthesize_non_stream_output_from_sse_deltas() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses")) @@ -803,13 +821,12 @@ async fn responses_should_reconstruct_non_stream_output_text_from_sse_deltas() { let body = response_json(response).await; assert_eq!(body["id"], "resp_text"); - assert_eq!(body["output"][0]["role"], "assistant"); - assert_eq!(body["output"][0]["content"][0]["text"], "hello from deltas"); - assert_eq!(body["output_text"], "hello from deltas"); + assert_eq!(body["output"], json!([])); + assert!(body.get("output_text").is_none()); } #[tokio::test] -async fn responses_should_use_done_output_items_when_completed_output_is_empty() { +async fn responses_should_not_inject_done_items_when_completed_output_is_empty() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses")) @@ -846,8 +863,8 @@ async fn responses_should_use_done_output_items_when_completed_output_is_empty() let body = response_json(response).await; assert_eq!(body["id"], "resp_item"); - assert_eq!(body["output"][0]["content"][0]["text"], "from done item"); - assert_eq!(body["output_text"], "from done item"); + assert_eq!(body["output"], json!([])); + assert!(body.get("output_text").is_none()); } #[tokio::test] @@ -1201,7 +1218,7 @@ async fn responses_should_prefer_session_affinity_account_for_previous_response( } #[tokio::test] -async fn responses_should_preserve_explicit_history_when_banned_affinity_switches_account() { +async fn responses_should_replay_full_history_when_banned_affinity_switches_account() { let (base_url, upstream) = spawn_single_websocket_completed_upstream("resp_after_banned_affinity").await; let (app, state, api_key, _pool, _dir) = @@ -1242,19 +1259,21 @@ async fn responses_should_preserve_explicit_history_when_banned_affinity_switche captured_header(&captured.headers, "chatgpt-account-id"), Some("chatgpt-primary") ); + assert!(captured_header(&captured.headers, "x-codex-turn-state").is_none()); + assert!(captured.payload.get("previous_response_id").is_none()); assert_eq!( - captured_header(&captured.headers, "x-codex-turn-state"), - Some("turn_affinity_risk") + captured.payload["input"][0]["content"], + "Original affinity history" ); assert_eq!( - captured.payload["previous_response_id"], - "resp_affinity_risk" + captured.payload["input"][1]["content"], + "Continue after ban" ); - assert!(affinity.is_none()); + assert!(affinity.is_some()); } #[tokio::test] -async fn responses_should_preserve_explicit_history_when_quota_affinity_switches_account() { +async fn responses_should_replay_full_history_when_quota_affinity_switches_account() { let (base_url, upstream) = spawn_single_websocket_completed_upstream("resp_after_quota_affinity").await; let (app, state, api_key, _pool, _dir) = @@ -1291,13 +1310,15 @@ async fn responses_should_preserve_explicit_history_when_quota_affinity_switches captured_header(&captured.headers, "authorization"), Some("Bearer access-primary") ); + assert!(captured_header(&captured.headers, "x-codex-turn-state").is_none()); + assert!(captured.payload.get("previous_response_id").is_none()); assert_eq!( - captured_header(&captured.headers, "x-codex-turn-state"), - Some("turn_affinity_risk") + captured.payload["input"][0]["content"], + "Original affinity history" ); assert_eq!( - captured.payload["previous_response_id"], - "resp_affinity_risk" + captured.payload["input"][1]["content"], + "Continue after quota" ); assert!(affinity.is_some()); } diff --git a/backend/tests/dispatch/service/responses_recovery.rs b/backend/tests/dispatch/service/responses_recovery.rs index c45d1d2d2..3805ae696 100644 --- a/backend/tests/dispatch/service/responses_recovery.rs +++ b/backend/tests/dispatch/service/responses_recovery.rs @@ -41,7 +41,7 @@ async fn responses_should_dispatch_to_codex_and_return_completed_response() { assert_eq!(status, StatusCode::OK); assert_eq!(body["id"], "resp_response_1"); - assert_eq!(body["output_text"], "response hello"); + assert!(body.get("output_text").is_none()); assert_eq!(body["usage"]["input_tokens"], 5); assert_eq!(body["usage"]["output_tokens"], 2); } @@ -130,8 +130,9 @@ async fn responses_should_preserve_upstream_client_error_status() { let body = response_json(response).await; assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["code"], "codex_client_error"); + assert_eq!(body["error"]["code"], "invalid_input"); + assert_eq!(body["error"]["message"], "A required parameter is invalid."); + assert!(body["error"].get("type").is_none()); } #[tokio::test] @@ -1074,9 +1075,8 @@ async fn responses_stream_should_return_model_unsupported_error_when_fallback_is .await .unwrap(); - assert_eq!(status, StatusCode::OK); - assert!(content_type.starts_with("text/event-stream")); - assert!(body.contains("event: response.failed")); + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(content_type.starts_with("application/json")); assert!(body.contains("\"code\":\"model_not_found\"")); assert!(body.contains("All accounts exhausted (1 model-unsupported)")); assert!(body.contains("model_not_available")); @@ -1129,6 +1129,156 @@ async fn responses_explicit_previous_response_should_fail_without_transparent_re assert!(!captured.retry_attempted); } +#[tokio::test] +async fn responses_external_previous_response_not_found_should_return_exact_upstream_400_once() { + let server = MockServer::start().await; + let upstream_error = json!({ + "type": "error", + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_external' not found.", + "param": "previous_response_id" + }, + "status": 400 + }); + Mock::given(method("POST")) + .and(path("/codex/responses")) + .respond_with(ResponseTemplate::new(400).set_body_json(upstream_error.clone())) + .expect(1) + .mount(&server) + .await; + let (app, api_key, _pool, _dir) = test_app_with_two_accounts(server.uri()).await; + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "gpt-5.5", + "previous_response_id": "resp_external", + "input": [{"role": "user", "content": "Continue"}], + "stream": false, + "use_websocket": false + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = response_json(response).await; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body, upstream_error); + assert_eq!(server.received_requests().await.unwrap().len(), 1); +} + +#[tokio::test] +async fn responses_managed_history_should_rebuild_full_replay_for_next_configured_candidate() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-primary")) + .respond_with(ResponseTemplate::new(401).set_body_json(json!({ + "error": {"code": "token_revoked", "message": "token revoked"} + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-secondary")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(RESPONSES_SUCCESS_SSE), + ) + .expect(1) + .mount(&server) + .await; + let (app, state, api_key, _pool, _dir) = + test_app_with_two_accounts_and_state(server.uri()).await; + state + .services + .session_affinity + .record( + "resp_managed".to_string(), + SessionAffinityEntry { + account_id: "acct_primary".to_string(), + conversation_id: "conversation-managed".to_string(), + turn_state: Some("account-bound-turn-state".to_string()), + instructions_hash: None, + input_tokens: Some(11), + function_call_ids: Vec::new(), + variant_hash: None, + continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted, + replay: Some(ResponseReplaySnapshot { + full_input: vec![ + json!({"role": "user", "content": "Earlier question"}), + json!({ + "type": "reasoning", + "id": "reasoning-old", + "encrypted_content": "account-bound-secret", + "summary": [] + }), + ], + }), + created_at: Utc::now(), + }, + ) + .await + .unwrap(); + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "gpt-5.5", + "previous_response_id": "resp_managed", + "turnState": "client-turn-state", + "input": [{"role": "user", "content": "Current question"}], + "stream": false, + "use_websocket": false + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let requests = server.received_requests().await.unwrap(); + let secondary = requests + .iter() + .find(|request| { + request + .headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + == Some("Bearer access-secondary") + }) + .expect("secondary request"); + let body: Value = serde_json::from_slice(&secondary.body).unwrap(); + assert!(body.get("previous_response_id").is_none()); + assert_eq!(body["input"][0]["content"], "Earlier question"); + assert_eq!(body["input"][1]["id"], "reasoning-old"); + assert!(body["input"][1].get("encrypted_content").is_none()); + assert_eq!(body["input"][2]["content"], "Current question"); + assert!(secondary.headers.get("x-codex-turn-state").is_none()); +} + #[tokio::test] async fn responses_explicit_unanswered_function_call_should_not_retry_without_full_context() { let (base_url, upstream) = @@ -1166,7 +1316,8 @@ async fn responses_explicit_unanswered_function_call_should_not_retry_without_fu let captured = upstream.await.unwrap(); assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(body["error"]["code"], "previous_response_unavailable"); + assert_eq!(body["error"]["code"], "codex_client_error"); + assert_eq!(body["error"]["message"], "Upstream Codex response failed"); assert_eq!(captured.payload["previous_response_id"], "resp_stale"); assert!(!captured.retry_attempted); } @@ -1270,7 +1421,7 @@ async fn responses_stream_structural_events_should_not_commit_before_history_fai let body = response_text(response).await; let captured = upstream.await.unwrap(); - assert_eq!(status, StatusCode::OK); + assert_eq!(status, StatusCode::BAD_REQUEST); assert!(body.contains("previous_response_unavailable")); assert!(!body.contains("response.created")); assert_eq!(captured.payload["previous_response_id"], "resp_stale"); @@ -1555,9 +1706,8 @@ async fn responses_stream_should_return_auth_error_when_401_fallback_is_exhauste .await .unwrap(); - assert_eq!(status, StatusCode::OK); - assert!(content_type.starts_with("text/event-stream")); - assert!(body.contains("event: response.failed")); + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert!(content_type.starts_with("application/json")); assert!(body.contains("\"code\":\"invalid_api_key\"")); assert!(body.contains("All accounts exhausted (1 expired)")); assert!(body.contains("token_revoked")); @@ -1853,6 +2003,62 @@ async fn responses_should_retry_same_account_after_5xx_before_fallback() { assert_eq!(secondary_requests, 0, "requests: {authorizations:?}"); } +#[tokio::test] +async fn responses_should_traverse_all_configured_strategy_candidates_without_fixed_cap() { + let server = MockServer::start().await; + for index in 0..6 { + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", format!("Bearer access-{index}"))) + .respond_with(ResponseTemplate::new(401).set_body_json(json!({ + "error": { + "type": "invalid_request_error", + "code": "token_invalidated", + "message": "token is no longer valid" + } + }))) + .expect(1) + .mount(&server) + .await; + } + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-6")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(RESPONSES_SUCCESS_SSE), + ) + .expect(1) + .mount(&server) + .await; + + let (app, api_key, _dir) = test_app_with_ranked_accounts(server.uri(), 7).await; + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "gpt-5.5", + "input": [{"role": "user", "content": "Say hello"}], + "stream": false, + "use_websocket": false + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(server.received_requests().await.unwrap().len(), 7); +} + #[tokio::test] async fn responses_stream_should_retry_same_account_after_5xx_before_fallback() { let server = MockServer::start().await; @@ -2240,9 +2446,8 @@ async fn responses_stream_should_return_rate_limit_error_when_429_fallback_is_ex .await .unwrap(); - assert_eq!(status, StatusCode::OK); - assert!(content_type.starts_with("text/event-stream")); - assert!(body.contains("event: response.failed")); + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + assert!(content_type.starts_with("application/json")); assert!(body.contains("rate_limit_exceeded")); assert!(body.contains("All accounts exhausted (1 rate-limited)")); assert!(body.contains("rate limited")); @@ -2316,9 +2521,8 @@ async fn responses_stream_should_return_quota_error_when_402_fallback_is_exhaust .await .unwrap(); - assert_eq!(status, StatusCode::OK); - assert!(content_type.starts_with("text/event-stream")); - assert!(body.contains("event: response.failed")); + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + assert!(content_type.starts_with("application/json")); assert!(body.contains("\"code\":\"insufficient_quota\"")); assert!(body.contains("All accounts exhausted (1 quota-exhausted)")); assert!(body.contains("quota reached")); diff --git a/backend/tests/dispatch/service/responses_websocket.rs b/backend/tests/dispatch/service/responses_websocket.rs index 0de9d52d1..c3229268b 100644 --- a/backend/tests/dispatch/service/responses_websocket.rs +++ b/backend/tests/dispatch/service/responses_websocket.rs @@ -14,8 +14,11 @@ async fn responses_websocket_should_stream_first_frame_before_terminal_event() { websocket .send(Message::Text( json!({ - "type": "response.output_text.delta", - "delta": "first websocket frame" + "type": "response.created", + "response": { + "id": "resp_ws_streaming", + "status": "in_progress" + } }) .to_string() .into(), @@ -72,7 +75,7 @@ async fn responses_websocket_should_stream_first_frame_before_terminal_event() { terminal_tx.send(()).unwrap(); let payload = upstream.await.unwrap(); - assert!(first_chunk.contains("event: response.output_text.delta")); + assert!(first_chunk.contains("event: response.created")); assert_eq!(payload["type"], "response.create"); } @@ -348,8 +351,8 @@ async fn responses_websocket_should_reuse_connection_for_recorded_conversation() } #[tokio::test] -async fn responses_websocket_should_retry_fresh_connection_when_reused_connection_dies_before_first_output( -) { +async fn responses_websocket_should_replay_history_when_reused_connection_dies_before_first_output() +{ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let base_url = format!("http://{}", listener.local_addr().unwrap()); let upstream = tokio::spawn(async move { @@ -419,14 +422,13 @@ async fn responses_websocket_should_retry_fresh_connection_when_reused_connectio stale_payload["previous_response_id"], "resp_stale_reuse_first" ); + assert!(fresh_payload.get("previous_response_id").is_none()); assert_eq!( - fresh_payload["previous_response_id"], - "resp_stale_reuse_first" - ); - assert_eq!( - fresh_payload["prompt_cache_key"], first_payload["prompt_cache_key"], - "fresh retry should keep the recorded conversation identity" + fresh_payload["input"][0]["content"], + "prime stale websocket reuse" ); + assert!(first_payload.get("prompt_cache_key").is_none()); + assert!(fresh_payload.get("prompt_cache_key").is_none()); } #[tokio::test] @@ -560,18 +562,17 @@ async fn responses_websocket_should_not_reuse_connection_when_pool_is_disabled() !reused_connection, "disabled pool reused the upstream websocket" ); + assert!(first_payload.get("prompt_cache_key").is_none()); + assert!(second_payload.get("prompt_cache_key").is_none()); + assert!(second_payload.get("previous_response_id").is_none()); assert_eq!( - second_payload["prompt_cache_key"], first_payload["prompt_cache_key"], - "disabling the pool must not change the recorded conversation key" - ); - assert_eq!( - second_payload["previous_response_id"], - "resp_disabled_pool_first" + second_payload["input"][0]["content"], + "do not reuse this upstream websocket" ); } #[tokio::test] -async fn responses_websocket_stream_should_record_metadata_turn_state_for_continuation() { +async fn responses_websocket_stream_should_strip_turn_state_when_replaying_on_new_connection() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let base_url = format!("http://{}", listener.local_addr().unwrap()); let upstream = tokio::spawn(async move { @@ -678,397 +679,8 @@ async fn responses_websocket_stream_should_record_metadata_turn_state_for_contin let (first_payload, second_payload, second_headers) = upstream.await.unwrap(); assert!(first_payload.get("previous_response_id").is_none()); - assert_eq!( - second_payload["previous_response_id"], - "resp_metadata_turn_state" - ); - assert_eq!( - captured_header(&second_headers, "x-codex-turn-state"), - Some("turn-from-metadata") - ); -} - -#[tokio::test] -async fn responses_websocket_should_implicitly_resume_full_history_with_reasoning_replay() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut websocket = accept_async(stream).await.unwrap(); - let first_payload = send_websocket_response_and_capture_payload( - &mut websocket, - WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY.trim().to_string(), - ) - .await; - let second_payload = accept_followup_websocket_response( - &listener, - &mut websocket, - websocket_completed_response("resp_implicit_resume_second", 4, 1), - ) - .await; - let _ = websocket.close(None).await; - (first_payload, second_payload) - }); - let (app, api_key, _dir) = test_app_with_account(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [{"role": "user", "content": "remember this"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [ - {"role": "user", "content": "remember this"}, - {"role": "assistant", "content": "cached answer"}, - {"role": "user", "content": "continue"} - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - - let (first_payload, mut second_payload) = upstream.await.unwrap(); - assert!(first_payload["prompt_cache_key"].as_str().is_some()); - assert_eq!( - second_payload["previous_response_id"], - "resp_implicit_resume_first" - ); - assert_eq!( - second_payload["prompt_cache_key"], - first_payload["prompt_cache_key"] - ); - assert_eq!( - second_payload["input"][0]["encrypted_content"], - "enc_reasoning_replay" - ); - assert!(second_payload["input"][0].get("content").is_none()); - assert_eq!(second_payload["input"][1]["content"], "continue"); - assert_eq!(second_payload["input"].as_array().unwrap().len(), 2); - - second_payload - .as_object_mut() - .unwrap() - .remove("prompt_cache_key"); - second_payload - .as_object_mut() - .unwrap() - .remove("client_metadata"); - let expected: Value = serde_json::from_str(REASONING_REPLAY_REQUEST_GOLDEN).unwrap(); - assert_eq!(second_payload, expected); -} - -#[tokio::test] -async fn responses_websocket_should_strip_implicit_disabled_affinity_when_switching_accounts() { - responses_websocket_should_strip_implicit_affinity_when_switching_accounts( - AccountStatus::Disabled, - ) - .await; -} - -#[tokio::test] -async fn responses_websocket_should_strip_implicit_banned_affinity_when_switching_accounts() { - responses_websocket_should_strip_implicit_affinity_when_switching_accounts( - AccountStatus::Banned, - ) - .await; -} - -async fn responses_websocket_should_strip_implicit_affinity_when_switching_accounts( - status: AccountStatus, -) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let status_name = status.as_str(); - let first_response_id = format!("resp_implicit_{status_name}_first"); - let second_response_id = format!("resp_implicit_{status_name}_second"); - let upstream_first_response_id = first_response_id.clone(); - let upstream_second_response_id = second_response_id.clone(); - let upstream = tokio::spawn(async move { - let (first_stream, _) = listener.accept().await.unwrap(); - let mut first_websocket = - accept_websocket_with_authorization(first_stream, "Bearer access-primary").await; - let first_payload = send_websocket_response_and_capture_payload( - &mut first_websocket, - websocket_completed_response(&upstream_first_response_id, 4, 1), - ) - .await; - first_websocket.close(None).await.unwrap(); - - let (second_stream, _) = listener.accept().await.unwrap(); - let second_headers = Arc::new(Mutex::new(Vec::new())); - let second_headers_for_callback = Arc::clone(&second_headers); - let mut second_websocket = accept_hdr_async(second_stream, move |request, _response| { - *second_headers_for_callback.lock().unwrap() = request_headers(request); - }) - .await - .unwrap(); - let second_payload = send_websocket_response_and_capture_payload( - &mut second_websocket, - websocket_completed_response(&upstream_second_response_id, 3, 1), - ) - .await; - second_websocket.close(None).await.unwrap(); - let captured_second_headers = second_headers.lock().unwrap().clone(); - - (first_payload, second_payload, captured_second_headers) - }); - let (app, state, api_key, _pool, _dir) = test_app_with_two_accounts_and_state(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [{"role": "user", "content": "remember this"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - assert!(response_text(first_response) - .await - .contains(&format!("\"id\":\"{first_response_id}\""))); - - assert!( - state - .services - .account_pool - .set_status("acct_primary", status) - .await - ); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [ - {"role": "user", "content": "remember this"}, - {"role": "assistant", "content": "cached answer"}, - {"role": "user", "content": "continue"} - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - assert!(response_text(second_response) - .await - .contains(&format!("\"id\":\"{second_response_id}\""))); - let (first_payload, second_payload, second_headers) = upstream.await.unwrap(); - let affinity = state - .services - .session_affinity - .lookup(&first_response_id, Utc::now()) - .await; - - assert!(first_payload["prompt_cache_key"].as_str().is_some()); - assert_eq!( - captured_header(&second_headers, "authorization"), - Some("Bearer access-secondary") - ); + assert!(second_payload.get("previous_response_id").is_none()); assert!(captured_header(&second_headers, "x-codex-turn-state").is_none()); - assert!(second_payload.get("previous_response_id").is_none()); - assert_eq!(second_payload["input"].as_array().unwrap().len(), 3); - assert!(affinity.is_none()); -} - -#[tokio::test] -async fn responses_websocket_should_not_implicitly_resume_unmatched_function_call_output() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut websocket = accept_async(stream).await.unwrap(); - let first_payload = send_websocket_response_and_capture_payload( - &mut websocket, - websocket_completed_function_call_response("resp_call_first", "call_expected"), - ) - .await; - let second_payload = accept_followup_websocket_response( - &listener, - &mut websocket, - websocket_completed_response("resp_call_mismatch_second", 4, 1), - ) - .await; - let _ = websocket.close(None).await; - (first_payload, second_payload) - }); - let (app, api_key, _dir) = test_app_with_account(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "use the lookup tool", - "stream": false, - "input": [{"role": "user", "content": "call the lookup tool"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "use the lookup tool", - "stream": false, - "input": [ - {"role": "user", "content": "call the lookup tool"}, - { - "type": "function_call", - "call_id": "call_expected", - "name": "lookup", - "arguments": "{}" - }, - { - "type": "function_call_output", - "call_id": "call_missing", - "output": "tool output" - } - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - - let (first_payload, second_payload) = upstream.await.unwrap(); - assert!(first_payload["prompt_cache_key"].as_str().is_some()); - assert!(second_payload.get("previous_response_id").is_none()); - assert_eq!(second_payload["input"].as_array().unwrap().len(), 3); - assert_eq!(second_payload["input"][2]["call_id"], "call_missing"); -} - -#[tokio::test] -async fn responses_websocket_should_implicitly_resume_after_redis_affinity_restore() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let server_base_url = base_url.clone(); - let upstream = tokio::spawn(async move { - let first_payload = accept_websocket_response_with_message( - &listener, - WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY.trim().to_string(), - ) - .await; - let second_payload = accept_websocket_response_with_message( - &listener, - websocket_completed_response("resp_restored_implicit_resume", 4, 1), - ) - .await; - (first_payload, second_payload) - }); - let (pool, _guard) = init_test_db("websocket-affinity-restart").await; - let redis = create_test_redis("websocket-affinity-restart").await; - let api_key = insert_client_api_key(&pool).await; - insert_account(&pool).await; - let config = test_config(test_database_url(), base_url); - let initial_state = test_app_state_with_storage( - config.clone(), - pool.clone(), - redis.clone(), - TEST_INSTALLATION_ID.to_string(), - UsageRecordOptions::from_config(&config), - ) - .await; - initial_state - .services - .account_pool - .restore_from_store() - .await - .unwrap(); - let app = router::router().with_state(initial_state); - - let first_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [{"role": "user", "content": "remember this"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let restored_config = test_config(test_database_url(), server_base_url); - let restored_state = test_app_state_with_storage( - restored_config.clone(), - pool.clone(), - redis, - TEST_INSTALLATION_ID.to_string(), - UsageRecordOptions::from_config(&restored_config), - ) - .await; - assert_eq!( - restored_state - .services - .account_pool - .restore_from_store() - .await - .unwrap(), - 1 - ); - let restored_app = router::router().with_state(restored_state); - - let second_response = restored_app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [ - {"role": "user", "content": "remember this"}, - {"role": "assistant", "content": "cached answer"}, - {"role": "user", "content": "continue"} - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - - let (first_payload, second_payload) = upstream.await.unwrap(); - assert_eq!( - second_payload["prompt_cache_key"], - first_payload["prompt_cache_key"] - ); - assert_eq!( - second_payload["previous_response_id"], - "resp_implicit_resume_first" - ); - assert_eq!(second_payload["input"].as_array().unwrap().len(), 1); - assert_eq!(second_payload["input"][0]["content"], "continue"); } #[tokio::test] @@ -1167,324 +779,6 @@ async fn responses_websocket_pool_should_be_evicted_after_admin_account_status_c ); } -#[tokio::test] -async fn responses_websocket_should_not_implicitly_resume_self_contained_function_call_replay() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut websocket = accept_async(stream).await.unwrap(); - let first_payload = send_websocket_response_and_capture_payload( - &mut websocket, - websocket_completed_function_call_response("resp_self_contained_first", "call_self"), - ) - .await; - let second_payload = accept_followup_websocket_response( - &listener, - &mut websocket, - websocket_completed_response("resp_self_contained_second", 4, 1), - ) - .await; - let _ = websocket.close(None).await; - (first_payload, second_payload) - }); - let (app, api_key, _dir) = test_app_with_account(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "use the lookup tool", - "stream": false, - "input": [{"role": "user", "content": "call the lookup tool"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "use the lookup tool", - "stream": false, - "input": [ - {"role": "user", "content": "call the lookup tool"}, - { - "type": "function_call", - "call_id": "call_self", - "name": "lookup", - "arguments": "{}" - }, - { - "type": "function_call_output", - "call_id": "call_self", - "output": "tool output" - } - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - - let (first_payload, second_payload) = upstream.await.unwrap(); - assert!(first_payload["prompt_cache_key"].as_str().is_some()); - assert!(second_payload.get("previous_response_id").is_none()); - assert_eq!(second_payload["input"].as_array().unwrap().len(), 3); - assert_eq!(second_payload["input"][2]["call_id"], "call_self"); -} - -#[tokio::test] -async fn responses_websocket_should_not_implicitly_resume_across_codex_windows() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut websocket = accept_async(stream).await.unwrap(); - let first_payload = send_websocket_response_and_capture_payload( - &mut websocket, - WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY.trim().to_string(), - ) - .await; - let second_payload = accept_followup_websocket_response( - &listener, - &mut websocket, - websocket_completed_response("resp_window_b", 8, 2), - ) - .await; - let _ = websocket.close(None).await; - (first_payload, second_payload) - }); - let (app, api_key, _dir) = test_app_with_account(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "prompt_cache_key": "shared-variant-session", - "codexWindowId": "window-a", - "stream": false, - "input": [{"role": "user", "content": "remember this"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "prompt_cache_key": "shared-variant-session", - "codexWindowId": "window-b", - "stream": false, - "input": [ - {"role": "user", "content": "remember this"}, - {"role": "assistant", "content": "cached answer"}, - {"role": "user", "content": "continue in another window"} - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - - let (first_payload, second_payload) = upstream.await.unwrap(); - assert_eq!( - second_payload["prompt_cache_key"], - first_payload["prompt_cache_key"] - ); - assert!(second_payload.get("previous_response_id").is_none()); - assert_eq!(second_payload["input"].as_array().unwrap().len(), 3); -} - -#[tokio::test] -async fn responses_websocket_should_evict_reasoning_replay_after_invalid_encrypted_content() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut websocket = accept_async(stream).await.unwrap(); - let first_payload = send_websocket_response_and_capture_payload( - &mut websocket, - WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY.trim().to_string(), - ) - .await; - let invalid_payload = accept_followup_websocket_response( - &listener, - &mut websocket, - WEBSOCKET_INVALID_ENCRYPTED_CONTENT.trim().to_string(), - ) - .await; - let _ = websocket.close(None).await; - let retried_payload = accept_websocket_response_with_message( - &listener, - websocket_completed_response("resp_after_replay_eviction", 4, 1), - ) - .await; - (first_payload, invalid_payload, retried_payload) - }); - let (app, api_key, _dir) = test_app_with_account(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [{"role": "user", "content": "remember this"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [ - {"role": "user", "content": "remember this"}, - {"role": "assistant", "content": "cached answer"}, - {"role": "user", "content": "continue"} - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - - let (first_payload, invalid_payload, retried_payload) = upstream.await.unwrap(); - assert_eq!( - invalid_payload["previous_response_id"], - "resp_implicit_resume_first" - ); - assert_eq!( - invalid_payload["input"][0]["encrypted_content"], - "enc_reasoning_replay" - ); - assert_eq!( - retried_payload["prompt_cache_key"], - first_payload["prompt_cache_key"] - ); - assert!(retried_payload.get("previous_response_id").is_none()); - let retried_input = retried_payload["input"].as_array().unwrap(); - assert!(retried_input - .iter() - .all(|item| item.get("encrypted_content").is_none())); - assert_eq!(retried_input.last().unwrap()["content"], "continue"); -} - -#[tokio::test] -async fn responses_websocket_should_restore_full_history_when_implicit_resume_previous_response_is_missing( -) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut websocket = accept_async(stream).await.unwrap(); - let first_payload = send_websocket_response_and_capture_payload( - &mut websocket, - WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY.trim().to_string(), - ) - .await; - let implicit_payload = accept_followup_websocket_response_sequence( - &listener, - &mut websocket, - vec![ - json!({ - "type": "response.created", - "response": { - "id": "resp_implicit_resume_failed", - "object": "response", - "status": "in_progress" - } - }) - .to_string(), - json!({ - "type": "response.content_part.added", - "response_id": "resp_implicit_resume_failed", - "output_index": 0, - "content_index": 0, - "part": {"type": "output_text", "text": "", "annotations": []} - }) - .to_string(), - WEBSOCKET_PREVIOUS_RESPONSE_NOT_FOUND.trim().to_string(), - ], - ) - .await; - let _ = websocket.close(None).await; - let restored_payload = accept_websocket_response_with_message( - &listener, - websocket_completed_response("resp_implicit_resume_restored", 10, 2), - ) - .await; - (first_payload, implicit_payload, restored_payload) - }); - let (app, api_key, _dir) = test_app_with_account(base_url).await; - - let first_response = app - .clone() - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": false, - "input": [{"role": "user", "content": "remember this"}] - }), - )) - .await - .unwrap(); - assert_eq!(first_response.status(), StatusCode::OK); - - let second_response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "instructions": "answer briefly", - "stream": true, - "input": [ - {"role": "user", "content": "remember this"}, - {"role": "assistant", "content": "cached answer"}, - {"role": "user", "content": "continue"} - ] - }), - )) - .await - .unwrap(); - assert_eq!(second_response.status(), StatusCode::OK); - let body = response_text(second_response).await; - let (_first_payload, implicit_payload, restored_payload) = upstream.await.unwrap(); - assert!(body.contains("resp_implicit_resume_restored")); - assert_eq!( - implicit_payload["previous_response_id"], - "resp_implicit_resume_first" - ); - assert_eq!(implicit_payload["input"].as_array().unwrap().len(), 2); - assert!(restored_payload.get("previous_response_id").is_none()); - assert_eq!(restored_payload["input"].as_array().unwrap().len(), 3); - assert_eq!(restored_payload["input"][0]["role"], "user"); - assert_eq!(restored_payload["input"][1]["role"], "assistant"); - assert_eq!(restored_payload["input"][2]["content"], "continue"); -} - #[tokio::test] async fn responses_websocket_should_route_previous_response_id_to_recorded_account() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -1551,15 +845,12 @@ async fn responses_websocket_should_route_previous_response_id_to_recorded_accou ); let (first_payload, second_payload) = upstream.await.unwrap(); - assert_ne!(first_payload["prompt_cache_key"], Value::Null); + assert!(first_payload.get("prompt_cache_key").is_none()); assert_eq!( second_payload["previous_response_id"], "resp_affinity_first" ); - assert_eq!( - second_payload["prompt_cache_key"], first_payload["prompt_cache_key"], - "previous_response_id should inherit the recorded conversation identity" - ); + assert!(second_payload.get("prompt_cache_key").is_none()); } #[tokio::test] @@ -1567,14 +858,19 @@ async fn responses_websocket_should_prefer_conversation_account_without_previous let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let base_url = format!("http://{}", listener.local_addr().unwrap()); let upstream = tokio::spawn(async move { - accept_successful_websocket_response_with_authorization( + accept_two_successful_websocket_responses_with_authorization( &listener, "Bearer access-secondary", + "resp_conversation_seed", "resp_conversation_affinity", ) .await }); - let (app, state, api_key, _pool, _dir) = test_app_with_two_accounts_and_state(base_url).await; + let (app, state, api_key, _pool, _dir) = + test_app_with_two_accounts_and_state_config(base_url, |config| { + config.auth.rotation_strategy = "round_robin".to_string(); + }) + .await; let request_body = json!({ "model": "gpt-5.5", "prompt_cache_key": "conv_conversation_affinity", @@ -1582,43 +878,46 @@ async fn responses_websocket_should_prefer_conversation_account_without_previous "stream": false, "use_websocket": true }); - let mut affinity_body = request_body.as_object().unwrap().clone(); - affinity_body.remove("use_websocket"); - let mut affinity_request = CodexResponsesRequest::from_body(affinity_body); - affinity_request.explicit_prompt_cache_key = true; - prepare_variant_identity(&mut affinity_request); - let variant_hash = compute_variant_hash(&affinity_request); - state - .services - .session_affinity - .record( - "resp_conversation_latest".to_string(), - SessionAffinityEntry { - account_id: "acct_secondary".to_string(), - conversation_id: "conv_conversation_affinity".to_string(), - turn_state: None, - instructions_hash: None, - input_tokens: None, - function_call_ids: Vec::new(), - variant_hash: Some(variant_hash), - created_at: Utc::now(), - }, - ) - .await - .unwrap(); - - let response = app + assert!( + state + .services + .account_pool + .set_status("acct_primary", AccountStatus::Disabled) + .await + ); + let first_response = app + .clone() .oneshot(responses_json_request(&api_key, &request_body)) .await .unwrap(); - let body = response_text(response).await; - let payload = upstream.await.unwrap(); + assert_eq!(first_response.status(), StatusCode::OK); + assert!(response_text(first_response) + .await + .contains("\"id\":\"resp_conversation_seed\"")); + assert!( + state + .services + .account_pool + .set_status("acct_primary", AccountStatus::Active) + .await + ); + + let second_response = app + .oneshot(responses_json_request(&api_key, &request_body)) + .await + .unwrap(); + let body = response_text(second_response).await; + let (_reused, first_payload, second_payload) = upstream.await.unwrap(); assert!(body.contains("\"id\":\"resp_conversation_affinity\"")); - assert!(payload["prompt_cache_key"] + assert!(first_payload["prompt_cache_key"] .as_str() - .is_some_and(|value| value.starts_with("cp_"))); - assert!(payload.get("previous_response_id").is_none()); + .is_some_and(|value| value.starts_with("wi_"))); + assert_eq!( + second_payload["prompt_cache_key"], + first_payload["prompt_cache_key"] + ); + assert!(second_payload.get("previous_response_id").is_none()); } #[tokio::test] @@ -1654,82 +953,6 @@ async fn responses_websocket_non_stream_should_forward_unknown_previous_response assert_eq!(payload["previous_response_id"], "resp_missing"); } -#[tokio::test] -async fn responses_websocket_should_preserve_known_history_on_fallback_account() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - accept_websocket_response_with_authorization_and_message( - &listener, - "Bearer access-secondary", - websocket_completed_response("resp_history_forwarded_to_fallback", 3, 1), - ) - .await - }); - let (app, state, api_key, _pool, _dir) = - test_app_with_two_accounts_and_state_config(base_url, |config| { - config.auth.max_concurrent_per_account = 1; - }) - .await; - let _held_primary_slot = seed_primary_affinity_and_hold_primary_slot(&state).await; - - let response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "input": [], - "stream": false, - "previous_response_id": "resp_owned_by_primary" - }), - )) - .await - .unwrap(); - let body = response_json(response).await; - let payload = upstream.await.unwrap(); - - assert_eq!(body["id"], "resp_history_forwarded_to_fallback"); - assert_eq!(payload["previous_response_id"], "resp_owned_by_primary"); -} - -#[tokio::test] -async fn responses_websocket_stream_should_preserve_known_history_on_fallback_account() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base_url = format!("http://{}", listener.local_addr().unwrap()); - let upstream = tokio::spawn(async move { - accept_websocket_response_with_authorization_and_message( - &listener, - "Bearer access-secondary", - websocket_completed_response("resp_stream_history_forwarded_to_fallback", 3, 1), - ) - .await - }); - let (app, state, api_key, _pool, _dir) = - test_app_with_two_accounts_and_state_config(base_url, |config| { - config.auth.max_concurrent_per_account = 1; - }) - .await; - let _held_primary_slot = seed_primary_affinity_and_hold_primary_slot(&state).await; - - let response = app - .oneshot(responses_json_request( - &api_key, - &json!({ - "model": "gpt-5.5", - "input": [], - "stream": true, - "previous_response_id": "resp_owned_by_primary" - }), - )) - .await - .unwrap(); - let body = response_text(response).await; - let payload = upstream.await.unwrap(); - - assert!(body.contains("resp_stream_history_forwarded_to_fallback")); - assert_eq!(payload["previous_response_id"], "resp_owned_by_primary"); -} - #[tokio::test] async fn responses_websocket_stream_should_forward_unknown_previous_response() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -1764,7 +987,7 @@ async fn responses_websocket_stream_should_forward_unknown_previous_response() { } #[tokio::test] -async fn responses_websocket_previous_response_id_should_retry_fallback_account_after_429() { +async fn responses_websocket_stream_should_not_fan_out_external_previous_after_429() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let base_url = format!("http://{}", listener.local_addr().unwrap()); let upstream = tokio::spawn(async move { @@ -1777,43 +1000,34 @@ async fn responses_websocket_previous_response_id_should_retry_fallback_account_ WEBSOCKET_HISTORY_RATE_LIMITED, ) .await; - accept_successful_websocket_response_with_authorization( - &listener, - "Bearer access-secondary", - "resp_history_fallback", - ) - .await }); - let (app, api_key, pool, _dir) = test_app_with_two_accounts(base_url).await; + let (app, api_key, _pool, _dir) = test_app_with_two_accounts(base_url).await; - let response = app - .oneshot(responses_json_request( + let response = timeout( + StdDuration::from_secs(2), + app.oneshot(responses_json_request( &api_key, &json!({ "model": "gpt-5.5", "input": [], + "stream": true, "previous_response_id": "resp_prev" }), - )) - .await - .unwrap(); + )), + ) + .await + .expect("external previous should stop after the selected account") + .unwrap(); + let status = response.status(); let body = response_text(response).await; - let fallback_payload = upstream.await.unwrap(); - let secondary_usage: (i64,) = - sqlx::query_as("select request_count from account_usage where account_id = $1") - .bind("acct_secondary") - .fetch_one(&pool) - .await - .unwrap(); + upstream.await.unwrap(); - assert!(body.contains("\"id\":\"resp_history_fallback\"")); - assert_eq!(fallback_payload["previous_response_id"], "resp_prev"); - assert_eq!(secondary_usage.0, 1); + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + assert!(body.contains("history account rate limited")); } #[tokio::test] -async fn responses_websocket_non_stream_previous_response_id_should_retry_fallback_account_after_429( -) { +async fn responses_websocket_non_stream_should_not_fan_out_external_previous_after_429() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let base_url = format!("http://{}", listener.local_addr().unwrap()); let upstream = tokio::spawn(async move { @@ -1826,17 +1040,12 @@ async fn responses_websocket_non_stream_previous_response_id_should_retry_fallba WEBSOCKET_HISTORY_RATE_LIMITED, ) .await; - accept_successful_websocket_response_with_authorization( - &listener, - "Bearer access-secondary", - "resp_history_fallback_non_stream", - ) - .await }); - let (app, api_key, pool, _dir) = test_app_with_two_accounts(base_url).await; + let (app, api_key, _pool, _dir) = test_app_with_two_accounts(base_url).await; - let response = app - .oneshot(responses_json_request( + let response = timeout( + StdDuration::from_secs(2), + app.oneshot(responses_json_request( &api_key, &json!({ "model": "gpt-5.5", @@ -1844,21 +1053,17 @@ async fn responses_websocket_non_stream_previous_response_id_should_retry_fallba "stream": false, "previous_response_id": "resp_prev" }), - )) - .await - .unwrap(); + )), + ) + .await + .expect("external previous should stop after the selected account") + .unwrap(); + let status = response.status(); let body = response_json(response).await; - let fallback_payload = upstream.await.unwrap(); - let secondary_usage: (i64,) = - sqlx::query_as("select request_count from account_usage where account_id = $1") - .bind("acct_secondary") - .fetch_one(&pool) - .await - .unwrap(); + upstream.await.unwrap(); - assert_eq!(body["id"], "resp_history_fallback_non_stream"); - assert_eq!(fallback_payload["previous_response_id"], "resp_prev"); - assert_eq!(secondary_usage.0, 1); + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + assert_eq!(body["error"]["message"], "history account rate limited"); } #[tokio::test] @@ -1899,6 +1104,7 @@ async fn responses_websocket_without_history_should_mark_expired_after_fallback_ )) .await .unwrap(); + let status = response.status(); let body = response_text(response).await; upstream.await.unwrap(); let secondary_status: (String,) = sqlx::query_as("select status from accounts where id = $1") @@ -1914,7 +1120,7 @@ async fn responses_websocket_without_history_should_mark_expired_after_fallback_ .await .unwrap(); - assert!(body.contains("event: response.failed")); + assert_eq!(status, StatusCode::UNAUTHORIZED); assert!(body.contains("\"type\":\"invalid_request_error\"")); assert!(body.contains("\"code\":\"invalid_api_key\"")); assert!(body.contains("All accounts exhausted")); @@ -1962,6 +1168,7 @@ async fn responses_websocket_without_history_should_return_rate_limit_stream_err )) .await .unwrap(); + let status = response.status(); let body = response_text(response).await; upstream.await.unwrap(); let primary_usage: (i64,) = @@ -1977,7 +1184,8 @@ async fn responses_websocket_without_history_should_return_rate_limit_stream_err .await .unwrap(); - assert_response_failed_stream( + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + assert_openai_error_body( &body, "rate_limit_error", "rate_limit_exceeded", @@ -2080,6 +1288,7 @@ async fn responses_websocket_without_history_should_return_quota_stream_error_wh )) .await .unwrap(); + let status = response.status(); let body = response_text(response).await; upstream.await.unwrap(); let account_status: (String,) = sqlx::query_as("select status from accounts where id = $1") @@ -2088,7 +1297,8 @@ async fn responses_websocket_without_history_should_return_quota_stream_error_wh .await .unwrap(); - assert_response_failed_stream( + assert_eq!(status, StatusCode::TOO_MANY_REQUESTS); + assert_openai_error_body( &body, "insufficient_quota", "insufficient_quota", @@ -2130,6 +1340,7 @@ async fn responses_websocket_without_history_should_return_model_unsupported_str )) .await .unwrap(); + let status = response.status(); let body = response_text(response).await; upstream.await.unwrap(); let account_status: (String,) = sqlx::query_as("select status from accounts where id = $1") @@ -2138,12 +1349,13 @@ async fn responses_websocket_without_history_should_return_model_unsupported_str .await .unwrap(); - assert_response_failed_stream( + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_openai_error_body( &body, "invalid_request_error", "model_not_found", &[ - "No accounts available", + "All accounts exhausted", "model_not_available", "not available", ], @@ -2180,14 +1392,16 @@ async fn responses_websocket_with_history_should_return_path_block_stream_error_ )) .await .unwrap(); + let status = response.status(); let body = response_text(response).await; upstream.await.unwrap(); - assert_response_failed_stream( + assert_eq!(status, StatusCode::NOT_FOUND); + assert_openai_error_body( &body, - "server_error", - "upstream_error", - &["No accounts available", "Cloudflare path-block"], + "invalid_request_error", + "codex_client_error", + &["Upstream Codex request failed"], ); } @@ -2345,7 +1559,7 @@ async fn responses_stream_with_previous_response_id_should_record_websocket_audi assert_eq!(event.output_tokens, Some(1)); assert!( event.first_token_ms.is_some_and(|value| value > 0), - "websocket stream usage metadata should include first token latency: {metadata:?}", + "websocket stream usage metadata should include initial event latency: {metadata:?}", ); assert_rate_limit_header(&metadata, "x-codex-primary-used-percent", "44"); assert_rate_limit_header(&metadata, "x-codex-primary-window-minutes", "5"); @@ -2361,35 +1575,3 @@ async fn responses_stream_with_previous_response_id_should_record_websocket_audi Some(44.0) ); } - -async fn seed_primary_affinity_and_hold_primary_slot(state: &AppState) -> AccountLease { - state - .services - .session_affinity - .record( - "resp_owned_by_primary".to_string(), - SessionAffinityEntry { - account_id: "acct_primary".to_string(), - conversation_id: "conv_primary".to_string(), - turn_state: Some("turn-primary".to_string()), - instructions_hash: None, - input_tokens: None, - function_call_ids: Vec::new(), - variant_hash: None, - created_at: Utc::now(), - }, - ) - .await - .unwrap(); - let held_primary_slot = state - .services - .account_pool - .acquire_with( - &AccountAcquireRequest::new("gpt-5.5", Utc::now()) - .with_required_account_id("acct_primary"), - ) - .await - .unwrap(); - assert_eq!(held_primary_slot.account.id, "acct_primary"); - held_primary_slot -} diff --git a/backend/tests/dispatch/service/usage_logging.rs b/backend/tests/dispatch/service/usage_logging.rs index 69c4cf887..53e271aa5 100644 --- a/backend/tests/dispatch/service/usage_logging.rs +++ b/backend/tests/dispatch/service/usage_logging.rs @@ -401,7 +401,7 @@ async fn responses_should_passively_cache_rate_limit_headers() { } #[tokio::test] -async fn responses_should_record_empty_response_attempts() { +async fn responses_should_record_empty_response_attempt() { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses")) @@ -410,7 +410,7 @@ async fn responses_should_record_empty_response_attempts() { .insert_header("content-type", "text/event-stream") .set_body_string(RESPONSES_EMPTY_COMPLETED_SSE), ) - .expect(3) + .expect(1) .mount(&server) .await; @@ -436,7 +436,7 @@ async fn responses_should_record_empty_response_attempts() { .await .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); let usage: (i64, i64, i64, i64) = sqlx::query_as( "select request_count, empty_response_count, input_tokens, output_tokens from account_usage where account_id = $1", ) @@ -444,7 +444,7 @@ async fn responses_should_record_empty_response_attempts() { .fetch_one(&pool) .await .unwrap(); - assert_eq!(usage, (3, 3, 0, 0)); + assert_eq!(usage, (1, 1, 0, 0)); } #[tokio::test] @@ -576,7 +576,7 @@ async fn responses_stream_should_record_usage_record_after_completed_stream() { assert_eq!(event.output_tokens, Some(5)); assert!( event.first_token_ms.is_some_and(|value| value > 0), - "stream usage metadata should include first token latency: {metadata:?}", + "stream usage metadata should include first output event latency: {metadata:?}", ); let first_token_bucket: (i64, i64) = sqlx::query_as( "select first_token_latency_sum, first_token_latency_count from request_time_buckets limit 1", @@ -858,10 +858,10 @@ async fn responses_should_record_request_count_when_5xx_retries_are_exhausted() .await .unwrap(); - assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); assert_eq!(authorizations.len(), 3, "requests: {authorizations:?}"); assert_eq!(failed_usage.map(|row| row.0).unwrap_or_default(), 1); - let event = latest_response_ops_error_log(&pool).await; + let event = latest_response_upstream_ops_error_log(&pool).await; let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); assert_eq!(event.account_id.as_deref(), Some("acct_chat")); assert_eq!(metadata["accountEmail"], "user@example.com"); @@ -912,10 +912,10 @@ async fn responses_should_record_attempt_trace_after_retrying_another_account() ) .await .unwrap(); - let event = latest_response_ops_error_log(&pool).await; + let event = latest_response_upstream_ops_error_log(&pool).await; let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert_eq!(event.level, "error"); assert_eq!(event.request_id.as_deref(), Some("req_attempt_trace_retry")); assert_eq!(event.account_id.as_deref(), Some("acct_secondary")); diff --git a/backend/tests/fleet/quota.rs b/backend/tests/fleet/quota.rs index 18f70bc6f..88b4d3e42 100644 --- a/backend/tests/fleet/quota.rs +++ b/backend/tests/fleet/quota.rs @@ -8,6 +8,7 @@ use codex_proxy_rs::fleet::quota::{ quota_snapshot_reset_at, QuotaRefreshService, }; use codex_proxy_rs::fleet::store::{NewAccount, PgAccountStore}; +use codex_proxy_rs::infra::identity::AccountPseudonymizer; use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore; use codex_proxy_rs::upstream::openai::transport::CodexBackendClient; use secrecy::SecretString; @@ -19,6 +20,10 @@ use wiremock::{ use crate::support::storage::init_test_db; +fn test_account_pseudonymizer() -> Arc { + Arc::new(AccountPseudonymizer::new([7; 32])) +} + #[test] fn quota_from_usage_should_preserve_spend_control_individual_limit() { let usage = json!({ @@ -316,6 +321,7 @@ async fn quota_refresh_service_should_send_usage_cookie_when_cookie_store_is_con store, PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), ) .with_cookie_store(cookies); @@ -397,6 +403,7 @@ async fn quota_refresh_service_should_fetch_usage_for_quota_locked_accounts_and_ store.clone(), PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), ); let mut last_refreshed = HashMap::new(); @@ -472,6 +479,7 @@ async fn quota_refresh_service_should_refresh_quota_exhausted_accounts() { store, PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), ); let mut last_refreshed = HashMap::new(); @@ -507,6 +515,7 @@ async fn quota_refresh_service_should_skip_recent_locked_account_before_cooldown store, PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), 1_800, ); @@ -546,6 +555,7 @@ async fn quota_refresh_service_should_skip_recent_locked_account_inside_cooldown store, PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), 1_800, ); @@ -602,6 +612,7 @@ async fn quota_refresh_service_should_bypass_recent_skip_after_cooldown_grace() store, PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), 1_800, ); @@ -656,6 +667,7 @@ async fn quota_refresh_service_should_stagger_multiple_locked_account_requests() store, PgAccountUsageStore::new(pool.clone()), Arc::new(codex), + test_account_pseudonymizer(), ) .with_request_spacing(StdDuration::from_millis(200)); diff --git a/backend/tests/infra/identity.rs b/backend/tests/infra/identity.rs new file mode 100644 index 000000000..cfe146378 --- /dev/null +++ b/backend/tests/infra/identity.rs @@ -0,0 +1,29 @@ +use codex_proxy_rs::infra::identity::AccountPseudonymizer; + +#[test] +fn account_pseudonyms_should_be_stable_and_domain_separated() { + let pseudonymizer = AccountPseudonymizer::new([7; 32]); + + let first = pseudonymizer.scoped("session-id", Some("acct-1"), "client-value"); + let repeated = pseudonymizer.scoped("session-id", Some("acct-1"), "client-value"); + let other_account = pseudonymizer.scoped("session-id", Some("acct-2"), "client-value"); + let other_domain = pseudonymizer.scoped("thread-id", Some("acct-1"), "client-value"); + + assert_eq!(first, repeated); + assert_ne!(first, other_account); + assert_ne!(first, other_domain); + assert!(!first.contains("client-value")); +} + +#[test] +fn installation_ids_should_be_stable_uuid_values_scoped_per_account() { + let pseudonymizer = AccountPseudonymizer::new([9; 32]); + + let first = pseudonymizer.installation_id("acct-1"); + let repeated = pseudonymizer.installation_id("acct-1"); + let other_account = pseudonymizer.installation_id("acct-2"); + + assert_eq!(first, repeated); + assert_ne!(first, other_account); + assert!(uuid::Uuid::parse_str(&first).is_ok()); +} diff --git a/backend/tests/infra/mod.rs b/backend/tests/infra/mod.rs index a7bf05c0e..026ac36f2 100644 --- a/backend/tests/infra/mod.rs +++ b/backend/tests/infra/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod database; pub(crate) mod format; +pub(crate) mod identity; pub(crate) mod logging; pub(crate) mod paths; pub(crate) mod time; diff --git a/backend/tests/infra/paths.rs b/backend/tests/infra/paths.rs index 426575577..c805a4bba 100644 --- a/backend/tests/infra/paths.rs +++ b/backend/tests/infra/paths.rs @@ -1,8 +1,8 @@ use std::{ffi::OsString, fs, sync::Mutex}; -use codex_proxy_rs::infra::paths::{ensure_data_dir, load_or_create_installation_id}; +use codex_proxy_rs::infra::paths::{ensure_data_dir, load_or_create_identity_secret}; -const INSTALLATION_ID_FILE_NAME: &str = "installation_id"; +const IDENTITY_SECRET_FILE_NAME: &str = "identity_hmac_secret"; static ENV_LOCK: Mutex<()> = Mutex::new(()); @@ -18,37 +18,33 @@ fn data_dir_should_use_xdg_data_home_directly() { } #[test] -fn load_or_create_installation_id_should_persist_under_data_dir_root() { +fn identity_secret_should_be_stable_and_owner_only() { let dir = tempfile::tempdir().unwrap(); - let installation_id = - load_or_create_installation_id(Some(dir.path())).expect("installation id should generate"); + let first = load_or_create_identity_secret(dir.path()).expect("secret should generate"); + let second = load_or_create_identity_secret(dir.path()).expect("secret should reload"); - assert_eq!( - fs::read_to_string(dir.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(), - installation_id - ); + assert_eq!(first, second); + let path = dir.path().join(IDENTITY_SECRET_FILE_NAME); + assert_eq!(fs::read_to_string(&path).unwrap(), hex::encode(first)); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } } #[test] -fn load_or_create_installation_id_should_seed_data_dir_from_codex_desktop_id() { - let _guard = ENV_LOCK.lock().unwrap(); - let home = tempfile::tempdir().unwrap(); - let data = tempfile::tempdir().unwrap(); - let codex_path = home.path().join(".codex").join(INSTALLATION_ID_FILE_NAME); - let desktop_id = "018f8f6b-1d7b-7b7c-b9c8-8c9f4c6d0e1a"; - fs::create_dir_all(codex_path.parent().unwrap()).unwrap(); - fs::write(&codex_path, desktop_id).unwrap(); - let _home = EnvVarGuard::set("HOME", home.path().as_os_str().to_owned()); +fn identity_secret_should_fail_closed_when_existing_file_is_invalid() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join(IDENTITY_SECRET_FILE_NAME), "not-a-secret").unwrap(); - let installation_id = - load_or_create_installation_id(Some(data.path())).expect("installation id should seed"); + let error = load_or_create_identity_secret(dir.path()).expect_err("invalid secret must fail"); - assert_eq!(installation_id, desktop_id); - assert_eq!( - fs::read_to_string(data.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(), - desktop_id - ); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); } struct EnvVarGuard { diff --git a/backend/tests/models/catalog.rs b/backend/tests/models/catalog.rs index 572e76bd7..c89879dc3 100644 --- a/backend/tests/models/catalog.rs +++ b/backend/tests/models/catalog.rs @@ -149,13 +149,12 @@ async fn model_service_should_refresh_plan_accounts_and_build_routing() { ); let result = service - .refresh_backend_models_with_installation_id( + .refresh_backend_models( &[ refresh_plan_account("plus", active_account("acct-plus-1", "plus")), refresh_plan_account("team", active_account("acct-team", "team")), ], "req-model-refresh", - None, ) .await .expect("refresh should succeed"); @@ -198,13 +197,29 @@ async fn model_service_should_return_no_accounts_when_no_plan_accounts_exist() { let service = ModelService::new(test_model_config(), Some(store), Some(upstream)); let error = service - .refresh_backend_models_with_installation_id(&[], "req-model-refresh", None) + .refresh_backend_models(&[], "req-model-refresh") .await .expect_err("refresh should fail"); assert!(matches!(error, ModelServiceError::NoAccounts)); } +#[tokio::test] +async fn model_service_should_notify_only_when_models_etag_changes() { + let service = ModelService::new(test_model_config(), None, None); + let mut changes = service.subscribe_models_etag_changes(); + + assert!(!service.observe_models_etag(None)); + assert!(service.observe_models_etag(Some("etag-1"))); + changes.changed().await.expect("first etag notification"); + assert_eq!(*changes.borrow_and_update(), 1); + + assert!(!service.observe_models_etag(Some("etag-1"))); + assert!(service.observe_models_etag(Some("etag-2"))); + changes.changed().await.expect("second etag notification"); + assert_eq!(*changes.borrow_and_update(), 2); +} + fn test_model_config() -> ModelConfig { ModelConfig { model_aliases: BTreeMap::new(), @@ -237,6 +252,7 @@ fn refresh_plan_account(plan_type: &str, account: Account) -> ModelRefreshPlanAc plan_type: plan_type.to_string(), access_token: account.access_token, account_id: account.account_id, + installation_id: "test-installation-id".to_string(), } } diff --git a/backend/tests/upstream/openai/protocol/events.rs b/backend/tests/upstream/openai/protocol/events.rs index 0f2bfeb53..77e4179d5 100644 --- a/backend/tests/upstream/openai/protocol/events.rs +++ b/backend/tests/upstream/openai/protocol/events.rs @@ -254,8 +254,9 @@ fn parse_rate_limit_headers_should_extract_primary_secondary_and_review_windows( let parsed = parse_rate_limit_headers(&headers).expect("rate limits should parse"); + let core = parsed.limits.get("codex").expect("core rate limit"); assert_eq!( - parsed.primary, + core.primary, Some(RateLimitWindow { used_percent: 100.0, window_minutes: Some(5), @@ -263,12 +264,13 @@ fn parse_rate_limit_headers_should_extract_primary_secondary_and_review_windows( }) ); assert_eq!( - parsed.secondary.expect("secondary window").window_minutes, + core.secondary.expect("secondary window").window_minutes, Some(10080) ); assert_eq!( parsed - .code_review + .limits + .get("codex_code_review") .expect("review window") .primary .expect("review primary") @@ -297,14 +299,91 @@ fn parse_rate_limits_event_should_extract_internal_websocket_rate_limits() { let parsed = parse_rate_limits_event(&event).expect("event should parse"); - let used_percent = parsed.primary.expect("primary window").used_percent; + let core = parsed.limits.get("codex").expect("core rate limit"); + let used_percent = core.primary.expect("primary window").used_percent; assert!((used_percent - 99.5).abs() < f64::EPSILON); assert_eq!( - parsed.secondary.expect("secondary window").reset_at, + core.secondary.expect("secondary window").reset_at, Some(1_894_056_000) ); } +#[test] +fn parse_rate_limit_headers_should_extract_arbitrary_limits_and_account_metadata() { + let headers = vec![ + ( + "x-codex-other-primary-used-percent".to_string(), + "63.5".to_string(), + ), + ( + "x-codex-other-primary-window-minutes".to_string(), + "1440".to_string(), + ), + ( + "x-codex-other-limit-name".to_string(), + "Codex Other".to_string(), + ), + ( + "x-codex-credits-has-credits".to_string(), + "true".to_string(), + ), + ("x-codex-credits-unlimited".to_string(), "false".to_string()), + ("x-codex-credits-balance".to_string(), "12.50".to_string()), + ("x-codex-plan-type".to_string(), "pro".to_string()), + ("x-codex-promo-message".to_string(), "bonus".to_string()), + ( + "x-codex-rate-limit-reached-type".to_string(), + "weekly".to_string(), + ), + ]; + + let parsed = parse_rate_limit_headers(&headers).expect("rate limits should parse"); + let other = parsed.limits.get("codex_other").expect("dynamic limit"); + assert_eq!(other.limit_name.as_deref(), Some("Codex Other")); + assert_eq!( + other.primary.expect("primary window").window_minutes, + Some(1440) + ); + let credits = parsed.credits.expect("credits"); + assert!(credits.has_credits); + assert!(!credits.unlimited); + assert_eq!(credits.balance.as_deref(), Some("12.50")); + assert_eq!(parsed.plan_type.as_deref(), Some("pro")); + assert_eq!(parsed.promo_message.as_deref(), Some("bonus")); + assert_eq!(parsed.rate_limit_reached_type.as_deref(), Some("weekly")); +} + +#[test] +fn parse_rate_limits_event_should_keep_metered_limit_credits_and_plan() { + let event = json!({ + "type": "codex.rate_limits", + "plan_type": "team", + "metered_limit_name": "codex-other", + "limit_name": "Other requests", + "rate_limits": { + "primary": { + "used_percent": 33, + "window_minutes": 60, + "reset_at": 1893456300 + } + }, + "credits": { + "has_credits": true, + "unlimited": false, + "balance": "9.25" + } + }); + + let parsed = parse_rate_limits_event(&event).expect("event should parse"); + let other = parsed.limits.get("codex_other").expect("metered limit"); + assert_eq!(other.limit_name.as_deref(), Some("Other requests")); + assert_eq!(parsed.plan_type.as_deref(), Some("team")); + assert_eq!( + parsed.credits.expect("credits").balance.as_deref(), + Some("9.25") + ); +} + #[test] fn rate_limit_quota_should_preserve_existing_monthly_and_credits_when_passive_data_lacks_them() { let headers = vec![ diff --git a/backend/tests/upstream/openai/protocol/mod.rs b/backend/tests/upstream/openai/protocol/mod.rs index ab39d9a23..4ee371fec 100644 --- a/backend/tests/upstream/openai/protocol/mod.rs +++ b/backend/tests/upstream/openai/protocol/mod.rs @@ -25,16 +25,14 @@ use codex_proxy_rs::upstream::openai::protocol::websocket::{ websocket_function_call_output_item_event_invalid_required_fields, websocket_function_call_output_payload_item_event_invalid_required_fields, websocket_image_generation_call_output_item_event_invalid_required_fields, - websocket_incomplete_response_reason, websocket_local_shell_call_output_item_event_invalid_required_fields, websocket_message_output_item_event_invalid_required_fields, websocket_metadata_turn_state, websocket_output_item_event_invalid_item_type_tag, websocket_output_item_event_invalid_metadata, websocket_output_item_event_missing_item, websocket_output_item_event_non_object_item, websocket_payload_audit_snapshot, websocket_reasoning_output_item_event_invalid_required_fields, - websocket_reasoning_summary_part_added_missing_summary_index, - websocket_response_completed_missing_response, websocket_response_completed_parse_error, - websocket_response_created_missing_response, + websocket_reasoning_summary_part_added_missing_summary_index, websocket_response_completed_id, + websocket_response_completed_missing_response, websocket_response_created_missing_response, websocket_response_output_text_delta_missing_delta, websocket_tool_search_call_output_item_event_invalid_required_fields, websocket_tool_search_output_item_event_invalid_required_fields, diff --git a/backend/tests/upstream/openai/protocol/responses.rs b/backend/tests/upstream/openai/protocol/responses.rs index 7f26da1db..24320b13b 100644 --- a/backend/tests/upstream/openai/protocol/responses.rs +++ b/backend/tests/upstream/openai/protocol/responses.rs @@ -83,18 +83,7 @@ fn response_from_codex_sse_should_collect_completed_response_with_deltas() { "usage": { "input_tokens": 2, "output_tokens": 3 - }, - "output": [{ - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{ - "type": "output_text", - "text": "hello", - "annotations": [] - }] - }], - "output_text": "hello" + } })) ); } @@ -107,7 +96,7 @@ fn response_from_codex_sse_should_collect_incomplete_response() { assert_eq!( response, - CollectedResponse::Completed(json!({ + CollectedResponse::Incomplete(json!({ "id": "resp_incomplete", "object": "response", "status": "incomplete", diff --git a/backend/tests/upstream/openai/protocol/websocket.rs b/backend/tests/upstream/openai/protocol/websocket.rs index d54e24c89..d29392c69 100644 --- a/backend/tests/upstream/openai/protocol/websocket.rs +++ b/backend/tests/upstream/openai/protocol/websocket.rs @@ -29,8 +29,6 @@ fn codex_websocket_payload_audit_snapshot_should_redact_user_content() { "model", "instructions", "input", - "stream", - "store", "previous_response_id", "service_tier", "prompt_cache_key", @@ -39,7 +37,8 @@ fn codex_websocket_payload_audit_snapshot_should_redact_user_content() { ); assert_eq!(snapshot.body["type"], "response.create"); assert_eq!(snapshot.body["model"], "gpt-5.5"); - assert_eq!(snapshot.body["stream"], true); + assert!(snapshot.body.get("stream").is_none()); + assert!(snapshot.body.get("store").is_none()); assert_eq!(snapshot.body["instructions"], ""); assert_eq!(snapshot.body["input"], ""); assert_eq!(snapshot.body["previous_response_id"], ""); @@ -78,8 +77,6 @@ fn codex_websocket_response_create_payload_text_should_preserve_canonical_field_ "\"model\":\"gpt-5.5\"", "\"instructions\":\"private capture instructions\"", "\"input\":", - "\"stream\":true", - "\"store\":false", "\"prompt_cache_key\":\"session-1\"", "\"client_metadata\":", ], @@ -187,6 +184,7 @@ fn codex_websocket_metadata_turn_state_should_extract_case_insensitive_header() #[test] fn codex_websocket_terminal_event_should_match_completed_failed_and_error() { assert!(is_terminal_websocket_event("response.completed")); + assert!(is_terminal_websocket_event("response.incomplete")); assert!(is_terminal_websocket_event("response.failed")); assert!(is_terminal_websocket_event("error")); assert!(!is_terminal_websocket_event("response.output_text.delta")); @@ -223,6 +221,9 @@ fn codex_websocket_error_frame_should_classify_ts_aligned_upstream_error_codes() ("account_banned", 403), ("banned", 403), ("previous_response_not_found", 400), + ("invalid_encrypted_content", 400), + ("missing_tool_output", 400), + ("no_tool_output", 400), ]; for (code, expected_status) in cases { @@ -256,7 +257,6 @@ fn codex_websocket_error_frame_should_ignore_legacy_extension_codes() { "invalid_prompt", "cyber_policy", "invalid_request", - "invalid_encrypted_content", "no_tool_output_found_for_function_call", "server_is_overloaded", "slow_down", @@ -376,7 +376,7 @@ fn codex_websocket_wrapped_error_headers_should_extract_retry_after() { } #[test] -fn codex_websocket_response_completed_parse_error_should_validate_shape() { +fn codex_websocket_response_completed_id_should_validate_shape() { let frame = json!({ "type": "response.completed", "response": { @@ -390,14 +390,14 @@ fn codex_websocket_response_completed_parse_error_should_validate_shape() { }) .to_string(); - let error = websocket_response_completed_parse_error(&frame) - .expect("invalid response.completed should report a parse error"); + let error = websocket_response_completed_id(&frame) + .expect_err("invalid response.completed should report a parse error"); assert!(error.contains("failed to parse ResponseCompleted")); } #[test] -fn codex_websocket_response_completed_parse_error_should_reject_missing_id_and_incomplete_usage() { +fn codex_websocket_response_completed_id_should_reject_missing_id_and_incomplete_usage() { let missing_id = json!({ "type": "response.completed", "response": { @@ -423,11 +423,11 @@ fn codex_websocket_response_completed_parse_error_should_reject_missing_id_and_i }) .to_string(); - assert!(websocket_response_completed_parse_error(&missing_id) - .expect("missing id should be rejected") + assert!(websocket_response_completed_id(&missing_id) + .expect_err("missing id should be rejected") .contains("missing field")); - assert!(websocket_response_completed_parse_error(&incomplete_usage) - .expect("incomplete usage should be rejected") + assert!(websocket_response_completed_id(&incomplete_usage) + .expect_err("incomplete usage should be rejected") .contains("missing field")); } @@ -528,22 +528,12 @@ fn codex_websocket_optional_null_fields_should_match_missing_field_behavior() { } #[test] -fn codex_websocket_response_control_events_should_detect_incomplete_and_missing_completed_response() -{ - let incomplete = json!({ - "type": "response.incomplete", - "response": { - "incomplete_details": { - "reason": "max_output_tokens" - } - } - }) - .to_string(); +fn codex_websocket_response_control_events_should_recognize_incomplete_as_terminal() { + assert!(is_terminal_websocket_event("response.incomplete")); +} - assert_eq!( - websocket_incomplete_response_reason(&incomplete).as_deref(), - Some("max_output_tokens") - ); +#[test] +fn codex_websocket_response_control_events_should_detect_missing_completed_response() { assert!(websocket_response_completed_missing_response( r#"{"type":"response.completed"}"# )); diff --git a/backend/tests/upstream/openai/transport/client.rs b/backend/tests/upstream/openai/transport/client.rs index 9c24b690c..5906476b1 100644 --- a/backend/tests/upstream/openai/transport/client.rs +++ b/backend/tests/upstream/openai/transport/client.rs @@ -64,8 +64,9 @@ async fn codex_backend_client_should_apply_configured_websocket_pool() { .with_websocket_pool(Arc::new(pool)); let mut request = CodexResponsesRequest::new_http_sse("gpt-5.5", "be brief", Vec::new()); request.force_http_sse = false; - request.set_previous_response_id(Some("resp_runtime_pool_previous".to_string())); + request.use_websocket = true; request.set_prompt_cache_key(Some("conversation-runtime-pool".to_string())); + request.local_conversation_id = Some("conversation-runtime-pool".to_string()); let first = backend .create_response( @@ -74,6 +75,7 @@ async fn codex_backend_client_should_apply_configured_websocket_pool() { ) .await .expect("first runtime websocket response should succeed"); + request.set_previous_response_id(Some("resp_runtime_pool_first".to_string())); let second = backend .create_response( &request, diff --git a/backend/tests/upstream/openai/transport/headers.rs b/backend/tests/upstream/openai/transport/headers.rs index c0427135d..4ce1f4963 100644 --- a/backend/tests/upstream/openai/transport/headers.rs +++ b/backend/tests/upstream/openai/transport/headers.rs @@ -301,6 +301,10 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an cookie_header: None, installation_id: Some("install-123"), session_id: Some("cp_derived"), + thread_id: None, + prompt_cache_key: Some("cp_derived"), + client_request_id: None, + turn_id: None, }, ) .await @@ -332,7 +336,6 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an "ignored_non_string": 42, "x-codex-installation-id": "install-123", "session_id": "cp_derived", - "thread_id": "cp_derived", "x-codex-window-id": "cw_derived", "x-codex-turn-metadata": "{\"thread_source\":\"subagent\"}", "x-codex-parent-thread-id": "parent-456" @@ -342,7 +345,7 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an let headers = received_headers.lock().unwrap().clone(); assert!(headers .iter() - .any(|(name, value)| name == "x-client-request-id" && value == "cp_derived")); + .any(|(name, value)| name == "x-client-request-id" && value == "req_ws_security")); assert!(headers .iter() .any(|(name, value)| name == "x-codex-installation-id" && value == "install-123")); @@ -373,9 +376,7 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an assert!(headers .iter() .any(|(name, value)| name == "session-id" && value == "cp_derived")); - assert!(headers - .iter() - .any(|(name, value)| name == "thread-id" && value == "cp_derived")); + assert!(headers.iter().all(|(name, _)| name != "thread-id")); assert!(headers.iter().all(|(name, _)| name != "content-type")); assert!(headers.iter().all(|(name, _)| name != "accept")); assert!(headers.iter().all(|(name, _)| name != "session_id")); @@ -402,9 +403,7 @@ async fn codex_backend_client_should_send_desktop_headers_and_capture_response_m .and(wiremock::matchers::body_json(json!({ "model": "gpt-5.5", "instructions": "", - "input": [], - "stream": true, - "store": false + "input": [] }))) .respond_with( wiremock::ResponseTemplate::new(200) @@ -448,6 +447,10 @@ async fn codex_backend_client_should_send_desktop_headers_and_capture_response_m cookie_header: Some("cf_clearance=old"), installation_id: None, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }, ) .await @@ -517,6 +520,10 @@ async fn codex_backend_client_should_use_latest_runtime_fingerprint_for_new_requ cookie_header: None, installation_id: None, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }; client @@ -592,6 +599,10 @@ async fn codex_backend_client_usage_should_use_wham_usage_headers() { cookie_header: Some("cf_clearance=old"), installation_id: Some("install-1"), session_id: Some("session-1"), + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }) .await .unwrap(); @@ -748,6 +759,10 @@ async fn codex_backend_client_should_send_http_sse_headers_in_fingerprint_order( cookie_header: Some("cf_clearance=old"), installation_id: Some("install-1"), session_id: Some("session-1"), + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }, ) .await @@ -778,7 +793,6 @@ async fn codex_backend_client_should_send_http_sse_headers_in_fingerprint_order( "x-client-request-id", "x-codex-installation-id", "session-id", - "thread-id", "x-codex-window-id", "x-codex-turn-state", "x-codex-turn-metadata", @@ -820,6 +834,12 @@ async fn codex_backend_client_should_send_compact_headers_in_fingerprint_order() client_ip: None, client_user_agent: None, client_api_key_id: None, + client_session_id: None, + client_thread_id: None, + client_request_id: None, + client_turn_id: None, + client_window_id: None, + client_parent_thread_id: None, }, CodexRequestContext { access_token: "access-token", @@ -835,6 +855,10 @@ async fn codex_backend_client_should_send_compact_headers_in_fingerprint_order() cookie_header: Some("cf_clearance=old"), installation_id: Some("install-1"), session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }, ) .await diff --git a/backend/tests/upstream/openai/transport/http_client.rs b/backend/tests/upstream/openai/transport/http_client.rs index d62baf5ec..3463520a1 100644 --- a/backend/tests/upstream/openai/transport/http_client.rs +++ b/backend/tests/upstream/openai/transport/http_client.rs @@ -134,6 +134,10 @@ async fn codex_backend_client_should_cap_non_success_error_body_at_one_mib() { cookie_header: None, installation_id: None, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }, ) .await; @@ -188,6 +192,10 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod cookie_header: None, installation_id: None, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }, ) .await; @@ -204,6 +212,101 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod assert_eq!(retry_after_seconds, Some(12)); } +#[tokio::test] +async fn codex_backend_client_should_capture_only_safe_response_metadata() { + let server = wiremock::MockServer::start().await; + let body = concat!( + "event: response.completed\n", + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n" + ); + wiremock::Mock::given(wiremock::matchers::method("POST")) + .and(wiremock::matchers::path("/codex/responses")) + .respond_with( + wiremock::ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .insert_header("x-request-id", "upstream-request") + .insert_header("openai-model", "gpt-5.5-2026-07-01") + .insert_header("x-models-etag", "models-v2") + .insert_header("x-reasoning-included", "true") + .insert_header("openai-processing-ms", "42") + .insert_header("set-cookie", "secret=value") + .insert_header("x-codex-primary-used-percent", "15") + .set_body_string(body), + ) + .mount(&server) + .await; + let client = CodexBackendClient::new( + reqwest::Client::builder().no_proxy().build().unwrap(), + server.uri(), + crate::support::fingerprint::runtime_test_fingerprint(), + ); + let mut request = + codex_proxy_rs::upstream::openai::protocol::responses::CodexResponsesRequest::new_http_sse( + "gpt-5.5", + "", + Vec::new(), + ); + request.force_http_sse = true; + + let response = client + .create_response( + &request, + CodexRequestContext { + access_token: "access-token", + account_id: Some("chatgpt-account"), + request_id: "req_response_metadata", + turn_state: None, + turn_metadata: None, + beta_features: None, + include_timing_metrics: None, + version: None, + codex_window_id: None, + parent_thread_id: None, + cookie_header: None, + installation_id: None, + session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, + }, + ) + .await + .expect("response should succeed"); + + assert_eq!( + response.response_metadata.effective_model.as_deref(), + Some("gpt-5.5-2026-07-01") + ); + assert_eq!( + response.response_metadata.models_etag.as_deref(), + Some("models-v2") + ); + assert!(response.response_metadata.reasoning_included); + let mut names = response + .response_metadata + .client_headers + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(); + names.sort_unstable(); + assert_eq!( + names, + vec![ + "openai-model", + "openai-processing-ms", + "x-models-etag", + "x-reasoning-included", + "x-request-id" + ] + ); + assert_eq!(response.set_cookie_headers, vec!["secret=value"]); + assert_eq!( + response.rate_limit_headers, + vec![("x-codex-primary-used-percent".to_string(), "15".to_string())] + ); +} + #[tokio::test] async fn build_reqwest_client_should_reuse_cached_connection_pool() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/backend/tests/upstream/openai/transport/mod.rs b/backend/tests/upstream/openai/transport/mod.rs index 5fdff49f1..b737cbd33 100644 --- a/backend/tests/upstream/openai/transport/mod.rs +++ b/backend/tests/upstream/openai/transport/mod.rs @@ -8,6 +8,7 @@ use std::{ }; use codex_proxy_rs::upstream::openai::{ + protocol::responses::{PreviousResponseScope, StreamCommitPolicy}, transport::websocket::{ execute_response_create_request, responses_websocket_endpoint, CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketRequest, @@ -176,6 +177,10 @@ pub(crate) fn request_context<'a>( cookie_header: None, installation_id: None, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, } } @@ -208,8 +213,9 @@ pub(crate) fn pooled_websocket_request( Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some(conversation_id.to_string())); - request.client_conversation_id = Some(conversation_id.to_string()); + request.local_conversation_id = Some(conversation_id.to_string()); request } @@ -247,7 +253,7 @@ pub(crate) fn websocket_pool_config_for_tests( ping_interval, ping_timeout: Duration::from_secs(1), liveness_timeout, - first_token_timeout: None, + initial_event_timeout: None, } } diff --git a/backend/tests/upstream/openai/transport/websocket.rs b/backend/tests/upstream/openai/transport/websocket.rs index 6bbf74672..ce3d97777 100644 --- a/backend/tests/upstream/openai/transport/websocket.rs +++ b/backend/tests/upstream/openai/transport/websocket.rs @@ -140,6 +140,7 @@ async fn codex_backend_client_should_decode_permessage_deflate_context_takeover_ Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); let response = backend .create_response( @@ -189,7 +190,8 @@ fn websocket_connection_should_prepare_response_create_payload_text() { assert_eq!(payload["model"], "gpt-5.5"); assert_eq!(payload["instructions"], "be brief"); assert_eq!(payload["input"][0]["content"], "hello"); - assert_eq!(payload["stream"], true); + assert!(payload.get("stream").is_none()); + assert!(payload.get("store").is_none()); } #[test] @@ -227,8 +229,6 @@ fn websocket_connection_should_prepare_capture_payload_with_canonical_field_orde "\"model\":\"gpt-5.5\"", "\"instructions\":\"private capture instructions\"", "\"input\":", - "\"stream\":true", - "\"store\":false", "\"prompt_cache_key\":\"session-1\"", "\"client_metadata\":", ], @@ -755,7 +755,7 @@ async fn codex_backend_client_should_not_reuse_websocket_rate_limit_headers() { } #[tokio::test] -async fn websocket_execute_response_create_request_should_surface_incomplete_response() { +async fn websocket_execute_response_create_request_should_forward_incomplete_response() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let server = tokio::spawn(async move { @@ -797,15 +797,13 @@ async fn websocket_execute_response_create_request_should_surface_incomplete_res ) .expect("payload should serialize"); - let error = execute_response_create_request(&prepared) + let exchange = execute_response_create_request(&prepared) .await - .expect_err("response.incomplete should be surfaced as websocket error"); + .expect("response.incomplete should remain a terminal Responses event"); server.await.unwrap(); - let CodexWebSocketExchangeError::IncompleteResponse { reason } = error else { - panic!("expected incomplete websocket response"); - }; - assert_eq!(reason, "max_output_tokens"); + assert!(exchange.body.contains("event: response.incomplete")); + assert!(exchange.body.contains("max_output_tokens")); } #[tokio::test] @@ -866,8 +864,7 @@ async fn websocket_execute_response_create_request_should_reject_invalid_complet } #[tokio::test] -async fn websocket_execute_response_create_request_should_forward_completed_without_response_as_terminal( -) { +async fn websocket_execute_response_create_request_should_reject_completed_without_response() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let server = tokio::spawn(async move { @@ -903,14 +900,15 @@ async fn websocket_execute_response_create_request_should_forward_completed_with ) .expect("payload should serialize"); - // 透明代理：WS 输出侧不再按 schema 丢弃事件，缺 `response` 的 `response.completed` - // 也被原样转发，并作为终止事件结束交换（信任上游终止信号）。 - let response = execute_response_create_request(&prepared) + let error = execute_response_create_request(&prepared) .await - .expect("completed frame should terminate the exchange"); + .expect_err("completed frame without response must be rejected"); server.await.unwrap(); - assert!(response.body.contains("event: response.completed")); + assert!(matches!( + error, + CodexWebSocketExchangeError::InvalidCompletedResponse { .. } + )); } #[tokio::test] @@ -966,37 +964,96 @@ async fn websocket_execute_response_create_request_should_pass_through_unclassif assert!(response.body.contains("No tool output found")); } -#[tokio::test(start_paused = true)] -async fn websocket_execute_response_create_request_should_timeout_when_upstream_is_silent() { +#[tokio::test] +async fn codex_backend_client_should_timeout_when_upstream_is_silent() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - let (request_seen_tx, request_seen_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let mut websockets = Vec::new(); + for _ in 0..2 { + let (stream, _) = listener.accept().await.unwrap(); + let mut websocket = accept_codex_test_websocket(stream).await; + let _message = websocket.next().await.unwrap().unwrap(); + websockets.push(websocket); + } + futures::future::pending::<()>().await; + }); + let backend = CodexBackendClient::new( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{addr}"), + crate::support::fingerprint::runtime_test_fingerprint(), + ) + .with_websocket_initial_event_timeout(Some(Duration::from_millis(30))); + let mut request = + codex_proxy_rs::upstream::openai::protocol::responses::CodexResponsesRequest::new_http_sse( + "gpt-5.5", + "be brief", + Vec::new(), + ); + request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); + + let error = backend + .create_response( + &request, + request_context("req_silent_websocket", Some("chatgpt-account")), + ) + .await + .expect_err("silent upstream should time out"); + server.abort(); + + assert!(matches!( + error, + CodexClientError::WebSocket(CodexWebSocketExchangeError::InitialEventTimeout { timeout }) + if timeout == Duration::from_millis(30) + )); +} + +#[tokio::test(start_paused = true)] +async fn websocket_response_created_should_switch_to_active_stream_idle_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (created_tx, created_rx) = tokio::sync::oneshot::channel(); let server = tokio::spawn(async move { let (stream, _) = listener.accept().await.unwrap(); let mut websocket = accept_codex_test_websocket(stream).await; let _message = websocket.next().await.unwrap().unwrap(); - request_seen_tx.send(()).unwrap(); - futures::future::pending::<()>().await; + websocket + .send(Message::Text( + json!({ + "type": "response.created", + "response": {"id": "resp_structural", "status": "in_progress"} + }) + .to_string() + .into(), + )) + .await + .unwrap(); + created_tx.send(()).unwrap(); + tokio::time::sleep(Duration::from_secs(30)).await; + websocket + .send(Message::Text( + completed_websocket_response("resp_structural", 3, 1).into(), + )) + .await + .unwrap(); + websocket.close(None).await.unwrap(); }); let prepared = prepared_websocket_request(&format!("http://{addr}")); - let response_task = tokio::spawn(async move { execute_response_create_request(&prepared).await }); - request_seen_rx.await.unwrap(); - tokio::task::yield_now().await; - tokio::time::advance(Duration::from_secs(20)).await; - tokio::task::yield_now().await; - let error = response_task + created_rx.await.unwrap(); + tokio::time::advance(Duration::from_secs(30)).await; + tokio::task::yield_now().await; + let response = response_task .await .expect("websocket task should finish") - .expect_err("silent upstream should time out"); - server.abort(); + .expect("structural activity should disable the initial timeout"); + server.await.unwrap(); - let CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout } = error else { - panic!("expected receive idle timeout"); - }; - assert_eq!(timeout, Duration::from_secs(20)); + assert!(response.body.contains("event: response.created")); + assert!(response.body.contains("resp_structural")); } #[tokio::test] @@ -1087,6 +1144,7 @@ async fn codex_backend_client_stream_should_reject_binary_websocket_event() { Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.force_http_sse = false; let backend = CodexBackendClient::new( reqwest::Client::builder().no_proxy().build().unwrap(), @@ -1094,15 +1152,19 @@ async fn codex_backend_client_stream_should_reject_binary_websocket_event() { crate::support::fingerprint::runtime_test_fingerprint(), ); - let Err(error) = backend + let mut response = backend .create_response_stream( &request, request_context("req_stream_binary", Some("chatgpt-account")), ) .await - else { - panic!("binary event before first output should reject stream open"); - }; + .expect("stream should open before consuming websocket frames"); + let error = response + .body + .next() + .await + .expect("binary frame should produce a stream item") + .expect_err("binary frame should fail the stream"); server.await.unwrap(); assert!(matches!( @@ -1112,7 +1174,7 @@ async fn codex_backend_client_stream_should_reject_binary_websocket_event() { } #[tokio::test] -async fn codex_backend_client_stream_should_retry_without_pool_when_first_output_times_out() { +async fn codex_backend_client_stream_should_keep_socket_after_structural_activity() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let accepted_connections = Arc::new(AtomicUsize::new(0)); @@ -1137,59 +1199,55 @@ async fn codex_backend_client_stream_should_retry_without_pool_when_first_output .await .unwrap(); - let (second_stream, _) = listener.accept().await.unwrap(); - accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); - let mut second_websocket = accept_codex_test_websocket(second_stream).await; - let _second_message = second_websocket.next().await.unwrap().unwrap(); - second_websocket + tokio::time::sleep(Duration::from_millis(50)).await; + first_websocket .send(Message::Text( json!({ "type": "response.output_text.delta", - "delta": "no-pool retry recovered" + "delta": "delayed output" }) .to_string() .into(), )) .await .unwrap(); - second_websocket + first_websocket .send(Message::Text( - completed_websocket_response("resp_no_pool_retry_recovered", 3, 1).into(), + completed_websocket_response("resp_no_pool_delayed", 3, 1).into(), )) .await .unwrap(); - second_websocket.close(None).await.unwrap(); - drop(first_websocket); + first_websocket.close(None).await.unwrap(); }); let backend = CodexBackendClient::new( reqwest::Client::builder().no_proxy().build().unwrap(), format!("http://{addr}"), crate::support::fingerprint::runtime_test_fingerprint(), ) - .with_websocket_first_token_timeout(Some(Duration::from_millis(30))); - let request = pooled_websocket_request("conversation-first-token-no-pool-timeout"); + .with_websocket_initial_event_timeout(Some(Duration::from_millis(30))); + let request = pooled_websocket_request("conversation-structural-no-pool"); let response = backend .create_response_stream( &request, - request_context("req_first_token_no_pool_timeout", Some("chatgpt-account")), + request_context("req_structural_no_pool", Some("chatgpt-account")), ) .await - .expect("no-pool first-token timeout should retry before returning stream"); + .expect("structural activity should keep the websocket stream open"); let decision = response.websocket_pool_decision; let mut stream = response.body; let mut body = String::new(); while let Some(chunk) = stream.next().await { - let chunk = chunk.expect("retried no-pool stream chunk should be valid"); + let chunk = chunk.expect("delayed websocket stream chunk should be valid"); body.push_str(std::str::from_utf8(&chunk).unwrap()); } server.await.unwrap(); - assert!(body.contains("no-pool retry recovered")); - assert!(body.contains("resp_no_pool_retry_recovered")); - assert!(!body.contains("resp_no_pool_first_token_stalled")); + assert!(body.contains("resp_no_pool_first_token_stalled")); + assert!(body.contains("delayed output")); + assert!(body.contains("resp_no_pool_delayed")); assert!(decision.is_none()); - assert_eq!(accepted_connections.load(Ordering::SeqCst), 2); + assert_eq!(accepted_connections.load(Ordering::SeqCst), 1); } #[tokio::test] @@ -1220,6 +1278,7 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.force_http_sse = false; let backend = CodexBackendClient::new( reqwest::Client::builder().no_proxy().build().unwrap(), @@ -1290,6 +1349,7 @@ async fn codex_backend_client_stream_should_wait_for_terminal_after_active_webso Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.force_http_sse = false; let backend = CodexBackendClient::new( reqwest::Client::builder().no_proxy().build().unwrap(), @@ -1354,6 +1414,7 @@ async fn codex_backend_client_stream_should_timeout_when_active_websocket_stalls Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.force_http_sse = false; let backend = CodexBackendClient::new( reqwest::Client::builder().no_proxy().build().unwrap(), @@ -1450,6 +1511,7 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_ Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1))); let backend = CodexBackendClient::new( reqwest::Client::builder().no_proxy().build().unwrap(), @@ -1475,6 +1537,10 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_ cookie_header: None, installation_id: None, session_id: None, + thread_id: None, + prompt_cache_key: None, + client_request_id: None, + turn_id: None, }, ) .await diff --git a/backend/tests/upstream/openai/transport/websocket_pool.rs b/backend/tests/upstream/openai/transport/websocket_pool.rs index f0c67dabd..81d68a289 100644 --- a/backend/tests/upstream/openai/transport/websocket_pool.rs +++ b/backend/tests/upstream/openai/transport/websocket_pool.rs @@ -49,6 +49,7 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool".to_string())); let first = backend @@ -127,6 +128,7 @@ async fn codex_backend_client_should_open_fresh_socket_when_idle_pooled_websocke Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool-silent".to_string())); let first = backend @@ -156,7 +158,7 @@ async fn codex_backend_client_should_open_fresh_socket_when_idle_pooled_websocke } #[tokio::test] -async fn codex_backend_client_stream_should_retry_fresh_socket_when_first_output_times_out() { +async fn codex_backend_client_stream_should_keep_fresh_socket_after_structural_activity() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let accepted_connections = Arc::new(AtomicUsize::new(0)); @@ -181,32 +183,28 @@ async fn codex_backend_client_stream_should_retry_fresh_socket_when_first_output .await .unwrap(); - let (second_stream, _) = listener.accept().await.unwrap(); - accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); - let mut second_websocket = accept_codex_test_websocket(second_stream).await; - let _second_message = second_websocket.next().await.unwrap().unwrap(); - second_websocket + tokio::time::sleep(Duration::from_millis(50)).await; + first_websocket .send(Message::Text( json!({ "type": "response.output_text.delta", - "delta": "fresh retry recovered" + "delta": "fresh delayed output" }) .to_string() .into(), )) .await .unwrap(); - second_websocket + first_websocket .send(Message::Text( - completed_websocket_response("resp_first_token_fresh_recovered", 3, 1).into(), + completed_websocket_response("resp_fresh_delayed", 3, 1).into(), )) .await .unwrap(); - second_websocket.close(None).await.unwrap(); - drop(first_websocket); + first_websocket.close(None).await.unwrap(); }); let pool = Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig { - first_token_timeout: Some(Duration::from_millis(30)), + initial_event_timeout: Some(Duration::from_millis(30)), ..websocket_pool_config_for_tests(None, None, None) })); let backend = CodexBackendClient::new( @@ -215,33 +213,33 @@ async fn codex_backend_client_stream_should_retry_fresh_socket_when_first_output crate::support::fingerprint::runtime_test_fingerprint(), ) .with_websocket_pool(Arc::clone(&pool)); - let request = pooled_websocket_request("conversation-first-token-fresh-timeout"); + let request = pooled_websocket_request("conversation-structural-fresh"); let response = backend .create_response_stream( &request, - request_context("req_first_token_fresh_timeout", Some("chatgpt-account")), + request_context("req_structural_fresh", Some("chatgpt-account")), ) .await - .expect("fresh first-token timeout should retry before returning stream"); + .expect("structural activity should keep the fresh websocket open"); let decision = response.websocket_pool_decision; let mut stream = response.body; let mut body = String::new(); while let Some(chunk) = stream.next().await { - let chunk = chunk.expect("retried stream chunk should be valid"); + let chunk = chunk.expect("delayed fresh stream chunk should be valid"); body.push_str(std::str::from_utf8(&chunk).unwrap()); } server.await.unwrap(); - assert!(body.contains("fresh retry recovered")); - assert!(body.contains("resp_first_token_fresh_recovered")); - assert!(!body.contains("resp_first_token_fresh_stalled")); + assert!(body.contains("resp_first_token_fresh_stalled")); + assert!(body.contains("fresh delayed output")); + assert!(body.contains("resp_fresh_delayed")); assert_eq!(decision.unwrap().kind(), "new"); - assert_eq!(accepted_connections.load(Ordering::SeqCst), 2); + assert_eq!(accepted_connections.load(Ordering::SeqCst), 1); } #[tokio::test] -async fn codex_backend_client_stream_should_retry_reused_socket_when_first_output_times_out() { +async fn codex_backend_client_stream_should_keep_reused_socket_after_structural_activity() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let accepted_connections = Arc::new(AtomicUsize::new(0)); @@ -274,32 +272,28 @@ async fn codex_backend_client_stream_should_retry_reused_socket_when_first_outpu .await .unwrap(); - let (second_stream, _) = listener.accept().await.unwrap(); - accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); - let mut second_websocket = accept_codex_test_websocket(second_stream).await; - let _retry_message = second_websocket.next().await.unwrap().unwrap(); - second_websocket + tokio::time::sleep(Duration::from_millis(50)).await; + first_websocket .send(Message::Text( json!({ "type": "response.output_text.delta", - "delta": "reuse retry recovered" + "delta": "reused delayed output" }) .to_string() .into(), )) .await .unwrap(); - second_websocket + first_websocket .send(Message::Text( - completed_websocket_response("resp_first_token_reuse_recovered", 3, 1).into(), + completed_websocket_response("resp_reused_delayed", 3, 1).into(), )) .await .unwrap(); - second_websocket.close(None).await.unwrap(); - drop(first_websocket); + first_websocket.close(None).await.unwrap(); }); let pool = Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig { - first_token_timeout: Some(Duration::from_millis(30)), + initial_event_timeout: Some(Duration::from_millis(30)), ..websocket_pool_config_for_tests(None, None, None) })); let backend = CodexBackendClient::new( @@ -308,41 +302,41 @@ async fn codex_backend_client_stream_should_retry_reused_socket_when_first_outpu crate::support::fingerprint::runtime_test_fingerprint(), ) .with_websocket_pool(Arc::clone(&pool)); - let request = pooled_websocket_request("conversation-first-token-reuse-timeout"); + let request = pooled_websocket_request("conversation-structural-reuse"); let seed = backend .create_response( &request, - request_context("req_first_token_reuse_seed", Some("chatgpt-account")), + request_context("req_structural_reuse_seed", Some("chatgpt-account")), ) .await .expect("seed response should populate pool"); let response = backend .create_response_stream( &request, - request_context("req_first_token_reuse_timeout", Some("chatgpt-account")), + request_context("req_structural_reuse", Some("chatgpt-account")), ) .await - .expect("reused first-token timeout should retry before returning stream"); + .expect("structural activity should keep the reused websocket open"); let decision = response.websocket_pool_decision; let mut stream = response.body; let mut body = String::new(); while let Some(chunk) = stream.next().await { - let chunk = chunk.expect("retried reused stream chunk should be valid"); + let chunk = chunk.expect("delayed reused stream chunk should be valid"); body.push_str(std::str::from_utf8(&chunk).unwrap()); } server.await.unwrap(); assert!(seed.body.contains("resp_first_token_reuse_seed")); - assert!(body.contains("reuse retry recovered")); - assert!(body.contains("resp_first_token_reuse_recovered")); - assert!(!body.contains("resp_first_token_reuse_stalled")); - assert_eq!(decision.unwrap().kind(), "retry_after_stale_reuse"); - assert_eq!(accepted_connections.load(Ordering::SeqCst), 2); + assert!(body.contains("resp_first_token_reuse_stalled")); + assert!(body.contains("reused delayed output")); + assert!(body.contains("resp_reused_delayed")); + assert_eq!(decision.unwrap().kind(), "reuse"); + assert_eq!(accepted_connections.load(Ordering::SeqCst), 1); } #[tokio::test] -async fn codex_backend_client_stream_should_retry_bypass_socket_when_first_output_times_out() { +async fn codex_backend_client_stream_should_keep_bypass_socket_after_structural_activity() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let accepted_connections = Arc::new(AtomicUsize::new(0)); @@ -367,33 +361,29 @@ async fn codex_backend_client_stream_should_retry_bypass_socket_when_first_outpu .await .unwrap(); - let (second_stream, _) = listener.accept().await.unwrap(); - accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); - let mut second_websocket = accept_codex_test_websocket(second_stream).await; - let _second_message = second_websocket.next().await.unwrap().unwrap(); - second_websocket + tokio::time::sleep(Duration::from_millis(50)).await; + first_websocket .send(Message::Text( json!({ "type": "response.output_text.delta", - "delta": "bypass retry recovered" + "delta": "bypass delayed output" }) .to_string() .into(), )) .await .unwrap(); - second_websocket + first_websocket .send(Message::Text( - completed_websocket_response("resp_bypass_retry_recovered", 3, 1).into(), + completed_websocket_response("resp_bypass_delayed", 3, 1).into(), )) .await .unwrap(); - second_websocket.close(None).await.unwrap(); - drop(first_websocket); + first_websocket.close(None).await.unwrap(); }); let pool = Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig { max_per_account: 0, - first_token_timeout: Some(Duration::from_millis(30)), + initial_event_timeout: Some(Duration::from_millis(30)), ..websocket_pool_config_for_tests(None, None, None) })); let backend = CodexBackendClient::new( @@ -402,31 +392,31 @@ async fn codex_backend_client_stream_should_retry_bypass_socket_when_first_outpu crate::support::fingerprint::runtime_test_fingerprint(), ) .with_websocket_pool(Arc::clone(&pool)); - let request = pooled_websocket_request("conversation-first-token-bypass-timeout"); + let request = pooled_websocket_request("conversation-structural-bypass"); let response = backend .create_response_stream( &request, - request_context("req_first_token_bypass_timeout", Some("chatgpt-account")), + request_context("req_structural_bypass", Some("chatgpt-account")), ) .await - .expect("bypass first-token timeout should retry before returning stream"); + .expect("structural activity should keep the bypass websocket open"); let decision = response.websocket_pool_decision; let mut stream = response.body; let mut body = String::new(); while let Some(chunk) = stream.next().await { - let chunk = chunk.expect("retried bypass stream chunk should be valid"); + let chunk = chunk.expect("delayed bypass stream chunk should be valid"); body.push_str(std::str::from_utf8(&chunk).unwrap()); } server.await.unwrap(); - assert!(body.contains("bypass retry recovered")); - assert!(body.contains("resp_bypass_retry_recovered")); - assert!(!body.contains("resp_bypass_first_token_stalled")); + assert!(body.contains("resp_bypass_first_token_stalled")); + assert!(body.contains("bypass delayed output")); + assert!(body.contains("resp_bypass_delayed")); let decision = decision.expect("bypass decision"); assert_eq!(decision.kind(), "bypass"); assert_eq!(decision.reason(), Some("cap")); - assert_eq!(accepted_connections.load(Ordering::SeqCst), 2); + assert_eq!(accepted_connections.load(Ordering::SeqCst), 1); } #[tokio::test] @@ -464,6 +454,7 @@ async fn codex_backend_client_should_not_reuse_pooled_websocket_across_local_acc Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool".to_string())); let first = backend @@ -631,7 +622,8 @@ async fn websocket_pool_should_release_fresh_reservation_when_prefetch_is_cancel crate::support::fingerprint::runtime_test_fingerprint(), ) .with_websocket_pool(pool); - let request = pooled_websocket_request("conversation-cancelled-prefetch"); + let mut request = pooled_websocket_request("conversation-cancelled-prefetch"); + request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal; { let first_request = backend.create_response_stream( @@ -756,7 +748,18 @@ async fn websocket_pool_should_bypass_new_keys_after_account_cap() { let accepted_connections = Arc::new(AtomicUsize::new(0)); let accepted_connections_for_server = Arc::clone(&accepted_connections); let server = tokio::spawn(async move { - for response_id in ["resp_cap_first", "resp_cap_second", "resp_cap_third"] { + let (first_stream, _) = listener.accept().await.unwrap(); + accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); + let mut first_websocket = accept_codex_test_websocket(first_stream).await; + let _first_message = first_websocket.next().await.unwrap().unwrap(); + first_websocket + .send(Message::Text( + completed_websocket_response("resp_cap_first", 2, 1).into(), + )) + .await + .unwrap(); + + for response_id in ["resp_cap_second", "resp_cap_third"] { let (stream, _) = listener.accept().await.unwrap(); accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); let mut websocket = accept_codex_test_websocket(stream).await; @@ -767,10 +770,9 @@ async fn websocket_pool_should_bypass_new_keys_after_account_cap() { )) .await .unwrap(); - if response_id == "resp_cap_third" { - websocket.close(None).await.unwrap(); - } + websocket.close(None).await.unwrap(); } + drop(first_websocket); }); let pool = Arc::new(CodexWebSocketPool::new(1, Duration::from_mins(1))); let backend = CodexBackendClient::new( @@ -1050,6 +1052,7 @@ async fn codex_backend_client_should_keep_idle_pooled_websocket_alive_across_rep Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool-background".to_string())); let first = backend @@ -1156,6 +1159,7 @@ async fn codex_backend_client_should_close_idle_pooled_websocket_when_account_is Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool-evict".to_string())); let first = backend @@ -1260,6 +1264,7 @@ async fn codex_backend_client_should_stop_reusing_pooled_websockets_after_shutdo Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool-shutdown".to_string())); let first = backend @@ -1364,6 +1369,7 @@ async fn codex_backend_client_should_close_idle_pooled_websocket_after_liveness_ Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool-liveness".to_string())); let first = backend @@ -1459,6 +1465,7 @@ async fn codex_backend_client_should_discard_pooled_websocket_after_upstream_err Vec::new(), ); request.set_previous_response_id(Some("resp_previous".to_string())); + request.previous_response_scope = Some(PreviousResponseScope::Persisted); request.set_prompt_cache_key(Some("conversation-pool".to_string())); let first_error = backend @@ -1487,17 +1494,17 @@ async fn codex_backend_client_should_discard_pooled_websocket_after_upstream_err } #[tokio::test] -async fn codex_backend_client_should_reuse_pooled_websocket_after_unmapped_response_failed() { +async fn codex_backend_client_should_discard_pooled_websocket_after_unmapped_response_failed() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let accepted_connections = Arc::new(AtomicUsize::new(0)); let accepted_connections_for_server = Arc::clone(&accepted_connections); let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); + let (first_stream, _) = listener.accept().await.unwrap(); accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); - let mut websocket = accept_codex_test_websocket(stream).await; - let _first_message = websocket.next().await.unwrap().unwrap(); - websocket + let mut first_websocket = accept_codex_test_websocket(first_stream).await; + let _first_message = first_websocket.next().await.unwrap().unwrap(); + first_websocket .send(Message::Text( json!({ "type": "response.failed", @@ -1516,14 +1523,17 @@ async fn codex_backend_client_should_reuse_pooled_websocket_after_unmapped_respo .await .unwrap(); - let _second_message = websocket.next().await.unwrap().unwrap(); - websocket + let (second_stream, _) = listener.accept().await.unwrap(); + accepted_connections_for_server.fetch_add(1, Ordering::SeqCst); + let mut second_websocket = accept_codex_test_websocket(second_stream).await; + let _second_message = second_websocket.next().await.unwrap().unwrap(); + second_websocket .send(Message::Text( completed_websocket_response("resp_pool_after_unmapped_failed", 5, 2).into(), )) .await .unwrap(); - websocket.close(None).await.unwrap(); + second_websocket.close(None).await.unwrap(); }); let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1))); let backend = CodexBackendClient::new( @@ -1547,10 +1557,11 @@ async fn codex_backend_client_should_reuse_pooled_websocket_after_unmapped_respo request_context("req_pool_after_unmapped_failed", Some("chatgpt-account")), ) .await - .expect("terminal failed websocket should be reusable"); + .expect("terminal failed websocket should be replaced"); server.await.unwrap(); assert!(first.body.contains("resp_pool_model_refusal")); assert!(second.body.contains("resp_pool_after_unmapped_failed")); - assert_eq!(accepted_connections.load(Ordering::SeqCst), 1); + assert_eq!(second.websocket_pool_decision.unwrap().kind(), "new"); + assert_eq!(accepted_connections.load(Ordering::SeqCst), 2); } diff --git a/docs/previous-response-not-found-fix.md b/docs/previous-response-not-found-fix.md new file mode 100644 index 000000000..0e69c159d --- /dev/null +++ b/docs/previous-response-not-found-fix.md @@ -0,0 +1,940 @@ +# Responses WebSocket previous_response_id 低延迟恢复实施方案 + +日期：2026-07-11 + +目标基线：edbedd31ffb502b432e3c61def2ba41b51e858cd + +文档状态：P0/P1 已实施并通过本地完整测试；真实链路验证待执行 + +适用问题： + + stream disconnected before completion: + Upstream stream closed before response.completed: + previous_response_not_found + +## 1. 结论 + +不要把本问题实现成 GPT-5.6 模型特判，也不要直接照搬提交 +1b5bfb37e171e7478f3d4c666100d03a501da4ff 的全局预取改法。 + +推荐实现由三个部分组成： + +1. 在 WebSocket 连接对象自身记录该连接当前可以续接的最新 response ID。 +2. 发送 store=false + previous_response_id 前，先验证当前取出的池连接是否仍持有这个 ID。 +3. 只有代理自己做过隐式续接、手里保留完整输入快照时，才允许清除旧 ID 并透明重放；客户端显式传入但代理没有完整历史时，返回明确错误。 + +流式预取不能对所有请求统一等待真实输出。应根据请求是否具备安全重放能力选择提交边界： + +- 普通请求、显式 previous_response_id：保持原有低延迟提交行为。 +- 代理生成的隐式续接：在首个不可逆输出或终态前暂存结构帧，以便失败时安全重放。 + +这个设计比“连接代次写入 Redis”更直接。OpenAI 的缓存语义是“一条具体 WebSocket 上最近的 response”，因此最准确的真相源就是池中的连接对象，而不是账号亲和表、模型名或持久化数据库。 + +## 2. 基线说明 + +本文以 `edbedd31ffb502b432e3c61def2ba41b51e858cd` 作为事故复现与缺陷定位基线；P0/P1 实现落在后续依赖治理基线 `977d94d5812eebe57b319b4f00a99af485d28040` 之上。 + +不要直接 cherry-pick 1b5bfb37： + +- 它的父提交是 ddd6fe160f64c40df15f6260ed2513ead54e04e6，不是目标基线。 +- 它同时包含依赖、配置、监控、测试整理和文档变化。 +- 它把所有 WebSocket 流都推进到“真实输出或终态”才返回，存在不必要的首包延迟和首输出超时语义变化。 + +## 3. 已确认的生产事实 + +### 3.1 故障范围 + +线上 previous_response_not_found 按模型统计： + +| 模型 | 错误数 | 结论 | +| --- | ---: | --- | +| gpt-5.6-sol | 116 | 当前唯一观测到错误的模型 | +| 其他模型 | 0 | 不能据此证明相同连接条件下不会失败 | + +116 次失败全部满足： + + model = gpt-5.6-sol + transport = websocket + websocket_pool_kind = new + previous_response_id present = true + store = false + +成功请求的关键对照： + +| 模型 | previous_response_id + reuse 成功 | previous_response_id + new 成功 | +| --- | ---: | ---: | +| gpt-5.5 | 162 | 0 | +| gpt-5.6-sol | 117 | 0 | + +因此目前只能确认“故障在 GPT-5.6 上被观测到”，不能确认“GPT-5.5 在相同的 new + previous_response_id + store=false 条件下会成功”。线上没有这个对照样本。 + +### 3.2 本次 response ID 的时间线 + +失效 ID： + + resp_0de25f0dd99612a3016a51087f8b5c8194b20293e08fb6aa5f + +关键请求： + +| 北京时间 | 请求 ID | 事实 | +| --- | --- | --- | +| 22:58:06 | req_3090a40b-200e-4997-a910-01d522974a59 | gpt-5.6-sol，复用 WebSocket，正常完成并产生上述 response ID | +| 22:58:30 | 同上 | completed=true，store=false，首输出约 5599ms | +| 22:58:31 | req_aee4bb05-d67e-454e-9913-10a5db5afd64 | 同账号、同 conversation hash、同 pool key，再次复用；HTTP response 已返回，但没有流终结记录 | +| 23:00:22 起 | 后续重试 | 连接已经变成 new，继续携带旧 response ID | +| 23:05:16 | req_0b171310-8b4e-4b0b-90fa-fd27d334eda9 | new WebSocket，返回嵌套的 previous_response_not_found | +| 23:05:22 | req_215248f4-d0ed-44c2-a565-4a6fdd393bee | 同样失败 | + +原响应和失败请求使用同一账号、同一 conversation hash、同一 pool key。账号轮换不是根因。 + +### 3.3 首包与首输出数据 + +线上两个 Rust 容器均未覆盖 first_token_timeout_ms，因此目标基线会使用默认 20000ms。 + +最近三天 5722 条 gpt-5.6-sol 成功流记录： + +| 指标 | first_token_ms | +| --- | ---: | +| p50 | 2716ms | +| p95 | 6929ms | +| p99 | 11275ms | +| 大于 15 秒 | 21 条 | +| 大于 20 秒 | 9 条 | +| 最大值 | 37334ms | + +first_token_ms 与 WebSocket 内部熔断的起算点不完全相同，因此不能断言这 9 条都会触发超时；但足以证明不能把“等待真实输出”直接套进现有 20 秒超时而不做语义拆分。 + +故障请求中，上游错误通常在服务端返回 HTTP response 后约 39 至 75ms 到达。对可恢复请求多等待这几十毫秒可以捕获错误，但把同一等待策略扩大到所有正常请求是不必要的。 + +## 4. 官方协议依据 + +OpenAI WebSocket Mode 文档明确说明： + +- 活跃 WebSocket 只在连接本地内存缓存最近一条 previous response。 +- store=false 没有持久化回退。 +- ID 不在连接缓存时返回 previous_response_not_found。 +- 连接关闭后，如果不能从持久化响应续接，应清空 previous_response_id 并发送完整输入。 + +参考： + +- https://developers.openai.com/api/docs/guides/websocket-mode/ +- https://developers.openai.com/api/docs/guides/conversation-state/ + +官方 Codex 0.144.1 的实现也把增量状态绑定到当前 WebSocket session： + +- 构造增量请求： + https://github.com/openai/codex/blob/rust-v0.144.1/codex-rs/core/src/client.rs#L1223-L1253 +- 创建新连接前清空 last_request 和 last_response_rx： + https://github.com/openai/codex/blob/rust-v0.144.1/codex-rs/core/src/client.rs#L1323-L1354 +- 连接错误后新连接发送完整输入的测试： + https://github.com/openai/codex/blob/rust-v0.144.1/codex-rs/core/tests/suite/client_websockets.rs#L1986-L2073 + +官方行为是协议级状态管理，没有 GPT-5.6 模型分支。 + +## 5. 目标基线的缺陷位置 + +目标提交 edbedd31 中的相关入口： + +| 文件 | 当前职责 | 缺陷 | +| --- | --- | --- | +| backend/src/upstream/openai/transport/websocket_pool.rs | 按账号和 conversation 保存池连接 | PooledWebSocketConnection 不记录该连接缓存的最新 response ID | +| backend/src/upstream/openai/transport/websocket.rs | acquire、复用、新建和 stale-reuse 重试 | 新连接仍原样发送 store=false 的旧 previous_response_id | +| backend/src/upstream/openai/transport/websocket/stream.rs | 预取和 live stream 转发 | 首内容、连接健康、真实输出和下游提交共用近似语义 | +| backend/src/upstream/openai/protocol/websocket.rs | WebSocket 帧解析 | 错误分类丢失结构化 code；没有提取 completed response ID 的公共函数 | +| backend/src/dispatch/responses/service.rs | 隐式续接与 history recovery | recover_request_history 会在没有完整快照时直接删除客户端显式 history | +| backend/src/dispatch/responses/service/stream.rs | 流式重试循环 | 结构帧可能先越过预取边界，错误进入 live stream 后无法回到调度循环 | +| backend/src/dispatch/responses/service/complete.rs | 非流式重试循环 | 显式 previous ID 也可能被无损性不足的 strip-history 重试 | +| backend/src/dispatch/errors.rs | 错误分类 | previous_response_not_found 依赖正文字符串扫描 | +| backend/src/dispatch/responses/errors.rs | 对外调度错误 | 没有独立 HistoryUnavailable 类型 | +| backend/src/api/client/errors.rs | OpenAI 兼容错误输出 | 无稳定 previous_response_unavailable 错误码 | + +目标基线的 recover_request_history： + + 1. 忘记 affinity 中的旧 response ID。 + 2. 如果有 ImplicitResumeSnapshot，则恢复完整输入。 + 3. 无论是否有快照，都清除 previous_response_id 和 turn_state。 + 4. 重试同一请求。 + +第 3 步对客户端显式增量请求是不安全的。客户端可能只发送了“本轮新增 input”，删除 ID 后直接上送会丢失全部历史。 + +## 6. 根因模型 + + 完成响应 A，store=false + -> 上游 WebSocket W1 内存缓存 latest_response_id = A + -> 代理 session affinity 只记住账号和 conversation + -> 下一轮下游取消，代理销毁 W1 + -> W1 的连接本地缓存随之消失 + -> 客户端仍携带 A + -> 连接池没有 W1，只能创建 W2 + -> 当前代码把 A 原样发给 W2 + -> 上游返回 previous_response_not_found + -> 非输出结构帧先提交 HTTP/SSE + -> 错误只能包装成 stream_disconnected + +需要修复两个独立问题： + +1. 不允许把连接本地 ID 发到错误连接。 +2. 只有确实拥有完整上下文时才允许透明重放。 + +## 7. 必须保持的设计不变量 + +1. store=false 的 previous_response_id 只能发送到明确缓存该 ID 的同一池连接。 +2. store=true 可以在新连接上尝试持久化 hydration，不做本地提前拒绝。 +3. session affinity 只负责选账号和 conversation，不负责证明 response ID 在某条连接上可用。 +4. 连接对象是 connection-local previous response 状态的唯一真相源。 +5. 透明重放必须有完整输入快照作为能力凭证。 +6. 客户端显式 previous ID 且没有完整历史时，禁止删除 ID 后重试。 +7. 真实输出、工具调用参数或有效 output item 已经发给客户端后，禁止透明重试。 +8. 同一个入站请求最多做一次 history replay。 +9. 不通过 model == gpt-5.6 判断行为。 +10. 不把 store=false 静默改成 store=true。 +11. 不把 response ID 原文写入普通日志、错误消息或 WebSocket audit。 +12. 正常请求不能因为修复而统一推迟到真实 token 才返回 HTTP response。 + +## 8. 方案比较 + +### 8.1 GPT-5.6 模型特判 + +做法：仅在模型为 gpt-5.6-sol 时捕获错误或删除 ID。 + +拒绝原因： + +- 官方语义属于 WebSocket 和 store，不属于模型。 +- gpt-5.5 没有 new + previous_response_id 的线上对照样本。 +- 后续模型会继续复制同一协议行为。 +- 模型别名、路由后缀和新模型会形成维护表。 + +### 8.2 所有流统一预取到真实输出或终态 + +做法：把 prefetch_first_content_stream_frames 改为对所有请求等待真实输出或终态。 + +优点：实现直接，能够在 HTTP response 返回前捕获结构帧之后的错误。 + +拒绝作为最终方案的原因： + +- 正常请求的 response.created 或结构事件可能被延迟。 +- 会把现有 20 秒首内容熔断变成更严格的真实输出熔断。 +- 线上已有成功请求的首输出超过 20 秒。 +- 对客户端显式、不可重放的请求，即使提前捕获错误也无法透明恢复。 + +### 8.3 连接本地状态 + 能力感知预取 + +这是本文推荐方案： + +- 池连接记录 latest_response_id。 +- 发送前主动验证 continuation。 +- 只有 Replayable 请求使用 UntilOutputOrTerminal 预取。 +- 其他请求保持 FirstForwardableContent。 +- 上游仍返回 previous_response_not_found 时作为后备保护。 + +## 9. 推荐的数据结构 + +### 9.1 WebSocketContinuationRequirement + +在 backend/src/upstream/openai/transport/websocket/types.rs 增加： + + enum WebSocketContinuationRequirement { + NewChain, + Persisted { response_id: String }, + ConnectionLocal { response_id: String }, + } + +构造规则： + + previous_response_id 为空 + -> NewChain + + previous_response_id 存在且 store=true + -> Persisted + + previous_response_id 存在且 store=false + -> ConnectionLocal + +把该字段加入 CodexWebSocketRequest。它是本地控制信息，不进入上游 JSON。 + +不要在 transport 中重新解析 payload_text 获取 store 和 previous ID。构造 prepared request 时已经持有 CodexResponsesRequest，应在这里一次性生成强类型 requirement。 + +### 9.2 WebSocketContinuationState + +在 backend/src/upstream/openai/transport/websocket_pool.rs 增加： + + struct WebSocketContinuationState { + latest_response_id: Option, + } + +把它作为独立字段加入 PooledWebSocketConnection： + + struct PooledWebSocketConnection { + websocket: PumpedWebSocket, + metadata: CodexWebSocketConnectionMetadata, + continuation: WebSocketContinuationState, + created_at: Instant, + } + +不要把 latest_response_id 塞入 CodexWebSocketConnectionMetadata。握手 metadata、限流头、turn state 与协议续接状态生命周期不同，独立类型更清楚。 + +状态只在收到并成功校验 response.completed 后更新。以下情况不能记录： + +- response.created +- response.in_progress +- response.failed +- error +- response.incomplete +- completed 帧解析失败 +- 下游发送失败导致连接被 discard +- 上游在 terminal 前关闭 + +每条连接只保存一个最新 ID，正好对应官方“一条连接缓存最近一条 previous response”的语义。不需要 response ID 集合。 + +### 9.3 PreviousResponseUnavailableReason + +增加内部原因枚举： + + enum PreviousResponseUnavailableReason { + PoolUnavailable, + FreshConnectionRequired, + ConnectionBusy, + LatestResponseMismatch, + ReusedConnectionLost, + UpstreamRejected, + } + +Display 不得包含 response ID 原文。 + +对外错误统一为： + + type: invalid_request_error + code: previous_response_unavailable + status: 400 + message: Previous response context is no longer available. Start a new chain and resend the full input context. + +ConnectionBusy 不表示历史已经丢失，应单独返回可重试错误： + + status: 503 + code: continuation_connection_busy + +不得为了避开 Busy 在新连接上发送 store=false 的旧 ID。 + +### 9.4 HistoryRecoveryPlan + +在 dispatch 层用显式类型代替“Option 是否刚好为 Some”的隐含约定： + + enum HistorySource { + None, + Explicit, + Implicit, + } + + struct HistoryRecoveryPlan { + source: HistorySource, + replay: Option, + } + +规则： + +- 客户端原请求带 previous_response_id：source=Explicit，replay=None。 +- 代理根据完整输入自行应用 implicit resume：source=Implicit，replay=Some(snapshot)。 +- 没有 history：source=None，replay=None。 + +take_replay 消耗 Option，天然保证最多恢复一次。 + +### 9.5 StreamCommitPolicy + +增加： + + enum StreamCommitPolicy { + FirstForwardableContent, + UntilOutputOrTerminal, + } + +选择规则： + + recovery_plan.replay.is_some() + -> UntilOutputOrTerminal + + 其他 + -> FirstForwardableContent + +恢复后完整输入请求已经不带 previous ID，下一次尝试必须回到 FirstForwardableContent。 + +## 10. 连接池发送前验证 + +在 execute_response_create_request_with_pool 和 +execute_response_create_request_stream_with_pool 中，必须在 send 之前执行验证。 + +行为矩阵： + +| requirement | acquire 结果 | 行为 | +| --- | --- | --- | +| NewChain | Reused | 允许发送 | +| NewChain | FreshReserved / Bypass | 保持现有行为 | +| Persisted | 任意 | 允许发送，由上游 hydration 决定 | +| ConnectionLocal | Reused 且 latest ID 相同 | 允许发送 | +| ConnectionLocal | Reused 但 latest ID 不同 | 将连接原样 put 回池，返回 LatestResponseMismatch | +| ConnectionLocal | FreshReserved | discard 预留 slot，不建连，返回 FreshConnectionRequired | +| ConnectionLocal | Bypass(Busy) | 不建新连接，返回 ConnectionBusy | +| ConnectionLocal | Bypass(Disabled/Cap) | 不建新连接，返回 PoolUnavailable | +| ConnectionLocal | pool=None | 不建新连接，返回 PoolUnavailable | + +关键点： + +- mismatch 时不能销毁一条仍然健康的连接；它可能仍可服务携带 latest ID 的正确请求。 +- FreshReserved 已在池里放入 Busy 占位，提前返回前必须 discard，避免 slot 永久卡住。 +- 验证必须同时覆盖流式和非流式入口。 + +## 11. stale-reuse 重试规则 + +目标基线当前行为： + + 复用连接在首帧前死亡或首 token 超时 + -> 使用同一个 payload 新建 WebSocket 重试 + +这个逻辑对 NewChain 合理，对 store=false + previous_response_id 错误。 + +修改为： + +| 请求类型 | 复用连接失效后的行为 | +| --- | --- | +| NewChain | 保持现有 fresh retry | +| Persisted | 可以 fresh retry | +| ConnectionLocal | 禁止 transport 原样 fresh retry，返回 ReusedConnectionLost | + +ReusedConnectionLost 交给 dispatch： + +- HistoryRecoveryPlan 有 replay：恢复完整请求并重试。 +- 无 replay：返回 previous_response_unavailable。 + +这样 transport 不需要理解完整上下文，dispatch 也不会错误猜测连接状态。 + +## 12. 成功响应如何更新 latest_response_id + +### 12.1 协议解析 + +在 backend/src/upstream/openai/protocol/websocket.rs 增加： + + websocket_response_completed_id(raw: &str) -> Result, String> + +行为： + +- 非 response.completed 返回 Ok(None)。 +- completed 先沿用现有 ResponseCompleted 强类型校验。 +- 校验成功后返回 response.id。 +- id 缺失或为空返回解析错误。 + +不要通过搜索原始字符串 resp_ 提取。 + +### 12.2 非流式 WebSocket + +collect_websocket_response 接收并返回 WebSocketContinuationState。 + +读取到 response.completed： + +1. 校验 completed shape。 +2. 提取 ID。 +3. continuation.latest_response_id = Some(id)。 +4. 返回 exchange、websocket、metadata、continuation。 +5. pool.put 时把 continuation 放回 PooledWebSocketConnection。 + +### 12.3 流式 WebSocket + +WebSocketStreamPoolReturn 增加 continuation 字段。 + +forward_websocket_response_stream 读取到 response.completed 时： + +1. 在发送 terminal SSE 前完成 ID 校验和状态更新。 +2. 如果下游发送成功，finish_stream_websocket 把新状态放回池。 +3. 如果下游发送失败，discard 连接，不保留状态。 + +新建池连接的 continuation 为 default；复用连接沿用取出的 continuation。 + +## 13. 错误必须结构化分类 + +目标基线的 is_history_recovery_signal 会扫描： + +- previous_response_not_found +- previous response + not found +- no tool output found +- invalid encrypted content + +本问题不能继续依赖 message 文本。 + +修改 protocol/websocket.rs 的 ClassifiedWebSocketError，使其保留 error.code。至少精确识别： + + previous_response_not_found + +WebSocket error 事件中 code 的读取路径继续支持现有官方形状： + + /response/error/code + /error/code + +ResponsesSseFailure 已有 upstream_code。增加精确函数： + + fn is_previous_response_not_found_failure(failure: &ResponsesSseFailure) -> bool { + failure.upstream_code.as_deref() == Some("previous_response_not_found") + } + +消息字符串只能用于诊断展示，不能决定是否清除历史和重放。 + +invalid_encrypted_content 和 tool output 缺失属于不同失败种类，可以继续走各自恢复策略，但不要再与 previous_response_not_found 合并为一个模糊 bool。 + +建议增加： + + enum HistoryFailureKind { + PreviousResponseNotFound, + InvalidEncryptedContent, + MissingToolOutput, + } + +## 14. dispatch 安全恢复 + +### 14.1 调整快照捕获时机 + +目标基线 apply_implicit_resume 在完成 affinity 查询前就 clone 完整 input。 + +将 ImplicitResumeSnapshot::capture(request) 移到所有资格检查通过之后、第一次修改 request 之前： + + 1. 找 continuation_start。 + 2. 查 conversation、variant、instructions、function call、account。 + 3. 构造 replay input。 + 4. 此时 capture 原始完整请求。 + 5. 再 set_previous_response_id、set_input、修改 transport。 + +这样未命中隐式续接时不会无意义 clone 大输入。 + +### 14.2 替换 recover_request_history + +删除“无快照也 strip history”的行为。 + +新函数语义： + + async fn try_replay_full_request( + request, + recovery_plan, + account_id, + evict_reasoning_replay, + ) -> bool + +流程： + +1. recovery_plan.take_replay()。 +2. 没有 snapshot，返回 false，不修改请求。 +3. 必要时清除 reasoning replay。 +4. 忘记失效 response ID 的 session affinity。 +5. snapshot.restore(request)。 +6. 清除 previous_response_id 和 turn_state。 +7. 返回 true。 + +只有返回 true 才能 continue 调度循环。 + +### 14.3 显式 previous ID + +客户端显式 previous ID 失败时： + +- 不删除 ID 后重试。 +- 不切换账号后用残缺输入重试。 +- 释放账号 lease。 +- 返回 ResponseDispatchError::HistoryUnavailable。 + +非流式响应： + + HTTP 400 + type = invalid_request_error + code = previous_response_unavailable + +流式请求如果尚未开始下游 body： + + 保持 Responses 流式兼容形状 + event = response.failed + code = previous_response_unavailable + +不得把完整上游 JSON 嵌套进 stream_disconnected message 返回给客户端。 + +### 14.4 同账号重放 + +有快照的恢复必须固定到刚才使用的账号： + + next_required_account_id = Some(release_account_id) + +原因： + +- reasoning replay 和 account scope 相关。 +- 避免恢复动作变成账号轮换。 +- 与现有 session affinity 语义一致。 + +## 15. 流式提交和超时必须拆成三个概念 + +不要再用 saw_content_frame 同时表示三个状态。 + +至少拆为： + + saw_upstream_activity + saw_irreversible_output + downstream_commit_ready + +含义： + +| 状态 | 用途 | +| --- | --- | +| saw_upstream_activity | 判断连接是否已经有响应，切换初始等待与 active idle timeout | +| saw_irreversible_output | 判断透明重试是否还安全，并记录真实 TTFT | +| downstream_commit_ready | 由 StreamCommitPolicy 决定何时返回下游 Response | + +事件分类： + +| 事件 | activity | irreversible output | terminal | +| --- | --- | --- | --- | +| response.created | 是 | 否 | 否 | +| response.in_progress | 是 | 否 | 否 | +| response.content_part.added 且文本为空 | 是 | 否 | 否 | +| metadata / rate_limits | 是 | 否 | 否 | +| 非空 reasoning delta | 是 | 是 | 否 | +| 非空 output_text delta | 是 | 是 | 否 | +| function_call_arguments delta | 是 | 是 | 否 | +| 有效 output_item.added | 是 | 是 | 否 | +| response.completed | 是 | 否 | 是 | +| response.failed / error | 是 | 否 | 是 | + +### 15.1 超时调整 + +目标基线的 first_token_timeout_ms 名字和行为已经混合“连接健康”与“真实输出”。 + +推荐在本次实现中拆分： + + ws_pool: + initial_event_timeout_ms: 20000 + first_output_timeout_ms: 0 + +语义： + +- initial_event_timeout_ms：发出 response.create 后，等待第一个有效上游活动事件的时间。 +- first_output_timeout_ms：等待不可逆输出的可选业务上限；0 表示禁用。 +- 收到任何有效活动后使用现有 active stream idle timeout。 + +生产配置目前没有显式 first_token_timeout_ms，因此可以直接把示例和默认配置迁移到新字段，不需要保留旧字段兼容分支。 + +如果暂时不想改配置结构，最低要求也是： + +- guarded prefetch 收到 response.created 或其他结构事件后，不得继续使用 20 秒“无活动”熔断。 +- 之后使用 active idle timeout 等待 output 或 terminal。 + +不要简单把默认值从 20 秒拍脑袋改成 60 秒，这只是在掩盖两个不同概念。 + +## 16. 按文件实施步骤 + +### 第一步：协议与纯类型 + +修改： + +- backend/src/upstream/openai/protocol/websocket.rs +- backend/src/upstream/openai/transport/websocket/types.rs + +增加： + +- completed response ID 提取。 +- 结构化 WebSocket error code。 +- WebSocketContinuationRequirement。 +- PreviousResponseUnavailableReason。 +- StreamCommitPolicy。 + +先写纯单元测试，再进入连接池代码。 + +### 第二步：连接池 continuation 状态 + +修改： + +- backend/src/upstream/openai/transport/websocket_pool.rs +- backend/tests/upstream/openai/transport/websocket_pool.rs + +增加 WebSocketContinuationState，并覆盖 acquire、put、discard、过期关闭的状态生命周期。 + +### 第三步：transport 发送前校验 + +修改： + +- backend/src/upstream/openai/transport/websocket.rs +- backend/src/upstream/openai/transport/client/requests.rs +- backend/src/upstream/openai/transport/client.rs + +确保流式、非流式、pool=None、FreshReserved、Bypass 和 stale-reuse 全部分支遵循行为矩阵。 + +### 第四步：完成事件更新连接状态 + +修改： + +- backend/src/upstream/openai/transport/websocket/stream.rs +- backend/src/upstream/openai/transport/websocket.rs + +让聚合和 live stream 两条路径都只在合法 response.completed 后更新 latest ID。 + +### 第五步：dispatch 恢复能力 + +修改： + +- backend/src/dispatch/implicit_resume.rs +- backend/src/dispatch/responses/service.rs +- backend/src/dispatch/responses/service/complete.rs +- backend/src/dispatch/responses/service/stream.rs +- backend/src/dispatch/responses/sse_failure.rs + +引入 HistoryRecoveryPlan，延后快照 clone，删除显式 history 的不安全 strip-and-retry。 + +### 第六步：客户端错误和可观测性 + +修改： + +- backend/src/dispatch/errors.rs +- backend/src/dispatch/responses/errors.rs +- backend/src/api/client/errors.rs +- backend/src/dispatch/responses/event_recording.rs +- backend/src/dispatch/responses/stream_lifecycle.rs + +增加 stable error code 和诊断字段。 + +### 第七步：超时语义 + +修改： + +- backend/src/bootstrap/config.rs +- backend/src/bootstrap/services.rs +- backend/src/upstream/openai/transport/websocket_pool.rs +- deploy/config.example.yaml + +拆分 initial activity 和 first output timeout。不要保留两套字段别名。 + +## 17. 必须新增的测试 + +### 17.1 协议单元测试 + +1. response.completed 能提取 response.id。 +2. 非 completed 返回 None。 +3. completed 缺少 ID 返回解析错误。 +4. error.code=previous_response_not_found 精确分类。 +5. 仅 message 包含相同文字但 code 不匹配时，不触发自动恢复。 + +### 17.2 连接池测试 + +1. 首次成功响应后，池连接保存 latest response ID。 +2. 相同 ID 在同一连接上允许续接。 +3. 不同 ID 被拒绝，并把健康连接放回池。 +4. FreshReserved + connection-local ID 不建立 TCP/WebSocket。 +5. Busy + connection-local ID 不 bypass 到新连接。 +6. store=true previous ID 允许新连接 hydration。 +7. discard 后 continuation state 一起消失。 +8. max-age 过期后旧 ID 不可继续发送。 + +### 17.3 transport 集成测试 + +1. new chain 完成后，第二轮在同一 socket 发送 previous ID。 +2. 第二轮前关闭池连接，断言不会在新 socket 发送旧 ID。 +3. reused socket 在 send 前死亡，connection-local 请求不得 fresh retry 原 payload。 +4. 无 previous ID 的 stale-reuse 仍可 fresh retry。 +5. store=true 的 previous ID 可以在 fresh socket 发送。 +6. 上游没有收到被提前拒绝请求的 payload。 + +### 17.4 dispatch 恢复测试 + +1. implicit resume + 池连接丢失：直接恢复完整 input，只向上游发送一次完整请求。 +2. explicit previous ID + 池连接丢失：返回 previous_response_unavailable，上游零请求。 +3. implicit resume 在匹配连接上收到： + + response.created + response.content_part.added + previous_response_not_found + + 断言结构事件未下发，完整请求重放一次。 + +4. explicit previous ID 收到同样错误：不重放。 +5. 真实 output_text.delta 后再失败：不重放。 +6. function call 参数产生后再失败：不重放。 +7. recovery plan 消耗后第二次错误不再重试。 +8. 恢复固定在原账号。 +9. 测试模型至少包含 gpt-5.6-sol，但实现断言不能检查模型名。 + +### 17.5 延迟和超时测试 + +使用 Tokio paused time 覆盖： + +1. FirstForwardableContent 策略在结构内容到达时立即返回，不等真实 delta。 +2. UntilOutputOrTerminal 策略缓存结构内容。 +3. 收到 response.created 后等待 30 秒再输出，不触发 20 秒“无活动”超时。 +4. 真实输出到达后立即提交，额外本地开销不包含固定 sleep。 +5. terminal error 到达后立即返回恢复循环。 + +## 18. 可观测性 + +普通日志和 ops error metadata 增加： + + history_source = none | explicit | implicit + history_replay_available = true | false + ws_continuation = new_chain | persisted | matched | missing | mismatch | busy + recovery_action = none | replay_full | reject + recovery_attempt = 0 | 1 + prefetch_policy = first_content | output_or_terminal + upstream_request_sent = true | false + initial_event_ms + first_output_ms + response_ready_ms + +response ID 和 connection 标识只记录稳定短 hash，不记录原值。 + +这些内部诊断不得写成额外 usage_records。usage_records 只代表真实的 /v1 请求事实；恢复原因写入现有请求的 metadata 或 ops error log。 + +WebSocket audit 继续脱敏： + +- instructions +- input +- previous_response_id +- prompt_cache_key +- client_metadata +- tools + +## 19. 延迟验收标准 + +需要分别比较： + + request_start -> downstream HTTP response ready + request_start -> first SSE event + request_start -> first irreversible output + upstream first output -> downstream first output + +验收要求： + +1. NewChain 和显式匹配续接的 response-ready 分布不得因修复系统性后移。 +2. upstream first output 到 downstream first output 不得出现固定等待窗口。 +3. normal success 的 p50/p95 TTFT 与基线差异应处于本地调度抖动范围。 +4. connection-local mismatch 应在建立新 WebSocket 前失败。 +5. replayable mismatch 应比“先请求上游再等 400”更快。 +6. first output 超过 20 秒但已有上游活动的请求不能被误判为死连接。 + +不要使用 50ms、100ms 之类固定 grace window 等待错误。生产错误碰巧在 39 至 75ms 内到达，不代表协议保证这个时序。 + +## 20. 验证命令 + +回退后先验证基线： + + cargo test --manifest-path backend/Cargo.toml previous_response + cargo test --manifest-path backend/Cargo.toml websocket_pool + +开发中按阶段运行： + + cargo test --manifest-path backend/Cargo.toml websocket_response_completed + cargo test --manifest-path backend/Cargo.toml previous_response + cargo test --manifest-path backend/Cargo.toml responses_websocket + cargo test --manifest-path backend/Cargo.toml websocket_pool + +最终验证： + + cargo fmt --manifest-path backend/Cargo.toml -- --check + cargo clippy --manifest-path backend/Cargo.toml --all-targets --all-features -- -D warnings + cargo test --manifest-path backend/Cargo.toml + bash backend/scripts/check_architecture.sh + +如果测试中确认真实缺陷，先修复该缺陷并重新跑当前阶段，再进入下一组测试。 + +## 21. next 实例真实链路验证 + +不要直接切主实例。先在 next 实例开启 WebSocket audit： + + CODEX_PROXY_WS_AUDIT_DIR=.runtime/ws-audit + +场景矩阵： + +| 场景 | 预期 | +| --- | --- | +| gpt-5.6-sol，同 socket 连续两轮 | 第二轮 matched，正常增量 | +| 第一轮完成后主动 discard socket，第二轮为显式 ID | 本地 previous_response_unavailable，上游零请求 | +| 第一轮完成后 discard，第二轮为代理 implicit resume | 直接发送完整 input，不发送旧 ID | +| store=true，第一轮 socket 丢失 | 允许新连接尝试 hydration | +| 同 conversation 并发导致 Busy | 不创建新连接发送 connection-local ID | +| 结构事件后 upstream not found，且可重放 | 客户端不看到旧结构事件，完整重放成功 | +| 首输出超过 20 秒但持续有生命周期活动 | 不因 initial activity timeout 失败 | +| 真实输出后 upstream 错误 | 不透明重放 | + +真实链路文档必须记录每个场景的 request ID、pool decision、是否真的发送上游、first output 和 response-ready 时间。 + +## 22. 部署观察 + +灰度时重点观察： + +- previous_response_not_found 按 ws_continuation 原因分组。 +- previous_response_unavailable 数量。 +- recovery_action=replay_full 成功率。 +- upstream_request_sent=false 的提前拦截数量。 +- gpt-5.6-sol 的 response-ready、first-output p50/p95/p99。 +- ConnectionBusy 数量。 +- WebSocket pool discard reason。 +- 是否仍出现同一个 response ID 连续重试风暴。 + +不应观察到： + +- gpt-5.6 专属分支命中日志。 +- 显式 previous ID 被 strip 后当作新链发送。 +- store=false 旧 ID 出现在 ws_pool=new 的上游 payload。 +- 正常请求统一延迟到文本 delta 才返回。 +- 因内部恢复产生额外 usage_records。 + +## 23. 回滚 + +本方案不引入数据库 schema，也不依赖 Redis 持久化，因此回滚只涉及二进制和配置。 + +如果采用新的超时字段，回滚旧版本前必须同步恢复旧配置字段，否则旧版本会因 deny_unknown_fields 拒绝启动。 + +回滚触发条件： + +- 正常请求 response-ready p95 明显回退。 +- gpt-5.6-sol 首输出超时增加。 +- pool Busy 导致大量可用请求失败。 +- store=true hydration 被错误提前拒绝。 +- full replay 出现重复工具调用或重复输出。 + +## 24. 实施完成定义 + +只有全部满足才算修复完成： + +- store=false 旧 ID 不会发送到新连接。 +- 连接只保存最近一个成功 completed response ID。 +- 显式增量请求没有完整历史时不会伪恢复。 +- 隐式续接丢失连接时能够用完整输入恢复。 +- 上游 exact code 结构化分类，不靠 message 判断本错误。 +- 已输出后不重试。 +- 正常请求不承担全局预取延迟。 +- initial activity 与 first output 超时语义已经分开。 +- 流式和非流式测试均覆盖。 +- GPT-5.6 真实链路验证通过。 +- ops 日志可解释每次 reject 或 replay。 +- 完整 fmt、clippy、test、architecture check 通过。 + +## 25. 参考工程的取舍 + +sub2api： + + /home/zyy/桌面/Codes/sub2api/backend/internal/service/openai_ws_forwarder_ingress.go + +其优点是保留 replay input、只做一次恢复，并避免在 function_call_output 无完整链路时盲目删除 ID。可以参考“有完整重放输入才恢复”的原则。 + +CLIProxyAPI： + + /home/zyy/桌面/Codes/CLIProxyAPI/sdk/api/handlers/openai/openai_responses_websocket.go + +其优点是保存 transcript，并在可恢复错误后释放 pinned auth、强制下一次完整 transcript replay。可以参考“连接失效后增量状态也必须失效”的原则。 + +不应直接照抄： + +- sub2api 的 message/场景启发式不能替代强类型连接状态。 +- CLIProxyAPI 的完整 transcript 常驻策略会增加内存和隐私边界。 +- 本工程已有 ImplicitResumeSnapshot，应该复用现有完整输入能力，而不是再建第二套 transcript 缓存。 + +## 26. 最终建议 + +实现时优先完成以下最小闭环： + + PooledWebSocketConnection.latest_response_id + + store=false 发送前精确校验 + + stale-reuse 禁止原 payload 新连接重试 + + HistoryRecoveryPlan + + 显式 history 不再 strip-and-retry + + 结构化 previous_response_unavailable + +能力感知预取和超时状态拆分必须与流式回归测试一起完成，不能只移植全局 UntilOutputOrTerminal。 + +这套实现既能解决当前 GPT-5.6 事故，也保持模型无关、低延迟、无额外持久化和清晰的状态所有权。 diff --git a/docs/previous-response-websocket-recovery-implementation.md b/docs/previous-response-websocket-recovery-implementation.md new file mode 100644 index 000000000..3b89e2b48 --- /dev/null +++ b/docs/previous-response-websocket-recovery-implementation.md @@ -0,0 +1,1088 @@ +# Codex 上游交互与 Responses WebSocket 恢复审计 + +日期：2026-07-11 + +工程基线：`977d94d5812eebe57b319b4f00a99af485d28040` + +上游基线：OpenAI Codex `5c19155cbd93bfa099016e7487259f61669823ff` + +桌面版本：Codex Desktop `26.707.41301` + +文档状态：P0/P1 已实施并通过本地完整测试；真实链路验证待执行。 + +## 1. 审计范围 + +本次审计同时对照以下三类证据： + +1. 当前工程的 `/v1/responses`、HTTP/SSE、Responses WebSocket、连接池、隐式续接、账号调度和遥测链路。 +2. `/home/zyy/桌面/Codes/codex-desktop-linux` 中锁定的最新桌面版本与其依赖信息。 +3. OpenAI Codex 官方源码在 `5c19155c` 的请求构造、连接状态机、响应解析、错误分类和默认阈值。 + +重点检查： + +- `previous_response_id` 在连接复用、断线、换连接和恢复时的所有权。 +- HTTP/SSE 与 WebSocket 的请求体、请求头和客户端元数据。 +- 根据响应头、流事件和错误字段做出的运行时判断。 +- 首输出、空闲、连接寿命、重试和账号轮换阈值。 +- 模型目录、限流、有效模型、安全元数据和诊断信息的更新链路。 + +本工程是透明代理，不应机械复制 Codex CLI 的请求构造器。官方源码用于确认协议语义和服务端约定；客户端请求的业务字段和上游响应的业务字段默认保持原样。 + +### 1.1 透明代理边界 + +这里的“透明”是 JSON、header 和 SSE 事件的协议语义透明。代理会解析并重新序列化 JSON，因此不承诺空白字符逐字节相同，但字段、值、类型、未知扩展和事件顺序不能被无理由修改。 + +允许的变化是封闭清单： + +1. 将客户端 API key 替换为选中上游账号的 Authorization，并设置对应 `chatgpt-account-id`。 +2. 剥离 hop-by-hop header、客户端本地 transport hint，以及不能泄漏给另一侧的 cookie、token 和账号身份。 +3. 注入上游连接必需的 Host、WebSocket upgrade、beta、指纹和受控风控 header。 +4. 在客户端显式使用代理公开的模型别名时，将别名解析成真实 model ID；同时保留 requested model 供审计。 +5. HTTP/SSE 与 WebSocket 之间只转换传输 envelope，事件 JSON 载荷保持不变。 +6. 在账号认证、额度、模型能力或单节点 Cloudflare 风控失败时更新账号状态，并按 failure scope 决定继续候选或触发级联保护。 +7. 对会跨账号建立关联的 identity/risk 字段做账号作用域伪名化；原始值只留在本地请求上下文，不直接暴露给上游账号。 + +以下行为默认禁止： + +- 给 body 注入客户端没有提供的业务默认值。 +- 给缺失的 `prompt_cache_key` 生成值，或生成/覆盖 `previous_response_id`、input、reasoning、tools、text、service tier。 +- 覆盖普通客户端 metadata；只有明确列入安全策略的 reserved identity key 可以被账号作用域值替换。 +- 在正常路径为了优化而把完整请求改写成增量请求；账号 failover 只有持有完整快照时才允许派生语义等价的完整请求。 +- 把可由客户端处理的上游 4xx code、message、param 或 JSON body 包装成另一种错误。 +- 给成功响应增加便利字段，或改写上游已经返回的 output/event 内容。 + +账号轮换是明确的代理职责，不受普通透明透传限制。history 所属账号不可用且账号池仍有可用账号时，代理不能直接中断客户端；它必须使用自己完整保存的上下文，在当前设置所选调度策略排出的新账号上执行语义等价的 full replay。约束是不能把原账号/原 socket 绑定的同一 `previous_response_id` 直接发给新账号，也不能用残缺上下文假装恢复。 + +### 1.2 账号池与账号保护不变量 + +“发挥账号池效果”和“保护账号”必须同时成立： + +1. 每个请求进入 dispatch 时冻结一份符合模型、计划、账号状态和冷却条件的候选账号 ID 快照；亲和账号和订阅层级只决定顺序，不得裁掉低优先级但可用的账号。 +2. node-local 账号失败必须更新账号状态、从本请求排除该账号并继续下一个候选。只有候选快照中的账号都已尝试或被实时状态排除，才能生成账号池耗尽错误；瞬时 slot/connection Busy 只改变租用时机，应先尝试其他账号再回访，不能冒充账号耗尽。本实现不设代理自定义的候选次数、换号时间预算或风险熔断。 +3. 固定数字只能限制单账号内部的网络重试，不能限制账号候选遍历。候选有 20 个时，quota verify、model unsupported 或 5xx failover 都不能在第 2 或第 5 个账号提前结束。 +4. 每次换账号都重新构造账号 envelope：Authorization、上游 account ID、cookie、Cloudflare 状态、连接、配额、冷却和受控指纹只来自新账号及代理运行时，不能继承客户端或前一账号的值。 +5. `previous_response_id`、turn state、连接本地状态和账号绑定的加密 reasoning 不能跨账号；history-bound 请求必须从不含凭据的完整业务快照派生 full replay。 +6. 客户端业务 body、工具调用和工具输出默认不变。风控剥离只处理账号/连接绑定信息、不可信身份 header 和明确的 reserved identity metadata，不能借机改写 prompt、reasoning 配置、tools 或响应内容。 +7. 账号禁用、封禁、额度耗尽、429 冷却、Cloudflare challenge/path block、cookie 更新和模型不支持都要反馈到账号池，避免后续请求继续伤害同一账号。 +8. 已经向客户端提交不可逆输出后，盲目换账号重放可能重复文本、工具调用或副作用，禁止这样做；账号池耗尽保证适用于请求仍可安全重放的阶段。 + +只有精确的账号信号才能修改持久/运行时账号状态。generic 5xx、网络断开、空响应和客户端请求错误只在当前请求 ledger 中排除该账号，不得据此把账号永久禁用、写长期冷却或污染模型能力；这类状态变化需要明确 code/header 或独立探测证据。 + +账号保护也意味着不能把确定性的客户端错误广播到整个账号池。请求结构非法、上下文超限、缺少 tool output 等与账号无关的 4xx 应保留首个上游原始错误并立即返回；账号认证、额度、风控、能力和可恢复传输故障才进入账号遍历。错误分类必须基于结构化 status/code，而不是 message 文本猜测。 + +### 1.3 故障域与风险剖离 + +账号池中的每个账号是独立故障节点，但账号共享代理出口、运行时指纹和请求来源，因此不能假设所有失败都只属于单节点。 + +| 故障域 | 识别依据 | 动作 | +| --- | --- | --- | +| 节点级 | 精确 token/quota/account/model capability，或只在一个账号出现的 cookie/连接故障 | 隔离该账号，清理其连接/状态，使用干净 envelope 继续下一候选 | +| 会话污染 | previous ID、turn state、加密 reasoning 与目标账号/socket 不匹配 | 不发送污染值；从完整快照派生 full replay | +| 请求级 | schema、param、context、tool output、policy 等确定性 4xx | 立即停止 fanout，原样返回，不触碰其他账号 | +| 单节点风控 | challenge/path-block/new-ban 等结构化信号 | 只隔离该账号的 cookie、连接和状态，重建干净 envelope 后继续遍历当前设置所选调度策略排序后的剩余候选 | +| 共享上游 | 多账号返回相同明确的 global overload/maintenance 信号 | 仅作为当前请求的结构化失败证据，不打开 endpoint circuit，不修改未证实异常的账号状态 | + +风险剖离不得变成请求级熔断。任何账号出现风控信号时，只清理该账号及该连接绑定的 cookie、turn state、previous ID、encrypted reasoning 和 identity；换号仍调用当前设置所选调度策略，不改成固定 Smart、插入顺序、tier 顺序或手工风险队列。只有确定性客户端请求错误、外部未知 history 缺少完整快照或输出已提交时才停止跨账号重放。 + +每次 failover 必须从不可变请求重新构造，不能在同一个可变 JSON 上连续剥字段。否则第一个节点的响应状态、cookie、turn state 或恢复 patch 会悄悄进入后续所有节点，形成真正的池级污染。 + +## 2. 审计基线结论（实施前） + +当前实现与最新官方行为及透明代理约束存在以下差异。 + +### P0 + +1. `previous_response_id` 仍未绑定到产生它的具体 WebSocket。连接池对象没有保存 `latest_response_id`，新连接、旁路连接或 stale-reuse 后的新连接仍可能发送代理已知源自旧连接 `store=false` 响应的 ID。 +2. history-bound 请求遇到 429 等账号错误后，现有实现和测试会换账号但仍原样发送同一 `previous_response_id`。账号应该轮换，但轮换请求必须由完整上下文重建为 full replay，不能复用旧 ID。 +3. 生产上的上游 `400 previous_response_not_found` 被改写成 HTTP 200 流中的代理 `stream_disconnected`，原始 code 只被塞进 message 字符串，客户端无法按官方错误结构处理。 +4. 所有 WebSocket 流当前都会预取到“首个真实输出或终态”，并受默认 20 秒绝对首输出超时约束。它已改变正常流的提交边界，且会把仍有生命周期活动但思考超过 20 秒的请求误判为故障。 +5. dispatch 没有“完整候选集已耗尽”的证明：quota verify 固定最多 5 个账号，model unsupported 只允许一次换号，单账号 5xx 重试结束后直接返回，普通可恢复 transport error 也直接返回。账号池可能仍有可用账号。 +6. 当前跨账号 history 清理只在临时 `ImplicitResumeSnapshot` 存在时生效。客户端显式 previous ID 或跨请求恢复没有完整快照，旧账号的 ID/turn state 仍可能进入新账号，账号轮换与会话隔离不能同时保证。 +7. Cloudflare/path-block 必须只作为账号级 tracker：隔离当前账号并重建 envelope，换号继续使用当前设置所选调度策略，不引入跨账号 risk guard 或 endpoint circuit。 + +### P1 + +1. `from_body()` 注入缺失的 input、stream、store；session logic 会给缺失的 `prompt_cache_key` 生成值；implicit resume 还会生成 previous ID 并替换 input。这些均超出透明代理职责。显式 identity 的账号作用域转换属于保护职责，但必须与业务默认注入分开。 +2. `response.incomplete` 会被 `completed_response_metadata()` 当作可续接完成响应，进而写入 affinity 和 reasoning replay；官方只允许 `response.completed` 推进续接状态。 +3. `session_id`、`thread_id` 和 `x-client-request-id` 被合并成一个账号作用域值，破坏了字段各自语义；同一个全局 installation ID 还会发送给池内全部账号，形成跨账号关联标识。 +4. WebSocket 包裹错误只保留状态码，丢失精确 `error.code`、完整 headers 和认证诊断；恢复逻辑仍需要扫描字符串。 +5. 非流式响应会补写 `output` 和 `output_text`，流异常会补写代理 failure；只有 transport 确实缺少等价表示时才允许合成，并必须保留原始上游字段。 +6. 未消费或未向客户端传递 `x-models-etag`、`openai-model`、`x-reasoning-included`、模型验证、审核/安全缓冲和 `end_turn` 等数据。 +7. 限流解析只覆盖 core 与 code review 固定族，缺少任意 metered limit、credits、plan type、promo message 和 reached type。 + +### P2 + +1. Responses Lite、memory consolidation、W3C trace context 和 attestation 只应做受控透传；代理不应复制官方客户端的 body 构造逻辑。 +2. 同账号 5xx 重试没有退避和抖动，账号内重试结束后也不会继续完整候选集；empty response 只计固定次数且不明确排除当前账号。 +3. 模型目录按三个端点顺序全量读取，没有利用响应 ETag 驱动刷新。 + +这些问题必须按协议语义修复，不能增加模型名特判或兼容分支。 + +## 3. 权威证据 + +### 3.1 本地桌面版本 + +`/home/zyy/桌面/Codes/codex-desktop-linux/flake.nix` 锁定： + +```text +codexVersion = "26.707.41301" +``` + +桌面包用于确认发布版本；协议行为以对应日期的官方 Rust 源码为主，因为桌面 JavaScript 包并不拥有 Responses 传输状态机。 + +### 3.2 官方源码 + +官方源码基线： + +```text +5c19155cbd93bfa099016e7487259f61669823ff +2026-07-11T04:15:42Z +Add ordinals to paginated rollout records (#32332) +``` + +主要证据： + +| 主题 | 官方文件 | +| --- | --- | +| WebSocket 会话与增量请求 | `codex-rs/core/src/client.rs` | +| WebSocket 协议、握手和包裹错误 | `codex-rs/codex-api/src/endpoint/responses_websocket.rs` | +| HTTP/SSE 响应解析 | `codex-rs/codex-api/src/sse/responses.rs` | +| 请求元数据 | `codex-rs/core/src/responses_metadata.rs` | +| 限流解析 | `codex-rs/codex-api/src/rate_limits.rs` | +| 请求与流默认阈值 | `codex-rs/model-provider-info/src/lib.rs` | +| WebSocket 行为测试 | `codex-rs/core/tests/suite/client_websockets.rs` | + +固定版本链接： + +- [WebSocket 会话状态](https://github.com/openai/codex/blob/5c19155cbd93bfa099016e7487259f61669823ff/codex-rs/core/src/client.rs#L289-L354) +- [增量输入判定与请求构造](https://github.com/openai/codex/blob/5c19155cbd93bfa099016e7487259f61669823ff/codex-rs/core/src/client.rs#L1167-L1254) +- [新连接清除增量状态](https://github.com/openai/codex/blob/5c19155cbd93bfa099016e7487259f61669823ff/codex-rs/core/src/client.rs#L1310-L1345) +- [WebSocket 响应与错误映射](https://github.com/openai/codex/blob/5c19155cbd93bfa099016e7487259f61669823ff/codex-rs/codex-api/src/endpoint/responses_websocket.rs) +- [限流解析](https://github.com/openai/codex/blob/5c19155cbd93bfa099016e7487259f61669823ff/codex-rs/codex-api/src/rate_limits.rs) + +## 4. 当前工程链路 + +### 4.1 入站请求 + +`backend/src/api/client/responses.rs`： + +- 接收原始 JSON object。 +- 剥离本地传输字段 `use_websocket`。 +- 提取 Codex 上下文字段供请求头使用，同时保留原始 body。 +- 必要时向 `client_metadata` 写入受控的 `x-openai-subagent`。 + +`backend/src/upstream/openai/protocol/responses.rs` 以 `Map` 作为上游 body 的唯一真相源，并启用 serde_json `preserve_order`。这是正确方向：未知字段可透传，协议升级不需要先扩充本地 DTO。 + +但 `CodexResponsesRequest::from_body()` 仍会为缺失字段注入 `input=[]`、`stream=true`、`store=false`。即使这些值通常符合 Codex 客户端行为，也不应写回 wire body；代理可以在本地读取协议默认值，但必须保留字段“缺失”的原始状态。 + +### 4.2 调度与会话恢复 + +`backend/src/dispatch/affinity/resolve.rs` 和 `backend/src/dispatch/recovery/implicit_resume.rs`： + +- 根据 `prompt_cache_key`、窗口和账号建立本地 conversation identity。 +- 代理生成隐式续接时保存完整输入快照。 +- 只有存在 `ImplicitResumeSnapshot` 时才会清除旧 history 并完整重放。 +- 亲和账号被禁用或封禁时执行 cascading ban defense，但没有快照就无法安全清除 history。 + +旧文档所述“客户端显式 previous ID 会被无条件剥离”已经不符合当前代码。现在显式 ID 没有快照时不会被伪恢复，这是已经修正的部分。 + +但隐式续接本身不符合当前确认的透明边界：客户端发送完整 input 且没有 previous ID 时，代理不应自行生成 previous ID、裁剪 input、插入 reasoning replay，再在失败后尝试恢复原请求。应删除这条优化链路，完整请求直接完整转发。 + +### 4.3 账号池与保护链路 + +当前 dispatch 已经会对以下账号级结果更新状态、排除当前账号并继续调度： + +- 429 和 quota exhausted。 +- token 失效、账号禁用或封禁。 +- Cloudflare challenge 和 path block。 +- 部分 model unsupported。 + +账号保护也已有正确基础：cookie 以账号 ID 隔离存储，`set-cookie` 被代理截留；quota cooldown、Cloudflare cooldown、账号状态和 WebSocket pool 都按账号管理；选中账号的凭据和 account ID 由 transport 重新生成，而不是透传客户端 Authorization。 + +但当前循环不是完整的候选遍历： + +- `candidates::filter()` 会按当前最高可用订阅层级收窄，并把瞬时达到并发上限的账号排除；它适合“选下一个”，不能直接作为“本请求全部候选”的快照。 +- `MAX_QUOTA_VERIFY_ATTEMPTS=5` 会在第 5 个 stale-quota 账号后直接返回。 +- `model_unsupported_retry_used` 是单个布尔值，第二个账号仍不支持时直接返回。 +- retryable 5xx 只在同一账号立即重试 2 次，之后走 generic error 并返回。 +- 其他可恢复 transport error 会直接返回，不会排除当前账号。 +- empty response 最多重试 2 次，但没有把产生空响应的账号加入排除集，可能重复选择同一账号。 +- history error 没有临时 implicit snapshot 时直接返回，也无法在其他账号上做完整重放。 +- stream/complete 循环持有同一个可变 `request`，history 清理和恢复 patch 会原地累积；后续账号没有从不可变基线重新派生，存在 attempt 间状态穿透。 +- Cloudflare challenge/path-block tracker 保持账号维度；这是有意设计，跨账号风险信号不得改写候选顺序或提前中断本请求。 + +目标实现必须先由 pool/scheduler 生成稳定的候选 ID 快照，再由 dispatch 维护逐账号 attempt ledger。快照包含所有基础可用层级，tier/affinity/策略只负责排序；账号凭据和运行状态在真正租用时重新读取，避免长请求持有过期 token。ledger 中每个候选只能进入一次账号级尝试，单账号内部允许独立的有界 transport retry；瞬时 Busy 候选先延后再回访。最终错误必须能证明候选总数、已尝试数和被状态排除数相等，而不是根据某个固定计数猜测“应该没有账号了”。 + +每个 attempt 都从不可变的客户端原始请求或完整 replay snapshot 构造新请求，并重新绑定所选账号的 Authorization、account ID、cookie、Cloudflare 上下文、WebSocket 和冷却状态。前一 attempt 的可变 request、turn state、连接 metadata 或加密 reasoning 不能被下一账号继承。 + +### 4.4 上游传输 + +`backend/src/upstream/openai/transport/client.rs` 与 `client_sse.rs`： + +- HTTP/SSE 发送原始 body，并按传输要求设置 `stream`。 +- WebSocket 在 body 外增加 `type: response.create`。 +- 有 `previous_response_id` 时强制使用 WebSocket，不允许 HTTP fallback。 +- 普通 `use_websocket=true` 请求在特定失败后允许回退 HTTP/SSE。 + +`response_upstream_request()` 当前会把显式 wire `prompt_cache_key` 改成账号作用域 conversation identity，并把 session/thread/window 等 reserved identity 写入 `client_metadata`。账号作用域方向符合账号保护，但当前实现没有独立安全策略：缺失 key 会先被生成，多个语义不同的 identity 被合并，同一个 installation ID 跨全部账号复用，账号作用域哈希也没有服务端密钥。应保留保护目标，重写实现边界。 + +`backend/src/upstream/openai/transport/websocket_pool.rs` 按 base URL、账号和本地 conversation identity 管理连接，但 `PooledWebSocketConnection` 当前只有： + +```text +websocket +metadata +created_at +``` + +它没有保存这条连接实际缓存的最后一个 response ID。 + +### 4.5 响应与记录 + +- `protocol/events.rs`：usage 与固定限流族。 +- `protocol/responses.rs`：SSE 收集、完成响应重构、续接元数据。 +- `transport/response_meta.rs`：turn state、cookie、限流头和首输出时间。 +- `dispatch/affinity/resolve.rs`：response ID affinity 与 reasoning replay。 +- `dispatch/recording.rs`、`telemetry/recorder.rs`：使用和错误记录。 + +API 响应层目前主要重建 content type 与 JSON/SSE body，没有把安全的上游 response headers 带回客户端。`set-cookie` 必须被账号层截留，但 request ID、effective model、turn state、限流等端到端 header 应建立明确 allowlist。 + +### 4.6 Wire 改写清单 + +| 当前行为 | 分类 | 结论 | +| --- | --- | --- | +| 替换 Authorization、设置上游 account ID | 账号必需 | 保留 | +| 管理上游 cookie、Cloudflare cooldown、账号禁用 | 风控必需 | 保留，cookie 不下发客户端 | +| 每次 attempt 重建账号 envelope | 账号隔离必需 | 保留；禁止复用前一账号状态 | +| 剥离 `use_websocket` | 本地 transport hint | 保留 | +| JSON 重新序列化并保持字段顺序 | transport 必需 | 允许，字段语义必须相同 | +| WebSocket 增加 `type=response.create` | 协议 envelope | 保留 | +| 模型别名映射 | 显式代理能力 | 仅对已公开配置的 alias 允许 | +| 缺省注入 input/stream/store | 业务 body 改写 | 删除；默认值只在本地解释 | +| 缺失时生成 `prompt_cache_key` | 非必要业务 body 改写 | 删除；缺失继续缺失 | +| 显式 identity 原样跨账号 | 账号关联风险 | 禁止；用带密钥、按账号和字段域分离的伪名替换 | +| 合并 session/thread/request identity | 协议语义破坏 | 拆分后分别做账号作用域转换 | +| 全池复用 installation ID | 账号关联风险 | 改成稳定的 per-account installation pseudonym | +| 覆盖普通 client metadata | 业务 body 改写 | 禁止；只处理 reserved identity key | +| 添加纯代理 telemetry metadata | transport metadata | 只允许保留在代理命名空间且不得覆盖 | +| implicit resume 改写 input/previous ID | 业务语义改写 | 删除 | +| history-bound 请求跨账号发送原 ID | 历史所有权破坏 | 改成基于完整上下文的 full replay | +| 确定性客户端 4xx 广播到全部账号 | 账号保护破坏 | 禁止；精确错误直接透传 | +| 为非流式响应添加 `output_text` | 成功响应改写 | 删除；只返回上游终态对象 | +| 上游 error 改成 `stream_disconnected` | 错误响应改写 | 删除；保留 status/type/code/message/param | +| 完全无上游终态时合成代理失败事件 | transport 必需 | 允许，但必须标识 `source=proxy` | + +## 5. 官方 WebSocket 状态机 + +最新官方实现的核心所有权是： + +```text +ModelClientSession + -> WebsocketSession.connection + -> WebsocketSession.last_request + -> WebsocketSession.last_response_rx + -> LastResponse { response_id, items_added } +``` + +规则如下： + +1. 只有 `response.completed` 才产生 `LastResponse`。 +2. 下一次请求必须与上一次请求的非输入属性完全匹配。 +3. 当前 input 必须以前次 input 加服务端完成输出为前缀，才能只发送增量 input。 +4. 满足条件时才写 `previous_response_id`。 +5. 连接关闭或创建新连接前，立即清除 `last_request`、`last_response_rx` 和 warmup 状态。 +6. 错误后的下一次请求使用新连接和完整 `response.create`，不携带旧 ID。 + +非输入属性比较包含： + +```text +model +instructions +tools +tool_choice +parallel_tool_calls +reasoning +store +stream +include +service_tier +prompt_cache_key +text +``` + +官方有意忽略 `client_metadata` 和 `stream_options`，因为它们不改变被 previous response 引用的上下文。 + +对应测试已覆盖： + +- 完成后前缀匹配，使用 `previous_response_id`。 +- 非输入字段变化，发送完整请求且不带 previous ID。 +- 前次流错误，下一次新连接发送完整请求且不带 previous ID。 + +## 6. P0：连接本地 previous response + +### 6.1 当前缺陷 + +本地 affinity 和 pool key 只能证明“这次请求属于同一会话和账号”，不能证明“当前 socket 仍持有这个 response ID”。以下路径都可能失去上游连接本地缓存： + +- socket EOF、Close、RST 或 pump 标记关闭。 +- 55 分钟主动过期。 +- stale-reuse 后丢弃旧连接并新建。 +- pool Busy 时旁路新建连接。 +- 进程重启或实例切换。 +- 客户端取消导致 socket 被丢弃。 + +当前新连接仍会原样发送请求 body，因此会出现： + +```text +store=false +previous_response_id= +websocket_pool_kind=new +actual socket=W2 +``` + +服务端正确返回 `previous_response_not_found`。 + +### 6.2 生产证据 + +2026-07-11 通过 `ssh oci` 对 `https://codex-proxy.aivify.cc/` 做了只读复核： + +```text +Nginx Proxy Manager: /data/nginx/proxy_host/26.conf +forward: 127.0.0.1:8082 +container: codex-proxy-rs +image: ghcr.io/zyycn/codex-proxy-rs:1.0.2 +runtime data: /srv/codex-proxy-rs/.runtime +``` + +日志中 `previous_response_not_found` 有 549 行，但每个失败通常同时写一条 WARN 和一条最终 ERROR，不能把 grep 行数当请求数。以 `live response stream finalized` 和 `request_id` 去重，并与 SQLite `ops_error_logs` 交叉确认后的结果是： + +| 指标 | 结果 | +| --- | ---: | +| 独立失败请求 | 211 | +| 失效 previous response ID | 16 | +| 涉及账号 | 13 | +| 2026-07-10 | 116 | +| 2026-07-11（截至 18:59:57 北京时间） | 95 | +| `gpt-5.6-sol` | 211 | +| WebSocket | 211 | +| 每请求上游账号 attempt | 1（211/211） | +| pool new | 206 | +| pool bypass | 5 | +| ops_error_logs | 211 | +| usage_records | 0 | + +失败延迟说明服务端会快速拒绝无效 ID： + +| 指标 | latency_ms | +| --- | ---: | +| min | 310 | +| p50 | 549 | +| p95 | 1826 | +| p99 | 2850 | +| max | 8138 | + +16 个失效 ID 均能在 `usage_records.response_id` 找到原始成功响应，且全部满足： + +```text +origin model = gpt-5.6-sol +origin transport = websocket +origin store = false +failure account = origin account +``` + +原响应连接中 14 个为 reuse、2 个为 new；后续 211 个失败全部在 new 或 bypass 连接发生。因此账号轮换不是这批生产事故的根因，连接本地缓存丢失才是共同条件。但每个失败请求也都只尝试了一个账号，说明 history error 分支没有发挥账号池；修复不能简单把旧 ID 继续发给下一账号，必须先从完整快照重建干净请求。 + +所有 211 个失败 wire request 都包含 previous ID，`store=false`，input item 数为 1 到 8（p50=4，p95=7），符合增量请求形态。现有日志没有记录 previous ID 是客户端显式提供还是代理 implicit resume 生成，因此后续必须记录 provenance，不能靠 input 长度推断。 + +用户给出的 response ID 的完整时间线： + +| 北京时间 | request ID | 事实 | +| --- | --- | --- | +| 22:58:30 | `req_3090a40b-200e-4997-a910-01d522974a59` | reuse socket 成功完成，产生 `resp_0de25...fb6aa5f`；store=false，610 个 input item，首输出 5599ms | +| 22:58:31 | `req_aee4bb05-d67e-454e-9913-10a5db5afd64` | 同账号、同 conversation/pool key 复用 socket；HTTP 200 已交给客户端，但之后没有流终态记录 | +| 23:00:21 | `req_20d5d48a-7f95-4bd8-a521-694af6ac939d` | 同账号改为 new socket，携带旧 ID 和 5 个增量 item | +| 23:00:22 | 同上 | 上游返回 400 `previous_response_not_found` | +| 23:34:52 | 多次客户端重试 | 同一 ID 累计 19 个独立失败请求 | + +这条链路说明：22:58:31 的客户端取消或未完成流使原 socket 未回池，下一次只能创建新 socket；代理只按 account/conversation 找池，无法知道旧 ID 已随 socket 消失。 + +生产还证实了错误透明性问题。上游原始错误是： + +```json +{ + "type": "error", + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_...' not found.", + "param": "previous_response_id" + }, + "status": 400 +} +``` + +客户端最终看到的却是 HTTP 200 流内 `response.failed`，code 变成 `stream_disconnected`，原始 JSON 被嵌入 message；`ops_error_logs.upstream_code` 也只记录 `stream_disconnected`。这是确定的响应改写，不符合透明代理原则。 + +这证明故障条件是“连接本地 ID 被发到新连接”，不能据此增加 `gpt-5.6-sol` 特判。 + +### 6.3 目标不变量 + +1. 代理已知由 `store=false` 响应产生的 previous ID，只能发到明确持有该 ID 的同一条 socket。 +2. 新连接的 continuation state 必须为空。 +3. 只有成功的 `response.completed` 才更新连接的 latest response ID。 +4. `response.incomplete`、`response.failed`、错误、取消和未完成 EOF 都必须使本轮无法成为续接基线。 +5. 正常路径原样转发客户端请求，不主动把完整请求优化成增量请求。 +6. 原账号或原 socket 不可用时必须继续账号轮换；有完整快照时清除旧 ID 和账号绑定状态，在本请求剩余的每个候选账号执行 full replay，直至成功或候选耗尽。 +7. stale-reuse 失败后不得把原增量 payload 或同一 previous ID 原样发给新连接/新账号。 +8. full replay 只能发生在任何真实输出提交给客户端之前，并且每次派生都必须可审计。 +9. `store=true` 的服务端持久化语义要与连接本地 fast path 分开处理，不能共用一个布尔判断。 + +必须显式记录 ID 来源，不能仅根据当前请求的 `store` 值推断前一响应是否持久化： + +```text +KnownConnectionLocal # 代理观察到前一响应 store=false，只能在原 socket 使用 +KnownPersisted # 代理观察到前一响应已持久化，可在新连接尝试 hydration +ExternalUnknown # 客户端外部提供，代理不知道来源 +``` + +`KnownConnectionLocal` 丢失原 socket 后，使用完整快照在原账号的新连接 full replay；若该账号同时发生账号级故障，再依次重放到其余候选账号。`KnownPersisted` 可以先在符合账号所有权的连接尝试 hydration，失败后同样走 full replay 和完整候选遍历。`ExternalUnknown` 没有代理历史时只能原样尝试；若上游返回 not found，必须原样返回，不能删除 ID 后把不完整 input 当新会话发送，更不能把未知 ID 广播给所有账号。 + +账号轮换需要新的完整恢复快照，而不是沿用当前只保存 reasoning/function call 的 replay cache： + +```text +ResponseReplaySnapshot + response_id + owner_account_id + source_store + request_properties + full_input_through_completed_response + owner_turn_state + created_at +``` + +快照应放在 Redis 独立 TTL key 中，与 affinity 同步提交和删除，不写 PostgreSQL，不进入日志。快照只保存恢复所需的业务上下文，不保存 access token、account cookie、上游 account ID、指纹或 Cloudflare 凭据。正常请求仍使用客户端原始 body；只有账号/连接恢复分支读取快照并派生 full replay。跨账号时只剥离明确账号绑定的 `previous_response_id`、turn state 和加密 reasoning 内容，保留 reasoning summary、tool call、tool output 及所有其他业务字段。 + +当前 `ImplicitResumeSnapshot` 只在代理主动做隐式续接时保存本次完整 input，无法覆盖客户端显式增量请求，也无法跨请求生存；它不能承担账号轮换恢复，应由上述完整快照替代。 + +### 6.4 推荐状态 + +连接对象应直接拥有 continuation state： + +```text +PooledWebSocketConnection + websocket + metadata + created_at + continuation + latest_response_id + owner_account_id + source_store + replay_snapshot_key +``` + +同 socket 快路径只校验精确 latest response ID；代理不替客户端判断或改写其他业务字段。需要换 socket 或换账号时，每个候选都通过 replay snapshot 构造独立的派生请求，不修改保留的客户端原始 request，也不复用上一个 attempt 的派生 request。 + +不要把连接代次或 latest response ID 写入 Redis。Redis 无法证明当前进程中的具体 socket 仍存活，反而会制造第二真相源。 + +## 7. P0：预取与超时 + +### 7.1 当前行为 + +`backend/src/upstream/openai/transport/websocket_frames.rs` 的 `prefetch_stream_frames_until_output_or_terminal()` 已用于 fresh、pooled 和旁路流式路径。 + +它会： + +1. 暂存 `response.created`、`response.in_progress` 等结构事件。 +2. 继续等待文本、reasoning、工具参数等真实输出，或等待终态。 +3. 从发送请求开始应用默认 20 秒绝对 `first_token_timeout`。 +4. 20 秒内即使不断收到生命周期事件，只要没有真实输出仍会失败。 + +最新官方 WebSocket 实现不会对每个流做这种全局预取。请求发送成功后返回流，由 5 分钟逐帧 idle timeout 保护活跃流。 + +### 7.2 延迟证据 + +最近三天 5722 条 `gpt-5.6-sol` 成功记录： + +| 指标 | first_token_ms | +| --- | ---: | +| p50 | 2716ms | +| p95 | 6929ms | +| p99 | 11275ms | +| 大于 15 秒 | 21 | +| 大于 20 秒 | 9 | +| 最大值 | 37334ms | + +这些数据不能精确等同于 WebSocket 内部计时点，但足以否定“20 秒内没有真实输出就是坏连接”。 + +### 7.3 正确拆分 + +必须区分三个概念： + +| 概念 | 含义 | 建议 | +| --- | --- | --- | +| initial activity timeout | 请求发出后迟迟没有任何有效上游活动 | 可保留短超时，但收到合法生命周期事件后应重置 | +| stream idle timeout | 活跃流在两帧之间长时间无数据 | 保持 5 分钟，与官方默认一致 | +| failover commit boundary | 哪一帧发给客户端后就不能做账号轮换 full replay | 在首个不可逆业务输出前允许恢复，提交后禁止重放 | + +正常请求和客户端显式 previous ID 不应被全局延迟到真实输出。账号 failover 可以在首个不可逆业务输出前短暂暂存纯 lifecycle 帧，但只等待“足以判断能否提交”的边界，不能等待首个文本 token，更不能受 20 秒绝对首输出阈值约束。 + +默认 `first_token_timeout_ms` 应取消，或重定义为 activity-aware 的诊断阈值；不能继续作为 20 秒绝对熔断。 + +## 8. 请求格式与身份 + +### 8.1 已对齐部分 + +- HTTP body 保留未知字段和客户端显式业务字段。 +- WebSocket body 增加 `type: response.create`。 +- `OpenAI-Beta: responses_websockets=2026-02-06` 与最新官方一致。 +- 支持 `x-codex-turn-state`、turn metadata、beta features、timing metrics、window、parent thread、installation ID 和受控 subagent。 +- WebSocket payload 会写 `x-codex-ws-stream-request-start-ms`。 + +官方 Codex 构造请求时还会设置 `store=false`、`stream=true`、`tool_choice=auto`、`parallel_tool_calls`、`include=[reasoning.encrypted_content]` 等值。它是客户端请求构造器，不是代理改写清单；代理必须转发客户端实际提供的字段，不能照抄这些默认值到缺失字段。 + +### 8.2 身份字段被合并 + +官方元数据明确区分： + +```text +installation_id +session_id +thread_id +turn_id +window_id +``` + +官方请求头映射是： + +```text +x-client-request-id = thread_id +session-id = session_id +thread-id = thread_id +``` + +当前 `CodexRequestContext` 只有 `session_id`，并把它同时写到三个位置。`response_client_metadata()` 也把 `thread_id` 强制覆盖成 `session_id`。 + +目标设计： + +1. `InboundIdentity` 分开保存客户端原始 `session_id`、`thread_id`、`client_request_id`、`turn_id`、window、parent thread 和 prompt cache key，只存在于受控请求上下文/恢复快照。 +2. `AccountScopedIdentity` 按字段 domain 和账号分别派生 wire 值，不能用一个 conversation hash 覆盖所有字段。 +3. 派生使用持久化服务端密钥的 HMAC，而不是可离线枚举的裸 SHA-256；密钥不进日志、不进响应，换账号必然得到不同伪名。 +4. installation ID 仅由持久化身份密钥和账号 ID 稳定派生；不保留全池共用值，也不维护本地基础 installation ID 文件。 +5. 客户端未提供的 body identity 默认保持缺失；仅 transport 必需的 request ID 可以使用代理本轮 request ID，不能回写成客户端 body 字段。 +6. `client_metadata` 的普通字段原样保留；reserved identity key 由安全策略替换为账号作用域值，并记录 `identity_transform=account_scoped`，但不记录原值。 +7. 本地 pool/affinity identity、客户端原始 identity 和上游账号作用域 identity 是三个不同类型，禁止用 `String` 在层间混传。 + +### 8.3 prompt_cache_key 被改写 + +当前 transport 会把 `prompt_cache_key` 改写为账号作用域的本地 conversation identity。这个保护方向正确，但实现仍有三个问题： + +- `ensure_prompt_cache_key()` 会在客户端缺失时主动生成业务字段。 +- 裸 SHA-256 同时承担本地索引和 wire 伪名，没有密钥和类型边界。 +- conversation/session/thread/window 被压成同一个 identity，协议语义与保护语义混在一起。 + +目标设计应保留三个独立字段： + +```text +client_prompt_cache_key # 客户端原值，仅受控本地使用 +local_conversation_identity # pool、affinity、恢复索引 +wire_account_scoped_prompt_cache_key # HMAC(field, account, client value) +``` + +客户端显式提供 key 时，上游只看到当前账号的稳定伪名；客户端未提供时 wire body 继续缺失。连接池和 affinity 使用独立 local identity，不能为了内部索引而给上游 body 生成 key。账号 failover 必须从 `client_prompt_cache_key` 为新账号重新派生，不能复用前一账号的 wire key。 + +HMAC identity secret 在首次启动时生成并以仅运行用户可读的权限持久化到 data directory；启动时无法安全读取/创建就 fail closed。它不能放进日志、PostgreSQL 业务表或客户端可见配置。镜像替换必须继续挂载同一 data directory，否则 wire identity 会改变，当前 Redis continuation 快照应一并失效，不能保留旧伪名兼容分支。 + +### 8.4 请求头差异 + +| 请求信息 | 当前 | 最新官方 | 处理建议 | +| --- | --- | --- | --- | +| Responses WS beta | 已支持 | 已支持 | 保持 | +| session/thread/client request ID | 合并成账号作用域 conversation ID | 分离 | 按字段分别做账号作用域伪名化 | +| installation ID | 持久密钥 + 账号 ID 派生 | 单客户端稳定 | 保持 per-account 稳定伪名，不落地基础 installation ID 文件 | +| `x-openai-subagent` | 白名单支持 | 支持 | 保持白名单 | +| `x-openai-memgen-request` | 缺失 | memory consolidation 使用 | 对受信客户端/内部 route 做 allowlist 透传，不改 body | +| Responses Lite header | 缺失 | 客户端与 Lite body 同步发送 | 受控透传客户端 marker，代理不重建 Lite body | +| WS traceparent/tracestate metadata | 缺失 | 支持 | 客户端 trace 只做本地关联；每个账号 attempt 生成独立合法 trace context | +| `x-oai-attestation` | 缺失 | 官方按请求生成 | 不伪造；仅在有真实 attestation provider 时实现 | +| Zstd 请求压缩 | 缺失 | ChatGPT backend 可启用 | P2 性能项，先做能力探测与指标 | + +## 9. 响应驱动的数据与判断 + +### 9.1 当前覆盖情况 + +| 数据 | 当前处理 | 最新官方用途 | 结论 | +| --- | --- | --- | --- | +| `x-codex-turn-state` | HTTP/WS 捕获并续传 | 同一 turn sticky routing | 已对齐 | +| `set-cookie` | 捕获并进入 Cloudflare/cookie 链路 | 会话恢复 | 已对齐 | +| `x-request-id`、`cf-ray` | 部分捕获 | 诊断与反馈 | 扩充字段 | +| usage | 输入、输出、缓存、reasoning、图片 token | 计量 | 基本对齐 | +| completed response ID | 写 affinity/replay | WS 增量续接 | 所有权仍错误 | +| `response.incomplete` | 被当作完成元数据 | 官方作为错误，不推进续接 | P1 修复 | +| `x-models-etag` | 未捕获 | 模型目录刷新 | P1 增加 | +| `openai-model` | 未捕获 | 服务端实际选择模型 | P1 增加 | +| `x-reasoning-included` | 未捕获 | 响应能力/解析提示 | P1 增加 | +| model verifications | 未解析 | 模型验证状态 | 按客户端契约保留 | +| turn moderation metadata | 未解析 | 审核信息 | 保留并定义展示边界 | +| safety buffering | 未解析 | 安全缓冲与重试模型提示 | 保留并定义行为 | +| `end_turn` | 反序列化后丢弃 | turn 生命周期 | 向内部流事件暴露 | + +### 9.2 `response.incomplete` 不能成为续接基线 + +`completed_response_metadata()` 当前匹配： + +```text +response.completed | response.incomplete +``` + +随后 `record_response_affinity()` 会把该 ID 写入 affinity 和 reasoning replay。这与官方状态机相反。 + +修复要求： + +1. `CompletedResponseMetadata` 只由 `response.completed` 产生。 +2. `response.incomplete` 可按 Responses API 契约透传给客户端，但不能写 continuation、affinity 或 replay。 +3. 流式与非流式路径共用同一终态分类，禁止各自维护不同规则。 + +### 9.3 有效模型 + +服务端可能通过 `openai-model` 返回实际执行模型，例如安全路由后的模型。当前 usage、账号模型统计和日志主要使用请求模型或本地 display model。 + +应新增 `effective_model`： + +- 调度仍按 requested model 选账号。 +- 计量和模型分布优先记录 server effective model。 +- 同时保留 `requested_model`，便于审计路由变化。 +- 不把 effective model 回写请求或影响本轮重试选择。 + +### 9.4 模型 ETag + +当前模型目录依次请求： + +```text +/codex/models?client_version=... +/models +/sentinel/chat-requirements +``` + +最新官方会从 Responses HTTP/WS 响应捕获 `x-models-etag` 并发出模型目录变更事件。建议让模型服务保存 ETag；发生变化时异步刷新 Redis 模型快照，避免每次响应同步请求模型端点。 + +## 10. 限流差异 + +当前 `ParsedRateLimits` 只包含： + +```text +primary +secondary +code_review +``` + +最新官方会： + +1. 枚举所有 `x--primary-*` header family。 +2. 保存 `limit_id` 和 `limit_name`。 +3. 解析 credits 的 `has_credits`、`unlimited`、`balance`。 +4. 从 `codex.rate_limits` 事件读取 `plan_type` 和 metered limit。 +5. 解析 promo message 与 rate-limit-reached-type。 + +改造后应以 `limit_id` 为键保存任意窗口，core 和 code review 只是普通已知 ID，不再写固定分支。数据库/Redis 结构若已能保存 JSON snapshot，应直接扩充领域类型，不新增旧格式兼容路径。 + +## 11. 错误与诊断 + +### 11.1 WebSocket 包裹错误 + +官方会把 WebSocket error event 中的 primitive `headers` map 完整转换为 HTTP HeaderMap，再走统一 TransportError。它还对精确 code `websocket_connection_limit_reached` 做专门重连判断。 + +当前 `ClassifiedWebSocketError` 只有： + +```text +status_code +``` + +随后 transport 只提取 retry-after，并用 `CodexUpstreamDiagnostics::with_status()` 构造诊断。因此会丢失： + +- 精确 `error.code` 和 `error.type`。 +- `x-request-id` / `x-oai-request-id`。 +- `cf-ray`。 +- `x-openai-authorization-error`。 +- base64 `x-error-json` 中的认证错误 code。 +- 包裹事件中的 set-cookie 和完整限流头。 + +目标错误类型至少包含： + +```text +status +code +error_type +message +headers +retry_after +request_id +cf_ray +``` + +`previous_response_not_found`、无 tool output、`invalid_encrypted_content` 和连接 60 分钟上限必须由精确 code 分类，不再扫描 message/body。 + +### 11.2 请求 ID + +`CodexUpstreamDiagnostics` 当前缺少官方使用的 `x-oai-request-id`。解析顺序应为： + +```text +x-request-id +x-oai-request-id +x-openai-request-id +openai-request-id +request-id +``` + +认证错误 header 只进入受控诊断字段，不将原始敏感 body 写入普通日志。 + +### 11.3 错误分类与账号池动作 + +中间账号的错误是 dispatch 决策输入，不是立即返回给客户端的结果。目标分类如下： + +| 错误来源 | 示例 | 账号池动作 | 最终客户端语义 | +| --- | --- | --- | --- | +| 确定性请求错误 | schema/param 非法、上下文超限、缺 tool output | 不广播到其他账号 | 原样保留上游 status/type/code/message/param/body | +| 外部未知 history | 无快照的 `previous_response_not_found` | 不把未知 ID 广播到其他账号 | 原样返回上游 400 | +| 可恢复 managed history | connection-local ID 丢失、invalid encrypted content | 从完整快照派生 full replay；账号级失败时继续剩余候选 | 成功响应，或候选耗尽后的统一错误 | +| 节点级认证 | 精确 token invalid/expired/revoked | 更新账号状态并排除，继续全部剩余候选 | 候选耗尽后返回 `no_available_accounts` | +| 高风险账号状态 | account deactivated/banned | 隔离当前账号与连接，重建干净 envelope 后继续当前设置所选调度策略 | 候选耗尽后返回 `no_available_accounts` | +| 账号额度/单节点风控 | 429、quota、Cloudflare challenge/path block | 写冷却/状态并排除，使用干净 envelope 继续遍历当前设置所选调度策略排序后的剩余候选 | 候选耗尽后返回 `no_available_accounts` | +| 账号模型能力 | 精确 model unsupported | 记录该账号能力并排除，继续全部剩余候选 | 全部候选均不支持时返回稳定 `model_not_supported` | +| 可恢复 5xx/transport | connect、reset、502/503/504 | 先做单账号有界重试；仍失败则仅在本请求排除并继续全部剩余候选，不持久禁用账号 | 候选耗尽后返回稳定 upstream unavailable | +| 多账号同类风控 | 多个账号在干净重放后出现 challenge/path-block/new-ban | 各自隔离，不传播账号绑定状态，继续当前设置所选调度策略 | 候选耗尽后返回脱敏汇总错误 | +| 明确共享上游故障 | global overload/maintenance 精确 code | 仅记录当前请求证据，不打开 endpoint circuit，不修改账号状态 | 候选耗尽后保留 upstream unavailable 语义 | +| 输出已提交后的流错误 | 任意晚期 EOF/error | 禁止盲目 full replay | 保留原上游失败；仅无等价上游事件时合成 proxy failure | + +账号池终态只能由 attempt ledger 证明候选已耗尽后产生。全为 model unsupported 时返回 `model_not_supported`，全为可恢复 5xx/transport 时返回 upstream unavailable，node-local 认证、额度或混合账号状态耗尽时返回 `no_available_accounts`。错误应携带脱敏失败类别汇总和最后一个上游 request ID 供诊断，但不能冒充某个中间账号的原始响应。客户端主动取消、确定性请求错误、输出已提交和进程关闭是独立终止条件，不应伪装成账号池耗尽。 + +## 12. 阈值对照 + +| 项目 | 当前工程 | 最新官方默认 | 结论 | +| --- | ---: | ---: | --- | +| WebSocket 最大寿命 | 55 分钟 | 服务端 60 分钟精确错误 | 55 分钟预留合理，保留 | +| WS 建连超时 | 依赖底层连接行为 | 15 秒 | 增加显式配置与指标 | +| 首次无活动等待 | 20 秒 | 无独立短阈值 | 改成 activity-aware | +| 首真实输出绝对超时 | 20 秒 | 无 | 默认取消 | +| 活跃流逐帧 idle | 5 分钟 | 5 分钟 | 对齐 | +| fresh WS 首输出重试 | 2 次 | stream retries 5 | 不直接照抄；先修复错误分类和提交边界 | +| 同账号 5xx 重试 | 2 次，无退避，结束后立即返回 | request retries 4，200ms 指数退避和抖动 | 加退避；单账号预算用尽后继续下一候选 | +| quota verify 账号数 | 固定最多 5 | 不适用 | 删除固定上限，遍历本请求完整候选集 | +| model unsupported 账号数 | 最多 2 | 不适用 | 删除布尔闸门，遍历本请求完整候选集 | +| 空响应重试 | 请求级固定 2 次 | 不适用 | 当前账号记一次并排除，继续完整候选；删除固定请求级上限 | +| 单账号 pool slot | 8 | session 单连接模型 | 代理特有，基于 Busy/饱和数据调整 | +| ping / ping timeout | 25 秒 / 5 秒 | 库和服务端管理 | 可保留，记录误杀率 | + +官方 `request_max_retries=4` 和 `stream_max_retries=5` 是 Codex 客户端策略，不是代理应无条件复制的数值。代理还会轮换账号，必须把单账号网络预算与账号候选预算分开，避免次数相乘，也不能让网络重试上限截断候选遍历。 + +推荐重试规则： + +1. 请求开始时冻结候选账号快照，并按亲和/调度策略排序；后续只从该快照取未尝试账号。 +2. 同账号 5xx 使用 200ms 起步的有界指数退避和 0.9 到 1.1 抖动；预算用尽后排除当前账号并继续下一候选。 +3. 对 429 使用服务端 retry-after 和账号冷却，不做同账号快速重试，立即继续下一候选。 +4. quota verify、model unsupported、empty response、token invalid/expired、Cloudflare 和 new-ban 都计入当前账号的 attempt，不能使用小于候选数的全局固定上限。 +5. managed previous history 错误在每个候选上只允许一次有完整快照证明的 full replay；不能复用旧 ID，也不能重复使用上一个账号的派生 request。 +6. 确定性客户端 4xx 和外部未知 history 不做账号 fanout，避免用无效请求伤害整个账号池。 +7. 一旦真实输出已经提交给客户端，不做可能重复输出或副作用的 full replay。 +8. 客户端取消、确定性请求错误、输出已提交、进程关闭或候选 ledger 已耗尽可以结束 dispatch；普通 node-local 重试计数不能提前返回客户端错误。 + +## 13. 其他上游差异 + +### 13.1 Responses Lite + +最新官方按模型目录的 `use_responses_lite` 同时改变 body 与 header： + +- tools 和 developer instructions 移入 input。 +- 顶层 instructions 置空。 +- 顶层 tools 清空。 +- reasoning context 使用 all-turns 语义。 +- 增加 `x-openai-internal-codex-responses-lite: true`。 + +这不是代理应该执行的单个 header 功能。客户端若已经按 Responses Lite 契约构造 body，代理只对受信客户端透传 marker 和原始 body；代理不能根据模型目录自行搬移 instructions/tools 或改变 reasoning context，否则会同时破坏透明性和账号 failover 的等价重放。 + +### 13.2 memory consolidation + +最新官方在 memory consolidation 请求上同时发送受控 subagent 和 `x-openai-memgen-request: true`。当前白名单已经允许 `memory_consolidation`，但缺少 companion header。只有受信客户端明确提交该请求类型时才成对透传；普通客户端提供的同名内部 header 必须剥离，代理不改写 body 来伪造 memory consolidation。 + +### 13.3 trace context + +最新官方把每次 WS 请求的 traceparent/tracestate 写进 `client_metadata`，因为握手头不能表达同一连接上的每一轮 trace。代理收到的客户端 trace 只用于本地链路关联，不能把同一 trace ID 透传到多个上游账号。每个账号 attempt 应生成独立的合法 W3C trace context，并在本地用 request ID 关联；tracestate 中不可信 vendor 数据剥离，日志和 audit artifact 继续脱敏。 + +### 13.4 attestation + +`x-oai-attestation` 是官方按请求生成的证明，不是固定指纹头。代理没有真实 attestation provider 时应保持缺失，不得伪造静态值或盲目转发不可信客户端值。 + +### 13.5 请求压缩 + +官方可对 ChatGPT Responses 请求启用 Zstd。该项只影响带宽和 CPU，不影响 P0 正确性。实施前应确认服务端 capability、正文阈值、压缩失败回退和端到端延迟收益。 + +## 14. 实施顺序 + +### 阶段一：P0 账号池耗尽、账号隔离与 continuation 所有权 + +修改： + +- `backend/src/fleet/pool/mod.rs` +- `backend/src/fleet/scheduler/candidates.rs` +- `backend/src/dispatch/service.rs` +- `backend/src/dispatch/stream/lifecycle.rs` +- `backend/src/dispatch/upstream_call.rs` +- `backend/src/dispatch/recovery/exhaustion.rs` +- `backend/src/dispatch/recovery/cloudflare.rs` +- `backend/src/upstream/openai/transport/websocket_pool.rs` +- `backend/src/upstream/openai/transport/websocket.rs` +- `backend/src/upstream/openai/transport/websocket_frames.rs` +- `backend/src/upstream/openai/protocol/websocket.rs` +- `backend/src/dispatch/affinity/types.rs` +- `backend/src/dispatch/affinity/store.rs` +- `backend/src/dispatch/recovery/implicit_resume.rs` +- `backend/src/dispatch/errors.rs` + +完成项： + +1. pool/scheduler 一次返回跨全部可用层级的有序候选 ID 快照；tier/affinity 只排序，瞬时 Busy 延后回访，dispatch 使用 attempt ledger 证明全部候选已处理。 +2. 删除 `MAX_QUOTA_VERIFY_ATTEMPTS` 和 model unsupported 单次换号闸门；账号级错误、5xx/transport retry exhausted 和 empty response 都继续剩余候选。 +3. 风控只修改当前账号的状态并清理其连接绑定信息；不引入 request risk guard、endpoint circuit 或换号次数/时间预算。 +4. 每个 attempt 从不可变原始请求/完整快照重新构造，并绑定该账号独立的凭据、cookie、风控状态和连接。 +5. 连接对象拥有 continuation state，发送前验证 previous ID、owner account、socket 和请求形态；新连接不携带 connection-local ID。 +6. Redis 保存唯一的完整 replay snapshot；删除主动 implicit resume 优化及其临时快照，不保留两套恢复模型。 +7. history failover 只从完整快照派生，跨账号剥离旧 ID、turn state 和账号绑定加密内容，业务上下文保持等价。 +8. 删除全局 output-prefetch，拆分 activity timeout 与 commit boundary；提交前可 failover，提交后禁止盲目重放。 + +### 阶段二：终态和结构化错误 + +修改： + +- `backend/src/upstream/openai/protocol/responses.rs` +- `backend/src/upstream/openai/protocol/websocket.rs` +- `backend/src/upstream/openai/transport/diagnostics.rs` +- `backend/src/dispatch/affinity/resolve.rs` +- `backend/src/api/client/errors.rs` + +完成项： + +1. completed 与 incomplete 使用判别明确的终态类型。 +2. exact error code 和 primitive headers 全链路保留。 +3. 中间账号错误只进入 attempt ledger；候选耗尽才生成稳定的 `no_available_accounts`/upstream unavailable。 +4. 可由客户端修复的上游 4xx 原样保留，外部未知 history 不做账号 fanout。 +5. 请求 ID、认证错误和 Cloudflare trace 统一提取。 + +### 阶段三：身份、响应元数据与限流 + +修改： + +- `backend/src/upstream/openai/transport/client.rs` +- `backend/src/upstream/openai/transport/client_sse.rs` +- `backend/src/upstream/openai/transport/response_meta.rs` +- `backend/src/upstream/openai/protocol/events.rs` +- `backend/src/models/` +- `backend/src/dispatch/recording.rs` +- `backend/src/telemetry/` + +完成项： + +1. session/thread/turn/client request ID 分离。 +2. local conversation identity、客户端原始 identity 与 wire account-scoped identity 使用不同强类型。 +3. 用持久 HMAC 密钥分别派生 prompt/session/thread/window/installation 伪名，删除裸哈希与全池 installation ID。 +4. 客户端普通 body 保持不可变，账号 envelope、reserved identity transform、代理 trace 与普通客户端 metadata 分层构造。 +5. 捕获 ETag、effective model、reasoning included、end_turn 和安全元数据。 +6. 任意 limit ID、credits 和 plan type 进入统一限流领域模型，并反馈账号保护状态。 + +### 阶段四：P2 协议能力 + +按独立评审实施 Responses Lite、memgen、trace context、attestation provider 和请求压缩。不得为未启用能力保留半成品兼容代码。 + +## 15. 测试矩阵 + +所有测试放在 `backend/tests`，禁止写入 `backend/src`。 + +### 15.1 continuation + +| 场景 | 预期 | +| --- | --- | +| 同 socket，完成响应后前缀扩展 | 发送 delta 和匹配的 previous ID | +| 请求非输入字段变化 | 同 socket 发送完整 input，不带 previous ID | +| 完成后主动丢弃 socket | 新连接不带 connection-local ID | +| stale-reuse 写失败 | 不把原增量 payload 发到新连接 | +| pool Busy | 旁路连接不携带池连接的 previous ID | +| `response.incomplete` | 不更新 latest ID、affinity 或 replay | +| completed 后客户端取消下一轮 | 被取消轮不推进 continuation | +| managed connection-local ID、连接缺失 | 从 Redis 完整快照重放，不带旧 ID/turn state/账号绑定加密内容 | +| managed ID 的快照意外缺失 | 视为内部不变量破坏，不把残缺请求发往任何账号 | +| 外部来源未知的显式 previous ID | 新连接尝试一次；not found 时原样返回，不广播未知 ID | +| 原账号新连接 full replay 仍发生账号级失败 | 排除原账号，并从同一业务快照依次派生到全部剩余候选 | + +### 15.2 账号池与账号保护 + +| 场景 | 预期 | +| --- | --- | +| 高优先级层级账号全部失败 | 继续尝试快照中的低优先级可用层级，不因 tier 收窄提前结束 | +| 某候选瞬时 slot/connection Busy | 先尝试其他账号，再回访该候选，不设代理自定义换号 deadline | +| 候选快照有 N 个账号，第一个 429 | 写该账号 cooldown，随后尝试其余 N-1 个 | +| 连续 quota verify limit reached 超过 5 个 | 不受固定 5 次限制，继续到候选耗尽或成功 | +| 连续 model unsupported 超过 2 个 | 不受布尔闸门限制，逐账号记录能力并继续 | +| 某账号 retryable 5xx | 完成有退避的单账号重试；仍失败则排除并尝试下一账号 | +| 某账号 transport reset/connect error | 有界同账号重试后继续下一账号，不立即返回 | +| 某账号返回空响应 | 记录失败并排除，继续下一账号，不重复选择同一账号 | +| token invalid/expired | 更新账号状态、清理/隔离连接并继续下一账号 | +| account deactivated/banned | 隔离该账号与其连接状态，重建干净 envelope 后继续遍历当前设置所选调度策略排序后的剩余候选 | +| Cloudflare challenge/path block | 捕获该账号 cookie 与冷却状态，排除并继续下一账号 | +| 第一个账号高风险、第二个账号在干净 envelope 下成功 | 正常返回成功，两个 attempt 间无 cookie/session/identity 穿透 | +| 多个账号在干净重放后出现同类高风险信号 | 逐个隔离已证实失败的账号，继续遍历当前设置所选调度策略排序后的剩余候选，不触发请求级熔断 | +| 同一账号内部重复同类高风险帧 | 只更新该账号状态，不影响其他候选的排序或健康状态 | +| 多账号连续 quota/model unsupported | 继续完整候选遍历 | +| 混合账号级错误，最后一个成功 | 客户端只看到成功结果，中间错误只进入脱敏 attempt trace | +| 全部候选发生 node-local 失败 | `attempted + state_excluded = candidate_count` 后才返回稳定池耗尽错误 | +| 确定性 invalid request 4xx | 只请求一个账号，原样返回错误，避免污染账号池 | +| 每次跨账号 attempt | Authorization/account ID/cookie/socket/turn state 均来自或匹配当前账号 | +| 跨账号 full replay | 业务 input/tools/tool outputs 保持等价，旧账号绑定字段全部不存在 | +| 客户端取消 | 终止请求但不伪报账号池耗尽，不错误修改未尝试账号状态 | + +### 15.3 流与超时 + +| 场景 | 预期 | +| --- | --- | +| 生命周期事件持续到 25 秒才有输出 | 不触发 20 秒绝对首输出失败 | +| 发送后完全无活动 | activity timeout 生效 | +| 首输出后 5 分钟无帧 | stream idle timeout 生效 | +| 结构事件后 history error 且有快照 | 客户端未见不可逆输出时按候选顺序完整重放 | +| 真实输出后上游错误 | 禁止 full replay，保留原上游错误语义 | +| `websocket_connection_limit_reached` | 精确 code 触发新连接，清空增量状态 | + +### 15.4 请求与响应元数据 + +- distinct session/thread/client request ID 不互相覆盖，并分别派生账号作用域值。 +- 显式 prompt cache key 的原值不出现在上游 wire；同账号伪名稳定，换账号伪名不同，缺失时不生成。 +- 同一个代理 identity secret 为不同账号派生不同且各自稳定的 installation ID。 +- `x-models-etag` 变化触发一次异步模型刷新。 +- `openai-model` 写入 effective model，同时保留 requested model。 +- WS error headers 中 string、number 和 bool 均能安全映射。 +- `x-oai-request-id`、auth error 和 `x-error-json` 可解析且日志不泄露原 body。 +- 任意 limit ID 和 credits 在 HTTP header 与 WS event 两种路径一致。 + +## 16. 可观测性 + +每次 WebSocket 请求至少记录： + +```text +candidate_count +attempted_account_count +state_excluded_account_count +pool_exhausted +attempt_index +attempt_account_id_hash +attempt_failure_class +account_state_transition +account_envelope_rebuilt +identity_transform +pool_decision +connection_id_hash +connection_age_ms +continuation_requirement +continuation_match +previous_response_id_present +previous_response_provenance +request_shape_match +replay_snapshot_available +derived_request_reason +recovery_action +commit_boundary +first_activity_ms +first_output_ms +upstream_request_id +upstream_error_code +effective_model +``` + +禁止记录 access token、cookie、完整 previous ID、prompt cache key、完整 input/tools/client metadata 或 attestation。 + +内部重连、full replay、模型刷新和 quota verify 不得产生额外 `usage_records`；usage 只代表真实 `/v1/...` 客户端调用的最终结果。 + +## 17. 验收标准 + +只有全部满足才算改造完成： + +- 代理已知源自 `store=false` 响应的旧 ID 不会出现在新 socket 的上游 payload。 +- continuation state 由具体连接拥有，连接销毁时同步销毁。 +- 只有 `response.completed` 推进续接状态。 +- 代理管理的每条 continuation 都有 Redis 完整 replay snapshot，快照不包含任何账号凭据或 cookie。 +- 显式增量请求没有完整历史时不会伪恢复，也不会把未知 ID 广播到账号池。 +- 普通业务 body 原样转发；reserved identity 做可审计的账号作用域转换，只有账号 failover 分支派生 full replay。 +- 每次 attempt 都重建账号 envelope，前一账号的 Authorization、account ID、cookie、turn state、socket 和加密 reasoning 不会进入下一账号。 +- prompt/session/thread/window/installation 等关联标识按账号隔离，普通 metadata 不被风控转换误伤。 +- node-local token 失效、额度、模型能力、可恢复 5xx/transport、空响应和高风险信号会隔离当前账号并继续遍历当前设置所选调度策略排序后的剩余候选；只有确定性请求错误、外部未知 history 无完整快照或输出已提交可以停止跨账号重放。 +- 不存在 request risk guard、endpoint circuit、候选次数上限或换号时间预算；风险剖离只保护已失败账号，不改写当前设置所选策略的候选顺序。 +- quota verify、model unsupported 或普通重试计数不会在候选耗尽前结束 dispatch。 +- 最终池耗尽错误有 `candidate_count = attempted + state_excluded` 的 ledger 证据。 +- 确定性客户端 4xx 原样返回且不会污染其余账号。 +- 全局 output-prefetch 与 20 秒绝对首输出熔断已移除。 +- exact upstream error code 和 headers 全链路保留。 +- session/thread/request identity 不再合并。 +- effective model、models ETag 和任意限流族有明确消费方。 +- 已提交真实输出后不重试。 +- 所有新增测试位于 `backend/tests`。 +- `cargo fmt --manifest-path backend/Cargo.toml -- --check` 通过。 +- `cargo clippy --manifest-path backend/Cargo.toml --all-targets --all-features -- -D warnings` 通过。 +- `cargo test --manifest-path backend/Cargo.toml` 通过。 + +## 18. 真实链路验证 + +先在 next 实例开启脱敏后的 WebSocket audit： + +```bash +CODEX_PROXY_WS_AUDIT_DIR=.runtime/ws-audit +``` + +至少验证： + +1. 同 socket 连续两轮正常增量。 +2. 完成后主动 discard，再发送代理已知 connection-local previous ID。 +3. 在新连接发送外部来源未知、可能已持久化的 previous ID。 +4. 完成后主动 discard，使用 Redis 完整快照在原账号新连接重放。 +5. 原账号分别制造 429、quota、auth、首次 Cloudflare、model unsupported 和 retryable 5xx，验证每类单节点错误都会进入下一候选。 +6. 让前 N-1 个候选发生 node-local 失败、最后一个成功，确认客户端只看到成功且 attempt ledger 完整。 +7. 让全部候选发生 node-local 失败，确认最后一个候选结束前不会返回客户端错误。 +8. 让两个独立账号在干净 envelope 下出现同类 challenge/path-block，确认两者各自隔离且第三个账号仍由当前设置所选调度策略正常选中。 +9. 跨账号 full replay 抓取脱敏 wire，确认业务上下文等价且无旧账号 ID、turn state、cookie、account ID 或加密 reasoning。 +10. 对同一客户端 identity 分别选两个账号，确认 prompt/session/thread/window/installation wire 伪名均不同且普通 metadata 相同。 +11. pool Busy 与 stale-reuse。 +12. 生命周期活动超过 20 秒后才出现首输出。 +13. 精确 `websocket_connection_limit_reached`。 +14. HTTP/SSE 与 WS 的 effective model、ETag、限流和 request ID 一致性。 + +每个场景记录下游 request ID、上游 request ID、pool decision、是否实际发送上游、恢复动作、首活动和首输出时间。确认问题后先修复并重跑当前场景，再进入下一场景。 + +## 19. 参考工程取舍 + +`sub2api` 的可取原则是：只有保存了完整 replay input 才允许清除 previous ID 后恢复；本工程还必须在此基础上遍历全部账号候选，而不是同账号恢复失败后立即结束。 + +`CLIProxyAPI` 的可取原则是：连接失效时增量状态同时失效，下一次使用完整 transcript。 + +本工程最终只保留一套 Redis `ResponseReplaySnapshot`。现有进程内 `ImplicitResumeSnapshot` 和不完整 reasoning replay cache 不能作为并存兼容路径，应在迁移完成后删除。也不应复制依赖消息文本的启发式分类；连接状态、错误 code、账号失败类别和 attempt ledger 必须使用强类型表达。 + +## 20. 最终建议 + +先完成以下最小正确闭环： + +```text +immutable client request + -> ordered candidate snapshot + -> account-local risk stripping + -> per-account isolated envelope + -> per-account identity pseudonyms + -> bounded same-account transport retry + -> exclude failed account and continue + -> full replay snapshot for history failover + -> success or proven candidate exhaustion + +connection-owned continuation state + + send-before-match validation + + completed-only continuation update + + new socket/new account clears bound state + + exact upstream error classification + + remove global output-prefetch +``` + +## 21. 实施状态 + +P0/P1 已按上述边界实施：完整候选 ledger 使用运行时设置当前选择的策略顺序（Smart / QuotaResetPriority / RoundRobin / Sticky），没有换号次数/时间预算或风险熔断；managed history 换号只从 Redis 完整快照重放，external unknown history 不广播；WebSocket continuation 由连接拥有，只有 `response.completed` 推进状态。 + +请求 body 不再注入代理默认值，identity 和 installation ID 使用持久 HMAC 密钥按字段域/账号隔离；`response.incomplete` 不写续接基线；HTTP/SSE/WS 捕获并消费 effective model、models ETag、reasoning included 和任意 limit ID/credits/plan/promo/reached type。P2 协议能力仍保持独立评审边界。