mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
fix: harden response recovery and account failover
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Generated
+12
-2
@@ -363,6 +363,7 @@ dependencies = [
|
||||
"flate2",
|
||||
"futures",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"indexmap",
|
||||
"ipnet",
|
||||
"percent-encoding",
|
||||
@@ -1061,7 +1062,16 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018"
|
||||
dependencies = [
|
||||
"hmac",
|
||||
"hmac 0.13.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2394,7 +2404,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hkdf",
|
||||
"hmac",
|
||||
"hmac 0.13.0",
|
||||
"itoa",
|
||||
"log",
|
||||
"md-5",
|
||||
|
||||
@@ -17,6 +17,7 @@ dirs = "6.0.0"
|
||||
flate2 = "1.1.9"
|
||||
futures = "0.3.32"
|
||||
hex = "0.4.3"
|
||||
hmac = "0.12.1"
|
||||
indexmap = { version = "2.0", features = ["serde"] }
|
||||
ipnet = "2.12.0"
|
||||
percent-encoding = "2.3.2"
|
||||
|
||||
@@ -10,6 +10,7 @@ use serde_json::{json, Value};
|
||||
use crate::{
|
||||
dispatch::errors::{DispatchFailureClass, ResponseDispatchError},
|
||||
upstream::openai::protocol::sse::response_failed_sse_event as encode_response_failed_sse_event,
|
||||
upstream::openai::transport::CodexClientError,
|
||||
};
|
||||
|
||||
const NO_ACTIVE_UPSTREAM_ACCOUNT_MESSAGE: &str = "No active upstream account is available";
|
||||
@@ -19,6 +20,8 @@ const UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE: &str = "Upstream Codex request fail
|
||||
const INVALID_UPSTREAM_CODEX_RESPONSE_MESSAGE: &str = "Invalid upstream Codex response";
|
||||
const UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE: &str = "Upstream Codex response failed";
|
||||
const HISTORY_UNAVAILABLE_MESSAGE: &str = "Previous response context is unavailable. Start a new conversation or resend the complete input without previous_response_id.";
|
||||
const CONTINUATION_BUSY_MESSAGE: &str =
|
||||
"The previous response connection is busy. Retry the request.";
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ResponseDispatchMessageStyle {
|
||||
@@ -72,6 +75,10 @@ const HISTORY_UNAVAILABLE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "invalid_request_error",
|
||||
code: "previous_response_unavailable",
|
||||
};
|
||||
const CONTINUATION_BUSY_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
code: "continuation_connection_busy",
|
||||
};
|
||||
const INVALID_UPSTREAM_RESPONSE_ERROR: OpenAiErrorKind = OpenAiErrorKind {
|
||||
error_type: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
@@ -105,11 +112,24 @@ pub fn openai_error_response(
|
||||
}
|
||||
|
||||
pub fn responses_dispatch_error_response(error: ResponseDispatchError) -> Response {
|
||||
responses_dispatch_error_response_ref(&error)
|
||||
}
|
||||
|
||||
pub fn responses_dispatch_error_response_ref(error: &ResponseDispatchError) -> Response {
|
||||
if let ResponseDispatchError::Upstream(CodexClientError::Upstream { status, body, .. }) = error
|
||||
{
|
||||
if status.is_client_error() {
|
||||
if let Ok(body) = serde_json::from_str::<Value>(body) {
|
||||
return (*status, Json(body)).into_response();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match error {
|
||||
ResponseDispatchError::NoActiveAccount => {
|
||||
responses_no_available_accounts_response().into_response()
|
||||
}
|
||||
error => response_dispatch_openai_error_response(&error),
|
||||
error => response_dispatch_openai_error_response(error),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,6 +293,7 @@ fn dispatch_failure_message(
|
||||
}
|
||||
DispatchFailureClass::ResponseFailed => UPSTREAM_CODEX_RESPONSE_FAILED_MESSAGE.to_owned(),
|
||||
DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_MESSAGE.to_owned(),
|
||||
DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_MESSAGE.to_owned(),
|
||||
_ => UPSTREAM_CODEX_REQUEST_FAILED_MESSAGE.to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -290,6 +311,7 @@ fn dispatch_failure_openai_error_kind(
|
||||
| DispatchFailureClass::Banned => AUTHENTICATION_ERROR,
|
||||
DispatchFailureClass::ModelUnsupported => MODEL_NOT_FOUND_ERROR,
|
||||
DispatchFailureClass::HistoryUnavailable => HISTORY_UNAVAILABLE_ERROR,
|
||||
DispatchFailureClass::ContinuationBusy => CONTINUATION_BUSY_ERROR,
|
||||
DispatchFailureClass::InvalidSse
|
||||
| DispatchFailureClass::MissingCompleted
|
||||
| DispatchFailureClass::EmptyUpstreamResponse => INVALID_UPSTREAM_RESPONSE_ERROR,
|
||||
@@ -297,7 +319,9 @@ fn dispatch_failure_openai_error_kind(
|
||||
DispatchFailureClass::Upstream if status.is_client_error() => {
|
||||
responses_error_kind_for_status(status)
|
||||
}
|
||||
DispatchFailureClass::Upstream => UPSTREAM_ERROR,
|
||||
DispatchFailureClass::Upstream | DispatchFailureClass::UpstreamUnavailable => {
|
||||
UPSTREAM_ERROR
|
||||
}
|
||||
DispatchFailureClass::CloudflareChallenge | DispatchFailureClass::CloudflarePathBlocked => {
|
||||
UPSTREAM_ERROR
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ use super::{
|
||||
errors::{
|
||||
invalid_responses_request_response, missing_client_api_key_response,
|
||||
model_not_found_response, responses_dispatch_error_response,
|
||||
responses_stream_dispatch_failed_sse_event,
|
||||
responses_dispatch_error_response_ref, responses_stream_dispatch_failed_sse_event,
|
||||
},
|
||||
models::model_catalog_for_state,
|
||||
sse::{done_sse_frame, event_stream_response as sse_event_stream_response, SseResponseOptions},
|
||||
@@ -101,10 +101,43 @@ pub fn build_codex_request(
|
||||
None => {}
|
||||
}
|
||||
apply_context_header_fields(&mut request, headers);
|
||||
apply_identity_fields(&mut request, headers);
|
||||
request.client_user_agent = user_agent_from_headers(headers);
|
||||
request
|
||||
}
|
||||
|
||||
fn apply_identity_fields(request: &mut CodexResponsesRequest, headers: &HeaderMap) {
|
||||
request.client_conversation_id = request.prompt_cache_key().map(ToString::to_string);
|
||||
request.client_session_id = identity_context_string(request.body(), "session_id")
|
||||
.or_else(|| header_string(headers, "session-id"));
|
||||
request.client_thread_id = identity_context_string(request.body(), "thread_id")
|
||||
.or_else(|| header_string(headers, "thread-id"));
|
||||
request.client_request_id = identity_context_string(request.body(), "x-client-request-id")
|
||||
.or_else(|| header_string(headers, "x-client-request-id"));
|
||||
request.client_turn_id = identity_context_string(request.body(), "turn_id")
|
||||
.or_else(|| header_string(headers, "x-codex-turn-id"));
|
||||
request.codex_window_id = request.codex_window_id.take().or_else(|| {
|
||||
identity_context_string(request.body(), "x-codex-window-id")
|
||||
.or_else(|| header_string(headers, "x-codex-window-id"))
|
||||
});
|
||||
request.parent_thread_id = request.parent_thread_id.take().or_else(|| {
|
||||
identity_context_string(request.body(), "x-codex-parent-thread-id")
|
||||
.or_else(|| header_string(headers, "x-codex-parent-thread-id"))
|
||||
});
|
||||
}
|
||||
|
||||
fn identity_context_string(body: &Map<String, Value>, key: &str) -> Option<String> {
|
||||
body_context_string(body, key).or_else(|| {
|
||||
body.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get(key))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
})
|
||||
}
|
||||
|
||||
/// 从客户端原始 Responses JSON body 构造 Codex compact 请求。
|
||||
///
|
||||
/// compact 端点只返回一次性 JSON(代理按非流式读取),故剥离 `stream` 这一
|
||||
@@ -115,6 +148,18 @@ pub fn build_compact_request(
|
||||
mut body: Map<String, Value>,
|
||||
headers: &HeaderMap,
|
||||
) -> CodexCompactRequest {
|
||||
let client_session_id = identity_context_string(&body, "session_id")
|
||||
.or_else(|| header_string(headers, "session-id"));
|
||||
let client_thread_id =
|
||||
identity_context_string(&body, "thread_id").or_else(|| header_string(headers, "thread-id"));
|
||||
let client_request_id = identity_context_string(&body, "x-client-request-id")
|
||||
.or_else(|| header_string(headers, "x-client-request-id"));
|
||||
let client_turn_id = identity_context_string(&body, "turn_id")
|
||||
.or_else(|| header_string(headers, "x-codex-turn-id"));
|
||||
let client_window_id = identity_context_string(&body, "x-codex-window-id")
|
||||
.or_else(|| header_string(headers, "x-codex-window-id"));
|
||||
let client_parent_thread_id = identity_context_string(&body, "x-codex-parent-thread-id")
|
||||
.or_else(|| header_string(headers, "x-codex-parent-thread-id"));
|
||||
body.remove(COMPACT_STREAM_KEY);
|
||||
for key in TRANSPORT_ONLY_KEYS {
|
||||
body.remove(key);
|
||||
@@ -124,6 +169,12 @@ pub fn build_compact_request(
|
||||
client_ip: None,
|
||||
client_user_agent: user_agent_from_headers(headers),
|
||||
client_api_key_id: None,
|
||||
client_session_id,
|
||||
client_thread_id,
|
||||
client_request_id,
|
||||
client_turn_id,
|
||||
client_window_id,
|
||||
client_parent_thread_id,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,7 +363,10 @@ async fn handle_responses(
|
||||
.complete(request_id.as_str(), route, codex_request, &model)
|
||||
.await
|
||||
{
|
||||
Ok(body) => (StatusCode::OK, Json(body)).into_response(),
|
||||
Ok(result) => apply_safe_response_headers(
|
||||
(StatusCode::OK, Json(result.body)).into_response(),
|
||||
result.response_headers,
|
||||
),
|
||||
Err(error) => responses_dispatch_error_response(error),
|
||||
}
|
||||
}
|
||||
@@ -352,7 +406,10 @@ async fn handle_compact_responses(
|
||||
.compact(request_id.as_str(), compact_request, &model)
|
||||
.await
|
||||
{
|
||||
Ok(body) => (StatusCode::OK, Json(body)).into_response(),
|
||||
Ok(result) => apply_safe_response_headers(
|
||||
(StatusCode::OK, Json(result.body)).into_response(),
|
||||
result.response_headers,
|
||||
),
|
||||
Err(error) => responses_dispatch_error_response(error),
|
||||
}
|
||||
}
|
||||
@@ -424,10 +481,37 @@ fn event_stream_response(mut body: String) -> Response {
|
||||
}
|
||||
|
||||
fn live_event_stream_response(stream: ResponseDispatchStream) -> Response {
|
||||
sse_event_stream_response(Body::from_stream(stream.body), SseResponseOptions::BASIC)
|
||||
let response =
|
||||
sse_event_stream_response(Body::from_stream(stream.body), SseResponseOptions::BASIC);
|
||||
apply_safe_response_headers(response, stream.response_headers)
|
||||
}
|
||||
|
||||
fn apply_safe_response_headers(mut response: Response, headers: Vec<(String, String)>) -> Response {
|
||||
for (name, value) in headers {
|
||||
let name = match name.as_str() {
|
||||
"x-request-id" => axum::http::header::HeaderName::from_static("x-request-id"),
|
||||
"openai-model" => axum::http::header::HeaderName::from_static("openai-model"),
|
||||
"x-models-etag" => axum::http::header::HeaderName::from_static("x-models-etag"),
|
||||
"x-reasoning-included" => {
|
||||
axum::http::header::HeaderName::from_static("x-reasoning-included")
|
||||
}
|
||||
"openai-processing-ms" => {
|
||||
axum::http::header::HeaderName::from_static("openai-processing-ms")
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let Ok(value) = axum::http::HeaderValue::from_str(&value) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().insert(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn response_dispatch_stream_error_response(error: &ResponseDispatchError) -> Response {
|
||||
if (400..=499).contains(&error.client_http_status_code()) {
|
||||
return responses_dispatch_error_response_ref(error);
|
||||
}
|
||||
event_stream_response(responses_stream_dispatch_failed_sse_event(error))
|
||||
}
|
||||
|
||||
|
||||
@@ -173,16 +173,16 @@ pub struct WebSocketPoolSettings {
|
||||
pub max_age_ms: u64,
|
||||
/// 单账号最大连接数。
|
||||
pub max_per_account: usize,
|
||||
/// 首个真实输出(首 token)到达前的绝对超时;`0` 表示禁用。
|
||||
/// 首个真实输出(首事件)到达前的绝对超时;`0` 表示禁用。
|
||||
///
|
||||
/// 从发出 `response.create` 起算,覆盖建连/发送/上游排队/思考的全程,直到收到
|
||||
/// 首个真实内容帧(`response.created`/`response.in_progress` 等纯生命周期帧不计)。
|
||||
/// 超时判定为连接落到病态上游后端,丢弃并换新连接重试。
|
||||
#[serde(default = "default_ws_first_token_timeout_ms")]
|
||||
pub first_token_timeout_ms: u64,
|
||||
#[serde(default = "default_ws_initial_event_timeout_ms")]
|
||||
pub initial_event_timeout_ms: u64,
|
||||
}
|
||||
|
||||
fn default_ws_first_token_timeout_ms() -> u64 {
|
||||
fn default_ws_initial_event_timeout_ms() -> u64 {
|
||||
20_000
|
||||
}
|
||||
|
||||
@@ -192,7 +192,7 @@ impl Default for WebSocketPoolSettings {
|
||||
enabled: true,
|
||||
max_age_ms: 3_300_000,
|
||||
max_per_account: 8,
|
||||
first_token_timeout_ms: default_ws_first_token_timeout_ms(),
|
||||
initial_event_timeout_ms: default_ws_initial_event_timeout_ms(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,8 +25,9 @@ use crate::{
|
||||
},
|
||||
infra::{
|
||||
database::connect,
|
||||
identity::AccountPseudonymizer,
|
||||
logging::{init_tracing, LogError, LogGuard, RotationConfig, TracingConfig},
|
||||
paths::{ensure_data_dir, load_or_create_installation_id},
|
||||
paths::{ensure_data_dir, load_or_create_identity_secret},
|
||||
redis::RedisConnection,
|
||||
},
|
||||
keys::{manage::KeyManageService, service::KeyVerifier, store::PgClientKeyStore},
|
||||
@@ -115,7 +116,7 @@ pub struct Services {
|
||||
pub codex: Arc<CodexBackendClient>,
|
||||
pub websocket_pool: Option<Arc<CodexWebSocketPool>>,
|
||||
pub fingerprint: RuntimeFingerprint,
|
||||
pub installation_id: Option<String>,
|
||||
pub account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
pub background_tasks: BackgroundTaskStores,
|
||||
pub(crate) account_pool_static: AccountPoolStaticSettings,
|
||||
}
|
||||
@@ -151,32 +152,42 @@ impl Services {
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
Self::try_with_installation_id(config, stores, fingerprint, None)
|
||||
}
|
||||
|
||||
pub(crate) fn try_with_installation_id(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
installation_id: Option<String>,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
Self::try_with_installation_id_and_usage_record_options(
|
||||
Self::try_with_usage_record_options(
|
||||
config,
|
||||
stores,
|
||||
fingerprint,
|
||||
installation_id,
|
||||
UsageRecordOptions::from_config(config),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn try_with_installation_id_and_usage_record_options(
|
||||
pub fn try_with_usage_record_options(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
installation_id: Option<String>,
|
||||
usage_record_options: UsageRecordOptions,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
let installation_id = installation_id.filter(|id| !id.trim().is_empty());
|
||||
let mut identity_secret = [0u8; 32];
|
||||
rand::Rng::fill_bytes(&mut rand::rng(), &mut identity_secret);
|
||||
Self::try_with_identity_secret_and_usage_record_options(
|
||||
config,
|
||||
stores,
|
||||
fingerprint,
|
||||
identity_secret,
|
||||
usage_record_options,
|
||||
)
|
||||
}
|
||||
|
||||
fn try_with_identity_secret_and_usage_record_options(
|
||||
config: &AppConfig,
|
||||
stores: BackgroundTaskStores,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
identity_secret: [u8; 32],
|
||||
usage_record_options: UsageRecordOptions,
|
||||
) -> Result<Self, CustomCaError> {
|
||||
let account_pseudonymizer = Arc::new(AccountPseudonymizer::new(identity_secret));
|
||||
let account_identity = Arc::new(crate::dispatch::affinity::AccountIdentityService::new(
|
||||
(*account_pseudonymizer).clone(),
|
||||
));
|
||||
let database = stores.accounts.pool().clone();
|
||||
let redis = stores.redis.clone();
|
||||
let account_store = Arc::new(stores.accounts.clone()) as Arc<dyn AccountStore>;
|
||||
@@ -187,21 +198,21 @@ impl Services {
|
||||
enabled: true,
|
||||
max_age: std::time::Duration::from_millis(config.ws_pool.max_age_ms),
|
||||
max_per_account: config.ws_pool.max_per_account,
|
||||
first_token_timeout: (config.ws_pool.first_token_timeout_ms > 0).then(|| {
|
||||
std::time::Duration::from_millis(config.ws_pool.first_token_timeout_ms)
|
||||
initial_event_timeout: (config.ws_pool.initial_event_timeout_ms > 0).then(|| {
|
||||
std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms)
|
||||
}),
|
||||
..CodexWebSocketPoolConfig::default()
|
||||
}))
|
||||
});
|
||||
let codex = {
|
||||
let websocket_first_token_timeout = (config.ws_pool.first_token_timeout_ms > 0)
|
||||
.then(|| std::time::Duration::from_millis(config.ws_pool.first_token_timeout_ms));
|
||||
let websocket_initial_event_timeout = (config.ws_pool.initial_event_timeout_ms > 0)
|
||||
.then(|| std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms));
|
||||
let client = CodexBackendClient::new(
|
||||
build_reqwest_client(config.tls.force_http11)?,
|
||||
config.api.base_url.clone(),
|
||||
fingerprint.clone(),
|
||||
)
|
||||
.with_websocket_first_token_timeout(websocket_first_token_timeout);
|
||||
.with_websocket_initial_event_timeout(websocket_initial_event_timeout);
|
||||
if let Some(pool) = &websocket_pool {
|
||||
Arc::new(client.with_websocket_pool(pool.clone()))
|
||||
} else {
|
||||
@@ -275,7 +286,7 @@ impl Services {
|
||||
config.auth.oauth_token_endpoint.clone(),
|
||||
),
|
||||
refresh_policy: refresh_policy.clone(),
|
||||
installation_id: installation_id.clone(),
|
||||
account_pseudonymizer: account_pseudonymizer.clone(),
|
||||
}));
|
||||
let usage = Arc::new(AccountUsageQueryService::new(stores.account_usage.clone()));
|
||||
let session_affinity =
|
||||
@@ -289,7 +300,7 @@ impl Services {
|
||||
codex: codex.clone(),
|
||||
session_affinity: session_affinity.clone(),
|
||||
recorder,
|
||||
installation_id: installation_id.clone(),
|
||||
account_identity,
|
||||
cloudflare: cloudflare_recovery,
|
||||
}));
|
||||
|
||||
@@ -314,7 +325,7 @@ impl Services {
|
||||
codex,
|
||||
websocket_pool,
|
||||
fingerprint,
|
||||
installation_id,
|
||||
account_pseudonymizer,
|
||||
background_tasks: stores,
|
||||
account_pool_static,
|
||||
})
|
||||
@@ -520,18 +531,19 @@ async fn build_router(
|
||||
config.admin.validate_default_password()?;
|
||||
let default_admin_password = config.admin.default_password.clone();
|
||||
let data_dir = ensure_data_dir()?;
|
||||
let installation_id = load_or_create_installation_id(Some(&data_dir))?;
|
||||
let identity_secret = load_or_create_identity_secret(&data_dir)?;
|
||||
let default_fingerprint = fingerprint_from_config(&config.fingerprint);
|
||||
let runtime_fingerprint = fingerprint_store
|
||||
.ensure_current_seed(&default_fingerprint)
|
||||
.await?;
|
||||
let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint);
|
||||
let runtime_config = crate::bootstrap::state::RuntimeConfig::from(config.clone());
|
||||
let services = Services::try_with_installation_id(
|
||||
let services = Services::try_with_identity_secret_and_usage_record_options(
|
||||
&config,
|
||||
stores,
|
||||
runtime_fingerprint,
|
||||
Some(installation_id),
|
||||
identity_secret,
|
||||
UsageRecordOptions::from_config(&config),
|
||||
)?;
|
||||
services.initialize_hot_path_state().await?;
|
||||
|
||||
|
||||
@@ -53,9 +53,23 @@ impl TaskCoordinator {
|
||||
);
|
||||
coordinator.push(
|
||||
"model_refresh",
|
||||
ModelRefreshTask::new(services.models.clone(), services.account_pool.clone())
|
||||
.with_installation_id(services.installation_id.clone())
|
||||
.start(),
|
||||
ModelRefreshTask::new(
|
||||
services.models.clone(),
|
||||
services.account_pool.clone(),
|
||||
services.account_pseudonymizer.clone(),
|
||||
)
|
||||
.start(),
|
||||
);
|
||||
coordinator.push(
|
||||
"model_etag_refresh",
|
||||
SchedulerHandle::spawn(
|
||||
ModelRefreshTask::new(
|
||||
services.models.clone(),
|
||||
services.account_pool.clone(),
|
||||
services.account_pseudonymizer.clone(),
|
||||
)
|
||||
.run_etag_refreshes(),
|
||||
),
|
||||
);
|
||||
if config.auth.refresh_enabled {
|
||||
coordinator.push(
|
||||
@@ -69,6 +83,7 @@ impl TaskCoordinator {
|
||||
stores.accounts.clone(),
|
||||
stores.account_usage.clone(),
|
||||
services.codex.clone(),
|
||||
services.account_pseudonymizer.clone(),
|
||||
config
|
||||
.quota
|
||||
.refresh_interval_minutes
|
||||
@@ -76,7 +91,6 @@ impl TaskCoordinator {
|
||||
.max(1),
|
||||
30 * 60,
|
||||
)
|
||||
.with_installation_id(services.installation_id.clone())
|
||||
.with_cookie_store(stores.cookies.clone())
|
||||
.with_account_pool(services.account_pool.clone())
|
||||
.start(),
|
||||
|
||||
@@ -8,6 +8,7 @@ use tokio::time::sleep;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::fleet::pool::AccountPoolService;
|
||||
use crate::infra::identity::AccountPseudonymizer;
|
||||
use crate::models::service::{
|
||||
ModelRefreshPlanAccount, ModelRefreshResult, ModelService, ModelServiceError,
|
||||
};
|
||||
@@ -25,7 +26,7 @@ pub struct ModelRefreshTask {
|
||||
initial_delay_ms: u64,
|
||||
retry_delay_ms: u64,
|
||||
max_retries: u32,
|
||||
installation_id: Option<String>,
|
||||
account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
}
|
||||
|
||||
const DEFAULT_REFRESH_INTERVAL_SECS: u64 = 3600;
|
||||
@@ -35,7 +36,11 @@ const MAX_RETRIES: u32 = 12;
|
||||
|
||||
impl ModelRefreshTask {
|
||||
/// 构造默认任务。
|
||||
pub fn new(model_service: Arc<ModelService>, account_pool: Arc<AccountPoolService>) -> Self {
|
||||
pub fn new(
|
||||
model_service: Arc<ModelService>,
|
||||
account_pool: Arc<AccountPoolService>,
|
||||
account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
model_service,
|
||||
account_pool,
|
||||
@@ -43,16 +48,10 @@ impl ModelRefreshTask {
|
||||
initial_delay_ms: INITIAL_DELAY_MS,
|
||||
retry_delay_ms: RETRY_DELAY_MS,
|
||||
max_retries: MAX_RETRIES,
|
||||
installation_id: None,
|
||||
account_pseudonymizer,
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置 Codex installation id。
|
||||
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
|
||||
self.installation_id = installation_id.filter(|id| !id.trim().is_empty());
|
||||
self
|
||||
}
|
||||
|
||||
/// 启动后台刷新任务。
|
||||
pub fn start(self) -> SchedulerHandle {
|
||||
let config = PeriodicTaskConfig::new(
|
||||
@@ -64,6 +63,27 @@ impl ModelRefreshTask {
|
||||
spawn_periodic_task(self, config)
|
||||
}
|
||||
|
||||
/// 在 Responses 响应声明模型目录 ETag 变化时异步刷新。
|
||||
pub async fn run_etag_refreshes(self) {
|
||||
let mut etag_changes = self.model_service.subscribe_models_etag_changes();
|
||||
while etag_changes.changed().await.is_ok() {
|
||||
// 合并同一批并发响应产生的重复通知。
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
etag_changes.borrow_and_update();
|
||||
match self.refresh_once().await {
|
||||
Ok(result) => info!(
|
||||
refreshed_plans = result.refreshed_plans,
|
||||
model_count = result.model_count,
|
||||
"模型 ETag 变化刷新完成"
|
||||
),
|
||||
Err(ModelServiceError::NoAccounts) => {
|
||||
info!("模型 ETag 变化刷新跳过:没有可用账号");
|
||||
}
|
||||
Err(error) => warn!(error = ?error, "模型 ETag 变化刷新失败"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一次模型刷新并同步账号池模型计划约束。
|
||||
pub async fn refresh_once(&self) -> Result<ModelRefreshResult, ModelServiceError> {
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
@@ -74,16 +94,15 @@ impl ModelRefreshTask {
|
||||
plan_type: selected.plan_type.clone(),
|
||||
access_token: selected.account.access_token.clone(),
|
||||
account_id: selected.account.account_id.clone(),
|
||||
installation_id: self
|
||||
.account_pseudonymizer
|
||||
.installation_id(&selected.account.id),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let result = self
|
||||
.model_service
|
||||
.refresh_backend_models_with_installation_id(
|
||||
&plan_accounts,
|
||||
&request_id,
|
||||
self.installation_id.as_deref(),
|
||||
)
|
||||
.refresh_backend_models(&plan_accounts, &request_id)
|
||||
.await;
|
||||
for selected in selected_accounts {
|
||||
selected.release().await;
|
||||
|
||||
@@ -12,6 +12,7 @@ use crate::{
|
||||
quota::{QuotaRefreshService, QuotaRefreshSummary},
|
||||
store::PgAccountStore,
|
||||
},
|
||||
infra::identity::AccountPseudonymizer,
|
||||
telemetry::account_usage::store::PgAccountUsageStore,
|
||||
upstream::openai::transport::CodexBackendClient,
|
||||
};
|
||||
@@ -34,6 +35,7 @@ impl QuotaRefreshTask {
|
||||
store: PgAccountStore,
|
||||
usage_store: PgAccountUsageStore,
|
||||
codex: Arc<CodexBackendClient>,
|
||||
account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
interval_secs: u64,
|
||||
min_refresh_interval_secs: u64,
|
||||
) -> Self {
|
||||
@@ -42,6 +44,7 @@ impl QuotaRefreshTask {
|
||||
store,
|
||||
usage_store,
|
||||
codex,
|
||||
account_pseudonymizer,
|
||||
min_refresh_interval_secs,
|
||||
),
|
||||
interval_secs,
|
||||
@@ -49,12 +52,6 @@ impl QuotaRefreshTask {
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置 Codex installation id。
|
||||
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
|
||||
self.service = self.service.with_installation_id(installation_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置 usage 请求可复用的账号 Cookie 存储。
|
||||
pub fn with_cookie_store(mut self, cookie_store: PgCookieStore) -> Self {
|
||||
self.service = self.service.with_cookie_store(cookie_store);
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
//! 客户端身份、本地会话索引与账号作用域 wire 身份隔离。
|
||||
|
||||
use crate::{
|
||||
infra::identity::AccountPseudonymizer,
|
||||
upstream::openai::protocol::responses::{CodexCompactRequest, CodexResponsesRequest},
|
||||
};
|
||||
|
||||
/// 持久化密钥驱动的账号身份隔离服务。
|
||||
#[derive(Clone)]
|
||||
pub struct AccountIdentityService {
|
||||
pseudonymizer: AccountPseudonymizer,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AccountIdentityService {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountIdentityService")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// 单次账号 attempt 的 wire 身份。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountScopedIdentity {
|
||||
pub prompt_cache_key: Option<String>,
|
||||
pub session_id: Option<String>,
|
||||
pub thread_id: Option<String>,
|
||||
pub client_request_id: String,
|
||||
pub turn_id: Option<String>,
|
||||
pub window_id: Option<String>,
|
||||
pub parent_thread_id: Option<String>,
|
||||
pub installation_id: String,
|
||||
}
|
||||
|
||||
struct IdentityValues<'a> {
|
||||
prompt_cache_key: Option<&'a str>,
|
||||
session_id: Option<&'a str>,
|
||||
thread_id: Option<&'a str>,
|
||||
client_request_id: Option<&'a str>,
|
||||
turn_id: Option<&'a str>,
|
||||
window_id: Option<&'a str>,
|
||||
parent_thread_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
impl AccountIdentityService {
|
||||
pub fn new(pseudonymizer: AccountPseudonymizer) -> Self {
|
||||
Self { pseudonymizer }
|
||||
}
|
||||
|
||||
pub fn prepare_local_identity(&self, request: &mut CodexResponsesRequest) {
|
||||
if request.local_conversation_id.is_some() {
|
||||
return;
|
||||
}
|
||||
let anchor = request
|
||||
.prompt_cache_key()
|
||||
.or(request.client_session_id.as_deref())
|
||||
.or(request.client_thread_id.as_deref())
|
||||
.or(request.client_conversation_id.as_deref())
|
||||
.map(ToString::to_string)
|
||||
.or_else(|| super::resolve::derive_stable_conversation_key(request));
|
||||
request.local_conversation_id = anchor
|
||||
.as_deref()
|
||||
.map(|anchor| format!("lc_{}", self.pseudonym("local-conversation", None, anchor)));
|
||||
}
|
||||
|
||||
pub fn scope(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
self.scope_values(
|
||||
account_id,
|
||||
proxy_request_id,
|
||||
IdentityValues {
|
||||
prompt_cache_key: request.prompt_cache_key(),
|
||||
session_id: request.client_session_id.as_deref(),
|
||||
thread_id: request.client_thread_id.as_deref(),
|
||||
client_request_id: request.client_request_id.as_deref(),
|
||||
turn_id: request.client_turn_id.as_deref(),
|
||||
window_id: request.codex_window_id.as_deref(),
|
||||
parent_thread_id: request.parent_thread_id.as_deref(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn scope_compact(
|
||||
&self,
|
||||
request: &CodexCompactRequest,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
self.scope_values(
|
||||
account_id,
|
||||
proxy_request_id,
|
||||
IdentityValues {
|
||||
prompt_cache_key: request.prompt_cache_key(),
|
||||
session_id: request.client_session_id.as_deref(),
|
||||
thread_id: request.client_thread_id.as_deref(),
|
||||
client_request_id: request.client_request_id.as_deref(),
|
||||
turn_id: request.client_turn_id.as_deref(),
|
||||
window_id: request.client_window_id.as_deref(),
|
||||
parent_thread_id: request.client_parent_thread_id.as_deref(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn scope_auxiliary(
|
||||
&self,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
AccountScopedIdentity {
|
||||
client_request_id: format!(
|
||||
"cr_{}",
|
||||
self.pseudonym("client-request-id", Some(account_id), proxy_request_id)
|
||||
),
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
..AccountScopedIdentity::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_values(
|
||||
&self,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
values: IdentityValues<'_>,
|
||||
) -> AccountScopedIdentity {
|
||||
AccountScopedIdentity {
|
||||
prompt_cache_key: self.scope_optional(
|
||||
"prompt-cache-key",
|
||||
account_id,
|
||||
values.prompt_cache_key,
|
||||
),
|
||||
session_id: self.scope_optional("session-id", account_id, values.session_id),
|
||||
thread_id: self.scope_optional("thread-id", account_id, values.thread_id),
|
||||
client_request_id: format!(
|
||||
"cr_{}",
|
||||
self.pseudonym(
|
||||
"client-request-id",
|
||||
Some(account_id),
|
||||
values.client_request_id.unwrap_or(proxy_request_id),
|
||||
)
|
||||
),
|
||||
turn_id: self.scope_optional("turn-id", account_id, values.turn_id),
|
||||
window_id: self.scope_optional("window-id", account_id, values.window_id),
|
||||
parent_thread_id: self.scope_optional(
|
||||
"parent-thread-id",
|
||||
account_id,
|
||||
values.parent_thread_id,
|
||||
),
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_optional(
|
||||
&self,
|
||||
domain: &str,
|
||||
account_id: &str,
|
||||
value: Option<&str>,
|
||||
) -> Option<String> {
|
||||
value
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| format!("wi_{}", self.pseudonym(domain, Some(account_id), value)))
|
||||
}
|
||||
|
||||
fn pseudonym(&self, domain: &str, account_id: Option<&str>, value: &str) -> String {
|
||||
self.pseudonymizer.scoped(domain, account_id, value)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
//! 会话亲和身份、存储、解析与运行时服务。
|
||||
|
||||
pub mod identity;
|
||||
pub mod resolve;
|
||||
pub mod service;
|
||||
pub mod store;
|
||||
pub mod types;
|
||||
|
||||
pub use identity::{AccountIdentityService, AccountScopedIdentity};
|
||||
pub use resolve::*;
|
||||
pub use service::*;
|
||||
pub use store::*;
|
||||
|
||||
@@ -1,89 +1,33 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::{types::ConversationIdentity, SessionAffinityEntry, SessionAffinityService},
|
||||
recovery::{
|
||||
implicit_resume::{restore_full_request_without_history, ImplicitResumeSnapshot},
|
||||
reasoning_replay::ReasoningReplayCache,
|
||||
},
|
||||
affinity::{SessionAffinityEntry, SessionAffinityService},
|
||||
recovery::history::HistoryRecoveryPlan,
|
||||
service::ResponseDispatchService,
|
||||
},
|
||||
fleet::account::AccountStatus,
|
||||
upstream::openai::protocol::{
|
||||
events::TokenUsage,
|
||||
responses::{completed_response_metadata, CodexResponsesRequest},
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(in crate::dispatch) struct AccountAffinityDecision {
|
||||
preferred_account_id: Option<String>,
|
||||
request_history_owner: RequestHistoryOwner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
enum RequestHistoryOwner {
|
||||
#[default]
|
||||
Absent,
|
||||
Known(String),
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountAffinityDecision {
|
||||
pub(in crate::dispatch) fn preferred_account_id(&self) -> Option<&str> {
|
||||
self.preferred_account_id.as_deref()
|
||||
}
|
||||
|
||||
fn should_strip_request_history_for(&self, account_id: &str) -> bool {
|
||||
match &self.request_history_owner {
|
||||
RequestHistoryOwner::Absent => false,
|
||||
RequestHistoryOwner::Known(history_account_id) => history_account_id != account_id,
|
||||
RequestHistoryOwner::Unknown => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponseDispatchService {
|
||||
pub(in crate::dispatch) async fn account_affinity_for_request(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
now: DateTime<Utc>,
|
||||
) -> AccountAffinityDecision {
|
||||
let has_previous_response_id = request.previous_response_id().is_some();
|
||||
if let Some(account_id) = self.previous_response_account_id(request, now).await {
|
||||
return AccountAffinityDecision {
|
||||
preferred_account_id: Some(account_id.clone()),
|
||||
request_history_owner: RequestHistoryOwner::Known(account_id),
|
||||
};
|
||||
}
|
||||
|
||||
let conversation_account_id = self.conversation_account_id(request, now).await;
|
||||
AccountAffinityDecision {
|
||||
preferred_account_id: conversation_account_id,
|
||||
request_history_owner: if has_previous_response_id {
|
||||
RequestHistoryOwner::Unknown
|
||||
} else {
|
||||
RequestHistoryOwner::Absent
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn previous_response_account_id(
|
||||
pub(in crate::dispatch) async fn preferred_account_id_for_request(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
history: &HistoryRecoveryPlan,
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<String> {
|
||||
let previous_response_id = request.previous_response_id()?;
|
||||
self.session_affinity
|
||||
.lookup_account(previous_response_id, now)
|
||||
.await
|
||||
if let Some(account_id) = history.preferred_account_id() {
|
||||
return Some(account_id.to_string());
|
||||
}
|
||||
if request.previous_response_id().is_some() {
|
||||
return None;
|
||||
}
|
||||
self.conversation_account_id(request, now).await
|
||||
}
|
||||
|
||||
async fn conversation_account_id(
|
||||
@@ -92,7 +36,8 @@ impl ResponseDispatchService {
|
||||
now: DateTime<Utc>,
|
||||
) -> Option<String> {
|
||||
let conversation_id = request
|
||||
.prompt_cache_key()
|
||||
.local_conversation_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let variant_hash = compute_variant_hash(request);
|
||||
@@ -100,89 +45,17 @@ impl ResponseDispatchService {
|
||||
.lookup_latest_account_by_conversation(conversation_id, None, Some(&variant_hash), now)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn strip_history_if_account_changed(
|
||||
request: &mut CodexResponsesRequest,
|
||||
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
||||
account_affinity: &AccountAffinityDecision,
|
||||
acquired_account_id: &str,
|
||||
) {
|
||||
if request.previous_response_id().is_none()
|
||||
|| !account_affinity.should_strip_request_history_for(acquired_account_id)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(snapshot) = implicit_resume.take() else {
|
||||
return;
|
||||
};
|
||||
restore_full_request_without_history(request, snapshot);
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) async fn apply_cascading_ban_defense(
|
||||
&self,
|
||||
request: &mut CodexResponsesRequest,
|
||||
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
||||
preferred_account_id: Option<&str>,
|
||||
acquired_account_id: &str,
|
||||
) -> bool {
|
||||
let Some(preferred_account_id) =
|
||||
preferred_account_id.filter(|account_id| *account_id != acquired_account_id)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let has_history = request.previous_response_id().is_some()
|
||||
|| request
|
||||
.turn_state
|
||||
.as_deref()
|
||||
.is_some_and(|value| !value.is_empty());
|
||||
if !has_history {
|
||||
return false;
|
||||
}
|
||||
let Some(preferred_account) = self
|
||||
.account_pool
|
||||
.account_snapshot(preferred_account_id)
|
||||
.await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !matches!(
|
||||
preferred_account.status,
|
||||
AccountStatus::Banned | AccountStatus::Disabled
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let response_id_to_forget = request.previous_response_id().map(str::to_string);
|
||||
let restored_full_request = if let Some(snapshot) = implicit_resume.take() {
|
||||
restore_full_request_without_history(request, snapshot);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if let Some(response_id) = response_id_to_forget {
|
||||
self.session_affinity.forget(&response_id).await;
|
||||
}
|
||||
restored_full_request
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn record_response_affinity(
|
||||
session_affinity: &Arc<SessionAffinityService>,
|
||||
reasoning_replay: &Arc<Mutex<ReasoningReplayCache>>,
|
||||
request: &CodexResponsesRequest,
|
||||
pub(in crate::dispatch) async fn record_response_affinity(
|
||||
session_affinity: &SessionAffinityService,
|
||||
history: &HistoryRecoveryPlan,
|
||||
original_request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
body: &str,
|
||||
turn_state: Option<String>,
|
||||
usage: Option<TokenUsage>,
|
||||
) {
|
||||
let Some(conversation_id) = request
|
||||
.prompt_cache_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let metadata = match completed_response_metadata(body) {
|
||||
Ok(Some(metadata)) => metadata,
|
||||
Ok(None) => return,
|
||||
@@ -195,17 +68,27 @@ pub(crate) async fn record_response_affinity(
|
||||
}
|
||||
};
|
||||
|
||||
let variant_hash = compute_variant_hash(request);
|
||||
let conversation_id = history
|
||||
.conversation_id(original_request)
|
||||
.unwrap_or(&metadata.response_id)
|
||||
.to_string();
|
||||
let variant_hash = compute_variant_hash(original_request);
|
||||
let entry = SessionAffinityEntry {
|
||||
account_id: account_id.to_string(),
|
||||
conversation_id: conversation_id.to_string(),
|
||||
conversation_id,
|
||||
turn_state: turn_state
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| request.turn_state.clone()),
|
||||
instructions_hash: Some(hash_instructions(Some(request.instructions()))),
|
||||
.or_else(|| original_request.turn_state.clone()),
|
||||
instructions_hash: Some(hash_instructions(Some(original_request.instructions()))),
|
||||
input_tokens: usage.map(|usage| usage.input_tokens),
|
||||
function_call_ids: metadata.function_call_ids,
|
||||
variant_hash: Some(variant_hash.clone()),
|
||||
variant_hash: Some(variant_hash),
|
||||
continuation_scope: if original_request.store() {
|
||||
crate::upstream::openai::protocol::responses::PreviousResponseScope::Persisted
|
||||
} else {
|
||||
crate::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal
|
||||
},
|
||||
replay: history.completed_replay(original_request, &metadata.output),
|
||||
created_at: Utc::now(),
|
||||
};
|
||||
if let Err(error) = session_affinity
|
||||
@@ -219,30 +102,6 @@ pub(crate) async fn record_response_affinity(
|
||||
"failed to record session affinity"
|
||||
);
|
||||
}
|
||||
|
||||
reasoning_replay.lock().await.record(
|
||||
metadata.response_id,
|
||||
account_id,
|
||||
conversation_id,
|
||||
&variant_hash,
|
||||
&metadata.replay_items,
|
||||
Utc::now(),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) async fn evict_reasoning_replay(
|
||||
reasoning_replay: &Arc<Mutex<ReasoningReplayCache>>,
|
||||
request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
) {
|
||||
let variant_hash = compute_variant_hash(request);
|
||||
let conversation_id = request.prompt_cache_key().unwrap_or("").to_string();
|
||||
reasoning_replay.lock().await.evict_by_identity(
|
||||
account_id,
|
||||
&conversation_id,
|
||||
&variant_hash,
|
||||
Utc::now(),
|
||||
);
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
@@ -309,58 +168,9 @@ pub fn hash_instructions(instructions: Option<&str>) -> String {
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// Conversation identity 构建器
|
||||
// ====================================================================
|
||||
|
||||
const LEADING_SYSTEM_REMINDER_OPEN: &str = "<system-reminder>";
|
||||
const LEADING_SYSTEM_REMINDER_CLOSE: &str = "</system-reminder>";
|
||||
|
||||
/// 从 prompt_cache_key 和可选的 window_id 构建 conversation identity
|
||||
pub fn build_conversation_identity(
|
||||
prompt_cache_key: Option<&str>,
|
||||
client_window_id: Option<&str>,
|
||||
account_scope: &str,
|
||||
) -> ConversationIdentity {
|
||||
let conversation_id = prompt_cache_key
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.map(|key| build_account_scoped_identity("conversation", account_scope, key));
|
||||
|
||||
let window_id = if let Some(client_win) = client_window_id.filter(|s| !s.trim().is_empty()) {
|
||||
Some(build_account_scoped_identity(
|
||||
"window",
|
||||
account_scope,
|
||||
client_win,
|
||||
))
|
||||
} else {
|
||||
conversation_id
|
||||
.as_ref()
|
||||
.map(|conv_id| format!("{}:0", conv_id))
|
||||
};
|
||||
|
||||
ConversationIdentity {
|
||||
conversation_id,
|
||||
window_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// 确保请求拥有上游可复用的 prompt cache key。
|
||||
pub fn ensure_prompt_cache_key(request: &mut CodexResponsesRequest) {
|
||||
if let Some(existing) = request
|
||||
.prompt_cache_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
let existing = existing.to_string();
|
||||
request.set_prompt_cache_key(Some(existing));
|
||||
return;
|
||||
}
|
||||
|
||||
request.set_prompt_cache_key(Some(
|
||||
derive_stable_conversation_key(request).unwrap_or_else(|| Uuid::new_v4().to_string()),
|
||||
));
|
||||
}
|
||||
|
||||
/// 按原版 `stable-conversation-key.ts` 的规则派生稳定 conversation key。
|
||||
pub fn derive_stable_conversation_key(request: &CodexResponsesRequest) -> Option<String> {
|
||||
let instructions = request
|
||||
@@ -436,25 +246,3 @@ fn normalize_conversation_anchor_text(text: &str) -> String {
|
||||
}
|
||||
rest.to_string()
|
||||
}
|
||||
|
||||
/// 构建账号作用域的身份哈希。
|
||||
fn build_account_scoped_identity(kind: &str, account_scope: &str, client_value: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(kind.as_bytes());
|
||||
hasher.update(b"\0");
|
||||
hasher.update(account_scope.as_bytes());
|
||||
hasher.update(b"\0");
|
||||
hasher.update(client_value.as_bytes());
|
||||
|
||||
let digest = hasher.finalize();
|
||||
let hex = hex::encode(digest);
|
||||
let truncated = &hex[..32];
|
||||
|
||||
let prefix = match kind {
|
||||
"conversation" => "cp",
|
||||
"window" => "cw",
|
||||
_ => "cx",
|
||||
};
|
||||
|
||||
format!("{}_{}", prefix, truncated)
|
||||
}
|
||||
|
||||
@@ -181,14 +181,15 @@ impl RedisSessionAffinityStore {
|
||||
}
|
||||
|
||||
fn response_key(&self, response_id: &str) -> String {
|
||||
self.redis.key(&format!("affinity:resp:{response_id}"))
|
||||
self.redis.key(&format!("affinity:v2:resp:{response_id}"))
|
||||
}
|
||||
|
||||
fn conversation_key(&self, conversation_id: &str) -> String {
|
||||
self.redis.key(&format!("affinity:conv:{conversation_id}"))
|
||||
self.redis
|
||||
.key(&format!("affinity:v2:conv:{conversation_id}"))
|
||||
}
|
||||
|
||||
fn account_key(&self, account_id: &str) -> String {
|
||||
self.redis.key(&format!("affinity:account:{account_id}"))
|
||||
self.redis.key(&format!("affinity:v2:account:{account_id}"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::upstream::openai::protocol::responses::PreviousResponseScope;
|
||||
|
||||
/// 会话亲和性条目。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -14,12 +17,14 @@ pub struct SessionAffinityEntry {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub function_call_ids: Vec<String>,
|
||||
pub variant_hash: Option<String>,
|
||||
pub continuation_scope: PreviousResponseScope,
|
||||
pub replay: Option<ResponseReplaySnapshot>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 从请求上下文派生的 conversation identity。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ConversationIdentity {
|
||||
pub conversation_id: Option<String>,
|
||||
pub window_id: Option<String>,
|
||||
/// 截止某个 completed response 的完整、无凭据业务输入快照。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ResponseReplaySnapshot {
|
||||
pub full_input: Vec<Value>,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
//! 请求级账号候选快照与尝试账本。
|
||||
|
||||
use std::collections::{BTreeSet, VecDeque};
|
||||
|
||||
use chrono::Utc;
|
||||
|
||||
use crate::fleet::pool::{
|
||||
AccountAcquireRequest, AccountCandidateLease, AccountLease, AccountPoolService,
|
||||
};
|
||||
|
||||
/// 请求开始时冻结的完整候选集,以及候选的最终处理状态。
|
||||
pub(in crate::dispatch) struct AccountAttemptLedger {
|
||||
model: String,
|
||||
candidate_ids: Vec<String>,
|
||||
pending: VecDeque<String>,
|
||||
busy: VecDeque<String>,
|
||||
attempted: BTreeSet<String>,
|
||||
state_excluded: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl AccountAttemptLedger {
|
||||
pub(in crate::dispatch) async fn freeze(
|
||||
account_pool: &AccountPoolService,
|
||||
request: &AccountAcquireRequest,
|
||||
) -> Self {
|
||||
let candidate_ids = account_pool.candidate_snapshot(request).await;
|
||||
Self {
|
||||
model: request.model.clone(),
|
||||
pending: candidate_ids.iter().cloned().collect(),
|
||||
candidate_ids,
|
||||
busy: VecDeque::new(),
|
||||
attempted: BTreeSet::new(),
|
||||
state_excluded: BTreeSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) async fn acquire_next(
|
||||
&mut self,
|
||||
account_pool: &AccountPoolService,
|
||||
) -> Option<AccountLease> {
|
||||
loop {
|
||||
while let Some(account_id) = self.pending.pop_front() {
|
||||
match account_pool
|
||||
.acquire_candidate(&self.model, &account_id, Utc::now())
|
||||
.await
|
||||
{
|
||||
AccountCandidateLease::Acquired(lease) => {
|
||||
self.attempted.insert(account_id);
|
||||
return Some(*lease);
|
||||
}
|
||||
AccountCandidateLease::Busy => self.busy.push_back(account_id),
|
||||
AccountCandidateLease::Unavailable => {
|
||||
self.state_excluded.insert(account_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self.busy.is_empty() {
|
||||
debug_assert_eq!(
|
||||
self.candidate_ids.len(),
|
||||
self.attempted.len() + self.state_excluded.len()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
account_pool.wait_for_candidate_change().await;
|
||||
std::mem::swap(&mut self.pending, &mut self.busy);
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn candidate_count(&self) -> usize {
|
||||
self.candidate_ids.len()
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn attempted_count(&self) -> usize {
|
||||
self.attempted.len()
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn state_excluded_count(&self) -> usize {
|
||||
self.state_excluded.len()
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,8 @@ pub(crate) enum DispatchFailureClass {
|
||||
CloudflareChallenge,
|
||||
CloudflarePathBlocked,
|
||||
ModelUnsupported,
|
||||
UpstreamUnavailable,
|
||||
ContinuationBusy,
|
||||
HistoryUnavailable,
|
||||
Upstream,
|
||||
InvalidSse,
|
||||
@@ -55,6 +57,8 @@ impl DispatchFailureClass {
|
||||
Self::CloudflareChallenge => "cloudflare_challenge",
|
||||
Self::CloudflarePathBlocked => "cloudflare_path_blocked",
|
||||
Self::ModelUnsupported => "model_unsupported",
|
||||
Self::UpstreamUnavailable => "upstream_unavailable",
|
||||
Self::ContinuationBusy => "continuation_busy",
|
||||
Self::HistoryUnavailable => "history_unavailable",
|
||||
Self::Upstream => "upstream",
|
||||
Self::InvalidSse => "invalid_sse",
|
||||
@@ -121,6 +125,7 @@ impl From<ExhaustedAccountKind> for DispatchFailureClass {
|
||||
ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge,
|
||||
ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked,
|
||||
ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported,
|
||||
ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -193,12 +198,25 @@ pub(crate) fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool {
|
||||
pub(crate) fn is_retryable_upstream_5xx_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { status, body, .. }
|
||||
CodexClientError::Upstream { status, .. }
|
||||
if status_code_is_transient_upstream(status.as_u16())
|
||||
&& !is_history_recovery_signal(body)
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_retryable_account_transport_error(error: &CodexClientError) -> bool {
|
||||
match error {
|
||||
CodexClientError::Http(error) => error.is_connect() || error.is_timeout(),
|
||||
CodexClientError::WebSocket(
|
||||
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::Transport(_)
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ClosedBeforeTerminal
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput { .. }
|
||||
| crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::InitialEventTimeout { .. },
|
||||
) => true,
|
||||
_ => is_retryable_upstream_5xx_error(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_quota_exhausted_upstream_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
@@ -217,9 +235,26 @@ pub(crate) fn is_model_unsupported_upstream_error(error: &CodexClientError) -> b
|
||||
}
|
||||
|
||||
pub(crate) fn is_history_recovery_upstream_error(error: &CodexClientError) -> bool {
|
||||
upstream_error_code(error).is_some_and(|code| is_history_recovery_code(&code))
|
||||
|| matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(
|
||||
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable { reason }
|
||||
) if !matches!(
|
||||
reason,
|
||||
crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_continuation_busy_error(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { body, .. } if is_history_recovery_signal(body)
|
||||
CodexClientError::WebSocket(
|
||||
crate::upstream::openai::transport::websocket::CodexWebSocketExchangeError::ContinuationUnavailable {
|
||||
reason: crate::upstream::openai::transport::websocket::PreviousResponseUnavailableReason::ConnectionBusy,
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -250,18 +285,27 @@ pub(crate) fn is_model_unsupported_signal(value: &str) -> bool {
|
||||
|| value.contains("not_available")))
|
||||
}
|
||||
|
||||
pub(crate) fn is_history_recovery_signal(value: &str) -> bool {
|
||||
let value = value.to_ascii_lowercase();
|
||||
value.contains("previous_response_not_found")
|
||||
|| (value.contains("previous response") && value.contains("not found"))
|
||||
|| value.contains("no tool output found for function call")
|
||||
|| is_invalid_encrypted_content_signal(&value)
|
||||
pub(crate) fn is_history_recovery_code(code: &str) -> bool {
|
||||
matches!(
|
||||
code,
|
||||
"previous_response_not_found"
|
||||
| "invalid_encrypted_content"
|
||||
| "missing_tool_output"
|
||||
| "no_tool_output"
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_invalid_encrypted_content_signal(value: &str) -> bool {
|
||||
let value = value.to_ascii_lowercase();
|
||||
value.contains("invalid_encrypted_content")
|
||||
|| (value.contains("invalid") && value.contains("encrypted") && value.contains("content"))
|
||||
pub(crate) fn upstream_error_code(error: &CodexClientError) -> Option<String> {
|
||||
let CodexClientError::Upstream { body, .. } = error else {
|
||||
return None;
|
||||
};
|
||||
let value = serde_json::from_str::<Value>(body).ok()?;
|
||||
value
|
||||
.pointer("/response/error/code")
|
||||
.or_else(|| value.pointer("/error/code"))
|
||||
.or_else(|| value.get("code"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
pub(crate) fn rate_limit_cooldown_until(
|
||||
@@ -349,6 +393,13 @@ pub enum ResponseDispatchError {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("all account candidates failed with transient upstream errors")]
|
||||
UpstreamUnavailable {
|
||||
count: usize,
|
||||
upstream_error: String,
|
||||
},
|
||||
#[error("the previous response connection is busy; retry the request")]
|
||||
ContinuationBusy,
|
||||
#[error("previous response context is unavailable: {upstream_error}")]
|
||||
HistoryUnavailable { upstream_error: String },
|
||||
#[error("upstream request failed: {0}")]
|
||||
@@ -399,6 +450,10 @@ impl ResponseDispatchError {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
ExhaustedAccountKind::UpstreamUnavailable => Self::UpstreamUnavailable {
|
||||
count: exhausted.count,
|
||||
upstream_error: exhausted.upstream_error,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +469,8 @@ impl ResponseDispatchError {
|
||||
| Self::InvalidSse(_)
|
||||
| Self::MissingCompleted
|
||||
| Self::EmptyUpstreamResponse => 502,
|
||||
Self::UpstreamUnavailable { .. } => 503,
|
||||
Self::ContinuationBusy => 503,
|
||||
Self::Failed(failure) => stream_failure_http_status(failure),
|
||||
Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400,
|
||||
Self::Upstream(error) => upstream_error_http_status(error),
|
||||
@@ -481,6 +538,17 @@ impl ResponseDispatchError {
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::UpstreamUnavailable {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Self::exhausted_metadata(
|
||||
ExhaustedAccountKind::UpstreamUnavailable,
|
||||
*count,
|
||||
upstream_error,
|
||||
),
|
||||
Self::ContinuationBusy => {
|
||||
DispatchErrorMetadata::simple(DispatchFailureClass::ContinuationBusy)
|
||||
}
|
||||
Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata {
|
||||
failure_class: DispatchFailureClass::HistoryUnavailable,
|
||||
exhausted_count: None,
|
||||
@@ -573,6 +641,14 @@ impl ResponseDispatchError {
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
Self::UpstreamUnavailable {
|
||||
count,
|
||||
upstream_error,
|
||||
} => Some(ExhaustedAccountRef {
|
||||
kind: ExhaustedAccountKind::UpstreamUnavailable,
|
||||
count: *count,
|
||||
upstream_error,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 请求调度模块(账号选择、回退、恢复、会话亲和性)。
|
||||
|
||||
pub mod affinity;
|
||||
mod attempts;
|
||||
pub mod errors;
|
||||
mod recording;
|
||||
pub mod recovery;
|
||||
|
||||
@@ -479,7 +479,30 @@ pub(super) async fn record_live_response_stream_event(
|
||||
rate_limit_headers: &[(String, String)],
|
||||
body: &str,
|
||||
) {
|
||||
let effective_model = context
|
||||
.response_metadata
|
||||
.effective_model
|
||||
.as_deref()
|
||||
.unwrap_or(&context.display_model);
|
||||
ensure_stream_metadata_flag(&mut metadata);
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
object.insert(
|
||||
"effectiveModel".to_string(),
|
||||
Value::String(effective_model.to_string()),
|
||||
);
|
||||
object.insert(
|
||||
"modelsEtag".to_string(),
|
||||
context
|
||||
.response_metadata
|
||||
.models_etag
|
||||
.as_ref()
|
||||
.map_or(Value::Null, |etag| Value::String(etag.clone())),
|
||||
);
|
||||
object.insert(
|
||||
"reasoningIncluded".to_string(),
|
||||
Value::Bool(context.response_metadata.reasoning_included),
|
||||
);
|
||||
}
|
||||
enrich_event_route_metadata(&mut metadata, &context.route);
|
||||
enrich_live_response_stream_metadata(
|
||||
context,
|
||||
@@ -492,7 +515,7 @@ pub(super) async fn record_live_response_stream_event(
|
||||
enrich_usage_record_identity(
|
||||
&mut metadata,
|
||||
Some(&context.requested_model),
|
||||
&context.display_model,
|
||||
effective_model,
|
||||
context.client_ip.as_deref(),
|
||||
context.request.client_user_agent.as_deref(),
|
||||
reasoning_effort_from_request(&context.request),
|
||||
@@ -507,7 +530,7 @@ pub(super) async fn record_live_response_stream_event(
|
||||
client_api_key_id: context.request.client_api_key_id.as_deref(),
|
||||
account_id: &context.account_id,
|
||||
route: &context.route,
|
||||
model: &context.display_model,
|
||||
model: effective_model,
|
||||
requested_model: Some(&context.requested_model),
|
||||
client_ip: context.client_ip.as_deref(),
|
||||
client_user_agent: context.request.client_user_agent.as_deref(),
|
||||
@@ -528,7 +551,7 @@ pub(super) async fn record_live_response_stream_event(
|
||||
event.provider = Some("openai".to_string());
|
||||
event.account_id = Some(context.account_id.clone());
|
||||
event.route = Some(context.route.clone());
|
||||
event.model = Some(context.display_model.clone());
|
||||
event.model = Some(effective_model.to_string());
|
||||
event.status_code = Some(status_code);
|
||||
event.latency_ms = Some(elapsed_millis_i64(context.started_at));
|
||||
event.metadata = metadata;
|
||||
|
||||
@@ -12,6 +12,7 @@ pub(crate) enum ExhaustedAccountKind {
|
||||
CloudflareChallenge,
|
||||
CloudflarePathBlocked,
|
||||
ModelUnsupported,
|
||||
UpstreamUnavailable,
|
||||
}
|
||||
|
||||
impl ExhaustedAccountKind {
|
||||
@@ -25,6 +26,7 @@ impl ExhaustedAccountKind {
|
||||
Self::CloudflareChallenge => "cloudflare-challenge",
|
||||
Self::CloudflarePathBlocked => "cloudflare-path-block",
|
||||
Self::ModelUnsupported => "model-unsupported",
|
||||
Self::UpstreamUnavailable => "upstream-unavailable",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,7 +58,7 @@ pub(crate) struct AccountExhaustionTracker {
|
||||
cloudflare_challenge: ExhaustedCounter,
|
||||
cloudflare_path_blocked: ExhaustedCounter,
|
||||
model_unsupported: ExhaustedCounter,
|
||||
model_unsupported_retry_used: bool,
|
||||
upstream_unavailable: ExhaustedCounter,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -143,7 +145,6 @@ impl AccountExhaustionTracker {
|
||||
account_id: Option<&str>,
|
||||
upstream_error: impl Into<String>,
|
||||
) {
|
||||
self.model_unsupported_retry_used = true;
|
||||
self.record(
|
||||
account_id,
|
||||
ExhaustedAccountKind::ModelUnsupported,
|
||||
@@ -152,16 +153,17 @@ impl AccountExhaustionTracker {
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn model_unsupported_retry_exhausted(
|
||||
&self,
|
||||
pub(crate) fn record_upstream_unavailable(
|
||||
&mut self,
|
||||
account_id: Option<&str>,
|
||||
upstream_error: impl Into<String>,
|
||||
) -> Option<ExhaustedAccount> {
|
||||
self.model_unsupported_retry_used.then(|| ExhaustedAccount {
|
||||
kind: ExhaustedAccountKind::ModelUnsupported,
|
||||
count: self.model_unsupported.count + 1,
|
||||
upstream_error: upstream_error.into(),
|
||||
status_code: None,
|
||||
})
|
||||
) {
|
||||
self.record(
|
||||
account_id,
|
||||
ExhaustedAccountKind::UpstreamUnavailable,
|
||||
upstream_error,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn last_exhausted(&self) -> Option<ExhaustedAccount> {
|
||||
@@ -204,6 +206,7 @@ impl AccountExhaustionTracker {
|
||||
ExhaustedAccountKind::CloudflareChallenge => &self.cloudflare_challenge,
|
||||
ExhaustedAccountKind::CloudflarePathBlocked => &self.cloudflare_path_blocked,
|
||||
ExhaustedAccountKind::ModelUnsupported => &self.model_unsupported,
|
||||
ExhaustedAccountKind::UpstreamUnavailable => &self.upstream_unavailable,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,6 +220,7 @@ impl AccountExhaustionTracker {
|
||||
ExhaustedAccountKind::CloudflareChallenge => &mut self.cloudflare_challenge,
|
||||
ExhaustedAccountKind::CloudflarePathBlocked => &mut self.cloudflare_path_blocked,
|
||||
ExhaustedAccountKind::ModelUnsupported => &mut self.model_unsupported,
|
||||
ExhaustedAccountKind::UpstreamUnavailable => &mut self.upstream_unavailable,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
//! previous response 所有权与完整上下文重放。
|
||||
|
||||
use chrono::Utc;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
dispatch::affinity::{
|
||||
types::ResponseReplaySnapshot, SessionAffinityEntry, SessionAffinityService,
|
||||
},
|
||||
upstream::openai::protocol::responses::{CodexResponsesRequest, StreamCommitPolicy},
|
||||
};
|
||||
|
||||
/// 当前请求的 previous response 来源。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(in crate::dispatch) enum HistorySource {
|
||||
None,
|
||||
Managed,
|
||||
ExternalUnknown,
|
||||
}
|
||||
|
||||
/// 请求级 previous response 恢复计划。
|
||||
pub(in crate::dispatch) struct HistoryRecoveryPlan {
|
||||
source: HistorySource,
|
||||
managed_entry: Option<SessionAffinityEntry>,
|
||||
external_attempted_account_id: Option<String>,
|
||||
replay_owner_account: bool,
|
||||
}
|
||||
|
||||
impl HistoryRecoveryPlan {
|
||||
pub(in crate::dispatch) async fn load(
|
||||
session_affinity: &SessionAffinityService,
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Self {
|
||||
let Some(previous_response_id) = request.previous_response_id() else {
|
||||
return Self {
|
||||
source: HistorySource::None,
|
||||
managed_entry: None,
|
||||
external_attempted_account_id: None,
|
||||
replay_owner_account: false,
|
||||
};
|
||||
};
|
||||
let managed_entry = session_affinity
|
||||
.lookup(previous_response_id, Utc::now())
|
||||
.await;
|
||||
Self {
|
||||
source: if managed_entry.is_some() {
|
||||
HistorySource::Managed
|
||||
} else {
|
||||
HistorySource::ExternalUnknown
|
||||
},
|
||||
managed_entry,
|
||||
external_attempted_account_id: None,
|
||||
replay_owner_account: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn preferred_account_id(&self) -> Option<&str> {
|
||||
self.managed_entry
|
||||
.as_ref()
|
||||
.map(|entry| entry.account_id.as_str())
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn prepare_attempt(
|
||||
&mut self,
|
||||
original: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
) -> Option<CodexResponsesRequest> {
|
||||
match self.source {
|
||||
HistorySource::None => Some(original.clone()),
|
||||
HistorySource::ExternalUnknown => match self.external_attempted_account_id.as_deref() {
|
||||
None => {
|
||||
self.external_attempted_account_id = Some(account_id.to_string());
|
||||
let mut request = original.clone();
|
||||
request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal;
|
||||
request.previous_response_scope = Some(
|
||||
crate::upstream::openai::protocol::responses::PreviousResponseScope::ExternalUnknown,
|
||||
);
|
||||
Some(request)
|
||||
}
|
||||
Some(first_account_id) if first_account_id == account_id => {
|
||||
let mut request = original.clone();
|
||||
request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal;
|
||||
request.previous_response_scope = Some(
|
||||
crate::upstream::openai::protocol::responses::PreviousResponseScope::ExternalUnknown,
|
||||
);
|
||||
Some(request)
|
||||
}
|
||||
Some(_) => None,
|
||||
},
|
||||
HistorySource::Managed => {
|
||||
let entry = self.managed_entry.as_ref()?;
|
||||
if entry.account_id == account_id && !self.replay_owner_account {
|
||||
let mut request = original.clone();
|
||||
request.local_conversation_id = Some(entry.conversation_id.clone());
|
||||
request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal;
|
||||
request.previous_response_scope = Some(entry.continuation_scope);
|
||||
if request.turn_state.as_deref().is_none_or(str::is_empty) {
|
||||
request.turn_state.clone_from(&entry.turn_state);
|
||||
}
|
||||
Some(request)
|
||||
} else {
|
||||
entry.replay.as_ref().map(|snapshot| {
|
||||
let mut request = full_replay_request(original, snapshot);
|
||||
request.local_conversation_id = Some(entry.conversation_id.clone());
|
||||
request
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// previous ID 在原连接不可用时,切换为同账号完整重放。
|
||||
pub(in crate::dispatch) fn recover_managed_history(&mut self, account_id: &str) -> bool {
|
||||
let Some(entry) = self.managed_entry.as_ref() else {
|
||||
return false;
|
||||
};
|
||||
if entry.account_id != account_id || entry.replay.is_none() || self.replay_owner_account {
|
||||
return false;
|
||||
}
|
||||
self.replay_owner_account = true;
|
||||
true
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn can_failover(&self) -> bool {
|
||||
!matches!(self.source, HistorySource::ExternalUnknown)
|
||||
&& self
|
||||
.managed_entry
|
||||
.as_ref()
|
||||
.is_none_or(|entry| entry.replay.is_some())
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn is_external_unknown(&self) -> bool {
|
||||
self.source == HistorySource::ExternalUnknown
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn completed_replay(
|
||||
&self,
|
||||
original: &CodexResponsesRequest,
|
||||
output: &[Value],
|
||||
) -> Option<ResponseReplaySnapshot> {
|
||||
let mut full_input = match self.source {
|
||||
HistorySource::None => original.input().to_vec(),
|
||||
HistorySource::Managed => self
|
||||
.managed_entry
|
||||
.as_ref()?
|
||||
.replay
|
||||
.as_ref()?
|
||||
.full_input
|
||||
.clone(),
|
||||
HistorySource::ExternalUnknown => return None,
|
||||
};
|
||||
if matches!(self.source, HistorySource::Managed) {
|
||||
full_input.extend_from_slice(original.input());
|
||||
}
|
||||
full_input.extend_from_slice(output);
|
||||
Some(ResponseReplaySnapshot { full_input })
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn conversation_id<'a>(
|
||||
&'a self,
|
||||
original: &'a CodexResponsesRequest,
|
||||
) -> Option<&'a str> {
|
||||
self.managed_entry
|
||||
.as_ref()
|
||||
.map(|entry| entry.conversation_id.as_str())
|
||||
.or_else(|| non_empty(original.local_conversation_id.as_deref()))
|
||||
}
|
||||
}
|
||||
|
||||
fn full_replay_request(
|
||||
original: &CodexResponsesRequest,
|
||||
snapshot: &ResponseReplaySnapshot,
|
||||
) -> CodexResponsesRequest {
|
||||
let mut request = original.clone();
|
||||
let mut input = snapshot.full_input.clone();
|
||||
input.extend_from_slice(original.input());
|
||||
strip_account_bound_encrypted_content(&mut input);
|
||||
request.set_input(input);
|
||||
request.set_previous_response_id(None);
|
||||
request.turn_state = None;
|
||||
request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal;
|
||||
request
|
||||
}
|
||||
|
||||
fn strip_account_bound_encrypted_content(values: &mut [Value]) {
|
||||
for value in values {
|
||||
match value {
|
||||
Value::Array(values) => strip_account_bound_encrypted_content(values),
|
||||
Value::Object(object) => {
|
||||
object.remove("encrypted_content");
|
||||
for value in object.values_mut() {
|
||||
strip_account_bound_encrypted_content(std::slice::from_mut(value));
|
||||
}
|
||||
}
|
||||
Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn non_empty(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
@@ -1,258 +0,0 @@
|
||||
//! Responses 隐式续接策略。
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use chrono::{Duration, Utc};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::{
|
||||
compute_variant_hash, ensure_prompt_cache_key, hash_instructions,
|
||||
prepare_variant_identity,
|
||||
},
|
||||
service::ResponseDispatchService,
|
||||
},
|
||||
upstream::openai::protocol::responses::CodexResponsesRequest,
|
||||
};
|
||||
|
||||
const IMPLICIT_RESUME_MAX_AGE_SECS: i64 = 55 * 60;
|
||||
|
||||
/// 隐式续接前的可恢复请求状态。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ImplicitResumeSnapshot {
|
||||
pub input: Vec<Value>,
|
||||
pub previous_response_id: Option<String>,
|
||||
pub turn_state: Option<String>,
|
||||
pub use_websocket: bool,
|
||||
pub force_http_sse: bool,
|
||||
}
|
||||
|
||||
impl ImplicitResumeSnapshot {
|
||||
/// 捕获当前请求中隐式续接会改写的字段。
|
||||
pub fn capture(request: &CodexResponsesRequest) -> Self {
|
||||
Self {
|
||||
input: request.input().to_vec(),
|
||||
previous_response_id: request.previous_response_id().map(ToString::to_string),
|
||||
turn_state: request.turn_state.clone(),
|
||||
use_websocket: request.use_websocket,
|
||||
force_http_sse: request.force_http_sse,
|
||||
}
|
||||
}
|
||||
|
||||
/// 恢复隐式续接前的请求字段。
|
||||
pub fn restore(self, request: &mut CodexResponsesRequest) {
|
||||
request.set_input(self.input);
|
||||
request.set_previous_response_id(self.previous_response_id);
|
||||
request.turn_state = self.turn_state;
|
||||
request.use_websocket = self.use_websocket;
|
||||
request.force_http_sse = self.force_http_sse;
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponseDispatchService {
|
||||
pub(in crate::dispatch) async fn prepare_response_session(
|
||||
&self,
|
||||
request: &mut CodexResponsesRequest,
|
||||
) -> Option<ImplicitResumeSnapshot> {
|
||||
prepare_variant_identity(request);
|
||||
if let Some(previous_response_id) = request.previous_response_id().map(ToString::to_string)
|
||||
{
|
||||
if request.prompt_cache_key().is_none() {
|
||||
let conversation_id = self
|
||||
.session_affinity
|
||||
.lookup_conversation_id(&previous_response_id, Utc::now())
|
||||
.await;
|
||||
request.set_prompt_cache_key(conversation_id);
|
||||
}
|
||||
if request.turn_state.as_deref().is_none_or(str::is_empty) {
|
||||
request.turn_state = self
|
||||
.session_affinity
|
||||
.lookup_turn_state(&previous_response_id, Utc::now())
|
||||
.await;
|
||||
}
|
||||
ensure_prompt_cache_key(request);
|
||||
return None;
|
||||
}
|
||||
|
||||
ensure_prompt_cache_key(request);
|
||||
self.apply_implicit_resume(request).await
|
||||
}
|
||||
|
||||
async fn apply_implicit_resume(
|
||||
&self,
|
||||
request: &mut CodexResponsesRequest,
|
||||
) -> Option<ImplicitResumeSnapshot> {
|
||||
let continuation_start = continuation_input_start(request.input());
|
||||
if continuation_start == 0 || continuation_start >= request.input().len() {
|
||||
return None;
|
||||
}
|
||||
let conversation_id = request
|
||||
.prompt_cache_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?
|
||||
.to_string();
|
||||
let snapshot = ImplicitResumeSnapshot::capture(request);
|
||||
let variant_hash = compute_variant_hash(request);
|
||||
let now = Utc::now();
|
||||
let previous_response_id = self
|
||||
.session_affinity
|
||||
.lookup_latest_response_by_conversation(
|
||||
&conversation_id,
|
||||
Some(Duration::seconds(IMPLICIT_RESUME_MAX_AGE_SECS)),
|
||||
Some(&variant_hash),
|
||||
now,
|
||||
)
|
||||
.await?;
|
||||
let current_instructions_hash = hash_instructions(Some(request.instructions()));
|
||||
if self
|
||||
.session_affinity
|
||||
.lookup_instructions_hash(&previous_response_id, now)
|
||||
.await
|
||||
.as_deref()
|
||||
!= Some(current_instructions_hash.as_str())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let stored_function_call_ids = self
|
||||
.session_affinity
|
||||
.lookup_function_call_ids(&previous_response_id, now)
|
||||
.await;
|
||||
if !implicit_resume_allowed(
|
||||
&request.input()[continuation_start..],
|
||||
request.input(),
|
||||
&stored_function_call_ids,
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
let account_id = self
|
||||
.session_affinity
|
||||
.lookup_account(&previous_response_id, now)
|
||||
.await?;
|
||||
let replay_items = self.reasoning_replay.lock().await.lookup(
|
||||
&previous_response_id,
|
||||
&account_id,
|
||||
&conversation_id,
|
||||
&variant_hash,
|
||||
now,
|
||||
);
|
||||
let continuation = request.input()[continuation_start..].to_vec();
|
||||
let mut input = replay_items;
|
||||
input.extend(continuation);
|
||||
|
||||
request.set_previous_response_id(Some(previous_response_id.clone()));
|
||||
request.use_websocket = true;
|
||||
request.force_http_sse = false;
|
||||
request.set_input(input);
|
||||
if let Some(turn_state) = self
|
||||
.session_affinity
|
||||
.lookup_turn_state(&previous_response_id, now)
|
||||
.await
|
||||
{
|
||||
request.turn_state = Some(turn_state);
|
||||
}
|
||||
|
||||
Some(snapshot)
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) async fn try_recover_implicit_resume(
|
||||
&self,
|
||||
request: &mut CodexResponsesRequest,
|
||||
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
||||
account_id: &str,
|
||||
evict_reasoning_replay: bool,
|
||||
) -> bool {
|
||||
let Some(snapshot) = implicit_resume.take() else {
|
||||
return false;
|
||||
};
|
||||
if evict_reasoning_replay {
|
||||
self.evict_reasoning_replay(request, account_id).await;
|
||||
}
|
||||
if let Some(previous_response_id) = request.previous_response_id() {
|
||||
self.session_affinity.forget(previous_response_id).await;
|
||||
}
|
||||
restore_full_request_without_history(request, snapshot);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回续接输入在完整输入中的起始位置。
|
||||
pub fn continuation_input_start(input: &[Value]) -> usize {
|
||||
let mut last_model_output_index = None;
|
||||
for (index, item) in input.iter().enumerate() {
|
||||
if item.get("role").is_some() {
|
||||
if item.get("role").and_then(Value::as_str) == Some("assistant") {
|
||||
last_model_output_index = Some(index);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if item.get("type").and_then(Value::as_str) == Some("function_call") {
|
||||
last_model_output_index = Some(index);
|
||||
}
|
||||
}
|
||||
last_model_output_index.map_or(0, |index| index.saturating_add(1))
|
||||
}
|
||||
|
||||
/// 判断完整输入历史是否可以隐式续接到已记录响应。
|
||||
pub fn implicit_resume_allowed(
|
||||
continuation_input: &[Value],
|
||||
full_input: &[Value],
|
||||
stored_function_call_ids: &[String],
|
||||
) -> bool {
|
||||
let required_call_ids = function_call_output_ids(continuation_input);
|
||||
if required_call_ids.is_empty() {
|
||||
return stored_function_call_ids.is_empty();
|
||||
}
|
||||
|
||||
let inline_call_ids = inline_function_call_ids(full_input);
|
||||
if required_call_ids
|
||||
.iter()
|
||||
.all(|call_id| inline_call_ids.contains(call_id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let stored_call_ids = stored_function_call_ids.iter().collect::<HashSet<_>>();
|
||||
let required_call_ids = required_call_ids.iter().collect::<HashSet<_>>();
|
||||
required_call_ids
|
||||
.iter()
|
||||
.all(|call_id| stored_call_ids.contains(*call_id))
|
||||
&& stored_call_ids
|
||||
.iter()
|
||||
.all(|call_id| required_call_ids.contains(*call_id))
|
||||
}
|
||||
|
||||
fn strip_request_history(request: &mut CodexResponsesRequest) {
|
||||
request.set_previous_response_id(None);
|
||||
request.turn_state = None;
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn restore_full_request_without_history(
|
||||
request: &mut CodexResponsesRequest,
|
||||
snapshot: ImplicitResumeSnapshot,
|
||||
) {
|
||||
snapshot.restore(request);
|
||||
strip_request_history(request);
|
||||
}
|
||||
|
||||
fn function_call_output_ids(input: &[Value]) -> Vec<String> {
|
||||
input
|
||||
.iter()
|
||||
.filter(|item| item.get("role").is_none())
|
||||
.filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call_output"))
|
||||
.filter_map(|item| item.get("call_id").and_then(Value::as_str))
|
||||
.filter(|call_id| !call_id.trim().is_empty())
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn inline_function_call_ids(input: &[Value]) -> HashSet<String> {
|
||||
input
|
||||
.iter()
|
||||
.filter(|item| item.get("role").is_none())
|
||||
.filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call"))
|
||||
.filter_map(|item| item.get("call_id").and_then(Value::as_str))
|
||||
.filter(|call_id| !call_id.trim().is_empty())
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
}
|
||||
@@ -3,5 +3,4 @@
|
||||
pub(crate) mod auth;
|
||||
pub mod cloudflare;
|
||||
pub(crate) mod exhaustion;
|
||||
pub mod implicit_resume;
|
||||
pub mod reasoning_replay;
|
||||
pub(crate) mod history;
|
||||
|
||||
@@ -1,274 +0,0 @@
|
||||
//! Responses reasoning replay 缓存策略。
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
const MAX_ENTRIES: usize = 512;
|
||||
const MAX_ENTRY_BYTES: usize = 256 * 1024;
|
||||
const MAX_TOTAL_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// 单条 reasoning replay 缓存记录。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct ReasoningReplayEntry {
|
||||
account_id: String,
|
||||
conversation_id: String,
|
||||
variant_hash: String,
|
||||
items: Vec<Value>,
|
||||
byte_size: usize,
|
||||
created_at: DateTime<Utc>,
|
||||
sequence: u64,
|
||||
}
|
||||
|
||||
/// 纯内存 reasoning replay 缓存。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReasoningReplayCache {
|
||||
entries: BTreeMap<String, ReasoningReplayEntry>,
|
||||
total_bytes: usize,
|
||||
next_sequence: u64,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl ReasoningReplayCache {
|
||||
pub fn new(ttl: Duration) -> Self {
|
||||
Self {
|
||||
entries: BTreeMap::new(),
|
||||
total_bytes: 0,
|
||||
next_sequence: 0,
|
||||
ttl,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record(
|
||||
&mut self,
|
||||
response_id: String,
|
||||
account_id: &str,
|
||||
conversation_id: &str,
|
||||
variant_hash: &str,
|
||||
items: &[Value],
|
||||
now: DateTime<Utc>,
|
||||
) -> usize {
|
||||
let sanitized = sanitize_reasoning_replay_items(items);
|
||||
self.cleanup_expired(now);
|
||||
self.delete_entry(&response_id);
|
||||
|
||||
if sanitized.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let byte_size = estimate_items_bytes(&sanitized);
|
||||
if byte_size > MAX_ENTRY_BYTES {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let sequence = self.next_sequence;
|
||||
self.next_sequence = self.next_sequence.saturating_add(1);
|
||||
self.total_bytes = self.total_bytes.saturating_add(byte_size);
|
||||
self.entries.insert(
|
||||
response_id,
|
||||
ReasoningReplayEntry {
|
||||
account_id: account_id.to_string(),
|
||||
conversation_id: conversation_id.to_string(),
|
||||
variant_hash: variant_hash.to_string(),
|
||||
items: sanitized,
|
||||
byte_size,
|
||||
created_at: now,
|
||||
sequence,
|
||||
},
|
||||
);
|
||||
self.evict_overflow();
|
||||
self.entries
|
||||
.values()
|
||||
.max_by_key(|entry| entry.sequence)
|
||||
.map_or(0, |entry| entry.items.len())
|
||||
}
|
||||
|
||||
pub fn lookup(
|
||||
&mut self,
|
||||
response_id: &str,
|
||||
account_id: &str,
|
||||
conversation_id: &str,
|
||||
variant_hash: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> Vec<Value> {
|
||||
self.cleanup_expired(now);
|
||||
let Some(entry) = self.entries.get(response_id) else {
|
||||
return Vec::new();
|
||||
};
|
||||
if entry.account_id != account_id
|
||||
|| entry.conversation_id != conversation_id
|
||||
|| entry.variant_hash != variant_hash
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
entry.items.clone()
|
||||
}
|
||||
|
||||
pub fn evict_by_identity(
|
||||
&mut self,
|
||||
account_id: &str,
|
||||
conversation_id: &str,
|
||||
variant_hash: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> usize {
|
||||
self.cleanup_expired(now);
|
||||
let response_ids = self
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| {
|
||||
entry.account_id == account_id
|
||||
&& entry.conversation_id == conversation_id
|
||||
&& entry.variant_hash == variant_hash
|
||||
})
|
||||
.map(|(response_id, _)| response_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let evicted = response_ids.len();
|
||||
for response_id in response_ids {
|
||||
self.delete_entry(&response_id);
|
||||
}
|
||||
evicted
|
||||
}
|
||||
|
||||
fn cleanup_expired(&mut self, now: DateTime<Utc>) {
|
||||
let ttl = self.ttl;
|
||||
let response_ids = self
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry_expired(entry.created_at, ttl, now))
|
||||
.map(|(response_id, _)| response_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
for response_id in response_ids {
|
||||
self.delete_entry(&response_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn evict_overflow(&mut self) {
|
||||
while self.entries.len() > MAX_ENTRIES || self.total_bytes > MAX_TOTAL_BYTES {
|
||||
let Some(response_id) = self.oldest_response_id() else {
|
||||
return;
|
||||
};
|
||||
self.delete_entry(&response_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn oldest_response_id(&self) -> Option<String> {
|
||||
self.entries
|
||||
.iter()
|
||||
.min_by_key(|(_, entry)| (entry.created_at, entry.sequence))
|
||||
.map(|(response_id, _)| response_id.clone())
|
||||
}
|
||||
|
||||
fn delete_entry(&mut self, response_id: &str) {
|
||||
let Some(entry) = self.entries.remove(response_id) else {
|
||||
return;
|
||||
};
|
||||
self.total_bytes = self.total_bytes.saturating_sub(entry.byte_size);
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_reasoning_replay_items(items: &[Value]) -> Vec<Value> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut sanitized = Vec::new();
|
||||
for item in items {
|
||||
let Some(replay_item) =
|
||||
sanitize_reasoning_item(item).or_else(|| sanitize_function_call(item))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some(key) = replay_item_key(&replay_item) else {
|
||||
continue;
|
||||
};
|
||||
if !seen.insert(key) {
|
||||
continue;
|
||||
}
|
||||
sanitized.push(replay_item);
|
||||
}
|
||||
sanitized
|
||||
}
|
||||
|
||||
fn sanitize_reasoning_item(value: &Value) -> Option<Value> {
|
||||
let object = value.as_object()?;
|
||||
if object.get("type").and_then(Value::as_str) != Some("reasoning") {
|
||||
return None;
|
||||
}
|
||||
let id = non_empty_str(object.get("id")?)?;
|
||||
let summary = sanitize_summary(object.get("summary")?)?;
|
||||
let encrypted_content = non_empty_str(object.get("encrypted_content")?)?;
|
||||
|
||||
let mut sanitized = Map::new();
|
||||
sanitized.insert("type".to_string(), json!("reasoning"));
|
||||
sanitized.insert("id".to_string(), json!(id));
|
||||
if let Some(status) = object
|
||||
.get("status")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|status| matches!(*status, "in_progress" | "completed" | "incomplete"))
|
||||
{
|
||||
sanitized.insert("status".to_string(), json!(status));
|
||||
}
|
||||
sanitized.insert("summary".to_string(), Value::Array(summary));
|
||||
sanitized.insert("encrypted_content".to_string(), json!(encrypted_content));
|
||||
Some(Value::Object(sanitized))
|
||||
}
|
||||
|
||||
fn sanitize_function_call(value: &Value) -> Option<Value> {
|
||||
let object = value.as_object()?;
|
||||
if object.get("type").and_then(Value::as_str) != Some("function_call") {
|
||||
return None;
|
||||
}
|
||||
let call_id = non_empty_str(object.get("call_id")?)?;
|
||||
let name = non_empty_str(object.get("name")?)?;
|
||||
let arguments = object.get("arguments")?.as_str()?;
|
||||
|
||||
let mut sanitized = Map::new();
|
||||
sanitized.insert("type".to_string(), json!("function_call"));
|
||||
if let Some(id) = object.get("id").and_then(non_empty_str) {
|
||||
sanitized.insert("id".to_string(), json!(id));
|
||||
}
|
||||
sanitized.insert("call_id".to_string(), json!(call_id));
|
||||
sanitized.insert("name".to_string(), json!(name));
|
||||
sanitized.insert("arguments".to_string(), json!(arguments));
|
||||
Some(Value::Object(sanitized))
|
||||
}
|
||||
|
||||
fn sanitize_summary(value: &Value) -> Option<Vec<Value>> {
|
||||
Some(
|
||||
value
|
||||
.as_array()?
|
||||
.iter()
|
||||
.filter_map(|part| {
|
||||
let object = part.as_object()?;
|
||||
if object.get("type").and_then(Value::as_str) != Some("summary_text") {
|
||||
return None;
|
||||
}
|
||||
let text = object.get("text")?.as_str()?;
|
||||
Some(json!({"type": "summary_text", "text": text}))
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn replay_item_key(item: &Value) -> Option<String> {
|
||||
match item.get("type").and_then(Value::as_str)? {
|
||||
"reasoning" => item
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.map(|id| format!("reasoning:{id}")),
|
||||
"function_call" => item
|
||||
.get("call_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(|call_id| format!("function_call:{call_id}")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn non_empty_str(value: &Value) -> Option<&str> {
|
||||
value.as_str().filter(|value| !value.trim().is_empty())
|
||||
}
|
||||
|
||||
fn estimate_items_bytes(items: &[Value]) -> usize {
|
||||
serde_json::to_vec(items).map_or(usize::MAX, |bytes| bytes.len())
|
||||
}
|
||||
|
||||
fn entry_expired(created_at: DateTime<Utc>, ttl: Duration, now: DateTime<Utc>) -> bool {
|
||||
now.signed_duration_since(created_at) >= ttl
|
||||
}
|
||||
+165
-205
@@ -10,20 +10,21 @@
|
||||
use std::{pin::Pin, sync::Arc, time::Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use chrono::{Duration, Utc};
|
||||
use chrono::Utc;
|
||||
use futures::stream::Stream;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::resolve::{evict_reasoning_replay, record_response_affinity},
|
||||
affinity::SessionAffinityService,
|
||||
affinity::resolve::{prepare_variant_identity, record_response_affinity},
|
||||
affinity::{AccountIdentityService, SessionAffinityService},
|
||||
attempts::AccountAttemptLedger,
|
||||
errors::{
|
||||
backend_transport_name, is_history_recovery_upstream_error,
|
||||
backend_transport_name, is_continuation_busy_error, is_history_recovery_upstream_error,
|
||||
is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error,
|
||||
is_rate_limit_upstream_error, rate_limit_cooldown_until, upstream_error_body,
|
||||
upstream_error_http_status, upstream_error_set_cookie_headers,
|
||||
is_rate_limit_upstream_error, is_retryable_account_transport_error,
|
||||
rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status,
|
||||
upstream_error_set_cookie_headers,
|
||||
},
|
||||
recovery::{
|
||||
auth::{auth_failure_account_status, is_auth_upstream_error},
|
||||
@@ -33,11 +34,11 @@ use crate::{
|
||||
CloudflareRecovery,
|
||||
},
|
||||
exhaustion::AccountExhaustionTracker,
|
||||
reasoning_replay::ReasoningReplayCache,
|
||||
history::HistoryRecoveryPlan,
|
||||
},
|
||||
upstream_call::{
|
||||
create_response_with_account_retrying_5xx, verify_acquired_quota_if_required,
|
||||
QuotaVerificationContext, QuotaVerificationDecision,
|
||||
AccountUpstreamContext, QuotaVerificationContext, QuotaVerificationDecision,
|
||||
QUOTA_VERIFY_LIMIT_REACHED_MESSAGE,
|
||||
},
|
||||
},
|
||||
@@ -73,10 +74,9 @@ use super::{
|
||||
},
|
||||
stream::{
|
||||
sse_failure::{
|
||||
auth_sse_failure_account_status, client_error_invalid_reasoning_replay,
|
||||
is_auth_sse_failure, is_history_recovery_sse_failure, is_model_unsupported_sse_failure,
|
||||
is_quota_exhausted_sse_failure, sse_failure_error_body,
|
||||
sse_failure_invalid_reasoning_replay, stream_failure_http_status,
|
||||
auth_sse_failure_account_status, is_auth_sse_failure, is_history_recovery_sse_failure,
|
||||
is_model_unsupported_sse_failure, is_quota_exhausted_sse_failure,
|
||||
sse_failure_error_body, stream_failure_http_status,
|
||||
},
|
||||
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
|
||||
},
|
||||
@@ -89,9 +89,8 @@ pub struct ResponseDispatchService {
|
||||
pub(in crate::dispatch) models: Arc<ModelService>,
|
||||
pub(in crate::dispatch) codex: Arc<CodexBackendClient>,
|
||||
pub(in crate::dispatch) session_affinity: Arc<SessionAffinityService>,
|
||||
pub(in crate::dispatch) reasoning_replay: Arc<Mutex<ReasoningReplayCache>>,
|
||||
pub(in crate::dispatch) account_identity: Arc<AccountIdentityService>,
|
||||
pub(in crate::dispatch) recorder: Arc<Recorder>,
|
||||
pub(in crate::dispatch) installation_id: Option<String>,
|
||||
pub(in crate::dispatch) cloudflare: CloudflareRecovery,
|
||||
}
|
||||
|
||||
@@ -100,13 +99,11 @@ pub(crate) struct ResponseDispatchServiceParts {
|
||||
pub models: Arc<ModelService>,
|
||||
pub codex: Arc<CodexBackendClient>,
|
||||
pub session_affinity: Arc<SessionAffinityService>,
|
||||
pub account_identity: Arc<AccountIdentityService>,
|
||||
pub recorder: Arc<Recorder>,
|
||||
pub installation_id: Option<String>,
|
||||
pub cloudflare: CloudflareRecovery,
|
||||
}
|
||||
|
||||
/// 默认 reasoning replay TTL 秒数。
|
||||
const DEFAULT_REASONING_REPLAY_TTL_SECS: i64 = 55 * 60;
|
||||
/// Responses live SSE 响应体流。
|
||||
pub type ResponseBodyStream =
|
||||
Pin<Box<dyn Stream<Item = Result<Bytes, ResponseDispatchStreamError>> + Send + 'static>>;
|
||||
@@ -114,6 +111,13 @@ pub type ResponseBodyStream =
|
||||
/// Responses live SSE 调度结果。
|
||||
pub struct ResponseDispatchStream {
|
||||
pub body: ResponseBodyStream,
|
||||
pub response_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Responses 非流式调度结果。
|
||||
pub struct ResponseDispatchResponse {
|
||||
pub body: Value,
|
||||
pub response_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl ResponseDispatchService {
|
||||
@@ -123,26 +127,16 @@ impl ResponseDispatchService {
|
||||
models: parts.models,
|
||||
codex: parts.codex,
|
||||
session_affinity: parts.session_affinity,
|
||||
reasoning_replay: Arc::new(Mutex::new(ReasoningReplayCache::new(Duration::seconds(
|
||||
DEFAULT_REASONING_REPLAY_TTL_SECS,
|
||||
)))),
|
||||
account_identity: parts.account_identity,
|
||||
recorder: parts.recorder,
|
||||
installation_id: parts.installation_id,
|
||||
cloudflare: parts.cloudflare,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) async fn evict_reasoning_replay(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
) {
|
||||
evict_reasoning_replay(&self.reasoning_replay, request, account_id).await;
|
||||
}
|
||||
|
||||
async fn record_response_affinity(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
history: &HistoryRecoveryPlan,
|
||||
original_request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
body: &str,
|
||||
turn_state: Option<String>,
|
||||
@@ -150,8 +144,8 @@ impl ResponseDispatchService {
|
||||
) {
|
||||
record_response_affinity(
|
||||
&self.session_affinity,
|
||||
&self.reasoning_replay,
|
||||
request,
|
||||
history,
|
||||
original_request,
|
||||
account_id,
|
||||
body,
|
||||
turn_state,
|
||||
@@ -169,7 +163,7 @@ impl ResponseDispatchService {
|
||||
route: &str,
|
||||
mut request: CodexResponsesRequest,
|
||||
requested_model: &str,
|
||||
) -> Result<Value, ResponseDispatchError> {
|
||||
) -> Result<ResponseDispatchResponse, ResponseDispatchError> {
|
||||
let started_at = Instant::now();
|
||||
let catalog = self.models.catalog().await;
|
||||
let display_model = catalog.resolve_model_id(requested_model);
|
||||
@@ -177,40 +171,44 @@ impl ResponseDispatchService {
|
||||
let tuple_schema = request.tuple_schema.clone();
|
||||
let image_generation_requested = request.expects_image_generation();
|
||||
let now = Utc::now();
|
||||
let mut implicit_resume = self.prepare_response_session(&mut request).await;
|
||||
let account_affinity = self.account_affinity_for_request(&request, now).await;
|
||||
prepare_variant_identity(&mut request);
|
||||
self.account_identity.prepare_local_identity(&mut request);
|
||||
let mut history = HistoryRecoveryPlan::load(&self.session_affinity, &request).await;
|
||||
let preferred_account_id = self
|
||||
.preferred_account_id_for_request(&request, &history, now)
|
||||
.await;
|
||||
let mut acquire_request = AccountAcquireRequest::new(request.model(), now);
|
||||
if let Some(preferred_account_id) = account_affinity.preferred_account_id() {
|
||||
if let Some(preferred_account_id) = preferred_account_id {
|
||||
acquire_request = acquire_request.with_preferred_account_id(preferred_account_id);
|
||||
}
|
||||
let mut excluded_account_ids = Vec::new();
|
||||
let mut candidates =
|
||||
AccountAttemptLedger::freeze(&self.account_pool, &acquire_request).await;
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut next_required_account_id: Option<String> = None;
|
||||
let mut empty_response_retries = 0u8;
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
const MAX_EMPTY_RESPONSE_RETRIES: u8 = 2;
|
||||
let (account, response, collected_response, attempt): (
|
||||
Account,
|
||||
CodexBackendResponse,
|
||||
CollectedResponse,
|
||||
ResponseDispatchAttempt,
|
||||
) = loop {
|
||||
let mut attempt_acquire_request = acquire_request
|
||||
.clone()
|
||||
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
||||
if let Some(account_id) = next_required_account_id.take() {
|
||||
attempt_acquire_request =
|
||||
attempt_acquire_request.with_required_account_id(account_id);
|
||||
}
|
||||
let allow_quota_verify_account_retry =
|
||||
attempt_acquire_request.required_account_id.is_none();
|
||||
attempt_acquire_request.now = Utc::now();
|
||||
let Some(acquired) = self
|
||||
.account_pool
|
||||
.acquire_with(&attempt_acquire_request)
|
||||
.await
|
||||
else {
|
||||
let acquired = if let Some(account_id) = next_required_account_id.take() {
|
||||
self.account_pool
|
||||
.acquire_with(
|
||||
&AccountAcquireRequest::new(request.model(), Utc::now())
|
||||
.with_required_account_id(account_id),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
candidates.acquire_next(&self.account_pool).await
|
||||
};
|
||||
let Some(acquired) = acquired else {
|
||||
tracing::info!(
|
||||
candidate_count = candidates.candidate_count(),
|
||||
attempted = candidates.attempted_count(),
|
||||
state_excluded = candidates.state_excluded_count(),
|
||||
"Responses account candidate ledger exhausted"
|
||||
);
|
||||
let error = exhausted_accounts
|
||||
.last_exhausted()
|
||||
.map(ResponseDispatchError::from_exhausted_account)
|
||||
@@ -235,18 +233,14 @@ impl ResponseDispatchService {
|
||||
return Err(error);
|
||||
};
|
||||
let acquired_account_id = acquired.account.id.clone();
|
||||
|
||||
// 配额验证
|
||||
let acquired = match verify_acquired_quota_if_required(
|
||||
QuotaVerificationContext {
|
||||
account_pool: self.account_pool.as_ref(),
|
||||
codex: self.codex.as_ref(),
|
||||
cloudflare: &self.cloudflare,
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
account_identity: &self.account_identity,
|
||||
request_id,
|
||||
excluded_account_ids: &mut excluded_account_ids,
|
||||
verify_attempts: &mut quota_verify_attempts,
|
||||
allow_retry_with_another_account: allow_quota_verify_account_retry,
|
||||
},
|
||||
acquired,
|
||||
)
|
||||
@@ -254,65 +248,35 @@ impl ResponseDispatchService {
|
||||
{
|
||||
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
||||
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
||||
exhausted_accounts
|
||||
.record_rate_limited(None, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE);
|
||||
continue;
|
||||
}
|
||||
QuotaVerificationDecision::MaxAttemptsReached
|
||||
| QuotaVerificationDecision::RequiredAccountUnavailable => {
|
||||
exhausted_accounts.record_rate_limited(
|
||||
Some(&acquired_account_id),
|
||||
QUOTA_VERIFY_LIMIT_REACHED_MESSAGE,
|
||||
);
|
||||
let error = exhausted_accounts
|
||||
.last_exhausted()
|
||||
.map(ResponseDispatchError::from_exhausted_account)
|
||||
.unwrap_or(ResponseDispatchError::NoActiveAccount);
|
||||
self.record_response_dispatch_error(
|
||||
request_id,
|
||||
route,
|
||||
requested_model,
|
||||
started_at,
|
||||
ResponseDispatchErrorDetails {
|
||||
client_api_key_id: request.client_api_key_id.as_deref(),
|
||||
account_id: Some(&acquired_account_id),
|
||||
stream: false,
|
||||
compact: false,
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&request),
|
||||
)),
|
||||
},
|
||||
&error,
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
self.apply_cascading_ban_defense(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
account_affinity.preferred_account_id(),
|
||||
&acquired.account.id,
|
||||
)
|
||||
.await;
|
||||
Self::strip_history_if_account_changed(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&account_affinity,
|
||||
&acquired.account.id,
|
||||
);
|
||||
let Some(attempt_request) = history.prepare_attempt(&request, &acquired.account.id)
|
||||
else {
|
||||
acquired.release_without_usage().await;
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: "previous response history cannot be sent to another account"
|
||||
.to_string(),
|
||||
});
|
||||
};
|
||||
self.account_pool.wait_for_request_interval(&acquired).await;
|
||||
let account = acquired.account.clone();
|
||||
let release_account_id = account.id.clone();
|
||||
let attempt = trace.start_attempt(&release_account_id);
|
||||
let response_result = create_response_with_account_retrying_5xx(
|
||||
&self.codex,
|
||||
self.installation_id.as_deref(),
|
||||
&self.cloudflare,
|
||||
&request,
|
||||
request_id,
|
||||
&account,
|
||||
AccountUpstreamContext {
|
||||
codex: &self.codex,
|
||||
account_identity: &self.account_identity,
|
||||
cloudflare: &self.cloudflare,
|
||||
request_id,
|
||||
account: &account,
|
||||
},
|
||||
&attempt_request,
|
||||
started_at,
|
||||
)
|
||||
.await;
|
||||
@@ -368,54 +332,31 @@ impl ResponseDispatchService {
|
||||
image_generation_requested,
|
||||
)
|
||||
.await;
|
||||
empty_response_retries += 1;
|
||||
if empty_response_retries <= MAX_EMPTY_RESPONSE_RETRIES {
|
||||
continue;
|
||||
exhausted_accounts.record_upstream_unavailable(
|
||||
Some(&release_account_id),
|
||||
"upstream response did not include visible output",
|
||||
);
|
||||
if !history.can_failover() {
|
||||
break (account, response, collected_response, attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let CollectedResponse::Failed(failure) = &collected_response {
|
||||
if is_history_recovery_sse_failure(failure)
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
sse_failure_invalid_reasoning_replay(failure),
|
||||
)
|
||||
.await
|
||||
&& history.recover_managed_history(&release_account_id)
|
||||
{
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
if is_model_unsupported_sse_failure(failure) {
|
||||
let upstream_error = sse_failure_error_body(failure);
|
||||
if let Some(exhausted) = exhausted_accounts
|
||||
.model_unsupported_retry_exhausted(upstream_error.clone())
|
||||
{
|
||||
let error =
|
||||
ResponseDispatchError::from_exhausted_account(exhausted);
|
||||
self.record_response_dispatch_error(
|
||||
request_id,
|
||||
route,
|
||||
requested_model,
|
||||
started_at,
|
||||
ResponseDispatchErrorDetails {
|
||||
client_api_key_id: request.client_api_key_id.as_deref(),
|
||||
account_id: Some(&release_account_id),
|
||||
stream: false,
|
||||
compact: false,
|
||||
transport: Some(backend_transport_name(response.transport)),
|
||||
},
|
||||
&error,
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
exhausted_accounts.record_model_unsupported(
|
||||
Some(&release_account_id),
|
||||
upstream_error,
|
||||
);
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
break (account, response, collected_response, attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_quota_exhausted_sse_failure(failure) {
|
||||
@@ -426,7 +367,9 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
break (account, response, collected_response, attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_auth_sse_failure(failure) {
|
||||
@@ -441,7 +384,9 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, account_status)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
break (account, response, collected_response, attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -456,7 +401,9 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
||||
exhausted_accounts.record_quota_exhausted(
|
||||
@@ -466,7 +413,9 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) if is_auth_upstream_error(&error) => {
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
@@ -480,7 +429,9 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, account_status)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
||||
exhausted_accounts.record_cloudflare_challenge(
|
||||
@@ -490,7 +441,9 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
||||
exhausted_accounts.record_cloudflare_path_blocked(
|
||||
@@ -500,36 +453,17 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
if let Some(exhausted) =
|
||||
exhausted_accounts.model_unsupported_retry_exhausted(upstream_error.clone())
|
||||
{
|
||||
let error = ResponseDispatchError::from_exhausted_account(exhausted);
|
||||
self.record_response_dispatch_error(
|
||||
request_id,
|
||||
route,
|
||||
requested_model,
|
||||
started_at,
|
||||
ResponseDispatchErrorDetails {
|
||||
client_api_key_id: request.client_api_key_id.as_deref(),
|
||||
account_id: Some(&release_account_id),
|
||||
stream: false,
|
||||
compact: false,
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&request),
|
||||
)),
|
||||
},
|
||||
&error,
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
exhausted_accounts
|
||||
.record_model_unsupported(Some(&release_account_id), upstream_error);
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) if is_banned_upstream_error(&error) => {
|
||||
exhausted_accounts.record_auth_failure(
|
||||
@@ -541,20 +475,16 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::Banned)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
if is_continuation_busy_error(&error) {
|
||||
return Err(ResponseDispatchError::ContinuationBusy);
|
||||
}
|
||||
let history_unavailable = is_history_recovery_upstream_error(&error);
|
||||
if history_unavailable
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
client_error_invalid_reasoning_replay(&error),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if history_unavailable && history.recover_managed_history(&release_account_id) {
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
@@ -567,14 +497,27 @@ impl ResponseDispatchService {
|
||||
model: requested_model,
|
||||
started_at,
|
||||
stream: false,
|
||||
transport: backend_transport_for_response_request(&request),
|
||||
request: &request,
|
||||
transport: backend_transport_for_response_request(&attempt_request),
|
||||
request: &attempt_request,
|
||||
error: &error,
|
||||
trace: &trace,
|
||||
attempt: Some(&attempt),
|
||||
})
|
||||
.await;
|
||||
if is_retryable_account_transport_error(&error) {
|
||||
if !history.can_failover() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
exhausted_accounts.record_upstream_unavailable(
|
||||
Some(&release_account_id),
|
||||
upstream_error_body(&error),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if history_unavailable {
|
||||
if history.is_external_unknown() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: upstream_error_body(&error),
|
||||
});
|
||||
@@ -584,8 +527,9 @@ impl ResponseDispatchService {
|
||||
}
|
||||
};
|
||||
|
||||
let completed = matches!(&collected_response, CollectedResponse::Completed(_));
|
||||
match collected_response {
|
||||
CollectedResponse::Completed(body) => {
|
||||
CollectedResponse::Completed(body) | CollectedResponse::Incomplete(body) => {
|
||||
let response_id = body.get("id").and_then(Value::as_str);
|
||||
self.cloudflare.reset_account_recovery(&account.id).await;
|
||||
if let Some(usage) = response.usage {
|
||||
@@ -598,20 +542,33 @@ impl ResponseDispatchService {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.record_response_affinity(
|
||||
&request,
|
||||
&account.id,
|
||||
&response.body,
|
||||
response.turn_state.clone(),
|
||||
response.usage,
|
||||
)
|
||||
.await;
|
||||
if completed {
|
||||
self.record_response_affinity(
|
||||
&history,
|
||||
&request,
|
||||
&account.id,
|
||||
&response.body,
|
||||
response.turn_state.clone(),
|
||||
response.usage,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let effective_model = response
|
||||
.response_metadata
|
||||
.effective_model
|
||||
.as_deref()
|
||||
.unwrap_or(&display_model);
|
||||
let mut metadata = json!({
|
||||
"responseId": response_id,
|
||||
"stream": false,
|
||||
"completed": completed,
|
||||
"incomplete": !completed,
|
||||
"transport": backend_transport_name(response.transport),
|
||||
"firstTokenMs": response.first_token_ms,
|
||||
"usage": response.usage,
|
||||
"effectiveModel": effective_model,
|
||||
"modelsEtag": response.response_metadata.models_etag.as_deref(),
|
||||
"reasoningIncluded": response.response_metadata.reasoning_included,
|
||||
});
|
||||
insert_response_status_metadata(
|
||||
&mut metadata,
|
||||
@@ -628,7 +585,7 @@ impl ResponseDispatchService {
|
||||
client_api_key_id: request.client_api_key_id.as_deref(),
|
||||
account_id: &account.id,
|
||||
route,
|
||||
model: &display_model,
|
||||
model: effective_model,
|
||||
requested_model: Some(requested_model),
|
||||
client_ip: request.client_ip.as_deref(),
|
||||
client_user_agent: request.client_user_agent.as_deref(),
|
||||
@@ -636,21 +593,24 @@ impl ResponseDispatchService {
|
||||
service_tier: request.service_tier(),
|
||||
started_at,
|
||||
status_code: 200,
|
||||
message: "v1 responses completed",
|
||||
message: if completed {
|
||||
"v1 responses completed"
|
||||
} else {
|
||||
"v1 responses incomplete"
|
||||
},
|
||||
metadata,
|
||||
rate_limit_headers: &response.rate_limit_headers,
|
||||
})
|
||||
.await;
|
||||
Ok(body)
|
||||
self.models
|
||||
.observe_models_etag(response.response_metadata.models_etag.as_deref());
|
||||
Ok(ResponseDispatchResponse {
|
||||
body,
|
||||
response_headers: response.response_metadata.client_headers,
|
||||
})
|
||||
}
|
||||
CollectedResponse::Failed(failure) => {
|
||||
let error = if is_history_recovery_sse_failure(&failure) {
|
||||
ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: sse_failure_error_body(&failure),
|
||||
}
|
||||
} else {
|
||||
ResponseDispatchError::Failed(failure)
|
||||
};
|
||||
let error = ResponseDispatchError::Failed(failure);
|
||||
self.record_response_dispatch_error(
|
||||
request_id,
|
||||
route,
|
||||
|
||||
@@ -4,11 +4,14 @@ use chrono::Utc;
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::prepare_variant_identity,
|
||||
attempts::AccountAttemptLedger,
|
||||
errors::{
|
||||
backend_transport_name, is_history_recovery_upstream_error,
|
||||
backend_transport_name, is_continuation_busy_error, is_history_recovery_upstream_error,
|
||||
is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error,
|
||||
is_rate_limit_upstream_error, rate_limit_cooldown_until, upstream_error_body,
|
||||
upstream_error_http_status, upstream_error_set_cookie_headers, ResponseDispatchError,
|
||||
is_rate_limit_upstream_error, is_retryable_account_transport_error,
|
||||
rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status,
|
||||
upstream_error_set_cookie_headers, ResponseDispatchError,
|
||||
},
|
||||
recording::{
|
||||
record_prefetched_response_stream_failure_event, record_response_upstream_error_event,
|
||||
@@ -22,23 +25,22 @@ use crate::{
|
||||
is_cloudflare_challenge_upstream_error, is_cloudflare_path_block_upstream_error,
|
||||
},
|
||||
exhaustion::AccountExhaustionTracker,
|
||||
history::HistoryRecoveryPlan,
|
||||
},
|
||||
service::{ResponseDispatchService, ResponseDispatchStream},
|
||||
stream::{
|
||||
live::{spawn_live_response_stream, LiveResponseStreamContext},
|
||||
prefetch::prefetch_first_sse_chunk,
|
||||
sse_failure::{
|
||||
auth_sse_failure_account_status, client_error_invalid_reasoning_replay,
|
||||
first_sse_failure, is_auth_sse_failure, is_history_recovery_sse_failure,
|
||||
is_model_unsupported_sse_failure, is_quota_exhausted_sse_failure,
|
||||
sse_failure_error_body, sse_failure_invalid_reasoning_replay,
|
||||
stream_failure_http_status,
|
||||
auth_sse_failure_account_status, first_sse_failure, is_auth_sse_failure,
|
||||
is_history_recovery_sse_failure, is_model_unsupported_sse_failure,
|
||||
is_quota_exhausted_sse_failure, sse_failure_error_body, stream_failure_http_status,
|
||||
},
|
||||
trace::ResponseDispatchTrace,
|
||||
},
|
||||
upstream_call::{
|
||||
create_response_stream_with_account_retrying_5xx, verify_acquired_quota_if_required,
|
||||
QuotaVerificationContext, QuotaVerificationDecision,
|
||||
AccountUpstreamContext, QuotaVerificationContext, QuotaVerificationDecision,
|
||||
QUOTA_VERIFY_LIMIT_REACHED_MESSAGE,
|
||||
},
|
||||
},
|
||||
@@ -65,16 +67,20 @@ impl ResponseDispatchService {
|
||||
request.set_stream(true);
|
||||
let tuple_schema = request.tuple_schema.clone();
|
||||
let now = Utc::now();
|
||||
let mut implicit_resume = self.prepare_response_session(&mut request).await;
|
||||
let account_affinity = self.account_affinity_for_request(&request, now).await;
|
||||
prepare_variant_identity(&mut request);
|
||||
self.account_identity.prepare_local_identity(&mut request);
|
||||
let mut history = HistoryRecoveryPlan::load(&self.session_affinity, &request).await;
|
||||
let preferred_account_id = self
|
||||
.preferred_account_id_for_request(&request, &history, now)
|
||||
.await;
|
||||
let mut acquire_request = AccountAcquireRequest::new(request.model(), now);
|
||||
if let Some(preferred_account_id) = account_affinity.preferred_account_id() {
|
||||
if let Some(preferred_account_id) = preferred_account_id {
|
||||
acquire_request = acquire_request.with_preferred_account_id(preferred_account_id);
|
||||
}
|
||||
let mut excluded_account_ids = Vec::new();
|
||||
let mut candidates =
|
||||
AccountAttemptLedger::freeze(&self.account_pool, &acquire_request).await;
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut next_required_account_id: Option<String> = None;
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
macro_rules! return_stream_dispatch_error {
|
||||
($error:expr) => {{
|
||||
@@ -119,21 +125,23 @@ impl ResponseDispatchService {
|
||||
}};
|
||||
}
|
||||
loop {
|
||||
let mut attempt_acquire_request = acquire_request
|
||||
.clone()
|
||||
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
||||
if let Some(account_id) = next_required_account_id.take() {
|
||||
attempt_acquire_request =
|
||||
attempt_acquire_request.with_required_account_id(account_id);
|
||||
}
|
||||
let allow_quota_verify_account_retry =
|
||||
attempt_acquire_request.required_account_id.is_none();
|
||||
attempt_acquire_request.now = Utc::now();
|
||||
let Some(acquired) = self
|
||||
.account_pool
|
||||
.acquire_with(&attempt_acquire_request)
|
||||
.await
|
||||
else {
|
||||
let acquired = if let Some(account_id) = next_required_account_id.take() {
|
||||
self.account_pool
|
||||
.acquire_with(
|
||||
&AccountAcquireRequest::new(request.model(), Utc::now())
|
||||
.with_required_account_id(account_id),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
candidates.acquire_next(&self.account_pool).await
|
||||
};
|
||||
let Some(acquired) = acquired else {
|
||||
tracing::info!(
|
||||
candidate_count = candidates.candidate_count(),
|
||||
attempted = candidates.attempted_count(),
|
||||
state_excluded = candidates.state_excluded_count(),
|
||||
"Responses stream account candidate ledger exhausted"
|
||||
);
|
||||
let error = exhausted_accounts
|
||||
.last_exhausted()
|
||||
.map(ResponseDispatchError::from_exhausted_account)
|
||||
@@ -146,11 +154,8 @@ impl ResponseDispatchService {
|
||||
account_pool: self.account_pool.as_ref(),
|
||||
codex: self.codex.as_ref(),
|
||||
cloudflare: &self.cloudflare,
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
account_identity: &self.account_identity,
|
||||
request_id,
|
||||
excluded_account_ids: &mut excluded_account_ids,
|
||||
verify_attempts: &mut quota_verify_attempts,
|
||||
allow_retry_with_another_account: allow_quota_verify_account_retry,
|
||||
},
|
||||
acquired,
|
||||
)
|
||||
@@ -158,55 +163,36 @@ impl ResponseDispatchService {
|
||||
{
|
||||
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
||||
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
||||
exhausted_accounts
|
||||
.record_rate_limited(None, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE);
|
||||
continue;
|
||||
}
|
||||
QuotaVerificationDecision::MaxAttemptsReached
|
||||
| QuotaVerificationDecision::RequiredAccountUnavailable => {
|
||||
exhausted_accounts.record_rate_limited(
|
||||
Some(&acquired_account_id),
|
||||
QUOTA_VERIFY_LIMIT_REACHED_MESSAGE,
|
||||
);
|
||||
let error = exhausted_accounts
|
||||
.last_exhausted()
|
||||
.map(ResponseDispatchError::from_exhausted_account)
|
||||
.unwrap_or(ResponseDispatchError::NoActiveAccount);
|
||||
return_stream_dispatch_error!(
|
||||
error,
|
||||
account_id: Some(&acquired_account_id),
|
||||
transport: Some(backend_transport_name(backend_transport_for_response_request(
|
||||
&request
|
||||
)))
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
self.apply_cascading_ban_defense(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
account_affinity.preferred_account_id(),
|
||||
&acquired.account.id,
|
||||
)
|
||||
.await;
|
||||
Self::strip_history_if_account_changed(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&account_affinity,
|
||||
&acquired.account.id,
|
||||
);
|
||||
let Some(attempt_request) = history.prepare_attempt(&request, &acquired.account.id)
|
||||
else {
|
||||
acquired.release_without_usage().await;
|
||||
return_stream_dispatch_error!(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: "previous response history cannot be sent to another account"
|
||||
.to_string(),
|
||||
});
|
||||
};
|
||||
|
||||
self.account_pool.wait_for_request_interval(&acquired).await;
|
||||
let account = acquired.account.clone();
|
||||
let release_account_id = account.id.clone();
|
||||
let attempt = trace.start_attempt(&release_account_id);
|
||||
let response_result = create_response_stream_with_account_retrying_5xx(
|
||||
&self.codex,
|
||||
self.installation_id.as_deref(),
|
||||
&self.cloudflare,
|
||||
&request,
|
||||
request_id,
|
||||
&account,
|
||||
AccountUpstreamContext {
|
||||
codex: &self.codex,
|
||||
account_identity: &self.account_identity,
|
||||
cloudflare: &self.cloudflare,
|
||||
request_id,
|
||||
account: &account,
|
||||
},
|
||||
&attempt_request,
|
||||
)
|
||||
.await;
|
||||
if let Err(error) = &response_result {
|
||||
@@ -228,6 +214,9 @@ impl ResponseDispatchService {
|
||||
let websocket_pool_decision = response.websocket_pool_decision;
|
||||
let turn_state = response.turn_state;
|
||||
let diagnostics = response.diagnostics;
|
||||
let response_metadata = response.response_metadata;
|
||||
self.models
|
||||
.observe_models_etag(response_metadata.models_etag.as_deref());
|
||||
self.cloudflare
|
||||
.capture_set_cookie_headers(&release_account_id, &set_cookie_headers)
|
||||
.await;
|
||||
@@ -248,7 +237,13 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
@@ -262,7 +257,13 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
@@ -280,7 +281,13 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, account_status)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
@@ -294,7 +301,13 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
@@ -308,7 +321,13 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
@@ -316,20 +335,17 @@ impl ResponseDispatchService {
|
||||
{
|
||||
acquired.complete().await;
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
if let Some(exhausted) = exhausted_accounts
|
||||
.model_unsupported_retry_exhausted(upstream_error.clone())
|
||||
{
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::from_exhausted_account(exhausted),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
exhausted_accounts.record_model_unsupported(
|
||||
Some(&release_account_id),
|
||||
upstream_error,
|
||||
);
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(ResponseDispatchError::Upstream(error))
|
||||
@@ -345,23 +361,29 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::Banned)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
acquired.complete().await;
|
||||
if let ResponseDispatchError::Upstream(upstream_error) = &error {
|
||||
if is_continuation_busy_error(upstream_error) {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::ContinuationBusy,
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
let history_unavailable =
|
||||
is_history_recovery_upstream_error(upstream_error);
|
||||
if history_unavailable
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
client_error_invalid_reasoning_replay(upstream_error),
|
||||
)
|
||||
.await
|
||||
&& history.recover_managed_history(&release_account_id)
|
||||
{
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
@@ -377,14 +399,27 @@ impl ResponseDispatchService {
|
||||
started_at,
|
||||
stream: true,
|
||||
transport,
|
||||
request: &request,
|
||||
request: &attempt_request,
|
||||
error: upstream_error,
|
||||
trace: &trace,
|
||||
attempt: Some(&attempt),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if is_retryable_account_transport_error(upstream_error) {
|
||||
exhausted_accounts.record_upstream_unavailable(
|
||||
Some(&release_account_id),
|
||||
upstream_error_body(upstream_error),
|
||||
);
|
||||
if history.is_external_unknown() {
|
||||
return Err(error);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if history_unavailable {
|
||||
if history.is_external_unknown() {
|
||||
return Err(error);
|
||||
}
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: upstream_error_body(upstream_error),
|
||||
});
|
||||
@@ -411,14 +446,7 @@ impl ResponseDispatchService {
|
||||
};
|
||||
if let Some(failure) = first_failure {
|
||||
if is_history_recovery_sse_failure(&failure)
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
sse_failure_invalid_reasoning_replay(&failure),
|
||||
)
|
||||
.await
|
||||
&& history.recover_managed_history(&release_account_id)
|
||||
{
|
||||
acquired.complete().await;
|
||||
next_required_account_id = Some(release_account_id);
|
||||
@@ -426,22 +454,18 @@ impl ResponseDispatchService {
|
||||
}
|
||||
if is_model_unsupported_sse_failure(&failure) {
|
||||
let upstream_error = sse_failure_error_body(&failure);
|
||||
if let Some(exhausted) = exhausted_accounts
|
||||
.model_unsupported_retry_exhausted(upstream_error.clone())
|
||||
{
|
||||
acquired.complete().await;
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::from_exhausted_account(exhausted),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
exhausted_accounts.record_model_unsupported(
|
||||
Some(&release_account_id),
|
||||
upstream_error,
|
||||
);
|
||||
excluded_account_ids.push(release_account_id);
|
||||
acquired.complete().await;
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Failed(failure),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_quota_exhausted_sse_failure(&failure) {
|
||||
@@ -452,8 +476,14 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
acquired.complete().await;
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Failed(failure),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_auth_sse_failure(&failure) {
|
||||
@@ -468,8 +498,14 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, account_status)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
acquired.complete().await;
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Failed(failure),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(transport))
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
acquired.complete().await;
|
||||
@@ -483,7 +519,7 @@ impl ResponseDispatchService {
|
||||
requested_model,
|
||||
started_at,
|
||||
transport,
|
||||
request: &request,
|
||||
request: &attempt_request,
|
||||
failure: &failure,
|
||||
diagnostics: &diagnostics,
|
||||
rate_limit_headers: &rate_limit_headers,
|
||||
@@ -493,7 +529,9 @@ impl ResponseDispatchService {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let error = if is_history_recovery_sse_failure(&failure) {
|
||||
let error = if is_history_recovery_sse_failure(&failure)
|
||||
&& !history.is_external_unknown()
|
||||
{
|
||||
ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: sse_failure_error_body(&failure),
|
||||
}
|
||||
@@ -507,7 +545,7 @@ impl ResponseDispatchService {
|
||||
account_pool: Arc::clone(&self.account_pool),
|
||||
account_lease: acquired,
|
||||
session_affinity: Arc::clone(&self.session_affinity),
|
||||
reasoning_replay: Arc::clone(&self.reasoning_replay),
|
||||
history,
|
||||
recorder: Arc::clone(&self.recorder),
|
||||
cloudflare: self.cloudflare.clone(),
|
||||
account_id: account.id,
|
||||
@@ -527,6 +565,7 @@ impl ResponseDispatchService {
|
||||
websocket_pool_decision,
|
||||
turn_state,
|
||||
diagnostics,
|
||||
response_metadata,
|
||||
attempt: attempt.clone(),
|
||||
attempts: trace.attempts().to_vec(),
|
||||
started_at,
|
||||
@@ -543,7 +582,15 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
||||
acquired.complete().await;
|
||||
@@ -554,7 +601,15 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) if is_auth_upstream_error(&error) => {
|
||||
acquired.complete().await;
|
||||
@@ -569,7 +624,15 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, account_status)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
||||
acquired.complete().await;
|
||||
@@ -580,7 +643,15 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
||||
acquired.complete().await;
|
||||
@@ -591,25 +662,30 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
||||
acquired.complete().await;
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
if let Some(exhausted) =
|
||||
exhausted_accounts.model_unsupported_retry_exhausted(upstream_error.clone())
|
||||
{
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::from_exhausted_account(exhausted),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(backend_transport_for_response_request(
|
||||
&request
|
||||
)))
|
||||
);
|
||||
}
|
||||
exhausted_accounts
|
||||
.record_model_unsupported(Some(&release_account_id), upstream_error);
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) if is_banned_upstream_error(&error) => {
|
||||
acquired.complete().await;
|
||||
@@ -622,21 +698,29 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::Banned)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
if history.is_external_unknown() {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::Upstream(error),
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
acquired.complete().await;
|
||||
if is_continuation_busy_error(&error) {
|
||||
return_stream_dispatch_error!(
|
||||
ResponseDispatchError::ContinuationBusy,
|
||||
account_id: Some(&release_account_id),
|
||||
transport: Some(backend_transport_name(
|
||||
backend_transport_for_response_request(&attempt_request)
|
||||
))
|
||||
);
|
||||
}
|
||||
let history_unavailable = is_history_recovery_upstream_error(&error);
|
||||
if history_unavailable
|
||||
&& self
|
||||
.try_recover_implicit_resume(
|
||||
&mut request,
|
||||
&mut implicit_resume,
|
||||
&release_account_id,
|
||||
client_error_invalid_reasoning_replay(&error),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if history_unavailable && history.recover_managed_history(&release_account_id) {
|
||||
next_required_account_id = Some(release_account_id);
|
||||
continue;
|
||||
}
|
||||
@@ -649,14 +733,27 @@ impl ResponseDispatchService {
|
||||
model: requested_model,
|
||||
started_at,
|
||||
stream: true,
|
||||
transport: backend_transport_for_response_request(&request),
|
||||
request: &request,
|
||||
transport: backend_transport_for_response_request(&attempt_request),
|
||||
request: &attempt_request,
|
||||
error: &error,
|
||||
trace: &trace,
|
||||
attempt: Some(&attempt),
|
||||
})
|
||||
.await;
|
||||
if is_retryable_account_transport_error(&error) {
|
||||
exhausted_accounts.record_upstream_unavailable(
|
||||
Some(&release_account_id),
|
||||
upstream_error_body(&error),
|
||||
);
|
||||
if history.is_external_unknown() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if history_unavailable {
|
||||
if history.is_external_unknown() {
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
}
|
||||
return Err(ResponseDispatchError::HistoryUnavailable {
|
||||
upstream_error: upstream_error_body(&error),
|
||||
});
|
||||
|
||||
@@ -8,20 +8,17 @@ use std::{
|
||||
use bytes::Bytes;
|
||||
use futures::{stream::Stream, StreamExt};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::{mpsc, oneshot, Mutex};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::{
|
||||
resolve::{evict_reasoning_replay, record_response_affinity},
|
||||
SessionAffinityService,
|
||||
},
|
||||
affinity::{resolve::record_response_affinity, SessionAffinityService},
|
||||
errors::{backend_transport_name, ResponseDispatchStreamError},
|
||||
recording::{
|
||||
insert_first_token_ms, live_response_rate_limit_headers, live_response_turn_state,
|
||||
record_live_response_stream_event,
|
||||
},
|
||||
recovery::{cloudflare::CloudflareRecovery, reasoning_replay::ReasoningReplayCache},
|
||||
recovery::{cloudflare::CloudflareRecovery, history::HistoryRecoveryPlan},
|
||||
service::ResponseDispatchStream,
|
||||
},
|
||||
fleet::pool::{AccountLease, AccountPoolService},
|
||||
@@ -42,16 +39,17 @@ use crate::{
|
||||
},
|
||||
transport::{
|
||||
CodexBackendSseStream, CodexBackendTransport, CodexRateLimitHeaderUpdates,
|
||||
CodexTurnStateUpdate, CodexUpstreamDiagnostics, WebSocketPoolDecision,
|
||||
CodexResponseMetadata, CodexTurnStateUpdate, CodexUpstreamDiagnostics,
|
||||
WebSocketPoolDecision,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
use super::{
|
||||
sse_failure::{
|
||||
sse_failure_invalid_reasoning_replay, status_code_for_stream_failure,
|
||||
stream_failure_metadata, stream_failure_source, synthetic_stream_disconnected_detail,
|
||||
STREAM_DISCONNECTED_CODE, STREAM_DISCONNECTED_MESSAGE,
|
||||
status_code_for_stream_failure, stream_failure_metadata, stream_failure_source,
|
||||
synthetic_stream_disconnected_detail, STREAM_DISCONNECTED_CODE,
|
||||
STREAM_DISCONNECTED_MESSAGE,
|
||||
},
|
||||
trace::ResponseDispatchAttempt,
|
||||
};
|
||||
@@ -82,6 +80,7 @@ pub(in crate::dispatch) fn spawn_live_response_stream(
|
||||
prefetched: Bytes,
|
||||
mut body: CodexBackendSseStream,
|
||||
) -> ResponseDispatchStream {
|
||||
let response_headers = context.response_metadata.client_headers.clone();
|
||||
let (sender, receiver) = mpsc::channel(8);
|
||||
let (cancel_sender, mut cancel_receiver) = oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
@@ -183,6 +182,7 @@ pub(in crate::dispatch) fn spawn_live_response_stream(
|
||||
receiver,
|
||||
cancel: Some(cancel_sender),
|
||||
}),
|
||||
response_headers,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,7 +342,7 @@ pub(in crate::dispatch) struct LiveResponseStreamContext {
|
||||
pub(in crate::dispatch) account_pool: Arc<AccountPoolService>,
|
||||
pub(in crate::dispatch) account_lease: AccountLease,
|
||||
pub(in crate::dispatch) session_affinity: Arc<SessionAffinityService>,
|
||||
pub(in crate::dispatch) reasoning_replay: Arc<Mutex<ReasoningReplayCache>>,
|
||||
pub(in crate::dispatch) history: HistoryRecoveryPlan,
|
||||
pub(in crate::dispatch) recorder: Arc<Recorder>,
|
||||
pub(in crate::dispatch) cloudflare: CloudflareRecovery,
|
||||
pub(in crate::dispatch) account_id: String,
|
||||
@@ -362,6 +362,7 @@ pub(in crate::dispatch) struct LiveResponseStreamContext {
|
||||
pub(in crate::dispatch) websocket_pool_decision: Option<WebSocketPoolDecision>,
|
||||
pub(in crate::dispatch) turn_state: Option<String>,
|
||||
pub(in crate::dispatch) diagnostics: CodexUpstreamDiagnostics,
|
||||
pub(in crate::dispatch) response_metadata: CodexResponseMetadata,
|
||||
pub(in crate::dispatch) attempt: ResponseDispatchAttempt,
|
||||
pub(in crate::dispatch) attempts: Vec<ResponseDispatchAttempt>,
|
||||
pub(in crate::dispatch) started_at: Instant,
|
||||
@@ -428,31 +429,40 @@ pub(in crate::dispatch) async fn finalize_live_response_stream(
|
||||
}
|
||||
};
|
||||
|
||||
match response_from_codex_sse(&body, context.tuple_schema.as_ref()) {
|
||||
Ok(CollectedResponse::Completed(completed)) => {
|
||||
let collected_response = response_from_codex_sse(&body, context.tuple_schema.as_ref());
|
||||
let completed_terminal = matches!(&collected_response, Ok(CollectedResponse::Completed(_)));
|
||||
match collected_response {
|
||||
Ok(CollectedResponse::Completed(terminal) | CollectedResponse::Incomplete(terminal)) => {
|
||||
context
|
||||
.cloudflare
|
||||
.reset_account_recovery(&context.account_id)
|
||||
.await;
|
||||
let response_id = completed.get("id").and_then(Value::as_str);
|
||||
record_response_affinity(
|
||||
&context.session_affinity,
|
||||
&context.reasoning_replay,
|
||||
&context.request,
|
||||
&context.account_id,
|
||||
&body,
|
||||
turn_state,
|
||||
usage,
|
||||
)
|
||||
.await;
|
||||
let response_id = terminal.get("id").and_then(Value::as_str);
|
||||
if completed_terminal {
|
||||
record_response_affinity(
|
||||
&context.session_affinity,
|
||||
&context.history,
|
||||
&context.request,
|
||||
&context.account_id,
|
||||
&body,
|
||||
turn_state,
|
||||
usage,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
record_live_response_stream_event(
|
||||
&context,
|
||||
200,
|
||||
UsageRecordLevel::Info,
|
||||
"v1 responses stream completed",
|
||||
if completed_terminal {
|
||||
"v1 responses stream completed"
|
||||
} else {
|
||||
"v1 responses stream incomplete"
|
||||
},
|
||||
serde_json::json!({
|
||||
"stream": true,
|
||||
"completed": true,
|
||||
"completed": completed_terminal,
|
||||
"incomplete": !completed_terminal,
|
||||
"responseId": response_id,
|
||||
"firstTokenMs": first_token_ms,
|
||||
"usage": usage,
|
||||
@@ -463,14 +473,6 @@ pub(in crate::dispatch) async fn finalize_live_response_stream(
|
||||
.await;
|
||||
}
|
||||
Ok(CollectedResponse::Failed(failure)) => {
|
||||
if sse_failure_invalid_reasoning_replay(&failure) {
|
||||
evict_reasoning_replay(
|
||||
&context.reasoning_replay,
|
||||
&context.request,
|
||||
&context.account_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let response_id = latest_response_id(&body);
|
||||
let latency_ms = elapsed_millis_i64(context.started_at);
|
||||
let failure_source = stream_failure_source(&failure);
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
dispatch::errors::{
|
||||
is_history_recovery_signal, is_invalid_encrypted_content_signal,
|
||||
is_model_unsupported_signal,
|
||||
},
|
||||
dispatch::errors::{is_history_recovery_code, is_model_unsupported_signal},
|
||||
fleet::account::AccountStatus,
|
||||
upstream::openai::{
|
||||
protocol::{
|
||||
@@ -12,7 +9,7 @@ use crate::{
|
||||
responses::{response_from_codex_sse, CollectedResponse, ResponsesSseFailure},
|
||||
sse::SseError,
|
||||
},
|
||||
transport::{is_banned_auth_signal, CodexClientError},
|
||||
transport::is_banned_auth_signal,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -93,20 +90,10 @@ pub(in crate::dispatch) fn is_model_unsupported_sse_failure(failure: &ResponsesS
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn is_history_recovery_sse_failure(failure: &ResponsesSseFailure) -> bool {
|
||||
sse_failure_matches(failure, is_history_recovery_signal)
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn sse_failure_invalid_reasoning_replay(
|
||||
failure: &ResponsesSseFailure,
|
||||
) -> bool {
|
||||
sse_failure_matches(failure, is_invalid_encrypted_content_signal)
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn client_error_invalid_reasoning_replay(error: &CodexClientError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexClientError::Upstream { body, .. } if is_invalid_encrypted_content_signal(body)
|
||||
)
|
||||
failure
|
||||
.upstream_code
|
||||
.as_deref()
|
||||
.is_some_and(is_history_recovery_code)
|
||||
}
|
||||
|
||||
pub(crate) fn auth_sse_failure_account_status(failure: &ResponsesSseFailure) -> AccountStatus {
|
||||
@@ -124,6 +111,7 @@ pub(in crate::dispatch) fn first_sse_failure(
|
||||
match response_from_codex_sse(&body, None)? {
|
||||
CollectedResponse::Failed(failure) => Ok(Some(failure)),
|
||||
CollectedResponse::Completed(_)
|
||||
| CollectedResponse::Incomplete(_)
|
||||
| CollectedResponse::MissingCompleted
|
||||
| CollectedResponse::Empty => Ok(None),
|
||||
}
|
||||
|
||||
@@ -7,12 +7,13 @@ use serde_json::{json, Value};
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::build_conversation_identity,
|
||||
affinity::{AccountIdentityService, AccountScopedIdentity},
|
||||
attempts::AccountAttemptLedger,
|
||||
errors::{
|
||||
is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error,
|
||||
is_rate_limit_upstream_error, is_retryable_upstream_5xx_error,
|
||||
rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status,
|
||||
upstream_error_set_cookie_headers, ResponseDispatchError,
|
||||
is_rate_limit_upstream_error, is_retryable_account_transport_error,
|
||||
is_retryable_upstream_5xx_error, rate_limit_cooldown_until, upstream_error_body,
|
||||
upstream_error_http_status, upstream_error_set_cookie_headers, ResponseDispatchError,
|
||||
},
|
||||
recording::{
|
||||
insert_response_status_metadata, insert_response_trace_metadata,
|
||||
@@ -28,7 +29,7 @@ use crate::{
|
||||
},
|
||||
exhaustion::AccountExhaustionTracker,
|
||||
},
|
||||
service::ResponseDispatchService,
|
||||
service::{ResponseDispatchResponse, ResponseDispatchService},
|
||||
stream::trace::ResponseDispatchTrace,
|
||||
},
|
||||
fleet::{
|
||||
@@ -56,27 +57,29 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
const MAX_QUOTA_VERIFY_ATTEMPTS: usize = 5;
|
||||
|
||||
pub(crate) const QUOTA_VERIFY_LIMIT_REACHED_MESSAGE: &str =
|
||||
"Upstream usage quota still reports limit_reached";
|
||||
|
||||
pub(crate) enum QuotaVerificationDecision {
|
||||
Ready(Box<AccountLease>),
|
||||
RetryWithAnotherAccount,
|
||||
RequiredAccountUnavailable,
|
||||
MaxAttemptsReached,
|
||||
}
|
||||
|
||||
pub(crate) struct QuotaVerificationContext<'a> {
|
||||
pub account_pool: &'a AccountPoolService,
|
||||
pub codex: &'a CodexBackendClient,
|
||||
pub cloudflare: &'a CloudflareRecovery,
|
||||
pub installation_id: Option<&'a str>,
|
||||
pub account_identity: &'a AccountIdentityService,
|
||||
pub request_id: &'a str,
|
||||
pub excluded_account_ids: &'a mut Vec<String>,
|
||||
pub verify_attempts: &'a mut usize,
|
||||
pub allow_retry_with_another_account: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct AccountUpstreamContext<'a> {
|
||||
pub codex: &'a CodexBackendClient,
|
||||
pub account_identity: &'a AccountIdentityService,
|
||||
pub cloudflare: &'a CloudflareRecovery,
|
||||
pub request_id: &'a str,
|
||||
pub account: &'a Account,
|
||||
}
|
||||
|
||||
pub(crate) async fn verify_acquired_quota_if_required(
|
||||
@@ -88,6 +91,9 @@ pub(crate) async fn verify_acquired_quota_if_required(
|
||||
}
|
||||
|
||||
let account_id = acquired.account.id.clone();
|
||||
let identity = context
|
||||
.account_identity
|
||||
.scope_auxiliary(&account_id, context.request_id);
|
||||
let cookie_header = context
|
||||
.cloudflare
|
||||
.cookie_header_for_request(&account_id, "/codex/usage")
|
||||
@@ -106,8 +112,12 @@ pub(crate) async fn verify_acquired_quota_if_required(
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id: context.installation_id,
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
turn_id: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -134,24 +144,14 @@ pub(crate) async fn verify_acquired_quota_if_required(
|
||||
.await;
|
||||
if quota_snapshot_limit_reached("a) {
|
||||
acquired.release_without_usage().await;
|
||||
context.excluded_account_ids.push(account_id.clone());
|
||||
*context.verify_attempts += 1;
|
||||
let max_attempts_reached = *context.verify_attempts >= MAX_QUOTA_VERIFY_ATTEMPTS;
|
||||
tracing::info!(
|
||||
request_id = %context.request_id,
|
||||
account_id = %account_id,
|
||||
quota_verify_required = true,
|
||||
quota_verify_result = "limit_reached",
|
||||
verify_attempts = *context.verify_attempts,
|
||||
retry_with_another_account = context.allow_retry_with_another_account && !max_attempts_reached,
|
||||
retry_with_another_account = true,
|
||||
"quota verification reported exhausted account before upstream request"
|
||||
);
|
||||
if !context.allow_retry_with_another_account {
|
||||
return QuotaVerificationDecision::RequiredAccountUnavailable;
|
||||
}
|
||||
if max_attempts_reached {
|
||||
return QuotaVerificationDecision::MaxAttemptsReached;
|
||||
}
|
||||
return QuotaVerificationDecision::RetryWithAnotherAccount;
|
||||
}
|
||||
|
||||
@@ -175,118 +175,212 @@ fn acquired_with_verified_quota(mut acquired: AccountLease, quota: &Value) -> Ac
|
||||
}
|
||||
|
||||
pub(crate) async fn create_response_with_account(
|
||||
codex: &CodexBackendClient,
|
||||
installation_id: Option<&str>,
|
||||
cloudflare: &CloudflareRecovery,
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexResponsesRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexBackendResponse, CodexClientError> {
|
||||
let cookie_header = cloudflare
|
||||
.cookie_header_for_request(&account.id, "/codex/responses")
|
||||
let cookie_header = context
|
||||
.cloudflare
|
||||
.cookie_header_for_request(&context.account.id, "/codex/responses")
|
||||
.await;
|
||||
let identity = build_conversation_identity(
|
||||
request.prompt_cache_key(),
|
||||
request.codex_window_id.as_deref(),
|
||||
&account.id,
|
||||
);
|
||||
codex
|
||||
let identity = context
|
||||
.account_identity
|
||||
.scope(request, &context.account.id, context.request_id);
|
||||
context
|
||||
.codex
|
||||
.create_response_with_pool_account_started_at(
|
||||
request,
|
||||
CodexRequestContext {
|
||||
access_token: &account.access_token,
|
||||
account_id: account.account_id.as_deref(),
|
||||
request_id,
|
||||
access_token: &context.account.access_token,
|
||||
account_id: context.account.account_id.as_deref(),
|
||||
request_id: context.request_id,
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
turn_metadata: request.turn_metadata.as_deref(),
|
||||
beta_features: request.beta_features.as_deref(),
|
||||
include_timing_metrics: request.include_timing_metrics.as_deref(),
|
||||
version: request.version.as_deref(),
|
||||
codex_window_id: identity.window_id.as_deref(),
|
||||
parent_thread_id: request.parent_thread_id.as_deref(),
|
||||
parent_thread_id: identity.parent_thread_id.as_deref(),
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id,
|
||||
session_id: identity.conversation_id.as_deref(),
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: identity.session_id.as_deref(),
|
||||
thread_id: identity.thread_id.as_deref(),
|
||||
prompt_cache_key: identity.prompt_cache_key.as_deref(),
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
turn_id: identity.turn_id.as_deref(),
|
||||
},
|
||||
Some(&account.id),
|
||||
Some(&context.account.id),
|
||||
started_at,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_response_stream_with_account(
|
||||
codex: &CodexBackendClient,
|
||||
installation_id: Option<&str>,
|
||||
cloudflare: &CloudflareRecovery,
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexResponsesRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
) -> Result<CodexBackendStreamingResponse, CodexClientError> {
|
||||
let cookie_header = cloudflare
|
||||
.cookie_header_for_request(&account.id, "/codex/responses")
|
||||
let cookie_header = context
|
||||
.cloudflare
|
||||
.cookie_header_for_request(&context.account.id, "/codex/responses")
|
||||
.await;
|
||||
let identity = build_conversation_identity(
|
||||
request.prompt_cache_key(),
|
||||
request.codex_window_id.as_deref(),
|
||||
&account.id,
|
||||
);
|
||||
codex
|
||||
let identity = context
|
||||
.account_identity
|
||||
.scope(request, &context.account.id, context.request_id);
|
||||
context
|
||||
.codex
|
||||
.create_response_stream_with_pool_account(
|
||||
request,
|
||||
CodexRequestContext {
|
||||
access_token: &account.access_token,
|
||||
account_id: account.account_id.as_deref(),
|
||||
request_id,
|
||||
access_token: &context.account.access_token,
|
||||
account_id: context.account.account_id.as_deref(),
|
||||
request_id: context.request_id,
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
turn_metadata: request.turn_metadata.as_deref(),
|
||||
beta_features: request.beta_features.as_deref(),
|
||||
include_timing_metrics: request.include_timing_metrics.as_deref(),
|
||||
version: request.version.as_deref(),
|
||||
codex_window_id: identity.window_id.as_deref(),
|
||||
parent_thread_id: request.parent_thread_id.as_deref(),
|
||||
parent_thread_id: identity.parent_thread_id.as_deref(),
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id,
|
||||
session_id: identity.conversation_id.as_deref(),
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: identity.session_id.as_deref(),
|
||||
thread_id: identity.thread_id.as_deref(),
|
||||
prompt_cache_key: identity.prompt_cache_key.as_deref(),
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
turn_id: identity.turn_id.as_deref(),
|
||||
},
|
||||
Some(&account.id),
|
||||
Some(&context.account.id),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_compact_response_with_account(
|
||||
codex: &CodexBackendClient,
|
||||
installation_id: Option<&str>,
|
||||
cloudflare: &CloudflareRecovery,
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexCompactRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
) -> Result<CodexCompactResponse, CodexClientError> {
|
||||
let cookie_header = cloudflare
|
||||
.cookie_header_for_request(&account.id, "/codex/responses/compact")
|
||||
let cookie_header = context
|
||||
.cloudflare
|
||||
.cookie_header_for_request(&context.account.id, "/codex/responses/compact")
|
||||
.await;
|
||||
codex
|
||||
let identity =
|
||||
context
|
||||
.account_identity
|
||||
.scope_compact(request, &context.account.id, context.request_id);
|
||||
let upstream_request = compact_upstream_request(request, &identity);
|
||||
context
|
||||
.codex
|
||||
.create_compact_response(
|
||||
request,
|
||||
&upstream_request,
|
||||
CodexRequestContext {
|
||||
access_token: &account.access_token,
|
||||
account_id: account.account_id.as_deref(),
|
||||
request_id,
|
||||
access_token: &context.account.access_token,
|
||||
account_id: context.account.account_id.as_deref(),
|
||||
request_id: context.request_id,
|
||||
turn_state: None,
|
||||
turn_metadata: None,
|
||||
beta_features: None,
|
||||
include_timing_metrics: None,
|
||||
version: None,
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
codex_window_id: identity.window_id.as_deref(),
|
||||
parent_thread_id: identity.parent_thread_id.as_deref(),
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id,
|
||||
session_id: None,
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: identity.session_id.as_deref(),
|
||||
thread_id: identity.thread_id.as_deref(),
|
||||
prompt_cache_key: identity.prompt_cache_key.as_deref(),
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
turn_id: identity.turn_id.as_deref(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn compact_upstream_request(
|
||||
request: &CodexCompactRequest,
|
||||
identity: &AccountScopedIdentity,
|
||||
) -> CodexCompactRequest {
|
||||
let mut upstream = request.clone();
|
||||
for (key, value) in [
|
||||
("prompt_cache_key", identity.prompt_cache_key.as_deref()),
|
||||
("session_id", identity.session_id.as_deref()),
|
||||
("thread_id", identity.thread_id.as_deref()),
|
||||
("turn_id", identity.turn_id.as_deref()),
|
||||
(
|
||||
"x-client-request-id",
|
||||
Some(identity.client_request_id.as_str()),
|
||||
),
|
||||
("x-codex-window-id", identity.window_id.as_deref()),
|
||||
(
|
||||
"x-codex-parent-thread-id",
|
||||
identity.parent_thread_id.as_deref(),
|
||||
),
|
||||
(
|
||||
"x-codex-installation-id",
|
||||
Some(identity.installation_id.as_str()),
|
||||
),
|
||||
] {
|
||||
replace_existing_identity_field(&mut upstream.body, key, value);
|
||||
}
|
||||
|
||||
let metadata = match upstream.body.get("client_metadata") {
|
||||
Some(Value::Object(metadata)) => Some(metadata.clone()),
|
||||
None => Some(serde_json::Map::new()),
|
||||
Some(_) => None,
|
||||
};
|
||||
if let Some(mut metadata) = metadata {
|
||||
for (key, value) in [
|
||||
(
|
||||
"x-codex-installation-id",
|
||||
Some(identity.installation_id.as_str()),
|
||||
),
|
||||
("session_id", identity.session_id.as_deref()),
|
||||
("thread_id", identity.thread_id.as_deref()),
|
||||
(
|
||||
"x-client-request-id",
|
||||
Some(identity.client_request_id.as_str()),
|
||||
),
|
||||
("turn_id", identity.turn_id.as_deref()),
|
||||
("x-codex-window-id", identity.window_id.as_deref()),
|
||||
(
|
||||
"x-codex-parent-thread-id",
|
||||
identity.parent_thread_id.as_deref(),
|
||||
),
|
||||
] {
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
Some(value) => {
|
||||
metadata.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
None => {
|
||||
metadata.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !metadata.is_empty() {
|
||||
upstream
|
||||
.body
|
||||
.insert("client_metadata".to_string(), Value::Object(metadata));
|
||||
}
|
||||
}
|
||||
upstream
|
||||
}
|
||||
|
||||
fn replace_existing_identity_field(
|
||||
body: &mut serde_json::Map<String, Value>,
|
||||
key: &str,
|
||||
value: Option<&str>,
|
||||
) {
|
||||
if !body.get(key).is_some_and(Value::is_string) {
|
||||
return;
|
||||
}
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
Some(value) => {
|
||||
body.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
None => {
|
||||
body.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_UPSTREAM_5XX_RETRIES_PER_ACCOUNT: usize = 2;
|
||||
|
||||
async fn retry_upstream_5xx<T, F, Fut>(
|
||||
@@ -392,87 +486,47 @@ fn log_upstream_failure(
|
||||
}
|
||||
|
||||
pub(crate) async fn create_response_with_account_retrying_5xx(
|
||||
codex: &CodexBackendClient,
|
||||
installation_id: Option<&str>,
|
||||
cloudflare: &CloudflareRecovery,
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexResponsesRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexBackendResponse, CodexClientError> {
|
||||
retry_upstream_5xx(
|
||||
request_id,
|
||||
&account.id,
|
||||
context.request_id,
|
||||
&context.account.id,
|
||||
None,
|
||||
"upstream response request failed with retryable 5xx",
|
||||
"upstream response request failed",
|
||||
|| {
|
||||
create_response_with_account(
|
||||
codex,
|
||||
installation_id,
|
||||
cloudflare,
|
||||
request,
|
||||
request_id,
|
||||
account,
|
||||
started_at,
|
||||
)
|
||||
},
|
||||
|| create_response_with_account(context, request, started_at),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn create_response_stream_with_account_retrying_5xx(
|
||||
codex: &CodexBackendClient,
|
||||
installation_id: Option<&str>,
|
||||
cloudflare: &CloudflareRecovery,
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexResponsesRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
) -> Result<CodexBackendStreamingResponse, CodexClientError> {
|
||||
retry_upstream_5xx(
|
||||
request_id,
|
||||
&account.id,
|
||||
context.request_id,
|
||||
&context.account.id,
|
||||
None,
|
||||
"upstream response stream request failed with retryable 5xx",
|
||||
"upstream response stream request failed",
|
||||
|| {
|
||||
create_response_stream_with_account(
|
||||
codex,
|
||||
installation_id,
|
||||
cloudflare,
|
||||
request,
|
||||
request_id,
|
||||
account,
|
||||
)
|
||||
},
|
||||
|| create_response_stream_with_account(context, request),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn create_compact_response_with_account_retrying_5xx(
|
||||
codex: &CodexBackendClient,
|
||||
installation_id: Option<&str>,
|
||||
cloudflare: &CloudflareRecovery,
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexCompactRequest,
|
||||
request_id: &str,
|
||||
account: &Account,
|
||||
) -> Result<CodexCompactResponse, CodexClientError> {
|
||||
retry_upstream_5xx(
|
||||
request_id,
|
||||
&account.id,
|
||||
context.request_id,
|
||||
&context.account.id,
|
||||
Some("compact"),
|
||||
"upstream compact request failed with retryable 5xx",
|
||||
"upstream compact request failed",
|
||||
|| {
|
||||
create_compact_response_with_account(
|
||||
codex,
|
||||
installation_id,
|
||||
cloudflare,
|
||||
request,
|
||||
request_id,
|
||||
account,
|
||||
)
|
||||
},
|
||||
|| create_compact_response_with_account(context, request),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -484,23 +538,28 @@ impl ResponseDispatchService {
|
||||
request_id: &str,
|
||||
mut request: CodexCompactRequest,
|
||||
requested_model: &str,
|
||||
) -> Result<Value, ResponseDispatchError> {
|
||||
) -> Result<ResponseDispatchResponse, ResponseDispatchError> {
|
||||
let started_at = Instant::now();
|
||||
let client_api_key_id = request.client_api_key_id.clone();
|
||||
let catalog = self.models.catalog().await;
|
||||
let display_model = catalog.resolve_model_id(requested_model);
|
||||
request.set_model(display_model.clone());
|
||||
let mut excluded_account_ids = Vec::new();
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
let acquire_request = AccountAcquireRequest::new(request.model(), Utc::now());
|
||||
let mut candidates =
|
||||
AccountAttemptLedger::freeze(&self.account_pool, &acquire_request).await;
|
||||
|
||||
loop {
|
||||
let acquire_request = AccountAcquireRequest::new(request.model(), Utc::now())
|
||||
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
||||
let acquired = match self.account_pool.acquire_with(&acquire_request).await {
|
||||
let acquired = match candidates.acquire_next(&self.account_pool).await {
|
||||
Some(acquired) => acquired,
|
||||
None => {
|
||||
tracing::info!(
|
||||
candidate_count = candidates.candidate_count(),
|
||||
attempted = candidates.attempted_count(),
|
||||
state_excluded = candidates.state_excluded_count(),
|
||||
"Responses compact account candidate ledger exhausted"
|
||||
);
|
||||
let error = exhausted_accounts
|
||||
.last_exhausted()
|
||||
.map(ResponseDispatchError::from_exhausted_account)
|
||||
@@ -523,11 +582,8 @@ impl ResponseDispatchService {
|
||||
account_pool: self.account_pool.as_ref(),
|
||||
codex: self.codex.as_ref(),
|
||||
cloudflare: &self.cloudflare,
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
account_identity: &self.account_identity,
|
||||
request_id,
|
||||
excluded_account_ids: &mut excluded_account_ids,
|
||||
verify_attempts: &mut quota_verify_attempts,
|
||||
allow_retry_with_another_account: true,
|
||||
},
|
||||
acquired,
|
||||
)
|
||||
@@ -535,42 +591,25 @@ impl ResponseDispatchService {
|
||||
{
|
||||
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
||||
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
||||
exhausted_accounts
|
||||
.record_rate_limited(None, QUOTA_VERIFY_LIMIT_REACHED_MESSAGE);
|
||||
continue;
|
||||
}
|
||||
QuotaVerificationDecision::MaxAttemptsReached
|
||||
| QuotaVerificationDecision::RequiredAccountUnavailable => {
|
||||
exhausted_accounts.record_rate_limited(
|
||||
Some(&acquired_account_id),
|
||||
QUOTA_VERIFY_LIMIT_REACHED_MESSAGE,
|
||||
);
|
||||
let error = exhausted_accounts
|
||||
.last_exhausted()
|
||||
.map(ResponseDispatchError::from_exhausted_account)
|
||||
.unwrap_or(ResponseDispatchError::NoActiveAccount);
|
||||
self.record_compact_dispatch_error(
|
||||
request_id,
|
||||
client_api_key_id.as_deref(),
|
||||
requested_model,
|
||||
started_at,
|
||||
Some(&acquired_account_id),
|
||||
&error,
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let account = acquired.account.clone();
|
||||
let release_account_id = account.id.clone();
|
||||
let attempt = trace.start_attempt(&release_account_id);
|
||||
let response_result = create_compact_response_with_account_retrying_5xx(
|
||||
&self.codex,
|
||||
self.installation_id.as_deref(),
|
||||
&self.cloudflare,
|
||||
AccountUpstreamContext {
|
||||
codex: &self.codex,
|
||||
account_identity: &self.account_identity,
|
||||
cloudflare: &self.cloudflare,
|
||||
request_id,
|
||||
account: &account,
|
||||
},
|
||||
&request,
|
||||
request_id,
|
||||
&account,
|
||||
)
|
||||
.await;
|
||||
acquired.complete().await;
|
||||
@@ -605,6 +644,9 @@ impl ResponseDispatchService {
|
||||
"stream": false,
|
||||
"compact": true,
|
||||
"usage": usage,
|
||||
"effectiveModel": response.response_metadata.effective_model.as_deref(),
|
||||
"modelsEtag": response.response_metadata.models_etag.as_deref(),
|
||||
"reasoningIncluded": response.response_metadata.reasoning_included,
|
||||
});
|
||||
insert_response_status_metadata(
|
||||
&mut metadata,
|
||||
@@ -620,7 +662,11 @@ impl ResponseDispatchService {
|
||||
client_api_key_id: request.client_api_key_id.as_deref(),
|
||||
account_id: &account.id,
|
||||
route: "/v1/responses/compact",
|
||||
model: &display_model,
|
||||
model: response
|
||||
.response_metadata
|
||||
.effective_model
|
||||
.as_deref()
|
||||
.unwrap_or(&display_model),
|
||||
requested_model: Some(requested_model),
|
||||
client_ip: request.client_ip.as_deref(),
|
||||
client_user_agent: request.client_user_agent.as_deref(),
|
||||
@@ -633,7 +679,12 @@ impl ResponseDispatchService {
|
||||
rate_limit_headers: &response.rate_limit_headers,
|
||||
})
|
||||
.await;
|
||||
return Ok(response.body);
|
||||
self.models
|
||||
.observe_models_etag(response.response_metadata.models_etag.as_deref());
|
||||
return Ok(ResponseDispatchResponse {
|
||||
body: response.body,
|
||||
response_headers: response.response_metadata.client_headers,
|
||||
});
|
||||
}
|
||||
Err(error) if is_rate_limit_upstream_error(&error) => {
|
||||
exhausted_accounts.record_rate_limited(
|
||||
@@ -644,7 +695,6 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
||||
exhausted_accounts.record_quota_exhausted(
|
||||
@@ -654,7 +704,6 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) if is_auth_upstream_error(&error) => {
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
@@ -668,7 +717,6 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, account_status)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
||||
exhausted_accounts.record_cloudflare_challenge(
|
||||
@@ -678,7 +726,6 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
||||
exhausted_accounts.record_cloudflare_path_blocked(
|
||||
@@ -688,28 +735,11 @@ impl ResponseDispatchService {
|
||||
self.cloudflare
|
||||
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
||||
let upstream_error = upstream_error_body(&error);
|
||||
if let Some(exhausted) =
|
||||
exhausted_accounts.model_unsupported_retry_exhausted(upstream_error.clone())
|
||||
{
|
||||
let error = ResponseDispatchError::from_exhausted_account(exhausted);
|
||||
self.record_compact_dispatch_error(
|
||||
request_id,
|
||||
client_api_key_id.as_deref(),
|
||||
requested_model,
|
||||
started_at,
|
||||
Some(&release_account_id),
|
||||
&error,
|
||||
)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
exhausted_accounts
|
||||
.record_model_unsupported(Some(&release_account_id), upstream_error);
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) if is_banned_upstream_error(&error) => {
|
||||
exhausted_accounts.record_auth_failure(
|
||||
@@ -721,9 +751,15 @@ impl ResponseDispatchService {
|
||||
self.account_pool
|
||||
.set_status(&release_account_id, AccountStatus::Banned)
|
||||
.await;
|
||||
excluded_account_ids.push(release_account_id);
|
||||
}
|
||||
Err(error) => {
|
||||
if is_retryable_account_transport_error(&error) {
|
||||
exhausted_accounts.record_upstream_unavailable(
|
||||
Some(&release_account_id),
|
||||
upstream_error_body(&error),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let error = ResponseDispatchError::Upstream(error);
|
||||
self.record_compact_dispatch_error(
|
||||
request_id,
|
||||
|
||||
@@ -226,6 +226,8 @@ impl AccountManageService {
|
||||
}
|
||||
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let installation_id =
|
||||
account_id.map(|account_id| self.account_pseudonymizer.installation_id(account_id));
|
||||
let context = crate::upstream::openai::transport::CodexRequestContext {
|
||||
access_token,
|
||||
account_id,
|
||||
@@ -238,8 +240,12 @@ impl AccountManageService {
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: None,
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
installation_id: installation_id.as_deref(),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
|
||||
match self.codex.fetch_usage(context).await {
|
||||
|
||||
@@ -82,14 +82,11 @@ impl AccountManageService {
|
||||
plan_type: plan_type.to_string(),
|
||||
access_token: account.access_token.expose_secret().to_string(),
|
||||
account_id: account.account_id.clone(),
|
||||
installation_id: self.account_pseudonymizer.installation_id(&account.id),
|
||||
};
|
||||
if let Err(error) = self
|
||||
.models
|
||||
.refresh_backend_models_with_installation_id(
|
||||
&[plan_account],
|
||||
&request_id,
|
||||
self.installation_id.as_deref(),
|
||||
)
|
||||
.refresh_backend_models(&[plan_account], &request_id)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -121,7 +118,7 @@ impl AccountManageService {
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
let installation_id = self.installation_id.clone();
|
||||
let installation_id = self.account_pseudonymizer.installation_id(&account.id);
|
||||
let codex = self.codex.clone();
|
||||
let service = self.clone();
|
||||
let stored_account_id = account.id.clone();
|
||||
@@ -161,8 +158,12 @@ impl AccountManageService {
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id: installation_id.as_deref(),
|
||||
installation_id: Some(&installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
|
||||
let outcome = match codex.create_response_stream(&request, context).await {
|
||||
@@ -265,6 +266,8 @@ fn test_responses_request(model: String) -> CodexResponsesRequest {
|
||||
]
|
||||
})],
|
||||
);
|
||||
request.set_stream(true);
|
||||
request.set_store(false);
|
||||
request.force_http_sse = true;
|
||||
request
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ impl AccountManageService {
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let token = stored.access_token.expose_secret().to_string();
|
||||
let cookie_header = self.usage_cookie_header(account_id).await;
|
||||
let installation_id = self.account_pseudonymizer.installation_id(account_id);
|
||||
let context = crate::upstream::openai::transport::CodexRequestContext {
|
||||
access_token: &token,
|
||||
account_id: stored.account_id.as_deref(),
|
||||
@@ -46,8 +47,12 @@ impl AccountManageService {
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
installation_id: Some(&installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
let raw = self
|
||||
.codex
|
||||
|
||||
@@ -14,6 +14,7 @@ use crate::{
|
||||
refresh::{RedisRefreshLeaseStore, RuntimeRefreshPolicy},
|
||||
store::PgAccountStore,
|
||||
},
|
||||
infra::identity::AccountPseudonymizer,
|
||||
models::service::ModelService,
|
||||
upstream::openai::{token_client::TokenRefresher, transport::CodexBackendClient},
|
||||
};
|
||||
@@ -32,7 +33,7 @@ pub struct AccountManageService {
|
||||
pub(crate) refresh_lease_owner_prefix: String,
|
||||
pub(crate) oauth: oauth::AccountOAuthService,
|
||||
pub(crate) refresh_policy: RuntimeRefreshPolicy,
|
||||
pub(crate) installation_id: Option<String>,
|
||||
pub(crate) account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
}
|
||||
|
||||
pub struct AccountManageServiceParts {
|
||||
@@ -45,7 +46,7 @@ pub struct AccountManageServiceParts {
|
||||
pub refresh_leases: RedisRefreshLeaseStore,
|
||||
pub oauth: oauth::AccountOAuthService,
|
||||
pub refresh_policy: RuntimeRefreshPolicy,
|
||||
pub installation_id: Option<String>,
|
||||
pub account_pseudonymizer: Arc<AccountPseudonymizer>,
|
||||
}
|
||||
|
||||
impl AccountManageService {
|
||||
@@ -64,7 +65,7 @@ impl AccountManageService {
|
||||
),
|
||||
oauth: parts.oauth,
|
||||
refresh_policy: parts.refresh_policy,
|
||||
installation_id: parts.installation_id,
|
||||
account_pseudonymizer: parts.account_pseudonymizer,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ pub struct AccountPoolService {
|
||||
store: Arc<dyn AccountStore>,
|
||||
usage_store: Arc<dyn AccountUsageStore>,
|
||||
request_interval: Arc<std::sync::RwLock<StdDuration>>,
|
||||
candidate_changed: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
/// 一次请求对账号池 slot 的唯一所有权凭据。
|
||||
@@ -79,6 +80,14 @@ pub struct AccountLease {
|
||||
slot: AccountSlotLease,
|
||||
}
|
||||
|
||||
/// 冻结候选后的单账号实时租用结果。
|
||||
#[derive(Debug)]
|
||||
pub enum AccountCandidateLease {
|
||||
Acquired(Box<AccountLease>),
|
||||
Busy,
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
impl AccountLease {
|
||||
fn new(service: AccountPoolService, acquired: AcquiredAccount) -> Self {
|
||||
let slot = AccountSlotLease::new(service, acquired.account.id.clone(), acquired.slot_id);
|
||||
@@ -190,6 +199,7 @@ impl AccountPoolService {
|
||||
request_interval: Arc::new(std::sync::RwLock::new(StdDuration::from_millis(
|
||||
request_interval_ms,
|
||||
))),
|
||||
candidate_changed: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,6 +310,7 @@ impl AccountPoolService {
|
||||
let Some(released) = released else {
|
||||
return;
|
||||
};
|
||||
self.candidate_changed.notify_one();
|
||||
if !record_request_usage {
|
||||
return;
|
||||
}
|
||||
@@ -387,6 +398,47 @@ impl AccountPoolService {
|
||||
lease
|
||||
}
|
||||
|
||||
/// 冻结请求开始时跨全部订阅层级的有序候选账号 ID。
|
||||
pub async fn candidate_snapshot(&self, request: &AccountAcquireRequest) -> Vec<String> {
|
||||
let refresh = self
|
||||
.pool
|
||||
.lock()
|
||||
.await
|
||||
.candidate_snapshot_with_status_refresh(request);
|
||||
self.persist_runtime_account_states(refresh.refreshed_accounts)
|
||||
.await;
|
||||
refresh.account_ids
|
||||
}
|
||||
|
||||
/// 按候选 ID 重新读取实时状态并租用账号。
|
||||
pub async fn acquire_candidate(
|
||||
&self,
|
||||
model: &str,
|
||||
account_id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AccountCandidateLease {
|
||||
let refresh = self
|
||||
.pool
|
||||
.lock()
|
||||
.await
|
||||
.acquire_candidate_with_status_refresh(model, account_id, now);
|
||||
let decision = match refresh.decision {
|
||||
AccountCandidateAcquireDecision::Acquired(acquired) => AccountCandidateLease::Acquired(
|
||||
Box::new(AccountLease::new(self.clone(), *acquired)),
|
||||
),
|
||||
AccountCandidateAcquireDecision::Busy => AccountCandidateLease::Busy,
|
||||
AccountCandidateAcquireDecision::Unavailable => AccountCandidateLease::Unavailable,
|
||||
};
|
||||
self.persist_runtime_account_states(refresh.refreshed_accounts)
|
||||
.await;
|
||||
decision
|
||||
}
|
||||
|
||||
/// 等待任一账号 slot 或运行状态发生变化后重试 Busy 候选。
|
||||
pub async fn wait_for_candidate_change(&self) {
|
||||
self.candidate_changed.notified().await;
|
||||
}
|
||||
|
||||
/// 按订阅计划各选一个可用账号,用于刷新模型列表。
|
||||
pub async fn distinct_plan_accounts(
|
||||
&self,
|
||||
|
||||
@@ -8,6 +8,25 @@ pub(super) struct AccountAcquireWithStatusRefresh {
|
||||
pub(super) refreshed_accounts: Vec<RefreshedAccountState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct AccountCandidateSnapshotWithStatusRefresh {
|
||||
pub(super) account_ids: Vec<String>,
|
||||
pub(super) refreshed_accounts: Vec<RefreshedAccountState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct AccountCandidateAcquireWithStatusRefresh {
|
||||
pub(super) decision: AccountCandidateAcquireDecision,
|
||||
pub(super) refreshed_accounts: Vec<RefreshedAccountState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum AccountCandidateAcquireDecision {
|
||||
Acquired(Box<AcquiredAccount>),
|
||||
Busy,
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct DistinctPlanAccountsWithStatusRefresh {
|
||||
pub(super) accounts: Vec<DistinctPlanAccount>,
|
||||
@@ -422,6 +441,65 @@ impl AccountPool {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn candidate_snapshot_with_status_refresh(
|
||||
&mut self,
|
||||
request: &AccountAcquireRequest,
|
||||
) -> AccountCandidateSnapshotWithStatusRefresh {
|
||||
self.cleanup_stale_slots(request.now);
|
||||
let refreshed_accounts = self.refresh_account_statuses(request.now);
|
||||
let candidates = self.snapshot_candidates(request);
|
||||
let mut ordered = self.scheduler.order(
|
||||
self.options.rotation_strategy,
|
||||
&candidates,
|
||||
&|account_id| self.slot_count(account_id),
|
||||
request.now,
|
||||
);
|
||||
ordered.sort_by_key(|account| {
|
||||
candidates::tier_rank(account.plan_type.as_deref(), &self.options.tier_priority)
|
||||
});
|
||||
|
||||
let mut account_ids = Vec::with_capacity(ordered.len());
|
||||
if let Some(preferred_account_id) = request.preferred_account_id.as_deref() {
|
||||
if let Some(index) = ordered
|
||||
.iter()
|
||||
.position(|account| account.id == preferred_account_id)
|
||||
{
|
||||
account_ids.push(ordered.remove(index).id);
|
||||
}
|
||||
}
|
||||
account_ids.extend(ordered.into_iter().map(|account| account.id));
|
||||
AccountCandidateSnapshotWithStatusRefresh {
|
||||
account_ids,
|
||||
refreshed_accounts,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn acquire_candidate_with_status_refresh(
|
||||
&mut self,
|
||||
model: &str,
|
||||
account_id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AccountCandidateAcquireWithStatusRefresh {
|
||||
let request = AccountAcquireRequest::new(model, now).with_required_account_id(account_id);
|
||||
self.cleanup_stale_slots(now);
|
||||
let refreshed_accounts = self.refresh_account_statuses(now);
|
||||
let snapshot_eligible = !self.snapshot_candidates(&request).is_empty();
|
||||
let decision = if !snapshot_eligible {
|
||||
AccountCandidateAcquireDecision::Unavailable
|
||||
} else if self.slot_count(account_id) >= self.options.max_concurrent_per_account {
|
||||
AccountCandidateAcquireDecision::Busy
|
||||
} else {
|
||||
match self.acquire_with_status_refresh(&request).acquired {
|
||||
Some(acquired) => AccountCandidateAcquireDecision::Acquired(Box::new(acquired)),
|
||||
None => AccountCandidateAcquireDecision::Unavailable,
|
||||
}
|
||||
};
|
||||
AccountCandidateAcquireWithStatusRefresh {
|
||||
decision,
|
||||
refreshed_accounts,
|
||||
}
|
||||
}
|
||||
|
||||
fn refresh_account_statuses(&mut self, now: DateTime<Utc>) -> Vec<RefreshedAccountState> {
|
||||
let mut refreshed_accounts = Vec::new();
|
||||
let mut expired_account_ids = Vec::new();
|
||||
@@ -707,6 +785,27 @@ impl AccountPool {
|
||||
)
|
||||
}
|
||||
|
||||
fn snapshot_candidates(&self, request: &AccountAcquireRequest) -> Vec<&Account> {
|
||||
let filter = CandidateFilter {
|
||||
max_concurrent_per_account: self.options.max_concurrent_per_account,
|
||||
skip_quota_limited: self.options.skip_quota_limited,
|
||||
tier_priority: &self.options.tier_priority,
|
||||
model_plan_allowlist: &self.options.model_plan_allowlist,
|
||||
fetched_model_plan_types: &self.options.fetched_model_plan_types,
|
||||
slot_count: &|account_id| self.slot_count(account_id),
|
||||
};
|
||||
candidates::snapshot(
|
||||
self.accounts.values(),
|
||||
&filter,
|
||||
&CandidateRequest {
|
||||
model: &request.model,
|
||||
exclude_account_ids: &request.exclude_account_ids,
|
||||
required_account_id: request.required_account_id.as_deref(),
|
||||
now: request.now,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// 按当前策略从候选集合中选择一个账号(委托 [`AccountScheduler::select`])。
|
||||
fn select_candidate(&self, candidates: &[&Account], now: DateTime<Utc>) -> Option<Account> {
|
||||
let slot_count = |account_id: &str| self.slots.get(account_id).map_or(0, VecDeque::len);
|
||||
|
||||
@@ -35,7 +35,7 @@ pub struct QuotaRefreshService {
|
||||
codex: Arc<CodexBackendClient>,
|
||||
min_refresh_interval_secs: u64,
|
||||
request_spacing: Duration,
|
||||
installation_id: Option<String>,
|
||||
account_pseudonymizer: Arc<crate::infra::identity::AccountPseudonymizer>,
|
||||
cookie_store: Option<PgCookieStore>,
|
||||
account_pool: Option<Arc<AccountPoolService>>,
|
||||
}
|
||||
@@ -46,6 +46,7 @@ impl QuotaRefreshService {
|
||||
store: PgAccountStore,
|
||||
usage_store: PgAccountUsageStore,
|
||||
codex: Arc<CodexBackendClient>,
|
||||
account_pseudonymizer: Arc<crate::infra::identity::AccountPseudonymizer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
@@ -53,7 +54,7 @@ impl QuotaRefreshService {
|
||||
codex,
|
||||
min_refresh_interval_secs: MIN_REFRESH_INTERVAL_SECS,
|
||||
request_spacing: Self::default_request_spacing(),
|
||||
installation_id: None,
|
||||
account_pseudonymizer,
|
||||
cookie_store: None,
|
||||
account_pool: None,
|
||||
}
|
||||
@@ -68,6 +69,7 @@ impl QuotaRefreshService {
|
||||
store: PgAccountStore,
|
||||
usage_store: PgAccountUsageStore,
|
||||
codex: Arc<CodexBackendClient>,
|
||||
account_pseudonymizer: Arc<crate::infra::identity::AccountPseudonymizer>,
|
||||
min_refresh_interval_secs: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
@@ -76,7 +78,7 @@ impl QuotaRefreshService {
|
||||
codex,
|
||||
min_refresh_interval_secs,
|
||||
request_spacing: Self::default_request_spacing(),
|
||||
installation_id: None,
|
||||
account_pseudonymizer,
|
||||
cookie_store: None,
|
||||
account_pool: None,
|
||||
}
|
||||
@@ -88,12 +90,6 @@ impl QuotaRefreshService {
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置 Codex installation id。
|
||||
pub fn with_installation_id(mut self, installation_id: Option<String>) -> Self {
|
||||
self.installation_id = installation_id.filter(|id| !id.trim().is_empty());
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置 usage 请求可复用的账号 Cookie 存储。
|
||||
pub fn with_cookie_store(mut self, cookie_store: PgCookieStore) -> Self {
|
||||
self.cookie_store = Some(cookie_store);
|
||||
@@ -155,6 +151,7 @@ impl QuotaRefreshService {
|
||||
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let cookie_header = self.usage_cookie_header(&account.id).await;
|
||||
let installation_id = self.account_pseudonymizer.installation_id(&account.id);
|
||||
match self
|
||||
.codex
|
||||
.fetch_usage(CodexRequestContext {
|
||||
@@ -169,8 +166,12 @@ impl QuotaRefreshService {
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
installation_id: self.installation_id.as_deref(),
|
||||
installation_id: Some(&installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -68,14 +68,40 @@ pub fn filter<'a>(
|
||||
candidates
|
||||
}
|
||||
|
||||
/// 冻结一次客户端请求可遍历的完整候选集。
|
||||
///
|
||||
/// 与 [`filter`] 不同,这里不按最高订阅层级收窄,也不因瞬时 slot 已满而删除账号。
|
||||
/// 层级和 slot 只影响后续顺序与租用时机,不能缩小请求级 failover 边界。
|
||||
pub fn snapshot<'a>(
|
||||
accounts: impl Iterator<Item = &'a Account>,
|
||||
filter: &CandidateFilter<'_>,
|
||||
request: &CandidateRequest<'_>,
|
||||
) -> Vec<&'a Account> {
|
||||
let mut candidates = accounts
|
||||
.filter(|account| is_snapshot_eligible(account, filter, request))
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(required_account_id) = request.required_account_id {
|
||||
candidates.retain(|account| account.id == required_account_id);
|
||||
}
|
||||
candidates
|
||||
}
|
||||
|
||||
/// 判断账号是否满足全部基础可用条件。
|
||||
fn is_base_available(
|
||||
account: &Account,
|
||||
filter: &CandidateFilter<'_>,
|
||||
request: &CandidateRequest<'_>,
|
||||
) -> bool {
|
||||
account.status == AccountStatus::Active
|
||||
is_snapshot_eligible(account, filter, request)
|
||||
&& (filter.slot_count)(&account.id) < filter.max_concurrent_per_account
|
||||
}
|
||||
|
||||
fn is_snapshot_eligible(
|
||||
account: &Account,
|
||||
filter: &CandidateFilter<'_>,
|
||||
request: &CandidateRequest<'_>,
|
||||
) -> bool {
|
||||
account.status == AccountStatus::Active
|
||||
&& quota_available_at(account, request.now, filter.skip_quota_limited)
|
||||
&& is_model_allowed(
|
||||
account,
|
||||
@@ -90,6 +116,12 @@ fn is_base_available(
|
||||
&& cloudflare_available_at(account, request.now)
|
||||
}
|
||||
|
||||
pub(crate) fn tier_rank(plan_type: Option<&str>, tier_priority: &[String]) -> usize {
|
||||
plan_type
|
||||
.and_then(|plan_type| tier_priority.iter().position(|tier| tier == plan_type))
|
||||
.unwrap_or(tier_priority.len())
|
||||
}
|
||||
|
||||
/// 判断模型是否允许该账号的订阅计划使用。
|
||||
fn is_model_allowed(
|
||||
account: &Account,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 账号运行时反馈:错误率与 TTFT 的 EWMA 统计。
|
||||
//!
|
||||
//! 每个账号维护两个指数加权移动平均(EWMA):错误率与首 token 延迟(TTFT,
|
||||
//! 每个账号维护两个指数加权移动平均(EWMA):错误率与首个有效输出事件延迟(TTFT,
|
||||
//! 毫秒)。请求结束后经 [`FeedbackStats::report`] 回灌,直接进 [`super::score`]
|
||||
//! 打分并可驱动 sticky escape。统计以 `AtomicU64`(存 `f64` 位模式)+ CAS 无锁更新,
|
||||
//! 对齐 sub2api 的设计,避免选择热路径上的锁与 DB 往返。
|
||||
@@ -127,7 +127,7 @@ impl FeedbackStats {
|
||||
}
|
||||
}
|
||||
|
||||
/// 回灌一次请求结果:成功/失败 + 首 token 延迟(毫秒)。
|
||||
/// 回灌一次请求结果:成功/失败 + 首个有效输出事件延迟(毫秒)。
|
||||
pub fn report(&self, account_id: &str, success: bool, first_token_ms: Option<u64>) {
|
||||
// 快路径:账号已存在时只需读锁 + 原子更新。
|
||||
{
|
||||
|
||||
@@ -129,6 +129,32 @@ impl AccountScheduler {
|
||||
select_with_strategy(strategy, &input, &mut cursor)
|
||||
}
|
||||
|
||||
/// 按当前策略为完整候选集排序,供单个请求依次执行账号 failover。
|
||||
pub fn order(
|
||||
&self,
|
||||
strategy: RotationStrategy,
|
||||
candidates: &[&Account],
|
||||
slot_count: &dyn Fn(&str) -> usize,
|
||||
now: DateTime<Utc>,
|
||||
) -> Vec<Account> {
|
||||
let input = SelectionInput {
|
||||
candidates,
|
||||
slot_count,
|
||||
feedback: &self.feedback,
|
||||
weights: &self.weights,
|
||||
now,
|
||||
};
|
||||
let mut cursor = self.lock_cursor();
|
||||
match strategy {
|
||||
RotationStrategy::Smart => strategy::smart::order(&input, &mut cursor),
|
||||
RotationStrategy::QuotaResetPriority => {
|
||||
strategy::quota_reset::order(&input, &mut cursor)
|
||||
}
|
||||
RotationStrategy::RoundRobin => strategy::round_robin::order(&input, &mut cursor),
|
||||
RotationStrategy::Sticky => strategy::sticky::order(&input),
|
||||
}
|
||||
}
|
||||
|
||||
/// 回灌一次请求结果到运行时 EWMA 反馈(错误率 / TTFT),供 Smart 打分。
|
||||
pub fn report_feedback(&self, account_id: &str, success: bool, first_token_ms: Option<u64>) {
|
||||
self.feedback.report(account_id, success, first_token_ms);
|
||||
@@ -181,6 +207,28 @@ pub(crate) fn select_by(
|
||||
Some((*sorted[index]).clone())
|
||||
}
|
||||
|
||||
pub(crate) fn order_by(
|
||||
candidates: &[&Account],
|
||||
cursor: &mut usize,
|
||||
compare: impl Fn(&Account, &Account) -> Ordering,
|
||||
) -> Vec<Account> {
|
||||
let mut sorted = candidates.to_vec();
|
||||
sorted.sort_by(|a, b| compare(a, b));
|
||||
let rotation = *cursor;
|
||||
*cursor = cursor.wrapping_add(1);
|
||||
|
||||
let mut start = 0;
|
||||
while start < sorted.len() {
|
||||
let mut end = start + 1;
|
||||
while end < sorted.len() && compare(sorted[start], sorted[end]) == Ordering::Equal {
|
||||
end += 1;
|
||||
}
|
||||
sorted[start..end].rotate_left(rotation % (end - start));
|
||||
start = end;
|
||||
}
|
||||
sorted.into_iter().cloned().collect()
|
||||
}
|
||||
|
||||
pub(crate) fn account_window_token_count(account: &Account) -> u64 {
|
||||
account
|
||||
.window_input_tokens
|
||||
|
||||
@@ -11,13 +11,20 @@
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use crate::fleet::account::Account;
|
||||
use crate::fleet::scheduler::{compare_last_used, compare_window_reset, select_by, SelectionInput};
|
||||
use crate::fleet::scheduler::{
|
||||
compare_last_used, compare_window_reset, order_by, select_by, SelectionInput,
|
||||
};
|
||||
|
||||
/// QuotaResetPriority 策略选择。
|
||||
pub fn select(input: &SelectionInput<'_>, cursor: &mut usize) -> Option<Account> {
|
||||
select_by(input.candidates, cursor, compare_quota_reset_priority)
|
||||
}
|
||||
|
||||
/// 按额度窗口重置优先级为完整候选集排序。
|
||||
pub fn order(input: &SelectionInput<'_>, cursor: &mut usize) -> Vec<Account> {
|
||||
order_by(input.candidates, cursor, compare_quota_reset_priority)
|
||||
}
|
||||
|
||||
/// 配额重置优先级比较:升序,最优(最应先用)排在最前。
|
||||
fn compare_quota_reset_priority(a: &Account, b: &Account) -> Ordering {
|
||||
a.quota_limit_reached
|
||||
|
||||
@@ -17,3 +17,19 @@ pub fn select(input: &SelectionInput<'_>, cursor: &mut usize) -> Option<Account>
|
||||
*cursor = cursor.wrapping_add(1);
|
||||
Some((*candidates[index]).clone())
|
||||
}
|
||||
|
||||
/// 从当前轮转位置开始返回完整候选顺序。
|
||||
pub fn order(input: &SelectionInput<'_>, cursor: &mut usize) -> Vec<Account> {
|
||||
let mut ordered = input
|
||||
.candidates
|
||||
.iter()
|
||||
.map(|account| (**account).clone())
|
||||
.collect::<Vec<_>>();
|
||||
if ordered.is_empty() {
|
||||
return ordered;
|
||||
}
|
||||
let candidate_count = ordered.len();
|
||||
ordered.rotate_left(*cursor % candidate_count);
|
||||
*cursor = cursor.wrapping_add(1);
|
||||
ordered
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ pub struct ScoreWeights {
|
||||
pub window_requests: f64,
|
||||
/// 错误率 EWMA(越健康越优先)。
|
||||
pub error_rate: f64,
|
||||
/// 首 token 延迟 EWMA(越快越优先)。
|
||||
/// 首个有效输出事件延迟 EWMA(越快越优先)。
|
||||
pub ttft: f64,
|
||||
/// 窗口重置临近度(越快重置越优先,use-it-or-lose-it,默认关闭)。
|
||||
pub reset: f64,
|
||||
@@ -97,6 +97,35 @@ pub fn select(input: &SelectionInput<'_>, cursor: &mut usize) -> Option<Account>
|
||||
Some((*input.candidates[account_index]).clone())
|
||||
}
|
||||
|
||||
/// 按 Smart 得分为完整候选集排序;同分账号沿用调度器轮转游标破并列。
|
||||
pub fn order(input: &SelectionInput<'_>, cursor: &mut usize) -> Vec<Account> {
|
||||
let mut ranked = rank_candidate_refs(
|
||||
input.candidates,
|
||||
input.weights,
|
||||
input.now.timestamp(),
|
||||
input.slot_count,
|
||||
input.feedback,
|
||||
);
|
||||
let rotation = *cursor;
|
||||
*cursor = cursor.wrapping_add(1);
|
||||
|
||||
let mut start = 0;
|
||||
while start < ranked.len() {
|
||||
let score = ranked[start].1.total.to_bits();
|
||||
let mut end = start + 1;
|
||||
while end < ranked.len() && ranked[end].1.total.to_bits() == score {
|
||||
end += 1;
|
||||
}
|
||||
ranked[start..end].rotate_left(rotation % (end - start));
|
||||
start = end;
|
||||
}
|
||||
|
||||
ranked
|
||||
.into_iter()
|
||||
.map(|(index, _)| (*input.candidates[index]).clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 对候选集合按加权得分排序(降序),返回 `(账号索引, 打分明细)` 列表。
|
||||
///
|
||||
/// 只做打分与排序,最终的并列破并列交给 [`select`]。公开供测试直接断言排序结果。
|
||||
|
||||
@@ -25,3 +25,14 @@ pub fn select(input: &SelectionInput<'_>, _cursor: &mut usize) -> Option<Account
|
||||
}
|
||||
Some(selected.clone())
|
||||
}
|
||||
|
||||
/// 按最近使用时间降序返回完整候选顺序。
|
||||
pub fn order(input: &SelectionInput<'_>) -> Vec<Account> {
|
||||
let mut ordered = input
|
||||
.candidates
|
||||
.iter()
|
||||
.map(|account| (**account).clone())
|
||||
.collect::<Vec<_>>();
|
||||
ordered.sort_by(|a, b| compare_last_used(b.last_used_at.as_deref(), a.last_used_at.as_deref()));
|
||||
ordered
|
||||
}
|
||||
|
||||
@@ -5,9 +5,63 @@ use argon2::{
|
||||
Argon2,
|
||||
};
|
||||
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
|
||||
use hmac::{Hmac, Mac};
|
||||
use rand::Rng;
|
||||
use sha2::{Digest, Sha256};
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
/// 持久密钥驱动的账号作用域伪名原语。
|
||||
#[derive(Clone)]
|
||||
pub struct AccountPseudonymizer {
|
||||
template: HmacSha256,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AccountPseudonymizer {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountPseudonymizer")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl AccountPseudonymizer {
|
||||
/// 使用服务端持久密钥构造伪名器。
|
||||
pub fn new(secret: [u8; 32]) -> Self {
|
||||
let template =
|
||||
HmacSha256::new_from_slice(&secret).expect("HMAC-SHA256 accepts every 32-byte key");
|
||||
Self { template }
|
||||
}
|
||||
|
||||
/// 按字段域和账号派生稳定伪名。
|
||||
pub fn scoped(&self, domain: &str, account_id: Option<&str>, value: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(self.digest(domain, account_id, value))
|
||||
}
|
||||
|
||||
/// 按账号派生 UUID 形态的 installation ID。
|
||||
pub fn installation_id(&self, account_id: &str) -> String {
|
||||
let digest = self.digest("installation-id", Some(account_id), "");
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes.copy_from_slice(&digest[..16]);
|
||||
bytes[6] = (bytes[6] & 0x0f) | 0x40;
|
||||
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
||||
Uuid::from_bytes(bytes).to_string()
|
||||
}
|
||||
|
||||
fn digest(&self, domain: &str, account_id: Option<&str>, value: &str) -> Vec<u8> {
|
||||
let mut mac = self.template.clone();
|
||||
mac.update(domain.as_bytes());
|
||||
mac.update(b"\0");
|
||||
if let Some(account_id) = account_id {
|
||||
mac.update(account_id.as_bytes());
|
||||
}
|
||||
mac.update(b"\0");
|
||||
mac.update(value.as_bytes());
|
||||
mac.finalize().into_bytes().to_vec()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// AuthError / AuthResult
|
||||
|
||||
+43
-66
@@ -1,13 +1,14 @@
|
||||
//! 数据目录、installation ID 等路径辅助。
|
||||
//! 数据目录与持久身份密钥路径辅助。
|
||||
|
||||
use std::{
|
||||
fs, io,
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use uuid::Uuid;
|
||||
use rand::Rng;
|
||||
|
||||
const INSTALLATION_ID_FILE_NAME: &str = "installation_id";
|
||||
const IDENTITY_SECRET_FILE_NAME: &str = "identity_hmac_secret";
|
||||
|
||||
fn data_dir() -> PathBuf {
|
||||
dirs::data_local_dir().unwrap_or_else(|| PathBuf::from(".runtime/data"))
|
||||
@@ -20,73 +21,49 @@ pub fn ensure_data_dir() -> std::io::Result<PathBuf> {
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
fn codex_desktop_installation_id_path() -> Option<PathBuf> {
|
||||
dirs::home_dir().map(|home| home.join(".codex").join(INSTALLATION_ID_FILE_NAME))
|
||||
}
|
||||
/// 读取或创建账号身份隔离使用的 256-bit HMAC 密钥。
|
||||
pub fn load_or_create_identity_secret(data_dir: &Path) -> io::Result<[u8; 32]> {
|
||||
let path = data_dir.join(IDENTITY_SECRET_FILE_NAME);
|
||||
match read_identity_secret(&path) {
|
||||
Ok(Some(secret)) => return Ok(secret),
|
||||
Ok(None) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
|
||||
fn data_installation_id_path(data_dir: &Path) -> PathBuf {
|
||||
data_dir.join(INSTALLATION_ID_FILE_NAME)
|
||||
}
|
||||
|
||||
/// 按读取优先级读取或生成 installation ID。
|
||||
///
|
||||
/// 顺序为本应用数据目录文件、真实 Codex Desktop 文件、生成并写入本应用数据目录。
|
||||
/// 当 `data_dir` 为 `None` 且没有可读文件时,返回一个不会持久化的新 UUID。
|
||||
pub fn load_or_create_installation_id(data_dir: Option<&Path>) -> io::Result<String> {
|
||||
let codex_path = codex_desktop_installation_id_path();
|
||||
let data_path = data_dir.map(data_installation_id_path);
|
||||
load_or_create_installation_id_from_paths(codex_path.as_deref(), data_path.as_deref())
|
||||
}
|
||||
|
||||
fn load_or_create_installation_id_from_paths(
|
||||
codex_desktop_path: Option<&Path>,
|
||||
data_path: Option<&Path>,
|
||||
) -> io::Result<String> {
|
||||
if let Some(path) = data_path {
|
||||
if let Some(id) = read_installation_id(path)? {
|
||||
return Ok(id);
|
||||
let mut secret = [0u8; 32];
|
||||
rand::rng().fill_bytes(&mut secret);
|
||||
let mut options = fs::OpenOptions::new();
|
||||
options.write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.mode(0o600);
|
||||
}
|
||||
match options.open(&path) {
|
||||
Ok(mut file) => {
|
||||
file.write_all(hex::encode(secret).as_bytes())?;
|
||||
file.sync_all()?;
|
||||
Ok(secret)
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(path) = codex_desktop_path {
|
||||
if let Some(id) = read_installation_id(path)? {
|
||||
if let Some(data_path) = data_path {
|
||||
persist_installation_id(data_path, &id)?;
|
||||
}
|
||||
return Ok(id);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(path) = data_path {
|
||||
let generated = generate_installation_id();
|
||||
persist_installation_id(path, &generated)?;
|
||||
return Ok(generated);
|
||||
}
|
||||
|
||||
Ok(generate_installation_id())
|
||||
}
|
||||
|
||||
fn read_installation_id(path: &Path) -> io::Result<Option<String>> {
|
||||
match fs::read_to_string(path) {
|
||||
Ok(content) => Ok(parse_installation_id(&content)),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => read_identity_secret(&path)?
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid identity secret")),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn persist_installation_id(path: &Path, installation_id: &str) -> io::Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
fs::write(path, installation_id)
|
||||
}
|
||||
|
||||
fn parse_installation_id(raw: &str) -> Option<String> {
|
||||
let trimmed = raw.trim();
|
||||
Uuid::parse_str(trimmed).ok()?;
|
||||
Some(trimmed.to_string())
|
||||
}
|
||||
|
||||
fn generate_installation_id() -> String {
|
||||
Uuid::new_v4().to_string()
|
||||
fn read_identity_secret(path: &Path) -> io::Result<Option<[u8; 32]>> {
|
||||
let raw = match fs::read_to_string(path) {
|
||||
Ok(raw) => raw,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let decoded = hex::decode(raw.trim())
|
||||
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
|
||||
let secret = decoded.try_into().map_err(|_| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"identity secret must be 32 bytes",
|
||||
)
|
||||
})?;
|
||||
Ok(Some(secret))
|
||||
}
|
||||
|
||||
@@ -80,6 +80,8 @@ pub struct ModelRefreshPlanAccount {
|
||||
pub access_token: String,
|
||||
/// 上游账号标识。
|
||||
pub account_id: Option<String>,
|
||||
/// 按账号作用域派生的 installation ID。
|
||||
pub installation_id: String,
|
||||
}
|
||||
|
||||
/// 模型目录服务。
|
||||
@@ -90,6 +92,9 @@ pub struct ModelService {
|
||||
catalog: Arc<RwLock<ModelCatalog>>,
|
||||
store: Option<Arc<dyn ModelSnapshotStore>>,
|
||||
upstream_client: Option<Arc<dyn CodexModelCatalogClient>>,
|
||||
models_etag: Arc<RwLock<Option<String>>>,
|
||||
etag_change_tx: watch::Sender<u64>,
|
||||
refresh_lock: Arc<tokio::sync::Mutex<()>>,
|
||||
}
|
||||
|
||||
impl ModelService {
|
||||
@@ -100,15 +105,44 @@ impl ModelService {
|
||||
upstream_client: Option<Arc<dyn CodexModelCatalogClient>>,
|
||||
) -> Self {
|
||||
let catalog = ModelCatalog::from_config(&config);
|
||||
let (etag_change_tx, _) = watch::channel(0);
|
||||
Self {
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
snapshots: Arc::new(RwLock::new(Vec::new())),
|
||||
catalog: Arc::new(RwLock::new(catalog)),
|
||||
store,
|
||||
upstream_client,
|
||||
models_etag: Arc::new(RwLock::new(None)),
|
||||
etag_change_tx,
|
||||
refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录 Responses 响应中观察到的模型目录 ETag。
|
||||
///
|
||||
/// 仅在值真正变化时通知后台刷新任务。
|
||||
pub fn observe_models_etag(&self, etag: Option<&str>) -> bool {
|
||||
let Some(etag) = etag.map(str::trim).filter(|etag| !etag.is_empty()) else {
|
||||
return false;
|
||||
};
|
||||
let mut current = self
|
||||
.models_etag
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if current.as_deref() == Some(etag) {
|
||||
return false;
|
||||
}
|
||||
*current = Some(etag.to_string());
|
||||
drop(current);
|
||||
self.etag_change_tx.send_modify(|revision| *revision += 1);
|
||||
true
|
||||
}
|
||||
|
||||
/// 订阅模型目录 ETag 变化。
|
||||
pub fn subscribe_models_etag_changes(&self) -> watch::Receiver<u64> {
|
||||
self.etag_change_tx.subscribe()
|
||||
}
|
||||
|
||||
/// 更新模型服务配置。
|
||||
pub fn update_config(&self, config: ModelConfig) {
|
||||
*self
|
||||
@@ -150,13 +184,13 @@ impl ModelService {
|
||||
self.cached_catalog()
|
||||
}
|
||||
|
||||
/// 使用运行时 installation id 刷新活跃账号对应的后端模型目录。
|
||||
pub async fn refresh_backend_models_with_installation_id(
|
||||
/// 刷新活跃账号对应的后端模型目录。
|
||||
pub async fn refresh_backend_models(
|
||||
&self,
|
||||
plan_accounts: &[ModelRefreshPlanAccount],
|
||||
request_id: &str,
|
||||
installation_id: Option<&str>,
|
||||
) -> Result<ModelRefreshResult, ModelServiceError> {
|
||||
let _refresh_guard = self.refresh_lock.lock().await;
|
||||
let store = self
|
||||
.store
|
||||
.as_ref()
|
||||
@@ -181,7 +215,7 @@ impl ModelService {
|
||||
access_token: &plan_account.access_token,
|
||||
account_id: plan_account.account_id.as_deref(),
|
||||
request_id,
|
||||
installation_id,
|
||||
installation_id: Some(&plan_account.installation_id),
|
||||
plan_type: &plan_account.plan_type,
|
||||
};
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ pub struct UsageRecord {
|
||||
pub upstream_request_id: Option<String>,
|
||||
/// 请求耗时毫秒。
|
||||
pub latency_ms: Option<i64>,
|
||||
/// 首 token 延迟毫秒。
|
||||
/// 首个有效输出事件延迟毫秒。
|
||||
pub first_token_ms: Option<i64>,
|
||||
/// 输入 token。
|
||||
pub input_tokens: Option<i64>,
|
||||
|
||||
@@ -166,9 +166,13 @@ pub struct RateLimitWindow {
|
||||
pub reset_at: Option<i64>,
|
||||
}
|
||||
|
||||
/// 复合限流信息,通常用于 code review 配额。
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
/// 单个计量项的限流信息。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RateLimitDetails {
|
||||
/// 稳定的计量项 ID。
|
||||
pub limit_id: String,
|
||||
/// 上游提供的可读名称。
|
||||
pub limit_name: Option<String>,
|
||||
/// 当前请求是否被允许。
|
||||
pub allowed: Option<bool>,
|
||||
/// 当前窗口是否已经触顶。
|
||||
@@ -179,15 +183,23 @@ pub struct RateLimitDetails {
|
||||
pub secondary: Option<RateLimitWindow>,
|
||||
}
|
||||
|
||||
/// 上游账户的 credits 快照。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreditsSnapshot {
|
||||
pub has_credits: bool,
|
||||
pub unlimited: bool,
|
||||
pub balance: Option<String>,
|
||||
}
|
||||
|
||||
/// 从 header 或内部事件中解析出的完整限流状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ParsedRateLimits {
|
||||
/// 主限流窗口。
|
||||
pub primary: Option<RateLimitWindow>,
|
||||
/// 次级限流窗口。
|
||||
pub secondary: Option<RateLimitWindow>,
|
||||
/// code review 限流窗口。
|
||||
pub code_review: Option<RateLimitDetails>,
|
||||
/// 以标准化 limit ID 为键的全部计量项。
|
||||
pub limits: BTreeMap<String, RateLimitDetails>,
|
||||
pub credits: Option<CreditsSnapshot>,
|
||||
pub plan_type: Option<String>,
|
||||
pub promo_message: Option<String>,
|
||||
pub rate_limit_reached_type: Option<String>,
|
||||
}
|
||||
|
||||
/// 从响应头对中解析限流信息。
|
||||
@@ -197,23 +209,60 @@ pub fn parse_rate_limit_headers(headers: &[(String, String)]) -> Option<ParsedRa
|
||||
normalized.insert(name.to_ascii_lowercase(), value.trim());
|
||||
}
|
||||
|
||||
let primary = parse_window_from_lookup(&normalized, "x-codex-primary");
|
||||
let secondary = parse_window_from_lookup(&normalized, "x-codex-secondary");
|
||||
let code_review = parse_details_from_lookup(&normalized, "x-codex-code-review")
|
||||
.or_else(|| parse_details_from_lookup(&normalized, "x-codex-review"))
|
||||
.or_else(|| parse_details_from_lookup(&normalized, "x-code-review"));
|
||||
let mut limit_ids = normalized
|
||||
.keys()
|
||||
.filter_map(|name| rate_limit_id_from_header_name(name))
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
limit_ids.insert("codex".to_string());
|
||||
let limits = limit_ids
|
||||
.into_iter()
|
||||
.filter_map(|limit_id| {
|
||||
let prefix = format!("x-{}", limit_id.replace('_', "-"));
|
||||
parse_details_from_lookup(&normalized, &prefix, &limit_id)
|
||||
.map(|details| (limit_id, details))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let credits = parse_credits_from_lookup(&normalized);
|
||||
let plan_type = lookup_non_empty(&normalized, "x-codex-plan-type");
|
||||
let promo_message = lookup_non_empty(&normalized, "x-codex-promo-message");
|
||||
let rate_limit_reached_type = lookup_non_empty(&normalized, "x-codex-rate-limit-reached-type");
|
||||
|
||||
if primary.is_none() && secondary.is_none() && code_review.is_none() {
|
||||
if limits.is_empty()
|
||||
&& credits.is_none()
|
||||
&& plan_type.is_none()
|
||||
&& promo_message.is_none()
|
||||
&& rate_limit_reached_type.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(ParsedRateLimits {
|
||||
primary,
|
||||
secondary,
|
||||
code_review,
|
||||
limits,
|
||||
credits,
|
||||
plan_type,
|
||||
promo_message,
|
||||
rate_limit_reached_type,
|
||||
})
|
||||
}
|
||||
|
||||
/// 判断响应头是否属于可被动同步的限流领域。
|
||||
pub(crate) fn is_rate_limit_header_name(name: &str) -> bool {
|
||||
let normalized = name.to_ascii_lowercase();
|
||||
normalized == "retry-after"
|
||||
|| normalized.contains("ratelimit")
|
||||
|| normalized.contains("rate-limit")
|
||||
|| matches!(
|
||||
normalized.as_str(),
|
||||
"x-codex-credits-has-credits"
|
||||
| "x-codex-credits-unlimited"
|
||||
| "x-codex-credits-balance"
|
||||
| "x-codex-plan-type"
|
||||
| "x-codex-promo-message"
|
||||
| "x-codex-rate-limit-reached-type"
|
||||
)
|
||||
|| rate_limit_id_from_header_name(&normalized).is_some()
|
||||
}
|
||||
|
||||
/// 从内部 `codex.rate_limits` 事件中解析限流信息。
|
||||
pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
if value
|
||||
@@ -224,33 +273,52 @@ pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
return None;
|
||||
}
|
||||
|
||||
let details = value.get("rate_limits").and_then(parse_details_from_object);
|
||||
let explicit_code_review = value
|
||||
.get("code_review_rate_limits")
|
||||
.and_then(parse_details_from_object)
|
||||
.or_else(|| {
|
||||
value
|
||||
.get("code_review_rate_limit")
|
||||
.and_then(parse_details_from_object)
|
||||
});
|
||||
let limit_id = rate_limit_name(value)
|
||||
.map(normalize_limit_id)
|
||||
.unwrap_or_else(|| "codex".to_string());
|
||||
let limit_name = value
|
||||
.get("metered_limit_name")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|_| value.get("limit_name"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(ToString::to_string);
|
||||
let details = value
|
||||
.get("rate_limits")
|
||||
.and_then(|details| parse_details_from_object(details, &limit_id, limit_name));
|
||||
let credits = value.get("credits").and_then(parse_credits_from_object);
|
||||
let plan_type = value
|
||||
.get("plan_type")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|plan| !plan.is_empty())
|
||||
.map(ToString::to_string);
|
||||
|
||||
let mut primary = details.and_then(|details| details.primary);
|
||||
let mut secondary = details.and_then(|details| details.secondary);
|
||||
let mut code_review = explicit_code_review;
|
||||
if details.is_some() && is_review_limit_name(rate_limit_name(value)) {
|
||||
code_review = code_review.or(details);
|
||||
primary = None;
|
||||
secondary = None;
|
||||
}
|
||||
|
||||
if primary.is_none() && secondary.is_none() && code_review.is_none() {
|
||||
if details.is_none() && credits.is_none() && plan_type.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut limits = BTreeMap::new();
|
||||
if let Some(details) = details {
|
||||
limits.insert(limit_id, details);
|
||||
}
|
||||
Some(ParsedRateLimits {
|
||||
primary,
|
||||
secondary,
|
||||
code_review,
|
||||
limits,
|
||||
credits,
|
||||
plan_type,
|
||||
promo_message: value
|
||||
.get("promo_message")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|message| !message.is_empty())
|
||||
.map(ToString::to_string),
|
||||
rate_limit_reached_type: value
|
||||
.get("rate_limit_reached_type")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|kind| !kind.is_empty())
|
||||
.map(ToString::to_string),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -267,41 +335,35 @@ pub fn rate_limit_quota(
|
||||
plan_type: Option<&str>,
|
||||
existing_quota: Option<&Value>,
|
||||
) -> Value {
|
||||
let mut snapshots = Vec::new();
|
||||
if let Some(snapshot) = quota_snapshot_from_windows(
|
||||
"core",
|
||||
None,
|
||||
None,
|
||||
rate_limits.primary,
|
||||
rate_limits.secondary,
|
||||
None,
|
||||
None,
|
||||
) {
|
||||
snapshots.push(snapshot);
|
||||
}
|
||||
|
||||
let has_code_review_update = rate_limits.code_review.is_some();
|
||||
if let Some(details) = rate_limits.code_review {
|
||||
if let Some(snapshot) = quota_snapshot_from_windows(
|
||||
"code_review",
|
||||
Some("code_review"),
|
||||
None,
|
||||
details.primary,
|
||||
details.secondary,
|
||||
details.allowed,
|
||||
details.limit_reached,
|
||||
) {
|
||||
snapshots.push(snapshot);
|
||||
}
|
||||
}
|
||||
let mut snapshots = rate_limits
|
||||
.limits
|
||||
.values()
|
||||
.filter_map(|details| {
|
||||
let (source, metered_feature) = quota_identity(&details.limit_id);
|
||||
let limit_name = details
|
||||
.limit_name
|
||||
.as_deref()
|
||||
.or_else(|| (source != "core").then_some(details.limit_id.as_str()));
|
||||
quota_snapshot_from_windows(
|
||||
source,
|
||||
limit_name,
|
||||
metered_feature,
|
||||
details.primary,
|
||||
details.secondary,
|
||||
details.allowed,
|
||||
details.limit_reached,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if let Some(existing_quota) = existing_quota {
|
||||
if let Some(existing_snapshots) = existing_quota.get("snapshots").and_then(Value::as_array)
|
||||
{
|
||||
for snapshot in existing_snapshots {
|
||||
let source = snapshot.get("source").and_then(Value::as_str);
|
||||
if source == Some("additional")
|
||||
|| (!has_code_review_update && source == Some("code_review"))
|
||||
if !rate_limits
|
||||
.limits
|
||||
.keys()
|
||||
.any(|limit_id| quota_snapshot_matches_limit(snapshot, limit_id))
|
||||
{
|
||||
snapshots.push(snapshot.clone());
|
||||
}
|
||||
@@ -312,42 +374,96 @@ pub fn rate_limit_quota(
|
||||
let monthly_limit = monthly_limit_from_snapshots(&snapshots)
|
||||
.or_else(|| existing_quota.and_then(|quota| quota.get("monthly_limit").cloned()))
|
||||
.unwrap_or(Value::Null);
|
||||
let credits = existing_quota
|
||||
.and_then(|quota| quota.get("credits").cloned())
|
||||
let credits = rate_limits
|
||||
.credits
|
||||
.as_ref()
|
||||
.map(|credits| {
|
||||
json!({
|
||||
"has_credits": credits.has_credits,
|
||||
"unlimited": credits.unlimited,
|
||||
"balance": credits.balance,
|
||||
})
|
||||
})
|
||||
.or_else(|| existing_quota.and_then(|quota| quota.get("credits").cloned()))
|
||||
.unwrap_or(Value::Null);
|
||||
let spend_control = existing_quota
|
||||
.and_then(|quota| quota.get("spend_control").cloned())
|
||||
.unwrap_or(Value::Null);
|
||||
|
||||
json!({
|
||||
"plan_type": plan_type.unwrap_or("unknown"),
|
||||
"plan_type": rate_limits.plan_type.as_deref().or(plan_type).unwrap_or("unknown"),
|
||||
"snapshots": snapshots,
|
||||
"monthly_limit": monthly_limit,
|
||||
"credits": credits,
|
||||
"spend_control": spend_control,
|
||||
"promo_message": rate_limits.promo_message,
|
||||
"rate_limit_reached_type": rate_limits.rate_limit_reached_type,
|
||||
})
|
||||
}
|
||||
|
||||
/// 将限流状态转换回 HTTP 头键值对。
|
||||
/// 将限流状态转换回内部传输头键值对。
|
||||
pub fn rate_limits_to_header_pairs(rate_limits: &ParsedRateLimits) -> Vec<(String, String)> {
|
||||
let mut headers = Vec::new();
|
||||
push_window_headers(&mut headers, "x-codex-primary", rate_limits.primary);
|
||||
push_window_headers(&mut headers, "x-codex-secondary", rate_limits.secondary);
|
||||
if let Some(code_review) = rate_limits.code_review {
|
||||
for details in rate_limits.limits.values() {
|
||||
let prefix = format!("x-{}", details.limit_id.replace('_', "-"));
|
||||
push_window_headers(&mut headers, &format!("{prefix}-primary"), details.primary);
|
||||
push_window_headers(
|
||||
&mut headers,
|
||||
"x-codex-code-review-primary",
|
||||
code_review.primary,
|
||||
);
|
||||
push_window_headers(
|
||||
&mut headers,
|
||||
"x-codex-code-review-secondary",
|
||||
code_review.secondary,
|
||||
&format!("{prefix}-secondary"),
|
||||
details.secondary,
|
||||
);
|
||||
if let Some(limit_name) = &details.limit_name {
|
||||
headers.push((format!("{prefix}-limit-name"), limit_name.clone()));
|
||||
}
|
||||
}
|
||||
if let Some(credits) = &rate_limits.credits {
|
||||
headers.push((
|
||||
"x-codex-credits-has-credits".to_string(),
|
||||
credits.has_credits.to_string(),
|
||||
));
|
||||
headers.push((
|
||||
"x-codex-credits-unlimited".to_string(),
|
||||
credits.unlimited.to_string(),
|
||||
));
|
||||
if let Some(balance) = &credits.balance {
|
||||
headers.push(("x-codex-credits-balance".to_string(), balance.clone()));
|
||||
}
|
||||
}
|
||||
if let Some(plan_type) = &rate_limits.plan_type {
|
||||
headers.push(("x-codex-plan-type".to_string(), plan_type.clone()));
|
||||
}
|
||||
if let Some(promo_message) = &rate_limits.promo_message {
|
||||
headers.push(("x-codex-promo-message".to_string(), promo_message.clone()));
|
||||
}
|
||||
if let Some(reached_type) = &rate_limits.rate_limit_reached_type {
|
||||
headers.push((
|
||||
"x-codex-rate-limit-reached-type".to_string(),
|
||||
reached_type.clone(),
|
||||
));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
fn quota_identity(limit_id: &str) -> (&'static str, Option<&str>) {
|
||||
if limit_id == "codex" {
|
||||
("core", None)
|
||||
} else if is_review_limit_name(Some(limit_id)) {
|
||||
("code_review", Some(limit_id))
|
||||
} else {
|
||||
("additional", Some(limit_id))
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_snapshot_matches_limit(snapshot: &Value, limit_id: &str) -> bool {
|
||||
let (source, metered_feature) = quota_identity(limit_id);
|
||||
if snapshot.get("source").and_then(Value::as_str) != Some(source) {
|
||||
return false;
|
||||
}
|
||||
source == "core"
|
||||
|| snapshot.get("metered_feature").and_then(Value::as_str) == metered_feature
|
||||
|| snapshot.get("limit_name").and_then(Value::as_str) == Some(limit_id)
|
||||
}
|
||||
|
||||
fn number_field(value: &Value, field: &str) -> Option<u64> {
|
||||
value.get(field)?.as_u64()
|
||||
}
|
||||
@@ -464,20 +580,88 @@ fn push_window_headers(
|
||||
fn parse_details_from_lookup(
|
||||
headers: &BTreeMap<String, &str>,
|
||||
prefix: &str,
|
||||
limit_id: &str,
|
||||
) -> Option<RateLimitDetails> {
|
||||
let primary = parse_window_from_lookup(headers, &format!("{prefix}-primary"));
|
||||
let secondary = parse_window_from_lookup(headers, &format!("{prefix}-secondary"));
|
||||
if primary.is_none() && secondary.is_none() {
|
||||
let limit_name = lookup_non_empty(headers, &format!("{prefix}-limit-name"));
|
||||
let allowed = lookup_bool(headers, &format!("{prefix}-allowed"));
|
||||
let limit_reached = lookup_bool(headers, &format!("{prefix}-limit-reached"));
|
||||
if primary.is_none()
|
||||
&& secondary.is_none()
|
||||
&& limit_name.is_none()
|
||||
&& allowed.is_none()
|
||||
&& limit_reached.is_none()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(RateLimitDetails {
|
||||
allowed: None,
|
||||
limit_reached: None,
|
||||
limit_id: limit_id.to_string(),
|
||||
limit_name,
|
||||
allowed,
|
||||
limit_reached,
|
||||
primary,
|
||||
secondary,
|
||||
})
|
||||
}
|
||||
|
||||
fn rate_limit_id_from_header_name(name: &str) -> Option<String> {
|
||||
const SUFFIXES: [&str; 9] = [
|
||||
"-primary-used-percent",
|
||||
"-primary-window-minutes",
|
||||
"-primary-reset-at",
|
||||
"-secondary-used-percent",
|
||||
"-secondary-window-minutes",
|
||||
"-secondary-reset-at",
|
||||
"-limit-name",
|
||||
"-allowed",
|
||||
"-limit-reached",
|
||||
];
|
||||
let name = name.strip_prefix("x-")?;
|
||||
let raw_id = SUFFIXES
|
||||
.iter()
|
||||
.find_map(|suffix| name.strip_suffix(suffix))?;
|
||||
(!raw_id.is_empty()).then(|| normalize_limit_id(raw_id))
|
||||
}
|
||||
|
||||
fn parse_credits_from_lookup(headers: &BTreeMap<String, &str>) -> Option<CreditsSnapshot> {
|
||||
Some(CreditsSnapshot {
|
||||
has_credits: lookup_bool(headers, "x-codex-credits-has-credits")?,
|
||||
unlimited: lookup_bool(headers, "x-codex-credits-unlimited")?,
|
||||
balance: lookup_non_empty(headers, "x-codex-credits-balance"),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_credits_from_object(value: &Value) -> Option<CreditsSnapshot> {
|
||||
Some(CreditsSnapshot {
|
||||
has_credits: value.get("has_credits")?.as_bool()?,
|
||||
unlimited: value.get("unlimited")?.as_bool()?,
|
||||
balance: value.get("balance").and_then(|balance| match balance {
|
||||
Value::String(balance) => Some(balance.clone()),
|
||||
Value::Number(balance) => Some(balance.to_string()),
|
||||
_ => None,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
fn lookup_non_empty(headers: &BTreeMap<String, &str>, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.map(|value| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn lookup_bool(headers: &BTreeMap<String, &str>, name: &str) -> Option<bool> {
|
||||
match headers.get(name)?.trim() {
|
||||
"1" => Some(true),
|
||||
"0" => Some(false),
|
||||
value if value.eq_ignore_ascii_case("true") => Some(true),
|
||||
value if value.eq_ignore_ascii_case("false") => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_window_from_lookup(
|
||||
headers: &BTreeMap<String, &str>,
|
||||
prefix: &str,
|
||||
@@ -499,15 +683,23 @@ fn parse_window_from_lookup(
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_details_from_object(value: &Value) -> Option<RateLimitDetails> {
|
||||
fn parse_details_from_object(
|
||||
value: &Value,
|
||||
limit_id: &str,
|
||||
limit_name: Option<String>,
|
||||
) -> Option<RateLimitDetails> {
|
||||
let primary = value.get("primary").and_then(parse_window_from_object);
|
||||
let secondary = value.get("secondary").and_then(parse_window_from_object);
|
||||
if primary.is_none() && secondary.is_none() {
|
||||
let allowed = value.get("allowed").and_then(Value::as_bool);
|
||||
let limit_reached = value.get("limit_reached").and_then(Value::as_bool);
|
||||
if primary.is_none() && secondary.is_none() && allowed.is_none() && limit_reached.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(RateLimitDetails {
|
||||
allowed: value.get("allowed").and_then(Value::as_bool),
|
||||
limit_reached: value.get("limit_reached").and_then(Value::as_bool),
|
||||
limit_id: limit_id.to_string(),
|
||||
limit_name,
|
||||
allowed,
|
||||
limit_reached,
|
||||
primary,
|
||||
secondary,
|
||||
})
|
||||
@@ -645,6 +837,10 @@ fn rate_limit_name(value: &Value) -> Option<&str> {
|
||||
.and_then(Value::as_str)
|
||||
}
|
||||
|
||||
fn normalize_limit_id(value: impl AsRef<str>) -> String {
|
||||
value.as_ref().trim().to_ascii_lowercase().replace('-', "_")
|
||||
}
|
||||
|
||||
fn is_review_limit_name(value: Option<&str>) -> bool {
|
||||
let Some(value) = value else {
|
||||
return false;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::{collections::BTreeSet, time::Instant};
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
infra::time::elapsed_millis_i64,
|
||||
@@ -16,8 +16,8 @@ use crate::{
|
||||
/// 发往上游的 Responses 请求。`body` 持有客户端原始 JSON object,逐字段(含顺序、
|
||||
/// 含未知字段)透传上游,是上游请求体的唯一来源;`use_websocket`/`force_http_sse`
|
||||
/// 仅用于本地传输选择,不写入 body。常用字段通过访问器方法读写。
|
||||
/// 代理只做最小 patch(`stream`/`store` 由传输层控制,`input` 字符串做官方兼容转换,
|
||||
/// implicit resume 时替换 `input`,模型后缀路由时写 `model`/`reasoning`/`service_tier`)。
|
||||
/// 普通客户端请求不修改 body;仅在代理持有完整历史并换号重放时替换历史字段,
|
||||
/// 或在模型后缀路由时写入 `model`/`reasoning`/`service_tier`。
|
||||
///
|
||||
/// 其余字段是代理控制状态,不进上游 body(原 `#[serde(skip)]` 字段)。
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -30,6 +30,16 @@ pub struct CodexResponsesRequest {
|
||||
pub explicit_prompt_cache_key: bool,
|
||||
/// 客户端会话 ID。
|
||||
pub client_conversation_id: Option<String>,
|
||||
/// 客户端 session ID,仅保留在受控本地上下文。
|
||||
pub client_session_id: Option<String>,
|
||||
/// 客户端 thread ID,仅保留在受控本地上下文。
|
||||
pub client_thread_id: Option<String>,
|
||||
/// 客户端 request ID,仅保留在受控本地上下文。
|
||||
pub client_request_id: Option<String>,
|
||||
/// 客户端 turn ID,仅保留在受控本地上下文。
|
||||
pub client_turn_id: Option<String>,
|
||||
/// 连接池和 affinity 使用的本地会话身份,不发送上游。
|
||||
pub local_conversation_id: Option<String>,
|
||||
/// 变体身份键。
|
||||
pub variant_identity: Option<String>,
|
||||
/// 代理侧识别的客户端 IP,仅用于管理端使用记录展示。
|
||||
@@ -56,6 +66,26 @@ pub struct CodexResponsesRequest {
|
||||
pub codex_window_id: Option<String>,
|
||||
/// 父线程 ID。
|
||||
pub parent_thread_id: Option<String>,
|
||||
/// 已知 previous response 的持久化范围,仅用于本地 transport 校验。
|
||||
pub previous_response_scope: Option<PreviousResponseScope>,
|
||||
/// 流式响应在交给下游前的本地提交策略。
|
||||
pub stream_commit_policy: StreamCommitPolicy,
|
||||
}
|
||||
|
||||
/// previous response 在上游的可续接范围。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PreviousResponseScope {
|
||||
Persisted,
|
||||
ConnectionLocal,
|
||||
ExternalUnknown,
|
||||
}
|
||||
|
||||
/// 流式请求的可逆提交边界。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum StreamCommitPolicy {
|
||||
FirstForwardableEvent,
|
||||
UntilOutputOrTerminal,
|
||||
}
|
||||
|
||||
impl Serialize for CodexResponsesRequest {
|
||||
@@ -114,7 +144,7 @@ pub fn response_body_has_first_output(body_bytes: &[u8]) -> bool {
|
||||
.is_ok_and(|events| events.iter().any(response_sse_event_has_first_output))
|
||||
}
|
||||
|
||||
/// 已收到首个有效 Responses SSE 输出事件时记录首 token 耗时。
|
||||
/// 已收到首个有效 Responses SSE 输出事件时记录耗时。
|
||||
pub fn update_first_response_output_ms(
|
||||
started_at: Instant,
|
||||
body_bytes: &[u8],
|
||||
@@ -180,8 +210,10 @@ fn complete_sse_body_prefix(body: &str) -> Option<&str> {
|
||||
/// 从 Codex SSE 收集出的非流式 Responses 结果。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum CollectedResponse {
|
||||
/// 收集到 `response.completed` 或 `response.incomplete`。
|
||||
/// 收集到 `response.completed`。
|
||||
Completed(Value),
|
||||
/// 收集到 `response.incomplete`。
|
||||
Incomplete(Value),
|
||||
/// 收集到 `response.failed` 或 `error`。
|
||||
Failed(ResponsesSseFailure),
|
||||
/// SSE 未包含成功终止响应。
|
||||
@@ -218,8 +250,8 @@ pub struct CompletedResponseMetadata {
|
||||
pub response_id: String,
|
||||
/// 完成响应中的 function call ids。
|
||||
pub function_call_ids: Vec<String>,
|
||||
/// 可用于 reasoning replay 的 output items。
|
||||
pub replay_items: Vec<Value>,
|
||||
/// 完成响应的完整 output items。
|
||||
pub output: Vec<Value>,
|
||||
}
|
||||
|
||||
/// 将 Codex Responses SSE 完成响应收集为非流式 Responses JSON。
|
||||
@@ -231,6 +263,7 @@ pub fn response_from_codex_sse(
|
||||
let mut output_text = String::new();
|
||||
let mut output_items = Vec::new();
|
||||
let mut terminal_response = None;
|
||||
let mut terminal_incomplete = false;
|
||||
let mut failed_response = None;
|
||||
|
||||
for event in events {
|
||||
@@ -248,8 +281,13 @@ pub fn response_from_codex_sse(
|
||||
output_items.push(item.clone());
|
||||
}
|
||||
}
|
||||
Some("response.completed" | "response.incomplete") => {
|
||||
Some("response.completed") => {
|
||||
terminal_response = value.get("response").cloned();
|
||||
terminal_incomplete = false;
|
||||
}
|
||||
Some("response.incomplete") => {
|
||||
terminal_response = value.get("response").cloned();
|
||||
terminal_incomplete = true;
|
||||
}
|
||||
Some(event_name @ ("error" | "response.failed")) if failed_response.is_none() => {
|
||||
failed_response = Some(ResponsesSseFailure::from_event(event_name, &value));
|
||||
@@ -264,13 +302,15 @@ pub fn response_from_codex_sse(
|
||||
let Some(mut response) = terminal_response else {
|
||||
return Ok(CollectedResponse::MissingCompleted);
|
||||
};
|
||||
if terminal_incomplete {
|
||||
reconvert_completed_response_tuple_values(&mut response, tuple_schema);
|
||||
return Ok(CollectedResponse::Incomplete(response));
|
||||
}
|
||||
if is_empty_response(&response, &output_text, &output_items) {
|
||||
return Ok(CollectedResponse::Empty);
|
||||
}
|
||||
|
||||
ensure_completed_response_output(&mut response, &output_items, &output_text);
|
||||
reconvert_completed_response_tuple_values(&mut response, tuple_schema);
|
||||
sync_output_text_from_output(&mut response);
|
||||
Ok(CollectedResponse::Completed(response))
|
||||
}
|
||||
|
||||
@@ -281,7 +321,8 @@ pub fn completed_response_metadata(
|
||||
let events = parse_sse_events(body)?;
|
||||
let mut response_id = None;
|
||||
let mut function_call_ids = BTreeSet::new();
|
||||
let mut replay_items = Vec::new();
|
||||
let mut output = Vec::new();
|
||||
let mut completed_items = Vec::new();
|
||||
|
||||
for event in events {
|
||||
let Ok(value) = serde_json::from_str::<Value>(&event.data) else {
|
||||
@@ -290,22 +331,25 @@ pub fn completed_response_metadata(
|
||||
match event.event.as_deref() {
|
||||
Some("response.output_item.done") => {
|
||||
if let Some(item) = value.get("item") {
|
||||
collect_response_replay_items(item, &mut replay_items);
|
||||
completed_items.push(item.clone());
|
||||
}
|
||||
if let Some(call_id) = value.pointer("/item/call_id").and_then(Value::as_str) {
|
||||
function_call_ids.insert(call_id.to_string());
|
||||
}
|
||||
}
|
||||
Some("response.completed" | "response.incomplete") => {
|
||||
Some("response.completed") => {
|
||||
response_id = value
|
||||
.pointer("/response/id")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string);
|
||||
collect_response_function_call_ids(&value, &mut function_call_ids);
|
||||
if let Some(output) = value.pointer("/response/output").and_then(Value::as_array) {
|
||||
for item in output {
|
||||
collect_response_replay_items(item, &mut replay_items);
|
||||
}
|
||||
if let Some(completed_output) =
|
||||
value.pointer("/response/output").and_then(Value::as_array)
|
||||
{
|
||||
completed_output.clone_into(&mut output);
|
||||
}
|
||||
if output.is_empty() {
|
||||
output.clone_from(&completed_items);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
@@ -315,7 +359,7 @@ pub fn completed_response_metadata(
|
||||
Ok(response_id.map(|response_id| CompletedResponseMetadata {
|
||||
response_id,
|
||||
function_call_ids: function_call_ids.into_iter().collect(),
|
||||
replay_items,
|
||||
output,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -337,7 +381,6 @@ pub fn reconvert_responses_sse_event_tuple_values(
|
||||
Some("response.completed" | "response.incomplete") => {
|
||||
if let Some(response) = data.get_mut("response") {
|
||||
reconvert_completed_response_tuple_values(response, Some(tuple_schema));
|
||||
sync_output_text_from_output(response);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
@@ -390,43 +433,6 @@ fn is_empty_response(response: &Value, output_text: &str, output_items: &[Value]
|
||||
== 0
|
||||
}
|
||||
|
||||
fn ensure_completed_response_output(
|
||||
response: &mut Value,
|
||||
output_items: &[Value],
|
||||
output_text: &str,
|
||||
) {
|
||||
let output_is_empty = response
|
||||
.get("output")
|
||||
.and_then(Value::as_array)
|
||||
.is_none_or(Vec::is_empty);
|
||||
if !output_is_empty {
|
||||
return;
|
||||
}
|
||||
|
||||
if !output_items.is_empty() {
|
||||
response["output"] = Value::Array(output_items.to_vec());
|
||||
return;
|
||||
}
|
||||
if output_text.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let item_status = response
|
||||
.get("status")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("completed");
|
||||
response["output"] = json!([{
|
||||
"type": "message",
|
||||
"status": item_status,
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "output_text",
|
||||
"text": output_text,
|
||||
"annotations": []
|
||||
}]
|
||||
}]);
|
||||
}
|
||||
|
||||
fn reconvert_completed_response_tuple_values(response: &mut Value, tuple_schema: Option<&Value>) {
|
||||
let Some(tuple_schema) = tuple_schema else {
|
||||
return;
|
||||
@@ -459,20 +465,6 @@ fn reconvert_output_item_tuple_values(item: &mut Value, tuple_schema: &Value) {
|
||||
}
|
||||
}
|
||||
|
||||
fn sync_output_text_from_output(response: &mut Value) {
|
||||
let Some(items) = response.get("output").and_then(Value::as_array) else {
|
||||
return;
|
||||
};
|
||||
let texts = items
|
||||
.iter()
|
||||
.filter_map(output_text_from_item)
|
||||
.collect::<Vec<_>>();
|
||||
if texts.is_empty() {
|
||||
return;
|
||||
}
|
||||
response["output_text"] = Value::String(texts.join("\n\n"));
|
||||
}
|
||||
|
||||
fn collect_response_function_call_ids(value: &Value, function_call_ids: &mut BTreeSet<String>) {
|
||||
let Some(output) = value.pointer("/response/output").and_then(Value::as_array) else {
|
||||
return;
|
||||
@@ -484,30 +476,6 @@ fn collect_response_function_call_ids(value: &Value, function_call_ids: &mut BTr
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_response_replay_items(item: &Value, replay_items: &mut Vec<Value>) {
|
||||
if matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some("reasoning" | "function_call")
|
||||
) {
|
||||
replay_items.push(item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
fn output_text_from_item(item: &Value) -> Option<String> {
|
||||
let content = item.get("content")?.as_array()?;
|
||||
let text = content
|
||||
.iter()
|
||||
.filter_map(|part| {
|
||||
let part_type = part.get("type")?.as_str()?;
|
||||
if part_type != "output_text" && part_type != "text" {
|
||||
return None;
|
||||
}
|
||||
part.get("text")?.as_str()
|
||||
})
|
||||
.collect::<String>();
|
||||
(!text.is_empty()).then_some(text)
|
||||
}
|
||||
|
||||
fn failure_message(value: &Value) -> Option<String> {
|
||||
value
|
||||
.pointer("/response/error/message")
|
||||
@@ -531,18 +499,19 @@ fn failure_code(value: &Value) -> Option<String> {
|
||||
impl CodexResponsesRequest {
|
||||
/// 从客户端原始 Responses JSON object 构造上游请求。
|
||||
///
|
||||
/// 只做最小规范化:`input` 缺省为 `[]`、`stream` 缺省 `true`、`store` 缺省 `false`。
|
||||
/// 其余字段(含未知字段)原样保留在 `body` 中透传上游。
|
||||
pub fn from_body(mut body: Map<String, Value>) -> Self {
|
||||
body.entry("input")
|
||||
.or_insert_with(|| Value::Array(Vec::new()));
|
||||
body.entry("stream").or_insert(Value::Bool(true));
|
||||
body.entry("store").or_insert(Value::Bool(false));
|
||||
/// 客户端提供的字段(含未知字段)原样保留在 `body` 中透传上游。
|
||||
/// 协议默认值仅由类型化访问器在本地解释,不写回上游正文。
|
||||
pub fn from_body(body: Map<String, Value>) -> Self {
|
||||
Self {
|
||||
body,
|
||||
tuple_schema: None,
|
||||
explicit_prompt_cache_key: false,
|
||||
client_conversation_id: None,
|
||||
client_session_id: None,
|
||||
client_thread_id: None,
|
||||
client_request_id: None,
|
||||
client_turn_id: None,
|
||||
local_conversation_id: None,
|
||||
variant_identity: None,
|
||||
client_ip: None,
|
||||
client_user_agent: None,
|
||||
@@ -556,6 +525,8 @@ impl CodexResponsesRequest {
|
||||
include_timing_metrics: None,
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
previous_response_scope: None,
|
||||
stream_commit_policy: StreamCommitPolicy::FirstForwardableEvent,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -612,7 +583,7 @@ impl CodexResponsesRequest {
|
||||
.map_or(&[], Vec::as_slice)
|
||||
}
|
||||
|
||||
/// 替换输入条目(implicit resume / 字符串输入兼容)。
|
||||
/// 替换输入条目(仅用于代理持有完整历史时的换号重放)。
|
||||
pub fn set_input(&mut self, input: Vec<Value>) {
|
||||
self.body.insert("input".to_string(), Value::Array(input));
|
||||
}
|
||||
@@ -638,6 +609,11 @@ impl CodexResponsesRequest {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// 设置上游存储标志。
|
||||
pub fn set_store(&mut self, store: bool) {
|
||||
self.body.insert("store".to_string(), Value::Bool(store));
|
||||
}
|
||||
|
||||
/// reasoning 配置(透传,不规整)。
|
||||
pub fn reasoning(&self) -> Option<&Value> {
|
||||
self.body.get("reasoning")
|
||||
@@ -691,6 +667,7 @@ impl CodexResponsesRequest {
|
||||
}
|
||||
None => {
|
||||
self.body.remove("previous_response_id");
|
||||
self.previous_response_scope = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -730,6 +707,22 @@ impl CodexResponsesRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅在客户端原本提供字符串字段时替换其账号作用域 wire 值。
|
||||
pub(crate) fn replace_existing_identity_field(&mut self, key: &str, value: Option<&str>) {
|
||||
if !self.body.get(key).is_some_and(Value::is_string) {
|
||||
return;
|
||||
}
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
Some(value) => {
|
||||
self.body
|
||||
.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
None => {
|
||||
self.body.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 判断请求是否声明了图片生成工具。
|
||||
pub fn expects_image_generation(&self) -> bool {
|
||||
self.tools().is_some_and(|tools| {
|
||||
@@ -755,6 +748,18 @@ pub struct CodexCompactRequest {
|
||||
pub client_user_agent: Option<String>,
|
||||
/// 已鉴权客户端 API key 的稳定 ID,仅用于事实归因。
|
||||
pub client_api_key_id: Option<String>,
|
||||
/// 客户端 session ID,仅保留在本地身份上下文。
|
||||
pub client_session_id: Option<String>,
|
||||
/// 客户端 thread ID,仅保留在本地身份上下文。
|
||||
pub client_thread_id: Option<String>,
|
||||
/// 客户端 request ID,仅保留在本地身份上下文。
|
||||
pub client_request_id: Option<String>,
|
||||
/// 客户端 turn ID,仅保留在本地身份上下文。
|
||||
pub client_turn_id: Option<String>,
|
||||
/// 客户端 window ID,仅保留在本地身份上下文。
|
||||
pub client_window_id: Option<String>,
|
||||
/// 客户端 parent thread ID,仅保留在本地身份上下文。
|
||||
pub client_parent_thread_id: Option<String>,
|
||||
}
|
||||
|
||||
impl Serialize for CodexCompactRequest {
|
||||
@@ -785,4 +790,8 @@ impl CodexCompactRequest {
|
||||
pub fn reasoning(&self) -> Option<&Value> {
|
||||
self.body.get("reasoning")
|
||||
}
|
||||
|
||||
pub fn prompt_cache_key(&self) -> Option<&str> {
|
||||
self.body.get("prompt_cache_key").and_then(Value::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,16 +51,26 @@ pub struct WebSocketAuditArtifact {
|
||||
}
|
||||
|
||||
/// 从 WebSocket 错误帧推导出的上游错误分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ClassifiedWebSocketError {
|
||||
/// 应映射给 HTTP 层的状态码。
|
||||
pub status_code: u16,
|
||||
/// 上游错误类型。
|
||||
pub error_type: Option<String>,
|
||||
/// 上游错误码。
|
||||
pub code: Option<String>,
|
||||
/// 上游错误消息。
|
||||
pub message: Option<String>,
|
||||
/// 上游错误参数。
|
||||
pub param: Option<String>,
|
||||
/// 错误帧携带的安全响应头。
|
||||
pub headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ResponseCompleted {
|
||||
#[serde(rename = "id")]
|
||||
_id: String,
|
||||
id: String,
|
||||
#[serde(default, rename = "usage")]
|
||||
_usage: Option<ResponseCompletedUsage>,
|
||||
#[serde(default, rename = "end_turn")]
|
||||
@@ -148,26 +158,34 @@ fn is_internal_websocket_event(event: &str) -> bool {
|
||||
/// 会话状态,不下发客户端。业务事件一律原样转发,不做 schema 校验。
|
||||
/// (`codex.rate_limits` 内部事件由 `is_internal_websocket_event` 单独剥离。)
|
||||
fn websocket_event_should_skip(raw: &str) -> bool {
|
||||
websocket_metadata_turn_state(raw).is_some()
|
||||
websocket_event_type(raw).as_deref() == Some("response.metadata")
|
||||
}
|
||||
|
||||
/// 提取 `response.metadata` 帧中的字符串响应头。
|
||||
pub fn websocket_metadata_headers(raw: &str) -> Vec<(String, String)> {
|
||||
let Ok(value) = serde_json::from_str::<Value>(raw) else {
|
||||
return Vec::new();
|
||||
};
|
||||
if value.get("type").and_then(Value::as_str) != Some("response.metadata") {
|
||||
return Vec::new();
|
||||
}
|
||||
value
|
||||
.get("headers")
|
||||
.and_then(Value::as_object)
|
||||
.into_iter()
|
||||
.flat_map(|headers| headers.iter())
|
||||
.filter_map(|(name, value)| json_value_as_string(value).map(|value| (name.clone(), value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 从 `response.metadata` 帧中提取 `x-codex-turn-state`。
|
||||
pub fn websocket_metadata_turn_state(raw: &str) -> Option<String> {
|
||||
let value = serde_json::from_str::<Value>(raw).ok()?;
|
||||
if value.get("type").and_then(Value::as_str) != Some("response.metadata") {
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.get("headers")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|headers| {
|
||||
headers.iter().find_map(|(name, value)| {
|
||||
if name.eq_ignore_ascii_case("x-codex-turn-state") {
|
||||
json_value_as_string(value)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
websocket_metadata_headers(raw)
|
||||
.into_iter()
|
||||
.find_map(|(name, value)| {
|
||||
name.eq_ignore_ascii_case("x-codex-turn-state")
|
||||
.then_some(value)
|
||||
})
|
||||
.or_else(|| {
|
||||
value
|
||||
@@ -186,34 +204,22 @@ fn json_value_as_string(value: &Value) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验 `response.completed` 的官方响应形状。
|
||||
pub fn websocket_response_completed_parse_error(raw: &str) -> Option<String> {
|
||||
let value = serde_json::from_str::<Value>(raw).ok()?;
|
||||
/// 校验 `response.completed` 的官方响应形状并提取 response ID。
|
||||
pub fn websocket_response_completed_id(raw: &str) -> Result<Option<String>, String> {
|
||||
let value = serde_json::from_str::<Value>(raw).map_err(|error| error.to_string())?;
|
||||
if value.get("type").and_then(Value::as_str) != Some("response.completed") {
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
let response = value.get("response")?;
|
||||
if response.is_null() {
|
||||
return None;
|
||||
let response = value
|
||||
.get("response")
|
||||
.filter(|response| !response.is_null())
|
||||
.ok_or_else(|| "response.completed is missing response".to_string())?;
|
||||
let completed = serde_json::from_value::<ResponseCompleted>(response.clone())
|
||||
.map_err(|error| format!("failed to parse ResponseCompleted: {error}"))?;
|
||||
if completed.id.trim().is_empty() {
|
||||
return Err("response.completed contains an empty response id".to_string());
|
||||
}
|
||||
match serde_json::from_value::<ResponseCompleted>(response.clone()) {
|
||||
Ok(_) => None,
|
||||
Err(error) => Some(format!("failed to parse ResponseCompleted: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `response.incomplete` 中提取 incomplete reason。
|
||||
pub fn websocket_incomplete_response_reason(raw: &str) -> Option<String> {
|
||||
let value = serde_json::from_str::<Value>(raw).ok()?;
|
||||
if value.get("type").and_then(Value::as_str) != Some("response.incomplete") {
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.pointer("/response/incomplete_details/reason")
|
||||
.or_else(|| value.pointer("/incomplete_details/reason"))
|
||||
.or_else(|| value.get("reason"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string)
|
||||
Ok(Some(completed.id))
|
||||
}
|
||||
|
||||
#[path = "websocket_errors.rs"]
|
||||
@@ -222,10 +228,13 @@ pub use errors::*;
|
||||
|
||||
/// 判断 WebSocket 事件是否会结束当前响应流。
|
||||
pub fn is_terminal_websocket_event(event: &str) -> bool {
|
||||
matches!(event, "response.completed" | "response.failed" | "error")
|
||||
matches!(
|
||||
event,
|
||||
"response.completed" | "response.incomplete" | "response.failed" | "error"
|
||||
)
|
||||
}
|
||||
|
||||
/// 将 WebSocket 错误帧映射为上游 HTTP 状态分类。
|
||||
/// 将 WebSocket 错误帧映射为结构化上游错误。
|
||||
pub fn classify_websocket_error_frame(raw: &str) -> Option<ClassifiedWebSocketError> {
|
||||
let value = serde_json::from_str::<Value>(raw).ok()?;
|
||||
let event_type = value.get("type").and_then(Value::as_str)?;
|
||||
@@ -233,37 +242,49 @@ pub fn classify_websocket_error_frame(raw: &str) -> Option<ClassifiedWebSocketEr
|
||||
return None;
|
||||
}
|
||||
|
||||
let code = websocket_error_code(&value);
|
||||
if code.as_deref() == Some("websocket_connection_limit_reached") {
|
||||
return Some(ClassifiedWebSocketError { status_code: 503 });
|
||||
let error = websocket_error_value(&value);
|
||||
let error_type = error
|
||||
.and_then(|error| error.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string);
|
||||
let code = error
|
||||
.and_then(|error| error.get("code"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string);
|
||||
let message = error
|
||||
.and_then(|error| error.get("message"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string);
|
||||
let param = error
|
||||
.and_then(|error| error.get("param"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string);
|
||||
let explicit_status =
|
||||
explicit_error_status_code(&value).filter(|status_code| !(200..=299).contains(status_code));
|
||||
if explicit_error_status_code(&value).is_some() && explicit_status.is_none() {
|
||||
return None;
|
||||
}
|
||||
let normalized_code = code.as_deref().map(str::to_ascii_lowercase);
|
||||
let normalized_type = error_type.as_deref().map(str::to_ascii_lowercase);
|
||||
let status_code = explicit_status
|
||||
.or_else(|| normalized_code.as_deref().and_then(error_status_code))
|
||||
.or_else(|| normalized_type.as_deref().and_then(error_type_status_code))?;
|
||||
|
||||
if event_type == "error" {
|
||||
if let Some(status_code) = explicit_error_status_code(&value) {
|
||||
if (200..=299).contains(&status_code) {
|
||||
return None;
|
||||
}
|
||||
return Some(ClassifiedWebSocketError { status_code });
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(code) = code {
|
||||
if let Some(status_code) = rotatable_error_status_code(&code) {
|
||||
return Some(ClassifiedWebSocketError { status_code });
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
Some(ClassifiedWebSocketError {
|
||||
status_code,
|
||||
error_type,
|
||||
code,
|
||||
message,
|
||||
param,
|
||||
headers: websocket_error_headers(&value),
|
||||
})
|
||||
}
|
||||
|
||||
fn websocket_error_code(value: &Value) -> Option<String> {
|
||||
fn websocket_error_value(value: &Value) -> Option<&Map<String, Value>> {
|
||||
value
|
||||
.pointer("/response/error/code")
|
||||
.or_else(|| value.pointer("/response/error/type"))
|
||||
.or_else(|| value.pointer("/error/code"))
|
||||
.or_else(|| value.pointer("/error/type"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_ascii_lowercase)
|
||||
.pointer("/response/error")
|
||||
.or_else(|| value.get("error"))
|
||||
.and_then(Value::as_object)
|
||||
}
|
||||
|
||||
fn explicit_error_status_code(value: &Value) -> Option<u16> {
|
||||
@@ -274,17 +295,42 @@ fn explicit_error_status_code(value: &Value) -> Option<u16> {
|
||||
.and_then(|status| u16::try_from(status).ok())
|
||||
}
|
||||
|
||||
fn rotatable_error_status_code(code: &str) -> Option<u16> {
|
||||
fn error_status_code(code: &str) -> Option<u16> {
|
||||
match code {
|
||||
"usage_limit_reached" | "rate_limit_exceeded" | "rate_limit_reached" => Some(429),
|
||||
"quota_exhausted" | "payment_required" => Some(402),
|
||||
"unauthorized" | "token_invalid" | "token_expired" | "account_deactivated" => Some(401),
|
||||
"forbidden" | "account_banned" | "banned" => Some(403),
|
||||
"previous_response_not_found" => Some(400),
|
||||
"previous_response_not_found"
|
||||
| "invalid_encrypted_content"
|
||||
| "missing_tool_output"
|
||||
| "no_tool_output" => Some(400),
|
||||
"websocket_connection_limit_reached" => Some(503),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_type_status_code(error_type: &str) -> Option<u16> {
|
||||
match error_type {
|
||||
"invalid_request_error" => Some(400),
|
||||
"authentication_error" => Some(401),
|
||||
"permission_error" => Some(403),
|
||||
"rate_limit_error" => Some(429),
|
||||
"server_error" | "api_error" => Some(500),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn websocket_error_headers(value: &Value) -> Vec<(String, String)> {
|
||||
value
|
||||
.get("headers")
|
||||
.and_then(Value::as_object)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|(name, value)| json_value_as_string(value).map(|value| (name.clone(), value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 从包裹在 WebSocket `error.headers` 中的 retry-after 值提取秒数。
|
||||
pub fn retry_after_seconds_from_wrapped_error_headers(raw: &str) -> Option<u64> {
|
||||
let value = serde_json::from_str::<Value>(raw).ok()?;
|
||||
|
||||
@@ -20,7 +20,6 @@ use reqwest::{
|
||||
use serde_json::{map::Map, Value};
|
||||
use thiserror::Error;
|
||||
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::upstream::openai::fingerprint::{Fingerprint, RuntimeFingerprint};
|
||||
use crate::upstream::openai::protocol::events::{extract_sse_usage, retry_after_seconds_from_body};
|
||||
@@ -47,7 +46,7 @@ use super::websocket::{
|
||||
CodexWebSocketRateLimitHeaderUpdates, CodexWebSocketTurnStateUpdate,
|
||||
};
|
||||
use super::websocket_pool::{
|
||||
CodexWebSocketPool, CodexWebSocketPoolKey, WebSocketPoolDecision, DEFAULT_FIRST_TOKEN_TIMEOUT,
|
||||
CodexWebSocketPool, CodexWebSocketPoolKey, WebSocketPoolDecision, DEFAULT_INITIAL_EVENT_TIMEOUT,
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -238,6 +237,27 @@ pub struct CodexRequestContext<'a> {
|
||||
pub installation_id: Option<&'a str>,
|
||||
/// session_id。
|
||||
pub session_id: Option<&'a str>,
|
||||
/// thread_id。
|
||||
pub thread_id: Option<&'a str>,
|
||||
/// 账号作用域 prompt cache key。
|
||||
pub prompt_cache_key: Option<&'a str>,
|
||||
/// 账号作用域 client request ID。
|
||||
pub client_request_id: Option<&'a str>,
|
||||
/// 账号作用域 turn ID。
|
||||
pub turn_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Codex Responses 上游响应。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CodexResponseMetadata {
|
||||
/// 上游实际选用的模型。
|
||||
pub effective_model: Option<String>,
|
||||
/// 模型目录版本。
|
||||
pub models_etag: Option<String>,
|
||||
/// 上游是否声明响应包含 reasoning。
|
||||
pub reasoning_included: bool,
|
||||
/// 允许透传给客户端的安全响应头。
|
||||
pub client_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Codex Responses 上游响应。
|
||||
@@ -261,6 +281,8 @@ pub struct CodexBackendResponse {
|
||||
pub websocket_pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
/// 安全响应元数据。
|
||||
pub response_metadata: CodexResponseMetadata,
|
||||
}
|
||||
|
||||
/// Codex Responses 实际使用的上游传输。
|
||||
@@ -310,6 +332,8 @@ pub struct CodexBackendStreamingResponse {
|
||||
pub websocket_pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
/// 安全响应元数据。
|
||||
pub response_metadata: CodexResponseMetadata,
|
||||
}
|
||||
|
||||
/// Codex compact 端点响应。
|
||||
@@ -323,6 +347,8 @@ pub struct CodexCompactResponse {
|
||||
pub rate_limit_headers: Vec<(String, String)>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
/// 安全响应元数据。
|
||||
pub response_metadata: CodexResponseMetadata,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -336,7 +362,7 @@ pub struct CodexBackendClient {
|
||||
pub(super) base_url: String,
|
||||
fingerprint: RuntimeFingerprint,
|
||||
websocket_pool: Option<Arc<CodexWebSocketPool>>,
|
||||
websocket_first_token_timeout: Option<Duration>,
|
||||
websocket_initial_event_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[path = "client_sse.rs"]
|
||||
@@ -362,6 +388,10 @@ impl CodexModelCatalogClient for CodexBackendClient {
|
||||
cookie_header: None,
|
||||
installation_id: request.installation_id,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
.await
|
||||
.map_err(|error| CodexModelCatalogClientError::RequestFailed {
|
||||
@@ -490,8 +520,19 @@ fn response_upstream_request(
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexResponsesRequest {
|
||||
let mut upstream = request.clone();
|
||||
if let Some(session_id) = context.session_id {
|
||||
upstream.set_prompt_cache_key(Some(session_id.to_string()));
|
||||
if request.prompt_cache_key().is_some() {
|
||||
upstream.set_prompt_cache_key(context.prompt_cache_key.map(ToString::to_string));
|
||||
}
|
||||
for (key, value) in [
|
||||
("session_id", context.session_id),
|
||||
("thread_id", context.thread_id),
|
||||
("turn_id", context.turn_id),
|
||||
("x-client-request-id", context.client_request_id),
|
||||
("x-codex-window-id", context.codex_window_id),
|
||||
("x-codex-parent-thread-id", context.parent_thread_id),
|
||||
("x-codex-installation-id", context.installation_id),
|
||||
] {
|
||||
upstream.replace_existing_identity_field(key, value);
|
||||
}
|
||||
upstream.set_client_metadata(response_client_metadata(request.client_metadata(), context));
|
||||
upstream
|
||||
@@ -506,7 +547,8 @@ fn websocket_upstream_request(request: &CodexResponsesRequest) -> CodexResponses
|
||||
fn stamp_ws_stream_request_start_ms(request: &mut CodexResponsesRequest) {
|
||||
let mut metadata = match request.client_metadata() {
|
||||
Some(Value::Object(metadata)) => metadata.clone(),
|
||||
_ => Map::new(),
|
||||
None => Map::new(),
|
||||
Some(_) => return,
|
||||
};
|
||||
metadata.insert(
|
||||
X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY.to_string(),
|
||||
@@ -528,25 +570,31 @@ fn response_client_metadata(
|
||||
) -> Option<Value> {
|
||||
// 以客户端原始 client_metadata 为基础(保留 number/bool/object 等所有值类型),
|
||||
// 在其上追加代理自身的上下文字段。
|
||||
let mut metadata = match client_metadata {
|
||||
Some(Value::Object(input)) => input.clone(),
|
||||
_ => Map::new(),
|
||||
let mut metadata = match client_metadata? {
|
||||
Value::Object(input) => input.clone(),
|
||||
value => return Some(value.clone()),
|
||||
};
|
||||
|
||||
insert_metadata_string(
|
||||
replace_metadata_string(
|
||||
&mut metadata,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
);
|
||||
insert_metadata_string(&mut metadata, "session_id", context.session_id);
|
||||
insert_metadata_string(&mut metadata, "thread_id", context.session_id);
|
||||
insert_metadata_string(&mut metadata, "x-codex-window-id", context.codex_window_id);
|
||||
replace_metadata_string(&mut metadata, "session_id", context.session_id);
|
||||
replace_metadata_string(&mut metadata, "thread_id", context.thread_id);
|
||||
replace_metadata_string(
|
||||
&mut metadata,
|
||||
"x-client-request-id",
|
||||
context.client_request_id,
|
||||
);
|
||||
replace_metadata_string(&mut metadata, "turn_id", context.turn_id);
|
||||
replace_metadata_string(&mut metadata, "x-codex-window-id", context.codex_window_id);
|
||||
insert_metadata_string(
|
||||
&mut metadata,
|
||||
"x-codex-turn-metadata",
|
||||
context.turn_metadata,
|
||||
);
|
||||
insert_metadata_string(
|
||||
replace_metadata_string(
|
||||
&mut metadata,
|
||||
"x-codex-parent-thread-id",
|
||||
context.parent_thread_id,
|
||||
@@ -565,6 +613,17 @@ fn insert_metadata_string(metadata: &mut Map<String, Value>, key: &str, value: O
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_metadata_string(metadata: &mut Map<String, Value>, key: &str, value: Option<&str>) {
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
Some(value) => {
|
||||
metadata.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
None => {
|
||||
metadata.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn openai_subagent_from_metadata(client_metadata: Option<&Value>) -> Option<String> {
|
||||
let value = client_metadata?
|
||||
.as_object()?
|
||||
|
||||
@@ -12,12 +12,12 @@ impl CodexBackendClient {
|
||||
base_url: base_url.into().trim_end_matches('/').to_string(),
|
||||
fingerprint,
|
||||
websocket_pool: None,
|
||||
websocket_first_token_timeout: Some(DEFAULT_FIRST_TOKEN_TIMEOUT),
|
||||
websocket_initial_event_timeout: Some(DEFAULT_INITIAL_EVENT_TIMEOUT),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_websocket_first_token_timeout(mut self, timeout: Option<Duration>) -> Self {
|
||||
self.websocket_first_token_timeout = timeout.filter(|timeout| !timeout.is_zero());
|
||||
pub fn with_websocket_initial_event_timeout(mut self, timeout: Option<Duration>) -> Self {
|
||||
self.websocket_initial_event_timeout = timeout.filter(|timeout| !timeout.is_zero());
|
||||
self
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ impl CodexBackendClient {
|
||||
let turn_state = response_meta::turn_state(response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
let response_metadata = response_meta::response_metadata(response.headers());
|
||||
let retry_after_seconds = retry_after_seconds(response.headers(), None);
|
||||
|
||||
if !status.is_success() {
|
||||
@@ -88,6 +89,7 @@ impl CodexBackendClient {
|
||||
first_token_ms,
|
||||
websocket_pool_decision: None,
|
||||
diagnostics,
|
||||
response_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -110,6 +112,7 @@ impl CodexBackendClient {
|
||||
let turn_state = response_meta::turn_state(response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
let response_metadata = response_meta::response_metadata(response.headers());
|
||||
let retry_after_seconds = retry_after_seconds(response.headers(), None);
|
||||
|
||||
if !status.is_success() {
|
||||
@@ -134,6 +137,7 @@ impl CodexBackendClient {
|
||||
turn_state_update: None,
|
||||
websocket_pool_decision: None,
|
||||
diagnostics,
|
||||
response_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -284,7 +288,7 @@ impl CodexBackendClient {
|
||||
&prepared,
|
||||
Some((pool, key)),
|
||||
started_at,
|
||||
self.websocket_first_token_timeout,
|
||||
self.websocket_initial_event_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -293,7 +297,7 @@ impl CodexBackendClient {
|
||||
&prepared,
|
||||
None,
|
||||
started_at,
|
||||
self.websocket_first_token_timeout,
|
||||
self.websocket_initial_event_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -316,6 +320,7 @@ impl CodexBackendClient {
|
||||
first_token_ms: exchange.first_token_ms,
|
||||
websocket_pool_decision: exchange.pool_decision,
|
||||
diagnostics: exchange.diagnostics,
|
||||
response_metadata: exchange.response_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -349,7 +354,7 @@ impl CodexBackendClient {
|
||||
execute_response_create_request_stream_with_pool(
|
||||
&prepared,
|
||||
Some((pool, key)),
|
||||
self.websocket_first_token_timeout,
|
||||
self.websocket_initial_event_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -357,7 +362,7 @@ impl CodexBackendClient {
|
||||
execute_response_create_request_stream_with_pool(
|
||||
&prepared,
|
||||
None,
|
||||
self.websocket_first_token_timeout,
|
||||
self.websocket_initial_event_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -384,6 +389,7 @@ impl CodexBackendClient {
|
||||
turn_state_update: Some(exchange.turn_state_update),
|
||||
websocket_pool_decision: exchange.pool_decision,
|
||||
diagnostics: exchange.diagnostics,
|
||||
response_metadata: exchange.response_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -395,8 +401,8 @@ impl CodexBackendClient {
|
||||
) -> Option<CodexWebSocketPoolKey> {
|
||||
let account_id = pool_account_id.or(context.account_id)?;
|
||||
let conversation_id = request
|
||||
.prompt_cache_key()
|
||||
.or(request.client_conversation_id.as_deref())
|
||||
.local_conversation_id
|
||||
.as_deref()
|
||||
.or(request.previous_response_id())?;
|
||||
Some(CodexWebSocketPoolKey::new(
|
||||
&self.base_url,
|
||||
@@ -424,6 +430,7 @@ impl CodexBackendClient {
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
let response_metadata = response_meta::response_metadata(response.headers());
|
||||
let retry_after_seconds = retry_after_seconds(response.headers(), None);
|
||||
let body = response.text().await?;
|
||||
|
||||
@@ -456,6 +463,7 @@ impl CodexBackendClient {
|
||||
set_cookie_headers,
|
||||
rate_limit_headers,
|
||||
diagnostics,
|
||||
response_metadata,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -513,7 +521,7 @@ impl CodexBackendClient {
|
||||
|
||||
fn request_headers(&self, context: CodexRequestContext<'_>) -> CodexClientResult<HeaderMap> {
|
||||
let fingerprint = self.fingerprint.current();
|
||||
let request_id = context.session_id.unwrap_or(context.request_id);
|
||||
let request_id = context.client_request_id.unwrap_or(context.request_id);
|
||||
let ordered_headers = build_ordered_codex_base_headers(
|
||||
&fingerprint,
|
||||
context.access_token,
|
||||
@@ -543,7 +551,8 @@ impl CodexBackendClient {
|
||||
context.installation_id,
|
||||
)?;
|
||||
insert_optional_header(&mut headers, "session-id", context.session_id)?;
|
||||
insert_optional_header(&mut headers, "thread-id", context.session_id)?;
|
||||
insert_optional_header(&mut headers, "thread-id", context.thread_id)?;
|
||||
insert_optional_header(&mut headers, "x-codex-turn-id", context.turn_id)?;
|
||||
insert_optional_header(&mut headers, "x-codex-window-id", context.codex_window_id)?;
|
||||
insert_optional_header(&mut headers, "x-codex-turn-state", context.turn_state)?;
|
||||
insert_optional_header(&mut headers, "x-codex-turn-metadata", context.turn_metadata)?;
|
||||
@@ -635,13 +644,22 @@ impl CodexBackendClient {
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-client-request-id"),
|
||||
HeaderValue::from_str(&Uuid::new_v4().to_string())?,
|
||||
HeaderValue::from_str(context.client_request_id.unwrap_or(context.request_id))?,
|
||||
);
|
||||
insert_optional_header(
|
||||
&mut headers,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
)?;
|
||||
insert_optional_header(&mut headers, "session-id", context.session_id)?;
|
||||
insert_optional_header(&mut headers, "thread-id", context.thread_id)?;
|
||||
insert_optional_header(&mut headers, "x-codex-turn-id", context.turn_id)?;
|
||||
insert_optional_header(&mut headers, "x-codex-window-id", context.codex_window_id)?;
|
||||
insert_optional_header(
|
||||
&mut headers,
|
||||
"x-codex-parent-thread-id",
|
||||
context.parent_thread_id,
|
||||
)?;
|
||||
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -40,6 +40,21 @@ impl CodexUpstreamDiagnostics {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_pairs(status_code: Option<u16>, headers: &[(String, String)]) -> Self {
|
||||
let request_id = UPSTREAM_REQUEST_ID_HEADERS
|
||||
.iter()
|
||||
.find_map(|name| pair_value(headers, name));
|
||||
let trace_headers = UPSTREAM_TRACE_HEADERS
|
||||
.iter()
|
||||
.filter_map(|name| pair_value(headers, name).map(|value| ((*name).to_string(), value)))
|
||||
.collect();
|
||||
Self {
|
||||
status_code,
|
||||
request_id,
|
||||
trace_headers,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.status_code.is_none() && self.request_id.is_none() && self.trace_headers.is_empty()
|
||||
}
|
||||
@@ -52,6 +67,15 @@ impl CodexUpstreamDiagnostics {
|
||||
}
|
||||
}
|
||||
|
||||
fn pair_value(headers: &[(String, String)], name: &str) -> Option<String> {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(candidate, _)| candidate.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn first_header(headers: &HeaderMap, names: &[&str]) -> Option<String> {
|
||||
names.iter().find_map(|name| header_value(headers, name))
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ pub use self::{
|
||||
CodexBackendStreamingResponse, CodexBackendTransport, CodexClientError, CodexClientResult,
|
||||
CodexCompactResponse, CodexModelCatalogClient, CodexModelCatalogClientError,
|
||||
CodexModelCatalogRequest, CodexRateLimitHeaderUpdates, CodexRequestContext,
|
||||
CodexTurnStateUpdate,
|
||||
CodexResponseMetadata, CodexTurnStateUpdate,
|
||||
},
|
||||
diagnostics::CodexUpstreamDiagnostics,
|
||||
endpoints::{
|
||||
|
||||
@@ -4,9 +4,19 @@ use std::time::Instant;
|
||||
|
||||
use reqwest::header::{HeaderMap, SET_COOKIE};
|
||||
|
||||
use crate::upstream::openai::protocol::responses::update_first_response_output_ms;
|
||||
use crate::upstream::openai::protocol::{
|
||||
events::is_rate_limit_header_name, responses::update_first_response_output_ms,
|
||||
};
|
||||
|
||||
use super::diagnostics::CodexUpstreamDiagnostics;
|
||||
use super::{client::CodexResponseMetadata, diagnostics::CodexUpstreamDiagnostics};
|
||||
|
||||
const CLIENT_RESPONSE_HEADERS: [&str; 5] = [
|
||||
"x-request-id",
|
||||
"openai-model",
|
||||
"x-models-etag",
|
||||
"x-reasoning-included",
|
||||
"openai-processing-ms",
|
||||
];
|
||||
|
||||
pub(super) fn diagnostics(
|
||||
status_code: Option<u16>,
|
||||
@@ -33,7 +43,7 @@ pub(super) fn set_cookie_headers(headers: &HeaderMap) -> Vec<String> {
|
||||
pub(super) fn rate_limit_headers(headers: &HeaderMap) -> Vec<(String, String)> {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| is_rate_limit_header(name.as_str()))
|
||||
.filter(|(name, _)| is_rate_limit_header_name(name.as_str()))
|
||||
.filter_map(|(name, value)| {
|
||||
value
|
||||
.to_str()
|
||||
@@ -43,6 +53,59 @@ pub(super) fn rate_limit_headers(headers: &HeaderMap) -> Vec<(String, String)> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn response_metadata(headers: &HeaderMap) -> CodexResponseMetadata {
|
||||
response_metadata_from_pairs(
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(name, value)| value.to_str().ok().map(|value| (name.as_str(), value))),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn merge_response_metadata(
|
||||
metadata: &mut CodexResponseMetadata,
|
||||
headers: impl IntoIterator<Item = (String, String)>,
|
||||
) {
|
||||
for (name, value) in headers {
|
||||
apply_response_header(metadata, &name, &value);
|
||||
}
|
||||
}
|
||||
|
||||
fn response_metadata_from_pairs<'a>(
|
||||
headers: impl IntoIterator<Item = (&'a str, &'a str)>,
|
||||
) -> CodexResponseMetadata {
|
||||
let mut metadata = CodexResponseMetadata::default();
|
||||
for (name, value) in headers {
|
||||
apply_response_header(&mut metadata, name, value);
|
||||
}
|
||||
metadata
|
||||
}
|
||||
|
||||
fn apply_response_header(metadata: &mut CodexResponseMetadata, name: &str, value: &str) {
|
||||
let name = name.to_ascii_lowercase();
|
||||
let value = value.trim();
|
||||
if value.is_empty() || !CLIENT_RESPONSE_HEADERS.contains(&name.as_str()) {
|
||||
return;
|
||||
}
|
||||
match name.as_str() {
|
||||
"openai-model" => metadata.effective_model = Some(value.to_string()),
|
||||
"x-models-etag" => metadata.models_etag = Some(value.to_string()),
|
||||
"x-reasoning-included" => metadata.reasoning_included = true,
|
||||
_ => {}
|
||||
}
|
||||
upsert_client_header(&mut metadata.client_headers, (name, value.to_string()));
|
||||
}
|
||||
|
||||
fn upsert_client_header(headers: &mut Vec<(String, String)>, header: (String, String)) {
|
||||
if let Some(existing) = headers
|
||||
.iter_mut()
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case(&header.0))
|
||||
{
|
||||
*existing = header;
|
||||
} else {
|
||||
headers.push(header);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn update_first_token_ms(
|
||||
started_at: Instant,
|
||||
body_bytes: &[u8],
|
||||
@@ -50,15 +113,3 @@ pub(super) fn update_first_token_ms(
|
||||
) {
|
||||
update_first_response_output_ms(started_at, body_bytes, first_token_ms);
|
||||
}
|
||||
|
||||
fn is_rate_limit_header(name: &str) -> bool {
|
||||
let name = name.to_ascii_lowercase();
|
||||
name == "retry-after"
|
||||
|| name.contains("ratelimit")
|
||||
|| name.contains("rate-limit")
|
||||
|| name.starts_with("x-codex-primary-")
|
||||
|| name.starts_with("x-codex-secondary-")
|
||||
|| name.starts_with("x-codex-code-review-")
|
||||
|| name.starts_with("x-codex-review-")
|
||||
|| name.starts_with("x-code-review-")
|
||||
}
|
||||
|
||||
@@ -28,23 +28,25 @@ use tungstenite::{
|
||||
use crate::infra::time::china_filename_timestamp_millis;
|
||||
use crate::upstream::openai::protocol::events::{self, TokenUsage};
|
||||
use crate::upstream::openai::protocol::responses::{
|
||||
response_body_has_first_output, CodexResponsesRequest,
|
||||
response_body_has_first_output, CodexResponsesRequest, PreviousResponseScope,
|
||||
StreamCommitPolicy,
|
||||
};
|
||||
use crate::upstream::openai::protocol::sse::SseError;
|
||||
use crate::upstream::openai::protocol::websocket::{
|
||||
classify_websocket_error_frame, is_terminal_websocket_event,
|
||||
retry_after_seconds_from_wrapped_error_headers, websocket_event_to_sse_frame,
|
||||
websocket_event_type, websocket_incomplete_response_reason, websocket_metadata_turn_state,
|
||||
websocket_response_completed_parse_error, websocket_response_create_payload_text,
|
||||
OpeningAuditHeader, OpeningAuditSnapshot, WebSocketAuditArtifact,
|
||||
websocket_event_type, websocket_metadata_headers, websocket_metadata_turn_state,
|
||||
websocket_response_completed_id, websocket_response_create_payload_text,
|
||||
ClassifiedWebSocketError, OpeningAuditHeader, OpeningAuditSnapshot, WebSocketAuditArtifact,
|
||||
};
|
||||
|
||||
use super::websocket_pool::{
|
||||
CodexWebSocketConnectionMetadata, CodexWebSocketPool, CodexWebSocketPoolKey,
|
||||
PooledWebSocketConnection, WebSocketPoolAcquire, WebSocketPoolDecision, WebSocketPoolLease,
|
||||
PooledWebSocketConnection, WebSocketContinuationState, WebSocketPoolAcquire,
|
||||
WebSocketPoolDecision, WebSocketPoolLease,
|
||||
};
|
||||
use super::websocket_pump::{PumpKeepalive, PumpedWebSocket, RawWsStream};
|
||||
use super::{diagnostics::CodexUpstreamDiagnostics, response_meta};
|
||||
use super::{client::CodexResponseMetadata, diagnostics::CodexUpstreamDiagnostics, response_meta};
|
||||
use uuid::Uuid;
|
||||
|
||||
const REDACTED_HEADER_VALUE: &str = "<redacted>";
|
||||
@@ -102,16 +104,89 @@ pub async fn write_websocket_audit_artifact_from_env(
|
||||
pub struct CodexWebSocketRequest {
|
||||
pub(super) connection: CodexWebSocketConnection,
|
||||
pub(super) payload_text: String,
|
||||
pub(super) continuation: WebSocketContinuationRequirement,
|
||||
pub(super) stream_commit_policy: StreamCommitPolicy,
|
||||
}
|
||||
|
||||
/// 当前 WebSocket 请求对 previous response 状态的要求。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum WebSocketContinuationRequirement {
|
||||
/// 不依赖任何已有响应状态。
|
||||
NewChain,
|
||||
/// 上游已持久化 response,可在新连接 hydration。
|
||||
Persisted { response_id: String },
|
||||
/// 代理没有所有权信息,只允许 dispatch 选定的单个账号原样尝试。
|
||||
ExternalUnknown { response_id: String },
|
||||
/// `store=false` response,只能在拥有该 ID 的原连接续接。
|
||||
ConnectionLocal { response_id: String },
|
||||
}
|
||||
|
||||
impl WebSocketContinuationRequirement {
|
||||
fn from_request(request: &CodexResponsesRequest) -> Self {
|
||||
match request.previous_response_id() {
|
||||
None => Self::NewChain,
|
||||
Some(response_id)
|
||||
if request.previous_response_scope == Some(PreviousResponseScope::Persisted)
|
||||
|| (request.previous_response_scope.is_none() && request.store()) =>
|
||||
{
|
||||
Self::Persisted {
|
||||
response_id: response_id.to_string(),
|
||||
}
|
||||
}
|
||||
Some(response_id)
|
||||
if request.previous_response_scope
|
||||
== Some(PreviousResponseScope::ExternalUnknown) =>
|
||||
{
|
||||
Self::ExternalUnknown {
|
||||
response_id: response_id.to_string(),
|
||||
}
|
||||
}
|
||||
Some(response_id) => Self::ConnectionLocal {
|
||||
response_id: response_id.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn permits_fresh_connection(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::NewChain | Self::Persisted { .. } | Self::ExternalUnknown { .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// 连接本地 previous response 无法满足的原因。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PreviousResponseUnavailableReason {
|
||||
PoolUnavailable,
|
||||
FreshConnectionRequired,
|
||||
ConnectionBusy,
|
||||
LatestResponseMismatch,
|
||||
ReusedConnectionLost,
|
||||
UpstreamRejected,
|
||||
}
|
||||
|
||||
impl fmt::Display for PreviousResponseUnavailableReason {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::PoolUnavailable => "pool_unavailable",
|
||||
Self::FreshConnectionRequired => "fresh_connection_required",
|
||||
Self::ConnectionBusy => "connection_busy",
|
||||
Self::LatestResponseMismatch => "latest_response_mismatch",
|
||||
Self::ReusedConnectionLost => "reused_connection_lost",
|
||||
Self::UpstreamRejected => "upstream_rejected",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[path = "websocket_frames.rs"]
|
||||
mod frames;
|
||||
|
||||
use frames::{
|
||||
audit_header_value, collect_websocket_response, is_first_token_timeout,
|
||||
audit_header_value, collect_websocket_response, is_initial_event_timeout,
|
||||
prefetch_stream_frames_until_output_or_terminal, reusable_websocket_metadata,
|
||||
reused_stream_prefetch_error, stream_websocket_response, websocket_audit_file_name,
|
||||
websocket_connection_metadata, WebSocketStreamPoolReturn,
|
||||
websocket_connection_metadata, WebSocketStreamPoolReturn, WebSocketTerminalKind,
|
||||
};
|
||||
pub use frames::{
|
||||
execute_response_create_request, CodexWebSocketExchange, CodexWebSocketExchangeError,
|
||||
@@ -129,6 +204,15 @@ impl CodexWebSocketRequest {
|
||||
pub fn payload_text(&self) -> &str {
|
||||
&self.payload_text
|
||||
}
|
||||
|
||||
/// 返回连接续接要求。
|
||||
pub fn continuation(&self) -> &WebSocketContinuationRequirement {
|
||||
&self.continuation
|
||||
}
|
||||
|
||||
pub fn stream_commit_policy(&self) -> StreamCommitPolicy {
|
||||
self.stream_commit_policy
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexWebSocketConnection {
|
||||
@@ -175,6 +259,8 @@ impl CodexWebSocketConnection {
|
||||
Ok(CodexWebSocketRequest {
|
||||
connection: Self::responses(base_url, websocket_key, business_headers),
|
||||
payload_text: websocket_response_create_payload_text(request)?,
|
||||
continuation: WebSocketContinuationRequirement::from_request(request),
|
||||
stream_commit_policy: request.stream_commit_policy,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -330,74 +416,79 @@ fn request_line_for_endpoint(endpoint: &str) -> String {
|
||||
async fn execute_fresh_response_create_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let (websocket, response) =
|
||||
connect_pumped_websocket(request.connection(), PumpKeepalive::disabled()).await?;
|
||||
let first_token_started_at = Instant::now();
|
||||
websocket
|
||||
.send(Message::Text(request.payload_text().to_string().into()))
|
||||
.await?;
|
||||
|
||||
let metadata = websocket_connection_metadata(&response);
|
||||
let (exchange, _websocket, _metadata) = collect_websocket_response(
|
||||
websocket,
|
||||
metadata,
|
||||
false,
|
||||
started_at,
|
||||
first_token_started_at,
|
||||
first_token_timeout,
|
||||
)
|
||||
.await?;
|
||||
let (exchange, mut websocket, _metadata, _continuation, _terminal) =
|
||||
collect_websocket_response(
|
||||
websocket,
|
||||
metadata,
|
||||
WebSocketContinuationState::default(),
|
||||
false,
|
||||
started_at,
|
||||
initial_event_timeout,
|
||||
)
|
||||
.await?;
|
||||
websocket.close().await;
|
||||
Ok(exchange)
|
||||
}
|
||||
|
||||
async fn execute_fresh_response_create_request_stream(
|
||||
request: &CodexWebSocketRequest,
|
||||
first_token_timeout: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
) -> Result<CodexWebSocketStreamingExchange, CodexWebSocketExchangeError> {
|
||||
let (mut websocket, response) =
|
||||
connect_pumped_websocket(request.connection(), PumpKeepalive::disabled()).await?;
|
||||
let first_token_started_at = Instant::now();
|
||||
websocket
|
||||
.send(Message::Text(request.payload_text().to_string().into()))
|
||||
.await?;
|
||||
|
||||
let mut metadata = websocket_connection_metadata(&response);
|
||||
let prefetched_frames = match prefetch_stream_frames_until_output_or_terminal(
|
||||
&mut websocket,
|
||||
&mut metadata,
|
||||
first_token_started_at,
|
||||
first_token_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prefetched_frames) => prefetched_frames,
|
||||
Err(error) => {
|
||||
websocket.close().await;
|
||||
return Err(error);
|
||||
let prefetched_frames = if should_prefetch_until_output_or_terminal(request) {
|
||||
match prefetch_stream_frames_until_output_or_terminal(
|
||||
&mut websocket,
|
||||
&mut metadata,
|
||||
initial_event_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prefetched_frames) => prefetched_frames,
|
||||
Err(error) => {
|
||||
websocket.close().await;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(stream_websocket_response(
|
||||
websocket,
|
||||
metadata,
|
||||
None,
|
||||
prefetched_frames,
|
||||
initial_event_timeout,
|
||||
))
|
||||
}
|
||||
|
||||
async fn execute_fresh_response_create_request_with_retries(
|
||||
request: &CodexWebSocketRequest,
|
||||
started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
attempts: usize,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let attempts = attempts.max(1);
|
||||
for attempt in 1..=attempts {
|
||||
match execute_fresh_response_create_request(request, started_at, first_token_timeout).await
|
||||
match execute_fresh_response_create_request(request, started_at, initial_event_timeout)
|
||||
.await
|
||||
{
|
||||
Ok(exchange) => return Ok(exchange),
|
||||
Err(error) if attempt < attempts && is_first_token_timeout(&error) => continue,
|
||||
Err(error) if attempt < attempts && is_initial_event_timeout(&error) => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
@@ -406,14 +497,14 @@ async fn execute_fresh_response_create_request_with_retries(
|
||||
|
||||
async fn execute_fresh_response_create_request_stream_with_retries(
|
||||
request: &CodexWebSocketRequest,
|
||||
first_token_timeout: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
attempts: usize,
|
||||
) -> Result<CodexWebSocketStreamingExchange, CodexWebSocketExchangeError> {
|
||||
let attempts = attempts.max(1);
|
||||
for attempt in 1..=attempts {
|
||||
match execute_fresh_response_create_request_stream(request, first_token_timeout).await {
|
||||
match execute_fresh_response_create_request_stream(request, initial_event_timeout).await {
|
||||
Ok(exchange) => return Ok(exchange),
|
||||
Err(error) if attempt < attempts && is_first_token_timeout(&error) => continue,
|
||||
Err(error) if attempt < attempts && is_initial_event_timeout(&error) => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
@@ -428,13 +519,18 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
request: &CodexWebSocketRequest,
|
||||
pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>,
|
||||
started_at: Instant,
|
||||
fallback_first_token_timeout: Option<Duration>,
|
||||
fallback_initial_event_timeout: Option<Duration>,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let Some((pool, key)) = pool else {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::PoolUnavailable,
|
||||
));
|
||||
}
|
||||
return execute_fresh_response_create_request_with_retries(
|
||||
request,
|
||||
started_at,
|
||||
fallback_first_token_timeout,
|
||||
fallback_initial_event_timeout,
|
||||
WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS,
|
||||
)
|
||||
.await;
|
||||
@@ -442,6 +538,16 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
|
||||
match pool.acquire(&key).await {
|
||||
WebSocketPoolAcquire::Reused { connection, lease } => {
|
||||
if let WebSocketContinuationRequirement::ConnectionLocal { response_id } =
|
||||
request.continuation()
|
||||
{
|
||||
if connection.continuation.latest_response_id() != Some(response_id.as_str()) {
|
||||
lease.put(*connection).await;
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::LatestResponseMismatch,
|
||||
));
|
||||
}
|
||||
}
|
||||
let result = execute_pooled_response_create_request(
|
||||
request,
|
||||
pool,
|
||||
@@ -454,11 +560,16 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput {
|
||||
..
|
||||
})
|
||||
| Err(CodexWebSocketExchangeError::FirstTokenTimeout { .. }) => {
|
||||
| Err(CodexWebSocketExchangeError::InitialEventTimeout { .. }) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::ReusedConnectionLost,
|
||||
));
|
||||
}
|
||||
let mut exchange = execute_fresh_response_create_request_with_retries(
|
||||
request,
|
||||
started_at,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS,
|
||||
)
|
||||
.await?;
|
||||
@@ -473,6 +584,12 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::FreshReserved(lease) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
lease.discard().await;
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::FreshConnectionRequired,
|
||||
));
|
||||
}
|
||||
let result =
|
||||
execute_fresh_pooled_response_create_request(request, pool, lease, started_at)
|
||||
.await;
|
||||
@@ -481,11 +598,11 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
exchange.pool_decision = Some(WebSocketPoolDecision::new());
|
||||
Ok(exchange)
|
||||
}
|
||||
Err(error) if is_first_token_timeout(&error) => {
|
||||
Err(error) if is_initial_event_timeout(&error) => {
|
||||
let mut exchange = execute_fresh_response_create_request_with_retries(
|
||||
request,
|
||||
started_at,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
1,
|
||||
)
|
||||
.await?;
|
||||
@@ -496,10 +613,21 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::Bypass(reason) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
return Err(continuation_unavailable(match reason {
|
||||
super::websocket_pool::WebSocketPoolBypassReason::Busy => {
|
||||
PreviousResponseUnavailableReason::ConnectionBusy
|
||||
}
|
||||
super::websocket_pool::WebSocketPoolBypassReason::Disabled
|
||||
| super::websocket_pool::WebSocketPoolBypassReason::Cap => {
|
||||
PreviousResponseUnavailableReason::PoolUnavailable
|
||||
}
|
||||
}));
|
||||
}
|
||||
let mut exchange = execute_fresh_response_create_request_with_retries(
|
||||
request,
|
||||
started_at,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS,
|
||||
)
|
||||
.await?;
|
||||
@@ -512,12 +640,17 @@ pub(crate) async fn execute_response_create_request_with_pool(
|
||||
pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
request: &CodexWebSocketRequest,
|
||||
pool: Option<(&CodexWebSocketPool, CodexWebSocketPoolKey)>,
|
||||
fallback_first_token_timeout: Option<Duration>,
|
||||
fallback_initial_event_timeout: Option<Duration>,
|
||||
) -> Result<CodexWebSocketStreamingExchange, CodexWebSocketExchangeError> {
|
||||
let Some((pool, key)) = pool else {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::PoolUnavailable,
|
||||
));
|
||||
}
|
||||
return execute_fresh_response_create_request_stream_with_retries(
|
||||
request,
|
||||
fallback_first_token_timeout,
|
||||
fallback_initial_event_timeout,
|
||||
WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS,
|
||||
)
|
||||
.await;
|
||||
@@ -525,6 +658,16 @@ pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
|
||||
match pool.acquire(&key).await {
|
||||
WebSocketPoolAcquire::Reused { connection, lease } => {
|
||||
if let WebSocketContinuationRequirement::ConnectionLocal { response_id } =
|
||||
request.continuation()
|
||||
{
|
||||
if connection.continuation.latest_response_id() != Some(response_id.as_str()) {
|
||||
lease.put(*connection).await;
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::LatestResponseMismatch,
|
||||
));
|
||||
}
|
||||
}
|
||||
let result =
|
||||
execute_pooled_response_create_request_stream(request, pool, lease, *connection)
|
||||
.await;
|
||||
@@ -532,10 +675,15 @@ pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
Err(CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput {
|
||||
..
|
||||
})
|
||||
| Err(CodexWebSocketExchangeError::FirstTokenTimeout { .. }) => {
|
||||
| Err(CodexWebSocketExchangeError::InitialEventTimeout { .. }) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::ReusedConnectionLost,
|
||||
));
|
||||
}
|
||||
let mut exchange = execute_fresh_response_create_request_stream_with_retries(
|
||||
request,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS,
|
||||
)
|
||||
.await?;
|
||||
@@ -550,6 +698,12 @@ pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::FreshReserved(lease) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
lease.discard().await;
|
||||
return Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::FreshConnectionRequired,
|
||||
));
|
||||
}
|
||||
let result =
|
||||
execute_fresh_pooled_response_create_request_stream(request, pool, lease).await;
|
||||
match result {
|
||||
@@ -557,10 +711,10 @@ pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
exchange.pool_decision = Some(WebSocketPoolDecision::new());
|
||||
Ok(exchange)
|
||||
}
|
||||
Err(error) if is_first_token_timeout(&error) => {
|
||||
Err(error) if is_initial_event_timeout(&error) => {
|
||||
let mut exchange = execute_fresh_response_create_request_stream_with_retries(
|
||||
request,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
1,
|
||||
)
|
||||
.await?;
|
||||
@@ -571,9 +725,20 @@ pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::Bypass(reason) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
return Err(continuation_unavailable(match reason {
|
||||
super::websocket_pool::WebSocketPoolBypassReason::Busy => {
|
||||
PreviousResponseUnavailableReason::ConnectionBusy
|
||||
}
|
||||
super::websocket_pool::WebSocketPoolBypassReason::Disabled
|
||||
| super::websocket_pool::WebSocketPoolBypassReason::Cap => {
|
||||
PreviousResponseUnavailableReason::PoolUnavailable
|
||||
}
|
||||
}));
|
||||
}
|
||||
let mut exchange = execute_fresh_response_create_request_stream_with_retries(
|
||||
request,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
WEBSOCKET_FIRST_TOKEN_FRESH_RETRY_ATTEMPTS,
|
||||
)
|
||||
.await?;
|
||||
@@ -583,6 +748,12 @@ pub(crate) async fn execute_response_create_request_stream_with_pool(
|
||||
}
|
||||
}
|
||||
|
||||
fn continuation_unavailable(
|
||||
reason: PreviousResponseUnavailableReason,
|
||||
) -> CodexWebSocketExchangeError {
|
||||
CodexWebSocketExchangeError::ContinuationUnavailable { reason }
|
||||
}
|
||||
|
||||
async fn execute_fresh_pooled_response_create_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
pool: &CodexWebSocketPool,
|
||||
@@ -597,7 +768,6 @@ async fn execute_fresh_pooled_response_create_request(
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let first_token_started_at = Instant::now();
|
||||
if let Err(error) = websocket
|
||||
.send(Message::Text(request.payload_text().to_string().into()))
|
||||
.await
|
||||
@@ -608,29 +778,39 @@ async fn execute_fresh_pooled_response_create_request(
|
||||
|
||||
let now = Instant::now();
|
||||
let metadata = websocket_connection_metadata(&response);
|
||||
let (exchange, websocket, metadata) = match collect_websocket_response(
|
||||
websocket,
|
||||
metadata,
|
||||
false,
|
||||
started_at,
|
||||
first_token_started_at,
|
||||
pool.first_token_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(exchange) => exchange,
|
||||
Err(error) => {
|
||||
lease.discard().await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
lease
|
||||
.put(PooledWebSocketConnection {
|
||||
let (exchange, mut websocket, metadata, continuation, terminal) =
|
||||
match collect_websocket_response(
|
||||
websocket,
|
||||
metadata: reusable_websocket_metadata(metadata),
|
||||
created_at: now,
|
||||
})
|
||||
.await;
|
||||
metadata,
|
||||
WebSocketContinuationState::default(),
|
||||
false,
|
||||
started_at,
|
||||
pool.initial_event_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(exchange) => exchange,
|
||||
Err(error) => {
|
||||
lease.discard().await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
match terminal {
|
||||
WebSocketTerminalKind::Completed => {
|
||||
lease
|
||||
.put(PooledWebSocketConnection {
|
||||
websocket,
|
||||
metadata: reusable_websocket_metadata(metadata),
|
||||
continuation,
|
||||
created_at: now,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
WebSocketTerminalKind::Incomplete | WebSocketTerminalKind::Failed => {
|
||||
lease.discard().await;
|
||||
websocket.close().await;
|
||||
}
|
||||
}
|
||||
Ok(exchange)
|
||||
}
|
||||
|
||||
@@ -647,7 +827,6 @@ async fn execute_fresh_pooled_response_create_request_stream(
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let first_token_started_at = Instant::now();
|
||||
if let Err(error) = websocket
|
||||
.send(Message::Text(request.payload_text().to_string().into()))
|
||||
.await
|
||||
@@ -658,20 +837,23 @@ async fn execute_fresh_pooled_response_create_request_stream(
|
||||
|
||||
let now = Instant::now();
|
||||
let mut metadata = websocket_connection_metadata(&response);
|
||||
let prefetched_frames = match prefetch_stream_frames_until_output_or_terminal(
|
||||
&mut websocket,
|
||||
&mut metadata,
|
||||
first_token_started_at,
|
||||
pool.first_token_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prefetched_frames) => prefetched_frames,
|
||||
Err(error) => {
|
||||
websocket.close().await;
|
||||
lease.discard().await;
|
||||
return Err(error);
|
||||
let prefetched_frames = if should_prefetch_until_output_or_terminal(request) {
|
||||
match prefetch_stream_frames_until_output_or_terminal(
|
||||
&mut websocket,
|
||||
&mut metadata,
|
||||
pool.initial_event_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prefetched_frames) => prefetched_frames,
|
||||
Err(error) => {
|
||||
websocket.close().await;
|
||||
lease.discard().await;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(stream_websocket_response(
|
||||
websocket,
|
||||
@@ -679,8 +861,10 @@ async fn execute_fresh_pooled_response_create_request_stream(
|
||||
Some(WebSocketStreamPoolReturn {
|
||||
lease,
|
||||
created_at: now,
|
||||
continuation: WebSocketContinuationState::default(),
|
||||
}),
|
||||
prefetched_frames,
|
||||
pool.initial_event_timeout(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -691,7 +875,6 @@ async fn execute_pooled_response_create_request(
|
||||
connection: PooledWebSocketConnection,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexWebSocketExchange, CodexWebSocketExchangeError> {
|
||||
let first_token_started_at = Instant::now();
|
||||
if let Err(error) = connection
|
||||
.websocket
|
||||
.send(Message::Text(request.payload_text().to_string().into()))
|
||||
@@ -709,21 +892,30 @@ async fn execute_pooled_response_create_request(
|
||||
match collect_websocket_response(
|
||||
connection.websocket,
|
||||
reusable_websocket_metadata(connection.metadata),
|
||||
connection.continuation,
|
||||
true,
|
||||
started_at,
|
||||
first_token_started_at,
|
||||
pool.first_token_timeout(),
|
||||
pool.initial_event_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((exchange, websocket, metadata)) => {
|
||||
lease
|
||||
.put(PooledWebSocketConnection {
|
||||
websocket,
|
||||
metadata: reusable_websocket_metadata(metadata),
|
||||
created_at,
|
||||
})
|
||||
.await;
|
||||
Ok((exchange, mut websocket, metadata, continuation, terminal)) => {
|
||||
match terminal {
|
||||
WebSocketTerminalKind::Completed => {
|
||||
lease
|
||||
.put(PooledWebSocketConnection {
|
||||
websocket,
|
||||
metadata: reusable_websocket_metadata(metadata),
|
||||
continuation,
|
||||
created_at,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
WebSocketTerminalKind::Incomplete | WebSocketTerminalKind::Failed => {
|
||||
lease.discard().await;
|
||||
websocket.close().await;
|
||||
}
|
||||
}
|
||||
Ok(exchange)
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -742,7 +934,7 @@ async fn execute_pooled_response_create_request_stream(
|
||||
let mut websocket = connection.websocket;
|
||||
let mut metadata = reusable_websocket_metadata(connection.metadata);
|
||||
let created_at = connection.created_at;
|
||||
let first_token_started_at = Instant::now();
|
||||
let continuation = connection.continuation;
|
||||
if let Err(error) = websocket
|
||||
.send(Message::Text(request.payload_text().to_string().into()))
|
||||
.await
|
||||
@@ -755,26 +947,41 @@ async fn execute_pooled_response_create_request_stream(
|
||||
);
|
||||
}
|
||||
|
||||
let prefetched_frames = match prefetch_stream_frames_until_output_or_terminal(
|
||||
&mut websocket,
|
||||
&mut metadata,
|
||||
first_token_started_at,
|
||||
pool.first_token_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prefetched_frames) => prefetched_frames,
|
||||
Err(error) => {
|
||||
lease.discard().await;
|
||||
websocket.close().await;
|
||||
return Err(reused_stream_prefetch_error(error));
|
||||
let prefetched_frames = if should_prefetch_until_output_or_terminal(request) {
|
||||
match prefetch_stream_frames_until_output_or_terminal(
|
||||
&mut websocket,
|
||||
&mut metadata,
|
||||
pool.initial_event_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prefetched_frames) => prefetched_frames,
|
||||
Err(error) => {
|
||||
lease.discard().await;
|
||||
websocket.close().await;
|
||||
return Err(reused_stream_prefetch_error(error));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
Ok(stream_websocket_response(
|
||||
websocket,
|
||||
metadata,
|
||||
Some(WebSocketStreamPoolReturn { lease, created_at }),
|
||||
Some(WebSocketStreamPoolReturn {
|
||||
lease,
|
||||
created_at,
|
||||
continuation,
|
||||
}),
|
||||
prefetched_frames,
|
||||
pool.initial_event_timeout(),
|
||||
))
|
||||
}
|
||||
|
||||
fn should_prefetch_until_output_or_terminal(request: &CodexWebSocketRequest) -> bool {
|
||||
matches!(
|
||||
request.stream_commit_policy(),
|
||||
StreamCommitPolicy::UntilOutputOrTerminal
|
||||
)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ pub struct CodexWebSocketExchange {
|
||||
pub pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
/// 安全响应元数据。
|
||||
pub response_metadata: CodexResponseMetadata,
|
||||
}
|
||||
|
||||
/// Responses WebSocket live SSE exchange result.
|
||||
@@ -41,6 +43,8 @@ pub struct CodexWebSocketStreamingExchange {
|
||||
pub pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
/// 安全响应元数据。
|
||||
pub response_metadata: CodexResponseMetadata,
|
||||
}
|
||||
|
||||
/// Responses WebSocket live SSE byte stream.
|
||||
@@ -51,6 +55,13 @@ pub type CodexWebSocketRateLimitHeaderUpdates = Arc<Mutex<Vec<(String, String)>>
|
||||
/// WebSocket live stream turn-state update.
|
||||
pub type CodexWebSocketTurnStateUpdate = Arc<Mutex<Option<String>>>;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum WebSocketTerminalKind {
|
||||
Completed,
|
||||
Incomplete,
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Responses WebSocket exchange error.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CodexWebSocketExchangeError {
|
||||
@@ -66,11 +77,10 @@ pub enum CodexWebSocketExchangeError {
|
||||
/// 上游 WebSocket 错误帧。
|
||||
#[error("{0}")]
|
||||
Upstream(Box<CodexWebSocketUpstreamError>),
|
||||
/// 上游返回 `response.incomplete`。
|
||||
#[error("Incomplete response returned, reason: {reason}")]
|
||||
IncompleteResponse {
|
||||
/// incomplete_details.reason。
|
||||
reason: String,
|
||||
/// 请求依赖的连接本地 previous response 无法在当前连接满足。
|
||||
#[error("websocket continuation unavailable: {reason}")]
|
||||
ContinuationUnavailable {
|
||||
reason: PreviousResponseUnavailableReason,
|
||||
},
|
||||
/// 上游返回无法按官方形状解析的 `response.completed`。
|
||||
#[error("{message}")]
|
||||
@@ -98,8 +108,8 @@ pub enum CodexWebSocketExchangeError {
|
||||
},
|
||||
/// 首个真实输出在配置的绝对超时内未到达(连接落到病态上游后端)。
|
||||
#[error("websocket first output not received within {timeout:?}")]
|
||||
FirstTokenTimeout {
|
||||
/// 首 token 超时时长。
|
||||
InitialEventTimeout {
|
||||
/// 首个上游事件超时时长。
|
||||
timeout: Duration,
|
||||
},
|
||||
}
|
||||
@@ -113,6 +123,16 @@ pub struct CodexWebSocketUpstreamError {
|
||||
pub retry_after_seconds: Option<u64>,
|
||||
/// 原始错误帧。
|
||||
pub body: String,
|
||||
/// 上游错误类型。
|
||||
pub error_type: Option<String>,
|
||||
/// 上游错误码。
|
||||
pub code: Option<String>,
|
||||
/// 上游错误消息。
|
||||
pub message: Option<String>,
|
||||
/// 上游错误参数。
|
||||
pub param: Option<String>,
|
||||
/// 错误事件携带的响应头。
|
||||
pub headers: Vec<(String, String)>,
|
||||
/// 上游透传的 `set-cookie` 列表。
|
||||
pub set_cookie_headers: Vec<String>,
|
||||
/// 上游诊断元数据。
|
||||
@@ -141,10 +161,36 @@ impl CodexWebSocketExchangeError {
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
error_type: None,
|
||||
code: None,
|
||||
message: None,
|
||||
param: None,
|
||||
headers: Vec::new(),
|
||||
set_cookie_headers,
|
||||
diagnostics,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn upstream_frame(
|
||||
classified: ClassifiedWebSocketError,
|
||||
retry_after_seconds: Option<u64>,
|
||||
body: String,
|
||||
) -> Self {
|
||||
let diagnostics =
|
||||
CodexUpstreamDiagnostics::from_pairs(Some(classified.status_code), &classified.headers);
|
||||
Self::Upstream(Box::new(CodexWebSocketUpstreamError {
|
||||
status_code: classified.status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
error_type: classified.error_type,
|
||||
code: classified.code,
|
||||
message: classified.message,
|
||||
param: classified.param,
|
||||
headers: classified.headers,
|
||||
set_cookie_headers: Vec::new(),
|
||||
diagnostics,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一次 prepared Responses WebSocket 请求并聚合为 SSE。
|
||||
@@ -161,40 +207,37 @@ pub async fn execute_response_create_request(
|
||||
pub(super) async fn collect_websocket_response(
|
||||
mut websocket: PumpedWebSocket,
|
||||
mut metadata: CodexWebSocketConnectionMetadata,
|
||||
mut continuation: WebSocketContinuationState,
|
||||
reused_connection: bool,
|
||||
started_at: Instant,
|
||||
first_token_started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
) -> Result<
|
||||
(
|
||||
CodexWebSocketExchange,
|
||||
PumpedWebSocket,
|
||||
CodexWebSocketConnectionMetadata,
|
||||
WebSocketContinuationState,
|
||||
WebSocketTerminalKind,
|
||||
),
|
||||
CodexWebSocketExchangeError,
|
||||
> {
|
||||
let mut body = String::new();
|
||||
let mut saw_first_output = false;
|
||||
let mut saw_upstream_activity = false;
|
||||
let mut first_token_ms = None;
|
||||
|
||||
loop {
|
||||
let receive_timeout = receive_idle_timeout_with_first_token_deadline(
|
||||
saw_first_output,
|
||||
first_token_started_at,
|
||||
first_token_timeout,
|
||||
)?;
|
||||
let receive_timeout = receive_idle_timeout(saw_upstream_activity, initial_event_timeout);
|
||||
let message = match next_websocket_message(&mut websocket, receive_timeout).await {
|
||||
Ok(message) => message,
|
||||
Err(error) if !saw_first_output => {
|
||||
if let Some(timeout) =
|
||||
elapsed_first_token_timeout(first_token_started_at, first_token_timeout)
|
||||
{
|
||||
return Err(CodexWebSocketExchangeError::FirstTokenTimeout { timeout });
|
||||
}
|
||||
Err(CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout })
|
||||
if !saw_upstream_activity =>
|
||||
{
|
||||
if reused_connection {
|
||||
return Err(reused_connection_died_before_first_output(&error));
|
||||
return Err(reused_connection_died_before_first_output(
|
||||
&CodexWebSocketExchangeError::InitialEventTimeout { timeout },
|
||||
));
|
||||
}
|
||||
return Err(error);
|
||||
return Err(CodexWebSocketExchangeError::InitialEventTimeout { timeout });
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
@@ -202,9 +245,12 @@ pub(super) async fn collect_websocket_response(
|
||||
break;
|
||||
};
|
||||
let text = match message {
|
||||
Message::Text(text) => text,
|
||||
Message::Text(text) => {
|
||||
saw_upstream_activity = true;
|
||||
text
|
||||
}
|
||||
Message::Binary(_) => return Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent),
|
||||
Message::Close(_) if reused_connection && !saw_first_output => {
|
||||
Message::Close(_) if reused_connection && !saw_upstream_activity => {
|
||||
return Err(
|
||||
CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput {
|
||||
message: "websocket closed".to_string(),
|
||||
@@ -218,34 +264,42 @@ pub(super) async fn collect_websocket_response(
|
||||
if let Some(classified) = classify_websocket_error_frame(&raw) {
|
||||
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&raw));
|
||||
return Err(CodexWebSocketExchangeError::upstream(
|
||||
classified.status_code,
|
||||
return Err(CodexWebSocketExchangeError::upstream_frame(
|
||||
classified,
|
||||
retry_after_seconds,
|
||||
raw,
|
||||
Vec::new(),
|
||||
CodexUpstreamDiagnostics::with_status(classified.status_code),
|
||||
));
|
||||
}
|
||||
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
|
||||
return Err(CodexWebSocketExchangeError::IncompleteResponse { reason });
|
||||
}
|
||||
if let Some(message) = websocket_response_completed_parse_error(&raw) {
|
||||
return Err(CodexWebSocketExchangeError::InvalidCompletedResponse { message });
|
||||
}
|
||||
if let Some(headers) = websocket_rate_limit_event_headers(&raw) {
|
||||
metadata.rate_limit_headers.extend(headers);
|
||||
continue;
|
||||
}
|
||||
update_websocket_response_metadata(&mut metadata, &raw);
|
||||
if let Some(metadata_turn_state) = websocket_metadata_turn_state(&raw) {
|
||||
metadata.turn_state = Some(metadata_turn_state);
|
||||
continue;
|
||||
}
|
||||
let event = websocket_event_type(&raw);
|
||||
if event.as_deref() == Some("response.completed") {
|
||||
let response_id = websocket_response_completed_id(&raw)
|
||||
.map_err(
|
||||
|message| CodexWebSocketExchangeError::InvalidCompletedResponse { message },
|
||||
)?
|
||||
.ok_or_else(|| CodexWebSocketExchangeError::InvalidCompletedResponse {
|
||||
message: "response.completed is missing response id".to_string(),
|
||||
})?;
|
||||
continuation.record_completed(response_id);
|
||||
}
|
||||
let terminal = match event.as_deref() {
|
||||
Some("response.completed") => Some(WebSocketTerminalKind::Completed),
|
||||
Some("response.incomplete") => Some(WebSocketTerminalKind::Incomplete),
|
||||
Some("response.failed" | "error") => Some(WebSocketTerminalKind::Failed),
|
||||
_ => None,
|
||||
};
|
||||
let forwarded = if let Some(frame) = websocket_event_to_sse_frame(&raw) {
|
||||
let has_first_output = response_body_has_first_output(frame.as_bytes());
|
||||
body.push_str(&frame);
|
||||
if has_first_output {
|
||||
saw_first_output = true;
|
||||
response_meta::update_first_token_ms(
|
||||
started_at,
|
||||
body.as_bytes(),
|
||||
@@ -256,7 +310,7 @@ pub(super) async fn collect_websocket_response(
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if forwarded && event.as_deref().is_some_and(is_terminal_websocket_event) {
|
||||
if let (true, Some(terminal)) = (forwarded, terminal) {
|
||||
let usage = events::extract_sse_usage(&body)?;
|
||||
let exchange = CodexWebSocketExchange {
|
||||
body,
|
||||
@@ -267,12 +321,13 @@ pub(super) async fn collect_websocket_response(
|
||||
first_token_ms,
|
||||
pool_decision: None,
|
||||
diagnostics: metadata.diagnostics.clone(),
|
||||
response_metadata: metadata.response_metadata.clone(),
|
||||
};
|
||||
return Ok((exchange, websocket, metadata));
|
||||
return Ok((exchange, websocket, metadata, continuation, terminal));
|
||||
}
|
||||
}
|
||||
|
||||
if reused_connection && !saw_first_output {
|
||||
if reused_connection && !saw_upstream_activity {
|
||||
return Err(
|
||||
CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstOutput {
|
||||
message: "websocket closed before terminal event".to_string(),
|
||||
@@ -286,28 +341,28 @@ pub(super) async fn collect_websocket_response(
|
||||
pub(super) async fn prefetch_stream_frames_until_output_or_terminal(
|
||||
websocket: &mut PumpedWebSocket,
|
||||
metadata: &mut CodexWebSocketConnectionMetadata,
|
||||
first_token_started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
) -> Result<Vec<String>, CodexWebSocketExchangeError> {
|
||||
let mut prefetched_frames = Vec::new();
|
||||
let mut saw_upstream_activity = false;
|
||||
loop {
|
||||
let receive_timeout =
|
||||
receive_timeout_before_first_output(first_token_started_at, first_token_timeout)?;
|
||||
let receive_timeout = receive_idle_timeout(saw_upstream_activity, initial_event_timeout);
|
||||
let message = match next_websocket_message(websocket, receive_timeout).await {
|
||||
Ok(Some(message)) => message,
|
||||
Ok(None) => return Err(CodexWebSocketExchangeError::ClosedBeforeTerminal),
|
||||
Err(error) => {
|
||||
if let Some(timeout) =
|
||||
elapsed_first_token_timeout(first_token_started_at, first_token_timeout)
|
||||
{
|
||||
return Err(CodexWebSocketExchangeError::FirstTokenTimeout { timeout });
|
||||
}
|
||||
return Err(error);
|
||||
Err(CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout })
|
||||
if !saw_upstream_activity =>
|
||||
{
|
||||
return Err(CodexWebSocketExchangeError::InitialEventTimeout { timeout });
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
|
||||
let raw = match message {
|
||||
Message::Text(text) => text.to_string(),
|
||||
Message::Text(text) => {
|
||||
saw_upstream_activity = true;
|
||||
text.to_string()
|
||||
}
|
||||
Message::Binary(_) => return Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent),
|
||||
Message::Close(_) => return Err(CodexWebSocketExchangeError::ClosedBeforeTerminal),
|
||||
_ => continue,
|
||||
@@ -316,24 +371,22 @@ pub(super) async fn prefetch_stream_frames_until_output_or_terminal(
|
||||
if let Some(classified) = classify_websocket_error_frame(&raw) {
|
||||
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&raw));
|
||||
return Err(CodexWebSocketExchangeError::upstream(
|
||||
classified.status_code,
|
||||
return Err(CodexWebSocketExchangeError::upstream_frame(
|
||||
classified,
|
||||
retry_after_seconds,
|
||||
raw,
|
||||
Vec::new(),
|
||||
CodexUpstreamDiagnostics::with_status(classified.status_code),
|
||||
));
|
||||
}
|
||||
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
|
||||
return Err(CodexWebSocketExchangeError::IncompleteResponse { reason });
|
||||
}
|
||||
if let Some(message) = websocket_response_completed_parse_error(&raw) {
|
||||
return Err(CodexWebSocketExchangeError::InvalidCompletedResponse { message });
|
||||
if websocket_event_type(&raw).as_deref() == Some("response.completed") {
|
||||
websocket_response_completed_id(&raw).map_err(|message| {
|
||||
CodexWebSocketExchangeError::InvalidCompletedResponse { message }
|
||||
})?;
|
||||
}
|
||||
if let Some(headers) = websocket_rate_limit_event_headers(&raw) {
|
||||
metadata.rate_limit_headers.extend(headers);
|
||||
continue;
|
||||
}
|
||||
update_websocket_response_metadata(metadata, &raw);
|
||||
if let Some(metadata_turn_state) = websocket_metadata_turn_state(&raw) {
|
||||
metadata.turn_state = Some(metadata_turn_state);
|
||||
continue;
|
||||
@@ -380,6 +433,7 @@ pub(super) fn websocket_connection_metadata(
|
||||
turn_state: response_meta::turn_state(response.headers()),
|
||||
set_cookie_headers: response_meta::set_cookie_headers(response.headers()),
|
||||
rate_limit_headers: response_meta::rate_limit_headers(response.headers()),
|
||||
response_metadata: response_meta::response_metadata(response.headers()),
|
||||
diagnostics: response_meta::diagnostics(
|
||||
Some(response.status().as_u16()),
|
||||
response.headers(),
|
||||
@@ -401,6 +455,7 @@ pub(super) fn reusable_websocket_metadata(
|
||||
pub(super) struct WebSocketStreamPoolReturn {
|
||||
pub(super) lease: WebSocketPoolLease,
|
||||
pub(super) created_at: Instant,
|
||||
pub(super) continuation: WebSocketContinuationState,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -408,6 +463,7 @@ enum StreamWebSocketDiscardReason {
|
||||
ClientCancelled,
|
||||
DownstreamSendFailed,
|
||||
IncompleteResponse,
|
||||
FailedResponse,
|
||||
InvalidCompletedResponse,
|
||||
UnexpectedBinaryEvent,
|
||||
UpstreamClosed,
|
||||
@@ -421,6 +477,7 @@ impl StreamWebSocketDiscardReason {
|
||||
Self::ClientCancelled => "client_cancelled",
|
||||
Self::DownstreamSendFailed => "downstream_send_failed",
|
||||
Self::IncompleteResponse => "incomplete_response",
|
||||
Self::FailedResponse => "failed_response",
|
||||
Self::InvalidCompletedResponse => "invalid_completed_response",
|
||||
Self::UnexpectedBinaryEvent => "unexpected_binary_event",
|
||||
Self::UpstreamClosed => "upstream_closed",
|
||||
@@ -435,6 +492,7 @@ pub(super) fn stream_websocket_response(
|
||||
metadata: CodexWebSocketConnectionMetadata,
|
||||
pool_return: Option<WebSocketStreamPoolReturn>,
|
||||
prefetched_frames: Vec<String>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
) -> CodexWebSocketStreamingExchange {
|
||||
let response_metadata = metadata.clone();
|
||||
let rate_limit_header_updates = Arc::new(Mutex::new(Vec::new()));
|
||||
@@ -443,15 +501,16 @@ pub(super) fn stream_websocket_response(
|
||||
let turn_state_update_for_task = Arc::clone(&turn_state_update);
|
||||
let (tx, rx) = mpsc::channel(WEBSOCKET_STREAM_BUFFER);
|
||||
tokio::spawn(async move {
|
||||
forward_websocket_response_stream(
|
||||
forward_websocket_response_stream(WebSocketStreamForwardState {
|
||||
websocket,
|
||||
metadata,
|
||||
pool_return,
|
||||
prefetched_frames,
|
||||
rate_limit_header_updates_for_task,
|
||||
turn_state_update_for_task,
|
||||
initial_event_timeout,
|
||||
rate_limit_header_updates: rate_limit_header_updates_for_task,
|
||||
turn_state_update: turn_state_update_for_task,
|
||||
tx,
|
||||
)
|
||||
})
|
||||
.await;
|
||||
});
|
||||
|
||||
@@ -468,21 +527,39 @@ pub(super) fn stream_websocket_response(
|
||||
turn_state_update,
|
||||
pool_decision: None,
|
||||
diagnostics: response_metadata.diagnostics,
|
||||
response_metadata: response_metadata.response_metadata,
|
||||
}
|
||||
}
|
||||
|
||||
async fn forward_websocket_response_stream(
|
||||
mut websocket: PumpedWebSocket,
|
||||
mut metadata: CodexWebSocketConnectionMetadata,
|
||||
struct WebSocketStreamForwardState {
|
||||
websocket: PumpedWebSocket,
|
||||
metadata: CodexWebSocketConnectionMetadata,
|
||||
pool_return: Option<WebSocketStreamPoolReturn>,
|
||||
prefetched_frames: Vec<String>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
rate_limit_header_updates: CodexWebSocketRateLimitHeaderUpdates,
|
||||
turn_state_update: CodexWebSocketTurnStateUpdate,
|
||||
tx: mpsc::Sender<Result<Bytes, CodexWebSocketExchangeError>>,
|
||||
) {
|
||||
}
|
||||
|
||||
async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
let WebSocketStreamForwardState {
|
||||
mut websocket,
|
||||
mut metadata,
|
||||
pool_return,
|
||||
prefetched_frames,
|
||||
initial_event_timeout,
|
||||
rate_limit_header_updates,
|
||||
turn_state_update,
|
||||
tx,
|
||||
} = state;
|
||||
let mut pool_return = pool_return;
|
||||
let mut continuation = pool_return
|
||||
.as_mut()
|
||||
.map(|pool_return| std::mem::take(&mut pool_return.continuation))
|
||||
.unwrap_or_default();
|
||||
let mut saw_upstream_activity = !prefetched_frames.is_empty();
|
||||
let mut prefetched_frames: VecDeque<String> = prefetched_frames.into();
|
||||
let mut saw_first_output = false;
|
||||
loop {
|
||||
let raw = if let Some(text) = prefetched_frames.pop_front() {
|
||||
text
|
||||
@@ -501,7 +578,7 @@ async fn forward_websocket_response_stream(
|
||||
}
|
||||
message = next_websocket_message(
|
||||
&mut websocket,
|
||||
receive_idle_timeout(saw_first_output),
|
||||
receive_idle_timeout(saw_upstream_activity, initial_event_timeout),
|
||||
) => message,
|
||||
};
|
||||
let message = match message {
|
||||
@@ -521,7 +598,10 @@ async fn forward_websocket_response_stream(
|
||||
break;
|
||||
};
|
||||
match message {
|
||||
Message::Text(text) => text.to_string(),
|
||||
Message::Text(text) => {
|
||||
saw_upstream_activity = true;
|
||||
text.to_string()
|
||||
}
|
||||
Message::Binary(_) => {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -559,60 +639,50 @@ async fn forward_websocket_response_stream(
|
||||
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&raw));
|
||||
let _ = tx
|
||||
.send(Err(CodexWebSocketExchangeError::upstream(
|
||||
classified.status_code,
|
||||
.send(Err(CodexWebSocketExchangeError::upstream_frame(
|
||||
classified,
|
||||
retry_after_seconds,
|
||||
raw,
|
||||
Vec::new(),
|
||||
CodexUpstreamDiagnostics::with_status(classified.status_code),
|
||||
)))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::IncompleteResponse,
|
||||
)
|
||||
.await;
|
||||
let _ = tx
|
||||
.send(Err(CodexWebSocketExchangeError::IncompleteResponse {
|
||||
reason,
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
if let Some(message) = websocket_response_completed_parse_error(&raw) {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::InvalidCompletedResponse,
|
||||
)
|
||||
.await;
|
||||
let _ = tx
|
||||
.send(Err(CodexWebSocketExchangeError::InvalidCompletedResponse {
|
||||
message,
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
if let Some(headers) = websocket_rate_limit_event_headers(&raw) {
|
||||
metadata.rate_limit_headers.extend(headers.iter().cloned());
|
||||
rate_limit_header_updates.lock().await.extend(headers);
|
||||
continue;
|
||||
}
|
||||
update_websocket_response_metadata(&mut metadata, &raw);
|
||||
if let Some(metadata_turn_state) = websocket_metadata_turn_state(&raw) {
|
||||
metadata.turn_state = Some(metadata_turn_state);
|
||||
*turn_state_update.lock().await = metadata.turn_state.clone();
|
||||
continue;
|
||||
}
|
||||
let event = websocket_event_type(&raw);
|
||||
if event.as_deref() == Some("response.completed") {
|
||||
match websocket_response_completed_id(&raw) {
|
||||
Ok(Some(response_id)) => continuation.record_completed(response_id),
|
||||
Ok(None) => unreachable!("event type was checked"),
|
||||
Err(message) => {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::InvalidCompletedResponse,
|
||||
)
|
||||
.await;
|
||||
let _ = tx
|
||||
.send(Err(CodexWebSocketExchangeError::InvalidCompletedResponse {
|
||||
message,
|
||||
}))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(frame) = websocket_event_to_sse_frame(&raw) else {
|
||||
continue;
|
||||
};
|
||||
let terminal = event.as_deref().is_some_and(is_terminal_websocket_event);
|
||||
let has_first_output = response_body_has_first_output(frame.as_bytes());
|
||||
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -622,11 +692,30 @@ async fn forward_websocket_response_stream(
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
if has_first_output {
|
||||
saw_first_output = true;
|
||||
}
|
||||
if terminal {
|
||||
finish_stream_websocket(websocket, metadata, pool_return.take()).await;
|
||||
match event.as_deref() {
|
||||
Some("response.completed") => {
|
||||
finish_stream_websocket(websocket, metadata, continuation, pool_return.take())
|
||||
.await;
|
||||
}
|
||||
Some("response.incomplete") => {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::IncompleteResponse,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Some("response.failed" | "error") => {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::FailedResponse,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
_ => unreachable!("terminal websocket event was matched above"),
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -642,55 +731,24 @@ async fn forward_websocket_response_stream(
|
||||
.await;
|
||||
}
|
||||
|
||||
fn receive_idle_timeout(saw_response_frame: bool) -> Duration {
|
||||
if saw_response_frame {
|
||||
fn receive_idle_timeout(
|
||||
saw_upstream_activity: bool,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
) -> Duration {
|
||||
if saw_upstream_activity {
|
||||
WEBSOCKET_ACTIVE_STREAM_IDLE_TIMEOUT
|
||||
} else {
|
||||
WEBSOCKET_RECEIVE_IDLE_TIMEOUT
|
||||
initial_event_timeout
|
||||
.filter(|timeout| !timeout.is_zero())
|
||||
.unwrap_or(WEBSOCKET_RECEIVE_IDLE_TIMEOUT)
|
||||
}
|
||||
}
|
||||
|
||||
fn receive_idle_timeout_with_first_token_deadline(
|
||||
saw_first_output: bool,
|
||||
first_token_started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
) -> Result<Duration, CodexWebSocketExchangeError> {
|
||||
if saw_first_output {
|
||||
return Ok(WEBSOCKET_ACTIVE_STREAM_IDLE_TIMEOUT);
|
||||
}
|
||||
receive_timeout_before_first_output(first_token_started_at, first_token_timeout)
|
||||
}
|
||||
|
||||
fn receive_timeout_before_first_output(
|
||||
first_token_started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
) -> Result<Duration, CodexWebSocketExchangeError> {
|
||||
let Some(first_token_timeout) = first_token_timeout.filter(|timeout| !timeout.is_zero()) else {
|
||||
return Ok(WEBSOCKET_RECEIVE_IDLE_TIMEOUT);
|
||||
};
|
||||
let Some(remaining) = first_token_timeout.checked_sub(first_token_started_at.elapsed()) else {
|
||||
return Err(CodexWebSocketExchangeError::FirstTokenTimeout {
|
||||
timeout: first_token_timeout,
|
||||
});
|
||||
};
|
||||
if remaining.is_zero() {
|
||||
return Err(CodexWebSocketExchangeError::FirstTokenTimeout {
|
||||
timeout: first_token_timeout,
|
||||
});
|
||||
}
|
||||
Ok(WEBSOCKET_RECEIVE_IDLE_TIMEOUT.min(remaining))
|
||||
}
|
||||
|
||||
fn elapsed_first_token_timeout(
|
||||
first_token_started_at: Instant,
|
||||
first_token_timeout: Option<Duration>,
|
||||
) -> Option<Duration> {
|
||||
let first_token_timeout = first_token_timeout.filter(|timeout| !timeout.is_zero())?;
|
||||
(first_token_started_at.elapsed() >= first_token_timeout).then_some(first_token_timeout)
|
||||
}
|
||||
|
||||
pub(super) fn is_first_token_timeout(error: &CodexWebSocketExchangeError) -> bool {
|
||||
matches!(error, CodexWebSocketExchangeError::FirstTokenTimeout { .. })
|
||||
pub(super) fn is_initial_event_timeout(error: &CodexWebSocketExchangeError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
CodexWebSocketExchangeError::InitialEventTimeout { .. }
|
||||
)
|
||||
}
|
||||
|
||||
async fn next_websocket_message(
|
||||
@@ -708,6 +766,7 @@ async fn next_websocket_message(
|
||||
async fn finish_stream_websocket(
|
||||
mut websocket: PumpedWebSocket,
|
||||
metadata: CodexWebSocketConnectionMetadata,
|
||||
continuation: WebSocketContinuationState,
|
||||
pool_return: Option<WebSocketStreamPoolReturn>,
|
||||
) {
|
||||
let Some(pool_return) = pool_return else {
|
||||
@@ -719,6 +778,7 @@ async fn finish_stream_websocket(
|
||||
.put(PooledWebSocketConnection {
|
||||
websocket,
|
||||
metadata: reusable_websocket_metadata(metadata),
|
||||
continuation,
|
||||
created_at: pool_return.created_at,
|
||||
})
|
||||
.await;
|
||||
@@ -749,6 +809,13 @@ fn websocket_rate_limit_event_headers(raw: &str) -> Option<Vec<(String, String)>
|
||||
.map(|parsed| events::rate_limits_to_header_pairs(&parsed))
|
||||
}
|
||||
|
||||
fn update_websocket_response_metadata(metadata: &mut CodexWebSocketConnectionMetadata, raw: &str) {
|
||||
response_meta::merge_response_metadata(
|
||||
&mut metadata.response_metadata,
|
||||
websocket_metadata_headers(raw),
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn audit_header_value(name: &str, value: &str) -> String {
|
||||
if is_sensitive_opening_header(name) {
|
||||
REDACTED_HEADER_VALUE.to_string()
|
||||
|
||||
@@ -18,7 +18,7 @@ const DEFAULT_MAX_AGE: Duration = Duration::from_mins(55);
|
||||
const DEFAULT_MAINTENANCE_INTERVAL: Duration = Duration::from_secs(25);
|
||||
const DEFAULT_PING_INTERVAL: Duration = Duration::from_secs(25);
|
||||
const DEFAULT_PING_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
pub(crate) const DEFAULT_FIRST_TOKEN_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
pub(crate) const DEFAULT_INITIAL_EVENT_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// WebSocket 连接池 key。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
@@ -89,8 +89,8 @@ pub struct CodexWebSocketPoolConfig {
|
||||
pub ping_timeout: Duration,
|
||||
/// idle socket 无活动多久后视为失活。
|
||||
pub liveness_timeout: Option<Duration>,
|
||||
/// 首个真实输出到达前的绝对超时;`None` 表示禁用首 token 熔断。
|
||||
pub first_token_timeout: Option<Duration>,
|
||||
/// 首个上游事件到达前的超时;`None` 表示禁用首事件超时。
|
||||
pub initial_event_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
impl Default for CodexWebSocketPoolConfig {
|
||||
@@ -105,7 +105,7 @@ impl Default for CodexWebSocketPoolConfig {
|
||||
// idle 连接不设失活截断:靠 ping/pong 保活,只在 max_age(55 分钟)
|
||||
// 或 ping 失败时关闭,最大化跨轮复用(对齐 Codex CLI 的长连接策略)。
|
||||
liveness_timeout: None,
|
||||
first_token_timeout: Some(DEFAULT_FIRST_TOKEN_TIMEOUT),
|
||||
initial_event_timeout: Some(DEFAULT_INITIAL_EVENT_TIMEOUT),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -148,9 +148,9 @@ impl CodexWebSocketPool {
|
||||
self.config.keepalive()
|
||||
}
|
||||
|
||||
/// 首个真实输出到达前的绝对超时;`None` 表示禁用首 token 熔断。
|
||||
pub(crate) fn first_token_timeout(&self) -> Option<Duration> {
|
||||
self.config.first_token_timeout
|
||||
/// 首个上游事件到达前的超时;`None` 表示禁用首事件超时。
|
||||
pub(crate) fn initial_event_timeout(&self) -> Option<Duration> {
|
||||
self.config.initial_event_timeout
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire(&self, key: &CodexWebSocketPoolKey) -> WebSocketPoolAcquire {
|
||||
@@ -376,15 +376,33 @@ pub(crate) struct CodexWebSocketConnectionMetadata {
|
||||
pub(crate) turn_state: Option<String>,
|
||||
pub(crate) set_cookie_headers: Vec<String>,
|
||||
pub(crate) rate_limit_headers: Vec<(String, String)>,
|
||||
pub(crate) response_metadata: super::client::CodexResponseMetadata,
|
||||
pub(crate) diagnostics: super::diagnostics::CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
pub(crate) struct PooledWebSocketConnection {
|
||||
pub(crate) websocket: PumpedWebSocket,
|
||||
pub(crate) metadata: CodexWebSocketConnectionMetadata,
|
||||
pub(crate) continuation: WebSocketContinuationState,
|
||||
pub(crate) created_at: Instant,
|
||||
}
|
||||
|
||||
/// 只随具体 WebSocket 生命周期存在的续接状态。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub(crate) struct WebSocketContinuationState {
|
||||
latest_response_id: Option<String>,
|
||||
}
|
||||
|
||||
impl WebSocketContinuationState {
|
||||
pub(crate) fn latest_response_id(&self) -> Option<&str> {
|
||||
self.latest_response_id.as_deref()
|
||||
}
|
||||
|
||||
pub(crate) fn record_completed(&mut self, response_id: String) {
|
||||
self.latest_response_id = Some(response_id);
|
||||
}
|
||||
}
|
||||
|
||||
enum WebSocketPoolSlot {
|
||||
Idle(Box<PooledWebSocketConnection>),
|
||||
Busy(WebSocketPoolReservation),
|
||||
|
||||
@@ -31,6 +31,8 @@ async fn admin_accounts_lifecycle_should_update_and_delete_accounts() {
|
||||
input_tokens: None,
|
||||
function_call_ids: Vec::new(),
|
||||
variant_hash: None,
|
||||
continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal,
|
||||
replay: None,
|
||||
created_at: chrono::Utc::now(),
|
||||
},
|
||||
chrono::Duration::hours(1),
|
||||
|
||||
@@ -21,7 +21,7 @@ async fn scheduler_handle_should_shutdown_websocket_pool() {
|
||||
max_per_account: 1,
|
||||
enabled: true,
|
||||
ping_timeout: Duration::ZERO,
|
||||
first_token_timeout: None,
|
||||
initial_event_timeout: None,
|
||||
}));
|
||||
|
||||
SchedulerHandle::from_websocket_pool(pool.clone())
|
||||
|
||||
@@ -7,6 +7,7 @@ use std::{
|
||||
|
||||
use codex_proxy_rs::{
|
||||
fleet::pool::{AccountAcquireRequest, AccountPoolOptions, AccountPoolService},
|
||||
infra::identity::AccountPseudonymizer,
|
||||
models::{
|
||||
store::{ModelSnapshotStore, ModelSnapshotStoreResult},
|
||||
types::{BackendModelEntry, ModelPlanSnapshot},
|
||||
@@ -30,6 +31,7 @@ async fn model_refresh_task_should_start_and_shutdown() {
|
||||
let handle = codex_proxy_rs::bootstrap::tasks::model_refresh::ModelRefreshTask::new(
|
||||
model_service,
|
||||
account_pool,
|
||||
Arc::new(AccountPseudonymizer::new([7; 32])),
|
||||
)
|
||||
.start();
|
||||
|
||||
@@ -78,6 +80,7 @@ async fn model_refresh_task_should_sync_model_plan_allowlist_to_account_pool() {
|
||||
codex_proxy_rs::bootstrap::tasks::model_refresh::ModelRefreshTask::new(
|
||||
model_service,
|
||||
account_pool.clone(),
|
||||
Arc::new(AccountPseudonymizer::new([7; 32])),
|
||||
)
|
||||
.refresh_once()
|
||||
.await
|
||||
|
||||
@@ -120,6 +120,8 @@ fn affinity(
|
||||
input_tokens: Some(7),
|
||||
function_call_ids: vec!["call_1".to_string()],
|
||||
variant_hash: variant_hash.map(ToString::to_string),
|
||||
continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::ConnectionLocal,
|
||||
replay: None,
|
||||
created_at,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,18 +16,10 @@ use codex_proxy_rs::{
|
||||
api::AppState,
|
||||
bootstrap::config::AppConfig,
|
||||
bootstrap::services::{Services, UsageRecordOptions},
|
||||
dispatch::affinity::{
|
||||
build_conversation_identity, compute_variant_hash, prepare_variant_identity,
|
||||
SessionAffinityEntry,
|
||||
},
|
||||
fleet::{
|
||||
account::AccountStatus,
|
||||
cookies::PgCookieStore,
|
||||
pool::{AccountAcquireRequest, AccountLease},
|
||||
},
|
||||
dispatch::affinity::{ResponseReplaySnapshot, SessionAffinityEntry},
|
||||
fleet::{account::AccountStatus, cookies::PgCookieStore},
|
||||
infra::identity::hash_credential,
|
||||
telemetry::usage::store::{PgUsageRecordStore, UsageRecordFilter},
|
||||
upstream::openai::protocol::responses::CodexResponsesRequest,
|
||||
};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use serde_json::{json, Value};
|
||||
@@ -63,8 +55,6 @@ use crate::support::{
|
||||
storage::{background_task_stores, create_test_redis, init_test_db, test_database_url},
|
||||
};
|
||||
|
||||
const TEST_INSTALLATION_ID: &str = "b4f9d503-07b1-457b-a0da-87e6836b1c43";
|
||||
|
||||
fn websocket_accept_config() -> WebSocketConfig {
|
||||
let mut extensions = ExtensionsConfig::default();
|
||||
extensions.permessage_deflate = Some(DeflateConfig::default());
|
||||
@@ -210,8 +200,6 @@ const RESPONSES_FAILED_MODEL_UNSUPPORTED_SSE: &str = concat!(
|
||||
"\n"
|
||||
);
|
||||
|
||||
const WEBSOCKET_COMPLETED_WITH_REASONING_REPLAY: &str =
|
||||
include_str!("../fixtures/responses/websocket/completed_with_reasoning_replay.json");
|
||||
const WEBSOCKET_HISTORY_RATE_LIMITED: &str =
|
||||
include_str!("../fixtures/responses/websocket/history_rate_limited.json");
|
||||
const WEBSOCKET_RATE_LIMITED: &str =
|
||||
@@ -222,57 +210,42 @@ const WEBSOCKET_FIRST_ACCOUNT_LIMITED: &str =
|
||||
include_str!("../fixtures/responses/websocket/first_account_limited.json");
|
||||
const WEBSOCKET_SECOND_ACCOUNT_LIMITED: &str =
|
||||
include_str!("../fixtures/responses/websocket/second_account_limited.json");
|
||||
const WEBSOCKET_INVALID_ENCRYPTED_CONTENT: &str =
|
||||
include_str!("../fixtures/responses/websocket/invalid_encrypted_content.json");
|
||||
const WEBSOCKET_PREVIOUS_RESPONSE_NOT_FOUND: &str =
|
||||
include_str!("../fixtures/responses/websocket/previous_response_not_found.json");
|
||||
const REASONING_REPLAY_REQUEST_GOLDEN: &str =
|
||||
include_str!("../fixtures/responses/golden/reasoning_replay_request.json");
|
||||
|
||||
mod compact_routes;
|
||||
mod responses_http;
|
||||
mod responses_recovery;
|
||||
mod responses_websocket;
|
||||
mod usage_logging;
|
||||
|
||||
async fn test_app_state_with_pool_and_installation_id(
|
||||
config: &AppConfig,
|
||||
pool: PgPool,
|
||||
installation_id: String,
|
||||
) -> AppState {
|
||||
test_app_state_with_pool_installation_id_and_usage_record_options(
|
||||
async fn test_app_state_with_pool(config: &AppConfig, pool: PgPool) -> AppState {
|
||||
test_app_state_with_pool_and_usage_record_options(
|
||||
config.clone(),
|
||||
pool,
|
||||
installation_id,
|
||||
UsageRecordOptions::from_config(config),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn test_app_state_with_pool_installation_id_and_usage_record_options(
|
||||
async fn test_app_state_with_pool_and_usage_record_options(
|
||||
config: AppConfig,
|
||||
pool: PgPool,
|
||||
installation_id: String,
|
||||
usage_record_options: UsageRecordOptions,
|
||||
) -> AppState {
|
||||
let redis = create_test_redis("chat-upstream").await;
|
||||
test_app_state_with_storage(config, pool, redis, installation_id, usage_record_options).await
|
||||
test_app_state_with_storage(config, pool, redis, usage_record_options).await
|
||||
}
|
||||
|
||||
async fn test_app_state_with_storage(
|
||||
config: AppConfig,
|
||||
pool: PgPool,
|
||||
redis: codex_proxy_rs::infra::redis::RedisConnection,
|
||||
installation_id: String,
|
||||
usage_record_options: UsageRecordOptions,
|
||||
) -> AppState {
|
||||
insert_model_snapshot(&redis).await;
|
||||
let stores = background_task_stores(pool, redis);
|
||||
let services = Services::try_with_installation_id_and_usage_record_options(
|
||||
let services = Services::try_with_usage_record_options(
|
||||
&config,
|
||||
stores,
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
Some(installation_id),
|
||||
usage_record_options,
|
||||
)
|
||||
.expect("failed to build runtime services with configured TLS transport");
|
||||
@@ -289,27 +262,11 @@ async fn test_app_with_account(
|
||||
axum::Router,
|
||||
String,
|
||||
crate::support::storage::TestDatabaseGuard,
|
||||
) {
|
||||
test_app_with_account_and_installation_id(base_url, TEST_INSTALLATION_ID.to_string()).await
|
||||
}
|
||||
|
||||
async fn test_app_with_account_and_installation_id(
|
||||
base_url: String,
|
||||
installation_id: String,
|
||||
) -> (
|
||||
axum::Router,
|
||||
String,
|
||||
crate::support::storage::TestDatabaseGuard,
|
||||
) {
|
||||
let (pool, dir) = init_test_db("openai-chat-upstream").await;
|
||||
let api_key = insert_client_api_key(&pool).await;
|
||||
insert_account(&pool).await;
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&test_config(test_database_url(), base_url),
|
||||
pool,
|
||||
installation_id,
|
||||
)
|
||||
.await;
|
||||
let state = test_app_state_with_pool(&test_config(test_database_url(), base_url), pool).await;
|
||||
state
|
||||
.services
|
||||
.account_pool
|
||||
@@ -352,12 +309,8 @@ async fn test_app_without_accounts(
|
||||
) {
|
||||
let (pool, dir) = init_test_db("openai-responses-no-accounts").await;
|
||||
let api_key = insert_client_api_key(&pool).await;
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&test_config(test_database_url(), base_url),
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
.await;
|
||||
let state =
|
||||
test_app_state_with_pool(&test_config(test_database_url(), base_url), pool.clone()).await;
|
||||
state
|
||||
.services
|
||||
.account_pool
|
||||
@@ -392,12 +345,8 @@ async fn test_app_with_account_pool_and_disabled_telemetry(
|
||||
let (pool, dir) = init_test_db("openai-record-disabled-logging").await;
|
||||
let api_key = insert_client_api_key(&pool).await;
|
||||
insert_account(&pool).await;
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&test_config(test_database_url(), base_url),
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
.await;
|
||||
let state =
|
||||
test_app_state_with_pool(&test_config(test_database_url(), base_url), pool.clone()).await;
|
||||
state
|
||||
.services
|
||||
.account_pool
|
||||
@@ -424,10 +373,9 @@ async fn test_app_with_account_pool_and_telemetry_capture_body(
|
||||
enabled: true,
|
||||
capture_body: true,
|
||||
};
|
||||
let state = test_app_state_with_pool_installation_id_and_usage_record_options(
|
||||
let state = test_app_state_with_pool_and_usage_record_options(
|
||||
config,
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
usage_record_options,
|
||||
)
|
||||
.await;
|
||||
@@ -467,12 +415,7 @@ async fn test_app_with_account_pool_config_and_affinity(
|
||||
insert_account(&pool).await;
|
||||
let mut config = test_config(test_database_url(), base_url);
|
||||
configure(&mut config);
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&config,
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
.await;
|
||||
let state = test_app_state_with_pool(&config, pool.clone()).await;
|
||||
if let Some(response_id) = affinity_response_id {
|
||||
insert_session_affinity(&state.services.session_affinity, response_id, "acct_chat").await;
|
||||
}
|
||||
@@ -497,12 +440,7 @@ async fn test_app_with_two_accounts_and_affinity(
|
||||
insert_named_account(&pool, "acct_a", "access-default", "chatgpt-default").await;
|
||||
insert_named_account(&pool, "acct_z", "access-affinity", "chatgpt-affinity").await;
|
||||
let now = Utc::now();
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&test_config(test_database_url(), base_url),
|
||||
pool,
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
.await;
|
||||
let state = test_app_state_with_pool(&test_config(test_database_url(), base_url), pool).await;
|
||||
state
|
||||
.services
|
||||
.session_affinity
|
||||
@@ -516,6 +454,13 @@ async fn test_app_with_two_accounts_and_affinity(
|
||||
input_tokens: None,
|
||||
function_call_ids: Vec::new(),
|
||||
variant_hash: None,
|
||||
continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted,
|
||||
replay: Some(ResponseReplaySnapshot {
|
||||
full_input: vec![json!({
|
||||
"role": "user",
|
||||
"content": "Original managed history"
|
||||
})],
|
||||
}),
|
||||
created_at: now,
|
||||
},
|
||||
)
|
||||
@@ -560,12 +505,8 @@ async fn test_app_with_two_accounts_and_affinity_status(
|
||||
.await
|
||||
.unwrap();
|
||||
let now = Utc::now();
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&test_config(test_database_url(), base_url),
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
.await;
|
||||
let state =
|
||||
test_app_state_with_pool(&test_config(test_database_url(), base_url), pool.clone()).await;
|
||||
state
|
||||
.services
|
||||
.session_affinity
|
||||
@@ -579,6 +520,13 @@ async fn test_app_with_two_accounts_and_affinity_status(
|
||||
input_tokens: None,
|
||||
function_call_ids: Vec::new(),
|
||||
variant_hash: None,
|
||||
continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted,
|
||||
replay: Some(ResponseReplaySnapshot {
|
||||
full_input: vec![json!({
|
||||
"role": "user",
|
||||
"content": "Original affinity history"
|
||||
})],
|
||||
}),
|
||||
created_at: now,
|
||||
},
|
||||
)
|
||||
@@ -661,12 +609,7 @@ async fn test_app_with_two_accounts_and_state_config(
|
||||
.await;
|
||||
let mut config = test_config(test_database_url(), base_url);
|
||||
configure(&mut config);
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&config,
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
.await;
|
||||
let state = test_app_state_with_pool(&config, pool.clone()).await;
|
||||
state
|
||||
.services
|
||||
.account_pool
|
||||
@@ -682,6 +625,46 @@ async fn test_app_with_two_accounts_and_state_config(
|
||||
)
|
||||
}
|
||||
|
||||
async fn test_app_with_ranked_accounts(
|
||||
base_url: String,
|
||||
account_count: usize,
|
||||
) -> (
|
||||
axum::Router,
|
||||
String,
|
||||
crate::support::storage::TestDatabaseGuard,
|
||||
) {
|
||||
let (pool, dir) = init_test_db("openai-responses-full-candidate-ledger").await;
|
||||
let api_key = insert_client_api_key(&pool).await;
|
||||
for index in 0..account_count {
|
||||
let account_id = format!("acct-{index}");
|
||||
insert_named_account(
|
||||
&pool,
|
||||
&account_id,
|
||||
&format!("access-{index}"),
|
||||
&format!("chatgpt-{index}"),
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"insert into account_usage (account_id, request_count, window_request_count) values ($1, $2, $2)",
|
||||
)
|
||||
.bind(&account_id)
|
||||
.bind(i64::try_from(index).unwrap())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let mut config = test_config(test_database_url(), base_url);
|
||||
config.auth.rotation_strategy = "quota_reset_priority".to_string();
|
||||
let state = test_app_state_with_pool(&config, pool).await;
|
||||
state
|
||||
.services
|
||||
.account_pool
|
||||
.restore_from_store()
|
||||
.await
|
||||
.expect("account pool should restore");
|
||||
(router::router().with_state(state), api_key, dir)
|
||||
}
|
||||
|
||||
async fn seed_openai_admin_key(pool: &PgPool, key: &str) {
|
||||
sqlx::query(
|
||||
"insert into runtime_settings (
|
||||
@@ -784,6 +767,8 @@ async fn insert_session_affinity(
|
||||
input_tokens: None,
|
||||
function_call_ids: Vec::new(),
|
||||
variant_hash: None,
|
||||
continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted,
|
||||
replay: None,
|
||||
created_at: Utc::now(),
|
||||
},
|
||||
)
|
||||
@@ -1145,66 +1130,6 @@ async fn accept_websocket_response_with_message(
|
||||
payload
|
||||
}
|
||||
|
||||
async fn accept_followup_websocket_response(
|
||||
listener: &TcpListener,
|
||||
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
|
||||
response_message: String,
|
||||
) -> Value {
|
||||
tokio::select! {
|
||||
message = websocket.next() => {
|
||||
match message {
|
||||
Some(Ok(message)) if message.is_text() => {
|
||||
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
|
||||
.expect("websocket payload should be json");
|
||||
websocket
|
||||
.send(Message::Text(response_message.into()))
|
||||
.await
|
||||
.unwrap();
|
||||
payload
|
||||
}
|
||||
_ => accept_websocket_response_with_message(listener, response_message).await,
|
||||
}
|
||||
}
|
||||
accepted = listener.accept() => {
|
||||
let (stream, _) = accepted.unwrap();
|
||||
let mut followup = accept_async(stream).await.unwrap();
|
||||
let payload =
|
||||
send_websocket_response_and_capture_payload(&mut followup, response_message).await;
|
||||
followup.close(None).await.unwrap();
|
||||
payload
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn accept_followup_websocket_response_sequence(
|
||||
listener: &TcpListener,
|
||||
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
|
||||
response_messages: Vec<String>,
|
||||
) -> Value {
|
||||
let followup = tokio::select! {
|
||||
message = websocket.next() => {
|
||||
match message {
|
||||
Some(Ok(message)) if message.is_text() => {
|
||||
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
|
||||
.expect("websocket payload should be json");
|
||||
send_websocket_messages(websocket, &response_messages).await;
|
||||
return payload;
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
accepted = listener.accept() => Some(accepted.unwrap().0),
|
||||
};
|
||||
|
||||
let stream = followup.expect("follow-up websocket connection should be available");
|
||||
let mut followup = accept_async(stream).await.unwrap();
|
||||
let message = followup.next().await.unwrap().unwrap();
|
||||
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
|
||||
.expect("websocket payload should be json");
|
||||
send_websocket_messages(&mut followup, &response_messages).await;
|
||||
payload
|
||||
}
|
||||
|
||||
async fn send_websocket_messages(
|
||||
websocket: &mut tokio_tungstenite::WebSocketStream<TcpStream>,
|
||||
messages: &[String],
|
||||
@@ -1270,13 +1195,12 @@ async fn read_http_upgrade_request(stream: &mut TcpStream) -> String {
|
||||
String::from_utf8(request).unwrap()
|
||||
}
|
||||
|
||||
fn assert_response_failed_stream(
|
||||
fn assert_openai_error_body(
|
||||
body: &str,
|
||||
expected_error_type: &str,
|
||||
expected_code: &str,
|
||||
expected_fragments: &[&str],
|
||||
) {
|
||||
assert!(body.contains("event: response.failed"));
|
||||
assert!(
|
||||
body.contains(&format!("\"type\":\"{expected_error_type}\""))
|
||||
|| body.contains(&format!("\"type\": \"{expected_error_type}\"")),
|
||||
@@ -1380,30 +1304,6 @@ fn websocket_completed_response(
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn websocket_completed_function_call_response(response_id: &str, call_id: &str) -> String {
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "function_call",
|
||||
"id": format!("fc_{call_id}"),
|
||||
"call_id": call_id,
|
||||
"name": "lookup",
|
||||
"arguments": "{}"
|
||||
}],
|
||||
"usage": {
|
||||
"input_tokens": 6,
|
||||
"output_tokens": 2,
|
||||
"total_tokens": 8
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn responses_previous_stream_request(api_key: &str, content: &str) -> Request<Body> {
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
@@ -1516,16 +1416,29 @@ async fn latest_response_usage_record(pool: &PgPool) -> ResponseUsageRecordSnaps
|
||||
}
|
||||
|
||||
async fn latest_response_ops_error_log(pool: &PgPool) -> ResponseUsageRecordSnapshot {
|
||||
latest_response_ops_error_log_with_message(pool, None).await
|
||||
}
|
||||
|
||||
async fn latest_response_upstream_ops_error_log(pool: &PgPool) -> ResponseUsageRecordSnapshot {
|
||||
latest_response_ops_error_log_with_message(pool, Some("v1 responses upstream request failed"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn latest_response_ops_error_log_with_message(
|
||||
pool: &PgPool,
|
||||
message: Option<&str>,
|
||||
) -> ResponseUsageRecordSnapshot {
|
||||
let row = sqlx::query_as::<_, OpsErrorLogSnapshot>(
|
||||
"select request_id, account_id, route, model, transport, status_code,
|
||||
client_status_code, upstream_status_code, failure_class, attempt_index, response_id,
|
||||
metadata_json::text as metadata_json
|
||||
from ops_error_logs
|
||||
where kind = $1
|
||||
where kind = $1 and ($2::text is null or message = $2)
|
||||
order by created_at desc, id desc
|
||||
limit 1",
|
||||
)
|
||||
.bind("v1.response")
|
||||
.bind(message)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -203,10 +203,12 @@ async fn responses_compact_should_post_json_to_codex_compact_upstream() {
|
||||
json!([{"type": "function", "name": "lookup"}])
|
||||
);
|
||||
assert_eq!(upstream_body["text"]["format"]["type"], "json_schema");
|
||||
// compact 只剥离 stream(端点不支持流式);store/prompt_cache_key 等业务字段原样透传。
|
||||
// compact 只剥离 stream;prompt_cache_key 按账号作用域伪名化。
|
||||
assert!(upstream_body.get("stream").is_none());
|
||||
assert_eq!(upstream_body["store"], true);
|
||||
assert_eq!(upstream_body["prompt_cache_key"], "session-seed");
|
||||
assert!(upstream_body["prompt_cache_key"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.starts_with("wi_") && value != "session-seed"));
|
||||
assert_eq!(upstream_body["input"].as_array().unwrap().len(), 4);
|
||||
assert_eq!(upstream_body["input"][1]["ignored"], "drop");
|
||||
assert_eq!(
|
||||
@@ -333,8 +335,11 @@ async fn responses_compact_should_preserve_upstream_client_error_status() {
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert_eq!(body["error"]["code"], "codex_client_error");
|
||||
assert_eq!(body["error"]["code"], "invalid_encrypted_content");
|
||||
assert_eq!(
|
||||
body["error"]["message"],
|
||||
"The encrypted content could not be verified."
|
||||
);
|
||||
let event = latest_response_ops_error_log(&pool).await;
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
assert_eq!(
|
||||
@@ -354,7 +359,7 @@ async fn responses_compact_should_preserve_upstream_client_error_status() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_compact_should_map_upstream_internal_error_to_bad_gateway() {
|
||||
async fn responses_compact_should_report_upstream_unavailable_after_all_5xx_retries() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses/compact"))
|
||||
@@ -387,7 +392,7 @@ async fn responses_compact_should_map_upstream_internal_error_to_bad_gateway() {
|
||||
let status = response.status();
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(body["error"]["type"], "server_error");
|
||||
assert_eq!(body["error"]["code"], "upstream_error");
|
||||
}
|
||||
|
||||
@@ -60,6 +60,11 @@ async fn responses_should_honor_explicit_http_sse_transport() {
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.insert_header("openai-model", "gpt-5.5-effective")
|
||||
.insert_header("x-models-etag", "models-etag-2")
|
||||
.insert_header("x-reasoning-included", "true")
|
||||
.insert_header("set-cookie", "account-secret=value")
|
||||
.insert_header("x-codex-primary-used-percent", "12")
|
||||
.set_body_string(RESPONSES_STREAM_USAGE_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
@@ -87,6 +92,26 @@ async fn responses_should_honor_explicit_http_sse_transport() {
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let effective_model = response
|
||||
.headers()
|
||||
.get("openai-model")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(ToString::to_string);
|
||||
let models_etag = response
|
||||
.headers()
|
||||
.get("x-models-etag")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(ToString::to_string);
|
||||
let reasoning_included = response
|
||||
.headers()
|
||||
.get("x-reasoning-included")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(ToString::to_string);
|
||||
assert!(response.headers().get("set-cookie").is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("x-codex-primary-used-percent")
|
||||
.is_none());
|
||||
let body = response_text(response).await;
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let upstream_body: Value = serde_json::from_slice(&requests[0].body).unwrap();
|
||||
@@ -96,6 +121,9 @@ async fn responses_should_honor_explicit_http_sse_transport() {
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert_eq!(requests[0].method.as_str(), "POST");
|
||||
assert_eq!(upstream_body["model"], "gpt-5.5");
|
||||
assert_eq!(effective_model.as_deref(), Some("gpt-5.5-effective"));
|
||||
assert_eq!(models_etag.as_deref(), Some("models-etag-2"));
|
||||
assert_eq!(reasoning_included.as_deref(), Some("true"));
|
||||
assert!(upstream_body.get("use_websocket").is_none());
|
||||
}
|
||||
|
||||
@@ -254,13 +282,14 @@ async fn responses_should_use_websocket_upstream_by_default_while_serving_sse()
|
||||
// 透明代理:`generate` 原样透传上游,不再被剥离。
|
||||
assert_eq!(captured.payload["generate"], false);
|
||||
assert!(captured.payload.get("previous_response_id").is_none());
|
||||
assert!(captured.payload["prompt_cache_key"]
|
||||
assert!(captured.payload.get("prompt_cache_key").is_none());
|
||||
let metadata = captured.payload["client_metadata"]
|
||||
.as_object()
|
||||
.expect("official websocket timing metadata should be present");
|
||||
assert_eq!(metadata.len(), 1);
|
||||
assert!(metadata["x-codex-ws-stream-request-start-ms"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.starts_with("cp_")));
|
||||
assert!(captured.payload["client_metadata"]["x-codex-installation-id"].is_string());
|
||||
assert!(captured.payload["client_metadata"]["x-codex-window-id"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.starts_with("cp_") && value.ends_with(":0")));
|
||||
.is_some_and(|value| value.parse::<u128>().is_ok()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -299,7 +328,7 @@ async fn responses_non_stream_should_record_websocket_transport_metadata() {
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
assert_eq!(metadata["stream"], false);
|
||||
assert_eq!(event.transport.as_deref(), Some("websocket"));
|
||||
assert_eq!(metadata["websocketPool"]["kind"], "new");
|
||||
assert!(metadata.get("websocketPool").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -388,8 +417,7 @@ async fn responses_should_forward_text_schema_verbatim_to_upstream() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_pass_through_tuple_schema_output_verbatim() {
|
||||
// 透明代理:不再重写请求 schema,因此也不再对响应做 tuple 回转换——
|
||||
// 上游返回的 object 形态 `{"point":{"0":1,"1":2}}` 原样透传给客户端。
|
||||
// 透明代理:completed 事件中 output_text 和 output 的原始值都不用 delta 覆盖。
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
@@ -420,7 +448,7 @@ async fn responses_should_pass_through_tuple_schema_output_verbatim() {
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body["id"], "resp_tuple");
|
||||
assert_eq!(body["output_text"], "{\"point\":{\"0\":1,\"1\":2}}");
|
||||
assert_eq!(body["output_text"], "{\"point\":{\"0\":0,\"1\":0}}");
|
||||
assert_eq!(
|
||||
body["output"][0]["content"][0]["text"],
|
||||
"{\"point\":{\"0\":1,\"1\":2}}"
|
||||
@@ -531,17 +559,6 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop
|
||||
.find(|request| request.url.path() == "/codex/responses")
|
||||
.expect("responses upstream request should be sent");
|
||||
let upstream_body: Value = serde_json::from_slice(&upstream.body).unwrap();
|
||||
// 透明代理:window-id 不再从 client_metadata 回退读取,客户端未通过 body
|
||||
// `codexWindowId` 或 `x-codex-window-id` 请求头提供时,window-id 由会话派生(`:0` 后缀)。
|
||||
let identity = build_conversation_identity(Some("pcache"), None, "acct_chat");
|
||||
let conversation_id = identity
|
||||
.conversation_id
|
||||
.as_deref()
|
||||
.expect("conversation identity should be scoped");
|
||||
let window_id = identity
|
||||
.window_id
|
||||
.as_deref()
|
||||
.expect("window identity should be scoped");
|
||||
let upstream_header = |name: &str| {
|
||||
upstream
|
||||
.headers
|
||||
@@ -554,38 +571,36 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop
|
||||
assert_eq!(upstream_body["service_tier"], "priority");
|
||||
assert_eq!(upstream_body["reasoning"], json!({"effort": "high"}));
|
||||
assert!(upstream_body.get("include").is_none());
|
||||
assert_eq!(upstream_body["prompt_cache_key"], conversation_id);
|
||||
let prompt_cache_key = upstream_body["prompt_cache_key"]
|
||||
.as_str()
|
||||
.expect("prompt cache key should be account scoped");
|
||||
assert_ne!(prompt_cache_key, "pcache");
|
||||
assert_eq!(upstream_body["client_metadata"]["safe"], "yes");
|
||||
assert_eq!(upstream_body["client_metadata"]["drop"], 42);
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["x-openai-subagent"],
|
||||
"review"
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["x-codex-installation-id"],
|
||||
TEST_INSTALLATION_ID
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["session_id"],
|
||||
conversation_id
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["thread_id"],
|
||||
conversation_id
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["x-codex-window-id"],
|
||||
window_id
|
||||
);
|
||||
let installation_id = upstream_body["client_metadata"]["x-codex-installation-id"]
|
||||
.as_str()
|
||||
.expect("installation id should be account scoped");
|
||||
assert!(uuid::Uuid::parse_str(installation_id).is_ok());
|
||||
assert!(upstream_body["client_metadata"].get("session_id").is_none());
|
||||
assert!(upstream_body["client_metadata"].get("thread_id").is_none());
|
||||
let window_id = upstream_body["client_metadata"]["x-codex-window-id"]
|
||||
.as_str()
|
||||
.expect("window id should be account scoped");
|
||||
assert_ne!(window_id, "window-metadata");
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["x-codex-turn-metadata"],
|
||||
"meta-metadata"
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["x-codex-parent-thread-id"],
|
||||
"parent-metadata"
|
||||
);
|
||||
assert_eq!(upstream_header("session-id"), Some(conversation_id));
|
||||
assert_eq!(upstream_header("thread-id"), Some(conversation_id));
|
||||
let parent_thread_id = upstream_body["client_metadata"]["x-codex-parent-thread-id"]
|
||||
.as_str()
|
||||
.expect("parent thread id should be account scoped");
|
||||
assert_ne!(parent_thread_id, "parent-metadata");
|
||||
assert_eq!(upstream_header("session-id"), None);
|
||||
assert_eq!(upstream_header("thread-id"), None);
|
||||
assert_eq!(upstream_header("session_id"), None);
|
||||
assert_eq!(upstream_header("x-codex-window-id"), Some(window_id));
|
||||
assert_eq!(upstream_header("x-codex-turn-state"), Some("turn-body"));
|
||||
@@ -598,7 +613,10 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop
|
||||
Some("true")
|
||||
);
|
||||
assert_eq!(upstream_header("version"), Some("2026-06-12"));
|
||||
assert_eq!(upstream_header("x-codex-parent-thread-id"), None);
|
||||
assert_eq!(
|
||||
upstream_header("x-codex-parent-thread-id"),
|
||||
Some(parent_thread_id)
|
||||
);
|
||||
assert_eq!(upstream_header("x-openai-subagent"), Some("review"));
|
||||
// 透明代理:上下文透传字段提取到代理控制状态用于加请求头,body 中同时原样保留。
|
||||
for context_field in [
|
||||
@@ -766,7 +784,7 @@ async fn responses_should_preserve_input_items_before_upstream() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_reconstruct_non_stream_output_text_from_sse_deltas() {
|
||||
async fn responses_should_not_synthesize_non_stream_output_from_sse_deltas() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
@@ -803,13 +821,12 @@ async fn responses_should_reconstruct_non_stream_output_text_from_sse_deltas() {
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(body["id"], "resp_text");
|
||||
assert_eq!(body["output"][0]["role"], "assistant");
|
||||
assert_eq!(body["output"][0]["content"][0]["text"], "hello from deltas");
|
||||
assert_eq!(body["output_text"], "hello from deltas");
|
||||
assert_eq!(body["output"], json!([]));
|
||||
assert!(body.get("output_text").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_use_done_output_items_when_completed_output_is_empty() {
|
||||
async fn responses_should_not_inject_done_items_when_completed_output_is_empty() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
@@ -846,8 +863,8 @@ async fn responses_should_use_done_output_items_when_completed_output_is_empty()
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(body["id"], "resp_item");
|
||||
assert_eq!(body["output"][0]["content"][0]["text"], "from done item");
|
||||
assert_eq!(body["output_text"], "from done item");
|
||||
assert_eq!(body["output"], json!([]));
|
||||
assert!(body.get("output_text").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1201,7 +1218,7 @@ async fn responses_should_prefer_session_affinity_account_for_previous_response(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_preserve_explicit_history_when_banned_affinity_switches_account() {
|
||||
async fn responses_should_replay_full_history_when_banned_affinity_switches_account() {
|
||||
let (base_url, upstream) =
|
||||
spawn_single_websocket_completed_upstream("resp_after_banned_affinity").await;
|
||||
let (app, state, api_key, _pool, _dir) =
|
||||
@@ -1242,19 +1259,21 @@ async fn responses_should_preserve_explicit_history_when_banned_affinity_switche
|
||||
captured_header(&captured.headers, "chatgpt-account-id"),
|
||||
Some("chatgpt-primary")
|
||||
);
|
||||
assert!(captured_header(&captured.headers, "x-codex-turn-state").is_none());
|
||||
assert!(captured.payload.get("previous_response_id").is_none());
|
||||
assert_eq!(
|
||||
captured_header(&captured.headers, "x-codex-turn-state"),
|
||||
Some("turn_affinity_risk")
|
||||
captured.payload["input"][0]["content"],
|
||||
"Original affinity history"
|
||||
);
|
||||
assert_eq!(
|
||||
captured.payload["previous_response_id"],
|
||||
"resp_affinity_risk"
|
||||
captured.payload["input"][1]["content"],
|
||||
"Continue after ban"
|
||||
);
|
||||
assert!(affinity.is_none());
|
||||
assert!(affinity.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_preserve_explicit_history_when_quota_affinity_switches_account() {
|
||||
async fn responses_should_replay_full_history_when_quota_affinity_switches_account() {
|
||||
let (base_url, upstream) =
|
||||
spawn_single_websocket_completed_upstream("resp_after_quota_affinity").await;
|
||||
let (app, state, api_key, _pool, _dir) =
|
||||
@@ -1291,13 +1310,15 @@ async fn responses_should_preserve_explicit_history_when_quota_affinity_switches
|
||||
captured_header(&captured.headers, "authorization"),
|
||||
Some("Bearer access-primary")
|
||||
);
|
||||
assert!(captured_header(&captured.headers, "x-codex-turn-state").is_none());
|
||||
assert!(captured.payload.get("previous_response_id").is_none());
|
||||
assert_eq!(
|
||||
captured_header(&captured.headers, "x-codex-turn-state"),
|
||||
Some("turn_affinity_risk")
|
||||
captured.payload["input"][0]["content"],
|
||||
"Original affinity history"
|
||||
);
|
||||
assert_eq!(
|
||||
captured.payload["previous_response_id"],
|
||||
"resp_affinity_risk"
|
||||
captured.payload["input"][1]["content"],
|
||||
"Continue after quota"
|
||||
);
|
||||
assert!(affinity.is_some());
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ async fn responses_should_dispatch_to_codex_and_return_completed_response() {
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(body["id"], "resp_response_1");
|
||||
assert_eq!(body["output_text"], "response hello");
|
||||
assert!(body.get("output_text").is_none());
|
||||
assert_eq!(body["usage"]["input_tokens"], 5);
|
||||
assert_eq!(body["usage"]["output_tokens"], 2);
|
||||
}
|
||||
@@ -130,8 +130,9 @@ async fn responses_should_preserve_upstream_client_error_status() {
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(body["error"]["type"], "invalid_request_error");
|
||||
assert_eq!(body["error"]["code"], "codex_client_error");
|
||||
assert_eq!(body["error"]["code"], "invalid_input");
|
||||
assert_eq!(body["error"]["message"], "A required parameter is invalid.");
|
||||
assert!(body["error"].get("type").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1074,9 +1075,8 @@ async fn responses_stream_should_return_model_unsupported_error_when_fallback_is
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(content_type.starts_with("text/event-stream"));
|
||||
assert!(body.contains("event: response.failed"));
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert!(content_type.starts_with("application/json"));
|
||||
assert!(body.contains("\"code\":\"model_not_found\""));
|
||||
assert!(body.contains("All accounts exhausted (1 model-unsupported)"));
|
||||
assert!(body.contains("model_not_available"));
|
||||
@@ -1129,6 +1129,156 @@ async fn responses_explicit_previous_response_should_fail_without_transparent_re
|
||||
assert!(!captured.retry_attempted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_external_previous_response_not_found_should_return_exact_upstream_400_once() {
|
||||
let server = MockServer::start().await;
|
||||
let upstream_error = json!({
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "previous_response_not_found",
|
||||
"message": "Previous response with id 'resp_external' not found.",
|
||||
"param": "previous_response_id"
|
||||
},
|
||||
"status": 400
|
||||
});
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(ResponseTemplate::new(400).set_body_json(upstream_error.clone()))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (app, api_key, _pool, _dir) = test_app_with_two_accounts(server.uri()).await;
|
||||
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/responses")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"previous_response_id": "resp_external",
|
||||
"input": [{"role": "user", "content": "Continue"}],
|
||||
"stream": false,
|
||||
"use_websocket": false
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(body, upstream_error);
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_managed_history_should_rebuild_full_replay_for_next_configured_candidate() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-primary"))
|
||||
.respond_with(ResponseTemplate::new(401).set_body_json(json!({
|
||||
"error": {"code": "token_revoked", "message": "token revoked"}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-secondary"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(RESPONSES_SUCCESS_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (app, state, api_key, _pool, _dir) =
|
||||
test_app_with_two_accounts_and_state(server.uri()).await;
|
||||
state
|
||||
.services
|
||||
.session_affinity
|
||||
.record(
|
||||
"resp_managed".to_string(),
|
||||
SessionAffinityEntry {
|
||||
account_id: "acct_primary".to_string(),
|
||||
conversation_id: "conversation-managed".to_string(),
|
||||
turn_state: Some("account-bound-turn-state".to_string()),
|
||||
instructions_hash: None,
|
||||
input_tokens: Some(11),
|
||||
function_call_ids: Vec::new(),
|
||||
variant_hash: None,
|
||||
continuation_scope: codex_proxy_rs::upstream::openai::protocol::responses::PreviousResponseScope::Persisted,
|
||||
replay: Some(ResponseReplaySnapshot {
|
||||
full_input: vec![
|
||||
json!({"role": "user", "content": "Earlier question"}),
|
||||
json!({
|
||||
"type": "reasoning",
|
||||
"id": "reasoning-old",
|
||||
"encrypted_content": "account-bound-secret",
|
||||
"summary": []
|
||||
}),
|
||||
],
|
||||
}),
|
||||
created_at: Utc::now(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/responses")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"previous_response_id": "resp_managed",
|
||||
"turnState": "client-turn-state",
|
||||
"input": [{"role": "user", "content": "Current question"}],
|
||||
"stream": false,
|
||||
"use_websocket": false
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let secondary = requests
|
||||
.iter()
|
||||
.find(|request| {
|
||||
request
|
||||
.headers
|
||||
.get("authorization")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
== Some("Bearer access-secondary")
|
||||
})
|
||||
.expect("secondary request");
|
||||
let body: Value = serde_json::from_slice(&secondary.body).unwrap();
|
||||
assert!(body.get("previous_response_id").is_none());
|
||||
assert_eq!(body["input"][0]["content"], "Earlier question");
|
||||
assert_eq!(body["input"][1]["id"], "reasoning-old");
|
||||
assert!(body["input"][1].get("encrypted_content").is_none());
|
||||
assert_eq!(body["input"][2]["content"], "Current question");
|
||||
assert!(secondary.headers.get("x-codex-turn-state").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_explicit_unanswered_function_call_should_not_retry_without_full_context() {
|
||||
let (base_url, upstream) =
|
||||
@@ -1166,7 +1316,8 @@ async fn responses_explicit_unanswered_function_call_should_not_retry_without_fu
|
||||
let captured = upstream.await.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(body["error"]["code"], "previous_response_unavailable");
|
||||
assert_eq!(body["error"]["code"], "codex_client_error");
|
||||
assert_eq!(body["error"]["message"], "Upstream Codex response failed");
|
||||
assert_eq!(captured.payload["previous_response_id"], "resp_stale");
|
||||
assert!(!captured.retry_attempted);
|
||||
}
|
||||
@@ -1270,7 +1421,7 @@ async fn responses_stream_structural_events_should_not_commit_before_history_fai
|
||||
let body = response_text(response).await;
|
||||
let captured = upstream.await.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert!(body.contains("previous_response_unavailable"));
|
||||
assert!(!body.contains("response.created"));
|
||||
assert_eq!(captured.payload["previous_response_id"], "resp_stale");
|
||||
@@ -1555,9 +1706,8 @@ async fn responses_stream_should_return_auth_error_when_401_fallback_is_exhauste
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(content_type.starts_with("text/event-stream"));
|
||||
assert!(body.contains("event: response.failed"));
|
||||
assert_eq!(status, StatusCode::UNAUTHORIZED);
|
||||
assert!(content_type.starts_with("application/json"));
|
||||
assert!(body.contains("\"code\":\"invalid_api_key\""));
|
||||
assert!(body.contains("All accounts exhausted (1 expired)"));
|
||||
assert!(body.contains("token_revoked"));
|
||||
@@ -1853,6 +2003,62 @@ async fn responses_should_retry_same_account_after_5xx_before_fallback() {
|
||||
assert_eq!(secondary_requests, 0, "requests: {authorizations:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_traverse_all_configured_strategy_candidates_without_fixed_cap() {
|
||||
let server = MockServer::start().await;
|
||||
for index in 0..6 {
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", format!("Bearer access-{index}")))
|
||||
.respond_with(ResponseTemplate::new(401).set_body_json(json!({
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "token_invalidated",
|
||||
"message": "token is no longer valid"
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
}
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-6"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(RESPONSES_SUCCESS_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let (app, api_key, _dir) = test_app_with_ranked_accounts(server.uri(), 7).await;
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/responses")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"input": [{"role": "user", "content": "Say hello"}],
|
||||
"stream": false,
|
||||
"use_websocket": false
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 7);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_stream_should_retry_same_account_after_5xx_before_fallback() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -2240,9 +2446,8 @@ async fn responses_stream_should_return_rate_limit_error_when_429_fallback_is_ex
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(content_type.starts_with("text/event-stream"));
|
||||
assert!(body.contains("event: response.failed"));
|
||||
assert_eq!(status, StatusCode::TOO_MANY_REQUESTS);
|
||||
assert!(content_type.starts_with("application/json"));
|
||||
assert!(body.contains("rate_limit_exceeded"));
|
||||
assert!(body.contains("All accounts exhausted (1 rate-limited)"));
|
||||
assert!(body.contains("rate limited"));
|
||||
@@ -2316,9 +2521,8 @@ async fn responses_stream_should_return_quota_error_when_402_fallback_is_exhaust
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(content_type.starts_with("text/event-stream"));
|
||||
assert!(body.contains("event: response.failed"));
|
||||
assert_eq!(status, StatusCode::TOO_MANY_REQUESTS);
|
||||
assert!(content_type.starts_with("application/json"));
|
||||
assert!(body.contains("\"code\":\"insufficient_quota\""));
|
||||
assert!(body.contains("All accounts exhausted (1 quota-exhausted)"));
|
||||
assert!(body.contains("quota reached"));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -401,7 +401,7 @@ async fn responses_should_passively_cache_rate_limit_headers() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_record_empty_response_attempts() {
|
||||
async fn responses_should_record_empty_response_attempt() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
@@ -410,7 +410,7 @@ async fn responses_should_record_empty_response_attempts() {
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(RESPONSES_EMPTY_COMPLETED_SSE),
|
||||
)
|
||||
.expect(3)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
@@ -436,7 +436,7 @@ async fn responses_should_record_empty_response_attempts() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
let usage: (i64, i64, i64, i64) = sqlx::query_as(
|
||||
"select request_count, empty_response_count, input_tokens, output_tokens from account_usage where account_id = $1",
|
||||
)
|
||||
@@ -444,7 +444,7 @@ async fn responses_should_record_empty_response_attempts() {
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(usage, (3, 3, 0, 0));
|
||||
assert_eq!(usage, (1, 1, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -576,7 +576,7 @@ async fn responses_stream_should_record_usage_record_after_completed_stream() {
|
||||
assert_eq!(event.output_tokens, Some(5));
|
||||
assert!(
|
||||
event.first_token_ms.is_some_and(|value| value > 0),
|
||||
"stream usage metadata should include first token latency: {metadata:?}",
|
||||
"stream usage metadata should include first output event latency: {metadata:?}",
|
||||
);
|
||||
let first_token_bucket: (i64, i64) = sqlx::query_as(
|
||||
"select first_token_latency_sum, first_token_latency_count from request_time_buckets limit 1",
|
||||
@@ -858,10 +858,10 @@ async fn responses_should_record_request_count_when_5xx_retries_are_exhausted()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(authorizations.len(), 3, "requests: {authorizations:?}");
|
||||
assert_eq!(failed_usage.map(|row| row.0).unwrap_or_default(), 1);
|
||||
let event = latest_response_ops_error_log(&pool).await;
|
||||
let event = latest_response_upstream_ops_error_log(&pool).await;
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
assert_eq!(event.account_id.as_deref(), Some("acct_chat"));
|
||||
assert_eq!(metadata["accountEmail"], "user@example.com");
|
||||
@@ -912,10 +912,10 @@ async fn responses_should_record_attempt_trace_after_retrying_another_account()
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = latest_response_ops_error_log(&pool).await;
|
||||
let event = latest_response_upstream_ops_error_log(&pool).await;
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(event.level, "error");
|
||||
assert_eq!(event.request_id.as_deref(), Some("req_attempt_trace_retry"));
|
||||
assert_eq!(event.account_id.as_deref(), Some("acct_secondary"));
|
||||
|
||||
@@ -8,6 +8,7 @@ use codex_proxy_rs::fleet::quota::{
|
||||
quota_snapshot_reset_at, QuotaRefreshService,
|
||||
};
|
||||
use codex_proxy_rs::fleet::store::{NewAccount, PgAccountStore};
|
||||
use codex_proxy_rs::infra::identity::AccountPseudonymizer;
|
||||
use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore;
|
||||
use codex_proxy_rs::upstream::openai::transport::CodexBackendClient;
|
||||
use secrecy::SecretString;
|
||||
@@ -19,6 +20,10 @@ use wiremock::{
|
||||
|
||||
use crate::support::storage::init_test_db;
|
||||
|
||||
fn test_account_pseudonymizer() -> Arc<AccountPseudonymizer> {
|
||||
Arc::new(AccountPseudonymizer::new([7; 32]))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_from_usage_should_preserve_spend_control_individual_limit() {
|
||||
let usage = json!({
|
||||
@@ -316,6 +321,7 @@ async fn quota_refresh_service_should_send_usage_cookie_when_cookie_store_is_con
|
||||
store,
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
)
|
||||
.with_cookie_store(cookies);
|
||||
|
||||
@@ -397,6 +403,7 @@ async fn quota_refresh_service_should_fetch_usage_for_quota_locked_accounts_and_
|
||||
store.clone(),
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
);
|
||||
|
||||
let mut last_refreshed = HashMap::new();
|
||||
@@ -472,6 +479,7 @@ async fn quota_refresh_service_should_refresh_quota_exhausted_accounts() {
|
||||
store,
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
);
|
||||
|
||||
let mut last_refreshed = HashMap::new();
|
||||
@@ -507,6 +515,7 @@ async fn quota_refresh_service_should_skip_recent_locked_account_before_cooldown
|
||||
store,
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
1_800,
|
||||
);
|
||||
|
||||
@@ -546,6 +555,7 @@ async fn quota_refresh_service_should_skip_recent_locked_account_inside_cooldown
|
||||
store,
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
1_800,
|
||||
);
|
||||
|
||||
@@ -602,6 +612,7 @@ async fn quota_refresh_service_should_bypass_recent_skip_after_cooldown_grace()
|
||||
store,
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
1_800,
|
||||
);
|
||||
|
||||
@@ -656,6 +667,7 @@ async fn quota_refresh_service_should_stagger_multiple_locked_account_requests()
|
||||
store,
|
||||
PgAccountUsageStore::new(pool.clone()),
|
||||
Arc::new(codex),
|
||||
test_account_pseudonymizer(),
|
||||
)
|
||||
.with_request_spacing(StdDuration::from_millis(200));
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
use codex_proxy_rs::infra::identity::AccountPseudonymizer;
|
||||
|
||||
#[test]
|
||||
fn account_pseudonyms_should_be_stable_and_domain_separated() {
|
||||
let pseudonymizer = AccountPseudonymizer::new([7; 32]);
|
||||
|
||||
let first = pseudonymizer.scoped("session-id", Some("acct-1"), "client-value");
|
||||
let repeated = pseudonymizer.scoped("session-id", Some("acct-1"), "client-value");
|
||||
let other_account = pseudonymizer.scoped("session-id", Some("acct-2"), "client-value");
|
||||
let other_domain = pseudonymizer.scoped("thread-id", Some("acct-1"), "client-value");
|
||||
|
||||
assert_eq!(first, repeated);
|
||||
assert_ne!(first, other_account);
|
||||
assert_ne!(first, other_domain);
|
||||
assert!(!first.contains("client-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn installation_ids_should_be_stable_uuid_values_scoped_per_account() {
|
||||
let pseudonymizer = AccountPseudonymizer::new([9; 32]);
|
||||
|
||||
let first = pseudonymizer.installation_id("acct-1");
|
||||
let repeated = pseudonymizer.installation_id("acct-1");
|
||||
let other_account = pseudonymizer.installation_id("acct-2");
|
||||
|
||||
assert_eq!(first, repeated);
|
||||
assert_ne!(first, other_account);
|
||||
assert!(uuid::Uuid::parse_str(&first).is_ok());
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
pub(crate) mod database;
|
||||
pub(crate) mod format;
|
||||
pub(crate) mod identity;
|
||||
pub(crate) mod logging;
|
||||
pub(crate) mod paths;
|
||||
pub(crate) mod time;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use std::{ffi::OsString, fs, sync::Mutex};
|
||||
|
||||
use codex_proxy_rs::infra::paths::{ensure_data_dir, load_or_create_installation_id};
|
||||
use codex_proxy_rs::infra::paths::{ensure_data_dir, load_or_create_identity_secret};
|
||||
|
||||
const INSTALLATION_ID_FILE_NAME: &str = "installation_id";
|
||||
const IDENTITY_SECRET_FILE_NAME: &str = "identity_hmac_secret";
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
@@ -18,37 +18,33 @@ fn data_dir_should_use_xdg_data_home_directly() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_or_create_installation_id_should_persist_under_data_dir_root() {
|
||||
fn identity_secret_should_be_stable_and_owner_only() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let installation_id =
|
||||
load_or_create_installation_id(Some(dir.path())).expect("installation id should generate");
|
||||
let first = load_or_create_identity_secret(dir.path()).expect("secret should generate");
|
||||
let second = load_or_create_identity_secret(dir.path()).expect("secret should reload");
|
||||
|
||||
assert_eq!(
|
||||
fs::read_to_string(dir.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(),
|
||||
installation_id
|
||||
);
|
||||
assert_eq!(first, second);
|
||||
let path = dir.path().join(IDENTITY_SECRET_FILE_NAME);
|
||||
assert_eq!(fs::read_to_string(&path).unwrap(), hex::encode(first));
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
assert_eq!(
|
||||
fs::metadata(path).unwrap().permissions().mode() & 0o777,
|
||||
0o600
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_or_create_installation_id_should_seed_data_dir_from_codex_desktop_id() {
|
||||
let _guard = ENV_LOCK.lock().unwrap();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
let data = tempfile::tempdir().unwrap();
|
||||
let codex_path = home.path().join(".codex").join(INSTALLATION_ID_FILE_NAME);
|
||||
let desktop_id = "018f8f6b-1d7b-7b7c-b9c8-8c9f4c6d0e1a";
|
||||
fs::create_dir_all(codex_path.parent().unwrap()).unwrap();
|
||||
fs::write(&codex_path, desktop_id).unwrap();
|
||||
let _home = EnvVarGuard::set("HOME", home.path().as_os_str().to_owned());
|
||||
fn identity_secret_should_fail_closed_when_existing_file_is_invalid() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(dir.path().join(IDENTITY_SECRET_FILE_NAME), "not-a-secret").unwrap();
|
||||
|
||||
let installation_id =
|
||||
load_or_create_installation_id(Some(data.path())).expect("installation id should seed");
|
||||
let error = load_or_create_identity_secret(dir.path()).expect_err("invalid secret must fail");
|
||||
|
||||
assert_eq!(installation_id, desktop_id);
|
||||
assert_eq!(
|
||||
fs::read_to_string(data.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(),
|
||||
desktop_id
|
||||
);
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
struct EnvVarGuard {
|
||||
|
||||
@@ -149,13 +149,12 @@ async fn model_service_should_refresh_plan_accounts_and_build_routing() {
|
||||
);
|
||||
|
||||
let result = service
|
||||
.refresh_backend_models_with_installation_id(
|
||||
.refresh_backend_models(
|
||||
&[
|
||||
refresh_plan_account("plus", active_account("acct-plus-1", "plus")),
|
||||
refresh_plan_account("team", active_account("acct-team", "team")),
|
||||
],
|
||||
"req-model-refresh",
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("refresh should succeed");
|
||||
@@ -198,13 +197,29 @@ async fn model_service_should_return_no_accounts_when_no_plan_accounts_exist() {
|
||||
let service = ModelService::new(test_model_config(), Some(store), Some(upstream));
|
||||
|
||||
let error = service
|
||||
.refresh_backend_models_with_installation_id(&[], "req-model-refresh", None)
|
||||
.refresh_backend_models(&[], "req-model-refresh")
|
||||
.await
|
||||
.expect_err("refresh should fail");
|
||||
|
||||
assert!(matches!(error, ModelServiceError::NoAccounts));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_service_should_notify_only_when_models_etag_changes() {
|
||||
let service = ModelService::new(test_model_config(), None, None);
|
||||
let mut changes = service.subscribe_models_etag_changes();
|
||||
|
||||
assert!(!service.observe_models_etag(None));
|
||||
assert!(service.observe_models_etag(Some("etag-1")));
|
||||
changes.changed().await.expect("first etag notification");
|
||||
assert_eq!(*changes.borrow_and_update(), 1);
|
||||
|
||||
assert!(!service.observe_models_etag(Some("etag-1")));
|
||||
assert!(service.observe_models_etag(Some("etag-2")));
|
||||
changes.changed().await.expect("second etag notification");
|
||||
assert_eq!(*changes.borrow_and_update(), 2);
|
||||
}
|
||||
|
||||
fn test_model_config() -> ModelConfig {
|
||||
ModelConfig {
|
||||
model_aliases: BTreeMap::new(),
|
||||
@@ -237,6 +252,7 @@ fn refresh_plan_account(plan_type: &str, account: Account) -> ModelRefreshPlanAc
|
||||
plan_type: plan_type.to_string(),
|
||||
access_token: account.access_token,
|
||||
account_id: account.account_id,
|
||||
installation_id: "test-installation-id".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -254,8 +254,9 @@ fn parse_rate_limit_headers_should_extract_primary_secondary_and_review_windows(
|
||||
|
||||
let parsed = parse_rate_limit_headers(&headers).expect("rate limits should parse");
|
||||
|
||||
let core = parsed.limits.get("codex").expect("core rate limit");
|
||||
assert_eq!(
|
||||
parsed.primary,
|
||||
core.primary,
|
||||
Some(RateLimitWindow {
|
||||
used_percent: 100.0,
|
||||
window_minutes: Some(5),
|
||||
@@ -263,12 +264,13 @@ fn parse_rate_limit_headers_should_extract_primary_secondary_and_review_windows(
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
parsed.secondary.expect("secondary window").window_minutes,
|
||||
core.secondary.expect("secondary window").window_minutes,
|
||||
Some(10080)
|
||||
);
|
||||
assert_eq!(
|
||||
parsed
|
||||
.code_review
|
||||
.limits
|
||||
.get("codex_code_review")
|
||||
.expect("review window")
|
||||
.primary
|
||||
.expect("review primary")
|
||||
@@ -297,14 +299,91 @@ fn parse_rate_limits_event_should_extract_internal_websocket_rate_limits() {
|
||||
|
||||
let parsed = parse_rate_limits_event(&event).expect("event should parse");
|
||||
|
||||
let used_percent = parsed.primary.expect("primary window").used_percent;
|
||||
let core = parsed.limits.get("codex").expect("core rate limit");
|
||||
let used_percent = core.primary.expect("primary window").used_percent;
|
||||
assert!((used_percent - 99.5).abs() < f64::EPSILON);
|
||||
assert_eq!(
|
||||
parsed.secondary.expect("secondary window").reset_at,
|
||||
core.secondary.expect("secondary window").reset_at,
|
||||
Some(1_894_056_000)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rate_limit_headers_should_extract_arbitrary_limits_and_account_metadata() {
|
||||
let headers = vec![
|
||||
(
|
||||
"x-codex-other-primary-used-percent".to_string(),
|
||||
"63.5".to_string(),
|
||||
),
|
||||
(
|
||||
"x-codex-other-primary-window-minutes".to_string(),
|
||||
"1440".to_string(),
|
||||
),
|
||||
(
|
||||
"x-codex-other-limit-name".to_string(),
|
||||
"Codex Other".to_string(),
|
||||
),
|
||||
(
|
||||
"x-codex-credits-has-credits".to_string(),
|
||||
"true".to_string(),
|
||||
),
|
||||
("x-codex-credits-unlimited".to_string(), "false".to_string()),
|
||||
("x-codex-credits-balance".to_string(), "12.50".to_string()),
|
||||
("x-codex-plan-type".to_string(), "pro".to_string()),
|
||||
("x-codex-promo-message".to_string(), "bonus".to_string()),
|
||||
(
|
||||
"x-codex-rate-limit-reached-type".to_string(),
|
||||
"weekly".to_string(),
|
||||
),
|
||||
];
|
||||
|
||||
let parsed = parse_rate_limit_headers(&headers).expect("rate limits should parse");
|
||||
let other = parsed.limits.get("codex_other").expect("dynamic limit");
|
||||
assert_eq!(other.limit_name.as_deref(), Some("Codex Other"));
|
||||
assert_eq!(
|
||||
other.primary.expect("primary window").window_minutes,
|
||||
Some(1440)
|
||||
);
|
||||
let credits = parsed.credits.expect("credits");
|
||||
assert!(credits.has_credits);
|
||||
assert!(!credits.unlimited);
|
||||
assert_eq!(credits.balance.as_deref(), Some("12.50"));
|
||||
assert_eq!(parsed.plan_type.as_deref(), Some("pro"));
|
||||
assert_eq!(parsed.promo_message.as_deref(), Some("bonus"));
|
||||
assert_eq!(parsed.rate_limit_reached_type.as_deref(), Some("weekly"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rate_limits_event_should_keep_metered_limit_credits_and_plan() {
|
||||
let event = json!({
|
||||
"type": "codex.rate_limits",
|
||||
"plan_type": "team",
|
||||
"metered_limit_name": "codex-other",
|
||||
"limit_name": "Other requests",
|
||||
"rate_limits": {
|
||||
"primary": {
|
||||
"used_percent": 33,
|
||||
"window_minutes": 60,
|
||||
"reset_at": 1893456300
|
||||
}
|
||||
},
|
||||
"credits": {
|
||||
"has_credits": true,
|
||||
"unlimited": false,
|
||||
"balance": "9.25"
|
||||
}
|
||||
});
|
||||
|
||||
let parsed = parse_rate_limits_event(&event).expect("event should parse");
|
||||
let other = parsed.limits.get("codex_other").expect("metered limit");
|
||||
assert_eq!(other.limit_name.as_deref(), Some("Other requests"));
|
||||
assert_eq!(parsed.plan_type.as_deref(), Some("team"));
|
||||
assert_eq!(
|
||||
parsed.credits.expect("credits").balance.as_deref(),
|
||||
Some("9.25")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limit_quota_should_preserve_existing_monthly_and_credits_when_passive_data_lacks_them() {
|
||||
let headers = vec![
|
||||
|
||||
@@ -25,16 +25,14 @@ use codex_proxy_rs::upstream::openai::protocol::websocket::{
|
||||
websocket_function_call_output_item_event_invalid_required_fields,
|
||||
websocket_function_call_output_payload_item_event_invalid_required_fields,
|
||||
websocket_image_generation_call_output_item_event_invalid_required_fields,
|
||||
websocket_incomplete_response_reason,
|
||||
websocket_local_shell_call_output_item_event_invalid_required_fields,
|
||||
websocket_message_output_item_event_invalid_required_fields, websocket_metadata_turn_state,
|
||||
websocket_output_item_event_invalid_item_type_tag,
|
||||
websocket_output_item_event_invalid_metadata, websocket_output_item_event_missing_item,
|
||||
websocket_output_item_event_non_object_item, websocket_payload_audit_snapshot,
|
||||
websocket_reasoning_output_item_event_invalid_required_fields,
|
||||
websocket_reasoning_summary_part_added_missing_summary_index,
|
||||
websocket_response_completed_missing_response, websocket_response_completed_parse_error,
|
||||
websocket_response_created_missing_response,
|
||||
websocket_reasoning_summary_part_added_missing_summary_index, websocket_response_completed_id,
|
||||
websocket_response_completed_missing_response, websocket_response_created_missing_response,
|
||||
websocket_response_output_text_delta_missing_delta,
|
||||
websocket_tool_search_call_output_item_event_invalid_required_fields,
|
||||
websocket_tool_search_output_item_event_invalid_required_fields,
|
||||
|
||||
@@ -83,18 +83,7 @@ fn response_from_codex_sse_should_collect_completed_response_with_deltas() {
|
||||
"usage": {
|
||||
"input_tokens": 2,
|
||||
"output_tokens": 3
|
||||
},
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "output_text",
|
||||
"text": "hello",
|
||||
"annotations": []
|
||||
}]
|
||||
}],
|
||||
"output_text": "hello"
|
||||
}
|
||||
}))
|
||||
);
|
||||
}
|
||||
@@ -107,7 +96,7 @@ fn response_from_codex_sse_should_collect_incomplete_response() {
|
||||
|
||||
assert_eq!(
|
||||
response,
|
||||
CollectedResponse::Completed(json!({
|
||||
CollectedResponse::Incomplete(json!({
|
||||
"id": "resp_incomplete",
|
||||
"object": "response",
|
||||
"status": "incomplete",
|
||||
|
||||
@@ -29,8 +29,6 @@ fn codex_websocket_payload_audit_snapshot_should_redact_user_content() {
|
||||
"model",
|
||||
"instructions",
|
||||
"input",
|
||||
"stream",
|
||||
"store",
|
||||
"previous_response_id",
|
||||
"service_tier",
|
||||
"prompt_cache_key",
|
||||
@@ -39,7 +37,8 @@ fn codex_websocket_payload_audit_snapshot_should_redact_user_content() {
|
||||
);
|
||||
assert_eq!(snapshot.body["type"], "response.create");
|
||||
assert_eq!(snapshot.body["model"], "gpt-5.5");
|
||||
assert_eq!(snapshot.body["stream"], true);
|
||||
assert!(snapshot.body.get("stream").is_none());
|
||||
assert!(snapshot.body.get("store").is_none());
|
||||
assert_eq!(snapshot.body["instructions"], "<redacted>");
|
||||
assert_eq!(snapshot.body["input"], "<redacted>");
|
||||
assert_eq!(snapshot.body["previous_response_id"], "<redacted>");
|
||||
@@ -78,8 +77,6 @@ fn codex_websocket_response_create_payload_text_should_preserve_canonical_field_
|
||||
"\"model\":\"gpt-5.5\"",
|
||||
"\"instructions\":\"private capture instructions\"",
|
||||
"\"input\":",
|
||||
"\"stream\":true",
|
||||
"\"store\":false",
|
||||
"\"prompt_cache_key\":\"session-1\"",
|
||||
"\"client_metadata\":",
|
||||
],
|
||||
@@ -187,6 +184,7 @@ fn codex_websocket_metadata_turn_state_should_extract_case_insensitive_header()
|
||||
#[test]
|
||||
fn codex_websocket_terminal_event_should_match_completed_failed_and_error() {
|
||||
assert!(is_terminal_websocket_event("response.completed"));
|
||||
assert!(is_terminal_websocket_event("response.incomplete"));
|
||||
assert!(is_terminal_websocket_event("response.failed"));
|
||||
assert!(is_terminal_websocket_event("error"));
|
||||
assert!(!is_terminal_websocket_event("response.output_text.delta"));
|
||||
@@ -223,6 +221,9 @@ fn codex_websocket_error_frame_should_classify_ts_aligned_upstream_error_codes()
|
||||
("account_banned", 403),
|
||||
("banned", 403),
|
||||
("previous_response_not_found", 400),
|
||||
("invalid_encrypted_content", 400),
|
||||
("missing_tool_output", 400),
|
||||
("no_tool_output", 400),
|
||||
];
|
||||
|
||||
for (code, expected_status) in cases {
|
||||
@@ -256,7 +257,6 @@ fn codex_websocket_error_frame_should_ignore_legacy_extension_codes() {
|
||||
"invalid_prompt",
|
||||
"cyber_policy",
|
||||
"invalid_request",
|
||||
"invalid_encrypted_content",
|
||||
"no_tool_output_found_for_function_call",
|
||||
"server_is_overloaded",
|
||||
"slow_down",
|
||||
@@ -376,7 +376,7 @@ fn codex_websocket_wrapped_error_headers_should_extract_retry_after() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_websocket_response_completed_parse_error_should_validate_shape() {
|
||||
fn codex_websocket_response_completed_id_should_validate_shape() {
|
||||
let frame = json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
@@ -390,14 +390,14 @@ fn codex_websocket_response_completed_parse_error_should_validate_shape() {
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let error = websocket_response_completed_parse_error(&frame)
|
||||
.expect("invalid response.completed should report a parse error");
|
||||
let error = websocket_response_completed_id(&frame)
|
||||
.expect_err("invalid response.completed should report a parse error");
|
||||
|
||||
assert!(error.contains("failed to parse ResponseCompleted"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_websocket_response_completed_parse_error_should_reject_missing_id_and_incomplete_usage() {
|
||||
fn codex_websocket_response_completed_id_should_reject_missing_id_and_incomplete_usage() {
|
||||
let missing_id = json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
@@ -423,11 +423,11 @@ fn codex_websocket_response_completed_parse_error_should_reject_missing_id_and_i
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert!(websocket_response_completed_parse_error(&missing_id)
|
||||
.expect("missing id should be rejected")
|
||||
assert!(websocket_response_completed_id(&missing_id)
|
||||
.expect_err("missing id should be rejected")
|
||||
.contains("missing field"));
|
||||
assert!(websocket_response_completed_parse_error(&incomplete_usage)
|
||||
.expect("incomplete usage should be rejected")
|
||||
assert!(websocket_response_completed_id(&incomplete_usage)
|
||||
.expect_err("incomplete usage should be rejected")
|
||||
.contains("missing field"));
|
||||
}
|
||||
|
||||
@@ -528,22 +528,12 @@ fn codex_websocket_optional_null_fields_should_match_missing_field_behavior() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_websocket_response_control_events_should_detect_incomplete_and_missing_completed_response()
|
||||
{
|
||||
let incomplete = json!({
|
||||
"type": "response.incomplete",
|
||||
"response": {
|
||||
"incomplete_details": {
|
||||
"reason": "max_output_tokens"
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
fn codex_websocket_response_control_events_should_recognize_incomplete_as_terminal() {
|
||||
assert!(is_terminal_websocket_event("response.incomplete"));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
websocket_incomplete_response_reason(&incomplete).as_deref(),
|
||||
Some("max_output_tokens")
|
||||
);
|
||||
#[test]
|
||||
fn codex_websocket_response_control_events_should_detect_missing_completed_response() {
|
||||
assert!(websocket_response_completed_missing_response(
|
||||
r#"{"type":"response.completed"}"#
|
||||
));
|
||||
|
||||
@@ -64,8 +64,9 @@ async fn codex_backend_client_should_apply_configured_websocket_pool() {
|
||||
.with_websocket_pool(Arc::new(pool));
|
||||
let mut request = CodexResponsesRequest::new_http_sse("gpt-5.5", "be brief", Vec::new());
|
||||
request.force_http_sse = false;
|
||||
request.set_previous_response_id(Some("resp_runtime_pool_previous".to_string()));
|
||||
request.use_websocket = true;
|
||||
request.set_prompt_cache_key(Some("conversation-runtime-pool".to_string()));
|
||||
request.local_conversation_id = Some("conversation-runtime-pool".to_string());
|
||||
|
||||
let first = backend
|
||||
.create_response(
|
||||
@@ -74,6 +75,7 @@ async fn codex_backend_client_should_apply_configured_websocket_pool() {
|
||||
)
|
||||
.await
|
||||
.expect("first runtime websocket response should succeed");
|
||||
request.set_previous_response_id(Some("resp_runtime_pool_first".to_string()));
|
||||
let second = backend
|
||||
.create_response(
|
||||
&request,
|
||||
|
||||
@@ -301,6 +301,10 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an
|
||||
cookie_header: None,
|
||||
installation_id: Some("install-123"),
|
||||
session_id: Some("cp_derived"),
|
||||
thread_id: None,
|
||||
prompt_cache_key: Some("cp_derived"),
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -332,7 +336,6 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an
|
||||
"ignored_non_string": 42,
|
||||
"x-codex-installation-id": "install-123",
|
||||
"session_id": "cp_derived",
|
||||
"thread_id": "cp_derived",
|
||||
"x-codex-window-id": "cw_derived",
|
||||
"x-codex-turn-metadata": "{\"thread_source\":\"subagent\"}",
|
||||
"x-codex-parent-thread-id": "parent-456"
|
||||
@@ -342,7 +345,7 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an
|
||||
let headers = received_headers.lock().unwrap().clone();
|
||||
assert!(headers
|
||||
.iter()
|
||||
.any(|(name, value)| name == "x-client-request-id" && value == "cp_derived"));
|
||||
.any(|(name, value)| name == "x-client-request-id" && value == "req_ws_security"));
|
||||
assert!(headers
|
||||
.iter()
|
||||
.any(|(name, value)| name == "x-codex-installation-id" && value == "install-123"));
|
||||
@@ -373,9 +376,7 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an
|
||||
assert!(headers
|
||||
.iter()
|
||||
.any(|(name, value)| name == "session-id" && value == "cp_derived"));
|
||||
assert!(headers
|
||||
.iter()
|
||||
.any(|(name, value)| name == "thread-id" && value == "cp_derived"));
|
||||
assert!(headers.iter().all(|(name, _)| name != "thread-id"));
|
||||
assert!(headers.iter().all(|(name, _)| name != "content-type"));
|
||||
assert!(headers.iter().all(|(name, _)| name != "accept"));
|
||||
assert!(headers.iter().all(|(name, _)| name != "session_id"));
|
||||
@@ -402,9 +403,7 @@ async fn codex_backend_client_should_send_desktop_headers_and_capture_response_m
|
||||
.and(wiremock::matchers::body_json(json!({
|
||||
"model": "gpt-5.5",
|
||||
"instructions": "",
|
||||
"input": [],
|
||||
"stream": true,
|
||||
"store": false
|
||||
"input": []
|
||||
})))
|
||||
.respond_with(
|
||||
wiremock::ResponseTemplate::new(200)
|
||||
@@ -448,6 +447,10 @@ async fn codex_backend_client_should_send_desktop_headers_and_capture_response_m
|
||||
cookie_header: Some("cf_clearance=old"),
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -517,6 +520,10 @@ async fn codex_backend_client_should_use_latest_runtime_fingerprint_for_new_requ
|
||||
cookie_header: None,
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
|
||||
client
|
||||
@@ -592,6 +599,10 @@ async fn codex_backend_client_usage_should_use_wham_usage_headers() {
|
||||
cookie_header: Some("cf_clearance=old"),
|
||||
installation_id: Some("install-1"),
|
||||
session_id: Some("session-1"),
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -748,6 +759,10 @@ async fn codex_backend_client_should_send_http_sse_headers_in_fingerprint_order(
|
||||
cookie_header: Some("cf_clearance=old"),
|
||||
installation_id: Some("install-1"),
|
||||
session_id: Some("session-1"),
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -778,7 +793,6 @@ async fn codex_backend_client_should_send_http_sse_headers_in_fingerprint_order(
|
||||
"x-client-request-id",
|
||||
"x-codex-installation-id",
|
||||
"session-id",
|
||||
"thread-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-turn-state",
|
||||
"x-codex-turn-metadata",
|
||||
@@ -820,6 +834,12 @@ async fn codex_backend_client_should_send_compact_headers_in_fingerprint_order()
|
||||
client_ip: None,
|
||||
client_user_agent: None,
|
||||
client_api_key_id: None,
|
||||
client_session_id: None,
|
||||
client_thread_id: None,
|
||||
client_request_id: None,
|
||||
client_turn_id: None,
|
||||
client_window_id: None,
|
||||
client_parent_thread_id: None,
|
||||
},
|
||||
CodexRequestContext {
|
||||
access_token: "access-token",
|
||||
@@ -835,6 +855,10 @@ async fn codex_backend_client_should_send_compact_headers_in_fingerprint_order()
|
||||
cookie_header: Some("cf_clearance=old"),
|
||||
installation_id: Some("install-1"),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -134,6 +134,10 @@ async fn codex_backend_client_should_cap_non_success_error_body_at_one_mib() {
|
||||
cookie_header: None,
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -188,6 +192,10 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod
|
||||
cookie_header: None,
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -204,6 +212,101 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod
|
||||
assert_eq!(retry_after_seconds, Some(12));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_capture_only_safe_response_metadata() {
|
||||
let server = wiremock::MockServer::start().await;
|
||||
let body = concat!(
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
|
||||
);
|
||||
wiremock::Mock::given(wiremock::matchers::method("POST"))
|
||||
.and(wiremock::matchers::path("/codex/responses"))
|
||||
.respond_with(
|
||||
wiremock::ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.insert_header("x-request-id", "upstream-request")
|
||||
.insert_header("openai-model", "gpt-5.5-2026-07-01")
|
||||
.insert_header("x-models-etag", "models-v2")
|
||||
.insert_header("x-reasoning-included", "true")
|
||||
.insert_header("openai-processing-ms", "42")
|
||||
.insert_header("set-cookie", "secret=value")
|
||||
.insert_header("x-codex-primary-used-percent", "15")
|
||||
.set_body_string(body),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
server.uri(),
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
);
|
||||
let mut request =
|
||||
codex_proxy_rs::upstream::openai::protocol::responses::CodexResponsesRequest::new_http_sse(
|
||||
"gpt-5.5",
|
||||
"",
|
||||
Vec::new(),
|
||||
);
|
||||
request.force_http_sse = true;
|
||||
|
||||
let response = client
|
||||
.create_response(
|
||||
&request,
|
||||
CodexRequestContext {
|
||||
access_token: "access-token",
|
||||
account_id: Some("chatgpt-account"),
|
||||
request_id: "req_response_metadata",
|
||||
turn_state: None,
|
||||
turn_metadata: None,
|
||||
beta_features: None,
|
||||
include_timing_metrics: None,
|
||||
version: None,
|
||||
codex_window_id: None,
|
||||
parent_thread_id: None,
|
||||
cookie_header: None,
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("response should succeed");
|
||||
|
||||
assert_eq!(
|
||||
response.response_metadata.effective_model.as_deref(),
|
||||
Some("gpt-5.5-2026-07-01")
|
||||
);
|
||||
assert_eq!(
|
||||
response.response_metadata.models_etag.as_deref(),
|
||||
Some("models-v2")
|
||||
);
|
||||
assert!(response.response_metadata.reasoning_included);
|
||||
let mut names = response
|
||||
.response_metadata
|
||||
.client_headers
|
||||
.iter()
|
||||
.map(|(name, _)| name.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
names.sort_unstable();
|
||||
assert_eq!(
|
||||
names,
|
||||
vec![
|
||||
"openai-model",
|
||||
"openai-processing-ms",
|
||||
"x-models-etag",
|
||||
"x-reasoning-included",
|
||||
"x-request-id"
|
||||
]
|
||||
);
|
||||
assert_eq!(response.set_cookie_headers, vec!["secret=value"]);
|
||||
assert_eq!(
|
||||
response.rate_limit_headers,
|
||||
vec![("x-codex-primary-used-percent".to_string(), "15".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_reqwest_client_should_reuse_cached_connection_pool() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::{
|
||||
};
|
||||
|
||||
use codex_proxy_rs::upstream::openai::{
|
||||
protocol::responses::{PreviousResponseScope, StreamCommitPolicy},
|
||||
transport::websocket::{
|
||||
execute_response_create_request, responses_websocket_endpoint, CodexWebSocketConnection,
|
||||
CodexWebSocketExchangeError, CodexWebSocketRequest,
|
||||
@@ -176,6 +177,10 @@ pub(crate) fn request_context<'a>(
|
||||
cookie_header: None,
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,8 +213,9 @@ pub(crate) fn pooled_websocket_request(
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some(conversation_id.to_string()));
|
||||
request.client_conversation_id = Some(conversation_id.to_string());
|
||||
request.local_conversation_id = Some(conversation_id.to_string());
|
||||
request
|
||||
}
|
||||
|
||||
@@ -247,7 +253,7 @@ pub(crate) fn websocket_pool_config_for_tests(
|
||||
ping_interval,
|
||||
ping_timeout: Duration::from_secs(1),
|
||||
liveness_timeout,
|
||||
first_token_timeout: None,
|
||||
initial_event_timeout: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -140,6 +140,7 @@ async fn codex_backend_client_should_decode_permessage_deflate_context_takeover_
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
|
||||
let response = backend
|
||||
.create_response(
|
||||
@@ -189,7 +190,8 @@ fn websocket_connection_should_prepare_response_create_payload_text() {
|
||||
assert_eq!(payload["model"], "gpt-5.5");
|
||||
assert_eq!(payload["instructions"], "be brief");
|
||||
assert_eq!(payload["input"][0]["content"], "hello");
|
||||
assert_eq!(payload["stream"], true);
|
||||
assert!(payload.get("stream").is_none());
|
||||
assert!(payload.get("store").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -227,8 +229,6 @@ fn websocket_connection_should_prepare_capture_payload_with_canonical_field_orde
|
||||
"\"model\":\"gpt-5.5\"",
|
||||
"\"instructions\":\"private capture instructions\"",
|
||||
"\"input\":",
|
||||
"\"stream\":true",
|
||||
"\"store\":false",
|
||||
"\"prompt_cache_key\":\"session-1\"",
|
||||
"\"client_metadata\":",
|
||||
],
|
||||
@@ -755,7 +755,7 @@ async fn codex_backend_client_should_not_reuse_websocket_rate_limit_headers() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_execute_response_create_request_should_surface_incomplete_response() {
|
||||
async fn websocket_execute_response_create_request_should_forward_incomplete_response() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
@@ -797,15 +797,13 @@ async fn websocket_execute_response_create_request_should_surface_incomplete_res
|
||||
)
|
||||
.expect("payload should serialize");
|
||||
|
||||
let error = execute_response_create_request(&prepared)
|
||||
let exchange = execute_response_create_request(&prepared)
|
||||
.await
|
||||
.expect_err("response.incomplete should be surfaced as websocket error");
|
||||
.expect("response.incomplete should remain a terminal Responses event");
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexWebSocketExchangeError::IncompleteResponse { reason } = error else {
|
||||
panic!("expected incomplete websocket response");
|
||||
};
|
||||
assert_eq!(reason, "max_output_tokens");
|
||||
assert!(exchange.body.contains("event: response.incomplete"));
|
||||
assert!(exchange.body.contains("max_output_tokens"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -866,8 +864,7 @@ async fn websocket_execute_response_create_request_should_reject_invalid_complet
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_execute_response_create_request_should_forward_completed_without_response_as_terminal(
|
||||
) {
|
||||
async fn websocket_execute_response_create_request_should_reject_completed_without_response() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
@@ -903,14 +900,15 @@ async fn websocket_execute_response_create_request_should_forward_completed_with
|
||||
)
|
||||
.expect("payload should serialize");
|
||||
|
||||
// 透明代理:WS 输出侧不再按 schema 丢弃事件,缺 `response` 的 `response.completed`
|
||||
// 也被原样转发,并作为终止事件结束交换(信任上游终止信号)。
|
||||
let response = execute_response_create_request(&prepared)
|
||||
let error = execute_response_create_request(&prepared)
|
||||
.await
|
||||
.expect("completed frame should terminate the exchange");
|
||||
.expect_err("completed frame without response must be rejected");
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(response.body.contains("event: response.completed"));
|
||||
assert!(matches!(
|
||||
error,
|
||||
CodexWebSocketExchangeError::InvalidCompletedResponse { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -966,37 +964,96 @@ async fn websocket_execute_response_create_request_should_pass_through_unclassif
|
||||
assert!(response.body.contains("No tool output found"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn websocket_execute_response_create_request_should_timeout_when_upstream_is_silent() {
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_timeout_when_upstream_is_silent() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (request_seen_tx, request_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut websockets = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _message = websocket.next().await.unwrap().unwrap();
|
||||
websockets.push(websocket);
|
||||
}
|
||||
futures::future::pending::<()>().await;
|
||||
});
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
)
|
||||
.with_websocket_initial_event_timeout(Some(Duration::from_millis(30)));
|
||||
let mut request =
|
||||
codex_proxy_rs::upstream::openai::protocol::responses::CodexResponsesRequest::new_http_sse(
|
||||
"gpt-5.5",
|
||||
"be brief",
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
|
||||
let error = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_silent_websocket", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect_err("silent upstream should time out");
|
||||
server.abort();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::InitialEventTimeout { timeout })
|
||||
if timeout == Duration::from_millis(30)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn websocket_response_created_should_switch_to_active_stream_idle_timeout() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (created_tx, created_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _message = websocket.next().await.unwrap().unwrap();
|
||||
request_seen_tx.send(()).unwrap();
|
||||
futures::future::pending::<()>().await;
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": "resp_structural", "status": "in_progress"}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
created_tx.send(()).unwrap();
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_structural", 3, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
});
|
||||
let prepared = prepared_websocket_request(&format!("http://{addr}"));
|
||||
|
||||
let response_task =
|
||||
tokio::spawn(async move { execute_response_create_request(&prepared).await });
|
||||
request_seen_rx.await.unwrap();
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(20)).await;
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let error = response_task
|
||||
created_rx.await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(30)).await;
|
||||
tokio::task::yield_now().await;
|
||||
let response = response_task
|
||||
.await
|
||||
.expect("websocket task should finish")
|
||||
.expect_err("silent upstream should time out");
|
||||
server.abort();
|
||||
.expect("structural activity should disable the initial timeout");
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout } = error else {
|
||||
panic!("expected receive idle timeout");
|
||||
};
|
||||
assert_eq!(timeout, Duration::from_secs(20));
|
||||
assert!(response.body.contains("event: response.created"));
|
||||
assert!(response.body.contains("resp_structural"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1087,6 +1144,7 @@ async fn codex_backend_client_stream_should_reject_binary_websocket_event() {
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.force_http_sse = false;
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
@@ -1094,15 +1152,19 @@ async fn codex_backend_client_stream_should_reject_binary_websocket_event() {
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
);
|
||||
|
||||
let Err(error) = backend
|
||||
let mut response = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_stream_binary", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
panic!("binary event before first output should reject stream open");
|
||||
};
|
||||
.expect("stream should open before consuming websocket frames");
|
||||
let error = response
|
||||
.body
|
||||
.next()
|
||||
.await
|
||||
.expect("binary frame should produce a stream item")
|
||||
.expect_err("binary frame should fail the stream");
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
@@ -1112,7 +1174,7 @@ async fn codex_backend_client_stream_should_reject_binary_websocket_event() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_stream_should_retry_without_pool_when_first_output_times_out() {
|
||||
async fn codex_backend_client_stream_should_keep_socket_after_structural_activity() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
@@ -1137,59 +1199,55 @@ async fn codex_backend_client_stream_should_retry_without_pool_when_first_output
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (second_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut second_websocket = accept_codex_test_websocket(second_stream).await;
|
||||
let _second_message = second_websocket.next().await.unwrap().unwrap();
|
||||
second_websocket
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": "no-pool retry recovered"
|
||||
"delta": "delayed output"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_no_pool_retry_recovered", 3, 1).into(),
|
||||
completed_websocket_response("resp_no_pool_delayed", 3, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket.close(None).await.unwrap();
|
||||
drop(first_websocket);
|
||||
first_websocket.close(None).await.unwrap();
|
||||
});
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
)
|
||||
.with_websocket_first_token_timeout(Some(Duration::from_millis(30)));
|
||||
let request = pooled_websocket_request("conversation-first-token-no-pool-timeout");
|
||||
.with_websocket_initial_event_timeout(Some(Duration::from_millis(30)));
|
||||
let request = pooled_websocket_request("conversation-structural-no-pool");
|
||||
|
||||
let response = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_first_token_no_pool_timeout", Some("chatgpt-account")),
|
||||
request_context("req_structural_no_pool", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("no-pool first-token timeout should retry before returning stream");
|
||||
.expect("structural activity should keep the websocket stream open");
|
||||
let decision = response.websocket_pool_decision;
|
||||
let mut stream = response.body;
|
||||
let mut body = String::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.expect("retried no-pool stream chunk should be valid");
|
||||
let chunk = chunk.expect("delayed websocket stream chunk should be valid");
|
||||
body.push_str(std::str::from_utf8(&chunk).unwrap());
|
||||
}
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(body.contains("no-pool retry recovered"));
|
||||
assert!(body.contains("resp_no_pool_retry_recovered"));
|
||||
assert!(!body.contains("resp_no_pool_first_token_stalled"));
|
||||
assert!(body.contains("resp_no_pool_first_token_stalled"));
|
||||
assert!(body.contains("delayed output"));
|
||||
assert!(body.contains("resp_no_pool_delayed"));
|
||||
assert!(decision.is_none());
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1220,6 +1278,7 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.force_http_sse = false;
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
@@ -1290,6 +1349,7 @@ async fn codex_backend_client_stream_should_wait_for_terminal_after_active_webso
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.force_http_sse = false;
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
@@ -1354,6 +1414,7 @@ async fn codex_backend_client_stream_should_timeout_when_active_websocket_stalls
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.force_http_sse = false;
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
@@ -1450,6 +1511,7 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
@@ -1475,6 +1537,10 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
|
||||
cookie_header: None,
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -49,6 +49,7 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -127,6 +128,7 @@ async fn codex_backend_client_should_open_fresh_socket_when_idle_pooled_websocke
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool-silent".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -156,7 +158,7 @@ async fn codex_backend_client_should_open_fresh_socket_when_idle_pooled_websocke
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_stream_should_retry_fresh_socket_when_first_output_times_out() {
|
||||
async fn codex_backend_client_stream_should_keep_fresh_socket_after_structural_activity() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
@@ -181,32 +183,28 @@ async fn codex_backend_client_stream_should_retry_fresh_socket_when_first_output
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (second_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut second_websocket = accept_codex_test_websocket(second_stream).await;
|
||||
let _second_message = second_websocket.next().await.unwrap().unwrap();
|
||||
second_websocket
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": "fresh retry recovered"
|
||||
"delta": "fresh delayed output"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_first_token_fresh_recovered", 3, 1).into(),
|
||||
completed_websocket_response("resp_fresh_delayed", 3, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket.close(None).await.unwrap();
|
||||
drop(first_websocket);
|
||||
first_websocket.close(None).await.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig {
|
||||
first_token_timeout: Some(Duration::from_millis(30)),
|
||||
initial_event_timeout: Some(Duration::from_millis(30)),
|
||||
..websocket_pool_config_for_tests(None, None, None)
|
||||
}));
|
||||
let backend = CodexBackendClient::new(
|
||||
@@ -215,33 +213,33 @@ async fn codex_backend_client_stream_should_retry_fresh_socket_when_first_output
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let request = pooled_websocket_request("conversation-first-token-fresh-timeout");
|
||||
let request = pooled_websocket_request("conversation-structural-fresh");
|
||||
|
||||
let response = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_first_token_fresh_timeout", Some("chatgpt-account")),
|
||||
request_context("req_structural_fresh", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("fresh first-token timeout should retry before returning stream");
|
||||
.expect("structural activity should keep the fresh websocket open");
|
||||
let decision = response.websocket_pool_decision;
|
||||
let mut stream = response.body;
|
||||
let mut body = String::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.expect("retried stream chunk should be valid");
|
||||
let chunk = chunk.expect("delayed fresh stream chunk should be valid");
|
||||
body.push_str(std::str::from_utf8(&chunk).unwrap());
|
||||
}
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(body.contains("fresh retry recovered"));
|
||||
assert!(body.contains("resp_first_token_fresh_recovered"));
|
||||
assert!(!body.contains("resp_first_token_fresh_stalled"));
|
||||
assert!(body.contains("resp_first_token_fresh_stalled"));
|
||||
assert!(body.contains("fresh delayed output"));
|
||||
assert!(body.contains("resp_fresh_delayed"));
|
||||
assert_eq!(decision.unwrap().kind(), "new");
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_stream_should_retry_reused_socket_when_first_output_times_out() {
|
||||
async fn codex_backend_client_stream_should_keep_reused_socket_after_structural_activity() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
@@ -274,32 +272,28 @@ async fn codex_backend_client_stream_should_retry_reused_socket_when_first_outpu
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (second_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut second_websocket = accept_codex_test_websocket(second_stream).await;
|
||||
let _retry_message = second_websocket.next().await.unwrap().unwrap();
|
||||
second_websocket
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": "reuse retry recovered"
|
||||
"delta": "reused delayed output"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_first_token_reuse_recovered", 3, 1).into(),
|
||||
completed_websocket_response("resp_reused_delayed", 3, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket.close(None).await.unwrap();
|
||||
drop(first_websocket);
|
||||
first_websocket.close(None).await.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig {
|
||||
first_token_timeout: Some(Duration::from_millis(30)),
|
||||
initial_event_timeout: Some(Duration::from_millis(30)),
|
||||
..websocket_pool_config_for_tests(None, None, None)
|
||||
}));
|
||||
let backend = CodexBackendClient::new(
|
||||
@@ -308,41 +302,41 @@ async fn codex_backend_client_stream_should_retry_reused_socket_when_first_outpu
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let request = pooled_websocket_request("conversation-first-token-reuse-timeout");
|
||||
let request = pooled_websocket_request("conversation-structural-reuse");
|
||||
|
||||
let seed = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_first_token_reuse_seed", Some("chatgpt-account")),
|
||||
request_context("req_structural_reuse_seed", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("seed response should populate pool");
|
||||
let response = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_first_token_reuse_timeout", Some("chatgpt-account")),
|
||||
request_context("req_structural_reuse", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("reused first-token timeout should retry before returning stream");
|
||||
.expect("structural activity should keep the reused websocket open");
|
||||
let decision = response.websocket_pool_decision;
|
||||
let mut stream = response.body;
|
||||
let mut body = String::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.expect("retried reused stream chunk should be valid");
|
||||
let chunk = chunk.expect("delayed reused stream chunk should be valid");
|
||||
body.push_str(std::str::from_utf8(&chunk).unwrap());
|
||||
}
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(seed.body.contains("resp_first_token_reuse_seed"));
|
||||
assert!(body.contains("reuse retry recovered"));
|
||||
assert!(body.contains("resp_first_token_reuse_recovered"));
|
||||
assert!(!body.contains("resp_first_token_reuse_stalled"));
|
||||
assert_eq!(decision.unwrap().kind(), "retry_after_stale_reuse");
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
assert!(body.contains("resp_first_token_reuse_stalled"));
|
||||
assert!(body.contains("reused delayed output"));
|
||||
assert!(body.contains("resp_reused_delayed"));
|
||||
assert_eq!(decision.unwrap().kind(), "reuse");
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_stream_should_retry_bypass_socket_when_first_output_times_out() {
|
||||
async fn codex_backend_client_stream_should_keep_bypass_socket_after_structural_activity() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
@@ -367,33 +361,29 @@ async fn codex_backend_client_stream_should_retry_bypass_socket_when_first_outpu
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (second_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut second_websocket = accept_codex_test_websocket(second_stream).await;
|
||||
let _second_message = second_websocket.next().await.unwrap().unwrap();
|
||||
second_websocket
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": "bypass retry recovered"
|
||||
"delta": "bypass delayed output"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_bypass_retry_recovered", 3, 1).into(),
|
||||
completed_websocket_response("resp_bypass_delayed", 3, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second_websocket.close(None).await.unwrap();
|
||||
drop(first_websocket);
|
||||
first_websocket.close(None).await.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig {
|
||||
max_per_account: 0,
|
||||
first_token_timeout: Some(Duration::from_millis(30)),
|
||||
initial_event_timeout: Some(Duration::from_millis(30)),
|
||||
..websocket_pool_config_for_tests(None, None, None)
|
||||
}));
|
||||
let backend = CodexBackendClient::new(
|
||||
@@ -402,31 +392,31 @@ async fn codex_backend_client_stream_should_retry_bypass_socket_when_first_outpu
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let request = pooled_websocket_request("conversation-first-token-bypass-timeout");
|
||||
let request = pooled_websocket_request("conversation-structural-bypass");
|
||||
|
||||
let response = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_first_token_bypass_timeout", Some("chatgpt-account")),
|
||||
request_context("req_structural_bypass", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("bypass first-token timeout should retry before returning stream");
|
||||
.expect("structural activity should keep the bypass websocket open");
|
||||
let decision = response.websocket_pool_decision;
|
||||
let mut stream = response.body;
|
||||
let mut body = String::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.expect("retried bypass stream chunk should be valid");
|
||||
let chunk = chunk.expect("delayed bypass stream chunk should be valid");
|
||||
body.push_str(std::str::from_utf8(&chunk).unwrap());
|
||||
}
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(body.contains("bypass retry recovered"));
|
||||
assert!(body.contains("resp_bypass_retry_recovered"));
|
||||
assert!(!body.contains("resp_bypass_first_token_stalled"));
|
||||
assert!(body.contains("resp_bypass_first_token_stalled"));
|
||||
assert!(body.contains("bypass delayed output"));
|
||||
assert!(body.contains("resp_bypass_delayed"));
|
||||
let decision = decision.expect("bypass decision");
|
||||
assert_eq!(decision.kind(), "bypass");
|
||||
assert_eq!(decision.reason(), Some("cap"));
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -464,6 +454,7 @@ async fn codex_backend_client_should_not_reuse_pooled_websocket_across_local_acc
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -631,7 +622,8 @@ async fn websocket_pool_should_release_fresh_reservation_when_prefetch_is_cancel
|
||||
crate::support::fingerprint::runtime_test_fingerprint(),
|
||||
)
|
||||
.with_websocket_pool(pool);
|
||||
let request = pooled_websocket_request("conversation-cancelled-prefetch");
|
||||
let mut request = pooled_websocket_request("conversation-cancelled-prefetch");
|
||||
request.stream_commit_policy = StreamCommitPolicy::UntilOutputOrTerminal;
|
||||
|
||||
{
|
||||
let first_request = backend.create_response_stream(
|
||||
@@ -756,7 +748,18 @@ async fn websocket_pool_should_bypass_new_keys_after_account_cap() {
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
let accepted_connections_for_server = Arc::clone(&accepted_connections);
|
||||
let server = tokio::spawn(async move {
|
||||
for response_id in ["resp_cap_first", "resp_cap_second", "resp_cap_third"] {
|
||||
let (first_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut first_websocket = accept_codex_test_websocket(first_stream).await;
|
||||
let _first_message = first_websocket.next().await.unwrap().unwrap();
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_cap_first", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for response_id in ["resp_cap_second", "resp_cap_third"] {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
@@ -767,10 +770,9 @@ async fn websocket_pool_should_bypass_new_keys_after_account_cap() {
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
if response_id == "resp_cap_third" {
|
||||
websocket.close(None).await.unwrap();
|
||||
}
|
||||
websocket.close(None).await.unwrap();
|
||||
}
|
||||
drop(first_websocket);
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(1, Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
@@ -1050,6 +1052,7 @@ async fn codex_backend_client_should_keep_idle_pooled_websocket_alive_across_rep
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool-background".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -1156,6 +1159,7 @@ async fn codex_backend_client_should_close_idle_pooled_websocket_when_account_is
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool-evict".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -1260,6 +1264,7 @@ async fn codex_backend_client_should_stop_reusing_pooled_websockets_after_shutdo
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool-shutdown".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -1364,6 +1369,7 @@ async fn codex_backend_client_should_close_idle_pooled_websocket_after_liveness_
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool-liveness".to_string()));
|
||||
|
||||
let first = backend
|
||||
@@ -1459,6 +1465,7 @@ async fn codex_backend_client_should_discard_pooled_websocket_after_upstream_err
|
||||
Vec::new(),
|
||||
);
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.set_prompt_cache_key(Some("conversation-pool".to_string()));
|
||||
|
||||
let first_error = backend
|
||||
@@ -1487,17 +1494,17 @@ async fn codex_backend_client_should_discard_pooled_websocket_after_upstream_err
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_reuse_pooled_websocket_after_unmapped_response_failed() {
|
||||
async fn codex_backend_client_should_discard_pooled_websocket_after_unmapped_response_failed() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
let accepted_connections_for_server = Arc::clone(&accepted_connections);
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let (first_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _first_message = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
let mut first_websocket = accept_codex_test_websocket(first_stream).await;
|
||||
let _first_message = first_websocket.next().await.unwrap().unwrap();
|
||||
first_websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
@@ -1516,14 +1523,17 @@ async fn codex_backend_client_should_reuse_pooled_websocket_after_unmapped_respo
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let _second_message = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
let (second_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut second_websocket = accept_codex_test_websocket(second_stream).await;
|
||||
let _second_message = second_websocket.next().await.unwrap().unwrap();
|
||||
second_websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_pool_after_unmapped_failed", 5, 2).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
second_websocket.close(None).await.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(8, Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
@@ -1547,10 +1557,11 @@ async fn codex_backend_client_should_reuse_pooled_websocket_after_unmapped_respo
|
||||
request_context("req_pool_after_unmapped_failed", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("terminal failed websocket should be reusable");
|
||||
.expect("terminal failed websocket should be replaced");
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(first.body.contains("resp_pool_model_refusal"));
|
||||
assert!(second.body.contains("resp_pool_after_unmapped_failed"));
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(second.websocket_pool_decision.unwrap().kind(), "new");
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,940 @@
|
||||
# Responses WebSocket previous_response_id 低延迟恢复实施方案
|
||||
|
||||
日期:2026-07-11
|
||||
|
||||
目标基线:edbedd31ffb502b432e3c61def2ba41b51e858cd
|
||||
|
||||
文档状态:P0/P1 已实施并通过本地完整测试;真实链路验证待执行
|
||||
|
||||
适用问题:
|
||||
|
||||
stream disconnected before completion:
|
||||
Upstream stream closed before response.completed:
|
||||
previous_response_not_found
|
||||
|
||||
## 1. 结论
|
||||
|
||||
不要把本问题实现成 GPT-5.6 模型特判,也不要直接照搬提交
|
||||
1b5bfb37e171e7478f3d4c666100d03a501da4ff 的全局预取改法。
|
||||
|
||||
推荐实现由三个部分组成:
|
||||
|
||||
1. 在 WebSocket 连接对象自身记录该连接当前可以续接的最新 response ID。
|
||||
2. 发送 store=false + previous_response_id 前,先验证当前取出的池连接是否仍持有这个 ID。
|
||||
3. 只有代理自己做过隐式续接、手里保留完整输入快照时,才允许清除旧 ID 并透明重放;客户端显式传入但代理没有完整历史时,返回明确错误。
|
||||
|
||||
流式预取不能对所有请求统一等待真实输出。应根据请求是否具备安全重放能力选择提交边界:
|
||||
|
||||
- 普通请求、显式 previous_response_id:保持原有低延迟提交行为。
|
||||
- 代理生成的隐式续接:在首个不可逆输出或终态前暂存结构帧,以便失败时安全重放。
|
||||
|
||||
这个设计比“连接代次写入 Redis”更直接。OpenAI 的缓存语义是“一条具体 WebSocket 上最近的 response”,因此最准确的真相源就是池中的连接对象,而不是账号亲和表、模型名或持久化数据库。
|
||||
|
||||
## 2. 基线说明
|
||||
|
||||
本文以 `edbedd31ffb502b432e3c61def2ba41b51e858cd` 作为事故复现与缺陷定位基线;P0/P1 实现落在后续依赖治理基线 `977d94d5812eebe57b319b4f00a99af485d28040` 之上。
|
||||
|
||||
不要直接 cherry-pick 1b5bfb37:
|
||||
|
||||
- 它的父提交是 ddd6fe160f64c40df15f6260ed2513ead54e04e6,不是目标基线。
|
||||
- 它同时包含依赖、配置、监控、测试整理和文档变化。
|
||||
- 它把所有 WebSocket 流都推进到“真实输出或终态”才返回,存在不必要的首包延迟和首输出超时语义变化。
|
||||
|
||||
## 3. 已确认的生产事实
|
||||
|
||||
### 3.1 故障范围
|
||||
|
||||
线上 previous_response_not_found 按模型统计:
|
||||
|
||||
| 模型 | 错误数 | 结论 |
|
||||
| --- | ---: | --- |
|
||||
| gpt-5.6-sol | 116 | 当前唯一观测到错误的模型 |
|
||||
| 其他模型 | 0 | 不能据此证明相同连接条件下不会失败 |
|
||||
|
||||
116 次失败全部满足:
|
||||
|
||||
model = gpt-5.6-sol
|
||||
transport = websocket
|
||||
websocket_pool_kind = new
|
||||
previous_response_id present = true
|
||||
store = false
|
||||
|
||||
成功请求的关键对照:
|
||||
|
||||
| 模型 | previous_response_id + reuse 成功 | previous_response_id + new 成功 |
|
||||
| --- | ---: | ---: |
|
||||
| gpt-5.5 | 162 | 0 |
|
||||
| gpt-5.6-sol | 117 | 0 |
|
||||
|
||||
因此目前只能确认“故障在 GPT-5.6 上被观测到”,不能确认“GPT-5.5 在相同的 new + previous_response_id + store=false 条件下会成功”。线上没有这个对照样本。
|
||||
|
||||
### 3.2 本次 response ID 的时间线
|
||||
|
||||
失效 ID:
|
||||
|
||||
resp_0de25f0dd99612a3016a51087f8b5c8194b20293e08fb6aa5f
|
||||
|
||||
关键请求:
|
||||
|
||||
| 北京时间 | 请求 ID | 事实 |
|
||||
| --- | --- | --- |
|
||||
| 22:58:06 | req_3090a40b-200e-4997-a910-01d522974a59 | gpt-5.6-sol,复用 WebSocket,正常完成并产生上述 response ID |
|
||||
| 22:58:30 | 同上 | completed=true,store=false,首输出约 5599ms |
|
||||
| 22:58:31 | req_aee4bb05-d67e-454e-9913-10a5db5afd64 | 同账号、同 conversation hash、同 pool key,再次复用;HTTP response 已返回,但没有流终结记录 |
|
||||
| 23:00:22 起 | 后续重试 | 连接已经变成 new,继续携带旧 response ID |
|
||||
| 23:05:16 | req_0b171310-8b4e-4b0b-90fa-fd27d334eda9 | new WebSocket,返回嵌套的 previous_response_not_found |
|
||||
| 23:05:22 | req_215248f4-d0ed-44c2-a565-4a6fdd393bee | 同样失败 |
|
||||
|
||||
原响应和失败请求使用同一账号、同一 conversation hash、同一 pool key。账号轮换不是根因。
|
||||
|
||||
### 3.3 首包与首输出数据
|
||||
|
||||
线上两个 Rust 容器均未覆盖 first_token_timeout_ms,因此目标基线会使用默认 20000ms。
|
||||
|
||||
最近三天 5722 条 gpt-5.6-sol 成功流记录:
|
||||
|
||||
| 指标 | first_token_ms |
|
||||
| --- | ---: |
|
||||
| p50 | 2716ms |
|
||||
| p95 | 6929ms |
|
||||
| p99 | 11275ms |
|
||||
| 大于 15 秒 | 21 条 |
|
||||
| 大于 20 秒 | 9 条 |
|
||||
| 最大值 | 37334ms |
|
||||
|
||||
first_token_ms 与 WebSocket 内部熔断的起算点不完全相同,因此不能断言这 9 条都会触发超时;但足以证明不能把“等待真实输出”直接套进现有 20 秒超时而不做语义拆分。
|
||||
|
||||
故障请求中,上游错误通常在服务端返回 HTTP response 后约 39 至 75ms 到达。对可恢复请求多等待这几十毫秒可以捕获错误,但把同一等待策略扩大到所有正常请求是不必要的。
|
||||
|
||||
## 4. 官方协议依据
|
||||
|
||||
OpenAI WebSocket Mode 文档明确说明:
|
||||
|
||||
- 活跃 WebSocket 只在连接本地内存缓存最近一条 previous response。
|
||||
- store=false 没有持久化回退。
|
||||
- ID 不在连接缓存时返回 previous_response_not_found。
|
||||
- 连接关闭后,如果不能从持久化响应续接,应清空 previous_response_id 并发送完整输入。
|
||||
|
||||
参考:
|
||||
|
||||
- https://developers.openai.com/api/docs/guides/websocket-mode/
|
||||
- https://developers.openai.com/api/docs/guides/conversation-state/
|
||||
|
||||
官方 Codex 0.144.1 的实现也把增量状态绑定到当前 WebSocket session:
|
||||
|
||||
- 构造增量请求:
|
||||
https://github.com/openai/codex/blob/rust-v0.144.1/codex-rs/core/src/client.rs#L1223-L1253
|
||||
- 创建新连接前清空 last_request 和 last_response_rx:
|
||||
https://github.com/openai/codex/blob/rust-v0.144.1/codex-rs/core/src/client.rs#L1323-L1354
|
||||
- 连接错误后新连接发送完整输入的测试:
|
||||
https://github.com/openai/codex/blob/rust-v0.144.1/codex-rs/core/tests/suite/client_websockets.rs#L1986-L2073
|
||||
|
||||
官方行为是协议级状态管理,没有 GPT-5.6 模型分支。
|
||||
|
||||
## 5. 目标基线的缺陷位置
|
||||
|
||||
目标提交 edbedd31 中的相关入口:
|
||||
|
||||
| 文件 | 当前职责 | 缺陷 |
|
||||
| --- | --- | --- |
|
||||
| backend/src/upstream/openai/transport/websocket_pool.rs | 按账号和 conversation 保存池连接 | PooledWebSocketConnection 不记录该连接缓存的最新 response ID |
|
||||
| backend/src/upstream/openai/transport/websocket.rs | acquire、复用、新建和 stale-reuse 重试 | 新连接仍原样发送 store=false 的旧 previous_response_id |
|
||||
| backend/src/upstream/openai/transport/websocket/stream.rs | 预取和 live stream 转发 | 首内容、连接健康、真实输出和下游提交共用近似语义 |
|
||||
| backend/src/upstream/openai/protocol/websocket.rs | WebSocket 帧解析 | 错误分类丢失结构化 code;没有提取 completed response ID 的公共函数 |
|
||||
| backend/src/dispatch/responses/service.rs | 隐式续接与 history recovery | recover_request_history 会在没有完整快照时直接删除客户端显式 history |
|
||||
| backend/src/dispatch/responses/service/stream.rs | 流式重试循环 | 结构帧可能先越过预取边界,错误进入 live stream 后无法回到调度循环 |
|
||||
| backend/src/dispatch/responses/service/complete.rs | 非流式重试循环 | 显式 previous ID 也可能被无损性不足的 strip-history 重试 |
|
||||
| backend/src/dispatch/errors.rs | 错误分类 | previous_response_not_found 依赖正文字符串扫描 |
|
||||
| backend/src/dispatch/responses/errors.rs | 对外调度错误 | 没有独立 HistoryUnavailable 类型 |
|
||||
| backend/src/api/client/errors.rs | OpenAI 兼容错误输出 | 无稳定 previous_response_unavailable 错误码 |
|
||||
|
||||
目标基线的 recover_request_history:
|
||||
|
||||
1. 忘记 affinity 中的旧 response ID。
|
||||
2. 如果有 ImplicitResumeSnapshot,则恢复完整输入。
|
||||
3. 无论是否有快照,都清除 previous_response_id 和 turn_state。
|
||||
4. 重试同一请求。
|
||||
|
||||
第 3 步对客户端显式增量请求是不安全的。客户端可能只发送了“本轮新增 input”,删除 ID 后直接上送会丢失全部历史。
|
||||
|
||||
## 6. 根因模型
|
||||
|
||||
完成响应 A,store=false
|
||||
-> 上游 WebSocket W1 内存缓存 latest_response_id = A
|
||||
-> 代理 session affinity 只记住账号和 conversation
|
||||
-> 下一轮下游取消,代理销毁 W1
|
||||
-> W1 的连接本地缓存随之消失
|
||||
-> 客户端仍携带 A
|
||||
-> 连接池没有 W1,只能创建 W2
|
||||
-> 当前代码把 A 原样发给 W2
|
||||
-> 上游返回 previous_response_not_found
|
||||
-> 非输出结构帧先提交 HTTP/SSE
|
||||
-> 错误只能包装成 stream_disconnected
|
||||
|
||||
需要修复两个独立问题:
|
||||
|
||||
1. 不允许把连接本地 ID 发到错误连接。
|
||||
2. 只有确实拥有完整上下文时才允许透明重放。
|
||||
|
||||
## 7. 必须保持的设计不变量
|
||||
|
||||
1. store=false 的 previous_response_id 只能发送到明确缓存该 ID 的同一池连接。
|
||||
2. store=true 可以在新连接上尝试持久化 hydration,不做本地提前拒绝。
|
||||
3. session affinity 只负责选账号和 conversation,不负责证明 response ID 在某条连接上可用。
|
||||
4. 连接对象是 connection-local previous response 状态的唯一真相源。
|
||||
5. 透明重放必须有完整输入快照作为能力凭证。
|
||||
6. 客户端显式 previous ID 且没有完整历史时,禁止删除 ID 后重试。
|
||||
7. 真实输出、工具调用参数或有效 output item 已经发给客户端后,禁止透明重试。
|
||||
8. 同一个入站请求最多做一次 history replay。
|
||||
9. 不通过 model == gpt-5.6 判断行为。
|
||||
10. 不把 store=false 静默改成 store=true。
|
||||
11. 不把 response ID 原文写入普通日志、错误消息或 WebSocket audit。
|
||||
12. 正常请求不能因为修复而统一推迟到真实 token 才返回 HTTP response。
|
||||
|
||||
## 8. 方案比较
|
||||
|
||||
### 8.1 GPT-5.6 模型特判
|
||||
|
||||
做法:仅在模型为 gpt-5.6-sol 时捕获错误或删除 ID。
|
||||
|
||||
拒绝原因:
|
||||
|
||||
- 官方语义属于 WebSocket 和 store,不属于模型。
|
||||
- gpt-5.5 没有 new + previous_response_id 的线上对照样本。
|
||||
- 后续模型会继续复制同一协议行为。
|
||||
- 模型别名、路由后缀和新模型会形成维护表。
|
||||
|
||||
### 8.2 所有流统一预取到真实输出或终态
|
||||
|
||||
做法:把 prefetch_first_content_stream_frames 改为对所有请求等待真实输出或终态。
|
||||
|
||||
优点:实现直接,能够在 HTTP response 返回前捕获结构帧之后的错误。
|
||||
|
||||
拒绝作为最终方案的原因:
|
||||
|
||||
- 正常请求的 response.created 或结构事件可能被延迟。
|
||||
- 会把现有 20 秒首内容熔断变成更严格的真实输出熔断。
|
||||
- 线上已有成功请求的首输出超过 20 秒。
|
||||
- 对客户端显式、不可重放的请求,即使提前捕获错误也无法透明恢复。
|
||||
|
||||
### 8.3 连接本地状态 + 能力感知预取
|
||||
|
||||
这是本文推荐方案:
|
||||
|
||||
- 池连接记录 latest_response_id。
|
||||
- 发送前主动验证 continuation。
|
||||
- 只有 Replayable 请求使用 UntilOutputOrTerminal 预取。
|
||||
- 其他请求保持 FirstForwardableContent。
|
||||
- 上游仍返回 previous_response_not_found 时作为后备保护。
|
||||
|
||||
## 9. 推荐的数据结构
|
||||
|
||||
### 9.1 WebSocketContinuationRequirement
|
||||
|
||||
在 backend/src/upstream/openai/transport/websocket/types.rs 增加:
|
||||
|
||||
enum WebSocketContinuationRequirement {
|
||||
NewChain,
|
||||
Persisted { response_id: String },
|
||||
ConnectionLocal { response_id: String },
|
||||
}
|
||||
|
||||
构造规则:
|
||||
|
||||
previous_response_id 为空
|
||||
-> NewChain
|
||||
|
||||
previous_response_id 存在且 store=true
|
||||
-> Persisted
|
||||
|
||||
previous_response_id 存在且 store=false
|
||||
-> ConnectionLocal
|
||||
|
||||
把该字段加入 CodexWebSocketRequest。它是本地控制信息,不进入上游 JSON。
|
||||
|
||||
不要在 transport 中重新解析 payload_text 获取 store 和 previous ID。构造 prepared request 时已经持有 CodexResponsesRequest,应在这里一次性生成强类型 requirement。
|
||||
|
||||
### 9.2 WebSocketContinuationState
|
||||
|
||||
在 backend/src/upstream/openai/transport/websocket_pool.rs 增加:
|
||||
|
||||
struct WebSocketContinuationState {
|
||||
latest_response_id: Option<String>,
|
||||
}
|
||||
|
||||
把它作为独立字段加入 PooledWebSocketConnection:
|
||||
|
||||
struct PooledWebSocketConnection {
|
||||
websocket: PumpedWebSocket,
|
||||
metadata: CodexWebSocketConnectionMetadata,
|
||||
continuation: WebSocketContinuationState,
|
||||
created_at: Instant,
|
||||
}
|
||||
|
||||
不要把 latest_response_id 塞入 CodexWebSocketConnectionMetadata。握手 metadata、限流头、turn state 与协议续接状态生命周期不同,独立类型更清楚。
|
||||
|
||||
状态只在收到并成功校验 response.completed 后更新。以下情况不能记录:
|
||||
|
||||
- response.created
|
||||
- response.in_progress
|
||||
- response.failed
|
||||
- error
|
||||
- response.incomplete
|
||||
- completed 帧解析失败
|
||||
- 下游发送失败导致连接被 discard
|
||||
- 上游在 terminal 前关闭
|
||||
|
||||
每条连接只保存一个最新 ID,正好对应官方“一条连接缓存最近一条 previous response”的语义。不需要 response ID 集合。
|
||||
|
||||
### 9.3 PreviousResponseUnavailableReason
|
||||
|
||||
增加内部原因枚举:
|
||||
|
||||
enum PreviousResponseUnavailableReason {
|
||||
PoolUnavailable,
|
||||
FreshConnectionRequired,
|
||||
ConnectionBusy,
|
||||
LatestResponseMismatch,
|
||||
ReusedConnectionLost,
|
||||
UpstreamRejected,
|
||||
}
|
||||
|
||||
Display 不得包含 response ID 原文。
|
||||
|
||||
对外错误统一为:
|
||||
|
||||
type: invalid_request_error
|
||||
code: previous_response_unavailable
|
||||
status: 400
|
||||
message: Previous response context is no longer available. Start a new chain and resend the full input context.
|
||||
|
||||
ConnectionBusy 不表示历史已经丢失,应单独返回可重试错误:
|
||||
|
||||
status: 503
|
||||
code: continuation_connection_busy
|
||||
|
||||
不得为了避开 Busy 在新连接上发送 store=false 的旧 ID。
|
||||
|
||||
### 9.4 HistoryRecoveryPlan
|
||||
|
||||
在 dispatch 层用显式类型代替“Option 是否刚好为 Some”的隐含约定:
|
||||
|
||||
enum HistorySource {
|
||||
None,
|
||||
Explicit,
|
||||
Implicit,
|
||||
}
|
||||
|
||||
struct HistoryRecoveryPlan {
|
||||
source: HistorySource,
|
||||
replay: Option<ImplicitResumeSnapshot>,
|
||||
}
|
||||
|
||||
规则:
|
||||
|
||||
- 客户端原请求带 previous_response_id:source=Explicit,replay=None。
|
||||
- 代理根据完整输入自行应用 implicit resume:source=Implicit,replay=Some(snapshot)。
|
||||
- 没有 history:source=None,replay=None。
|
||||
|
||||
take_replay 消耗 Option,天然保证最多恢复一次。
|
||||
|
||||
### 9.5 StreamCommitPolicy
|
||||
|
||||
增加:
|
||||
|
||||
enum StreamCommitPolicy {
|
||||
FirstForwardableContent,
|
||||
UntilOutputOrTerminal,
|
||||
}
|
||||
|
||||
选择规则:
|
||||
|
||||
recovery_plan.replay.is_some()
|
||||
-> UntilOutputOrTerminal
|
||||
|
||||
其他
|
||||
-> FirstForwardableContent
|
||||
|
||||
恢复后完整输入请求已经不带 previous ID,下一次尝试必须回到 FirstForwardableContent。
|
||||
|
||||
## 10. 连接池发送前验证
|
||||
|
||||
在 execute_response_create_request_with_pool 和
|
||||
execute_response_create_request_stream_with_pool 中,必须在 send 之前执行验证。
|
||||
|
||||
行为矩阵:
|
||||
|
||||
| requirement | acquire 结果 | 行为 |
|
||||
| --- | --- | --- |
|
||||
| NewChain | Reused | 允许发送 |
|
||||
| NewChain | FreshReserved / Bypass | 保持现有行为 |
|
||||
| Persisted | 任意 | 允许发送,由上游 hydration 决定 |
|
||||
| ConnectionLocal | Reused 且 latest ID 相同 | 允许发送 |
|
||||
| ConnectionLocal | Reused 但 latest ID 不同 | 将连接原样 put 回池,返回 LatestResponseMismatch |
|
||||
| ConnectionLocal | FreshReserved | discard 预留 slot,不建连,返回 FreshConnectionRequired |
|
||||
| ConnectionLocal | Bypass(Busy) | 不建新连接,返回 ConnectionBusy |
|
||||
| ConnectionLocal | Bypass(Disabled/Cap) | 不建新连接,返回 PoolUnavailable |
|
||||
| ConnectionLocal | pool=None | 不建新连接,返回 PoolUnavailable |
|
||||
|
||||
关键点:
|
||||
|
||||
- mismatch 时不能销毁一条仍然健康的连接;它可能仍可服务携带 latest ID 的正确请求。
|
||||
- FreshReserved 已在池里放入 Busy 占位,提前返回前必须 discard,避免 slot 永久卡住。
|
||||
- 验证必须同时覆盖流式和非流式入口。
|
||||
|
||||
## 11. stale-reuse 重试规则
|
||||
|
||||
目标基线当前行为:
|
||||
|
||||
复用连接在首帧前死亡或首 token 超时
|
||||
-> 使用同一个 payload 新建 WebSocket 重试
|
||||
|
||||
这个逻辑对 NewChain 合理,对 store=false + previous_response_id 错误。
|
||||
|
||||
修改为:
|
||||
|
||||
| 请求类型 | 复用连接失效后的行为 |
|
||||
| --- | --- |
|
||||
| NewChain | 保持现有 fresh retry |
|
||||
| Persisted | 可以 fresh retry |
|
||||
| ConnectionLocal | 禁止 transport 原样 fresh retry,返回 ReusedConnectionLost |
|
||||
|
||||
ReusedConnectionLost 交给 dispatch:
|
||||
|
||||
- HistoryRecoveryPlan 有 replay:恢复完整请求并重试。
|
||||
- 无 replay:返回 previous_response_unavailable。
|
||||
|
||||
这样 transport 不需要理解完整上下文,dispatch 也不会错误猜测连接状态。
|
||||
|
||||
## 12. 成功响应如何更新 latest_response_id
|
||||
|
||||
### 12.1 协议解析
|
||||
|
||||
在 backend/src/upstream/openai/protocol/websocket.rs 增加:
|
||||
|
||||
websocket_response_completed_id(raw: &str) -> Result<Option<String>, String>
|
||||
|
||||
行为:
|
||||
|
||||
- 非 response.completed 返回 Ok(None)。
|
||||
- completed 先沿用现有 ResponseCompleted 强类型校验。
|
||||
- 校验成功后返回 response.id。
|
||||
- id 缺失或为空返回解析错误。
|
||||
|
||||
不要通过搜索原始字符串 resp_ 提取。
|
||||
|
||||
### 12.2 非流式 WebSocket
|
||||
|
||||
collect_websocket_response 接收并返回 WebSocketContinuationState。
|
||||
|
||||
读取到 response.completed:
|
||||
|
||||
1. 校验 completed shape。
|
||||
2. 提取 ID。
|
||||
3. continuation.latest_response_id = Some(id)。
|
||||
4. 返回 exchange、websocket、metadata、continuation。
|
||||
5. pool.put 时把 continuation 放回 PooledWebSocketConnection。
|
||||
|
||||
### 12.3 流式 WebSocket
|
||||
|
||||
WebSocketStreamPoolReturn 增加 continuation 字段。
|
||||
|
||||
forward_websocket_response_stream 读取到 response.completed 时:
|
||||
|
||||
1. 在发送 terminal SSE 前完成 ID 校验和状态更新。
|
||||
2. 如果下游发送成功,finish_stream_websocket 把新状态放回池。
|
||||
3. 如果下游发送失败,discard 连接,不保留状态。
|
||||
|
||||
新建池连接的 continuation 为 default;复用连接沿用取出的 continuation。
|
||||
|
||||
## 13. 错误必须结构化分类
|
||||
|
||||
目标基线的 is_history_recovery_signal 会扫描:
|
||||
|
||||
- previous_response_not_found
|
||||
- previous response + not found
|
||||
- no tool output found
|
||||
- invalid encrypted content
|
||||
|
||||
本问题不能继续依赖 message 文本。
|
||||
|
||||
修改 protocol/websocket.rs 的 ClassifiedWebSocketError,使其保留 error.code。至少精确识别:
|
||||
|
||||
previous_response_not_found
|
||||
|
||||
WebSocket error 事件中 code 的读取路径继续支持现有官方形状:
|
||||
|
||||
/response/error/code
|
||||
/error/code
|
||||
|
||||
ResponsesSseFailure 已有 upstream_code。增加精确函数:
|
||||
|
||||
fn is_previous_response_not_found_failure(failure: &ResponsesSseFailure) -> bool {
|
||||
failure.upstream_code.as_deref() == Some("previous_response_not_found")
|
||||
}
|
||||
|
||||
消息字符串只能用于诊断展示,不能决定是否清除历史和重放。
|
||||
|
||||
invalid_encrypted_content 和 tool output 缺失属于不同失败种类,可以继续走各自恢复策略,但不要再与 previous_response_not_found 合并为一个模糊 bool。
|
||||
|
||||
建议增加:
|
||||
|
||||
enum HistoryFailureKind {
|
||||
PreviousResponseNotFound,
|
||||
InvalidEncryptedContent,
|
||||
MissingToolOutput,
|
||||
}
|
||||
|
||||
## 14. dispatch 安全恢复
|
||||
|
||||
### 14.1 调整快照捕获时机
|
||||
|
||||
目标基线 apply_implicit_resume 在完成 affinity 查询前就 clone 完整 input。
|
||||
|
||||
将 ImplicitResumeSnapshot::capture(request) 移到所有资格检查通过之后、第一次修改 request 之前:
|
||||
|
||||
1. 找 continuation_start。
|
||||
2. 查 conversation、variant、instructions、function call、account。
|
||||
3. 构造 replay input。
|
||||
4. 此时 capture 原始完整请求。
|
||||
5. 再 set_previous_response_id、set_input、修改 transport。
|
||||
|
||||
这样未命中隐式续接时不会无意义 clone 大输入。
|
||||
|
||||
### 14.2 替换 recover_request_history
|
||||
|
||||
删除“无快照也 strip history”的行为。
|
||||
|
||||
新函数语义:
|
||||
|
||||
async fn try_replay_full_request(
|
||||
request,
|
||||
recovery_plan,
|
||||
account_id,
|
||||
evict_reasoning_replay,
|
||||
) -> bool
|
||||
|
||||
流程:
|
||||
|
||||
1. recovery_plan.take_replay()。
|
||||
2. 没有 snapshot,返回 false,不修改请求。
|
||||
3. 必要时清除 reasoning replay。
|
||||
4. 忘记失效 response ID 的 session affinity。
|
||||
5. snapshot.restore(request)。
|
||||
6. 清除 previous_response_id 和 turn_state。
|
||||
7. 返回 true。
|
||||
|
||||
只有返回 true 才能 continue 调度循环。
|
||||
|
||||
### 14.3 显式 previous ID
|
||||
|
||||
客户端显式 previous ID 失败时:
|
||||
|
||||
- 不删除 ID 后重试。
|
||||
- 不切换账号后用残缺输入重试。
|
||||
- 释放账号 lease。
|
||||
- 返回 ResponseDispatchError::HistoryUnavailable。
|
||||
|
||||
非流式响应:
|
||||
|
||||
HTTP 400
|
||||
type = invalid_request_error
|
||||
code = previous_response_unavailable
|
||||
|
||||
流式请求如果尚未开始下游 body:
|
||||
|
||||
保持 Responses 流式兼容形状
|
||||
event = response.failed
|
||||
code = previous_response_unavailable
|
||||
|
||||
不得把完整上游 JSON 嵌套进 stream_disconnected message 返回给客户端。
|
||||
|
||||
### 14.4 同账号重放
|
||||
|
||||
有快照的恢复必须固定到刚才使用的账号:
|
||||
|
||||
next_required_account_id = Some(release_account_id)
|
||||
|
||||
原因:
|
||||
|
||||
- reasoning replay 和 account scope 相关。
|
||||
- 避免恢复动作变成账号轮换。
|
||||
- 与现有 session affinity 语义一致。
|
||||
|
||||
## 15. 流式提交和超时必须拆成三个概念
|
||||
|
||||
不要再用 saw_content_frame 同时表示三个状态。
|
||||
|
||||
至少拆为:
|
||||
|
||||
saw_upstream_activity
|
||||
saw_irreversible_output
|
||||
downstream_commit_ready
|
||||
|
||||
含义:
|
||||
|
||||
| 状态 | 用途 |
|
||||
| --- | --- |
|
||||
| saw_upstream_activity | 判断连接是否已经有响应,切换初始等待与 active idle timeout |
|
||||
| saw_irreversible_output | 判断透明重试是否还安全,并记录真实 TTFT |
|
||||
| downstream_commit_ready | 由 StreamCommitPolicy 决定何时返回下游 Response |
|
||||
|
||||
事件分类:
|
||||
|
||||
| 事件 | activity | irreversible output | terminal |
|
||||
| --- | --- | --- | --- |
|
||||
| response.created | 是 | 否 | 否 |
|
||||
| response.in_progress | 是 | 否 | 否 |
|
||||
| response.content_part.added 且文本为空 | 是 | 否 | 否 |
|
||||
| metadata / rate_limits | 是 | 否 | 否 |
|
||||
| 非空 reasoning delta | 是 | 是 | 否 |
|
||||
| 非空 output_text delta | 是 | 是 | 否 |
|
||||
| function_call_arguments delta | 是 | 是 | 否 |
|
||||
| 有效 output_item.added | 是 | 是 | 否 |
|
||||
| response.completed | 是 | 否 | 是 |
|
||||
| response.failed / error | 是 | 否 | 是 |
|
||||
|
||||
### 15.1 超时调整
|
||||
|
||||
目标基线的 first_token_timeout_ms 名字和行为已经混合“连接健康”与“真实输出”。
|
||||
|
||||
推荐在本次实现中拆分:
|
||||
|
||||
ws_pool:
|
||||
initial_event_timeout_ms: 20000
|
||||
first_output_timeout_ms: 0
|
||||
|
||||
语义:
|
||||
|
||||
- initial_event_timeout_ms:发出 response.create 后,等待第一个有效上游活动事件的时间。
|
||||
- first_output_timeout_ms:等待不可逆输出的可选业务上限;0 表示禁用。
|
||||
- 收到任何有效活动后使用现有 active stream idle timeout。
|
||||
|
||||
生产配置目前没有显式 first_token_timeout_ms,因此可以直接把示例和默认配置迁移到新字段,不需要保留旧字段兼容分支。
|
||||
|
||||
如果暂时不想改配置结构,最低要求也是:
|
||||
|
||||
- guarded prefetch 收到 response.created 或其他结构事件后,不得继续使用 20 秒“无活动”熔断。
|
||||
- 之后使用 active idle timeout 等待 output 或 terminal。
|
||||
|
||||
不要简单把默认值从 20 秒拍脑袋改成 60 秒,这只是在掩盖两个不同概念。
|
||||
|
||||
## 16. 按文件实施步骤
|
||||
|
||||
### 第一步:协议与纯类型
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/upstream/openai/protocol/websocket.rs
|
||||
- backend/src/upstream/openai/transport/websocket/types.rs
|
||||
|
||||
增加:
|
||||
|
||||
- completed response ID 提取。
|
||||
- 结构化 WebSocket error code。
|
||||
- WebSocketContinuationRequirement。
|
||||
- PreviousResponseUnavailableReason。
|
||||
- StreamCommitPolicy。
|
||||
|
||||
先写纯单元测试,再进入连接池代码。
|
||||
|
||||
### 第二步:连接池 continuation 状态
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/upstream/openai/transport/websocket_pool.rs
|
||||
- backend/tests/upstream/openai/transport/websocket_pool.rs
|
||||
|
||||
增加 WebSocketContinuationState,并覆盖 acquire、put、discard、过期关闭的状态生命周期。
|
||||
|
||||
### 第三步:transport 发送前校验
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/upstream/openai/transport/websocket.rs
|
||||
- backend/src/upstream/openai/transport/client/requests.rs
|
||||
- backend/src/upstream/openai/transport/client.rs
|
||||
|
||||
确保流式、非流式、pool=None、FreshReserved、Bypass 和 stale-reuse 全部分支遵循行为矩阵。
|
||||
|
||||
### 第四步:完成事件更新连接状态
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/upstream/openai/transport/websocket/stream.rs
|
||||
- backend/src/upstream/openai/transport/websocket.rs
|
||||
|
||||
让聚合和 live stream 两条路径都只在合法 response.completed 后更新 latest ID。
|
||||
|
||||
### 第五步:dispatch 恢复能力
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/dispatch/implicit_resume.rs
|
||||
- backend/src/dispatch/responses/service.rs
|
||||
- backend/src/dispatch/responses/service/complete.rs
|
||||
- backend/src/dispatch/responses/service/stream.rs
|
||||
- backend/src/dispatch/responses/sse_failure.rs
|
||||
|
||||
引入 HistoryRecoveryPlan,延后快照 clone,删除显式 history 的不安全 strip-and-retry。
|
||||
|
||||
### 第六步:客户端错误和可观测性
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/dispatch/errors.rs
|
||||
- backend/src/dispatch/responses/errors.rs
|
||||
- backend/src/api/client/errors.rs
|
||||
- backend/src/dispatch/responses/event_recording.rs
|
||||
- backend/src/dispatch/responses/stream_lifecycle.rs
|
||||
|
||||
增加 stable error code 和诊断字段。
|
||||
|
||||
### 第七步:超时语义
|
||||
|
||||
修改:
|
||||
|
||||
- backend/src/bootstrap/config.rs
|
||||
- backend/src/bootstrap/services.rs
|
||||
- backend/src/upstream/openai/transport/websocket_pool.rs
|
||||
- deploy/config.example.yaml
|
||||
|
||||
拆分 initial activity 和 first output timeout。不要保留两套字段别名。
|
||||
|
||||
## 17. 必须新增的测试
|
||||
|
||||
### 17.1 协议单元测试
|
||||
|
||||
1. response.completed 能提取 response.id。
|
||||
2. 非 completed 返回 None。
|
||||
3. completed 缺少 ID 返回解析错误。
|
||||
4. error.code=previous_response_not_found 精确分类。
|
||||
5. 仅 message 包含相同文字但 code 不匹配时,不触发自动恢复。
|
||||
|
||||
### 17.2 连接池测试
|
||||
|
||||
1. 首次成功响应后,池连接保存 latest response ID。
|
||||
2. 相同 ID 在同一连接上允许续接。
|
||||
3. 不同 ID 被拒绝,并把健康连接放回池。
|
||||
4. FreshReserved + connection-local ID 不建立 TCP/WebSocket。
|
||||
5. Busy + connection-local ID 不 bypass 到新连接。
|
||||
6. store=true previous ID 允许新连接 hydration。
|
||||
7. discard 后 continuation state 一起消失。
|
||||
8. max-age 过期后旧 ID 不可继续发送。
|
||||
|
||||
### 17.3 transport 集成测试
|
||||
|
||||
1. new chain 完成后,第二轮在同一 socket 发送 previous ID。
|
||||
2. 第二轮前关闭池连接,断言不会在新 socket 发送旧 ID。
|
||||
3. reused socket 在 send 前死亡,connection-local 请求不得 fresh retry 原 payload。
|
||||
4. 无 previous ID 的 stale-reuse 仍可 fresh retry。
|
||||
5. store=true 的 previous ID 可以在 fresh socket 发送。
|
||||
6. 上游没有收到被提前拒绝请求的 payload。
|
||||
|
||||
### 17.4 dispatch 恢复测试
|
||||
|
||||
1. implicit resume + 池连接丢失:直接恢复完整 input,只向上游发送一次完整请求。
|
||||
2. explicit previous ID + 池连接丢失:返回 previous_response_unavailable,上游零请求。
|
||||
3. implicit resume 在匹配连接上收到:
|
||||
|
||||
response.created
|
||||
response.content_part.added
|
||||
previous_response_not_found
|
||||
|
||||
断言结构事件未下发,完整请求重放一次。
|
||||
|
||||
4. explicit previous ID 收到同样错误:不重放。
|
||||
5. 真实 output_text.delta 后再失败:不重放。
|
||||
6. function call 参数产生后再失败:不重放。
|
||||
7. recovery plan 消耗后第二次错误不再重试。
|
||||
8. 恢复固定在原账号。
|
||||
9. 测试模型至少包含 gpt-5.6-sol,但实现断言不能检查模型名。
|
||||
|
||||
### 17.5 延迟和超时测试
|
||||
|
||||
使用 Tokio paused time 覆盖:
|
||||
|
||||
1. FirstForwardableContent 策略在结构内容到达时立即返回,不等真实 delta。
|
||||
2. UntilOutputOrTerminal 策略缓存结构内容。
|
||||
3. 收到 response.created 后等待 30 秒再输出,不触发 20 秒“无活动”超时。
|
||||
4. 真实输出到达后立即提交,额外本地开销不包含固定 sleep。
|
||||
5. terminal error 到达后立即返回恢复循环。
|
||||
|
||||
## 18. 可观测性
|
||||
|
||||
普通日志和 ops error metadata 增加:
|
||||
|
||||
history_source = none | explicit | implicit
|
||||
history_replay_available = true | false
|
||||
ws_continuation = new_chain | persisted | matched | missing | mismatch | busy
|
||||
recovery_action = none | replay_full | reject
|
||||
recovery_attempt = 0 | 1
|
||||
prefetch_policy = first_content | output_or_terminal
|
||||
upstream_request_sent = true | false
|
||||
initial_event_ms
|
||||
first_output_ms
|
||||
response_ready_ms
|
||||
|
||||
response ID 和 connection 标识只记录稳定短 hash,不记录原值。
|
||||
|
||||
这些内部诊断不得写成额外 usage_records。usage_records 只代表真实的 /v1 请求事实;恢复原因写入现有请求的 metadata 或 ops error log。
|
||||
|
||||
WebSocket audit 继续脱敏:
|
||||
|
||||
- instructions
|
||||
- input
|
||||
- previous_response_id
|
||||
- prompt_cache_key
|
||||
- client_metadata
|
||||
- tools
|
||||
|
||||
## 19. 延迟验收标准
|
||||
|
||||
需要分别比较:
|
||||
|
||||
request_start -> downstream HTTP response ready
|
||||
request_start -> first SSE event
|
||||
request_start -> first irreversible output
|
||||
upstream first output -> downstream first output
|
||||
|
||||
验收要求:
|
||||
|
||||
1. NewChain 和显式匹配续接的 response-ready 分布不得因修复系统性后移。
|
||||
2. upstream first output 到 downstream first output 不得出现固定等待窗口。
|
||||
3. normal success 的 p50/p95 TTFT 与基线差异应处于本地调度抖动范围。
|
||||
4. connection-local mismatch 应在建立新 WebSocket 前失败。
|
||||
5. replayable mismatch 应比“先请求上游再等 400”更快。
|
||||
6. first output 超过 20 秒但已有上游活动的请求不能被误判为死连接。
|
||||
|
||||
不要使用 50ms、100ms 之类固定 grace window 等待错误。生产错误碰巧在 39 至 75ms 内到达,不代表协议保证这个时序。
|
||||
|
||||
## 20. 验证命令
|
||||
|
||||
回退后先验证基线:
|
||||
|
||||
cargo test --manifest-path backend/Cargo.toml previous_response
|
||||
cargo test --manifest-path backend/Cargo.toml websocket_pool
|
||||
|
||||
开发中按阶段运行:
|
||||
|
||||
cargo test --manifest-path backend/Cargo.toml websocket_response_completed
|
||||
cargo test --manifest-path backend/Cargo.toml previous_response
|
||||
cargo test --manifest-path backend/Cargo.toml responses_websocket
|
||||
cargo test --manifest-path backend/Cargo.toml websocket_pool
|
||||
|
||||
最终验证:
|
||||
|
||||
cargo fmt --manifest-path backend/Cargo.toml -- --check
|
||||
cargo clippy --manifest-path backend/Cargo.toml --all-targets --all-features -- -D warnings
|
||||
cargo test --manifest-path backend/Cargo.toml
|
||||
bash backend/scripts/check_architecture.sh
|
||||
|
||||
如果测试中确认真实缺陷,先修复该缺陷并重新跑当前阶段,再进入下一组测试。
|
||||
|
||||
## 21. next 实例真实链路验证
|
||||
|
||||
不要直接切主实例。先在 next 实例开启 WebSocket audit:
|
||||
|
||||
CODEX_PROXY_WS_AUDIT_DIR=.runtime/ws-audit
|
||||
|
||||
场景矩阵:
|
||||
|
||||
| 场景 | 预期 |
|
||||
| --- | --- |
|
||||
| gpt-5.6-sol,同 socket 连续两轮 | 第二轮 matched,正常增量 |
|
||||
| 第一轮完成后主动 discard socket,第二轮为显式 ID | 本地 previous_response_unavailable,上游零请求 |
|
||||
| 第一轮完成后 discard,第二轮为代理 implicit resume | 直接发送完整 input,不发送旧 ID |
|
||||
| store=true,第一轮 socket 丢失 | 允许新连接尝试 hydration |
|
||||
| 同 conversation 并发导致 Busy | 不创建新连接发送 connection-local ID |
|
||||
| 结构事件后 upstream not found,且可重放 | 客户端不看到旧结构事件,完整重放成功 |
|
||||
| 首输出超过 20 秒但持续有生命周期活动 | 不因 initial activity timeout 失败 |
|
||||
| 真实输出后 upstream 错误 | 不透明重放 |
|
||||
|
||||
真实链路文档必须记录每个场景的 request ID、pool decision、是否真的发送上游、first output 和 response-ready 时间。
|
||||
|
||||
## 22. 部署观察
|
||||
|
||||
灰度时重点观察:
|
||||
|
||||
- previous_response_not_found 按 ws_continuation 原因分组。
|
||||
- previous_response_unavailable 数量。
|
||||
- recovery_action=replay_full 成功率。
|
||||
- upstream_request_sent=false 的提前拦截数量。
|
||||
- gpt-5.6-sol 的 response-ready、first-output p50/p95/p99。
|
||||
- ConnectionBusy 数量。
|
||||
- WebSocket pool discard reason。
|
||||
- 是否仍出现同一个 response ID 连续重试风暴。
|
||||
|
||||
不应观察到:
|
||||
|
||||
- gpt-5.6 专属分支命中日志。
|
||||
- 显式 previous ID 被 strip 后当作新链发送。
|
||||
- store=false 旧 ID 出现在 ws_pool=new 的上游 payload。
|
||||
- 正常请求统一延迟到文本 delta 才返回。
|
||||
- 因内部恢复产生额外 usage_records。
|
||||
|
||||
## 23. 回滚
|
||||
|
||||
本方案不引入数据库 schema,也不依赖 Redis 持久化,因此回滚只涉及二进制和配置。
|
||||
|
||||
如果采用新的超时字段,回滚旧版本前必须同步恢复旧配置字段,否则旧版本会因 deny_unknown_fields 拒绝启动。
|
||||
|
||||
回滚触发条件:
|
||||
|
||||
- 正常请求 response-ready p95 明显回退。
|
||||
- gpt-5.6-sol 首输出超时增加。
|
||||
- pool Busy 导致大量可用请求失败。
|
||||
- store=true hydration 被错误提前拒绝。
|
||||
- full replay 出现重复工具调用或重复输出。
|
||||
|
||||
## 24. 实施完成定义
|
||||
|
||||
只有全部满足才算修复完成:
|
||||
|
||||
- store=false 旧 ID 不会发送到新连接。
|
||||
- 连接只保存最近一个成功 completed response ID。
|
||||
- 显式增量请求没有完整历史时不会伪恢复。
|
||||
- 隐式续接丢失连接时能够用完整输入恢复。
|
||||
- 上游 exact code 结构化分类,不靠 message 判断本错误。
|
||||
- 已输出后不重试。
|
||||
- 正常请求不承担全局预取延迟。
|
||||
- initial activity 与 first output 超时语义已经分开。
|
||||
- 流式和非流式测试均覆盖。
|
||||
- GPT-5.6 真实链路验证通过。
|
||||
- ops 日志可解释每次 reject 或 replay。
|
||||
- 完整 fmt、clippy、test、architecture check 通过。
|
||||
|
||||
## 25. 参考工程的取舍
|
||||
|
||||
sub2api:
|
||||
|
||||
/home/zyy/桌面/Codes/sub2api/backend/internal/service/openai_ws_forwarder_ingress.go
|
||||
|
||||
其优点是保留 replay input、只做一次恢复,并避免在 function_call_output 无完整链路时盲目删除 ID。可以参考“有完整重放输入才恢复”的原则。
|
||||
|
||||
CLIProxyAPI:
|
||||
|
||||
/home/zyy/桌面/Codes/CLIProxyAPI/sdk/api/handlers/openai/openai_responses_websocket.go
|
||||
|
||||
其优点是保存 transcript,并在可恢复错误后释放 pinned auth、强制下一次完整 transcript replay。可以参考“连接失效后增量状态也必须失效”的原则。
|
||||
|
||||
不应直接照抄:
|
||||
|
||||
- sub2api 的 message/场景启发式不能替代强类型连接状态。
|
||||
- CLIProxyAPI 的完整 transcript 常驻策略会增加内存和隐私边界。
|
||||
- 本工程已有 ImplicitResumeSnapshot,应该复用现有完整输入能力,而不是再建第二套 transcript 缓存。
|
||||
|
||||
## 26. 最终建议
|
||||
|
||||
实现时优先完成以下最小闭环:
|
||||
|
||||
PooledWebSocketConnection.latest_response_id
|
||||
+ store=false 发送前精确校验
|
||||
+ stale-reuse 禁止原 payload 新连接重试
|
||||
+ HistoryRecoveryPlan
|
||||
+ 显式 history 不再 strip-and-retry
|
||||
+ 结构化 previous_response_unavailable
|
||||
|
||||
能力感知预取和超时状态拆分必须与流式回归测试一起完成,不能只移植全局 UntilOutputOrTerminal。
|
||||
|
||||
这套实现既能解决当前 GPT-5.6 事故,也保持模型无关、低延迟、无额外持久化和清晰的状态所有权。
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user