mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 16:05:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2640a03219 | ||
|
|
7d0e9a0a20 | ||
|
|
09982d7221 | ||
|
|
c4db1bb244 | ||
|
|
e7c2f025cc | ||
|
|
4ee294b208 | ||
|
|
93f1a82d3f | ||
|
|
c46f98cd2b |
@@ -144,7 +144,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"codex_version: '0.147.0-alpha.6.6'",
|
||||
"codex_version: '0.151.0-alpha.7.2'",
|
||||
"codex_version: 'latest'",
|
||||
));
|
||||
}
|
||||
@@ -165,7 +165,7 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
assert_rejected(valid_config().replace("desktop_build: '6415'", "desktop_build: 'build'"));
|
||||
assert_rejected(valid_config().replace("desktop_build: '7377'", "desktop_build: 'build'"));
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -15,8 +15,18 @@ use super::error::RequestDecodeError;
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
/// Responses 请求进入共享解码内核时的下游传输来源。
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum RequestDecodeSource {
|
||||
/// 单次 HTTP 请求,请求头与正文属于同一请求。
|
||||
Http,
|
||||
/// 复用连接上的 WebSocket `response.create` 帧。
|
||||
WebSocketFrame,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct OpenAiRequestHeaders {
|
||||
turn_state: Option<String>,
|
||||
@@ -71,10 +81,18 @@ impl OpenAiRequestHeaders {
|
||||
}
|
||||
}
|
||||
|
||||
fn protocol_context(&self, use_websocket: Option<bool>) -> Map<String, Value> {
|
||||
fn protocol_context(
|
||||
&self,
|
||||
use_websocket: Option<bool>,
|
||||
frame_turn_metadata: Option<&String>,
|
||||
) -> Map<String, Value> {
|
||||
let mut context = Map::new();
|
||||
insert_protocol_context(&mut context, "turn_state", self.turn_state.as_ref());
|
||||
insert_protocol_context(&mut context, "turn_metadata", self.turn_metadata.as_ref());
|
||||
insert_protocol_context(
|
||||
&mut context,
|
||||
"turn_metadata",
|
||||
frame_turn_metadata.or(self.turn_metadata.as_ref()),
|
||||
);
|
||||
insert_protocol_context(&mut context, "beta_features", self.beta_features.as_ref());
|
||||
insert_protocol_context(&mut context, "version", self.version.as_ref());
|
||||
insert_protocol_context(
|
||||
@@ -304,14 +322,15 @@ pub(super) fn decode_request_inner(
|
||||
let Value::Object(object) = value else {
|
||||
return Err(RequestDecodeError::ExpectedObject);
|
||||
};
|
||||
decode_request_object(object, review, request_headers)
|
||||
decode_request_object(object, review, request_headers, RequestDecodeSource::Http)
|
||||
}
|
||||
|
||||
/// 解码已解析的顶层 object;WebSocket 帧解码复用此内核,避免序列化往返。
|
||||
/// 解码已解析的顶层 object;按下游传输来源恢复连接级协议上下文。
|
||||
pub(super) fn decode_request_object(
|
||||
mut object: Map<String, Value>,
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
source: RequestDecodeSource,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
// 仅消费已识别的本地 transport 开关;未知同名值保留给未来上游协议。
|
||||
let use_websocket = object.get("use_websocket").and_then(Value::as_bool);
|
||||
@@ -341,7 +360,11 @@ pub(super) fn decode_request_object(
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
request_headers.apply_subagent(&mut object, review.then_some(REVIEW_SUBAGENT));
|
||||
let protocol_context = request_headers.protocol_context(use_websocket);
|
||||
let frame_turn_metadata = match source {
|
||||
RequestDecodeSource::Http => None,
|
||||
RequestDecodeSource::WebSocketFrame => frame_turn_metadata(&object),
|
||||
};
|
||||
let protocol_context = request_headers.protocol_context(use_websocket, frame_turn_metadata);
|
||||
|
||||
let payload = ProtocolPayload::json_object(OPENAI_PROTOCOL, object)
|
||||
.map_err(|_| RequestDecodeError::CanonicalContract {
|
||||
@@ -362,6 +385,16 @@ pub(super) fn decode_request_object(
|
||||
})
|
||||
}
|
||||
|
||||
fn frame_turn_metadata(body: &Map<String, Value>) -> Option<&String> {
|
||||
body.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get(CODEX_TURN_METADATA_KEY))
|
||||
.and_then(|value| match value {
|
||||
Value::String(value) if !value.trim().is_empty() => Some(value),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn inject_subagent_metadata(body: &mut Map<String, Value>, subagent: &str) {
|
||||
let metadata = body
|
||||
.entry("client_metadata".to_owned())
|
||||
@@ -455,10 +488,11 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent
|
||||
// 不能透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent/version
|
||||
// 不能作为不透明头透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
// 设备 attestation/integrity 头只能由官方客户端或网关自身生成,
|
||||
// 客户端注入的 x-oai-attestation / X-OAI-IS 不得透传上游。
|
||||
| "x-oai-attestation"
|
||||
|
||||
@@ -6,7 +6,8 @@ use serde_json::{Map, Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
use super::super::{
|
||||
DecodedResponsesRequest, ProtocolErrorBody, RequestDecodeError, request::OpenAiRequestHeaders,
|
||||
DecodedResponsesRequest, ProtocolErrorBody, RequestDecodeError,
|
||||
request::{OpenAiRequestHeaders, RequestDecodeSource},
|
||||
};
|
||||
|
||||
/// 使用连接级请求头和 Provider 上下文解码官方 `response.create` 文本帧。
|
||||
@@ -40,8 +41,13 @@ fn decode_response_create_inner(
|
||||
if matches!(body.get("stream"), Some(value) if value.as_bool() != Some(true)) {
|
||||
return Err(ResponseCreateFrameError::StreamingRequired);
|
||||
}
|
||||
super::super::request::decode_request_object(body, false, request_headers)
|
||||
.map_err(ResponseCreateFrameError::Request)
|
||||
super::super::request::decode_request_object(
|
||||
body,
|
||||
false,
|
||||
request_headers,
|
||||
RequestDecodeSource::WebSocketFrame,
|
||||
)
|
||||
.map_err(ResponseCreateFrameError::Request)
|
||||
}
|
||||
|
||||
/// `response.create` 帧的稳定安全错误。
|
||||
|
||||
@@ -196,7 +196,13 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
HeaderValue::from_static("{\"kind\":\"review\"}"),
|
||||
);
|
||||
headers.insert("conversation-id", HeaderValue::from_static("conversation"));
|
||||
let body = json!({"model": "smart-code", "input": "hello"});
|
||||
let body = json!({
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": "{\"kind\":\"turn\"}"
|
||||
}
|
||||
});
|
||||
|
||||
let decoded = decode_request_with_headers(body.to_string().as_bytes(), &headers)
|
||||
.expect("request should decode");
|
||||
@@ -320,8 +326,8 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
vec![b"future-ascii".to_vec(), b"\x80\xff".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("version"),
|
||||
vec![b"future-v1".to_vec(), b"future-\x80".to_vec()]
|
||||
openai_protocol_context(&decoded).get("version"),
|
||||
Some(&json!("future-v1"))
|
||||
);
|
||||
assert_eq!(
|
||||
values("openai-beta"),
|
||||
@@ -349,6 +355,7 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
|
||||
"user-agent",
|
||||
"originator",
|
||||
"version",
|
||||
"x-oai-attestation",
|
||||
"x-oai-is",
|
||||
"x-oai-is-update",
|
||||
|
||||
@@ -9,11 +9,14 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{HeaderMap, HeaderValue, Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::{SinkExt, StreamExt, future::BoxFuture};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_api::openai::responses::{
|
||||
DecodedResponsesRequest, OpenAiRequestHeaders, ResponseCreateFrameError,
|
||||
decode_response_create_with_context,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
@@ -25,6 +28,7 @@ use gateway_core::event::{
|
||||
};
|
||||
use gateway_core::operation::Operation;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_protocol::openai::codex_responses_request_semantics;
|
||||
use serde_json::{Value, json};
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Message as ClientMessage;
|
||||
@@ -34,6 +38,22 @@ use tower::ServiceExt;
|
||||
use super::decode_response_create;
|
||||
use crate::openai::{api_router, authenticated_client, models::ModelsExecution};
|
||||
|
||||
fn decode_response_create_with_turn_header(
|
||||
payload: Value,
|
||||
opening_turn_metadata: &'static str,
|
||||
) -> DecodedResponsesRequest {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-codex-turn-metadata",
|
||||
HeaderValue::from_static(opening_turn_metadata),
|
||||
);
|
||||
decode_response_create_with_context(
|
||||
&payload.to_string(),
|
||||
&OpenAiRequestHeaders::from_headers(&headers),
|
||||
)
|
||||
.expect("response.create should decode")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_default_to_the_websocket_streaming_contract() {
|
||||
let decoded = decode_response_create(
|
||||
@@ -118,6 +138,87 @@ fn response_create_should_preserve_compaction_trigger_for_openai() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_prefer_frame_turn_metadata_over_opening_headers() {
|
||||
let decoded = decode_response_create_with_turn_header(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": r#"{"request_kind":"turn"}"#
|
||||
}
|
||||
}),
|
||||
r#"{"request_kind":"compaction"}"#,
|
||||
);
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("Responses must map to Generate");
|
||||
};
|
||||
|
||||
let semantics = codex_responses_request_semantics(
|
||||
request.protocol_payload().body(),
|
||||
request.protocol_payload().context(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(semantics.request_kind.as_deref(), semantics.compact),
|
||||
(Some("turn"), false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_accept_frame_compaction_after_a_turn_opening_header() {
|
||||
let decoded = decode_response_create_with_turn_header(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": r#"{"request_kind":"compaction"}"#
|
||||
}
|
||||
}),
|
||||
r#"{"request_kind":"turn"}"#,
|
||||
);
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("Responses must map to Generate");
|
||||
};
|
||||
|
||||
let semantics = codex_responses_request_semantics(
|
||||
request.protocol_payload().body(),
|
||||
request.protocol_payload().context(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(semantics.request_kind.as_deref(), semantics.compact),
|
||||
(Some("compaction"), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_fall_back_to_opening_turn_metadata_when_the_frame_omits_it() {
|
||||
let decoded = decode_response_create_with_turn_header(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "smart-code",
|
||||
"input": "hello"
|
||||
}),
|
||||
r#"{"request_kind":"compaction"}"#,
|
||||
);
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("Responses must map to Generate");
|
||||
};
|
||||
|
||||
let semantics = codex_responses_request_semantics(
|
||||
request.protocol_payload().body(),
|
||||
request.protocol_payload().context(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(semantics.request_kind.as_deref(), semantics.compact),
|
||||
(Some("compaction"), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_reject_explicit_non_streaming_requests() {
|
||||
let error = decode_response_create(
|
||||
|
||||
@@ -162,11 +162,12 @@ where
|
||||
account_state_owner,
|
||||
cancellation,
|
||||
attempts: 0,
|
||||
routing_attempts: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
transport_fallback_account: None,
|
||||
transport_recovery: None,
|
||||
current: None,
|
||||
send_state_watermark: UpstreamSendState::NotSent,
|
||||
downstream_committed_at: None,
|
||||
@@ -217,6 +218,13 @@ struct FailureFinalization {
|
||||
retry_after_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingTransportRecovery {
|
||||
account: crate::engine::credential::ProviderAccountId,
|
||||
transport: AttemptTransport,
|
||||
delay: Duration,
|
||||
}
|
||||
|
||||
/// API 可逐事件消费的 Core 执行会话。
|
||||
///
|
||||
/// API 只能提交下游 delivery 边界;账号重试、断流终结与
|
||||
@@ -238,7 +246,10 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
continuation_attempt: ContinuationAttempt,
|
||||
account_state_owner: Option<ProviderAccountStateOwner>,
|
||||
cancellation: CancellationToken,
|
||||
/// 所有实际上游 attempt 数;包含同账号传输重试,作为持久化序号。
|
||||
attempts: u32,
|
||||
/// 路由预算只统计正常选号/账号恢复,不被 Provider-owned 传输预算消耗。
|
||||
routing_attempts: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
@@ -246,9 +257,9 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
/// attempt 建立时即被消费,后续可重试错误仍可换号消耗剩余重试预算。
|
||||
/// 与 `required_account`(外部指定、贯穿整个请求)语义不同,不可合并。
|
||||
recovery_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
/// 上游传输在提交边界前失败后的一次性同账号备用传输钉选。
|
||||
/// 与凭据恢复分开保存,避免把 transport fallback 误记为 OAuth 恢复。
|
||||
transport_fallback_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
/// 上游传输在提交边界前失败后的一次性同账号重试或备用传输钉选。
|
||||
/// 与凭据恢复分开保存,避免把 transport recovery 误记为 OAuth 恢复。
|
||||
transport_recovery: Option<PendingTransportRecovery>,
|
||||
current: Option<CurrentAttempt>,
|
||||
/// 请求级发送状态水位;跨 attempt 单调不降,终态写回不得低于此档。
|
||||
send_state_watermark: UpstreamSendState,
|
||||
@@ -590,9 +601,25 @@ where
|
||||
}
|
||||
|
||||
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
|
||||
if self.attempts >= self.plan.max_attempts().get() {
|
||||
let transport_recovery = self.transport_recovery.take();
|
||||
if transport_recovery.is_none() && self.routing_attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
if let Some(recovery) = transport_recovery.as_ref()
|
||||
&& !recovery.delay.is_zero()
|
||||
{
|
||||
match poll_retry_delay(recovery.delay, self.cancellation.clone(), self.deadline).await {
|
||||
RetryDelayBoundary::Elapsed => {}
|
||||
RetryDelayBoundary::Cancelled => {
|
||||
self.finish_interruption(EngineError::Cancelled).await?;
|
||||
return Err(EngineError::Cancelled);
|
||||
}
|
||||
RetryDelayBoundary::Deadline => {
|
||||
self.finish_interruption(EngineError::Deadline).await?;
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(candidate) = self.plan.candidates().get(self.candidate_index).cloned() else {
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
@@ -616,23 +643,20 @@ where
|
||||
.ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
// 请求局部恢复钉选在此被一次性消费,只绑定本次 replay attempt;
|
||||
// 外部 required_account 每次 attempt 都重新生效。
|
||||
let (pinned_account, attempt_transport) = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => {
|
||||
(Some(account.clone()), AttemptTransport::Default)
|
||||
}
|
||||
AccountSelection::Scheduled(_) => {
|
||||
let transport_fallback_account = self.transport_fallback_account.take();
|
||||
let transport = if transport_fallback_account.is_some() {
|
||||
AttemptTransport::Fallback
|
||||
} else {
|
||||
AttemptTransport::Default
|
||||
};
|
||||
(
|
||||
transport_fallback_account
|
||||
.or_else(|| self.recovery_account.take())
|
||||
let is_transport_recovery = transport_recovery.is_some();
|
||||
let (pinned_account, attempt_transport) = if let Some(recovery) = transport_recovery {
|
||||
(Some(recovery.account), recovery.transport)
|
||||
} else {
|
||||
match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => {
|
||||
(Some(account.clone()), AttemptTransport::Default)
|
||||
}
|
||||
AccountSelection::Scheduled(_) => (
|
||||
self.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
transport,
|
||||
)
|
||||
AttemptTransport::Default,
|
||||
),
|
||||
}
|
||||
};
|
||||
let account_context = match &self.account_selection {
|
||||
@@ -881,6 +905,9 @@ where
|
||||
}
|
||||
}
|
||||
self.attempts = next_attempt.get();
|
||||
if !is_transport_recovery {
|
||||
self.routing_attempts = self.routing_attempts.saturating_add(1);
|
||||
}
|
||||
self.current = Some(CurrentAttempt {
|
||||
stream,
|
||||
metadata,
|
||||
@@ -1034,35 +1061,50 @@ where
|
||||
attempt_send_state,
|
||||
provider_proved_replay_safe,
|
||||
);
|
||||
let pre_delivery_retry = self.account_selection.required_account().is_none()
|
||||
let account_rotation_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& error.allows_pre_delivery_retry()
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
let transport_fallback_retry = pre_delivery_retry
|
||||
&& matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
);
|
||||
Some(crate::error::PreDeliveryRetry::AccountRotation)
|
||||
)
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportRetry {
|
||||
retry_index,
|
||||
delay,
|
||||
}) if self.downstream_committed_at.is_none() && !self.delivery_pending => {
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if self.downstream_committed_at.is_none() && !self.delivery_pending =>
|
||||
{
|
||||
Some((AttemptTransport::Fallback, Duration::ZERO))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let ordinary_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& provider_proved_replay_safe
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
let same_account_retry = error.retries_same_account()
|
||||
&& provider_proved_replay_safe
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& self.attempts < self.plan.max_attempts().get()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& !self
|
||||
.credential_recovery_attempted_accounts
|
||||
.contains(current.metadata.provider_account_id());
|
||||
let retryable =
|
||||
continuation_retry || same_account_retry || ordinary_retry || pre_delivery_retry;
|
||||
let retryable = continuation_retry
|
||||
|| same_account_retry
|
||||
|| ordinary_retry
|
||||
|| account_rotation_retry
|
||||
|| transport_recovery.is_some();
|
||||
|
||||
if retryable {
|
||||
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
|
||||
@@ -1075,9 +1117,12 @@ where
|
||||
// 只钉住紧随其后的 replay attempt;replay 再遇可重试错误时,
|
||||
// ordinary/continuation 重试门不受影响,仍可换号。
|
||||
self.recovery_account = Some(account);
|
||||
} else if transport_fallback_retry {
|
||||
self.transport_fallback_account =
|
||||
Some(current.metadata.provider_account_id().clone());
|
||||
} else if let Some((transport, delay)) = transport_recovery {
|
||||
self.transport_recovery = Some(PendingTransportRecovery {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport,
|
||||
delay,
|
||||
});
|
||||
} else if !continuation_retry {
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
@@ -1155,7 +1200,7 @@ where
|
||||
|| self.delivery_pending
|
||||
|| send_state == UpstreamSendState::Ambiguous
|
||||
|| !provider_proved_replay_safe
|
||||
|| self.attempts >= self.plan.max_attempts().get()
|
||||
|| self.routing_attempts >= self.plan.max_attempts().get()
|
||||
|| self
|
||||
.operation
|
||||
.provider_session_state(current.metadata.provider().as_str())
|
||||
@@ -1636,6 +1681,31 @@ enum ProviderBoundary {
|
||||
Deadline,
|
||||
}
|
||||
|
||||
enum RetryDelayBoundary {
|
||||
Elapsed,
|
||||
Cancelled,
|
||||
Deadline,
|
||||
}
|
||||
|
||||
async fn poll_retry_delay(
|
||||
delay: Duration,
|
||||
cancellation: CancellationToken,
|
||||
deadline: SystemTime,
|
||||
) -> RetryDelayBoundary {
|
||||
let Ok(remaining) = deadline.duration_since(SystemTime::now()) else {
|
||||
return RetryDelayBoundary::Deadline;
|
||||
};
|
||||
let retry_delay = Delay::new(delay).fuse();
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let timeout = Delay::new(remaining).fuse();
|
||||
pin_mut!(retry_delay, cancelled, timeout);
|
||||
select_biased! {
|
||||
_ = cancelled => RetryDelayBoundary::Cancelled,
|
||||
_ = timeout => RetryDelayBoundary::Deadline,
|
||||
_ = retry_delay => RetryDelayBoundary::Elapsed,
|
||||
}
|
||||
}
|
||||
|
||||
async fn poll_provider(
|
||||
provider: Arc<dyn Provider>,
|
||||
request: ProviderRequest,
|
||||
|
||||
@@ -112,6 +112,8 @@ pub enum AttemptTransport {
|
||||
/// 使用 Provider 的默认传输策略。
|
||||
#[default]
|
||||
Default,
|
||||
/// 固定账号重试 Provider 的首选传输;序号由 Provider 的独立预算驱动。
|
||||
Retry(NonZeroU32),
|
||||
/// 使用 Provider 定义的备用传输。
|
||||
Fallback,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 网关核心使用的稳定错误分类。
|
||||
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
@@ -202,6 +203,13 @@ pub enum ContinuationFailure {
|
||||
pub enum PreDeliveryRetry {
|
||||
/// 排除本次账号,按普通调度策略重新选号。
|
||||
AccountRotation,
|
||||
/// 固定本次账号,并按 Provider 给出的序号重试当前传输。
|
||||
SameAccountTransportRetry {
|
||||
/// Provider-owned 传输重试序号,从 1 开始。
|
||||
retry_index: NonZeroU32,
|
||||
/// 发起下一次传输尝试前的退避时长。
|
||||
delay: Duration,
|
||||
},
|
||||
/// 固定本次账号,并要求 Provider 使用备用传输。
|
||||
SameAccountTransportFallback,
|
||||
}
|
||||
@@ -475,10 +483,39 @@ impl ProviderError {
|
||||
/// 下游已提交后的隐藏重放。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_transport_fallback(mut self) -> Self {
|
||||
self.pre_delivery_retry = Some(PreDeliveryRetry::SameAccountTransportFallback);
|
||||
self.set_pre_delivery_transport_fallback();
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地设置同账号备用传输恢复,保留当前错误携带的原客户端响应。
|
||||
pub const fn set_pre_delivery_transport_fallback(&mut self) {
|
||||
self.pre_delivery_retry = Some(PreDeliveryRetry::SameAccountTransportFallback);
|
||||
}
|
||||
|
||||
/// 允许 Core 在首个客户端事件前固定原账号并重试当前 Provider 传输。
|
||||
///
|
||||
/// 重试预算和退避策略由 Provider 所属协议定义;Core 只负责同账号钉选、
|
||||
/// deadline/取消边界以及下游 commit barrier。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_transport_retry(
|
||||
mut self,
|
||||
retry_index: NonZeroU32,
|
||||
delay: Duration,
|
||||
) -> Self {
|
||||
self.set_pre_delivery_transport_retry(retry_index, delay);
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地设置同账号当前传输重试,保留当前错误携带的原客户端响应。
|
||||
pub const fn set_pre_delivery_transport_retry(
|
||||
&mut self,
|
||||
retry_index: NonZeroU32,
|
||||
delay: Duration,
|
||||
) {
|
||||
self.pre_delivery_retry =
|
||||
Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, delay });
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次。
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
|
||||
@@ -2797,6 +2797,66 @@ fn pre_delivery_transport_fallback_retries_the_same_account_with_fallback_transp
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_owned_transport_retries_keep_the_same_account_until_fallback() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let retry_one = NonZeroU32::new(1).expect("non-zero retry index");
|
||||
let retry_two = NonZeroU32::new(2).expect("non-zero retry index");
|
||||
let transport_error =
|
||||
|| ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transport_error().with_pre_delivery_transport_retry(retry_one, Duration::ZERO)
|
||||
)],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transport_error().with_pre_delivery_transport_retry(retry_two, Duration::ZERO)
|
||||
)],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(transport_error().with_pre_delivery_transport_fallback())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("fallback succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
|
||||
|
||||
let original = ProviderAccountId::new("acct_first").expect("account id");
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 4);
|
||||
assert_eq!(contexts[0].transport(), AttemptTransport::Default);
|
||||
assert_eq!(contexts[1].transport(), AttemptTransport::Retry(retry_one));
|
||||
assert_eq!(contexts[2].transport(), AttemptTransport::Retry(retry_two));
|
||||
assert_eq!(contexts[3].transport(), AttemptTransport::Fallback);
|
||||
for context in contexts.iter().skip(1) {
|
||||
assert_eq!(context.required_account(), Some(&original));
|
||||
assert!(!context.excluded_accounts().contains(&original));
|
||||
}
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 4);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_capacity_exhaustion_returns_the_last_retryable_upstream_failure() {
|
||||
let operation = generate_operation();
|
||||
|
||||
@@ -18,6 +18,15 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
/// 本服务的流式请求默认重试次数。
|
||||
pub const DEFAULT_STREAM_MAX_RETRIES: u64 = 3;
|
||||
/// 防止错误配置产生无界隐藏重放;与官方 Codex 的硬上限一致。
|
||||
pub const MAX_STREAM_MAX_RETRIES: u64 = 100;
|
||||
|
||||
const fn default_stream_max_retries() -> u64 {
|
||||
DEFAULT_STREAM_MAX_RETRIES
|
||||
}
|
||||
|
||||
/// OpenAI Provider 唯一启动配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -30,6 +39,8 @@ pub struct OpenAiConfig {
|
||||
pub quota: CodexQuotaSettings,
|
||||
#[serde(default)]
|
||||
pub auth: CodexAuthSettings,
|
||||
#[serde(default = "default_stream_max_retries")]
|
||||
pub stream_max_retries: u64,
|
||||
pub wire_profile: CodexWireProfileConfig,
|
||||
#[serde(skip)]
|
||||
identity_secret_path: PathBuf,
|
||||
@@ -88,6 +99,13 @@ impl OpenAiConfig {
|
||||
self.auth.refresh_enabled
|
||||
}
|
||||
|
||||
/// 返回经官方同款硬上限约束后的上游流重试预算。
|
||||
#[must_use]
|
||||
pub fn stream_max_retries(&self) -> u32 {
|
||||
u32::try_from(self.stream_max_retries.min(MAX_STREAM_MAX_RETRIES))
|
||||
.unwrap_or(MAX_STREAM_MAX_RETRIES as u32)
|
||||
}
|
||||
|
||||
pub(crate) fn session_identity(
|
||||
&self,
|
||||
) -> Result<CodexSessionIdentity, CodexSessionIdentityError> {
|
||||
@@ -102,6 +120,7 @@ impl Default for OpenAiConfig {
|
||||
ws_pool: CodexWebSocketPoolSettings::default(),
|
||||
quota: CodexQuotaSettings::default(),
|
||||
auth: CodexAuthSettings::default(),
|
||||
stream_max_retries: DEFAULT_STREAM_MAX_RETRIES,
|
||||
wire_profile: CodexWireProfileConfig::default(),
|
||||
identity_secret_path: PathBuf::new(),
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! OpenAI 根会话锚点到 Store 不透明账号亲和键及诊断上下文的单向派生。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -8,6 +10,7 @@ use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
use crate::transport::request::derive_conversation_anchor;
|
||||
|
||||
const AFFINITY_KEY_HASH_LENGTH: usize = 12;
|
||||
pub(crate) const CODEX_ROOT_SESSION_TTL: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
|
||||
/// 一次请求派生出的账号亲和键及其结构化日志上下文。
|
||||
pub(crate) struct CodexSessionAffinity {
|
||||
|
||||
@@ -16,7 +16,8 @@ pub mod token_client;
|
||||
mod types;
|
||||
|
||||
pub(crate) use affinity::{
|
||||
CodexSessionAffinity, derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
|
||||
derive_codex_session_affinity,
|
||||
};
|
||||
pub(crate) use oauth::oauth_owner_ref;
|
||||
pub(crate) use types::parse_access_token_expiration;
|
||||
|
||||
@@ -22,7 +22,7 @@ use secrecy::ExposeSecret;
|
||||
use thiserror::Error;
|
||||
use url::Url;
|
||||
|
||||
use super::affinity::CodexSessionAffinity;
|
||||
use super::affinity::{CODEX_ROOT_SESSION_TTL, CodexSessionAffinity};
|
||||
use super::catalog::CodexCredentialCatalogService;
|
||||
use super::cookie::CodexCookiePolicy;
|
||||
use super::quota::CodexCredentialQuotaService;
|
||||
@@ -41,7 +41,6 @@ const CLOUDFLARE_CHALLENGE_BACKOFF: [Duration; 4] = [
|
||||
Duration::from_secs(120),
|
||||
];
|
||||
const CLOUDFLARE_PATH_BLOCK_THRESHOLD: u32 = 3;
|
||||
const SESSION_AFFINITY_TTL: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
const SESSION_AFFINITY_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const CYBER_POLICY_SESSION_TTL: Duration = Duration::from_secs(60 * 60);
|
||||
|
||||
@@ -681,7 +680,7 @@ impl CodexCredentialSelector {
|
||||
&self.provider_kind,
|
||||
key,
|
||||
selected_account_id,
|
||||
SESSION_AFFINITY_TTL,
|
||||
CODEX_ROOT_SESSION_TTL,
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -955,7 +954,7 @@ impl CodexCredentialSelector {
|
||||
key,
|
||||
expected_affinity_account_id,
|
||||
account.id(),
|
||||
SESSION_AFFINITY_TTL,
|
||||
CODEX_ROOT_SESSION_TTL,
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
mod admin;
|
||||
pub mod config;
|
||||
mod provider;
|
||||
mod session_transport;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -138,6 +139,7 @@ pub async fn initialize(
|
||||
profile.clone(),
|
||||
config.base_url().to_owned(),
|
||||
Arc::clone(&websocket_pool),
|
||||
config.stream_max_retries(),
|
||||
)
|
||||
.map_err(OpenAiInitializeError::Provider)?
|
||||
.with_session_identity(session_identity),
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
@@ -57,6 +58,7 @@ use crate::credential::{
|
||||
SelectCodexCredential, SelectCodexProviderEndpointCredential,
|
||||
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
};
|
||||
use crate::session_transport::CodexSessionTransportFallbacks;
|
||||
use crate::transport::canonical::{
|
||||
CodexCanonicalDecoder, CodexCanonicalError, CodexCanonicalOutcome,
|
||||
};
|
||||
@@ -71,8 +73,9 @@ use crate::transport::profile::{
|
||||
};
|
||||
use crate::transport::protocol::responses::{
|
||||
CodexResponsesRequest, PREVIOUS_RESPONSE_NOT_FOUND_CODE, PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE,
|
||||
PreviousResponseScope, ResponseEventSignals, transport_requirement,
|
||||
PreviousResponseScope, ResponseEventSignals, TransportRequirement, transport_requirement,
|
||||
};
|
||||
use crate::transport::protocol::websocket::WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE;
|
||||
use crate::transport::request::{
|
||||
CodexRequestEncodeError, RequestAccountScope, encode_generate_request, scope_request_to_account,
|
||||
};
|
||||
@@ -126,6 +129,8 @@ pub struct CodexProvider {
|
||||
image_generations_url: Url,
|
||||
image_edits_url: Url,
|
||||
session_identity: Option<CodexSessionIdentity>,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
stream_max_retries: u32,
|
||||
}
|
||||
|
||||
impl CodexProvider {
|
||||
@@ -140,6 +145,7 @@ impl CodexProvider {
|
||||
profile: CodexWireProfileState,
|
||||
base_url: String,
|
||||
websocket_pool: Arc<CodexWebSocketPool>,
|
||||
stream_max_retries: u32,
|
||||
) -> Result<Self, CodexProviderConfigError> {
|
||||
let responses_url = Url::parse(&endpoint_url(&base_url, CODEX_RESPONSES_PATH))
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
@@ -160,6 +166,8 @@ impl CodexProvider {
|
||||
image_generations_url,
|
||||
image_edits_url,
|
||||
session_identity: None,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks::default(),
|
||||
stream_max_retries,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -288,11 +296,6 @@ impl Provider for CodexProvider {
|
||||
derive_codex_session_affinity(&upstream_request, context.client_api_key_ref());
|
||||
let cyber_policy_session_key =
|
||||
derive_codex_cyber_policy_session_key(&upstream_request, context.client_api_key_ref());
|
||||
let transport = match context.transport() {
|
||||
AttemptTransport::Default => selected_transport(&upstream_request),
|
||||
AttemptTransport::Fallback => CodexProviderTransport::HttpOnly,
|
||||
};
|
||||
apply_transport(&mut upstream_request, transport);
|
||||
|
||||
let selection_started_at = Instant::now();
|
||||
let lease = self
|
||||
@@ -388,6 +391,39 @@ impl Provider for CodexProvider {
|
||||
lease.installation_id(),
|
||||
account_scope,
|
||||
);
|
||||
let requirement = transport_requirement(&upstream_request);
|
||||
let requested_transport = selected_transport(&upstream_request);
|
||||
let sticky_http_fallback = context.transport() == AttemptTransport::Default
|
||||
&& requested_transport == CodexProviderTransport::PreferWebSocket
|
||||
&& !requirement.requires_websocket()
|
||||
&& session_affinity.as_ref().is_some_and(|affinity| {
|
||||
self.session_transport_fallbacks
|
||||
.is_http_only(affinity.key())
|
||||
});
|
||||
let transport = if requirement.requires_websocket() {
|
||||
CodexProviderTransport::PreferWebSocket
|
||||
} else {
|
||||
match context.transport() {
|
||||
AttemptTransport::Default if sticky_http_fallback => {
|
||||
CodexProviderTransport::HttpOnly
|
||||
}
|
||||
AttemptTransport::Default => requested_transport,
|
||||
AttemptTransport::Retry(_) => requested_transport,
|
||||
AttemptTransport::Fallback => CodexProviderTransport::HttpOnly,
|
||||
}
|
||||
};
|
||||
if sticky_http_fallback {
|
||||
tracing::info!(
|
||||
request_id = %context.request_id(),
|
||||
attempt_index = context.attempt_index().get(),
|
||||
session_affinity_key_hash = session_affinity
|
||||
.as_ref()
|
||||
.map_or("", CodexSessionAffinity::key_hash),
|
||||
transport_requirement = requirement.as_str(),
|
||||
"Reusing session-sticky OpenAI upstream HTTP fallback"
|
||||
);
|
||||
}
|
||||
apply_transport(&mut upstream_request, transport);
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
provider_kind,
|
||||
upstream_model.clone(),
|
||||
@@ -411,7 +447,14 @@ impl Provider for CodexProvider {
|
||||
continuation_scope: None,
|
||||
});
|
||||
let allows_account_state_mutation = lease.allows_account_state_mutation();
|
||||
let session_affinity_key_hash = session_affinity
|
||||
.as_ref()
|
||||
.map(|affinity| affinity.key_hash().to_owned());
|
||||
let session_affinity_key = session_affinity.map(CodexSessionAffinity::into_key);
|
||||
let websocket_retry_count = match context.transport() {
|
||||
AttemptTransport::Retry(retry_index) => retry_index.get(),
|
||||
AttemptTransport::Default | AttemptTransport::Fallback => 0,
|
||||
};
|
||||
let events = cold_response_stream(ColdResponse {
|
||||
client: self.client.clone(),
|
||||
response_origin: self.responses_url.clone(),
|
||||
@@ -425,6 +468,10 @@ impl Provider for CodexProvider {
|
||||
lease: Arc::clone(&lease),
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_affinity_key_hash,
|
||||
session_transport_fallbacks: self.session_transport_fallbacks.clone(),
|
||||
websocket_retry_count,
|
||||
stream_max_retries: self.stream_max_retries,
|
||||
session_capture,
|
||||
});
|
||||
let stream = ProviderStream::new(metadata, events, lease);
|
||||
@@ -522,6 +569,10 @@ struct ColdResponse {
|
||||
lease: Arc<CodexCredentialLease>,
|
||||
output_started_at: Instant,
|
||||
session_affinity_key: Option<ProviderSessionAffinityKey>,
|
||||
session_affinity_key_hash: Option<String>,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
websocket_retry_count: u32,
|
||||
stream_max_retries: u32,
|
||||
session_capture: Option<OpenAiSessionCapture>,
|
||||
}
|
||||
|
||||
@@ -663,7 +714,7 @@ async fn create_response_attempt(
|
||||
biased;
|
||||
_ = cancellation.cancelled() => Err(CodexHandshakeAttemptError::Cancelled),
|
||||
_ = tokio::time::sleep(handshake_deadline) => Err(CodexHandshakeAttemptError::Timeout),
|
||||
response = client.create_response_stream_with_pool_account(
|
||||
response = client.create_response_stream_with_deferred_websocket_recovery(
|
||||
request,
|
||||
request_context,
|
||||
Some(account_id),
|
||||
@@ -839,6 +890,10 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
lease,
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_affinity_key_hash,
|
||||
session_transport_fallbacks,
|
||||
websocket_retry_count,
|
||||
stream_max_retries,
|
||||
mut session_capture,
|
||||
} = response;
|
||||
Box::pin(async_stream::try_stream! {
|
||||
@@ -870,6 +925,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
lease.escape_reason(),
|
||||
lease.account_switch(),
|
||||
);
|
||||
let request_transport_requirement = transport_requirement(&request);
|
||||
let response = create_response_attempt(
|
||||
&client,
|
||||
&request,
|
||||
@@ -887,6 +943,14 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
&cancellation,
|
||||
)
|
||||
.await;
|
||||
let websocket_failure_policy = match &response {
|
||||
Err(CodexHandshakeAttemptError::Client(error))
|
||||
if transport_policy == CodexProviderTransport::PreferWebSocket =>
|
||||
{
|
||||
websocket_client_failure_policy(error)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if let Err(CodexHandshakeAttemptError::Client(error)) = &response {
|
||||
log_client_upstream_error(
|
||||
UpstreamErrorLogContext::new(&context, &active_account, None),
|
||||
@@ -897,6 +961,23 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
let response = match response {
|
||||
Ok(response) => response,
|
||||
Err(mut failure) => {
|
||||
if let Some(policy) = websocket_failure_policy {
|
||||
apply_websocket_recovery_policy(
|
||||
&mut failure,
|
||||
WebSocketRecoveryContext {
|
||||
policy,
|
||||
requirement: request_transport_requirement,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
request_id: context.request_id().as_str(),
|
||||
attempt_index: context.attempt_index().get(),
|
||||
account_id: active_account.id().as_str(),
|
||||
session_affinity_key: session_affinity_key.as_ref(),
|
||||
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
|
||||
session_transport_fallbacks: &session_transport_fallbacks,
|
||||
},
|
||||
);
|
||||
}
|
||||
if let Some(observation) = failure.observation.take() {
|
||||
yield ProviderEvent::observation(observation);
|
||||
}
|
||||
@@ -1029,12 +1110,25 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
continue;
|
||||
}
|
||||
Err(mut failure) => {
|
||||
if failure.error.allows_pre_delivery_retry()
|
||||
if failure.websocket_transport_retryable
|
||||
&& response_transport == CodexBackendTransport::WebSocket
|
||||
&& transport_requirement(&request).allows_pre_delivery_http_fallback()
|
||||
&& !pre_commit_events.is_committed()
|
||||
{
|
||||
failure.error = failure.error.with_pre_delivery_transport_fallback();
|
||||
apply_websocket_recovery_policy(
|
||||
&mut failure,
|
||||
WebSocketRecoveryContext {
|
||||
policy: WebSocketFailurePolicy::Budgeted,
|
||||
requirement: request_transport_requirement,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
request_id: context.request_id().as_str(),
|
||||
attempt_index: context.attempt_index().get(),
|
||||
account_id: active_account.id().as_str(),
|
||||
session_affinity_key: session_affinity_key.as_ref(),
|
||||
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
|
||||
session_transport_fallbacks: &session_transport_fallbacks,
|
||||
},
|
||||
);
|
||||
}
|
||||
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
|
||||
if !updates.is_empty() {
|
||||
@@ -1645,7 +1739,11 @@ fn codex_response_observation(
|
||||
WebSocketPoolKind::New
|
||||
});
|
||||
}
|
||||
if let Some(status_code) = diagnostics.status_code {
|
||||
// WebSocket opening 的 101 是成功升级事实,不是业务请求的失败 HTTP 状态。
|
||||
// opening 明确拒绝仍由 `codex_error_observation` 保存真实上游状态。
|
||||
if transport != CodexBackendTransport::WebSocket
|
||||
&& let Some(status_code) = diagnostics.status_code
|
||||
{
|
||||
observation = observation.with_status_code(status_code);
|
||||
}
|
||||
if let Some(request_id) = diagnostics.request_id.as_deref() {
|
||||
@@ -2084,6 +2182,7 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
|
||||
|
||||
struct MappedProviderFailure {
|
||||
error: ProviderError,
|
||||
websocket_transport_retryable: bool,
|
||||
account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,仅在凭据错误状态下持久化。
|
||||
error_message: Option<String>,
|
||||
@@ -2098,6 +2197,7 @@ impl MappedProviderFailure {
|
||||
fn plain(error: ProviderError) -> Self {
|
||||
Self {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
@@ -2311,7 +2411,7 @@ fn log_client_upstream_error(context: UpstreamErrorLogContext<'_>, error: &Codex
|
||||
.websocket_connection_id
|
||||
.map(|connection_id| connection_id.to_string())
|
||||
.unwrap_or_default();
|
||||
let upstream_error_raw = close.reason().unwrap_or_default();
|
||||
let close_reason_bytes = close.reason().map_or(0, str::len);
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = %context.account_id,
|
||||
@@ -2321,8 +2421,11 @@ fn log_client_upstream_error(context: UpstreamErrorLogContext<'_>, error: &Codex
|
||||
upstream_error_kind = "websocket_close",
|
||||
upstream_close_code = close.code().unwrap_or_default(),
|
||||
upstream_close_code_present = close.code().is_some(),
|
||||
upstream_error_raw,
|
||||
upstream_error_raw_bytes = upstream_error_raw.len(),
|
||||
upstream_last_event_type = close.last_event_type().unwrap_or_default(),
|
||||
upstream_last_event_type_present = close.last_event_type().is_some(),
|
||||
upstream_terminal_seen = false,
|
||||
upstream_close_reason_bytes = close_reason_bytes,
|
||||
upstream_close_reason_present = close.reason().is_some(),
|
||||
upstream_error_raw_present = close.reason().is_some(),
|
||||
"OpenAI upstream WebSocket closed before terminal event"
|
||||
);
|
||||
@@ -2480,6 +2583,151 @@ fn map_handshake_error(error: CodexClientError) -> MappedProviderFailure {
|
||||
map_client_error(error, UpstreamSendState::Ambiguous, true)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum WebSocketFailurePolicy {
|
||||
Budgeted,
|
||||
ImmediateFallback,
|
||||
}
|
||||
|
||||
struct WebSocketRecoveryContext<'a> {
|
||||
policy: WebSocketFailurePolicy,
|
||||
requirement: TransportRequirement,
|
||||
retry_count: u32,
|
||||
max_retries: u32,
|
||||
request_id: &'a str,
|
||||
attempt_index: u32,
|
||||
account_id: &'a str,
|
||||
session_affinity_key: Option<&'a ProviderSessionAffinityKey>,
|
||||
session_affinity_key_hash: Option<&'a str>,
|
||||
session_transport_fallbacks: &'a CodexSessionTransportFallbacks,
|
||||
}
|
||||
|
||||
fn websocket_client_failure_policy(error: &CodexClientError) -> Option<WebSocketFailurePolicy> {
|
||||
match error {
|
||||
CodexClientError::Upstream {
|
||||
status,
|
||||
transport: CodexBackendTransport::WebSocket,
|
||||
..
|
||||
} if *status == reqwest::StatusCode::UPGRADE_REQUIRED => {
|
||||
Some(WebSocketFailurePolicy::ImmediateFallback)
|
||||
}
|
||||
CodexClientError::Upstream {
|
||||
status,
|
||||
transport: CodexBackendTransport::WebSocket,
|
||||
..
|
||||
} if status.is_server_error()
|
||||
|| matches!(
|
||||
*status,
|
||||
reqwest::StatusCode::REQUEST_TIMEOUT
|
||||
| reqwest::StatusCode::CONFLICT
|
||||
| reqwest::StatusCode::TOO_EARLY
|
||||
| reqwest::StatusCode::TOO_MANY_REQUESTS
|
||||
) =>
|
||||
{
|
||||
Some(WebSocketFailurePolicy::Budgeted)
|
||||
}
|
||||
CodexClientError::WebSocket(error)
|
||||
if !matches!(error, CodexWebSocketExchangeError::InvalidRequest(_)) =>
|
||||
{
|
||||
Some(WebSocketFailurePolicy::Budgeted)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_websocket_recovery_policy(
|
||||
failure: &mut MappedProviderFailure,
|
||||
context: WebSocketRecoveryContext<'_>,
|
||||
) {
|
||||
let recovery_is_safe = match failure.error.send_state() {
|
||||
UpstreamSendState::NotSent => context.requirement.allows_pre_send_http_fallback(),
|
||||
UpstreamSendState::Sent | UpstreamSendState::Ambiguous => {
|
||||
context.requirement.allows_pre_delivery_http_fallback()
|
||||
}
|
||||
};
|
||||
if !recovery_is_safe {
|
||||
return;
|
||||
}
|
||||
|
||||
// 传输恢复必须保持原账号可调度;最终 HTTP attempt 若仍失败,再按真实 HTTP
|
||||
// 结果更新账号健康度,避免中间 WS 错误把同账号钉选提前冷却掉。
|
||||
failure.account_failure = None;
|
||||
let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback
|
||||
|| context.retry_count >= context.max_retries;
|
||||
if !fallback_now {
|
||||
let Some(retry_index) = context.retry_count.checked_add(1).and_then(NonZeroU32::new) else {
|
||||
return;
|
||||
};
|
||||
let delay = failure
|
||||
.error
|
||||
.retry_after()
|
||||
.unwrap_or_else(|| websocket_retry_backoff(retry_index));
|
||||
failure
|
||||
.error
|
||||
.set_pre_delivery_transport_retry(retry_index, delay);
|
||||
tracing::warn!(
|
||||
request_id = context.request_id,
|
||||
attempt_index = context.attempt_index,
|
||||
account_id = context.account_id,
|
||||
websocket_failure_kind = failure.error.kind().as_str(),
|
||||
websocket_failure_code = failure
|
||||
.error
|
||||
.upstream_code()
|
||||
.map_or("", OpaqueUpstreamValue::as_str),
|
||||
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
|
||||
upstream_status_code_present = failure.error.upstream_status().is_some(),
|
||||
websocket_retry_count = retry_index.get(),
|
||||
websocket_max_retries = context.max_retries,
|
||||
websocket_retry_delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
|
||||
transport_requirement = context.requirement.as_str(),
|
||||
"OpenAI upstream WebSocket failed; retrying the same account"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
failure.error.set_pre_delivery_transport_fallback();
|
||||
let sticky_state_transition = context
|
||||
.session_affinity_key
|
||||
.map(|key| context.session_transport_fallbacks.disable_websocket(key));
|
||||
tracing::warn!(
|
||||
request_id = context.request_id,
|
||||
attempt_index = context.attempt_index,
|
||||
account_id = context.account_id,
|
||||
websocket_failure_kind = failure.error.kind().as_str(),
|
||||
websocket_failure_code = failure
|
||||
.error
|
||||
.upstream_code()
|
||||
.map_or("", OpaqueUpstreamValue::as_str),
|
||||
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
|
||||
upstream_status_code_present = failure.error.upstream_status().is_some(),
|
||||
websocket_retry_count = context.retry_count,
|
||||
websocket_max_retries = context.max_retries,
|
||||
websocket_fallback_reason = match context.policy {
|
||||
WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted",
|
||||
WebSocketFailurePolicy::ImmediateFallback => "upgrade_required",
|
||||
},
|
||||
session_affinity_present = context.session_affinity_key.is_some(),
|
||||
session_affinity_key_hash = context.session_affinity_key_hash.unwrap_or(""),
|
||||
sticky_http_enabled = sticky_state_transition.is_some(),
|
||||
sticky_state_transition = sticky_state_transition.unwrap_or(false),
|
||||
"OpenAI upstream WebSocket disabled for the current session"
|
||||
);
|
||||
}
|
||||
|
||||
fn websocket_retry_backoff(retry_index: NonZeroU32) -> Duration {
|
||||
const INITIAL_DELAY_MS: u64 = 200;
|
||||
|
||||
let shift = retry_index.get().saturating_sub(1).min(63);
|
||||
let base_ms = INITIAL_DELAY_MS.saturating_mul(1_u64.checked_shl(shift).unwrap_or(u64::MAX));
|
||||
let mut random = [0_u8; 2];
|
||||
let jitter_per_mille = if getrandom::fill(&mut random).is_ok() {
|
||||
900_u64 + u64::from(u16::from_le_bytes(random) % 200)
|
||||
} else {
|
||||
1_000
|
||||
};
|
||||
Duration::from_millis(base_ms.saturating_mul(jitter_per_mille) / 1_000)
|
||||
}
|
||||
|
||||
fn continuation_replay_required_error() -> ProviderError {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
@@ -2495,8 +2743,10 @@ fn continuation_replay_required_error() -> ProviderError {
|
||||
|
||||
fn map_stream_error(error: CodexClientError) -> MappedProviderFailure {
|
||||
let allows_pre_delivery_retry = stream_transport_allows_pre_delivery_retry(&error);
|
||||
let websocket_failure = error.transport() == Some(CodexBackendTransport::WebSocket);
|
||||
let mut failure = map_client_error(error, UpstreamSendState::Sent, false);
|
||||
if allows_pre_delivery_retry {
|
||||
failure.websocket_transport_retryable = allows_pre_delivery_retry && websocket_failure;
|
||||
if allows_pre_delivery_retry && !websocket_failure {
|
||||
failure.error = failure.error.with_pre_delivery_retry();
|
||||
}
|
||||
failure
|
||||
@@ -2507,15 +2757,11 @@ fn stream_transport_allows_pre_delivery_retry(error: &CodexClientError) -> bool
|
||||
CodexClientError::Http(_)
|
||||
| CodexClientError::HttpJson(_)
|
||||
| CodexClientError::StreamIdleTimeout { .. } => true,
|
||||
CodexClientError::WebSocket(error) => matches!(
|
||||
CodexClientError::WebSocket(error) => !matches!(
|
||||
error,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
| CodexWebSocketExchangeError::PostSendAmbiguous { .. }
|
||||
| CodexWebSocketExchangeError::SendTimeout { .. }
|
||||
| CodexWebSocketExchangeError::ClosedBeforeTerminal(_)
|
||||
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
|
||||
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. }
|
||||
| CodexWebSocketExchangeError::InitialEventTimeout { .. }
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
| CodexWebSocketExchangeError::Upstream(_)
|
||||
| CodexWebSocketExchangeError::ContinuationUnavailable { .. }
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
@@ -2617,6 +2863,11 @@ fn map_client_error(
|
||||
"websocket_close_{close_code}"
|
||||
)));
|
||||
}
|
||||
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
|
||||
failure.error = failure.error.with_upstream_code(OpaqueUpstreamValue::new(
|
||||
WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(),
|
||||
));
|
||||
}
|
||||
if let Some(client_visible_error) = client_visible_error {
|
||||
failure.error = failure
|
||||
.error
|
||||
@@ -2637,6 +2888,13 @@ fn map_client_error(
|
||||
fn websocket_client_visible_error(
|
||||
error: &CodexWebSocketExchangeError,
|
||||
) -> Option<ClientVisibleUpstreamError> {
|
||||
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
|
||||
return Some(ClientVisibleUpstreamError::new(
|
||||
"websocket connection limit reached",
|
||||
Some(WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned()),
|
||||
Some("websocket_error".to_owned()),
|
||||
));
|
||||
}
|
||||
let close = error.close_before_terminal()?;
|
||||
let message = close
|
||||
.reason()
|
||||
@@ -2711,6 +2969,7 @@ fn map_upstream_failure(
|
||||
}
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
account_failure: account_failure(
|
||||
category,
|
||||
failure.retry_after_seconds,
|
||||
@@ -2822,6 +3081,7 @@ const fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSe
|
||||
| CodexWebSocketExchangeError::SharedConnectFailed
|
||||
| CodexWebSocketExchangeError::ContinuationUnavailable { .. } => UpstreamSendState::NotSent,
|
||||
CodexWebSocketExchangeError::Upstream(_)
|
||||
| CodexWebSocketExchangeError::ConnectionLimitReached
|
||||
| CodexWebSocketExchangeError::InvalidSse(_)
|
||||
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => UpstreamSendState::Sent,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
@@ -2851,6 +3111,7 @@ const fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderEr
|
||||
ProviderErrorKind::Unavailable
|
||||
}
|
||||
CodexWebSocketExchangeError::Upstream(_) => ProviderErrorKind::Unavailable,
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached => ProviderErrorKind::RateLimited,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
| CodexWebSocketExchangeError::Connect(_)
|
||||
| CodexWebSocketExchangeError::PostSendAmbiguous { .. }
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
//! OpenAI 根会话级上游传输降级状态。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
|
||||
use crate::credential::CODEX_ROOT_SESSION_TTL;
|
||||
|
||||
/// Provider 是进程级单例;限制不可显式观测结束的客户端会话占用。
|
||||
const MAX_SESSION_HTTP_FALLBACKS: usize = 16_384;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CodexSessionTransportFallbacks {
|
||||
state: Arc<Mutex<SessionTransportFallbackState>>,
|
||||
ttl: Duration,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SessionTransportFallbackState {
|
||||
http_only: HashMap<ProviderSessionAffinityKey, Instant>,
|
||||
}
|
||||
|
||||
impl Default for CodexSessionTransportFallbacks {
|
||||
fn default() -> Self {
|
||||
Self::with_limits(CODEX_ROOT_SESSION_TTL, MAX_SESSION_HTTP_FALLBACKS)
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexSessionTransportFallbacks {
|
||||
fn with_limits(ttl: Duration, capacity: usize) -> Self {
|
||||
debug_assert!(!ttl.is_zero());
|
||||
debug_assert!(capacity > 0);
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(SessionTransportFallbackState::default())),
|
||||
ttl,
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回该根会话是否已粘性切到 HTTP,并为活跃会话续期。
|
||||
pub(crate) fn is_http_only(&self, key: &ProviderSessionAffinityKey) -> bool {
|
||||
self.is_http_only_at(key, Instant::now())
|
||||
}
|
||||
|
||||
/// 在当前 WebSocket 重试预算耗尽后,为根会话启用粘性 HTTP。
|
||||
///
|
||||
/// 返回 `true` 表示本次发生了禁用状态转换;已禁用会话只续期。
|
||||
pub(crate) fn disable_websocket(&self, key: &ProviderSessionAffinityKey) -> bool {
|
||||
self.disable_websocket_at(key, Instant::now())
|
||||
}
|
||||
|
||||
fn is_http_only_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
|
||||
let mut state = self.lock();
|
||||
let Some(last_seen) = state.http_only.get_mut(key) else {
|
||||
return false;
|
||||
};
|
||||
if now.saturating_duration_since(*last_seen) >= self.ttl {
|
||||
state.http_only.remove(key);
|
||||
return false;
|
||||
}
|
||||
*last_seen = now;
|
||||
true
|
||||
}
|
||||
|
||||
fn disable_websocket_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
|
||||
let mut state = self.lock();
|
||||
let already_disabled = state
|
||||
.http_only
|
||||
.get(key)
|
||||
.is_some_and(|last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
|
||||
if !already_disabled && !state.http_only.contains_key(key) {
|
||||
self.make_room(&mut state, now);
|
||||
}
|
||||
state.http_only.insert(key.clone(), now);
|
||||
!already_disabled
|
||||
}
|
||||
|
||||
fn make_room(&self, state: &mut SessionTransportFallbackState, now: Instant) {
|
||||
if state.http_only.len() < self.capacity {
|
||||
return;
|
||||
}
|
||||
state
|
||||
.http_only
|
||||
.retain(|_, last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
|
||||
if state.http_only.len() < self.capacity {
|
||||
return;
|
||||
}
|
||||
let oldest = state
|
||||
.http_only
|
||||
.iter()
|
||||
.min_by_key(|(_, last_seen)| **last_seen)
|
||||
.map(|(key, _)| key.clone());
|
||||
if let Some(oldest) = oldest {
|
||||
state.http_only.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
fn lock(&self) -> MutexGuard<'_, SessionTransportFallbackState> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
@@ -402,7 +402,7 @@ pub struct CodexRequestContext<'a> {
|
||||
pub beta_features: Option<&'a str>,
|
||||
/// x-responsesapi-include-timing-metrics 头值。
|
||||
pub include_timing_metrics: Option<&'a str>,
|
||||
/// 客户端版本号。
|
||||
/// 下游 `version` 扩展头;存在时上游值由 Desktop 版本画像统一生成。
|
||||
pub version: Option<&'a str>,
|
||||
/// 客户端 window ID。
|
||||
pub codex_window_id: Option<&'a str>,
|
||||
@@ -599,6 +599,7 @@ pub(crate) struct PreparedResponseTransport {
|
||||
pub(super) requirement: TransportRequirement,
|
||||
pub(super) route: PreparedResponseRoute,
|
||||
pub(super) metrics: CodexTransportMetrics,
|
||||
pub(super) defer_websocket_recovery: bool,
|
||||
}
|
||||
|
||||
pub(super) enum PreparedResponseRoute {
|
||||
|
||||
@@ -182,6 +182,20 @@ impl CodexBackendClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Provider 生产路径:保留所有 WebSocket 失败,交由统一重试预算处理。
|
||||
pub(crate) async fn create_response_stream_with_deferred_websocket_recovery(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
let prepared = self
|
||||
.prepare_response_transport(request, context, pool_account_id, false, true)
|
||||
.await?;
|
||||
self.create_response_stream_with_prepared(request, context, prepared)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 在发送 payload 前完成 transport 选择和可取消的 WebSocket opening。
|
||||
#[doc(hidden)]
|
||||
pub(crate) async fn prepare_response_transport_with_pool_account(
|
||||
@@ -190,7 +204,7 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
self.prepare_response_transport(request, context, pool_account_id, false)
|
||||
self.prepare_response_transport(request, context, pool_account_id, false, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -202,12 +216,14 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
force_fresh: bool,
|
||||
defer_websocket_recovery: bool,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
let requirement = transport_requirement(request);
|
||||
if requirement == TransportRequirement::HttpRequired {
|
||||
return Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
route: PreparedResponseRoute::Http,
|
||||
defer_websocket_recovery,
|
||||
metrics: CodexTransportMetrics {
|
||||
decision: Some(CodexTransportDecision::HttpRequired),
|
||||
..CodexTransportMetrics::default()
|
||||
@@ -268,7 +284,8 @@ impl CodexBackendClient {
|
||||
let prepared = match prepared {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error)
|
||||
if requirement.allows_pre_send_http_fallback()
|
||||
if (!defer_websocket_recovery || error.requires_immediate_pool_fallback())
|
||||
&& requirement.allows_pre_send_http_fallback()
|
||||
&& error.allows_pre_send_http_fallback() =>
|
||||
{
|
||||
let decision = http_fallback_decision(&error);
|
||||
@@ -291,6 +308,7 @@ impl CodexBackendClient {
|
||||
transport_decision_wait_ms: Some(wait_ms),
|
||||
..CodexTransportMetrics::default()
|
||||
},
|
||||
defer_websocket_recovery,
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(websocket_exchange_error_to_client_error(error)),
|
||||
@@ -328,6 +346,7 @@ impl CodexBackendClient {
|
||||
prepared,
|
||||
})),
|
||||
metrics,
|
||||
defer_websocket_recovery,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -342,6 +361,7 @@ impl CodexBackendClient {
|
||||
requirement,
|
||||
route,
|
||||
metrics,
|
||||
defer_websocket_recovery,
|
||||
} = prepared;
|
||||
match route {
|
||||
PreparedResponseRoute::Http => self
|
||||
@@ -365,7 +385,8 @@ impl CodexBackendClient {
|
||||
{
|
||||
Ok(exchange) => exchange,
|
||||
Err(error)
|
||||
if requirement.allows_pre_delivery_http_fallback()
|
||||
if !defer_websocket_recovery
|
||||
&& requirement.allows_pre_delivery_http_fallback()
|
||||
&& error.allows_pre_delivery_http_fallback() =>
|
||||
{
|
||||
return self
|
||||
@@ -381,29 +402,20 @@ impl CodexBackendClient {
|
||||
}
|
||||
Err(error) => return Err(websocket_exchange_error_to_client_error(error)),
|
||||
};
|
||||
// 仅普通新链请求允许交付前降级/重试;延续类请求保持旧语义:
|
||||
// 不进入交付边界,错误由 exchange 流原样上抛给下游。
|
||||
if requirement.allows_pre_delivery_http_fallback() {
|
||||
let mut fresh_retry_used = false;
|
||||
loop {
|
||||
match await_websocket_delivery_boundary(&mut exchange).await {
|
||||
Ok(DeliveryBoundary::Ready) => break,
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached {
|
||||
message,
|
||||
raw_frame,
|
||||
}) => {
|
||||
let upstream_error_raw = String::from_utf8_lossy(&raw_frame);
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
websocket_connection_id = %exchange.websocket_connection_id,
|
||||
upstream_transport = "websocket",
|
||||
upstream_error_kind = "websocket_error_frame",
|
||||
upstream_error_raw = %upstream_error_raw,
|
||||
"OpenAI upstream returned a recoverable WebSocket error frame"
|
||||
);
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached)
|
||||
if defer_websocket_recovery =>
|
||||
{
|
||||
return Err(CodexClientError::WebSocket(
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached,
|
||||
));
|
||||
}
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached) => {
|
||||
if fresh_retry_used {
|
||||
// 新连接仍被限流:与官方重试预算耗尽一致,降级同账号 HTTP/SSE。
|
||||
return self
|
||||
.http_fallback_before_delivery(
|
||||
request,
|
||||
@@ -411,20 +423,18 @@ impl CodexBackendClient {
|
||||
requirement,
|
||||
metrics,
|
||||
delivery_wait_started_at,
|
||||
&message,
|
||||
"websocket connection limit reached",
|
||||
)
|
||||
.await
|
||||
;
|
||||
.await;
|
||||
}
|
||||
fresh_retry_used = true;
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
%message,
|
||||
"WebSocket connection limit reached; retrying on a fresh connection"
|
||||
);
|
||||
drop(exchange);
|
||||
match self
|
||||
.prepare_response_transport(request, context, None, true)
|
||||
.prepare_response_transport(request, context, None, true, false)
|
||||
.await
|
||||
{
|
||||
Ok(PreparedResponseTransport {
|
||||
@@ -493,19 +503,21 @@ impl CodexBackendClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) if !defer_websocket_recovery
|
||||
&& error.allows_pre_delivery_http_fallback() =>
|
||||
{
|
||||
return self
|
||||
.fallback_to_http_before_websocket_delivery(
|
||||
request,
|
||||
context,
|
||||
requirement,
|
||||
metrics,
|
||||
delivery_wait_started_at,
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
if error.allows_pre_delivery_http_fallback() {
|
||||
return self
|
||||
.fallback_to_http_before_websocket_delivery(
|
||||
request,
|
||||
context,
|
||||
requirement,
|
||||
metrics,
|
||||
delivery_wait_started_at,
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
return Err(websocket_exchange_error_to_client_error(
|
||||
post_send_ambiguous(error),
|
||||
));
|
||||
@@ -561,7 +573,6 @@ impl CodexBackendClient {
|
||||
delivery_wait_started_at: Instant,
|
||||
error: CodexWebSocketExchangeError,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
log_raw_websocket_close(context, &error);
|
||||
self.http_fallback_before_delivery(
|
||||
request,
|
||||
context,
|
||||
@@ -573,7 +584,6 @@ impl CodexBackendClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// 交付前降级同账号 HTTP/SSE 的公共路径:统一标记 decision 与等待耗时。
|
||||
async fn http_fallback_before_delivery(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
@@ -688,15 +698,16 @@ impl CodexBackendClient {
|
||||
X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
request.memgen_request.as_deref(),
|
||||
);
|
||||
// 客户端确实携带的普通协议头优先;没有携带时才使用上面的 Codex profile 默认值。
|
||||
// 客户端确实携带的普通协议头优先;画像身份头只能由 Codex profile 生成。
|
||||
for name in request.passthrough_headers.keys() {
|
||||
// 上游指纹必须由运行时画像统一生成:originator/User-Agent 即使绕过
|
||||
// API 透传黑名单也不能覆盖画像,避免下游客户端暴露不一致指纹。
|
||||
// 上游指纹必须由运行时画像统一生成:originator/User-Agent/version 即使
|
||||
// 绕过 API 透传黑名单也不能覆盖画像,避免下游客户端暴露不一致指纹。
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"openai-beta"
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
@@ -770,7 +781,17 @@ impl CodexBackendClient {
|
||||
"x-responsesapi-include-timing-metrics",
|
||||
context.include_timing_metrics,
|
||||
);
|
||||
insert_optional_protocol_header(&mut headers, "version", context.version);
|
||||
// 官方 Desktop 的版本画像来自 clientInfo.version。保留下游是否携带
|
||||
// `version` 扩展头的协议形状,但上游值统一使用当前 Desktop 制品版本。
|
||||
if context
|
||||
.version
|
||||
.is_some_and(|version| HeaderValue::from_str(version).is_ok())
|
||||
{
|
||||
headers.insert(
|
||||
HeaderName::from_static("version"),
|
||||
HeaderValue::from_str(&profile.desktop_version)?,
|
||||
);
|
||||
}
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-parent-thread-id",
|
||||
@@ -818,14 +839,8 @@ impl CodexBackendClient {
|
||||
enum DeliveryBoundary {
|
||||
/// 已越过边界,可开始向下游投递。
|
||||
Ready,
|
||||
/// 首个可投递帧是上游连接寿命限制错误;该帧已放回流,
|
||||
/// 调用方可选择丢弃并换新连接重试,或原样投递。
|
||||
ConnectionLimitReached {
|
||||
/// 上游错误说明。
|
||||
message: String,
|
||||
/// 从 WebSocket 文本事件构造的完整 SSE 错误帧。
|
||||
raw_frame: bytes::Bytes,
|
||||
},
|
||||
/// 首个可投递帧是上游连接寿命限制错误。
|
||||
ConnectionLimitReached,
|
||||
}
|
||||
|
||||
async fn await_websocket_delivery_boundary(
|
||||
@@ -836,19 +851,16 @@ async fn await_websocket_delivery_boundary(
|
||||
match exchange.body.next().await {
|
||||
Some(Ok(frame)) if is_websocket_lifecycle_prelude(&frame) => prelude.push(frame),
|
||||
Some(Ok(frame)) => {
|
||||
let limit_message = websocket_connection_limit_message(&frame);
|
||||
let raw_frame = limit_message.as_ref().map(|_| frame.clone());
|
||||
let connection_limit_reached = websocket_connection_limit_message(&frame).is_some();
|
||||
prelude.push(frame);
|
||||
let remaining =
|
||||
std::mem::replace(&mut exchange.body, Box::pin(futures::stream::empty()));
|
||||
exchange.body =
|
||||
Box::pin(futures::stream::iter(prelude.into_iter().map(Ok)).chain(remaining));
|
||||
return Ok(match limit_message {
|
||||
Some(message) => DeliveryBoundary::ConnectionLimitReached {
|
||||
message,
|
||||
raw_frame: raw_frame.unwrap_or_default(),
|
||||
},
|
||||
None => DeliveryBoundary::Ready,
|
||||
return Ok(if connection_limit_reached {
|
||||
DeliveryBoundary::ConnectionLimitReached
|
||||
} else {
|
||||
DeliveryBoundary::Ready
|
||||
});
|
||||
}
|
||||
Some(Err(error)) => return Err(error),
|
||||
@@ -857,34 +869,13 @@ async fn await_websocket_delivery_boundary(
|
||||
exchange.websocket_connection_id,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn log_raw_websocket_close(context: CodexRequestContext<'_>, error: &CodexWebSocketExchangeError) {
|
||||
let Some(close) = error.close_before_terminal() else {
|
||||
return;
|
||||
};
|
||||
let websocket_connection_id = close
|
||||
.connection_id()
|
||||
.map(|connection_id| connection_id.to_string())
|
||||
.unwrap_or_default();
|
||||
let upstream_error_raw = close.reason().unwrap_or_default();
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
websocket_connection_id,
|
||||
upstream_transport = "websocket",
|
||||
upstream_error_kind = "websocket_close",
|
||||
upstream_close_code = ?close.code(),
|
||||
upstream_error_raw,
|
||||
upstream_error_raw_present = close.reason().is_some(),
|
||||
"OpenAI upstream WebSocket closed before delivery"
|
||||
);
|
||||
}
|
||||
|
||||
fn is_websocket_lifecycle_prelude(frame: &[u8]) -> bool {
|
||||
frame.starts_with(b"event: response.created\n")
|
||||
|| frame.starts_with(b"event: response.in_progress\n")
|
||||
|
||||
@@ -72,7 +72,7 @@ pub struct CodexResponsesRequest {
|
||||
pub turn_metadata: Option<String>,
|
||||
/// beta features 透传头。
|
||||
pub beta_features: Option<String>,
|
||||
/// 客户端版本头。
|
||||
/// 下游 `version` 扩展头;存在时上游值由 Desktop 版本画像统一生成。
|
||||
pub version: Option<String>,
|
||||
/// timing metrics 透传头。
|
||||
pub include_timing_metrics: Option<String>,
|
||||
|
||||
@@ -66,6 +66,9 @@ pub enum CodexWebSocketExchangeError {
|
||||
/// 上游 WebSocket 错误帧。
|
||||
#[error("{0}")]
|
||||
Upstream(Box<CodexWebSocketUpstreamError>),
|
||||
/// 上游要求结束当前 WebSocket 连接并在新连接上重试。
|
||||
#[error("websocket connection limit reached")]
|
||||
ConnectionLimitReached,
|
||||
/// 请求依赖的连接本地 previous response 无法在当前连接满足。
|
||||
#[error("websocket continuation unavailable: {reason}")]
|
||||
ContinuationUnavailable {
|
||||
@@ -108,6 +111,7 @@ pub struct CodexWebSocketCloseError {
|
||||
connection_id: Option<Uuid>,
|
||||
code: Option<u16>,
|
||||
reason: Option<String>,
|
||||
last_event_type: Option<String>,
|
||||
}
|
||||
|
||||
impl CodexWebSocketCloseError {
|
||||
@@ -116,6 +120,7 @@ impl CodexWebSocketCloseError {
|
||||
connection_id: None,
|
||||
code,
|
||||
reason,
|
||||
last_event_type: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,6 +129,11 @@ impl CodexWebSocketCloseError {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_last_event_type(mut self, event_type: Option<String>) -> Self {
|
||||
self.last_event_type = event_type;
|
||||
self
|
||||
}
|
||||
|
||||
/// 返回承载该 close frame 的 WebSocket 连接标识。
|
||||
#[must_use]
|
||||
pub const fn connection_id(&self) -> Option<Uuid> {
|
||||
@@ -141,6 +151,12 @@ impl CodexWebSocketCloseError {
|
||||
pub fn reason(&self) -> Option<&str> {
|
||||
self.reason.as_deref()
|
||||
}
|
||||
|
||||
/// 返回关闭前最后一条经过白名单校验的事件类型;不包含事件 payload。
|
||||
#[must_use]
|
||||
pub fn last_event_type(&self) -> Option<&str> {
|
||||
self.last_event_type.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexWebSocketCloseError {
|
||||
@@ -150,6 +166,7 @@ impl fmt::Debug for CodexWebSocketCloseError {
|
||||
.field("connection_id", &self.connection_id)
|
||||
.field("code", &self.code)
|
||||
.field("reason", &self.reason.as_ref().map(|_| "<redacted>"))
|
||||
.field("last_event_type", &self.last_event_type)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -214,9 +231,12 @@ impl CodexWebSocketExchangeError {
|
||||
connection_id: Uuid,
|
||||
code: Option<u16>,
|
||||
reason: Option<String>,
|
||||
last_event_type: Option<String>,
|
||||
) -> Self {
|
||||
Self::ClosedBeforeTerminal(
|
||||
CodexWebSocketCloseError::new(code, reason).with_connection_id(connection_id),
|
||||
CodexWebSocketCloseError::new(code, reason)
|
||||
.with_connection_id(connection_id)
|
||||
.with_last_event_type(last_event_type),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -269,6 +289,11 @@ impl CodexWebSocketExchangeError {
|
||||
)
|
||||
}
|
||||
|
||||
/// 精确连接状态不可用属于本地池路由事实,不消耗上游 WS 重试预算。
|
||||
pub(in crate::transport) fn requires_immediate_pool_fallback(&self) -> bool {
|
||||
matches!(self, Self::ContinuationUnavailable { .. })
|
||||
}
|
||||
|
||||
/// 首个业务事件交付前,普通 WebSocket 失败可切到同账号 HTTP。
|
||||
pub(in crate::transport) fn allows_pre_delivery_http_fallback(&self) -> bool {
|
||||
match self {
|
||||
|
||||
@@ -29,20 +29,32 @@ pub(super) enum ExchangeAction {
|
||||
Ignore,
|
||||
}
|
||||
|
||||
pub(super) struct ReducedWebSocketEvent {
|
||||
pub(super) action: ExchangeAction,
|
||||
pub(super) diagnostic_event_type: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn reduce_websocket_event(
|
||||
raw: &str,
|
||||
metadata: &mut CodexWebSocketConnectionMetadata,
|
||||
continuation: &mut WebSocketContinuationState,
|
||||
) -> Result<ExchangeAction, CodexWebSocketExchangeError> {
|
||||
) -> Result<ReducedWebSocketEvent, CodexWebSocketExchangeError> {
|
||||
// 每帧只解析一次 JSON,后续提取全部复用同一 Value;
|
||||
// 不可解析的帧不承载可路由的事件类型,忽略。
|
||||
let Ok(value) = serde_json::from_str::<Value>(raw) else {
|
||||
return Ok(ExchangeAction::Ignore);
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::Ignore,
|
||||
diagnostic_event_type: None,
|
||||
});
|
||||
};
|
||||
let diagnostic_event_type = diagnostic_event_type(websocket_event_type(&value));
|
||||
if let Some(parsed) = events::parse_rate_limits_event(&value) {
|
||||
let headers = events::rate_limits_to_header_pairs(&parsed);
|
||||
metadata.rate_limit_headers.extend(headers);
|
||||
return Ok(ExchangeAction::RateLimits(parsed));
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::RateLimits(parsed),
|
||||
diagnostic_event_type,
|
||||
});
|
||||
}
|
||||
|
||||
response_meta::merge_response_metadata(
|
||||
@@ -51,7 +63,10 @@ pub(super) fn reduce_websocket_event(
|
||||
);
|
||||
if let Some(turn_state) = websocket_metadata_turn_state(&value) {
|
||||
metadata.turn_state = Some(turn_state.clone());
|
||||
return Ok(ExchangeAction::TurnState(turn_state));
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::TurnState(turn_state),
|
||||
diagnostic_event_type,
|
||||
});
|
||||
}
|
||||
|
||||
let event = websocket_event_type(&value);
|
||||
@@ -67,8 +82,24 @@ pub(super) fn reduce_websocket_event(
|
||||
Some("response.failed" | "error") => Some(WebSocketTerminalKind::Failed),
|
||||
_ => None,
|
||||
};
|
||||
Ok(match websocket_event_frame(&value, raw) {
|
||||
let action = match websocket_event_frame(&value, raw) {
|
||||
Some(frame) => ExchangeAction::Forward { frame, terminal },
|
||||
None => ExchangeAction::Ignore,
|
||||
};
|
||||
Ok(ReducedWebSocketEvent {
|
||||
action,
|
||||
diagnostic_event_type,
|
||||
})
|
||||
}
|
||||
|
||||
fn diagnostic_event_type(event_type: Option<&str>) -> Option<String> {
|
||||
const MAX_EVENT_TYPE_BYTES: usize = 128;
|
||||
|
||||
let event_type = event_type?;
|
||||
(!event_type.is_empty()
|
||||
&& event_type.len() <= MAX_EVENT_TYPE_BYTES
|
||||
&& event_type.bytes().all(|byte| {
|
||||
byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'_' | b'-')
|
||||
}))
|
||||
.then(|| event_type.to_owned())
|
||||
}
|
||||
|
||||
@@ -143,6 +143,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
.map(|pool_return| std::mem::take(&mut pool_return.continuation))
|
||||
.unwrap_or_default();
|
||||
let mut saw_upstream_activity = false;
|
||||
let mut last_event_type = None;
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
biased;
|
||||
@@ -219,6 +220,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
websocket_connection_id,
|
||||
frame.as_ref().map(|frame| u16::from(frame.code)),
|
||||
frame.map(|frame| frame.reason.to_string()),
|
||||
last_event_type.clone(),
|
||||
);
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
@@ -230,18 +232,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let (frame, terminal) = match reduce_websocket_event(&raw, &mut metadata, &mut continuation)
|
||||
{
|
||||
Ok(ExchangeAction::RateLimits(rate_limits)) => {
|
||||
rate_limit_updates.lock().await.push(rate_limits);
|
||||
continue;
|
||||
}
|
||||
Ok(ExchangeAction::TurnState(turn_state)) => {
|
||||
*turn_state_update.lock().await = Some(turn_state);
|
||||
continue;
|
||||
}
|
||||
Ok(ExchangeAction::Forward { frame, terminal }) => (frame, terminal),
|
||||
Ok(ExchangeAction::Ignore) => continue,
|
||||
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
|
||||
Ok(reduced) => reduced,
|
||||
Err(error) => {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -253,6 +245,21 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Some(event_type) = reduced.diagnostic_event_type {
|
||||
last_event_type = Some(event_type);
|
||||
}
|
||||
let (frame, terminal) = match reduced.action {
|
||||
ExchangeAction::RateLimits(rate_limits) => {
|
||||
rate_limit_updates.lock().await.push(rate_limits);
|
||||
continue;
|
||||
}
|
||||
ExchangeAction::TurnState(turn_state) => {
|
||||
*turn_state_update.lock().await = Some(turn_state);
|
||||
continue;
|
||||
}
|
||||
ExchangeAction::Forward { frame, terminal } => (frame, terminal),
|
||||
ExchangeAction::Ignore => continue,
|
||||
};
|
||||
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -300,16 +307,17 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = upstream_close.map_or_else(
|
||||
|| {
|
||||
CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
websocket_connection_id,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
},
|
||||
CodexWebSocketExchangeError::ClosedBeforeTerminal,
|
||||
);
|
||||
let error = match upstream_close {
|
||||
Some(close) => CodexWebSocketExchangeError::ClosedBeforeTerminal(
|
||||
close.with_last_event_type(last_event_type),
|
||||
),
|
||||
None => CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
websocket_connection_id,
|
||||
None,
|
||||
None,
|
||||
last_event_type,
|
||||
),
|
||||
};
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
} else {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use provider_openai::config::{CodexWireProfileConfig, OpenAiConfig};
|
||||
use provider_openai::config::{
|
||||
CodexWireProfileConfig, DEFAULT_STREAM_MAX_RETRIES, MAX_STREAM_MAX_RETRIES, OpenAiConfig,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn openai_config_builds_the_audited_wire_profile() {
|
||||
@@ -95,6 +97,7 @@ fn openai_config_restricts_upstream_base_url_to_https_or_loopback_http() {
|
||||
#[test]
|
||||
fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
let config = OpenAiConfig::default();
|
||||
assert_eq!(DEFAULT_STREAM_MAX_RETRIES, 3);
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -107,6 +110,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
config.auth.refresh_enabled,
|
||||
config.auth.oauth_client_id.as_str(),
|
||||
config.auth.oauth_token_endpoint.as_str(),
|
||||
config.stream_max_retries(),
|
||||
),
|
||||
(
|
||||
"https://chatgpt.com/backend-api",
|
||||
@@ -118,6 +122,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
true,
|
||||
"app_EMoamEEZ73f0CkXaXp7hrann",
|
||||
"https://auth.openai.com/oauth/token",
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -126,6 +131,17 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_stream_retry_budget_uses_the_official_hard_cap() {
|
||||
let mut config = OpenAiConfig::default();
|
||||
config.stream_max_retries = MAX_STREAM_MAX_RETRIES + 1;
|
||||
|
||||
assert_eq!(
|
||||
config.stream_max_retries(),
|
||||
u32::try_from(MAX_STREAM_MAX_RETRIES).expect("retry cap fits u32")
|
||||
);
|
||||
}
|
||||
|
||||
fn valid_config() -> OpenAiConfig {
|
||||
let mut config = OpenAiConfig::default();
|
||||
config.wire_profile = CodexWireProfileConfig {
|
||||
|
||||
@@ -34,6 +34,7 @@ use gateway_core::routing::{
|
||||
ProviderModel, PublicModelId, RoutingContext, RuntimeAccount, RuntimeAccountDirectory,
|
||||
RuntimeSnapshot, UpstreamModelId,
|
||||
};
|
||||
use provider_openai::config::DEFAULT_STREAM_MAX_RETRIES;
|
||||
use provider_openai::credential::{
|
||||
CodexCookiePolicy, CodexCredentialCatalogService, CodexCredentialQuotaService,
|
||||
CodexCredentialSelector, ImportCodexOAuthCredential,
|
||||
@@ -148,7 +149,26 @@ fn provider_with_affinity(
|
||||
}
|
||||
|
||||
fn provider_with_base_url(store: &Arc<MemoryAccountStore>, base_url: String) -> CodexProvider {
|
||||
provider_with_affinity_and_base_url(store, Arc::new(MemorySessionAffinity::default()), base_url)
|
||||
provider_with_base_url_and_retry_budget(
|
||||
store,
|
||||
base_url,
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
}
|
||||
|
||||
fn provider_with_base_url_and_retry_budget(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
base_url: String,
|
||||
stream_max_retries: u32,
|
||||
) -> CodexProvider {
|
||||
provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
base_url,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
stream_max_retries,
|
||||
)
|
||||
.0
|
||||
}
|
||||
|
||||
fn provider_with_leases(
|
||||
@@ -187,6 +207,7 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
session_affinity,
|
||||
base_url,
|
||||
leases,
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
.0
|
||||
}
|
||||
@@ -196,6 +217,7 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
stream_max_retries: u32,
|
||||
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder()
|
||||
@@ -239,6 +261,7 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
profile,
|
||||
base_url,
|
||||
websocket_pool,
|
||||
stream_max_retries,
|
||||
)
|
||||
.expect("official OpenAI provider");
|
||||
(provider, quota)
|
||||
@@ -769,21 +792,25 @@ async fn truncated_chunked_sse_server() -> (String, tokio::task::JoinHandle<()>)
|
||||
(base_url, server)
|
||||
}
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) {
|
||||
async fn capture_http_request(stream: &mut TcpStream) -> Vec<u8> {
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 1024];
|
||||
loop {
|
||||
let read = stream.read(&mut buffer).await.expect("read HTTP request");
|
||||
if read == 0 {
|
||||
return;
|
||||
return request;
|
||||
}
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
if request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
return;
|
||||
return request;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) {
|
||||
drop(capture_http_request(stream).await);
|
||||
}
|
||||
|
||||
async fn write_http_chunk(stream: &mut TcpStream, body: &str) {
|
||||
stream
|
||||
.write_all(format!("{:X}\r\n", body.len()).as_bytes())
|
||||
@@ -1131,7 +1158,7 @@ async fn selection_infrastructure_errors_have_a_distinct_classification() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
async fn websocket_close_after_delivery_preserves_details_without_enabling_sticky_http() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
@@ -1170,11 +1197,49 @@ async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
}))
|
||||
.await
|
||||
.expect("close WebSocket");
|
||||
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept next-turn WebSocket connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("next-turn WebSocket request")
|
||||
.expect("valid next-turn WebSocket request");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_after_committed_close",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("complete next-turn WebSocket request");
|
||||
});
|
||||
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let first_operation = Operation::Generate(generate_with_session_context(
|
||||
"committed-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
));
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", generate_operation()),
|
||||
planned_request("openai", first_operation),
|
||||
context("req_websocket_close", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
@@ -1186,13 +1251,16 @@ async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
None => panic!("WebSocket close must surface a provider error"),
|
||||
}
|
||||
};
|
||||
server.await.expect("WebSocket server");
|
||||
drop(stream);
|
||||
|
||||
assert!(!format!("{error:?}").contains("message too big"));
|
||||
assert!(!error.to_string().contains("message too big"));
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Transport);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
|
||||
assert!(error.allows_pre_delivery_retry());
|
||||
assert!(
|
||||
!error.allows_pre_delivery_retry(),
|
||||
"a close after the delivery boundary must not request a hidden replay"
|
||||
);
|
||||
let detail = error
|
||||
.client_visible_upstream_error()
|
||||
.expect("WebSocket close detail");
|
||||
@@ -1203,10 +1271,28 @@ async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some("websocket_close_1009")
|
||||
);
|
||||
|
||||
let second_operation = Operation::Generate(generate_with_session_context(
|
||||
"committed-websocket-session",
|
||||
Some("thread-second"),
|
||||
None,
|
||||
));
|
||||
let mut next_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", second_operation),
|
||||
context("req_after_committed_close", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("post-delivery close must not enable sticky HTTP fallback");
|
||||
assert_eq!(next_stream.metadata().transport().as_str(), "websocket");
|
||||
while let Some(event) = next_stream.next().await {
|
||||
event.expect("next turn WebSocket response");
|
||||
}
|
||||
server.await.expect("WebSocket server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_normal_close_before_semantic_output_requests_same_account_http_fallback() {
|
||||
async fn websocket_retry_budget_exhaustion_is_sticky_only_for_the_current_session() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
@@ -1217,79 +1303,350 @@ async fn websocket_normal_close_before_semantic_output_requests_same_account_htt
|
||||
listener.local_addr().expect("listener address")
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
for retry_count in 0..=2 {
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept WebSocket retry connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("WebSocket request")
|
||||
.expect("valid WebSocket request");
|
||||
for event in [
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": format!("resp_structural_{retry_count}"), "model": "gpt-5.4"}
|
||||
}),
|
||||
json!({
|
||||
"type": "response.in_progress",
|
||||
"response": {"id": format!("resp_structural_{retry_count}"), "model": "gpt-5.4"}
|
||||
}),
|
||||
] {
|
||||
websocket
|
||||
.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.expect("send structural WebSocket event");
|
||||
}
|
||||
if retry_count == 1 {
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "websocket_connection_limit_reached",
|
||||
"message": "create a new websocket connection"
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send connection-limit error");
|
||||
continue;
|
||||
}
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"id": format!("msg_structural_{retry_count}"),
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "in_progress",
|
||||
"content": []
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send structural output event");
|
||||
let close_code = match retry_count {
|
||||
0 => CloseCode::Normal,
|
||||
_ => CloseCode::Error,
|
||||
};
|
||||
websocket
|
||||
.close(Some(CloseFrame {
|
||||
code: close_code,
|
||||
reason: "".into(),
|
||||
}))
|
||||
.await
|
||||
.expect("close WebSocket retry attempt");
|
||||
}
|
||||
|
||||
let (mut http, _) = listener.accept().await.expect("accept sticky HTTP request");
|
||||
let request = capture_http_request(&mut http).await;
|
||||
let request_head = String::from_utf8_lossy(&request).to_ascii_lowercase();
|
||||
assert!(!request_head.contains("upgrade: websocket"));
|
||||
http.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
|
||||
CAPTURE_COMPLETED_SSE.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write sticky HTTP response");
|
||||
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept WebSocket connection");
|
||||
.expect("accept other-session WebSocket connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("WebSocket request")
|
||||
.expect("valid WebSocket request");
|
||||
for event in [
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": "resp_structural", "model": "gpt-5.4"}
|
||||
}),
|
||||
json!({
|
||||
"type": "response.in_progress",
|
||||
"response": {"id": "resp_structural", "model": "gpt-5.4"}
|
||||
}),
|
||||
json!({
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"id": "msg_structural",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "in_progress",
|
||||
"content": []
|
||||
}
|
||||
}),
|
||||
] {
|
||||
websocket
|
||||
.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.expect("send structural WebSocket event");
|
||||
}
|
||||
.expect("other-session WebSocket request")
|
||||
.expect("valid other-session WebSocket request");
|
||||
websocket
|
||||
.close(Some(CloseFrame {
|
||||
code: CloseCode::Normal,
|
||||
reason: "".into(),
|
||||
}))
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_other_session",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("close WebSocket");
|
||||
.expect("complete other-session WebSocket request");
|
||||
});
|
||||
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
let provider = provider_with_base_url_and_retry_budget(&store, base_url, 2);
|
||||
let mut saw_websocket_observation = false;
|
||||
for retry_count in 0..=2 {
|
||||
let operation = Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
));
|
||||
let attempt_context = match NonZeroU32::new(retry_count) {
|
||||
Some(retry_index) => context(
|
||||
&format!("req_websocket_normal_close_{retry_count}"),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.with_transport(AttemptTransport::Retry(retry_index)),
|
||||
None => context("req_websocket_normal_close_0", CancellationToken::new()),
|
||||
};
|
||||
let mut stream = provider
|
||||
.execute(planned_request("openai", operation), attempt_context)
|
||||
.await
|
||||
.expect("prepare WebSocket provider stream");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(event)) => {
|
||||
if let Some(observation) = event.response_observation() {
|
||||
saw_websocket_observation = true;
|
||||
assert_eq!(observation.status_code(), None);
|
||||
if let Some(provider_metadata) = observation.provider_metadata() {
|
||||
let metadata: Value = serde_json::from_str(provider_metadata.as_json())
|
||||
.expect("OpenAI provider metadata JSON");
|
||||
assert_eq!(
|
||||
metadata.get("upstreamStatus").and_then(Value::as_u64),
|
||||
Some(101),
|
||||
"successful upgrade remains provider-owned diagnostics"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!event.has_client_event(),
|
||||
"structural events must remain behind the replay boundary"
|
||||
);
|
||||
}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("WebSocket close must surface a provider error"),
|
||||
}
|
||||
};
|
||||
drop(stream);
|
||||
|
||||
if retry_count < 2 {
|
||||
let Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, delay }) =
|
||||
error.pre_delivery_retry()
|
||||
else {
|
||||
panic!("expected a same-account WebSocket retry: {error:?}");
|
||||
};
|
||||
assert_eq!(retry_index.get(), retry_count + 1);
|
||||
assert!(!delay.is_zero());
|
||||
} else {
|
||||
assert_eq!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback),
|
||||
"unexpected provider error at retry {retry_count}: {error:?}"
|
||||
);
|
||||
}
|
||||
let expected_close_code = match retry_count {
|
||||
0 => "websocket_close_1000",
|
||||
1 => "websocket_connection_limit_reached",
|
||||
_ => "websocket_close_1011",
|
||||
};
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some(expected_close_code)
|
||||
);
|
||||
}
|
||||
|
||||
assert!(saw_websocket_observation);
|
||||
|
||||
let second_operation = Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-second"),
|
||||
None,
|
||||
));
|
||||
let mut sticky_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", generate_operation()),
|
||||
context("req_websocket_normal_close", CancellationToken::new()),
|
||||
planned_request("openai", second_operation),
|
||||
context("req_websocket_sticky_http", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare WebSocket provider stream");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(event)) => assert!(
|
||||
!event.has_client_event(),
|
||||
"structural events must remain behind the replay boundary"
|
||||
.expect("same session should prepare an HTTP stream");
|
||||
assert_eq!(sticky_stream.metadata().transport().as_str(), "http_sse");
|
||||
while let Some(event) = sticky_stream.next().await {
|
||||
event.expect("sticky HTTP response");
|
||||
}
|
||||
drop(sticky_stream);
|
||||
|
||||
let other_operation = Operation::Generate(generate_with_session_context(
|
||||
"other-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
));
|
||||
let mut other_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", other_operation),
|
||||
context("req_other_session_websocket", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("another session should still prepare a WebSocket stream");
|
||||
assert_eq!(other_stream.metadata().transport().as_str(), "websocket");
|
||||
while let Some(event) = other_stream.next().await {
|
||||
event.expect("other-session WebSocket response");
|
||||
}
|
||||
drop(other_stream);
|
||||
|
||||
let warmup_operation = Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("warm up")),
|
||||
("session_id".to_owned(), json!("sticky-websocket-session")),
|
||||
("generate".to_owned(), json!(false)),
|
||||
("store".to_owned(), json!(false)),
|
||||
]),
|
||||
)
|
||||
.expect("warmup payload"),
|
||||
));
|
||||
let warmup_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", warmup_operation),
|
||||
context(
|
||||
"req_sticky_session_required_warmup",
|
||||
CancellationToken::new(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("required warmup must bypass sticky HTTP fallback");
|
||||
assert_eq!(warmup_stream.metadata().transport().as_str(), "websocket");
|
||||
drop(warmup_stream);
|
||||
server.await.expect("WebSocket and HTTP server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_upgrade_required_immediately_enables_session_http_fallback() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind upstream listener");
|
||||
let base_url = format!(
|
||||
"http://{}",
|
||||
listener.local_addr().expect("listener address")
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut opening, _) = listener.accept().await.expect("accept WS opening");
|
||||
let request = capture_http_request(&mut opening).await;
|
||||
assert!(String::from_utf8_lossy(&request).starts_with("GET /codex/responses"));
|
||||
let body = r#"{"error":{"code":"upgrade_required","message":"use HTTP"}}"#;
|
||||
opening
|
||||
.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 426 Upgrade Required\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write 426 response");
|
||||
|
||||
let (mut http, _) = listener.accept().await.expect("accept sticky HTTP request");
|
||||
let request = capture_http_request(&mut http).await;
|
||||
assert!(String::from_utf8_lossy(&request).starts_with("POST /codex/responses"));
|
||||
http.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
|
||||
CAPTURE_COMPLETED_SSE.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write HTTP fallback response");
|
||||
});
|
||||
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let operation = || {
|
||||
Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
))
|
||||
};
|
||||
let mut first = provider
|
||||
.execute(
|
||||
planned_request("openai", operation()),
|
||||
context("req_upgrade_required", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare WebSocket stream");
|
||||
let error = loop {
|
||||
match first.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("WebSocket close must surface a provider error"),
|
||||
None => panic!("426 opening must surface a fallback signal"),
|
||||
}
|
||||
};
|
||||
server.await.expect("WebSocket server");
|
||||
|
||||
assert_eq!(error.upstream_status(), Some(426));
|
||||
assert_eq!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback),
|
||||
"unexpected provider error: {error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some("websocket_close_1000")
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback)
|
||||
);
|
||||
drop(first);
|
||||
|
||||
let mut second = provider
|
||||
.execute(
|
||||
planned_request("openai", operation()),
|
||||
context("req_upgrade_required_next_turn", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("same session should select HTTP");
|
||||
assert_eq!(second.metadata().transport().as_str(), "http_sse");
|
||||
while let Some(event) = second.next().await {
|
||||
event.expect("HTTP fallback response");
|
||||
}
|
||||
server.await.expect("upstream server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2230,6 +2587,7 @@ async fn affinity_quota_switch_should_clear_old_turn_state_without_a_provider_st
|
||||
Arc::clone(&affinity),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
);
|
||||
let session_id = "stable-affinity-switch-session";
|
||||
let first_payload = ProtocolPayload::json_object(
|
||||
@@ -3585,6 +3943,7 @@ async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
);
|
||||
quota
|
||||
.prepare_scheduling(std::slice::from_ref(&account))
|
||||
|
||||
@@ -32,6 +32,8 @@ fn request_with_opaque_headers(use_websocket: bool) -> CodexResponsesRequest {
|
||||
],
|
||||
["user-agent", STANDARD.encode(b"Codex future-client")],
|
||||
["originator", STANDARD.encode(b"future-originator")],
|
||||
["version", STANDARD.encode(b"26.999.10001")],
|
||||
["version", STANDARD.encode(b"26.999.10002")],
|
||||
["openai-beta", STANDARD.encode(b"future_responses=v2")],
|
||||
["openai-beta", STANDARD.encode(b"future_tools=v3")],
|
||||
[
|
||||
@@ -321,7 +323,7 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
("x-codex-turn-metadata", "{\"thread_source\":\"subagent\"}"),
|
||||
("x-codex-beta-features", "feature-a"),
|
||||
("x-responsesapi-include-timing-metrics", "true"),
|
||||
("version", "26.318.11754"),
|
||||
("version", "1.2.3"),
|
||||
("x-codex-window-id", "cw_derived"),
|
||||
("x-codex-parent-thread-id", "parent-456"),
|
||||
("x-openai-subagent", "future_codex_mode"),
|
||||
@@ -356,14 +358,17 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
request
|
||||
});
|
||||
let request = request_with_opaque_headers(false);
|
||||
let expected_user_agent = test_wire_profile().snapshot().user_agent();
|
||||
let profile = test_wire_profile();
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_desktop_version = profile_snapshot.desktop_version;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("HTTP client"),
|
||||
format!("http://{address}"),
|
||||
test_wire_profile(),
|
||||
profile,
|
||||
);
|
||||
|
||||
client
|
||||
@@ -374,6 +379,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
account_id: Some("lease-account"),
|
||||
installation_id: Some("lease-installation"),
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
version: Some("26.825.51511"),
|
||||
..request_context("req_opaque_http", Some("lease-account"))
|
||||
},
|
||||
)
|
||||
@@ -413,6 +419,10 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
raw_header_values(&raw, "originator"),
|
||||
vec![b"codex_cli_rs".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, "version"),
|
||||
vec![expected_desktop_version.into_bytes()]
|
||||
);
|
||||
for dropped in [
|
||||
"openai-beta",
|
||||
"x-oai-attestation",
|
||||
@@ -490,6 +500,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
account_id: Some("lease-account"),
|
||||
installation_id: Some("lease-installation"),
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
version: Some("26.825.51511"),
|
||||
..request_context("req_opaque_ws", Some("lease-account"))
|
||||
},
|
||||
)
|
||||
@@ -526,6 +537,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
vec![test_wire_profile().snapshot().user_agent().into_bytes()]
|
||||
);
|
||||
assert_eq!(values("originator"), vec![b"codex_cli_rs".to_vec()]);
|
||||
assert_eq!(values("version"), vec![b"1.2.3".to_vec()]);
|
||||
assert_eq!(
|
||||
values("x-openai-internal-codex-residency"),
|
||||
vec![b"us".to_vec()]
|
||||
@@ -569,7 +581,9 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
request.codex_window_id = Some("cw_1".to_owned());
|
||||
request.parent_thread_id = Some("parent-1".to_owned());
|
||||
let profile = test_wire_profile();
|
||||
let expected_user_agent = profile.snapshot().user_agent();
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_desktop_version = profile_snapshot.desktop_version;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -629,6 +643,10 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
Some("turn-state"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
read_header_value(&raw_request, "version"),
|
||||
Some(expected_desktop_version.as_str())
|
||||
);
|
||||
for required in [
|
||||
"authorization",
|
||||
"chatgpt-account-id",
|
||||
|
||||
@@ -1507,6 +1507,17 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "secret@example.com",
|
||||
"payload": "must not become a diagnostic event type"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
});
|
||||
let mut request = codex_request("gpt-5.5", "be brief", Vec::new());
|
||||
@@ -1533,14 +1544,27 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
|
||||
.expect("stream should yield partial frame")
|
||||
.expect("partial frame should be valid");
|
||||
assert!(std::str::from_utf8(&first).unwrap().contains("partial"));
|
||||
let error = response
|
||||
.body
|
||||
.next()
|
||||
.await
|
||||
.expect("stream should yield close-before-terminal error")
|
||||
.expect_err("close before terminal should be an error");
|
||||
let error = loop {
|
||||
match response.body.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("stream should yield close-before-terminal error"),
|
||||
}
|
||||
};
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
|
||||
source: Some(source),
|
||||
..
|
||||
}) = &error
|
||||
else {
|
||||
panic!("close-before-terminal should preserve its typed source: {error:?}");
|
||||
};
|
||||
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = source.as_ref() else {
|
||||
panic!("post-send ambiguity should wrap a close-before-terminal error: {source:?}");
|
||||
};
|
||||
assert_eq!(close.last_event_type(), Some("response.output_text.delta"));
|
||||
|
||||
std::assert_matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
|
||||
|
||||
@@ -63,25 +63,25 @@ openai:
|
||||
# 版本元组来自同一个官方 macOS Desktop ZIP;Provider 每 24 小时检查 appcast,
|
||||
# 仅在包内 Core 核验和 Redis 条件覆盖都成功后原子更新。
|
||||
originator: 'Codex Desktop'
|
||||
codex_version: '0.147.0-alpha.6.6'
|
||||
desktop_version: '26.803.81509'
|
||||
desktop_build: '6415'
|
||||
codex_version: '0.151.0-alpha.7.2'
|
||||
desktop_version: '26.825.51511'
|
||||
desktop_build: '7377'
|
||||
os_type: 'Mac OS'
|
||||
os_version: '15.7.1'
|
||||
arch: 'arm64'
|
||||
terminal: 'unknown'
|
||||
verified_at: '2026-08-03T00:00:00+08:00'
|
||||
verified_at: '2026-08-31T00:00:00+08:00'
|
||||
|
||||
xai:
|
||||
# OAuth 端点、额度与模型目录策略由 xAI Provider 固定管理;这里只声明请求画像启动基线。
|
||||
wire_profile:
|
||||
# Provider 每 24 小时读取官方 npm metadata,仅在进程内更新 client_version。
|
||||
client_identifier: 'grok-shell'
|
||||
client_version: '0.2.118'
|
||||
client_identifier: 'GrokShell'
|
||||
client_version: '1.0.13'
|
||||
client_mode: 'headless'
|
||||
target_os: 'linux'
|
||||
target_arch: 'x86_64'
|
||||
verified_at: '2026-07-21T00:00:00+08:00'
|
||||
verified_at: '2026-08-31T00:00:00+08:00'
|
||||
|
||||
# Compose 凭据桥接区;密码仍只维护在 store 中。
|
||||
services:
|
||||
|
||||
+53
-35
@@ -1,47 +1,65 @@
|
||||
import type { ESLint, Rule } from 'eslint'
|
||||
import antfu from '@antfu/eslint-config'
|
||||
|
||||
const templateStylePlugin = {
|
||||
rules: {
|
||||
'single-line-simple-interpolation': {
|
||||
meta: {
|
||||
type: 'layout',
|
||||
fixable: 'whitespace',
|
||||
schema: [],
|
||||
messages: {
|
||||
expectedSingleLine: 'Expected a simple mustache interpolation to stay on one line.',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
const { parserServices } = context.sourceCode
|
||||
if (!parserServices.defineTemplateBodyVisitor)
|
||||
return {}
|
||||
type LocatedRuleNode = Rule.Node & {
|
||||
loc: NonNullable<Rule.Node['loc']>
|
||||
}
|
||||
|
||||
return parserServices.defineTemplateBodyVisitor({
|
||||
'VExpressionContainer[expression!=null]': function (node) {
|
||||
if (node.parent.type !== 'VElement')
|
||||
return
|
||||
interface TemplateExpressionContainer {
|
||||
type: 'VExpressionContainer'
|
||||
parent: { type: string }
|
||||
expression: LocatedRuleNode
|
||||
loc: NonNullable<Rule.Node['loc']>
|
||||
range: NonNullable<Rule.Node['range']>
|
||||
}
|
||||
|
||||
const expression = node.expression
|
||||
if (
|
||||
expression.loc.start.line !== expression.loc.end.line
|
||||
|| node.loc.start.line === node.loc.end.line
|
||||
) {
|
||||
return
|
||||
}
|
||||
const singleLineSimpleInterpolationRule = {
|
||||
meta: {
|
||||
type: 'layout',
|
||||
fixable: 'whitespace',
|
||||
schema: [],
|
||||
messages: {
|
||||
expectedSingleLine: 'Expected a simple mustache interpolation to stay on one line.',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
const { parserServices } = context.sourceCode
|
||||
if (!parserServices.defineTemplateBodyVisitor)
|
||||
return {}
|
||||
|
||||
context.report({
|
||||
node,
|
||||
messageId: 'expectedSingleLine',
|
||||
fix(fixer) {
|
||||
return fixer.replaceText(node, `{{ ${context.sourceCode.getText(expression)} }}`)
|
||||
},
|
||||
})
|
||||
return parserServices.defineTemplateBodyVisitor({
|
||||
'VExpressionContainer[expression!=null]': function (node: TemplateExpressionContainer) {
|
||||
if (node.parent.type !== 'VElement')
|
||||
return
|
||||
|
||||
const expression = node.expression
|
||||
if (
|
||||
expression.loc.start.line !== expression.loc.end.line
|
||||
|| node.loc.start.line === node.loc.end.line
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
context.report({
|
||||
node,
|
||||
messageId: 'expectedSingleLine',
|
||||
fix(fixer) {
|
||||
return fixer.replaceTextRange(
|
||||
node.range,
|
||||
`{{ ${context.sourceCode.getText(expression)} }}`,
|
||||
)
|
||||
},
|
||||
})
|
||||
},
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
} satisfies Rule.RuleModule
|
||||
|
||||
const templateStylePlugin = {
|
||||
rules: {
|
||||
'single-line-simple-interpolation': singleLineSimpleInterpolationRule,
|
||||
},
|
||||
} satisfies ESLint.Plugin
|
||||
|
||||
export default antfu(
|
||||
{
|
||||
|
||||
@@ -7,7 +7,7 @@ const props = withDefaults(
|
||||
defineProps<{
|
||||
provider?: string | null
|
||||
authenticationKind?: string | null
|
||||
size?: 'sm' | 'md'
|
||||
size?: 'xs' | 'sm' | 'md'
|
||||
}>(),
|
||||
{
|
||||
size: 'md',
|
||||
@@ -17,9 +17,21 @@ const props = withDefaults(
|
||||
const normalizedProvider = computed(() => (props.provider ?? '').trim().toLowerCase())
|
||||
const normalizedAuthenticationKind = computed(() => (props.authenticationKind ?? '').trim().toLowerCase())
|
||||
const showAuthenticationKind = computed(() => props.authenticationKind !== undefined)
|
||||
const groupGapClass = computed(() => (props.size === 'sm' ? 'gap-1' : 'gap-2'))
|
||||
const iconContainerClass = computed(() => (props.size === 'sm' ? 'size-5 rounded-md' : 'size-7 rounded-lg'))
|
||||
const iconClass = computed(() => (props.size === 'sm' ? 'size-3' : 'size-4'))
|
||||
const groupGapClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'gap-0.75'
|
||||
return props.size === 'sm' ? 'gap-1' : 'gap-2'
|
||||
})
|
||||
const iconContainerClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'size-4.5 rounded-md'
|
||||
return props.size === 'sm' ? 'size-5 rounded-md' : 'size-7 rounded-lg'
|
||||
})
|
||||
const iconClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'size-2.5'
|
||||
return props.size === 'sm' ? 'size-3' : 'size-4'
|
||||
})
|
||||
|
||||
const providerLabel = computed(() => formatProviderLabel(props.provider, '未知平台'))
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import type { LineSeriesOption } from 'echarts'
|
||||
|
||||
type TemporalValue = number | null | undefined
|
||||
|
||||
interface GapRun {
|
||||
startIndex: number
|
||||
endIndex: number
|
||||
}
|
||||
|
||||
interface GapBridgeOptions {
|
||||
name: string
|
||||
color: string
|
||||
xAxisIndex?: number
|
||||
yAxisIndex?: number
|
||||
width?: number
|
||||
z?: number
|
||||
maxGapBuckets?: number
|
||||
}
|
||||
|
||||
const defaultShortGapBuckets = 2
|
||||
|
||||
export function sampleGapBridgeSeries(
|
||||
values: TemporalValue[],
|
||||
options: GapBridgeOptions,
|
||||
): LineSeriesOption[] {
|
||||
const data = values.map(normalizeValue)
|
||||
const maxGapBuckets = options.maxGapBuckets ?? defaultShortGapBuckets
|
||||
const series: LineSeriesOption[] = []
|
||||
|
||||
for (const [runIndex, run] of boundedGapRuns(data)
|
||||
.filter(run => gapLength(run) <= maxGapBuckets)
|
||||
.entries()) {
|
||||
const beforeIndex = run.startIndex - 1
|
||||
const afterIndex = run.endIndex + 1
|
||||
const beforeValue = data[beforeIndex]
|
||||
const afterValue = data[afterIndex]
|
||||
const bridge = Array.from<number | null>({ length: data.length }).fill(null)
|
||||
|
||||
if (beforeValue == null || afterValue == null)
|
||||
continue
|
||||
|
||||
const span = afterIndex - beforeIndex
|
||||
for (let index = beforeIndex; index <= afterIndex; index += 1) {
|
||||
const progress = (index - beforeIndex) / span
|
||||
bridge[index] = beforeValue + (afterValue - beforeValue) * progress
|
||||
}
|
||||
|
||||
series.push({
|
||||
name: `${options.name}-gap-${runIndex}`,
|
||||
type: 'line',
|
||||
data: bridge,
|
||||
connectNulls: false,
|
||||
smooth: false,
|
||||
showSymbol: false,
|
||||
silent: true,
|
||||
xAxisIndex: options.xAxisIndex ?? 0,
|
||||
yAxisIndex: options.yAxisIndex ?? 0,
|
||||
z: Math.max(0, (options.z ?? 2) - 1),
|
||||
lineStyle: {
|
||||
color: options.color,
|
||||
type: 'dashed',
|
||||
width: Math.max(1, (options.width ?? 2.2) - 0.7),
|
||||
opacity: 0.32,
|
||||
},
|
||||
itemStyle: { color: options.color, opacity: 0 },
|
||||
emphasis: { disabled: true },
|
||||
tooltip: { show: false },
|
||||
})
|
||||
}
|
||||
|
||||
return series
|
||||
}
|
||||
|
||||
export function requestActivityByBucket(
|
||||
buckets: string[],
|
||||
activity: Array<{ bucket: string, totalRequests: number }>,
|
||||
) {
|
||||
const requestsByBucket = new Map(
|
||||
activity.map(point => [point.bucket, Math.max(0, point.totalRequests)]),
|
||||
)
|
||||
return buckets.map(bucket => (requestsByBucket.get(bucket) ?? 0) > 0)
|
||||
}
|
||||
|
||||
export function zeroInactiveValues(
|
||||
values: TemporalValue[],
|
||||
active: boolean[],
|
||||
) {
|
||||
return values.map((value, index) => normalizeValue(value) ?? (active[index] === false ? 0 : null))
|
||||
}
|
||||
|
||||
function normalizeValue(value: TemporalValue) {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
||||
}
|
||||
|
||||
function boundedGapRuns(values: Array<number | null>) {
|
||||
return gapRuns(values.map(value => value == null)).filter(
|
||||
run => run.startIndex > 0 && run.endIndex < values.length - 1,
|
||||
)
|
||||
}
|
||||
|
||||
function gapRuns(isGap: boolean[]) {
|
||||
const runs: GapRun[] = []
|
||||
let startIndex = -1
|
||||
|
||||
for (let index = 0; index <= isGap.length; index += 1) {
|
||||
if (isGap[index] && startIndex < 0) {
|
||||
startIndex = index
|
||||
continue
|
||||
}
|
||||
if (isGap[index] || startIndex < 0)
|
||||
continue
|
||||
runs.push({ startIndex, endIndex: index - 1 })
|
||||
startIndex = -1
|
||||
}
|
||||
|
||||
return runs
|
||||
}
|
||||
|
||||
function gapLength(run: GapRun) {
|
||||
return run.endIndex - run.startIndex + 1
|
||||
}
|
||||
@@ -16,12 +16,14 @@ const props = withDefaults(
|
||||
showPlan?: boolean
|
||||
titleMode?: 'local-part' | 'email'
|
||||
metaPosition?: 'title' | 'secondary'
|
||||
metaSize?: 'xs' | 'sm'
|
||||
}>(),
|
||||
{
|
||||
size: 'md',
|
||||
showPlan: false,
|
||||
titleMode: 'local-part',
|
||||
metaPosition: 'title',
|
||||
metaSize: 'sm',
|
||||
},
|
||||
)
|
||||
|
||||
@@ -54,6 +56,8 @@ const secondaryClass = computed(() =>
|
||||
: 'mt-0.5 font-mono text-cp-xs text-cp-text-quaternary',
|
||||
)
|
||||
|
||||
const metaGapClass = computed(() => props.metaSize === 'xs' ? 'gap-1' : 'gap-1.5')
|
||||
|
||||
const avatarToneClass = computed(() => {
|
||||
const identity = props.account.id || props.account.email || displayTitle.value
|
||||
return stablePresetVisualToneClass(identity)
|
||||
@@ -75,18 +79,20 @@ const avatarToneClass = computed(() => {
|
||||
</span>
|
||||
<span
|
||||
v-if="metaPosition === 'title' && (showPlan || $slots.meta)"
|
||||
class="inline-flex shrink-0 items-center justify-end gap-1.5"
|
||||
class="inline-flex shrink-0 items-center justify-end"
|
||||
:class="metaGapClass"
|
||||
>
|
||||
<slot name="meta" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" size="sm" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" :size="metaSize" />
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
v-if="metaPosition === 'secondary' && (showPlan || $slots.meta)"
|
||||
class="mt-0.5 inline-flex min-w-0 items-center gap-1.5"
|
||||
class="mt-0.5 inline-flex min-w-0 items-center"
|
||||
:class="metaGapClass"
|
||||
>
|
||||
<slot name="meta" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" size="sm" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" :size="metaSize" />
|
||||
</div>
|
||||
<div v-else-if="secondaryText" class="truncate font-emphasis" :class="secondaryClass">
|
||||
{{ secondaryText }}
|
||||
|
||||
@@ -6,7 +6,7 @@ import { stableVisualIndex } from '../utils/visualTone'
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
planType?: string | null
|
||||
size?: 'sm' | 'md'
|
||||
size?: 'xs' | 'sm' | 'md'
|
||||
}>(),
|
||||
{
|
||||
planType: null,
|
||||
@@ -29,11 +29,13 @@ const fallbackPalettes = [
|
||||
|
||||
const label = computed(() => props.planType?.trim() || 'Free')
|
||||
|
||||
const sizeClass = computed(() =>
|
||||
props.size === 'sm'
|
||||
const sizeClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'h-4.5 rounded-full px-1.5 text-[10px] font-bold'
|
||||
return props.size === 'sm'
|
||||
? 'h-5 rounded-full px-1.75 text-cp-xs font-bold'
|
||||
: 'h-5.5 rounded-full px-2 text-cp-xs font-heavy',
|
||||
)
|
||||
: 'h-5.5 rounded-full px-2 text-cp-xs font-heavy'
|
||||
})
|
||||
|
||||
const paletteClass = computed(() => {
|
||||
const key = label.value.toLowerCase()
|
||||
|
||||
@@ -157,7 +157,7 @@ const {
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex h-full min-h-0 w-full flex-col overflow-hidden">
|
||||
<div class="flex min-h-0 w-full flex-col xl:h-full xl:overflow-hidden">
|
||||
<BasePageHeader
|
||||
class="h-17"
|
||||
title="账号管理"
|
||||
@@ -167,7 +167,7 @@ const {
|
||||
<AccountOverviewCards :summary="accountSummary" />
|
||||
|
||||
<BaseCard
|
||||
class="mt-4 flex h-[calc(100dvh-250px)] min-h-125 flex-col"
|
||||
class="mt-4 flex flex-col xl:h-[calc(100dvh-250px)] xl:min-h-125"
|
||||
>
|
||||
<template #header>
|
||||
<AccountFilters
|
||||
@@ -188,9 +188,9 @@ const {
|
||||
</template>
|
||||
|
||||
<template #body>
|
||||
<div class="flex h-full min-h-0 flex-col">
|
||||
<div class="flex min-h-0 flex-col xl:h-full">
|
||||
<BaseTable
|
||||
class="min-h-0 flex-1 [--cp-table-row-height:72px]"
|
||||
class="h-100! min-h-100 flex-none [--cp-table-row-height:72px] xl:h-auto! xl:min-h-0 xl:flex-1"
|
||||
:columns="accountColumns"
|
||||
:rows="accounts"
|
||||
:loading="loading"
|
||||
|
||||
@@ -240,13 +240,14 @@ const statusBars = computed(() => {
|
||||
show-plan
|
||||
title-mode="email"
|
||||
meta-position="secondary"
|
||||
meta-size="xs"
|
||||
class="min-w-0"
|
||||
>
|
||||
<template #meta>
|
||||
<ProviderIconGroup
|
||||
:provider="account.provider"
|
||||
:authentication-kind="account.authenticationKind"
|
||||
size="sm"
|
||||
size="xs"
|
||||
/>
|
||||
</template>
|
||||
</AccountIdentityCell>
|
||||
|
||||
@@ -373,10 +373,10 @@ function aggregateUsageTrend(points: DashboardTrendPoint[]) {
|
||||
)
|
||||
const uncachedInputTokensValue = inputTokensValue - cachedTokensValue
|
||||
const effectiveTokensValue = uncachedInputTokensValue + outputTokensValue
|
||||
const cacheHitRateValue = inputTokensValue ? cachedTokensValue / inputTokensValue : null
|
||||
const cacheHitRateValue = inputTokensValue ? cachedTokensValue / inputTokensValue : 0
|
||||
const successRateValue = requestsValue
|
||||
? ((requestsValue - errorsValue) / requestsValue) * 100
|
||||
: null
|
||||
: 0
|
||||
|
||||
return {
|
||||
...first,
|
||||
@@ -392,12 +392,12 @@ function aggregateUsageTrend(points: DashboardTrendPoint[]) {
|
||||
uncachedInputTokensValue,
|
||||
effectiveTokens: formatCompactNumber(effectiveTokensValue),
|
||||
effectiveTokensValue: effectiveTokensValue > 0 ? effectiveTokensValue : null,
|
||||
cacheHitRate: cacheHitRateValue === null ? '—' : formatDashboardRate(cacheHitRateValue),
|
||||
cacheHitRate: formatDashboardRate(cacheHitRateValue),
|
||||
cacheHitRateValue,
|
||||
tokensValue: inputTokensValue + outputTokensValue,
|
||||
errors: formatCompactNumber(errorsValue),
|
||||
errorsValue,
|
||||
successRate: successRateValue === null ? '—' : `${successRateValue.toFixed(1)}%`,
|
||||
successRate: `${successRateValue.toFixed(1)}%`,
|
||||
successRateValue,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { dashboardTrendView, normalizeDashboardTrendKind } from './useDashb
|
||||
import { usePreferredReducedMotion } from '@vueuse/core'
|
||||
import { computed, shallowRef, watch } from 'vue'
|
||||
|
||||
import { sampleGapBridgeSeries } from '@/components/charts/timeSeriesGap'
|
||||
import { chartTooltipStyle, tooltipIndex } from '@/components/charts/tooltip'
|
||||
import { useChartPalette } from '@/composables/useChartPalette'
|
||||
|
||||
@@ -197,7 +198,7 @@ export function useRequestTrendChart(options: {
|
||||
),
|
||||
tooltipItem(
|
||||
'缓存命中',
|
||||
trendPointText(point, 'cacheHitRate'),
|
||||
zeroAwareRateText(point, 'cacheHitRate', 'inputTokensValue'),
|
||||
trendColor('缓存', '--cp-color-text-tertiary', '#94A3B8'),
|
||||
),
|
||||
tooltipItem('请求', point?.requests, palette.value.textSecondary),
|
||||
@@ -235,19 +236,19 @@ export function useRequestTrendChart(options: {
|
||||
if (name === '错误数')
|
||||
return point?.errors ?? value
|
||||
if (name === '成功率')
|
||||
return point?.successRate ?? value
|
||||
return zeroAwareRateText(point, 'successRate', 'requestsValue') ?? value
|
||||
if (name === '总请求')
|
||||
return point?.requests ?? value
|
||||
return value
|
||||
}
|
||||
|
||||
function getSeries() {
|
||||
function getSeries(): Array<BarSeriesOption | LineSeriesOption> {
|
||||
if (activeKind.value === 'usage') {
|
||||
const cacheColor = trendColor('缓存', '--cp-color-text-tertiary', '#94A3B8')
|
||||
const inputColor = trendColor('输入', '--cp-color-blue-solid', '#5983F4')
|
||||
const outputColor = trendColor('输出', '--cp-color-green-solid', '#12B981')
|
||||
return [
|
||||
lineSeries('缓存', activeSeriesValues('cachedTokensValue'), cacheColor, {
|
||||
lineSeries('缓存', seriesValues('cachedTokensValue'), cacheColor, {
|
||||
area: true,
|
||||
areaStartAlpha: '30',
|
||||
areaEndAlpha: '08',
|
||||
@@ -257,7 +258,7 @@ export function useRequestTrendChart(options: {
|
||||
width: 1.25,
|
||||
z: 1,
|
||||
}),
|
||||
lineSeries('输入', activeSeriesValues('uncachedInputTokensValue'), inputColor, {
|
||||
lineSeries('输入', seriesValues('uncachedInputTokensValue'), inputColor, {
|
||||
area: true,
|
||||
areaStartAlpha: '2A',
|
||||
areaEndAlpha: '05',
|
||||
@@ -266,7 +267,7 @@ export function useRequestTrendChart(options: {
|
||||
width: 2.3,
|
||||
z: 3,
|
||||
}),
|
||||
lineSeries('输出', activeSeriesValues('outputTokensValue'), outputColor, {
|
||||
lineSeries('输出', seriesValues('outputTokensValue'), outputColor, {
|
||||
smooth: 0.24,
|
||||
width: 2.1,
|
||||
yAxisIndex: 1,
|
||||
@@ -277,19 +278,19 @@ export function useRequestTrendChart(options: {
|
||||
|
||||
if (activeKind.value === 'latency') {
|
||||
return [
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'总耗时 P95',
|
||||
activeSeriesValues('latencyP95Ms'),
|
||||
trendColor('总耗时', '--cp-color-orange-solid', '#F59E0B'),
|
||||
{ lineType: 'dashed', smooth: 0.2, width: 1.8, z: 2 },
|
||||
),
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'首字 P95',
|
||||
activeSeriesValues('firstTokenP95Ms'),
|
||||
trendColor('首字', '--cp-color-cyan-solid', '#13C2C2'),
|
||||
{ area: true, smooth: 0.24, width: 2.5, z: 4 },
|
||||
),
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'吞吐 P50',
|
||||
activeSeriesValues('outputThroughputP50'),
|
||||
trendColor('吞吐', '--cp-color-green-solid', '#12B981'),
|
||||
@@ -304,9 +305,9 @@ export function useRequestTrendChart(options: {
|
||||
seriesValues('errorsValue'),
|
||||
trendColor('错误', '--cp-color-red-solid', '#EF4444'),
|
||||
),
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'成功率',
|
||||
seriesValues('successRateValue'),
|
||||
activeSeriesValues('successRateValue'),
|
||||
trendColor('成功', '--cp-color-green-solid', '#12B981'),
|
||||
{ width: 2.4, yAxisIndex: 1 },
|
||||
),
|
||||
@@ -325,7 +326,7 @@ export function useRequestTrendChart(options: {
|
||||
|
||||
function activeSeriesValues(key: string) {
|
||||
return points.value.map(point =>
|
||||
Number(point.requestsValue) > 0 ? trendPointNumber(point, key) : null,
|
||||
Number(point.requestsValue) > 0 ? trendPointNumber(point, key) : 0,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -377,6 +378,26 @@ export function useRequestTrendChart(options: {
|
||||
}
|
||||
}
|
||||
|
||||
function gapLineSeries(
|
||||
name: string,
|
||||
data: (number | null)[],
|
||||
color: string,
|
||||
lineOptions: LineSeriesOptions = {},
|
||||
) {
|
||||
const primary = lineSeries(name, data, color, lineOptions)
|
||||
return [
|
||||
primary,
|
||||
...sampleGapBridgeSeries(data, {
|
||||
name,
|
||||
color,
|
||||
width: lineOptions.width,
|
||||
xAxisIndex: lineOptions.xAxisIndex,
|
||||
yAxisIndex: lineOptions.yAxisIndex,
|
||||
z: lineOptions.z,
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
function barSeries(
|
||||
name: string,
|
||||
data: (number | null)[],
|
||||
@@ -490,7 +511,7 @@ function tooltipValue(source: unknown, key: string) {
|
||||
function trendPointProperty(point: TrendPoint | undefined, key: string) {
|
||||
if (!point || !(key in point))
|
||||
return undefined
|
||||
return (point as Record<string, unknown>)[key]
|
||||
return (point as unknown as Record<string, unknown>)[key]
|
||||
}
|
||||
|
||||
function trendPointText(point: TrendPoint | undefined, key: string) {
|
||||
@@ -498,6 +519,17 @@ function trendPointText(point: TrendPoint | undefined, key: string) {
|
||||
return typeof value === 'string' ? value : undefined
|
||||
}
|
||||
|
||||
function zeroAwareRateText(
|
||||
point: TrendPoint | undefined,
|
||||
valueKey: string,
|
||||
activityKey: string,
|
||||
) {
|
||||
const activity = trendPointProperty(point, activityKey)
|
||||
if (typeof activity === 'number' && activity <= 0)
|
||||
return '0.0%'
|
||||
return trendPointText(point, valueKey)
|
||||
}
|
||||
|
||||
function trendPointNumber(point: TrendPoint, key: string) {
|
||||
const value = trendPointProperty(point, key)
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
||||
|
||||
@@ -7,6 +7,10 @@ import BaseCard from '@/components/base/BaseCard.vue'
|
||||
import BaseEmpty from '@/components/base/BaseEmpty.vue'
|
||||
import BaseSegmented from '@/components/base/BaseSegmented.vue'
|
||||
import BaseChart from '@/components/charts/BaseChart.vue'
|
||||
import {
|
||||
requestActivityByBucket,
|
||||
zeroInactiveValues,
|
||||
} from '@/components/charts/timeSeriesGap'
|
||||
import { useChartPalette } from '@/composables/useChartPalette'
|
||||
import { formatLocalizedCompactNumber as formatCompactNumber } from '@/utils/number'
|
||||
|
||||
@@ -14,10 +18,11 @@ import {
|
||||
tooltipIndex,
|
||||
tooltipRows,
|
||||
usageCategoryAxis,
|
||||
usageGapAwareLineSeries,
|
||||
usageLegend,
|
||||
usageLineSeries,
|
||||
usageTooltip,
|
||||
usageTooltipContent,
|
||||
usageTooltipItem,
|
||||
usageValueAxis,
|
||||
} from '../utils/chart'
|
||||
import {
|
||||
@@ -28,11 +33,13 @@ import {
|
||||
} from '../utils/format'
|
||||
|
||||
type Cost = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['cost']
|
||||
type Activity = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['health']['points']
|
||||
type UsageChartPalette = ReturnType<typeof useChartPalette>['palette']['value']
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
cost: Cost
|
||||
activity: Activity
|
||||
loading?: boolean
|
||||
}>(),
|
||||
{
|
||||
@@ -43,6 +50,10 @@ const props = withDefaults(
|
||||
const activeView = shallowRef('cost')
|
||||
const { palette } = useChartPalette()
|
||||
const points = computed(() => props.cost.points)
|
||||
const requestActivity = computed(() => requestActivityByBucket(
|
||||
points.value.map(point => point.bucket),
|
||||
props.activity,
|
||||
))
|
||||
const hasNoCacheCost = computed(() =>
|
||||
points.value.some(point => point.noCacheCost != null),
|
||||
)
|
||||
@@ -74,6 +85,7 @@ const hasData = computed(() => {
|
||||
const chartOption = computed<EChartsOption>(() => {
|
||||
const theme = palette.value
|
||||
const legend = legendNames()
|
||||
const series = chartSeries(theme)
|
||||
|
||||
return {
|
||||
animationDuration: 240,
|
||||
@@ -95,7 +107,7 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
min: activeView.value === 'cache' ? 0 : undefined,
|
||||
max: activeView.value === 'cache' ? 1 : undefined,
|
||||
}),
|
||||
series: chartSeries(theme),
|
||||
series,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -119,9 +131,12 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
const chartPoints = points.value
|
||||
if (activeView.value === 'savings') {
|
||||
return [
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'缓存节省',
|
||||
chartPoints.map(point => decimalDisplayNumber(point.cacheSavings)),
|
||||
zeroInactiveValues(
|
||||
chartPoints.map(point => decimalDisplayNumber(point.cacheSavings)),
|
||||
requestActivity.value,
|
||||
),
|
||||
theme.success,
|
||||
{ area: 'strong' },
|
||||
),
|
||||
@@ -130,12 +145,12 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
|
||||
if (activeView.value === 'cache') {
|
||||
return [
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'缓存 Token 占比',
|
||||
chartPoints.map(point => point.cachedTokenRate),
|
||||
theme.normal,
|
||||
),
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'命中请求率',
|
||||
chartPoints.map(point => point.cacheHitRequestRate),
|
||||
theme.success,
|
||||
@@ -144,18 +159,24 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
}
|
||||
|
||||
const series = [
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'实际费用',
|
||||
chartPoints.map(point => decimalDisplayNumber(point.estimatedCost)),
|
||||
zeroInactiveValues(
|
||||
chartPoints.map(point => decimalDisplayNumber(point.estimatedCost)),
|
||||
requestActivity.value,
|
||||
),
|
||||
theme.success,
|
||||
{ area: 'strong' },
|
||||
),
|
||||
]
|
||||
if (hasNoCacheCost.value) {
|
||||
series.push(
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'无缓存费用',
|
||||
chartPoints.map(point => decimalDisplayNumber(point.noCacheCost)),
|
||||
zeroInactiveValues(
|
||||
chartPoints.map(point => decimalDisplayNumber(point.noCacheCost)),
|
||||
requestActivity.value,
|
||||
),
|
||||
theme.textMuted,
|
||||
),
|
||||
)
|
||||
@@ -165,29 +186,48 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
|
||||
function formatTooltip(params: unknown) {
|
||||
const rows = tooltipRows(params)
|
||||
const point = points.value[tooltipIndex(rows[0])]
|
||||
const pointIndex = tooltipIndex(rows[0])
|
||||
const point = points.value[pointIndex]
|
||||
if (!point)
|
||||
return ''
|
||||
const theme = palette.value
|
||||
|
||||
if (activeView.value === 'savings') {
|
||||
return usageTooltipContent(palette.value, point.label, [
|
||||
`缓存节省: ${formatUsd(point.cacheSavings)}`,
|
||||
return usageTooltipContent(theme, point.label, [
|
||||
usageTooltipItem(
|
||||
'缓存节省',
|
||||
formatUsd(costAmount(point.cacheSavings, pointIndex)),
|
||||
theme.success,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
if (activeView.value === 'cache') {
|
||||
return usageTooltipContent(palette.value, point.label, [
|
||||
`缓存 Token 占比: ${formatPercent(point.cachedTokenRate)}`,
|
||||
`命中请求率: ${formatPercent(point.cacheHitRequestRate)}`,
|
||||
return usageTooltipContent(theme, point.label, [
|
||||
usageTooltipItem('缓存 Token 占比', formatPercent(point.cachedTokenRate), theme.normal),
|
||||
usageTooltipItem('命中请求率', formatPercent(point.cacheHitRequestRate), theme.success),
|
||||
])
|
||||
}
|
||||
|
||||
const lines = [
|
||||
`实际费用: ${formatUsd(point.estimatedCost)}`,
|
||||
usageTooltipItem(
|
||||
'实际费用',
|
||||
formatUsd(costAmount(point.estimatedCost, pointIndex)),
|
||||
theme.success,
|
||||
),
|
||||
]
|
||||
if (point.noCacheCost != null)
|
||||
lines.push(`无缓存费用: ${formatUsd(point.noCacheCost)}`)
|
||||
return usageTooltipContent(palette.value, point.label, lines)
|
||||
if (point.noCacheCost != null) {
|
||||
lines.push(usageTooltipItem(
|
||||
'无缓存费用',
|
||||
formatUsd(costAmount(point.noCacheCost, pointIndex)),
|
||||
theme.textMuted,
|
||||
))
|
||||
}
|
||||
return usageTooltipContent(theme, point.label, lines)
|
||||
}
|
||||
|
||||
function costAmount(value: string | null, pointIndex: number) {
|
||||
return value ?? (requestActivity.value[pointIndex] === false ? '0' : null)
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -196,7 +236,7 @@ function formatTooltip(params: unknown) {
|
||||
as="article"
|
||||
title="成本效率"
|
||||
description="实际费用、缓存节省、服务层溢价与单位成本"
|
||||
class="h-full min-h-90"
|
||||
class="min-h-90 xl:h-full"
|
||||
>
|
||||
<template #actions>
|
||||
<BaseSegmented v-model="activeView" label="成本视图" :options="viewOptions" :disabled="loading" class="w-50" />
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import type { EChartsOption } from 'echarts'
|
||||
import type { BarSeriesOption, EChartsOption, LineSeriesOption } from 'echarts'
|
||||
import type { getUsageRecordInsightsOverview } from '@/api'
|
||||
import { BarChart } from 'echarts/charts'
|
||||
import { use } from 'echarts/core'
|
||||
@@ -15,9 +15,11 @@ import {
|
||||
tooltipIndex,
|
||||
tooltipRows,
|
||||
usageCategoryAxis,
|
||||
usageGapAwareLineSeries,
|
||||
usageLegend,
|
||||
usageTooltip,
|
||||
usageTooltipContent,
|
||||
usageTooltipItem,
|
||||
usageValueAxis,
|
||||
} from '../utils/chart'
|
||||
import { formatPercent } from '../utils/format'
|
||||
@@ -60,6 +62,26 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
const theme = palette.value
|
||||
const chartPoints = points.value
|
||||
const activePointCount = chartPoints.filter(point => requestCount(point) > 0).length
|
||||
const successRateSeries = usageGapAwareLineSeries(
|
||||
'服务成功率',
|
||||
chartPoints.map(serviceSuccessRate),
|
||||
theme.success,
|
||||
{ yAxisIndex: 1 },
|
||||
)
|
||||
const series: Array<BarSeriesOption | LineSeriesOption> = [
|
||||
outcomeSeries('成功', chartPoints.map(point => point.successRequests), theme.success),
|
||||
outcomeSeries('服务失败', chartPoints.map(point => point.failedRequests), theme.danger),
|
||||
outcomeSeries('取消', chartPoints.map(point => point.cancelledRequests), theme.info),
|
||||
outcomeSeries('未完成', chartPoints.map(point => point.incompleteRequests), theme.warning),
|
||||
outcomeSeries('调用方错误', chartPoints.map(point => point.callerErrorRequests), theme.normal),
|
||||
...successRateSeries.map((item, index) => index === 0
|
||||
? {
|
||||
...item,
|
||||
symbol: activePointCount <= 16 ? 'circle' : 'none',
|
||||
symbolSize: 4,
|
||||
}
|
||||
: item),
|
||||
]
|
||||
|
||||
return {
|
||||
animationDuration: 240,
|
||||
@@ -85,25 +107,7 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
splitLine: false,
|
||||
}),
|
||||
],
|
||||
series: [
|
||||
outcomeSeries('成功', chartPoints.map(point => point.successRequests), theme.success),
|
||||
outcomeSeries('服务失败', chartPoints.map(point => point.failedRequests), theme.danger),
|
||||
outcomeSeries('取消', chartPoints.map(point => point.cancelledRequests), theme.info),
|
||||
outcomeSeries('未完成', chartPoints.map(point => point.incompleteRequests), theme.warning),
|
||||
outcomeSeries('调用方错误', chartPoints.map(point => point.callerErrorRequests), theme.normal),
|
||||
{
|
||||
name: '服务成功率',
|
||||
type: 'line',
|
||||
yAxisIndex: 1,
|
||||
data: chartPoints.map(serviceSuccessRate),
|
||||
connectNulls: true,
|
||||
smooth: 0.25,
|
||||
symbol: activePointCount <= 16 ? 'circle' : 'none',
|
||||
symbolSize: 4,
|
||||
lineStyle: { color: theme.success, width: 2.2 },
|
||||
itemStyle: { color: theme.success },
|
||||
},
|
||||
],
|
||||
series,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -113,7 +117,9 @@ function requestCount(point: HealthPoint) {
|
||||
|
||||
function serviceSuccessRate(point: HealthPoint) {
|
||||
const total = Math.max(0, point.successRequests) + Math.max(0, point.failedRequests)
|
||||
return total > 0 ? Math.max(0, point.successRequests ?? 0) / total : null
|
||||
if (total > 0)
|
||||
return Math.max(0, point.successRequests ?? 0) / total
|
||||
return requestCount(point) === 0 ? 0 : null
|
||||
}
|
||||
|
||||
function outcomeSeries(name: string, data: number[], color: string) {
|
||||
@@ -134,15 +140,20 @@ function formatTooltip(params: unknown) {
|
||||
return ''
|
||||
|
||||
const successRate = serviceSuccessRate(point)
|
||||
const theme = palette.value
|
||||
|
||||
return usageTooltipContent(palette.value, point.label, [
|
||||
`请求量: ${formatCompactNumber(requestCount(point))}`,
|
||||
`成功: ${formatCompactNumber(point.successRequests)}`,
|
||||
`服务失败: ${formatCompactNumber(point.failedRequests)}`,
|
||||
`取消: ${formatCompactNumber(point.cancelledRequests)}`,
|
||||
`未完成: ${formatCompactNumber(point.incompleteRequests)}`,
|
||||
`调用方错误: ${formatCompactNumber(point.callerErrorRequests)}`,
|
||||
`服务成功率: ${successRate == null ? '无服务结果' : formatPercent(successRate)}`,
|
||||
return usageTooltipContent(theme, point.label, [
|
||||
usageTooltipItem('请求量', formatCompactNumber(requestCount(point)), theme.textSecondary),
|
||||
usageTooltipItem('成功', formatCompactNumber(point.successRequests), theme.success),
|
||||
usageTooltipItem('服务失败', formatCompactNumber(point.failedRequests), theme.danger),
|
||||
usageTooltipItem('取消', formatCompactNumber(point.cancelledRequests), theme.info),
|
||||
usageTooltipItem('未完成', formatCompactNumber(point.incompleteRequests), theme.warning),
|
||||
usageTooltipItem('调用方错误', formatCompactNumber(point.callerErrorRequests), theme.normal),
|
||||
usageTooltipItem(
|
||||
'服务成功率',
|
||||
successRate == null ? '无服务结果' : formatPercent(successRate),
|
||||
theme.success,
|
||||
),
|
||||
])
|
||||
}
|
||||
</script>
|
||||
@@ -152,7 +163,7 @@ function formatTooltip(params: unknown) {
|
||||
as="article"
|
||||
title="请求健康"
|
||||
:description="`按${granularityText}区分服务结果、取消、未完成与调用方错误`"
|
||||
class="h-full min-h-90"
|
||||
class="min-h-90 xl:h-full"
|
||||
>
|
||||
<template #body>
|
||||
<div class="grid min-h-66" :class="hasData ? 'gap-3' : 'h-full'">
|
||||
|
||||
@@ -28,7 +28,7 @@ const diagnosticDimension = defineModel('diagnosticDimension', {
|
||||
|
||||
<template>
|
||||
<section
|
||||
class="mt-5 grid grid-cols-1 gap-3 xl:auto-rows-[24.25rem] xl:grid-cols-2"
|
||||
class="mt-5 grid grid-cols-1 gap-3 xl:auto-rows-97 xl:grid-cols-2"
|
||||
aria-label="使用统计观测"
|
||||
>
|
||||
<UsageHealthCard
|
||||
@@ -43,8 +43,16 @@ const diagnosticDimension = defineModel('diagnosticDimension', {
|
||||
:loading="loading"
|
||||
/>
|
||||
|
||||
<UsagePerformanceCard :performance="overview.performance" :loading="loading" />
|
||||
<UsagePerformanceCard
|
||||
:performance="overview.performance"
|
||||
:activity="overview.health.points"
|
||||
:loading="loading"
|
||||
/>
|
||||
|
||||
<UsageCostCard :cost="overview.cost" :loading="loading" />
|
||||
<UsageCostCard
|
||||
:cost="overview.cost"
|
||||
:activity="overview.health.points"
|
||||
:loading="loading"
|
||||
/>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
@@ -7,6 +7,7 @@ import BaseCard from '@/components/base/BaseCard.vue'
|
||||
import BaseEmpty from '@/components/base/BaseEmpty.vue'
|
||||
import BaseSegmented from '@/components/base/BaseSegmented.vue'
|
||||
import BaseChart from '@/components/charts/BaseChart.vue'
|
||||
import { requestActivityByBucket, zeroInactiveValues } from '@/components/charts/timeSeriesGap'
|
||||
import { useChartPalette } from '@/composables/useChartPalette'
|
||||
import { formatLocalizedCompactNumber as formatCompactNumber } from '@/utils/number'
|
||||
|
||||
@@ -14,20 +15,23 @@ import {
|
||||
tooltipIndex,
|
||||
tooltipRows,
|
||||
usageCategoryAxis,
|
||||
usageGapAwareLineSeries,
|
||||
usageLegend,
|
||||
usageLineSeries,
|
||||
usageTooltip,
|
||||
usageTooltipContent,
|
||||
usageTooltipItem,
|
||||
usageValueAxis,
|
||||
} from '../utils/chart'
|
||||
import { formatDuration, formatDurationAxis, formatPercent } from '../utils/format'
|
||||
|
||||
type Performance = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['performance']
|
||||
type PerformancePoint = Performance['points'][number]
|
||||
type Activity = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['health']['points']
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
performance: Performance
|
||||
activity: Activity
|
||||
loading?: boolean
|
||||
}>(),
|
||||
{
|
||||
@@ -38,6 +42,10 @@ const props = withDefaults(
|
||||
const activeView = shallowRef('total')
|
||||
const { palette } = useChartPalette()
|
||||
const performancePoints = computed<PerformancePoint[]>(() => props.performance.points)
|
||||
const requestActivity = computed(() => requestActivityByBucket(
|
||||
performancePoints.value.map(point => point.bucket),
|
||||
props.activity,
|
||||
))
|
||||
|
||||
const viewOptions = [
|
||||
{ label: '总耗时', value: 'total' },
|
||||
@@ -54,14 +62,28 @@ const seriesLabels = computed(() => {
|
||||
return ['P50', 'P95', 'P99'] as const
|
||||
})
|
||||
|
||||
const selectedPoints = computed(() =>
|
||||
performancePoints.value.map(point => ({
|
||||
const selectedPoints = computed(() => {
|
||||
const points = performancePoints.value
|
||||
const active = requestActivity.value
|
||||
const firstValues = zeroInactiveValues(
|
||||
points.map(point => seriesValue(point, 'first')),
|
||||
active,
|
||||
)
|
||||
const secondValues = zeroInactiveValues(
|
||||
points.map(point => seriesValue(point, 'second')),
|
||||
active,
|
||||
)
|
||||
const thirdValues = zeroInactiveValues(
|
||||
points.map(point => seriesValue(point, 'third')),
|
||||
active,
|
||||
)
|
||||
return points.map((point, index) => ({
|
||||
point,
|
||||
first: seriesValue(point, 'first'),
|
||||
second: seriesValue(point, 'second'),
|
||||
third: seriesValue(point, 'third'),
|
||||
})),
|
||||
)
|
||||
first: firstValues[index] ?? null,
|
||||
second: secondValues[index] ?? null,
|
||||
third: thirdValues[index] ?? null,
|
||||
}))
|
||||
})
|
||||
|
||||
const summaryMetrics = computed(() => {
|
||||
const performance = props.performance
|
||||
@@ -98,6 +120,25 @@ const hasData = computed(
|
||||
const chartOption = computed<EChartsOption>(() => {
|
||||
const theme = palette.value
|
||||
const points = selectedPoints.value
|
||||
const series = [
|
||||
...usageGapAwareLineSeries(
|
||||
seriesLabels.value[0],
|
||||
points.map(point => point.first),
|
||||
theme.info,
|
||||
{ area: 'subtle' },
|
||||
),
|
||||
...usageGapAwareLineSeries(
|
||||
seriesLabels.value[1],
|
||||
points.map(point => point.second),
|
||||
theme.warning,
|
||||
),
|
||||
...usageGapAwareLineSeries(
|
||||
seriesLabels.value[2],
|
||||
points.map(point => point.third),
|
||||
activeView.value === 'scheduling' ? theme.success : theme.danger,
|
||||
{ yAxisIndex: activeView.value === 'scheduling' ? 1 : 0 },
|
||||
),
|
||||
]
|
||||
|
||||
return {
|
||||
animationDuration: 240,
|
||||
@@ -130,27 +171,7 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
? value => `${formatCompactNumber(value)}/s`
|
||||
: formatDurationAxis,
|
||||
),
|
||||
series: [
|
||||
usageLineSeries(
|
||||
seriesLabels.value[0],
|
||||
points.map(point => point.first),
|
||||
theme.info,
|
||||
{ area: 'subtle' },
|
||||
),
|
||||
usageLineSeries(
|
||||
seriesLabels.value[1],
|
||||
points.map(point => point.second),
|
||||
theme.warning,
|
||||
),
|
||||
{
|
||||
...usageLineSeries(
|
||||
seriesLabels.value[2],
|
||||
points.map(point => point.third),
|
||||
activeView.value === 'scheduling' ? theme.success : theme.danger,
|
||||
),
|
||||
yAxisIndex: activeView.value === 'scheduling' ? 1 : 0,
|
||||
},
|
||||
],
|
||||
series,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -180,10 +201,23 @@ function formatTooltip(params: unknown) {
|
||||
if (!selected)
|
||||
return ''
|
||||
|
||||
return usageTooltipContent(palette.value, selected.point.label, [
|
||||
`${seriesLabels.value[0]}: ${formatSeriesValue(selected.first, 'first')}`,
|
||||
`${seriesLabels.value[1]}: ${formatSeriesValue(selected.second, 'second')}`,
|
||||
`${seriesLabels.value[2]}: ${formatSeriesValue(selected.third, 'third')}`,
|
||||
const theme = palette.value
|
||||
return usageTooltipContent(theme, selected.point.label, [
|
||||
usageTooltipItem(
|
||||
seriesLabels.value[0],
|
||||
formatSeriesValue(selected.first, 'first'),
|
||||
theme.info,
|
||||
),
|
||||
usageTooltipItem(
|
||||
seriesLabels.value[1],
|
||||
formatSeriesValue(selected.second, 'second'),
|
||||
theme.warning,
|
||||
),
|
||||
usageTooltipItem(
|
||||
seriesLabels.value[2],
|
||||
formatSeriesValue(selected.third, 'third'),
|
||||
activeView.value === 'scheduling' ? theme.success : theme.danger,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
@@ -205,7 +239,7 @@ function formatThroughput(value: number | null) {
|
||||
as="article"
|
||||
title="响应速度"
|
||||
description="延迟、吞吐与调度分位"
|
||||
class="h-full min-h-90"
|
||||
class="min-h-90 xl:h-full"
|
||||
>
|
||||
<template #actions>
|
||||
<BaseSegmented v-model="activeView" label="性能指标" :options="viewOptions" :disabled="loading" class="w-68" />
|
||||
|
||||
@@ -36,20 +36,20 @@ export const usageRecordColumns = defineTableColumns<UsageDisplayRecord>([
|
||||
export const opsErrorColumns = defineTableColumns<OpsErrorRow>([
|
||||
{ key: 'accountId', label: '账号', kind: 'mono', size: '3xl' },
|
||||
{ key: 'provider', label: '平台/类型', kind: 'custom', size: 'sm' },
|
||||
{ key: 'createdAtDisplay', label: '时间', kind: 'datetime' },
|
||||
{ key: 'message', label: '错误摘要', kind: 'custom', size: '3xl' },
|
||||
{ key: 'failureClass', label: '失败分类', kind: 'mono' },
|
||||
{ key: 'upstreamStatusCode', label: '上游状态', kind: 'status', size: 'sm' },
|
||||
{ key: 'clientStatusCode', label: '客户端状态', kind: 'status', size: 'md' },
|
||||
{ key: 'failureClass', label: '失败分类', kind: 'mono' },
|
||||
{ key: 'kind', label: '事件', kind: 'mono' },
|
||||
{ key: 'route', label: '端点', kind: 'mono' },
|
||||
{ key: 'model', label: '模型', kind: 'custom', size: 'xl' },
|
||||
{ key: 'reasoningEffort', label: '推理强度', kind: 'custom' },
|
||||
{ key: 'route', label: '端点', kind: 'mono' },
|
||||
{ key: 'transport', label: '上游', kind: 'custom', size: 'md' },
|
||||
{ key: 'clientTransport', label: '接入', kind: 'custom', size: 'md' },
|
||||
{ key: 'createdAtDisplay', label: '时间', kind: 'datetime' },
|
||||
{ key: 'requestId', label: '请求 ID', kind: 'mono', size: '2xl' },
|
||||
{ key: 'kind', label: '事件', kind: 'mono' },
|
||||
{ key: 'clientIp', label: 'IP', kind: 'custom', size: '3xl' },
|
||||
{ key: 'userAgent', label: 'User-Agent', kind: 'custom', size: '4xl' },
|
||||
{ key: 'requestId', label: '请求 ID', kind: 'mono', size: '2xl' },
|
||||
{ key: 'message', label: '错误摘要', kind: 'custom', size: '3xl' },
|
||||
{ key: 'actions', label: '操作', kind: 'actions', size: 'sm' },
|
||||
])
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { LineSeriesOption } from 'echarts'
|
||||
import type { useChartPalette } from '@/composables/useChartPalette'
|
||||
import { sampleGapBridgeSeries } from '@/components/charts/timeSeriesGap'
|
||||
import { chartTooltipStyle } from '@/components/charts/tooltip'
|
||||
import { escapeTooltip } from './format'
|
||||
|
||||
@@ -9,6 +10,8 @@ type UsageAreaStrength = 'strong' | 'subtle'
|
||||
interface UsageLineSeriesOptions {
|
||||
stack?: string
|
||||
area?: UsageAreaStrength | false
|
||||
xAxisIndex?: number
|
||||
yAxisIndex?: number
|
||||
}
|
||||
|
||||
const areaAlpha: Record<UsageAreaStrength, readonly [string, string]> = {
|
||||
@@ -36,6 +39,8 @@ export function usageLineSeries(
|
||||
data: data.map(value => value ?? null),
|
||||
connectNulls: false,
|
||||
stack: options.stack,
|
||||
xAxisIndex: options.xAxisIndex ?? 0,
|
||||
yAxisIndex: options.yAxisIndex ?? 0,
|
||||
smooth: true,
|
||||
showSymbol: showSymbols,
|
||||
showAllSymbol: showSymbols,
|
||||
@@ -61,6 +66,26 @@ export function usageLineSeries(
|
||||
}
|
||||
}
|
||||
|
||||
export function usageGapAwareLineSeries(
|
||||
name: string,
|
||||
data: Array<number | null | undefined>,
|
||||
color: string,
|
||||
options: UsageLineSeriesOptions = {},
|
||||
) {
|
||||
const primary = usageLineSeries(name, data, color, options)
|
||||
return [
|
||||
primary,
|
||||
...sampleGapBridgeSeries(data, {
|
||||
name,
|
||||
color,
|
||||
xAxisIndex: options.xAxisIndex,
|
||||
yAxisIndex: options.yAxisIndex,
|
||||
width: 2.2,
|
||||
z: 2,
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
export function usageTooltip(
|
||||
theme: UsageChartPalette,
|
||||
formatter: (params: unknown) => string,
|
||||
@@ -81,6 +106,10 @@ export function usageTooltipContent(
|
||||
return `<div style="margin:0 0 7px;padding:0 0 7px;border-bottom:1px solid ${theme.divider};color:${theme.textPrimary};font-family:'JetBrains Mono Variable','JetBrains Mono',monospace;font-size:11px;font-weight:750;line-height:1.2">${title}</div><div style="line-height:1.55">${lines.join('<br/>')}</div>`
|
||||
}
|
||||
|
||||
export function usageTooltipItem(label: string, value: string, color: string) {
|
||||
return `<span style="display:inline-block;width:7px;height:7px;margin-right:6px;border-radius:999px;background:${escapeTooltip(color)}"></span>${escapeTooltip(label)}: ${escapeTooltip(value)}`
|
||||
}
|
||||
|
||||
export function usageCategoryAxis(labels: string[], theme: UsageChartPalette) {
|
||||
return {
|
||||
type: 'category' as const,
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.3.12"
|
||||
version: "3.3.14"
|
||||
|
||||
Reference in New Issue
Block a user