mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 08:05:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c2332bc8d0 | ||
|
|
8f242e2875 | ||
|
|
51e9981bee | ||
|
|
65fbbcb3ff | ||
|
|
bd086a7e3c | ||
|
|
3554ba13db | ||
|
|
bcccebc9fa | ||
|
|
d67b46a9e2 | ||
|
|
20c6d33cf4 | ||
|
|
cf0e84c548 | ||
|
|
2332af9a08 | ||
|
|
024fc9fcf0 | ||
|
|
4220f4e095 | ||
|
|
a709acb149 | ||
|
|
466842acd8 | ||
|
|
b91f28f95b | ||
|
|
87164db3da | ||
|
|
ad16dec3db | ||
|
|
a0da0f4713 | ||
|
|
65e5cde60d | ||
|
|
0f0b464f4e |
@@ -23,7 +23,7 @@
|
||||
|
||||
| 领域 | 当前能力 |
|
||||
| --- | --- |
|
||||
| 数据面 | Responses JSON、SSE、WebSocket、review 子代理、Images 生成/编辑与模型目录 |
|
||||
| 数据面 | Responses JSON、SSE、WebSocket(含 Codex 子代理语义)、Images 生成/编辑与模型目录 |
|
||||
| Provider | 固定 `openai`、`xai` 两个 Provider,各自拥有 OAuth、credential、额度、目录与 transport |
|
||||
| 路由 | Client Key 限定账号分组;按模型能力编译 Provider 候选;支持全局精确模型映射、会话亲和和安全 fallback |
|
||||
| 透明边界 | OpenAI 保留未知请求字段及字段顺序,并原样转发 SSE、WebSocket 与 Images 业务字节;xAI 在 Provider 内转换协议 |
|
||||
@@ -136,12 +136,9 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
| --- | --- |
|
||||
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
|
||||
| `GET /v1/responses` | Responses WebSocket 升级 |
|
||||
| `POST /v1/responses/review` | review 子代理请求 |
|
||||
| `POST /v1/images/generations` | OpenAI Images 生成 JSON 透明转发 |
|
||||
| `POST /v1/images/edits` | OpenAI Images 编辑 JSON 透明转发 |
|
||||
| `GET /v1/models` | 当前 Client Key 账号范围内的模型列表 |
|
||||
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
|
||||
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
|
||||
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
|
||||
|
||||
所有 `/v1/*` 路由都需要 Client Key。Images 是 OpenAI Provider 自有端点,不读取模型字段,也不参与
|
||||
|
||||
@@ -230,6 +230,8 @@ pub(super) fn account_usage_view(
|
||||
output_tokens_display: display_optional_tokens(usage.output_tokens),
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cached_tokens_display: display_optional_tokens(usage.cached_tokens),
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
reasoning_tokens_display: display_optional_tokens(usage.reasoning_tokens),
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_input_tokens_display: display_optional_tokens(usage.image_input_tokens),
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
@@ -347,6 +349,8 @@ pub(super) fn empty_account_usage() -> AccountUsageView {
|
||||
output_tokens_display: "—".to_owned(),
|
||||
cached_tokens: None,
|
||||
cached_tokens_display: "—".to_owned(),
|
||||
reasoning_tokens: None,
|
||||
reasoning_tokens_display: "—".to_owned(),
|
||||
image_input_tokens: None,
|
||||
image_input_tokens_display: "—".to_owned(),
|
||||
image_output_tokens: None,
|
||||
|
||||
@@ -276,6 +276,8 @@ pub struct AccountUsageView {
|
||||
pub output_tokens_display: String,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cached_tokens_display: String,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub reasoning_tokens_display: String,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_input_tokens_display: String,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
|
||||
@@ -3,22 +3,21 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
body::Bytes,
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
http::HeaderMap,
|
||||
response::Response,
|
||||
};
|
||||
use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::operation::{ImageRequest, ImageRequestKind, Operation, RawJsonPayload};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, client_access_error_response},
|
||||
error::{engine_error_response, gateway_error_response, protocol_error_response},
|
||||
responses::{PendingExecution, ProtocolError, ProtocolErrorBody, request_client_context},
|
||||
error::gateway_error_response,
|
||||
provider_endpoint::collect_raw_json_response,
|
||||
responses::request_client_context,
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
@@ -82,13 +81,13 @@ async fn handle_image_request(
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
let started = match service
|
||||
.start_image(client, operation, client_ip, user_agent, endpoint)
|
||||
.start_provider_endpoint(client, operation, client_ip, user_agent, endpoint)
|
||||
.await
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
collect_image_response(started).await
|
||||
collect_raw_json_response(started).await
|
||||
}
|
||||
|
||||
fn image_operation(
|
||||
@@ -118,116 +117,3 @@ fn image_operation(
|
||||
kind, payload,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn collect_image_response(started: StartedExecution) -> Response {
|
||||
let mut execution = PendingExecution::new(started.session);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
let body = match raw_json_body(events) {
|
||||
Some(body) => body,
|
||||
None => {
|
||||
let response = invalid_upstream_response();
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
};
|
||||
let status = session
|
||||
.response_status_code()
|
||||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.filter(StatusCode::is_success)
|
||||
.unwrap_or(StatusCode::OK);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = json_body_response(body, status, &response_headers);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
execution.cancel_and_finalize().await;
|
||||
return invalid_upstream_response();
|
||||
}
|
||||
execution.disarm();
|
||||
response
|
||||
}
|
||||
|
||||
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut body = None;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = wire.into_raw_json_body() else {
|
||||
continue;
|
||||
};
|
||||
if body.replace(raw).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn json_body_response(
|
||||
body: Bytes,
|
||||
status: StatusCode,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options =
|
||||
crate::openai::responses::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
if !crate::openai::responses::response_header_is_forwardable(
|
||||
header.name(),
|
||||
&connection_options,
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn invalid_upstream_response() -> Response {
|
||||
protocol_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream image response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ pub mod auth;
|
||||
pub mod error;
|
||||
pub mod images;
|
||||
pub mod models;
|
||||
mod provider_endpoint;
|
||||
pub mod responses;
|
||||
pub mod router;
|
||||
pub mod search;
|
||||
pub(crate) mod service;
|
||||
|
||||
@@ -97,50 +97,6 @@ pub(crate) async fn model_detail(
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `GET /v1/models/catalog`。
|
||||
///
|
||||
/// 新架构的目录由 core 按客户端 Provider 范围裁剪;这里不读取或重建任一
|
||||
/// Provider 的 wire catalog,只恢复 Codex 客户端依赖的稳定展示合同。
|
||||
pub(crate) async fn model_catalog(State(state): State<ApiState>, headers: HeaderMap) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let models = service
|
||||
.public_models(&client)
|
||||
.into_iter()
|
||||
.map(|model| catalog_model_json(model.as_str()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
(StatusCode::OK, Json(models)).into_response()
|
||||
}
|
||||
|
||||
/// `GET /v1/models/{model_id}/info`。
|
||||
pub(crate) async fn model_info(
|
||||
State(state): State<ApiState>,
|
||||
headers: HeaderMap,
|
||||
Path(model_id): Path<String>,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let Ok(public_model) = PublicModelId::new(model_id) else {
|
||||
return model_not_found_response().into_response();
|
||||
};
|
||||
if !service.contains_public_model(&client, &public_model) {
|
||||
return model_not_found_response().into_response();
|
||||
}
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(catalog_model_json(public_model.as_str())),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn openai_model_json(id: &str) -> Value {
|
||||
json!({
|
||||
"id": id,
|
||||
@@ -150,22 +106,6 @@ fn openai_model_json(id: &str) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
fn catalog_model_json(id: &str) -> Value {
|
||||
json!({
|
||||
"id": id,
|
||||
"displayName": id,
|
||||
"description": "",
|
||||
"isDefault": false,
|
||||
"supportedReasoningEfforts": [],
|
||||
"defaultReasoningEffort": "",
|
||||
"inputModalities": ["text"],
|
||||
"outputModalities": ["text"],
|
||||
"supportsPersonality": false,
|
||||
"upgrade": Value::Null,
|
||||
"source": "gateway",
|
||||
})
|
||||
}
|
||||
|
||||
fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
let id = profile.model().as_str();
|
||||
let presentation = profile.presentation();
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
//! OpenAI Provider 原生非流式 JSON 端点的公共交付边界。
|
||||
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
http::{HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
use super::{
|
||||
error::{engine_error_response, protocol_error_response},
|
||||
responses::{PendingExecution, ProtocolError, ProtocolErrorBody},
|
||||
};
|
||||
|
||||
/// 收集一个 Provider 原生 JSON 响应,并在 Core commit 后按原始 bytes 交付。
|
||||
pub(super) async fn collect_raw_json_response(started: StartedExecution) -> Response {
|
||||
let mut execution = PendingExecution::new(started.session);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
let body = match raw_json_body(events) {
|
||||
Some(body) => body,
|
||||
None => {
|
||||
let response = invalid_upstream_response();
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
};
|
||||
let status = session
|
||||
.response_status_code()
|
||||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.filter(StatusCode::is_success)
|
||||
.unwrap_or(StatusCode::OK);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = json_body_response(body, status, &response_headers);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
execution.cancel_and_finalize().await;
|
||||
return invalid_upstream_response();
|
||||
}
|
||||
execution.disarm();
|
||||
response
|
||||
}
|
||||
|
||||
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut body = None;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = wire.into_raw_json_body() else {
|
||||
continue;
|
||||
};
|
||||
if body.replace(raw).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn json_body_response(
|
||||
body: Bytes,
|
||||
status: StatusCode,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options = super::responses::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
if !super::responses::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn invalid_upstream_response() -> Response {
|
||||
protocol_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream JSON response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -14,8 +14,8 @@ use axum::{
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::stream;
|
||||
use gateway_core::engine::CommitRequirement;
|
||||
use gateway_core::engine::execution::{ClientTransport, ExecutionSession, StartedExecution};
|
||||
use gateway_core::engine::{CommitRequirement, EngineError};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::lifecycle::ConnectionGuard;
|
||||
@@ -32,7 +32,7 @@ use crate::openai::{
|
||||
|
||||
use super::{
|
||||
OpenAiResponsesEncoder, ProtocolErrorBody, ResponseEncodeError,
|
||||
request::{decode_request_with_headers, decode_review_request_with_headers},
|
||||
request::decode_request_with_headers,
|
||||
};
|
||||
|
||||
/// `POST /v1/responses`。
|
||||
@@ -41,37 +41,13 @@ pub(crate) async fn responses(
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_responses(state, connect_info, headers, body, false).await
|
||||
}
|
||||
|
||||
/// `POST /v1/responses/review`。
|
||||
pub(crate) async fn review_responses(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_responses(state, connect_info, headers, body, true).await
|
||||
}
|
||||
|
||||
async fn handle_responses(
|
||||
state: ApiState,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
review: bool,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let decoded = match if review {
|
||||
decode_review_request_with_headers(&body, &headers)
|
||||
} else {
|
||||
decode_request_with_headers(&body, &headers)
|
||||
} {
|
||||
let decoded = match decode_request_with_headers(&body, &headers) {
|
||||
Ok(decoded) => decoded,
|
||||
Err(error) => {
|
||||
return protocol_error_response(StatusCode::BAD_REQUEST, error.protocol_body());
|
||||
@@ -100,17 +76,26 @@ async fn handle_responses(
|
||||
} else {
|
||||
ClientTransport::HttpJson
|
||||
},
|
||||
if review {
|
||||
"/v1/responses/review"
|
||||
} else {
|
||||
"/v1/responses"
|
||||
},
|
||||
"/v1/responses",
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
request_id = %started.request_id,
|
||||
request_direction = "downstream",
|
||||
request_transport = if streaming { "http_sse" } else { "http_json" },
|
||||
request_method = "POST",
|
||||
request_path = "/v1/responses",
|
||||
request_headers = ?headers,
|
||||
request_body = %String::from_utf8_lossy(&body),
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
let StartedExecution {
|
||||
stream, session, ..
|
||||
} = started;
|
||||
@@ -180,7 +165,8 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = engine_error_response_with_headers(&error, &response_headers);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -206,7 +192,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
let response = engine_error_response_with_headers(&error, &response_headers);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
@@ -274,6 +260,13 @@ fn apply_response_headers(
|
||||
response
|
||||
}
|
||||
|
||||
fn engine_error_response_with_headers(
|
||||
error: &EngineError,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
apply_response_headers(engine_error_response(error), response_headers)
|
||||
}
|
||||
|
||||
/// 编码首个 SSE frame 后提交下游,再持续驱动同一执行会话。
|
||||
pub async fn stream_execution_response(
|
||||
session: Box<dyn ExecutionSession>,
|
||||
@@ -295,7 +288,8 @@ pub async fn stream_execution_response(
|
||||
return response;
|
||||
}
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = engine_error_response_with_headers(&error, &response_headers);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -327,7 +321,7 @@ pub async fn stream_execution_response(
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
let response = engine_error_response_with_headers(&error, &response_headers);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
let Some(session) = execution.into_session() else {
|
||||
|
||||
@@ -7,9 +7,9 @@ mod response;
|
||||
pub mod websocket;
|
||||
|
||||
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
|
||||
pub(crate) use http::responses;
|
||||
pub(super) use http::{PendingExecution, request_client_context};
|
||||
pub use http::{collect_execution_response, stream_execution_response};
|
||||
pub(crate) use http::{responses, review_responses};
|
||||
pub use request::{
|
||||
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, ResponsesRequestMetadata,
|
||||
decode_request_with_headers,
|
||||
|
||||
@@ -13,10 +13,10 @@ use serde_json::{Map, Value};
|
||||
use super::error::RequestDecodeError;
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
const DOWNSTREAM_WEBSOCKET_CONNECTION_ID_CONTEXT_KEY: &str = "downstream_websocket_connection_id";
|
||||
|
||||
/// Responses 请求进入共享解码内核时的下游传输来源。
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -36,6 +36,7 @@ pub struct OpenAiRequestHeaders {
|
||||
include_timing_metrics: Option<String>,
|
||||
codex_window_id: Option<String>,
|
||||
parent_thread_id: Option<String>,
|
||||
downstream_websocket_connection_id: Option<String>,
|
||||
|
||||
conversation_id: Option<String>,
|
||||
session_id: Option<String>,
|
||||
@@ -61,6 +62,7 @@ impl OpenAiRequestHeaders {
|
||||
include_timing_metrics: header_string(headers, "x-responsesapi-include-timing-metrics"),
|
||||
codex_window_id: header_string(headers, "x-codex-window-id"),
|
||||
parent_thread_id: header_string(headers, "x-codex-parent-thread-id"),
|
||||
downstream_websocket_connection_id: None,
|
||||
conversation_id: header_string(headers, "conversation-id")
|
||||
.or_else(|| header_string(headers, "conversation_id")),
|
||||
session_id: header_string(headers, "session-id")
|
||||
@@ -75,8 +77,18 @@ impl OpenAiRequestHeaders {
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_subagent(&self, body: &mut Map<String, Value>, forced_subagent: Option<&str>) {
|
||||
if let Some(subagent) = forced_subagent.or(self.subagent.as_deref()) {
|
||||
/// 绑定代理为当前下游 WebSocket 分配的连接身份。
|
||||
#[must_use]
|
||||
pub(super) fn with_downstream_websocket_connection_id(
|
||||
mut self,
|
||||
connection_id: impl Into<String>,
|
||||
) -> Self {
|
||||
self.downstream_websocket_connection_id = Some(connection_id.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn apply_subagent(&self, body: &mut Map<String, Value>) {
|
||||
if let Some(subagent) = self.subagent.as_deref() {
|
||||
inject_subagent_metadata(body, subagent);
|
||||
}
|
||||
}
|
||||
@@ -110,6 +122,11 @@ impl OpenAiRequestHeaders {
|
||||
"parent_thread_id",
|
||||
self.parent_thread_id.as_ref(),
|
||||
);
|
||||
insert_protocol_context(
|
||||
&mut context,
|
||||
DOWNSTREAM_WEBSOCKET_CONNECTION_ID_CONTEXT_KEY,
|
||||
self.downstream_websocket_connection_id.as_ref(),
|
||||
);
|
||||
insert_protocol_context(
|
||||
&mut context,
|
||||
"conversation_id",
|
||||
@@ -301,20 +318,11 @@ pub fn decode_request_with_headers(
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
decode_request_inner(body, false, &OpenAiRequestHeaders::from_headers(headers))
|
||||
}
|
||||
|
||||
/// 解码 `POST /v1/responses/review` 请求并冻结 review subagent 语义。
|
||||
pub(super) fn decode_review_request_with_headers(
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
decode_request_inner(body, true, &OpenAiRequestHeaders::from_headers(headers))
|
||||
decode_request_inner(body, &OpenAiRequestHeaders::from_headers(headers))
|
||||
}
|
||||
|
||||
pub(super) fn decode_request_inner(
|
||||
body: &[u8],
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
let value =
|
||||
@@ -322,13 +330,12 @@ pub(super) fn decode_request_inner(
|
||||
let Value::Object(object) = value else {
|
||||
return Err(RequestDecodeError::ExpectedObject);
|
||||
};
|
||||
decode_request_object(object, review, request_headers, RequestDecodeSource::Http)
|
||||
decode_request_object(object, request_headers, RequestDecodeSource::Http)
|
||||
}
|
||||
|
||||
/// 解码已解析的顶层 object;按下游传输来源恢复连接级协议上下文。
|
||||
pub(super) fn decode_request_object(
|
||||
mut object: Map<String, Value>,
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
source: RequestDecodeSource,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
@@ -359,7 +366,7 @@ pub(super) fn decode_request_object(
|
||||
.map(|response_id| ContinuationIntent::PreviousResponseId(response_id.to_owned()))
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
request_headers.apply_subagent(&mut object, review.then_some(REVIEW_SUBAGENT));
|
||||
request_headers.apply_subagent(&mut object);
|
||||
let frame_turn_metadata = match source {
|
||||
RequestDecodeSource::Http => None,
|
||||
RequestDecodeSource::WebSocketFrame => frame_turn_metadata(&object),
|
||||
|
||||
@@ -77,8 +77,8 @@ pub(super) async fn forward_execution(
|
||||
return send_gateway_error(connection, &error, &request_id).await;
|
||||
}
|
||||
ActiveInput::Event(Err(error)) => {
|
||||
let error = gateway_error_from_engine(&error);
|
||||
return send_gateway_error(connection, &error, &request_id).await;
|
||||
return send_initial_engine_error(connection, &mut execution, &error, &request_id)
|
||||
.await;
|
||||
}
|
||||
ActiveInput::Control => continue,
|
||||
ActiveInput::Disconnect => return ForwardOutcome::Disconnect,
|
||||
@@ -271,6 +271,30 @@ pub(super) async fn forward_execution(
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_initial_engine_error(
|
||||
connection: &mut ResponsesWebSocketConnection,
|
||||
execution: &mut PendingExecution,
|
||||
error: &EngineError,
|
||||
request_id: &Arc<str>,
|
||||
) -> ForwardOutcome {
|
||||
let response_headers = execution
|
||||
.session_mut()
|
||||
.map(|session| session.response_headers().to_vec())
|
||||
.unwrap_or_default();
|
||||
if !response_headers.is_empty()
|
||||
&& !send_metadata(
|
||||
connection,
|
||||
request_id,
|
||||
response_metadata_event(request_id, &response_headers),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return ForwardOutcome::Disconnect;
|
||||
}
|
||||
let error = gateway_error_from_engine(error);
|
||||
send_gateway_error(connection, &error, request_id).await
|
||||
}
|
||||
|
||||
fn commit_connection_replay(
|
||||
replay: &mut ConnectionReplaySnapshot,
|
||||
encoder: &OpenAiResponsesEncoder,
|
||||
|
||||
@@ -60,12 +60,14 @@ pub(crate) async fn responses_websocket(
|
||||
&headers,
|
||||
connect_info.map(|Extension(ConnectInfo(address))| address),
|
||||
);
|
||||
let request_headers = OpenAiRequestHeaders::from_headers(&headers);
|
||||
ResponsesWebSocketAdapter::new(service).upgrade_with_client_context(
|
||||
websocket,
|
||||
client,
|
||||
client_ip,
|
||||
user_agent,
|
||||
OpenAiRequestHeaders::from_headers(&headers),
|
||||
request_headers,
|
||||
headers,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -89,15 +91,29 @@ impl ResponsesWebSocketAdapter {
|
||||
client_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
request_headers: OpenAiRequestHeaders,
|
||||
raw_headers: HeaderMap,
|
||||
) -> Response {
|
||||
let connection_guard = match self.service.try_register_connection() {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => return runtime_unavailable_response().into_response(),
|
||||
};
|
||||
let connection_id = self.service.next_request_id().replacen("req_", "ws_", 1);
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
websocket_connection_id = %connection_id,
|
||||
request_direction = "downstream",
|
||||
request_transport = "websocket_handshake",
|
||||
request_method = "GET",
|
||||
request_path = "/v1/responses",
|
||||
request_headers = ?raw_headers,
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
let session = ResponsesWebSocketSession {
|
||||
service: self.service.clone(),
|
||||
client,
|
||||
connection_id: self.service.next_request_id().replacen("req_", "ws_", 1),
|
||||
connection_id,
|
||||
client_ip,
|
||||
user_agent,
|
||||
request_headers,
|
||||
@@ -143,6 +159,8 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
);
|
||||
let _connection_guard = connection_guard;
|
||||
let cancellation = lifecycle.cancellation();
|
||||
let request_headers =
|
||||
request_headers.with_downstream_websocket_connection_id(connection_id.clone());
|
||||
let mut connection = ResponsesWebSocketConnection::new(socket, connection_id, cancellation);
|
||||
let mut request_count = 0_u64;
|
||||
let mut replay = ConnectionReplaySnapshot::default();
|
||||
@@ -169,6 +187,18 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
}
|
||||
request_count = request_count.saturating_add(1);
|
||||
let correlation_id = Arc::<str>::from(service.next_request_id());
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
websocket_connection_id = connection.id(),
|
||||
correlation_id = %correlation_id,
|
||||
request_index = request_count,
|
||||
request_direction = "downstream",
|
||||
request_transport = "websocket",
|
||||
request_payload = %payload,
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
let decoded = match decode_response_create_with_context(&payload, &request_headers) {
|
||||
Ok(decoded) => decoded.with_client_context(client_ip, user_agent.clone()),
|
||||
Err(error) => {
|
||||
@@ -217,6 +247,14 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
continue;
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
websocket_connection_id = connection.id(),
|
||||
correlation_id = %correlation_id,
|
||||
request_id = %started.request_id,
|
||||
"request dump correlation"
|
||||
);
|
||||
|
||||
if forward_execution(&mut connection, started, &mut replay).await
|
||||
== ForwardOutcome::Disconnect
|
||||
|
||||
@@ -43,7 +43,6 @@ fn decode_response_create_inner(
|
||||
}
|
||||
super::super::request::decode_request_object(
|
||||
body,
|
||||
false,
|
||||
request_headers,
|
||||
RequestDecodeSource::WebSocketFrame,
|
||||
)
|
||||
|
||||
@@ -8,8 +8,9 @@ use axum::{
|
||||
|
||||
use super::{
|
||||
images::{image_edits, image_generations},
|
||||
models::{model_catalog, model_detail, model_info, models},
|
||||
responses::{responses, responses_websocket, review_responses},
|
||||
models::{model_detail, models},
|
||||
responses::{responses, responses_websocket},
|
||||
search::standalone_search,
|
||||
};
|
||||
|
||||
use crate::ApiState;
|
||||
@@ -19,11 +20,9 @@ pub(crate) fn router() -> Router<ApiState> {
|
||||
Router::new()
|
||||
.route("/v1/images/generations", post(image_generations))
|
||||
.route("/v1/images/edits", post(image_edits))
|
||||
.route("/v1/alpha/search", post(standalone_search))
|
||||
.route("/v1/responses", get(responses_websocket).post(responses))
|
||||
.route("/v1/responses/review", post(review_responses))
|
||||
.route("/v1/models", get(models))
|
||||
.route("/v1/models/catalog", get(model_catalog))
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
// OpenAI 数据面正文属于客户端/上游协议;代理不能用私有大小上限提前拒绝
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
//! Codex standalone search 非流式 HTTP adapter。
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::{
|
||||
body::Bytes,
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::HeaderMap,
|
||||
response::Response,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::operation::{Operation, RawJsonPayload, StandaloneSearchRequest};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, client_access_error_response},
|
||||
error::gateway_error_response,
|
||||
provider_endpoint::collect_raw_json_response,
|
||||
responses::request_client_context,
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const TURN_METADATA_CONTEXT_KEY: &str = "turn_metadata";
|
||||
|
||||
/// `POST /v1/alpha/search`。
|
||||
pub(crate) async fn standalone_search(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let (client_ip, user_agent) = request_client_context(
|
||||
&headers,
|
||||
connect_info.map(|Extension(ConnectInfo(address))| address),
|
||||
);
|
||||
let operation = match search_operation(body, &headers) {
|
||||
Ok(operation) => operation,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
let started = match service
|
||||
.start_provider_endpoint(client, operation, client_ip, user_agent, "/v1/alpha/search")
|
||||
.await
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
collect_raw_json_response(started).await
|
||||
}
|
||||
|
||||
fn search_operation(body: Bytes, headers: &HeaderMap) -> Result<Operation, GatewayError> {
|
||||
let mut context = Map::new();
|
||||
if let Some(turn_metadata) = headers
|
||||
.get("x-codex-turn-metadata")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
context.insert(
|
||||
TURN_METADATA_CONTEXT_KEY.to_owned(),
|
||||
Value::String(turn_metadata.to_owned()),
|
||||
);
|
||||
}
|
||||
let payload = RawJsonPayload::new(OPENAI_PROTOCOL, body)
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI protocol identifier is invalid",
|
||||
)
|
||||
})?
|
||||
.with_context(context);
|
||||
Ok(Operation::Search(StandaloneSearchRequest::from_raw_json(
|
||||
payload,
|
||||
)))
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Codex standalone search HTTP adapter。
|
||||
|
||||
mod http;
|
||||
|
||||
pub(crate) use http::standalone_search;
|
||||
@@ -106,7 +106,7 @@ impl OpenAiService {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn start_image(
|
||||
pub(crate) async fn start_provider_endpoint(
|
||||
&self,
|
||||
client: AuthenticatedClient,
|
||||
operation: gateway_core::operation::Operation,
|
||||
|
||||
@@ -348,6 +348,8 @@ mod response {
|
||||
output_tokens_display: "-".to_owned(),
|
||||
cached_tokens: None,
|
||||
cached_tokens_display: "-".to_owned(),
|
||||
reasoning_tokens: None,
|
||||
reasoning_tokens_display: "-".to_owned(),
|
||||
image_input_tokens: None,
|
||||
image_input_tokens_display: "-".to_owned(),
|
||||
image_output_tokens: None,
|
||||
@@ -374,6 +376,8 @@ mod response {
|
||||
let value = serde_json::to_value(view).expect("serialize account usage");
|
||||
assert!(value["inputTokens"].is_null());
|
||||
assert!(value["totalTokens"].is_null());
|
||||
assert!(value["reasoningTokens"].is_null());
|
||||
assert_eq!(value["reasoningTokensDisplay"], "-");
|
||||
assert_eq!(value["createdTokensDisplay"], "-");
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/openai/images/mod.rs",
|
||||
"src/openai/mod.rs",
|
||||
"src/openai/models.rs",
|
||||
"src/openai/provider_endpoint.rs",
|
||||
"src/openai/responses/error.rs",
|
||||
"src/openai/responses/http.rs",
|
||||
"src/openai/responses/mod.rs",
|
||||
@@ -71,6 +72,8 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/openai/responses/websocket/mod.rs",
|
||||
"src/openai/responses/websocket/protocol.rs",
|
||||
"src/openai/router.rs",
|
||||
"src/openai/search/http.rs",
|
||||
"src/openai/search/mod.rs",
|
||||
"src/openai/service.rs",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
@@ -104,12 +107,14 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/openai/images/mod.rs",
|
||||
"tests/openai/mod.rs",
|
||||
"tests/openai/models.rs",
|
||||
"tests/openai/provider_endpoint.rs",
|
||||
"tests/openai/responses/http.rs",
|
||||
"tests/openai/responses/mod.rs",
|
||||
"tests/openai/responses/websocket/connection.rs",
|
||||
"tests/openai/responses/websocket/mod.rs",
|
||||
"tests/openai/responses/websocket/protocol.rs",
|
||||
"tests/openai/router.rs",
|
||||
"tests/openai/search/mod.rs",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
|
||||
|
||||
@@ -2,8 +2,10 @@ mod auth;
|
||||
mod error;
|
||||
mod images;
|
||||
mod models;
|
||||
mod provider_endpoint;
|
||||
mod responses;
|
||||
mod router;
|
||||
mod search;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
@@ -383,46 +383,6 @@ async fn model_detail_should_hide_unknown_models() {
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_catalog_should_keep_the_codex_catalog_contract() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models/catalog"))
|
||||
.await
|
||||
.expect("catalog response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read catalog body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("catalog JSON");
|
||||
assert_eq!(value[0]["id"], "model-a");
|
||||
assert_eq!(value[0]["displayName"], "model-a");
|
||||
assert_eq!(value[0]["source"], "gateway");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_info_should_return_a_single_visible_catalog_entry() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models/model-a/info"))
|
||||
.await
|
||||
.expect("model info response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_info_should_hide_unknown_models() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models/model-private/info"))
|
||||
.await
|
||||
.expect("missing model info response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_should_authenticate_before_applying_the_version_gate() {
|
||||
let response = api_router(ModelsExecution::with_cli_min())
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
use std::sync::{Arc, Mutex, atomic::AtomicBool, atomic::Ordering};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::execution::ExecutionSession;
|
||||
use gateway_core::engine::{CoordinatedEvent, EngineError};
|
||||
use gateway_core::error::ProviderError;
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
pub(super) struct BufferedJsonSession {
|
||||
body: Option<Bytes>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
response_status: Option<u16>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
finalized: AtomicBool,
|
||||
failure: Option<ProviderError>,
|
||||
}
|
||||
|
||||
impl BufferedJsonSession {
|
||||
pub(super) fn success(
|
||||
body: Bytes,
|
||||
response_status: u16,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
body: Some(body),
|
||||
response_headers,
|
||||
response_status: Some(response_status),
|
||||
committed_statuses,
|
||||
finalized: AtomicBool::new(false),
|
||||
failure: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn failure(error: ProviderError) -> Self {
|
||||
Self {
|
||||
body: None,
|
||||
response_headers: Vec::new(),
|
||||
response_status: None,
|
||||
committed_statuses: Arc::new(Mutex::new(Vec::new())),
|
||||
finalized: AtomicBool::new(false),
|
||||
failure: Some(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionSession for BufferedJsonSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async { unreachable!("buffered JSON delivery does not stream") })
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
if let Some(error) = self.failure.take() {
|
||||
return Err(EngineError::Provider(error));
|
||||
}
|
||||
Ok(vec![ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_json(
|
||||
"openai",
|
||||
self.body.take().expect("single buffered JSON response"),
|
||||
)
|
||||
.expect("OpenAI protocol"),
|
||||
)])
|
||||
})
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.response_headers
|
||||
}
|
||||
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
self.response_status
|
||||
}
|
||||
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.committed_statuses
|
||||
.lock()
|
||||
.expect("committed status lock")
|
||||
.push(client_status_code.expect("buffered JSON response status"));
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn cancel(&self) {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ use serde_json::{Value, json};
|
||||
use gateway_api::openai::responses::{collect_execution_response, stream_execution_response};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::openai::{api_router, authenticated_client, authenticated_client_for_provider};
|
||||
use crate::openai::{api_router, authenticated_client_for_provider};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Trace {
|
||||
@@ -689,92 +689,6 @@ async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
assert!(captured.prompt_cache_key.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_inject_review_subagent_and_record_its_endpoint() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.header("x-openai-subagent", "compact")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": {"existing": "preserved"}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(captured.endpoint, "/v1/responses/review");
|
||||
assert_eq!(
|
||||
captured
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent")),
|
||||
Some(&json!("review"))
|
||||
);
|
||||
assert_eq!(
|
||||
captured
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("existing")),
|
||||
Some(&json!("preserved"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_not_replace_non_object_client_metadata() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": ["opaque", "client", "value"]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(
|
||||
captured.client_metadata,
|
||||
Some(json!(["opaque", "client", "value"]))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -979,6 +893,41 @@ async fn streaming_rate_limit_before_first_frame_should_persist_the_returned_429
|
||||
assert_eq!(trace.client_statuses(), vec![429]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_failure_before_first_frame_should_return_safe_observed_headers() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![NextStep::Error(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)))],
|
||||
)
|
||||
.with_response_headers(vec![
|
||||
ProviderResponseHeader::new(
|
||||
"x-codex-turn-state",
|
||||
Bytes::from_static(b"turn-state-before-close"),
|
||||
),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
Bytes::from_static(b"must-not-cross-boundary"),
|
||||
),
|
||||
]);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-codex-turn-state")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("turn-state-before-close")
|
||||
);
|
||||
assert!(response.headers().get("authorization").is_none());
|
||||
assert_eq!(trace.client_statuses(), vec![502]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_upstream_http_failure_before_first_frame_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -1065,11 +1014,29 @@ async fn buffered_upstream_http_failure_should_preserve_raw_response() {
|
||||
raw_body.clone(),
|
||||
));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_collect_error(EngineError::Provider(error));
|
||||
.with_collect_error(EngineError::Provider(error))
|
||||
.with_response_headers(vec![
|
||||
ProviderResponseHeader::new(
|
||||
"x-codex-turn-state",
|
||||
Bytes::from_static(b"turn-state-before-close"),
|
||||
),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
Bytes::from_static(b"must-not-cross-boundary"),
|
||||
),
|
||||
]);
|
||||
|
||||
let response = collect_execution_response(Box::new(session)).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-codex-turn-state")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("turn-state-before-close")
|
||||
);
|
||||
assert!(response.headers().get("authorization").is_none());
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
|
||||
@@ -428,6 +428,7 @@ async fn idle_connection_reaches_the_official_limit_without_starting_an_executio
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace: Arc::clone(&trace),
|
||||
response_headers: Vec::new(),
|
||||
fail_before_first_event: false,
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
|
||||
@@ -2,7 +2,7 @@ mod connection;
|
||||
mod protocol;
|
||||
|
||||
use std::sync::{
|
||||
Arc,
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime};
|
||||
@@ -280,17 +280,26 @@ struct AtomicFailureTrace {
|
||||
next_calls: AtomicUsize,
|
||||
committed: AtomicBool,
|
||||
finalized: AtomicBool,
|
||||
downstream_websocket_connection_ids: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
struct AtomicFailureSession {
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
fail_before_first_event: bool,
|
||||
}
|
||||
|
||||
impl ExecutionSession for AtomicFailureSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
match self.trace.next_calls.fetch_add(1, Ordering::AcqRel) {
|
||||
let next_call = self.trace.next_calls.fetch_add(1, Ordering::AcqRel);
|
||||
if self.fail_before_first_event && next_call == 0 {
|
||||
return Err(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)));
|
||||
}
|
||||
match next_call {
|
||||
0 => Ok(Some(atomic_failure_batch())),
|
||||
1 => {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
@@ -340,6 +349,7 @@ struct AtomicFailureExecution {
|
||||
client: AuthenticatedClient,
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
fail_before_first_event: bool,
|
||||
}
|
||||
|
||||
impl ExecutionService for AtomicFailureExecution {
|
||||
@@ -365,6 +375,19 @@ impl ExecutionService for AtomicFailureExecution {
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
self.trace.starts.fetch_add(1, Ordering::AcqRel);
|
||||
if let Operation::Generate(generate) = &request.operation
|
||||
&& let Some(connection_id) = generate
|
||||
.protocol_payload()
|
||||
.context()
|
||||
.get("downstream_websocket_connection_id")
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
self.trace
|
||||
.downstream_websocket_connection_ids
|
||||
.lock()
|
||||
.expect("downstream connection trace lock")
|
||||
.push(connection_id.to_owned());
|
||||
}
|
||||
Box::pin(async move {
|
||||
Ok(StartedExecution {
|
||||
request_id: gateway_core::engine::ModelRequestId::new("req_ws_atomic")
|
||||
@@ -374,6 +397,7 @@ impl ExecutionService for AtomicFailureExecution {
|
||||
session: Box::new(AtomicFailureSession {
|
||||
trace: Arc::clone(&self.trace),
|
||||
response_headers: self.response_headers.clone(),
|
||||
fail_before_first_event: self.fail_before_first_event,
|
||||
}),
|
||||
})
|
||||
})
|
||||
@@ -439,6 +463,7 @@ async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace: Arc::clone(&trace),
|
||||
response_headers: Vec::new(),
|
||||
fail_before_first_event: false,
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
@@ -510,6 +535,14 @@ async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
|
||||
assert!(trace.committed.load(Ordering::Acquire));
|
||||
assert_eq!(trace.next_calls.load(Ordering::Acquire), 2);
|
||||
assert!(trace.finalized.load(Ordering::Acquire));
|
||||
{
|
||||
let downstream_connection_ids = trace
|
||||
.downstream_websocket_connection_ids
|
||||
.lock()
|
||||
.expect("downstream connection trace lock");
|
||||
assert_eq!(downstream_connection_ids.len(), 1);
|
||||
assert!(downstream_connection_ids[0].starts_with("ws_"));
|
||||
}
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
@@ -540,6 +573,7 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req_upstream")),
|
||||
],
|
||||
fail_before_first_event: false,
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
@@ -624,6 +658,91 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_failure_before_first_event_should_return_observed_metadata_then_error() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace: Arc::clone(&trace),
|
||||
response_headers: vec![
|
||||
ProviderResponseHeader::new(
|
||||
"x-codex-turn-state",
|
||||
Bytes::from_static(b"turn-state-before-close"),
|
||||
),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
Bytes::from_static(b"must-not-cross-boundary"),
|
||||
),
|
||||
],
|
||||
fail_before_first_event: true,
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut messages = Vec::<Value>::new();
|
||||
while messages.len() < 2 {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
messages.push(serde_json::from_str(&text).expect("JSON WebSocket event"));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
messages
|
||||
.iter()
|
||||
.filter_map(|message| message.get("type").and_then(Value::as_str))
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["response.metadata", "error"]
|
||||
);
|
||||
let headers = messages[0]
|
||||
.get("headers")
|
||||
.and_then(Value::as_object)
|
||||
.expect("response metadata headers");
|
||||
assert_eq!(
|
||||
headers.get("x-codex-turn-state"),
|
||||
Some(&json!("turn-state-before-close"))
|
||||
);
|
||||
assert!(!headers.contains_key("authorization"));
|
||||
assert!(!trace.committed.load(Ordering::Acquire));
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
@@ -631,6 +750,7 @@ async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit(
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: Vec::new(),
|
||||
fail_before_first_event: false,
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode},
|
||||
http::{Method, Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
@@ -9,6 +9,54 @@ use super::models::ModelsExecution;
|
||||
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_responses_review_route_should_not_reach_the_responses_handler() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed review request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_models_catalog_extension_should_use_standard_model_lookup() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/catalog")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model catalog request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model catalog request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_model_info_route_should_return_not_found() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/model-a/info")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model info request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model info request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::ModelRequestId;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionService,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::ProviderResponseHeader;
|
||||
use gateway_core::operation::Operation;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::provider_endpoint::BufferedJsonSession;
|
||||
use super::{api_router, authenticated_client};
|
||||
|
||||
const SEARCH_RESPONSE: &[u8] = br#"{ "encrypted_output":"ciphertext", "output":"search result", "results":[{"type":"text_result","ref_id":"turn0search0","future":9007199254740993}] }"#;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct CapturedSearchRequest {
|
||||
provider: String,
|
||||
endpoint: String,
|
||||
transport: ClientTransport,
|
||||
body: Bytes,
|
||||
context: Value,
|
||||
}
|
||||
|
||||
struct SearchExecution {
|
||||
client: AuthenticatedClient,
|
||||
captured: Mutex<Vec<CapturedSearchRequest>>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
fail_with_upstream_response: bool,
|
||||
}
|
||||
|
||||
impl SearchExecution {
|
||||
fn new(fail_with_upstream_response: bool) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_search_test"),
|
||||
captured: Mutex::new(Vec::new()),
|
||||
committed_statuses: Arc::new(Mutex::new(Vec::new())),
|
||||
fail_with_upstream_response,
|
||||
})
|
||||
}
|
||||
|
||||
fn captured(&self) -> Vec<CapturedSearchRequest> {
|
||||
self.captured.lock().expect("capture lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionService for SearchExecution {
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
(plaintext == "sk_search_test")
|
||||
.then(|| self.client.clone())
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"search test must use provider endpoint execution",
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let Operation::Search(search) = &request.operation else {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"search test received a non-search operation",
|
||||
));
|
||||
};
|
||||
let payload = search.payload();
|
||||
self.captured
|
||||
.lock()
|
||||
.expect("capture lock")
|
||||
.push(CapturedSearchRequest {
|
||||
provider: request.provider.as_str().to_owned(),
|
||||
endpoint: request.metadata.endpoint,
|
||||
transport: request.metadata.transport,
|
||||
body: payload.body().clone(),
|
||||
context: Value::Object(payload.context().clone()),
|
||||
});
|
||||
let session = if self.fail_with_upstream_response {
|
||||
BufferedJsonSession::failure(
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(StatusCode::UNPROCESSABLE_ENTITY.as_u16())
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
StatusCode::UNPROCESSABLE_ENTITY.as_u16(),
|
||||
Some(b"application/problem+json".to_vec()),
|
||||
Bytes::from_static(
|
||||
br#"{ "error":{"message":"future search validation"}, "future":9007199254740993 }"#,
|
||||
),
|
||||
)
|
||||
.with_headers(vec![ProviderResponseHeader::new(
|
||||
"x-future-search-error",
|
||||
Bytes::from_static(b"preserved"),
|
||||
)]),
|
||||
),
|
||||
)
|
||||
} else {
|
||||
BufferedJsonSession::success(
|
||||
Bytes::from_static(SEARCH_RESPONSE),
|
||||
StatusCode::CREATED.as_u16(),
|
||||
vec![ProviderResponseHeader::new(
|
||||
"x-search-rate-limit",
|
||||
Bytes::from_static(b"42"),
|
||||
)],
|
||||
Arc::clone(&self.committed_statuses),
|
||||
)
|
||||
};
|
||||
Ok(StartedExecution {
|
||||
request_id: ModelRequestId::new("req_search_test").expect("request ID"),
|
||||
created_at: SystemTime::now(),
|
||||
stream: false,
|
||||
session: Box::new(session),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_route_should_preserve_request_and_success_response_bytes() {
|
||||
let execution = SearchExecution::new(false);
|
||||
let request_body = br#"{ "id":"search-session", "model":"gpt-future", "commands":{"search_query":[{"q":"private query"}]}, "future":1, "future":2 }"#;
|
||||
let turn_metadata =
|
||||
r#"{"session_id":"session","thread_id":"thread","turn_id":"turn","future":true}"#;
|
||||
let response = api_router(execution.clone())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/alpha/search")
|
||||
.header(AUTHORIZATION, "Bearer sk_search_test")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-codex-turn-metadata", turn_metadata)
|
||||
.body(Body::from(request_body.to_vec()))
|
||||
.expect("search request"),
|
||||
)
|
||||
.await
|
||||
.expect("search response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
assert_eq!(
|
||||
response.headers().get("x-search-rate-limit"),
|
||||
Some(&"42".parse().expect("header value"))
|
||||
);
|
||||
let response_body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read search response");
|
||||
assert_eq!(response_body.as_ref(), SEARCH_RESPONSE);
|
||||
|
||||
let captured = execution.captured();
|
||||
assert_eq!(captured.len(), 1);
|
||||
assert_eq!(captured[0].provider, "openai");
|
||||
assert_eq!(captured[0].endpoint, "/v1/alpha/search");
|
||||
assert_eq!(captured[0].transport, ClientTransport::HttpJson);
|
||||
assert_eq!(captured[0].body.as_ref(), request_body);
|
||||
assert_eq!(captured[0].context, json!({"turn_metadata": turn_metadata}));
|
||||
assert_eq!(
|
||||
*execution
|
||||
.committed_statuses
|
||||
.lock()
|
||||
.expect("committed status lock"),
|
||||
vec![StatusCode::CREATED.as_u16()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_route_should_return_the_exact_upstream_error_response() {
|
||||
let response = api_router(SearchExecution::new(true))
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/alpha/search")
|
||||
.header(AUTHORIZATION, "Bearer sk_search_test")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
br#"{ "model":"gpt-future", "future_invalid":9007199254740993 }"#.to_vec(),
|
||||
))
|
||||
.expect("search request"),
|
||||
)
|
||||
.await
|
||||
.expect("search response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(
|
||||
response.headers().get("content-type"),
|
||||
Some(&"application/problem+json".parse().expect("content type"))
|
||||
);
|
||||
assert_eq!(
|
||||
response.headers().get("x-future-search-error"),
|
||||
Some(&"preserved".parse().expect("header value"))
|
||||
);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read search response");
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
br#"{ "error":{"message":"future search validation"}, "future":9007199254740993 }"#
|
||||
);
|
||||
}
|
||||
@@ -210,6 +210,8 @@ struct CurrentAttempt {
|
||||
index: NonZeroU32,
|
||||
started_at: SystemTime,
|
||||
send_observed: bool,
|
||||
/// Provider 已为当前 attempt 交付可供同请求恢复的不透明状态检查点。
|
||||
session_checkpoint_observed: bool,
|
||||
response_observation: Option<ProviderResponseObservation>,
|
||||
}
|
||||
|
||||
@@ -570,6 +572,9 @@ where
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
PollBoundary::Item(Some(Ok(mut event))) => {
|
||||
if let Some(state) = event.session_update() {
|
||||
self.observe_session_checkpoint(state);
|
||||
}
|
||||
if let Some(observation) = event.take_observation() {
|
||||
self.observe_response(observation);
|
||||
}
|
||||
@@ -931,6 +936,7 @@ where
|
||||
index: next_attempt,
|
||||
started_at: SystemTime::now(),
|
||||
send_observed: false,
|
||||
session_checkpoint_observed: false,
|
||||
response_observation: None,
|
||||
});
|
||||
Ok(None)
|
||||
@@ -972,6 +978,17 @@ where
|
||||
self.mark_send_observed().await;
|
||||
}
|
||||
|
||||
fn observe_session_checkpoint(&mut self, state: &ProviderSessionState) {
|
||||
let Some(current) = self.current.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if state.provider() != current.metadata.provider().as_str() {
|
||||
return;
|
||||
}
|
||||
current.session_checkpoint_observed = true;
|
||||
self.operation.set_provider_session_state(state.clone());
|
||||
}
|
||||
|
||||
async fn mark_send_observed(&mut self) {
|
||||
let Some(current) = self.current.as_mut() else {
|
||||
return;
|
||||
@@ -1071,6 +1088,11 @@ where
|
||||
.await;
|
||||
}
|
||||
let provider_proved_replay_safe = provider_proved_replay_safe(&error);
|
||||
// Ambiguous 仍是默认 fail-closed。唯一例外是 Provider 在同一 attempt 中先
|
||||
// 交付了不透明恢复检查点,并明确证明从该检查点重放安全;该例外只用于
|
||||
// 固定原账号的 transport recovery,不放开普通换号或 continuation 重放。
|
||||
let checkpointed_transport_replay =
|
||||
current.session_checkpoint_observed && error.replay_is_safe();
|
||||
let continuation_retry = self.prepare_continuation_retry(
|
||||
¤t,
|
||||
&error,
|
||||
@@ -1093,14 +1115,16 @@ where
|
||||
delay,
|
||||
}) if self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous =>
|
||||
&& (attempt_send_state != UpstreamSendState::Ambiguous
|
||||
|| checkpointed_transport_replay) =>
|
||||
{
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous =>
|
||||
&& (attempt_send_state != UpstreamSendState::Ambiguous
|
||||
|| checkpointed_transport_replay) =>
|
||||
{
|
||||
Some((AttemptTransport::Fallback, Duration::ZERO))
|
||||
}
|
||||
|
||||
@@ -623,13 +623,21 @@ impl ProviderError {
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
|
||||
}
|
||||
|
||||
/// 标记 Provider 已证明本次拒绝没有执行生成,可在下游提交前重放。
|
||||
/// 标记 Provider 已证明本次失败可在下游提交前安全重放。
|
||||
///
|
||||
/// 证明可以来自明确的“未执行”拒绝,也可以来自同 attempt 已交付、可恢复
|
||||
/// 同一业务请求的状态检查点;后者仍须由 Core 的同账号恢复门禁单独校验。
|
||||
#[must_use]
|
||||
pub const fn with_replay_safe(mut self) -> Self {
|
||||
self.replay_safe = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地标记 Provider 已证明本次失败可从已交付的状态检查点安全恢复。
|
||||
pub const fn set_replay_safe(&mut self) {
|
||||
self.replay_safe = true;
|
||||
}
|
||||
|
||||
/// 允许 Core 仅在客户端尚未收到任何事件时执行一次受预算约束的换号恢复。
|
||||
///
|
||||
/// 该标记不证明上游未执行,也不等价于 [`Self::with_replay_safe`];它只表达
|
||||
|
||||
@@ -763,7 +763,8 @@ impl ProviderEvent {
|
||||
}
|
||||
}
|
||||
|
||||
/// 把本事件标记为 Provider 连接内状态的提交边界。
|
||||
/// 附加 Provider 私有状态检查点;Core 可将其用于同请求恢复,协议连接可在
|
||||
/// terminal 事件上持有它供下一轮使用。
|
||||
pub fn attach_session_update(&mut self, state: ProviderSessionState) {
|
||||
self.session_update = Some(Box::new(state));
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ pub enum OperationKind {
|
||||
Generate,
|
||||
/// 图像生成。
|
||||
GenerateImage,
|
||||
/// Provider 原生 standalone search。
|
||||
Search,
|
||||
}
|
||||
|
||||
impl OperationKind {
|
||||
@@ -31,6 +33,7 @@ impl OperationKind {
|
||||
match self {
|
||||
Self::Generate => "generate",
|
||||
Self::GenerateImage => "generate_image",
|
||||
Self::Search => "search",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -501,6 +504,35 @@ impl fmt::Debug for ImageRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 原生 standalone search 请求。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct StandaloneSearchRequest {
|
||||
payload: RawJsonPayload,
|
||||
}
|
||||
|
||||
impl StandaloneSearchRequest {
|
||||
/// 创建携带原始协议 JSON 正文的 standalone search 请求。
|
||||
#[must_use]
|
||||
pub const fn from_raw_json(payload: RawJsonPayload) -> Self {
|
||||
Self { payload }
|
||||
}
|
||||
|
||||
/// 返回未经重编码的协议 JSON 正文。
|
||||
#[must_use]
|
||||
pub const fn payload(&self) -> &RawJsonPayload {
|
||||
&self.payload
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for StandaloneSearchRequest {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("StandaloneSearchRequest")
|
||||
.field("payload", &"<not included in Debug>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 网关内部业务请求;不包含任何客户端 wire 或 Provider SDK 类型。
|
||||
#[derive(Clone, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
@@ -509,6 +541,8 @@ pub enum Operation {
|
||||
Generate(GenerateRequest),
|
||||
/// 图像生成。
|
||||
GenerateImage(ImageRequest),
|
||||
/// Provider 原生 standalone search。
|
||||
Search(StandaloneSearchRequest),
|
||||
}
|
||||
|
||||
impl Operation {
|
||||
@@ -534,6 +568,7 @@ impl Operation {
|
||||
match self {
|
||||
Self::Generate(_) => OperationKind::Generate,
|
||||
Self::GenerateImage(_) => OperationKind::GenerateImage,
|
||||
Self::Search(_) => OperationKind::Search,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -543,6 +578,7 @@ impl Operation {
|
||||
match self {
|
||||
Self::Generate(request) => request.requirements(),
|
||||
Self::GenerateImage(_) => CapabilityRequirements::new(OperationKind::GenerateImage),
|
||||
Self::Search(_) => CapabilityRequirements::new(OperationKind::Search),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -552,6 +588,7 @@ impl Operation {
|
||||
match self {
|
||||
Self::Generate(request) => request.image_generation_requested(),
|
||||
Self::GenerateImage(_) => true,
|
||||
Self::Search(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -560,7 +597,7 @@ impl Operation {
|
||||
pub fn provider_session_state(&self, provider: &str) -> Option<&ProviderSessionState> {
|
||||
match self {
|
||||
Self::Generate(request) => request.provider_session_state(provider),
|
||||
Self::GenerateImage(_) => None,
|
||||
Self::GenerateImage(_) | Self::Search(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,6 +270,7 @@ enum Script {
|
||||
struct ScriptedProvider {
|
||||
scripts: Mutex<VecDeque<Script>>,
|
||||
contexts: Mutex<Vec<AttemptContext>>,
|
||||
operations: Mutex<Vec<Operation>>,
|
||||
}
|
||||
|
||||
impl ScriptedProvider {
|
||||
@@ -277,6 +278,7 @@ impl ScriptedProvider {
|
||||
Self {
|
||||
scripts: Mutex::new(scripts.into()),
|
||||
contexts: Mutex::new(Vec::new()),
|
||||
operations: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -302,6 +304,10 @@ impl Provider for ScriptedProvider {
|
||||
request: ProviderRequest,
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
self.operations
|
||||
.lock()
|
||||
.expect("operations lock")
|
||||
.push(request.operation().clone());
|
||||
self.contexts.lock().expect("contexts lock").push(context);
|
||||
let script = self
|
||||
.scripts
|
||||
@@ -2838,14 +2844,25 @@ fn ambiguous_send_state_stops_retry() {
|
||||
fn ambiguous_pre_delivery_retry_marker_does_not_rotate_account() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let checkpoint = ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([("checkpoint".to_owned(), json!("opaque-state"))]),
|
||||
)
|
||||
.expect("provider checkpoint");
|
||||
let mut checkpoint_event = ProviderEvent::observation(ProviderResponseObservation::new(
|
||||
UpstreamTransport::new("websocket").expect("transport"),
|
||||
));
|
||||
checkpoint_event.attach_session_update(checkpoint);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)
|
||||
.with_pre_delivery_retry())],
|
||||
items: vec![
|
||||
Ok(checkpoint_event),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_retry(),
|
||||
),
|
||||
],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
@@ -2874,6 +2891,70 @@ fn ambiguous_pre_delivery_retry_marker_does_not_rotate_account() {
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_transport_retry_uses_provider_session_checkpoint_on_same_account() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let checkpoint = ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([("checkpoint".to_owned(), json!("opaque-state"))]),
|
||||
)
|
||||
.expect("provider checkpoint");
|
||||
let mut checkpoint_event = ProviderEvent::observation(ProviderResponseObservation::new(
|
||||
UpstreamTransport::new("websocket").expect("transport"),
|
||||
));
|
||||
checkpoint_event.attach_session_update(checkpoint.clone());
|
||||
let retry_index = NonZeroU32::new(1).expect("retry index");
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
items: vec![
|
||||
Ok(checkpoint_event),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_replay_safe()
|
||||
.with_pre_delivery_transport_retry(retry_index, Duration::ZERO),
|
||||
),
|
||||
],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("checkpointed retry succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
|
||||
|
||||
let original = ProviderAccountId::new("acct_first").expect("account id");
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[1].transport(),
|
||||
AttemptTransport::Retry(retry_index)
|
||||
);
|
||||
assert_eq!(contexts[1].required_account(), Some(&original));
|
||||
drop(contexts);
|
||||
let operations = provider.operations.lock().expect("operations lock");
|
||||
assert_eq!(
|
||||
operations[1].provider_session_state("openai"),
|
||||
Some(&checkpoint)
|
||||
);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 2);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_delivery_transport_fallback_retries_the_same_account_with_fallback_transport() {
|
||||
let operation = generate_operation();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::operation::{
|
||||
Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind,
|
||||
ProtocolPayload, ProviderSessionState, RawJsonPayload,
|
||||
ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
@@ -121,10 +121,43 @@ fn operation_kind_should_remain_stable() {
|
||||
ImageRequestKind::Generation,
|
||||
json!({"model": "gpt-image-2", "prompt": "draw"}),
|
||||
));
|
||||
let search = Operation::Search(StandaloneSearchRequest::from_raw_json(
|
||||
RawJsonPayload::new(
|
||||
"openai",
|
||||
Bytes::from_static(br#"{ "model":"gpt-test", "commands":{} }"#),
|
||||
)
|
||||
.expect("OpenAI search payload"),
|
||||
));
|
||||
|
||||
assert_eq!(generate.kind(), OperationKind::Generate);
|
||||
assert_eq!(image.kind(), OperationKind::GenerateImage);
|
||||
assert_eq!(search.kind(), OperationKind::Search);
|
||||
assert_eq!(OperationKind::Search.as_str(), "search");
|
||||
assert!(image.image_generation_requested());
|
||||
assert!(!search.image_generation_requested());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standalone_search_should_keep_raw_body_and_context_opaque() {
|
||||
let body = Bytes::from_static(
|
||||
br#"{ "model":"gpt-future", "commands":{}, "future":9007199254740993, "future":2 }"#,
|
||||
);
|
||||
let secret = "private-turn-metadata";
|
||||
let payload = RawJsonPayload::new("openai", body.clone())
|
||||
.expect("OpenAI search payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"turn_metadata".to_owned(),
|
||||
Value::String(secret.to_owned()),
|
||||
)]));
|
||||
let request = StandaloneSearchRequest::from_raw_json(payload);
|
||||
|
||||
assert_eq!(request.payload().body(), &body);
|
||||
assert_eq!(
|
||||
request.payload().context().get("turn_metadata"),
|
||||
Some(&Value::String(secret.to_owned()))
|
||||
);
|
||||
assert!(!format!("{request:?}").contains(secret));
|
||||
assert!(!format!("{request:?}").contains("gpt-future"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -118,6 +118,9 @@ pub struct LoggingConfig {
|
||||
pub level: String,
|
||||
pub stdout: bool,
|
||||
pub file: FileLoggingConfig,
|
||||
/// 将完整请求原文写入独立诊断日志;默认关闭,因为内容包含凭据。
|
||||
#[serde(default)]
|
||||
pub request_dump: bool,
|
||||
}
|
||||
|
||||
impl LoggingConfig {
|
||||
|
||||
@@ -6,16 +6,19 @@ use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::Utc;
|
||||
use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
|
||||
use tracing_appender::non_blocking::{NonBlocking, NonBlockingBuilder, WorkerGuard};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _};
|
||||
|
||||
use crate::config::LoggingConfig;
|
||||
use crate::config::{FileLoggingConfig, LoggingConfig};
|
||||
|
||||
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
|
||||
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump";
|
||||
/// 请求转储含完整凭据与正文,只允许在显式开启时写入独立文件。
|
||||
const REQUEST_DUMP_LOG_TARGET: &str = "request_dump";
|
||||
|
||||
/// non-blocking 日志 writer 的进程级守卫。
|
||||
pub struct LogGuard {
|
||||
@@ -39,6 +42,7 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = application_file_filter(&directive)?;
|
||||
let recovery_file_filter = oauth_recovery_file_filter();
|
||||
let request_dump_file_filter = request_dump_file_filter();
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
@@ -52,37 +56,30 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
writer
|
||||
});
|
||||
let (file_writer, recovery_file_writer) = if config.file.enabled {
|
||||
let maximum_bytes = config
|
||||
.file
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let recovery_file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
let (file_writer, file_guard) =
|
||||
rotating_file_writer(&config.file, LOG_FILE_PREFIX, "gateway-log-file")?;
|
||||
let (recovery_file_writer, recovery_file_guard) = rotating_file_writer(
|
||||
&config.file,
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
"gateway-log-oauth-recovery-file",
|
||||
)?;
|
||||
let (file_writer, file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-file")
|
||||
.finish(file_writer);
|
||||
let (recovery_file_writer, recovery_file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-oauth-recovery-file")
|
||||
.finish(recovery_file_writer);
|
||||
guards.push(file_guard);
|
||||
guards.push(recovery_file_guard);
|
||||
(Some(file_writer), Some(recovery_file_writer))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
let request_dump_file_writer = if config.request_dump {
|
||||
let (writer, guard) = rotating_file_writer(
|
||||
&config.file,
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX,
|
||||
"gateway-log-request-dump-file",
|
||||
)?;
|
||||
guards.push(guard);
|
||||
Some(writer)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let stdout_layer = stdout_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
@@ -118,10 +115,24 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
.with_span_list(true)
|
||||
.with_filter(recovery_file_filter)
|
||||
});
|
||||
let request_dump_file_layer = request_dump_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(request_dump_file_filter)
|
||||
});
|
||||
tracing_subscriber::registry()
|
||||
.with(stdout_layer)
|
||||
.with(file_layer)
|
||||
.with(recovery_file_layer)
|
||||
.with(request_dump_file_layer)
|
||||
.try_init()
|
||||
.map_err(|_| LogError::AlreadyInitialized)?;
|
||||
Ok(LogGuard { _writers: guards })
|
||||
@@ -131,8 +142,12 @@ fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log directive is valid");
|
||||
let request_dump_directive = format!("{REQUEST_DUMP_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static request dump log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map(|filter| filter.add_directive(request_dump_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
@@ -142,14 +157,41 @@ fn oauth_recovery_file_filter() -> EnvFilter {
|
||||
.expect("static OAuth recovery log filter is valid")
|
||||
}
|
||||
|
||||
fn request_dump_file_filter() -> EnvFilter {
|
||||
format!("off,{REQUEST_DUMP_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static request dump log filter is valid")
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
directive.to_owned()
|
||||
format!("{directive},{REQUEST_DUMP_LOG_TARGET}=off")
|
||||
} else {
|
||||
"off,gateway_startup=info".to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
fn rotating_file_writer(
|
||||
config: &FileLoggingConfig,
|
||||
file_prefix: &'static str,
|
||||
thread_name: &'static str,
|
||||
) -> Result<(NonBlocking, WorkerGuard), LogError> {
|
||||
let maximum_bytes = config
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let writer = RotatingLogWriter::open(
|
||||
config.directory.clone(),
|
||||
file_prefix,
|
||||
maximum_bytes,
|
||||
config.retention_days,
|
||||
config.max_files,
|
||||
)?;
|
||||
Ok(NonBlockingBuilder::default()
|
||||
.thread_name(thread_name)
|
||||
.finish(writer))
|
||||
}
|
||||
|
||||
struct RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
|
||||
@@ -103,6 +103,7 @@ fn valid_config() -> HostConfig {
|
||||
max_file_size_mb: 100,
|
||||
max_files: 30,
|
||||
},
|
||||
request_dump: false,
|
||||
},
|
||||
system_update,
|
||||
drain_timeout_seconds: 30,
|
||||
|
||||
@@ -8,13 +8,18 @@ use gateway_host::system_update::SystemUpdateConfig;
|
||||
|
||||
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
|
||||
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
|
||||
const REQUEST_DUMP_ENABLED_ENV: &str = "CPR_LOGGING_TEST_REQUEST_DUMP_ENABLED";
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
|
||||
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump.";
|
||||
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
|
||||
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
const REQUEST_DUMP_LOG_TARGET: &str = "request_dump";
|
||||
const REQUEST_DUMP_LOG_MARKER: &str = "request-dump-file-filter-test";
|
||||
const REQUEST_DUMP_SECRET: &str = "unredacted-authorization-value";
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_sink() {
|
||||
@@ -34,6 +39,7 @@ fn logging_requires_at_least_one_sink() {
|
||||
max_file_size_mb: 100,
|
||||
max_files: 30,
|
||||
},
|
||||
request_dump: false,
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
@@ -48,11 +54,12 @@ fn logging_requires_at_least_one_sink() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
fn sensitive_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
write_oauth_recovery_log(PathBuf::from(
|
||||
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
|
||||
));
|
||||
write_sensitive_logs(
|
||||
PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).expect("child log directory")),
|
||||
env::var(REQUEST_DUMP_ENABLED_ENV).is_ok_and(|value| value == "true"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -60,14 +67,15 @@ fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
let output = Command::new(env::current_exe().expect("current test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::oauth_recovery_file_logging_is_separate_and_overrides_global_log_level",
|
||||
"logging::sensitive_file_logging_is_separate_and_overrides_global_log_level",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env(REQUEST_DUMP_ENABLED_ENV, "true")
|
||||
.env(
|
||||
"RUST_LOG",
|
||||
"off,logging_test_application=info,oauth_recovery=off",
|
||||
"off,logging_test_application=info,oauth_recovery=off,request_dump=trace",
|
||||
)
|
||||
.output()
|
||||
.expect("run logging child process");
|
||||
@@ -84,6 +92,8 @@ fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
assert!(application_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(!application_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(!application_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
assert!(!application_log.contains(&json_target_field(REQUEST_DUMP_LOG_TARGET)));
|
||||
assert!(!application_log.contains(REQUEST_DUMP_SECRET));
|
||||
|
||||
let recovery_log = read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX);
|
||||
assert!(recovery_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
@@ -91,9 +101,49 @@ fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
assert!(!recovery_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(!recovery_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(recovery_log.contains(&format!(r#""provider":"{OAUTH_RECOVERY_PROVIDER}""#)));
|
||||
assert!(!recovery_log.contains(&json_target_field(REQUEST_DUMP_LOG_TARGET)));
|
||||
assert!(!recovery_log.contains(REQUEST_DUMP_SECRET));
|
||||
|
||||
let request_dump_log = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
assert!(request_dump_log.contains(&json_target_field(REQUEST_DUMP_LOG_TARGET)));
|
||||
assert!(request_dump_log.contains(REQUEST_DUMP_LOG_MARKER));
|
||||
assert!(request_dump_log.contains(REQUEST_DUMP_SECRET));
|
||||
assert!(!request_dump_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(!request_dump_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(!request_dump_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(!request_dump_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
|
||||
let disabled_directory = tempfile::tempdir().expect("create disabled log directory");
|
||||
let output = Command::new(env::current_exe().expect("current test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::sensitive_file_logging_is_separate_and_overrides_global_log_level",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, disabled_directory.path())
|
||||
.env(REQUEST_DUMP_ENABLED_ENV, "false")
|
||||
.env(
|
||||
"RUST_LOG",
|
||||
"off,logging_test_application=info,oauth_recovery=off,request_dump=trace",
|
||||
)
|
||||
.output()
|
||||
.expect("run logging child process with request dump disabled");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"logging child process failed:\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr),
|
||||
);
|
||||
assert!(!log_file_set_exists(
|
||||
disabled_directory.path(),
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX
|
||||
));
|
||||
let application_log = read_log_file_set(disabled_directory.path(), APPLICATION_LOG_FILE_PREFIX);
|
||||
assert!(!application_log.contains(REQUEST_DUMP_SECRET));
|
||||
}
|
||||
|
||||
fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
fn write_sensitive_logs(directory: PathBuf, request_dump: bool) {
|
||||
let config = HostConfig {
|
||||
listen: ListenConfig {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
@@ -110,6 +160,7 @@ fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
max_file_size_mb: 1,
|
||||
max_files: 1,
|
||||
},
|
||||
request_dump,
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
@@ -134,6 +185,12 @@ fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OAuth recovery test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: REQUEST_DUMP_LOG_TARGET,
|
||||
marker = REQUEST_DUMP_LOG_MARKER,
|
||||
authorization = REQUEST_DUMP_SECRET,
|
||||
"request dump test record"
|
||||
);
|
||||
drop(bundle);
|
||||
}
|
||||
|
||||
@@ -156,6 +213,18 @@ fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn log_file_set_exists(directory: &Path, file_prefix: &str) -> bool {
|
||||
fs::read_dir(directory)
|
||||
.expect("read log directory")
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with(file_prefix))
|
||||
})
|
||||
}
|
||||
|
||||
fn json_target_field(target: &str) -> String {
|
||||
format!(r#""target":"{target}""#)
|
||||
}
|
||||
|
||||
@@ -728,7 +728,8 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
upstream_connection_idle_ms = $47
|
||||
where id = $1 and outcome = 'running'
|
||||
returning id, client_api_key_ref, continuation_affinity_hash,
|
||||
continuation_requested, outcome, started_at, completed_at
|
||||
continuation_requested, provider_kind, upstream_transport,
|
||||
outcome, started_at, completed_at
|
||||
), recovery_target as (
|
||||
select prior.id
|
||||
from model_requests prior
|
||||
@@ -775,9 +776,47 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
where prior.id = target.id
|
||||
and prior.recovered_at is null
|
||||
returning prior.id
|
||||
), session_transport_recovery_target as (
|
||||
select prior.id
|
||||
from model_requests prior
|
||||
cross join finalized current
|
||||
where current.outcome = 'succeeded'
|
||||
and current.provider_kind = 'openai'
|
||||
and current.upstream_transport = 'http_sse'
|
||||
and current.continuation_affinity_hash is not null
|
||||
and prior.client_api_key_ref = current.client_api_key_ref
|
||||
and prior.continuation_affinity_hash = current.continuation_affinity_hash
|
||||
and prior.provider_kind = current.provider_kind
|
||||
and prior.outcome = 'failed'
|
||||
and prior.error_kind = 'upstream_unavailable'
|
||||
and prior.upstream_transport = 'websocket'
|
||||
and prior.upstream_send_state = 'ambiguous'
|
||||
and prior.downstream_committed_at is null
|
||||
and prior.recovered_at is null
|
||||
and current.started_at >= prior.completed_at
|
||||
and current.started_at <= prior.completed_at + interval '30 seconds'
|
||||
), session_transport_recovery_update as (
|
||||
update model_requests prior
|
||||
set recovery_attempt_count = prior.recovery_attempt_count + 1,
|
||||
recovery_request_id = current.id,
|
||||
recovered_at = current.completed_at,
|
||||
recovery_retry_delay_ms = greatest(
|
||||
0,
|
||||
floor(extract(epoch from (current.started_at - prior.completed_at)) * 1000)
|
||||
)::bigint,
|
||||
recovery_total_latency_ms = greatest(
|
||||
0,
|
||||
floor(extract(epoch from (current.completed_at - prior.completed_at)) * 1000)
|
||||
)::bigint
|
||||
from finalized current
|
||||
join session_transport_recovery_target target on true
|
||||
where prior.id = target.id
|
||||
and prior.recovered_at is null
|
||||
returning prior.id
|
||||
)
|
||||
select (select count(*) from finalized)::bigint
|
||||
+ (select count(*) * 0 from recovery_update)::bigint",
|
||||
+ (select count(*) * 0 from recovery_update)::bigint
|
||||
+ (select count(*) * 0 from session_transport_recovery_update)::bigint",
|
||||
)
|
||||
.bind(&finalization.model_request_id)
|
||||
.bind(finalization.outcome.as_str())
|
||||
|
||||
@@ -128,7 +128,6 @@ fn push_request_error_predicates(
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
statement.push(" and mr.recovered_at is null");
|
||||
push_range(statement, "mr.completed_at", range);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
@@ -171,7 +170,6 @@ fn push_ops_event_predicates(
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
statement.push(" and (mr.id is null or mr.recovered_at is null)");
|
||||
push_range(statement, "oe.created_at", range);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
|
||||
@@ -88,7 +88,19 @@ pub(crate) fn push_usage_filter(
|
||||
query.push(format!(" escape '\\' or {alias}.{column} like "));
|
||||
query.push_bind(pattern.clone());
|
||||
}
|
||||
query.push(" escape '\\')");
|
||||
if value.starts_with("sk_") {
|
||||
query.push(format!(
|
||||
" escape '\\' or exists (
|
||||
select 1 from client_api_keys searched_client_key
|
||||
where searched_client_key.id = {alias}.client_api_key_ref
|
||||
and searched_client_key.key like "
|
||||
));
|
||||
query.push_bind(pattern);
|
||||
query.push(" escape '\\')");
|
||||
} else {
|
||||
query.push(" escape '\\'");
|
||||
}
|
||||
query.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -719,6 +719,165 @@ async fn successful_new_chain_should_mark_recent_continuation_failure_recovered(
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_http_downgrade_should_mark_pending_websocket_failures_recovered() {
|
||||
let Some(database) = TestDatabase::create("execution_transport_downgrade_recovered").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let affinity_hash = "d".repeat(64);
|
||||
let first_failure_at = Utc::now() - Duration::seconds(8);
|
||||
let second_failure_at = first_failure_at + Duration::seconds(1);
|
||||
seed_websocket_transport_failure(
|
||||
&database.pool,
|
||||
"req_websocket_failed_first",
|
||||
&affinity_hash,
|
||||
first_failure_at,
|
||||
)
|
||||
.await;
|
||||
seed_websocket_transport_failure(
|
||||
&database.pool,
|
||||
"req_websocket_failed_second",
|
||||
&affinity_hash,
|
||||
second_failure_at,
|
||||
)
|
||||
.await;
|
||||
seed_websocket_transport_failure(
|
||||
&database.pool,
|
||||
"req_websocket_failed_after_delivery",
|
||||
&affinity_hash,
|
||||
second_failure_at,
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set downstream_committed_at = completed_at
|
||||
where id = 'req_websocket_failed_after_delivery'",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("mark control failure delivered downstream");
|
||||
|
||||
let recovery_started_at = first_failure_at + Duration::seconds(4);
|
||||
seed_transport_recovery_request(
|
||||
&database.pool,
|
||||
"req_http_fallback_succeeded",
|
||||
&affinity_hash,
|
||||
recovery_started_at,
|
||||
)
|
||||
.await;
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
let mut recovery = successful_core_finalization("req_http_fallback_succeeded");
|
||||
recovery.upstream_transport = Some("http_sse".to_owned());
|
||||
recovery.downstream_committed_at = Some(std::time::SystemTime::now());
|
||||
ExecutionStore::finalize_model_request(&store, recovery)
|
||||
.await
|
||||
.expect("finalize successful HTTP fallback");
|
||||
|
||||
type PersistedRecovery = (
|
||||
String,
|
||||
Option<String>,
|
||||
Option<chrono::DateTime<Utc>>,
|
||||
i32,
|
||||
Option<i64>,
|
||||
Option<i64>,
|
||||
);
|
||||
let recovered: Vec<PersistedRecovery> = sqlx::query_as(
|
||||
"select id, recovery_request_id, recovered_at, recovery_attempt_count,
|
||||
recovery_retry_delay_ms, recovery_total_latency_ms
|
||||
from model_requests
|
||||
where id like 'req_websocket_failed_%'
|
||||
order by id",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.expect("load WebSocket recovery observations");
|
||||
|
||||
assert_eq!(recovered.len(), 3);
|
||||
let delivered = &recovered[0];
|
||||
assert_eq!(delivered.0, "req_websocket_failed_after_delivery");
|
||||
assert_eq!((&delivered.1, delivered.2, delivered.3), (&None, None, 0));
|
||||
for (row, expected_delay_ms) in recovered[1..].iter().zip([4_000, 3_000]) {
|
||||
assert_eq!(row.1.as_deref(), Some("req_http_fallback_succeeded"));
|
||||
assert!(row.2.is_some());
|
||||
assert_eq!(row.3, 1);
|
||||
assert_eq!(row.4, Some(expected_delay_ms));
|
||||
assert!(row.5.is_some_and(|latency| latency >= expected_delay_ms));
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_success_or_late_http_success_should_not_hide_transport_failure() {
|
||||
let Some(database) = TestDatabase::create("execution_transport_downgrade_unresolved").await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let failure_at = Utc::now() - Duration::seconds(60);
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
|
||||
let websocket_affinity = "e".repeat(64);
|
||||
seed_websocket_transport_failure(
|
||||
&database.pool,
|
||||
"req_websocket_control_failure",
|
||||
&websocket_affinity,
|
||||
failure_at,
|
||||
)
|
||||
.await;
|
||||
seed_transport_recovery_request(
|
||||
&database.pool,
|
||||
"req_websocket_control_success",
|
||||
&websocket_affinity,
|
||||
failure_at + Duration::seconds(4),
|
||||
)
|
||||
.await;
|
||||
ExecutionStore::finalize_model_request(
|
||||
&store,
|
||||
successful_core_finalization("req_websocket_control_success"),
|
||||
)
|
||||
.await
|
||||
.expect("finalize successful WebSocket control request");
|
||||
|
||||
let late_affinity = "f".repeat(64);
|
||||
seed_websocket_transport_failure(
|
||||
&database.pool,
|
||||
"req_websocket_late_failure",
|
||||
&late_affinity,
|
||||
failure_at,
|
||||
)
|
||||
.await;
|
||||
seed_transport_recovery_request(
|
||||
&database.pool,
|
||||
"req_http_late_success",
|
||||
&late_affinity,
|
||||
failure_at + Duration::seconds(31),
|
||||
)
|
||||
.await;
|
||||
let mut late_recovery = successful_core_finalization("req_http_late_success");
|
||||
late_recovery.upstream_transport = Some("http_sse".to_owned());
|
||||
ExecutionStore::finalize_model_request(&store, late_recovery)
|
||||
.await
|
||||
.expect("finalize late HTTP request");
|
||||
|
||||
type UnresolvedTransportFailure = (String, Option<String>, Option<chrono::DateTime<Utc>>, i32);
|
||||
let unresolved: Vec<UnresolvedTransportFailure> = sqlx::query_as(
|
||||
"select id, recovery_request_id, recovered_at, recovery_attempt_count
|
||||
from model_requests
|
||||
where id in ('req_websocket_control_failure', 'req_websocket_late_failure')
|
||||
order by id",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.expect("load unresolved transport failures");
|
||||
assert_eq!(unresolved.len(), 2);
|
||||
assert!(
|
||||
unresolved
|
||||
.iter()
|
||||
.all(|row| row.1.is_none() && row.2.is_none() && row.3 == 0)
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() {
|
||||
let Some(database) = TestDatabase::create("execution_continuation_unresolved").await else {
|
||||
@@ -842,6 +1001,61 @@ async fn seed_recovery_request(
|
||||
.expect("prepare seeded recovery request");
|
||||
}
|
||||
|
||||
async fn seed_websocket_transport_failure(
|
||||
pool: &sqlx::PgPool,
|
||||
id: &str,
|
||||
affinity_hash: &str,
|
||||
completed_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
seed_running_request(pool, id)
|
||||
.await
|
||||
.expect("seed WebSocket transport failure request");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set client_api_key_ref = 'key_transport_recovery',
|
||||
client_transport = 'http_sse',
|
||||
continuation_affinity_hash = $2,
|
||||
attempt_count = 1, upstream_send_state = 'ambiguous',
|
||||
upstream_transport = 'websocket', outcome = 'failed',
|
||||
error_kind = 'upstream_unavailable',
|
||||
provider_error_code = 'websocket_close_1000',
|
||||
started_at = $3 - interval '1 second', deadline_at = $3,
|
||||
completed_at = $3
|
||||
where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(affinity_hash)
|
||||
.bind(completed_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("finalize seeded WebSocket transport failure");
|
||||
}
|
||||
|
||||
async fn seed_transport_recovery_request(
|
||||
pool: &sqlx::PgPool,
|
||||
id: &str,
|
||||
affinity_hash: &str,
|
||||
started_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
seed_running_request(pool, id)
|
||||
.await
|
||||
.expect("seed transport recovery request");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set client_api_key_ref = 'key_transport_recovery',
|
||||
client_transport = 'http_sse',
|
||||
continuation_affinity_hash = $2,
|
||||
started_at = $3, deadline_at = $3 + interval '1 minute'
|
||||
where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(affinity_hash)
|
||||
.bind(started_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("prepare seeded transport recovery request");
|
||||
}
|
||||
|
||||
async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
|
||||
@@ -186,6 +186,48 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_search_should_match_client_api_key_prefix() {
|
||||
let Some(database) = TestDatabase::create("usage_client_api_key_search").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now)
|
||||
.await
|
||||
.expect("seed observability facts");
|
||||
let plaintext_key = format!("sk_{}", "K".repeat(43));
|
||||
sqlx::query(
|
||||
"insert into client_api_keys (id, name, key, enabled, created_at, updated_at)
|
||||
values ('key_observe', 'usage search', $1, true, $2, $2)",
|
||||
)
|
||||
.bind(&plaintext_key)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed searchable client API key");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
|
||||
for search in [plaintext_key.as_str(), &plaintext_key[..10]] {
|
||||
let page = observability_repository(&database.pool)
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
search: Some(search.to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("search usage by client API key");
|
||||
|
||||
assert_eq!(page.total, 1);
|
||||
assert_eq!(page.items[0].id, "req_observe_success");
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_search_should_treat_sql_wildcards_as_literals() {
|
||||
let Some(database) = TestDatabase::create("ops_literal_wildcard_search").await else {
|
||||
@@ -216,7 +258,8 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_continuation_failure_should_be_hidden_from_default_business_metrics() {
|
||||
async fn recovered_continuation_failure_should_be_visible_in_ops_but_hidden_from_business_metrics()
|
||||
{
|
||||
let Some(database) = TestDatabase::create("observability_recovered_continuation").await else {
|
||||
return;
|
||||
};
|
||||
@@ -276,8 +319,13 @@ async fn recovered_continuation_failure_should_be_hidden_from_default_business_m
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("ops errors without recovered intermediates");
|
||||
assert_eq!(errors.total, 0);
|
||||
.expect("ops errors retain recovered failures");
|
||||
assert_eq!(errors.total, 2);
|
||||
assert!(errors.items.iter().all(|error| {
|
||||
error.recovery_request_id.as_deref() == Some("req_observe_success")
|
||||
&& error.recovered_at.is_some()
|
||||
&& error.recovery_attempt_count == 1
|
||||
}));
|
||||
|
||||
let retained: (String, Option<String>) = sqlx::query_as(
|
||||
"select outcome, recovery_request_id
|
||||
|
||||
@@ -16,8 +16,8 @@ pub mod token_client;
|
||||
mod types;
|
||||
|
||||
pub(crate) use affinity::{
|
||||
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
|
||||
derive_codex_session_affinity, derive_previous_response_id_hash,
|
||||
CodexSessionAffinity, derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
derive_previous_response_id_hash,
|
||||
};
|
||||
pub(crate) use oauth::oauth_owner_ref;
|
||||
pub(crate) use types::parse_access_token_expiration;
|
||||
|
||||
@@ -22,11 +22,66 @@ impl CodexProvider {
|
||||
),
|
||||
ImageRequestKind::Edit => (CODEX_IMAGE_EDITS_PATH, self.image_edits_url.clone()),
|
||||
};
|
||||
let image_turn_id = image
|
||||
.payload()
|
||||
.context()
|
||||
.get("image_turn_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
self.execute_raw_json_endpoint(
|
||||
context,
|
||||
RawJsonEndpointRequest {
|
||||
response_origin,
|
||||
endpoint_path,
|
||||
body: image.payload().body().clone(),
|
||||
image_turn_id,
|
||||
turn_metadata: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn execute_search(
|
||||
&self,
|
||||
search: &StandaloneSearchRequest,
|
||||
candidate: &ProviderCandidate,
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
if search.payload().protocol() != PROVIDER_NAME || candidate.upstream_model().is_some() {
|
||||
return Err(provider_error(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
}
|
||||
let turn_metadata = search
|
||||
.payload()
|
||||
.context()
|
||||
.get("turn_metadata")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
self.execute_raw_json_endpoint(
|
||||
context,
|
||||
RawJsonEndpointRequest {
|
||||
response_origin: self.search_url.clone(),
|
||||
endpoint_path: CODEX_ALPHA_SEARCH_PATH,
|
||||
body: search.payload().body().clone(),
|
||||
image_turn_id: None,
|
||||
turn_metadata,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn execute_raw_json_endpoint(
|
||||
&self,
|
||||
context: AttemptContext,
|
||||
request: RawJsonEndpointRequest,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
let selection_started_at = Instant::now();
|
||||
let lease = self
|
||||
.selector
|
||||
.select_for_provider_endpoint(&SelectCodexProviderEndpointCredential {
|
||||
request_url: &response_origin,
|
||||
request_url: &request.response_origin,
|
||||
attempt: &context,
|
||||
})
|
||||
.await
|
||||
@@ -48,18 +103,18 @@ impl CodexProvider {
|
||||
account_selection_wait_ms,
|
||||
lease.capacity_snapshot(),
|
||||
));
|
||||
let image_turn_id = image
|
||||
.payload()
|
||||
.context()
|
||||
.get("image_turn_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
// Standalone Provider 端点没有可证明的账号 owner;Search metadata 必须按
|
||||
// 跨账号输入收敛到当前 lease,不能沿用下游声明的账号或 installation identity。
|
||||
let turn_metadata = request.turn_metadata.as_deref().and_then(|metadata| {
|
||||
crate::transport::request::scope_turn_metadata(metadata, lease.installation_id(), true)
|
||||
});
|
||||
let events = cold_json_response_stream(ColdJsonResponse {
|
||||
client: self.client.clone(),
|
||||
response_origin,
|
||||
endpoint_path,
|
||||
body: image.payload().body().clone(),
|
||||
image_turn_id,
|
||||
response_origin: request.response_origin,
|
||||
endpoint_path: request.endpoint_path,
|
||||
body: request.body,
|
||||
image_turn_id: request.image_turn_id,
|
||||
turn_metadata,
|
||||
context,
|
||||
selector: Arc::clone(&self.selector),
|
||||
quota: Arc::clone(&self.quota),
|
||||
@@ -78,12 +133,21 @@ impl CodexProvider {
|
||||
}
|
||||
}
|
||||
|
||||
struct RawJsonEndpointRequest {
|
||||
response_origin: Url,
|
||||
endpoint_path: &'static str,
|
||||
body: Bytes,
|
||||
image_turn_id: Option<String>,
|
||||
turn_metadata: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) struct ColdResponse {
|
||||
pub(super) client: CodexBackendClient,
|
||||
pub(super) response_origin: Url,
|
||||
pub(super) request: CodexResponsesRequest,
|
||||
pub(super) upstream_model: UpstreamModelId,
|
||||
pub(super) transport_policy: CodexProviderTransport,
|
||||
pub(super) websocket_connection_preference: WebSocketConnectionPreference,
|
||||
pub(super) context: AttemptContext,
|
||||
pub(super) selector: Arc<CodexCredentialSelector>,
|
||||
pub(super) quota: Arc<CodexCredentialQuotaService>,
|
||||
@@ -92,7 +156,8 @@ pub(super) struct ColdResponse {
|
||||
pub(super) output_started_at: Instant,
|
||||
pub(super) session_affinity_key: Option<ProviderSessionAffinityKey>,
|
||||
pub(super) session_affinity_key_hash: Option<String>,
|
||||
pub(super) session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
pub(super) session_transport_recovery: CodexSessionTransportRecovery,
|
||||
pub(super) session_recovery_probe: Option<CodexSessionWebSocketProbe>,
|
||||
pub(super) websocket_retry_count: u32,
|
||||
pub(super) stream_max_retries: u32,
|
||||
pub(super) session_capture: Option<OpenAiSessionCapture>,
|
||||
@@ -104,6 +169,7 @@ pub(super) struct ColdJsonResponse {
|
||||
pub(super) endpoint_path: &'static str,
|
||||
pub(super) body: Bytes,
|
||||
pub(super) image_turn_id: Option<String>,
|
||||
pub(super) turn_metadata: Option<String>,
|
||||
pub(super) context: AttemptContext,
|
||||
pub(super) selector: Arc<CodexCredentialSelector>,
|
||||
pub(super) quota: Arc<CodexCredentialQuotaService>,
|
||||
@@ -120,6 +186,13 @@ pub(super) struct OpenAiSessionState {
|
||||
#[serde(default)]
|
||||
pub(super) client_turn_id: Option<String>,
|
||||
pub(super) continuation_scope: OpenAiContinuationScope,
|
||||
/// 该状态来自同一业务请求的 metadata 检查点,只允许被紧随其后的恢复 attempt 消费。
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub(super) retry_checkpoint: bool,
|
||||
}
|
||||
|
||||
const fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -166,6 +239,26 @@ pub(super) fn encode_openai_session_state(
|
||||
.map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::Sent))
|
||||
}
|
||||
|
||||
fn encode_openai_session_capture(
|
||||
capture: &OpenAiSessionCapture,
|
||||
retry_checkpoint: bool,
|
||||
) -> Result<ProviderSessionState, ProviderError> {
|
||||
let Some(continuation_scope) = capture.continuation_scope else {
|
||||
return Err(provider_error(
|
||||
ProviderErrorKind::Protocol,
|
||||
UpstreamSendState::Sent,
|
||||
));
|
||||
};
|
||||
encode_openai_session_state(OpenAiSessionState {
|
||||
account_id: capture.account_id.clone(),
|
||||
conversation_id: capture.conversation_id.clone(),
|
||||
turn_state: capture.turn_state.clone(),
|
||||
client_turn_id: capture.client_turn_id.clone(),
|
||||
continuation_scope,
|
||||
retry_checkpoint,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn attach_openai_session_update(
|
||||
events: &mut [ProviderEvent],
|
||||
capture: &mut Option<OpenAiSessionCapture>,
|
||||
@@ -179,16 +272,7 @@ pub(super) fn attach_openai_session_update(
|
||||
let Some(capture) = capture.take() else {
|
||||
return;
|
||||
};
|
||||
let Some(continuation_scope) = capture.continuation_scope else {
|
||||
return;
|
||||
};
|
||||
let Ok(update) = encode_openai_session_state(OpenAiSessionState {
|
||||
account_id: capture.account_id,
|
||||
conversation_id: capture.conversation_id,
|
||||
turn_state: capture.turn_state,
|
||||
client_turn_id: capture.client_turn_id,
|
||||
continuation_scope,
|
||||
}) else {
|
||||
let Ok(update) = encode_openai_session_capture(&capture, false) else {
|
||||
return;
|
||||
};
|
||||
events[terminal_index].attach_session_update(update);
|
||||
@@ -228,6 +312,7 @@ pub(super) async fn create_response_attempt(
|
||||
account_id: &str,
|
||||
deadline: SystemTime,
|
||||
cancellation: &CancellationToken,
|
||||
connection_preference: WebSocketConnectionPreference,
|
||||
) -> Result<CodexBackendStreamingResponse, CodexHandshakeAttemptError> {
|
||||
let Some(handshake_deadline) = remaining(deadline) else {
|
||||
return Err(CodexHandshakeAttemptError::Timeout);
|
||||
@@ -240,6 +325,7 @@ pub(super) async fn create_response_attempt(
|
||||
request,
|
||||
request_context,
|
||||
Some(account_id),
|
||||
connection_preference,
|
||||
) => response.map_err(CodexHandshakeAttemptError::Client),
|
||||
}
|
||||
}
|
||||
@@ -279,6 +365,7 @@ pub(super) async fn create_json_attempt(
|
||||
Some(installation_id),
|
||||
);
|
||||
request_context.cookie_header = cookie_header.map(ExposeSecret::expose_secret);
|
||||
request_context.turn_metadata = request.turn_metadata.as_deref();
|
||||
request_context.account_selection = account_selection;
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -382,7 +469,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
|
||||
tracing::warn!(
|
||||
account_id = %active_account.id(),
|
||||
error = %error,
|
||||
"Failed to persist OpenAI image response cookies"
|
||||
"Failed to persist OpenAI provider endpoint response cookies"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -407,6 +494,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
request,
|
||||
upstream_model,
|
||||
transport_policy,
|
||||
websocket_connection_preference,
|
||||
context,
|
||||
selector,
|
||||
quota,
|
||||
@@ -415,7 +503,8 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_affinity_key_hash,
|
||||
session_transport_fallbacks,
|
||||
session_transport_recovery,
|
||||
mut session_recovery_probe,
|
||||
websocket_retry_count,
|
||||
stream_max_retries,
|
||||
mut session_capture,
|
||||
@@ -465,6 +554,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
active_account.id().as_str(),
|
||||
context.deadline(),
|
||||
&cancellation,
|
||||
websocket_connection_preference,
|
||||
)
|
||||
.await;
|
||||
let websocket_failure_policy = match &response {
|
||||
@@ -491,6 +581,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
WebSocketRecoveryContext {
|
||||
policy,
|
||||
requirement: request_transport_requirement,
|
||||
replay_checkpoint_available: false,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
request_id: context.request_id().as_str(),
|
||||
@@ -498,7 +589,8 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
account_id: active_account.id().as_str(),
|
||||
session_affinity_key: session_affinity_key.as_ref(),
|
||||
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
|
||||
session_transport_fallbacks: &session_transport_fallbacks,
|
||||
session_transport_recovery: &session_transport_recovery,
|
||||
session_recovery_probe: &mut session_recovery_probe,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -521,6 +613,9 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
Err(failure.error)?;
|
||||
return;
|
||||
}
|
||||
let mut turn_state_checkpoint_observed = response.transport
|
||||
== CodexBackendTransport::WebSocket
|
||||
&& response.turn_state.is_some();
|
||||
if let Some(capture) = session_capture.as_mut() {
|
||||
capture.continuation_scope = Some(if capture.response_store {
|
||||
OpenAiContinuationScope::Persisted
|
||||
@@ -634,15 +729,62 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
continue;
|
||||
}
|
||||
Err(mut failure) => {
|
||||
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
|
||||
let rate_limits_changed = if updates.is_empty() {
|
||||
false
|
||||
} else {
|
||||
passive_quota_observation.observe(&updates);
|
||||
let update_headers = rate_limit_update_headers(&updates);
|
||||
observation_state.merge_rate_limit_headers(&update_headers)
|
||||
};
|
||||
let turn_state_merge = merge_turn_state_update(
|
||||
turn_state_updates.as_ref(),
|
||||
&mut session_capture,
|
||||
&mut observation_state,
|
||||
)
|
||||
.await;
|
||||
turn_state_checkpoint_observed |= turn_state_merge.is_some();
|
||||
let retry_checkpoint_update = if turn_state_checkpoint_observed
|
||||
&& failure.websocket_checkpoint_retryable
|
||||
{
|
||||
session_capture
|
||||
.as_ref()
|
||||
.and_then(|capture| encode_openai_session_capture(capture, true).ok())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut observation_event = if rate_limits_changed
|
||||
|| turn_state_merge.is_some()
|
||||
|| retry_checkpoint_update.is_some()
|
||||
{
|
||||
observation_state
|
||||
.observation()
|
||||
.map(ProviderEvent::observation)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let (Some(event), Some(update)) =
|
||||
(observation_event.as_mut(), retry_checkpoint_update)
|
||||
{
|
||||
event.attach_session_update(update);
|
||||
}
|
||||
let replay_checkpoint_available = observation_event
|
||||
.as_ref()
|
||||
.and_then(ProviderEvent::session_update)
|
||||
.is_some();
|
||||
if failure.websocket_transport_retryable
|
||||
&& response_transport == CodexBackendTransport::WebSocket
|
||||
&& !pre_commit_events.is_committed()
|
||||
&& (matches!(
|
||||
failure.error.send_state(),
|
||||
UpstreamSendState::Sent | UpstreamSendState::Ambiguous
|
||||
) || !pre_commit_events.is_committed())
|
||||
{
|
||||
apply_websocket_recovery_policy(
|
||||
&mut failure,
|
||||
WebSocketRecoveryContext {
|
||||
policy: WebSocketFailurePolicy::Budgeted,
|
||||
requirement: request_transport_requirement,
|
||||
replay_checkpoint_available,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
request_id: context.request_id().as_str(),
|
||||
@@ -650,19 +792,13 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
account_id: active_account.id().as_str(),
|
||||
session_affinity_key: session_affinity_key.as_ref(),
|
||||
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
|
||||
session_transport_fallbacks: &session_transport_fallbacks,
|
||||
session_transport_recovery: &session_transport_recovery,
|
||||
session_recovery_probe: &mut session_recovery_probe,
|
||||
},
|
||||
);
|
||||
}
|
||||
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
|
||||
if !updates.is_empty() {
|
||||
passive_quota_observation.observe(&updates);
|
||||
let update_headers = rate_limit_update_headers(&updates);
|
||||
if observation_state.merge_rate_limit_headers(&update_headers)
|
||||
&& let Some(observation) = observation_state.observation()
|
||||
{
|
||||
yield ProviderEvent::observation(observation);
|
||||
}
|
||||
if let Some(event) = observation_event {
|
||||
yield event;
|
||||
}
|
||||
if allows_account_state_mutation {
|
||||
synchronize_passive_quota(
|
||||
@@ -686,16 +822,14 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
passive_quota_observation.observe(&updates);
|
||||
observation_state.merge_rate_limit_headers(&rate_limit_update_headers(&updates))
|
||||
};
|
||||
let turn_state_changed = if let Some(updates) = turn_state_updates.as_ref()
|
||||
&& let Some(turn_state) = updates.lock().await.take()
|
||||
{
|
||||
if let Some(capture) = session_capture.as_mut() {
|
||||
capture.turn_state = Some(turn_state.clone());
|
||||
}
|
||||
observation_state.merge_client_header("x-codex-turn-state", &turn_state)
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let turn_state_merge = merge_turn_state_update(
|
||||
turn_state_updates.as_ref(),
|
||||
&mut session_capture,
|
||||
&mut observation_state,
|
||||
)
|
||||
.await;
|
||||
turn_state_checkpoint_observed |= turn_state_merge.is_some();
|
||||
let turn_state_changed = turn_state_merge.unwrap_or(false);
|
||||
let first_event_changed =
|
||||
observation_state.observe_stream_chunk(&chunk, output_started_at);
|
||||
let chunk_len = chunk.len();
|
||||
@@ -743,6 +877,21 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
let terminal_changed = completed
|
||||
&& observation_state.mark_completed(terminal_response_is_incomplete(&events));
|
||||
if response_transport == CodexBackendTransport::WebSocket
|
||||
&& completed
|
||||
&& terminal_failure.is_none()
|
||||
&& let Some(probe) = session_recovery_probe.take()
|
||||
{
|
||||
let session_recovery_cleared = probe.succeed();
|
||||
tracing::info!(
|
||||
request_id = %context.request_id(),
|
||||
attempt_index = context.attempt_index().get(),
|
||||
session_affinity_key_hash = session_affinity_key_hash.as_deref().unwrap_or(""),
|
||||
session_transport_action = "websocket_recovered",
|
||||
session_recovery_cleared,
|
||||
"OpenAI session fresh WebSocket probe reached a terminal response"
|
||||
);
|
||||
}
|
||||
if allows_account_state_mutation && (completed || terminal_failure.is_some()) {
|
||||
synchronize_passive_quota(
|
||||
"a,
|
||||
@@ -857,16 +1006,13 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
apply_failure(&failure_context, &active_account, failure)
|
||||
.await;
|
||||
}
|
||||
let turn_state_changed = if let Some(updates) = turn_state_updates.as_ref()
|
||||
&& let Some(turn_state) = updates.lock().await.take()
|
||||
{
|
||||
if let Some(capture) = session_capture.as_mut() {
|
||||
capture.turn_state = Some(turn_state.clone());
|
||||
}
|
||||
observation_state.merge_client_header("x-codex-turn-state", &turn_state)
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let turn_state_changed = merge_turn_state_update(
|
||||
turn_state_updates.as_ref(),
|
||||
&mut session_capture,
|
||||
&mut observation_state,
|
||||
)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
attach_openai_session_update(&mut events, &mut session_capture);
|
||||
let completed = events
|
||||
.iter()
|
||||
@@ -874,6 +1020,21 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
let terminal_changed = completed
|
||||
&& observation_state.mark_completed(terminal_response_is_incomplete(&events));
|
||||
if response_transport == CodexBackendTransport::WebSocket
|
||||
&& completed
|
||||
&& terminal_failure.is_none()
|
||||
&& let Some(probe) = session_recovery_probe.take()
|
||||
{
|
||||
let session_recovery_cleared = probe.succeed();
|
||||
tracing::info!(
|
||||
request_id = %context.request_id(),
|
||||
attempt_index = context.attempt_index().get(),
|
||||
session_affinity_key_hash = session_affinity_key_hash.as_deref().unwrap_or(""),
|
||||
session_transport_action = "websocket_recovered",
|
||||
session_recovery_cleared,
|
||||
"OpenAI session fresh WebSocket probe reached a terminal response"
|
||||
);
|
||||
}
|
||||
if allows_account_state_mutation && completed && terminal_failure.is_none() {
|
||||
// 同上:尾部 finish() 也可能产出 completed,亲和记录必须先于任何下游 yield。
|
||||
selector
|
||||
@@ -917,3 +1078,16 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn merge_turn_state_update(
|
||||
updates: Option<&CodexTurnStateUpdate>,
|
||||
session_capture: &mut Option<OpenAiSessionCapture>,
|
||||
observation_state: &mut OpenAiResponseObservationState,
|
||||
) -> Option<bool> {
|
||||
let updates = updates?;
|
||||
let turn_state = updates.lock().await.take()?;
|
||||
if let Some(capture) = session_capture.as_mut() {
|
||||
capture.turn_state = Some(turn_state.clone());
|
||||
}
|
||||
Some(observation_state.merge_client_header("x-codex-turn-state", &turn_state))
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@ pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
|
||||
pub(super) struct MappedProviderFailure {
|
||||
pub(super) error: ProviderError,
|
||||
pub(super) websocket_transport_retryable: bool,
|
||||
/// 兼容少数链路在 metadata 后、终态前关闭;仅在捕获 turn-state 时允许安全恢复。
|
||||
pub(super) websocket_checkpoint_retryable: bool,
|
||||
pub(super) account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,仅在凭据错误状态下持久化。
|
||||
pub(super) error_message: Option<String>,
|
||||
@@ -54,6 +56,7 @@ impl MappedProviderFailure {
|
||||
Self {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
websocket_checkpoint_retryable: false,
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
@@ -451,6 +454,8 @@ pub(super) enum WebSocketFailurePolicy {
|
||||
pub(super) struct WebSocketRecoveryContext<'a> {
|
||||
pub(super) policy: WebSocketFailurePolicy,
|
||||
pub(super) requirement: TransportRequirement,
|
||||
/// 本 attempt 已从 metadata 捕获可恢复同一请求的 turn-state 检查点。
|
||||
pub(super) replay_checkpoint_available: bool,
|
||||
pub(super) retry_count: u32,
|
||||
pub(super) max_retries: u32,
|
||||
pub(super) request_id: &'a str,
|
||||
@@ -458,7 +463,8 @@ pub(super) struct WebSocketRecoveryContext<'a> {
|
||||
pub(super) account_id: &'a str,
|
||||
pub(super) session_affinity_key: Option<&'a ProviderSessionAffinityKey>,
|
||||
pub(super) session_affinity_key_hash: Option<&'a str>,
|
||||
pub(super) session_transport_fallbacks: &'a CodexSessionTransportFallbacks,
|
||||
pub(super) session_transport_recovery: &'a CodexSessionTransportRecovery,
|
||||
pub(super) session_recovery_probe: &'a mut Option<CodexSessionWebSocketProbe>,
|
||||
}
|
||||
|
||||
pub(super) fn websocket_client_failure_policy(
|
||||
@@ -504,15 +510,24 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
context: WebSocketRecoveryContext<'_>,
|
||||
) {
|
||||
let send_state = failure.error.send_state();
|
||||
if matches!(
|
||||
let post_send = matches!(
|
||||
send_state,
|
||||
UpstreamSendState::Sent | UpstreamSendState::Ambiguous
|
||||
) {
|
||||
);
|
||||
if post_send && !context.replay_checkpoint_available {
|
||||
// payload 已经发送或发送结果不确定时,代理不能用同一业务输入自动重放。
|
||||
// 只为客户端下一次完整请求记住 HTTP 偏好,当前请求保持终态失败。
|
||||
let sticky_state_transition = context
|
||||
.session_affinity_key
|
||||
.map(|key| context.session_transport_fallbacks.disable_websocket(key));
|
||||
// 当前请求保持终态失败;恢复状态只裁决客户端主动发起的下一次请求。
|
||||
let session_recovery_transition = match context.session_recovery_probe.take() {
|
||||
Some(probe) => Some(probe.post_send_failed()),
|
||||
None => context.session_affinity_key.map(|key| {
|
||||
context
|
||||
.session_transport_recovery
|
||||
.record_post_send_failure(key)
|
||||
}),
|
||||
};
|
||||
let session_http_cooldown_ms = session_recovery_transition
|
||||
.and_then(CodexSessionRecoveryTransition::cooldown)
|
||||
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX));
|
||||
tracing::warn!(
|
||||
request_id = context.request_id,
|
||||
attempt_index = context.attempt_index,
|
||||
@@ -527,8 +542,12 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
continuation_recovery_action = "client_replay_required",
|
||||
session_affinity_present = context.session_affinity_key.is_some(),
|
||||
session_affinity_key_hash = context.session_affinity_key_hash.unwrap_or(""),
|
||||
sticky_http_enabled = sticky_state_transition.is_some(),
|
||||
sticky_state_transition = sticky_state_transition.unwrap_or(false),
|
||||
session_transport_action = session_recovery_transition
|
||||
.map_or("untracked", CodexSessionRecoveryTransition::action),
|
||||
session_websocket_failure_count = session_recovery_transition
|
||||
.map_or(0, CodexSessionRecoveryTransition::failure_count),
|
||||
session_http_cooldown_ms = session_http_cooldown_ms.unwrap_or_default(),
|
||||
session_http_cooldown_present = session_http_cooldown_ms.is_some(),
|
||||
"OpenAI upstream WebSocket failed after payload send; proxy replay was suppressed"
|
||||
);
|
||||
return;
|
||||
@@ -542,6 +561,11 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
// 传输恢复必须保持原账号可调度;最终 HTTP attempt 若仍失败,再按真实 HTTP
|
||||
// 结果更新账号健康度,避免中间 WS 错误把同账号钉选提前冷却掉。
|
||||
failure.account_failure = None;
|
||||
if post_send {
|
||||
// Core 对 Ambiguous 默认仍然 fail-closed;只有同一 attempt 的 session
|
||||
// checkpoint 与这个 Provider proof 同时存在时,才允许同账号传输恢复。
|
||||
failure.error.set_replay_safe();
|
||||
}
|
||||
let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback
|
||||
|| context.retry_count >= context.max_retries;
|
||||
if !fallback_now {
|
||||
@@ -570,15 +594,28 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
websocket_max_retries = context.max_retries,
|
||||
websocket_retry_delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
|
||||
transport_requirement = context.requirement.as_str(),
|
||||
replay_checkpoint_available = context.replay_checkpoint_available,
|
||||
"OpenAI upstream WebSocket failed; retrying the same account"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
failure.error.set_pre_delivery_transport_fallback();
|
||||
let sticky_state_transition = context
|
||||
.session_affinity_key
|
||||
.map(|key| context.session_transport_fallbacks.disable_websocket(key));
|
||||
let session_fallback = match context.policy {
|
||||
WebSocketFailurePolicy::Budgeted => SessionWebSocketFallback::RetryBudgetExhausted,
|
||||
WebSocketFailurePolicy::ImmediateFallback => SessionWebSocketFallback::UpgradeRequired,
|
||||
};
|
||||
let session_recovery_transition = match context.session_recovery_probe.take() {
|
||||
Some(probe) => Some(probe.fallback(session_fallback)),
|
||||
None => context.session_affinity_key.map(|key| {
|
||||
context
|
||||
.session_transport_recovery
|
||||
.record_websocket_fallback(key, session_fallback)
|
||||
}),
|
||||
};
|
||||
let session_http_cooldown_ms = session_recovery_transition
|
||||
.and_then(CodexSessionRecoveryTransition::cooldown)
|
||||
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX));
|
||||
tracing::warn!(
|
||||
request_id = context.request_id,
|
||||
attempt_index = context.attempt_index,
|
||||
@@ -596,11 +633,16 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted",
|
||||
WebSocketFailurePolicy::ImmediateFallback => "upgrade_required",
|
||||
},
|
||||
replay_checkpoint_available = context.replay_checkpoint_available,
|
||||
session_affinity_present = context.session_affinity_key.is_some(),
|
||||
session_affinity_key_hash = context.session_affinity_key_hash.unwrap_or(""),
|
||||
sticky_http_enabled = sticky_state_transition.is_some(),
|
||||
sticky_state_transition = sticky_state_transition.unwrap_or(false),
|
||||
"OpenAI upstream WebSocket disabled for the current session"
|
||||
session_transport_action =
|
||||
session_recovery_transition.map_or("untracked", CodexSessionRecoveryTransition::action),
|
||||
session_websocket_failure_count =
|
||||
session_recovery_transition.map_or(0, CodexSessionRecoveryTransition::failure_count),
|
||||
session_http_cooldown_ms = session_http_cooldown_ms.unwrap_or_default(),
|
||||
session_http_cooldown_present = session_http_cooldown_ms.is_some(),
|
||||
"OpenAI upstream WebSocket entered session HTTP cooldown"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -644,14 +686,31 @@ pub(super) fn continuation_replay_required_error(reason: &'static str) -> Provid
|
||||
pub(super) fn map_stream_error(error: CodexClientError) -> MappedProviderFailure {
|
||||
let allows_pre_delivery_retry = stream_transport_allows_pre_delivery_retry(&error);
|
||||
let websocket_failure = error.transport() == Some(CodexBackendTransport::WebSocket);
|
||||
let websocket_checkpoint_retryable = websocket_metadata_close_can_resume(&error);
|
||||
let mut failure = map_client_error(error, UpstreamSendState::Sent, false);
|
||||
failure.websocket_transport_retryable = allows_pre_delivery_retry && websocket_failure;
|
||||
failure.websocket_checkpoint_retryable =
|
||||
allows_pre_delivery_retry && websocket_checkpoint_retryable;
|
||||
if allows_pre_delivery_retry && !websocket_failure {
|
||||
failure.error = failure.error.with_pre_delivery_retry();
|
||||
}
|
||||
failure
|
||||
}
|
||||
|
||||
fn websocket_metadata_close_can_resume(error: &CodexClientError) -> bool {
|
||||
let CodexClientError::WebSocket(error) = error else {
|
||||
return false;
|
||||
};
|
||||
let Some(close) = error.close_before_terminal() else {
|
||||
return false;
|
||||
};
|
||||
close.code() == Some(1000)
|
||||
&& matches!(
|
||||
close.last_event_type(),
|
||||
Some("codex.response.metadata" | "response.metadata")
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn stream_transport_allows_pre_delivery_retry(error: &CodexClientError) -> bool {
|
||||
match error {
|
||||
CodexClientError::Http(_)
|
||||
@@ -870,9 +929,6 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
|
||||
CodexWebSocketExchangeError::ConnectTimeout { timeout } => {
|
||||
format!("OpenAI WebSocket connection timed out after {timeout:?}")
|
||||
}
|
||||
CodexWebSocketExchangeError::FastPathTimeout { timeout } => {
|
||||
format!("OpenAI WebSocket fast-path budget expired after {timeout:?}")
|
||||
}
|
||||
CodexWebSocketExchangeError::OriginCircuitOpen => {
|
||||
"OpenAI WebSocket origin circuit is open".to_owned()
|
||||
}
|
||||
@@ -1042,6 +1098,7 @@ pub(super) fn map_upstream_failure(
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
websocket_checkpoint_retryable: false,
|
||||
account_failure: account_failure(
|
||||
category,
|
||||
failure.retry_after_seconds,
|
||||
@@ -1147,7 +1204,6 @@ pub(super) fn websocket_send_state(error: &CodexWebSocketExchangeError) -> Upstr
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
| CodexWebSocketExchangeError::Connect(_)
|
||||
| CodexWebSocketExchangeError::ConnectTimeout { .. }
|
||||
| CodexWebSocketExchangeError::FastPathTimeout { .. }
|
||||
| CodexWebSocketExchangeError::OriginCircuitOpen
|
||||
| CodexWebSocketExchangeError::OriginHalfOpenBusy
|
||||
| CodexWebSocketExchangeError::SharedConnectFailed
|
||||
@@ -1175,7 +1231,6 @@ pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> Provi
|
||||
| CodexWebSocketExchangeError::InvalidSse(_)
|
||||
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => ProviderErrorKind::Protocol,
|
||||
CodexWebSocketExchangeError::ConnectTimeout { .. }
|
||||
| CodexWebSocketExchangeError::FastPathTimeout { .. }
|
||||
| CodexWebSocketExchangeError::SendTimeout { .. }
|
||||
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
|
||||
| CodexWebSocketExchangeError::InitialEventTimeout { .. } => ProviderErrorKind::Timeout,
|
||||
|
||||
@@ -29,7 +29,8 @@ use gateway_core::event::{
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::operation::{
|
||||
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind, ProviderSessionState,
|
||||
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind,
|
||||
ProviderSessionState, StandaloneSearchRequest,
|
||||
};
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
use gateway_core::routing::{
|
||||
@@ -60,7 +61,10 @@ use crate::credential::{
|
||||
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
derive_previous_response_id_hash,
|
||||
};
|
||||
use crate::session_transport::CodexSessionTransportFallbacks;
|
||||
use crate::session_transport::{
|
||||
CodexSessionRecoveryTransition, CodexSessionTransportDecision, CodexSessionTransportRecovery,
|
||||
CodexSessionWebSocketProbe, SessionWebSocketFallback,
|
||||
};
|
||||
use crate::transport::canonical::{
|
||||
CodexCanonicalDecoder, CodexCanonicalError, CodexCanonicalOutcome,
|
||||
};
|
||||
@@ -85,11 +89,12 @@ use crate::transport::session::CodexSessionIdentity;
|
||||
use crate::transport::usage::normalize_service_tier;
|
||||
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason};
|
||||
use crate::transport::{
|
||||
CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH, CODEX_RESPONSES_PATH,
|
||||
CodexAccountSelectionTelemetry, CodexBackendClient, CodexBackendJsonResponse,
|
||||
CodexBackendStreamingResponse, CodexBackendTransport, CodexClientError, CodexRateLimitUpdates,
|
||||
CodexRequestContext, CodexResponseMetadata, CodexTransportMetrics, CodexUpstreamDiagnostics,
|
||||
CodexWebSocketPool, endpoint_url,
|
||||
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
|
||||
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
|
||||
CodexBackendJsonResponse, CodexBackendStreamingResponse, CodexBackendTransport,
|
||||
CodexClientError, CodexRateLimitUpdates, CodexRequestContext, CodexResponseMetadata,
|
||||
CodexTransportMetrics, CodexTurnStateUpdate, CodexUpstreamDiagnostics, CodexWebSocketPool,
|
||||
WebSocketConnectionPreference, endpoint_url,
|
||||
};
|
||||
|
||||
mod execution;
|
||||
@@ -142,8 +147,9 @@ pub struct CodexProvider {
|
||||
responses_url: Url,
|
||||
image_generations_url: Url,
|
||||
image_edits_url: Url,
|
||||
search_url: Url,
|
||||
session_identity: Option<CodexSessionIdentity>,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
session_transport_recovery: CodexSessionTransportRecovery,
|
||||
stream_max_retries: u32,
|
||||
}
|
||||
|
||||
@@ -168,6 +174,8 @@ impl CodexProvider {
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
let image_edits_url = Url::parse(&endpoint_url(&base_url, CODEX_IMAGE_EDITS_PATH))
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
let search_url = Url::parse(&endpoint_url(&base_url, CODEX_ALPHA_SEARCH_PATH))
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
let client =
|
||||
CodexBackendClient::new(http, base_url, profile).with_websocket_pool(websocket_pool);
|
||||
Ok(Self {
|
||||
@@ -179,8 +187,9 @@ impl CodexProvider {
|
||||
responses_url,
|
||||
image_generations_url,
|
||||
image_edits_url,
|
||||
search_url,
|
||||
session_identity: None,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks::default(),
|
||||
session_transport_recovery: CodexSessionTransportRecovery::default(),
|
||||
stream_max_retries,
|
||||
})
|
||||
}
|
||||
@@ -283,6 +292,9 @@ impl Provider for CodexProvider {
|
||||
if let Operation::GenerateImage(image) = request.operation() {
|
||||
return self.execute_image(image, candidate, context).await;
|
||||
}
|
||||
if let Operation::Search(search) = request.operation() {
|
||||
return self.execute_search(search, candidate, context).await;
|
||||
}
|
||||
let Operation::Generate(generate) = request.operation() else {
|
||||
return Err(provider_error(
|
||||
ProviderErrorKind::Unsupported,
|
||||
@@ -308,15 +320,22 @@ impl Provider for CodexProvider {
|
||||
if let Some(identity) = &self.session_identity {
|
||||
identity.prepare_local_conversation(&mut upstream_request);
|
||||
}
|
||||
if previous_session.as_ref().is_some_and(|state| {
|
||||
same_client_turn(
|
||||
state.client_turn_id.as_deref(),
|
||||
upstream_request.client_turn_id.as_deref(),
|
||||
)
|
||||
}) {
|
||||
upstream_request.turn_state = previous_session
|
||||
.as_ref()
|
||||
.and_then(|state| state.turn_state.clone());
|
||||
if let Some(previous_session) = previous_session.as_ref() {
|
||||
upstream_request.turn_state = if previous_session.retry_checkpoint
|
||||
|| same_client_turn(
|
||||
previous_session.client_turn_id.as_deref(),
|
||||
upstream_request.client_turn_id.as_deref(),
|
||||
) {
|
||||
// 客户端可能携带 metadata-close 失败后才拿到的新状态;同一
|
||||
// turn 内显式回传值比最后一次成功响应保存的状态更新。同请求
|
||||
// 内部恢复由一次性 retry checkpoint 明确证明,不依赖客户端 turn ID。
|
||||
upstream_request
|
||||
.turn_state
|
||||
.take()
|
||||
.or_else(|| previous_session.turn_state.clone())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
}
|
||||
let session_affinity =
|
||||
derive_codex_session_affinity(&upstream_request, context.client_api_key_ref());
|
||||
@@ -434,26 +453,46 @@ impl Provider for CodexProvider {
|
||||
);
|
||||
let requirement = transport_requirement(&upstream_request);
|
||||
let requested_transport = selected_transport(&upstream_request);
|
||||
let sticky_http_fallback = context.transport() == AttemptTransport::Default
|
||||
&& requested_transport == CodexProviderTransport::PreferWebSocket
|
||||
&& requirement.allows_sticky_http_fallback()
|
||||
&& session_affinity.as_ref().is_some_and(|affinity| {
|
||||
self.session_transport_fallbacks
|
||||
.is_http_only(affinity.key())
|
||||
});
|
||||
let session_transport_decision = if matches!(
|
||||
context.transport(),
|
||||
AttemptTransport::Default | AttemptTransport::Retry(_)
|
||||
) && requested_transport
|
||||
== CodexProviderTransport::PreferWebSocket
|
||||
&& requirement.allows_session_transport_recovery()
|
||||
&& let Some(affinity) = session_affinity.as_ref()
|
||||
{
|
||||
self.session_transport_recovery.decide(affinity.key())
|
||||
} else {
|
||||
CodexSessionTransportDecision::Default
|
||||
};
|
||||
let session_transport_action = session_transport_decision.action();
|
||||
let session_websocket_failure_count = session_transport_decision.failure_count();
|
||||
let session_http_cooldown_ms = session_transport_decision
|
||||
.retry_after()
|
||||
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX));
|
||||
let mut websocket_connection_preference = WebSocketConnectionPreference::ReuseOrConnect;
|
||||
let mut session_recovery_probe = None;
|
||||
let transport = if requirement.requires_websocket() {
|
||||
CodexProviderTransport::PreferWebSocket
|
||||
} else {
|
||||
match context.transport() {
|
||||
AttemptTransport::Default if sticky_http_fallback => {
|
||||
CodexProviderTransport::HttpOnly
|
||||
AttemptTransport::Default | AttemptTransport::Retry(_) => {
|
||||
match session_transport_decision {
|
||||
CodexSessionTransportDecision::Default => requested_transport,
|
||||
CodexSessionTransportDecision::FreshWebSocket { probe, .. } => {
|
||||
websocket_connection_preference = WebSocketConnectionPreference::Fresh;
|
||||
session_recovery_probe = Some(probe);
|
||||
CodexProviderTransport::PreferWebSocket
|
||||
}
|
||||
CodexSessionTransportDecision::HttpSse { .. } => {
|
||||
CodexProviderTransport::HttpOnly
|
||||
}
|
||||
}
|
||||
}
|
||||
AttemptTransport::Default => requested_transport,
|
||||
AttemptTransport::Retry(_) => requested_transport,
|
||||
AttemptTransport::Fallback => CodexProviderTransport::HttpOnly,
|
||||
}
|
||||
};
|
||||
if sticky_http_fallback {
|
||||
if let Some(session_transport_action) = session_transport_action {
|
||||
tracing::info!(
|
||||
request_id = %context.request_id(),
|
||||
attempt_index = context.attempt_index().get(),
|
||||
@@ -461,7 +500,11 @@ impl Provider for CodexProvider {
|
||||
.as_ref()
|
||||
.map_or("", CodexSessionAffinity::key_hash),
|
||||
transport_requirement = requirement.as_str(),
|
||||
"Reusing session-sticky OpenAI upstream HTTP fallback"
|
||||
session_transport_action,
|
||||
session_websocket_failure_count = session_websocket_failure_count.unwrap_or_default(),
|
||||
session_http_cooldown_ms = session_http_cooldown_ms.unwrap_or_default(),
|
||||
session_http_cooldown_present = session_http_cooldown_ms.is_some(),
|
||||
"Applied OpenAI session transport recovery decision"
|
||||
);
|
||||
}
|
||||
apply_transport(&mut upstream_request, transport);
|
||||
@@ -510,7 +553,9 @@ impl Provider for CodexProvider {
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_affinity_key_hash,
|
||||
session_transport_fallbacks: self.session_transport_fallbacks.clone(),
|
||||
websocket_connection_preference,
|
||||
session_transport_recovery: self.session_transport_recovery.clone(),
|
||||
session_recovery_probe,
|
||||
websocket_retry_count,
|
||||
stream_max_retries: self.stream_max_retries,
|
||||
session_capture,
|
||||
|
||||
@@ -1,104 +1,457 @@
|
||||
//! OpenAI 根会话级上游传输降级状态。
|
||||
//! OpenAI 根会话级上游传输恢复状态。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
use tokio::time::Instant;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::credential::CODEX_ROOT_SESSION_TTL;
|
||||
|
||||
/// 未成功恢复的失败链需要跨过最长冷却,避免 4 小时档在半开前被清理。
|
||||
const SESSION_RECOVERY_RETENTION: Duration = Duration::from_secs(8 * 60 * 60);
|
||||
const HTTP_COOLDOWNS: [Duration; 4] = [
|
||||
Duration::from_secs(5 * 60),
|
||||
Duration::from_secs(30 * 60),
|
||||
Duration::from_secs(60 * 60),
|
||||
Duration::from_secs(4 * 60 * 60),
|
||||
];
|
||||
/// Provider 是进程级单例;限制不可显式观测结束的客户端会话占用。
|
||||
const MAX_SESSION_HTTP_FALLBACKS: usize = 16_384;
|
||||
const MAX_SESSION_RECOVERIES: usize = 16_384;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CodexSessionTransportFallbacks {
|
||||
state: Arc<Mutex<SessionTransportFallbackState>>,
|
||||
ttl: Duration,
|
||||
pub(crate) struct CodexSessionTransportRecovery {
|
||||
state: Arc<Mutex<SessionTransportRecoveryState>>,
|
||||
http_cooldowns: [Duration; 4],
|
||||
retention: Duration,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SessionTransportFallbackState {
|
||||
http_only: HashMap<ProviderSessionAffinityKey, Instant>,
|
||||
struct SessionTransportRecoveryState {
|
||||
sessions: HashMap<ProviderSessionAffinityKey, SessionCircuit>,
|
||||
}
|
||||
|
||||
impl Default for CodexSessionTransportFallbacks {
|
||||
enum SessionCircuit {
|
||||
FreshPending {
|
||||
failure_count: u8,
|
||||
last_failure_at: Instant,
|
||||
},
|
||||
Open {
|
||||
failure_count: u8,
|
||||
last_failure_at: Instant,
|
||||
until: Instant,
|
||||
},
|
||||
HalfOpen {
|
||||
failure_count: u8,
|
||||
last_failure_at: Instant,
|
||||
probe_id: Uuid,
|
||||
},
|
||||
}
|
||||
|
||||
impl SessionCircuit {
|
||||
const fn failure_count(&self) -> u8 {
|
||||
match self {
|
||||
Self::FreshPending { failure_count, .. }
|
||||
| Self::Open { failure_count, .. }
|
||||
| Self::HalfOpen { failure_count, .. } => *failure_count,
|
||||
}
|
||||
}
|
||||
|
||||
const fn last_failure_at(&self) -> Instant {
|
||||
match self {
|
||||
Self::FreshPending {
|
||||
last_failure_at, ..
|
||||
}
|
||||
| Self::Open {
|
||||
last_failure_at, ..
|
||||
}
|
||||
| Self::HalfOpen {
|
||||
last_failure_at, ..
|
||||
} => *last_failure_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CodexSessionTransportRecovery {
|
||||
fn default() -> Self {
|
||||
Self::with_limits(CODEX_ROOT_SESSION_TTL, MAX_SESSION_HTTP_FALLBACKS)
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(SessionTransportRecoveryState::default())),
|
||||
http_cooldowns: HTTP_COOLDOWNS,
|
||||
retention: SESSION_RECOVERY_RETENTION,
|
||||
capacity: MAX_SESSION_RECOVERIES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexSessionTransportFallbacks {
|
||||
fn with_limits(ttl: Duration, capacity: usize) -> Self {
|
||||
debug_assert!(!ttl.is_zero());
|
||||
debug_assert!(capacity > 0);
|
||||
#[derive(Default)]
|
||||
pub(crate) enum CodexSessionTransportDecision {
|
||||
#[default]
|
||||
Default,
|
||||
FreshWebSocket {
|
||||
failure_count: u8,
|
||||
probe: CodexSessionWebSocketProbe,
|
||||
},
|
||||
HttpSse {
|
||||
reason: SessionHttpFallbackReason,
|
||||
failure_count: u8,
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
}
|
||||
|
||||
impl CodexSessionTransportDecision {
|
||||
pub(crate) const fn action(&self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Default => None,
|
||||
Self::FreshWebSocket { .. } => Some("fresh_websocket_probe"),
|
||||
Self::HttpSse {
|
||||
reason: SessionHttpFallbackReason::Cooldown,
|
||||
..
|
||||
} => Some("http_cooldown"),
|
||||
Self::HttpSse {
|
||||
reason: SessionHttpFallbackReason::ProbeInFlight,
|
||||
..
|
||||
} => Some("http_probe_in_flight"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn failure_count(&self) -> Option<u8> {
|
||||
match self {
|
||||
Self::Default => None,
|
||||
Self::FreshWebSocket { failure_count, .. } | Self::HttpSse { failure_count, .. } => {
|
||||
Some(*failure_count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn retry_after(&self) -> Option<Duration> {
|
||||
match self {
|
||||
Self::HttpSse { retry_after, .. } => *retry_after,
|
||||
Self::Default | Self::FreshWebSocket { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum SessionWebSocketFallback {
|
||||
RetryBudgetExhausted,
|
||||
UpgradeRequired,
|
||||
}
|
||||
|
||||
impl SessionWebSocketFallback {
|
||||
const fn minimum_failure_count(self) -> u8 {
|
||||
match self {
|
||||
Self::RetryBudgetExhausted => 2,
|
||||
Self::UpgradeRequired => 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum SessionHttpFallbackReason {
|
||||
Cooldown,
|
||||
ProbeInFlight,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct CodexSessionRecoveryTransition {
|
||||
failure_count: u8,
|
||||
cooldown: Option<Duration>,
|
||||
}
|
||||
|
||||
impl CodexSessionRecoveryTransition {
|
||||
pub(crate) const fn action(self) -> &'static str {
|
||||
if self.cooldown.is_some() {
|
||||
"http_cooldown"
|
||||
} else {
|
||||
"fresh_websocket_next"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn failure_count(self) -> u8 {
|
||||
self.failure_count
|
||||
}
|
||||
|
||||
pub(crate) const fn cooldown(self) -> Option<Duration> {
|
||||
self.cooldown
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次根会话级 fresh WebSocket 许可。
|
||||
///
|
||||
/// 许可未得到明确结果就被释放时,恢复状态会重新允许下一次半开探测。
|
||||
pub(crate) struct CodexSessionWebSocketProbe {
|
||||
recovery: CodexSessionTransportRecovery,
|
||||
key: ProviderSessionAffinityKey,
|
||||
probe_id: Uuid,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl CodexSessionWebSocketProbe {
|
||||
fn new(
|
||||
recovery: CodexSessionTransportRecovery,
|
||||
key: ProviderSessionAffinityKey,
|
||||
probe_id: Uuid,
|
||||
) -> Self {
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(SessionTransportFallbackState::default())),
|
||||
ttl,
|
||||
capacity,
|
||||
recovery,
|
||||
key,
|
||||
probe_id,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回该根会话是否已粘性切到 HTTP,并为活跃会话续期。
|
||||
pub(crate) fn is_http_only(&self, key: &ProviderSessionAffinityKey) -> bool {
|
||||
self.is_http_only_at(key, Instant::now())
|
||||
pub(crate) fn succeed(mut self) -> bool {
|
||||
let recovered = self.recovery.record_probe_success(&self.key, self.probe_id);
|
||||
self.armed = false;
|
||||
recovered
|
||||
}
|
||||
|
||||
/// 在当前 WebSocket 重试预算耗尽后,为根会话启用粘性 HTTP。
|
||||
///
|
||||
/// 返回 `true` 表示本次发生了禁用状态转换;已禁用会话只续期。
|
||||
pub(crate) fn disable_websocket(&self, key: &ProviderSessionAffinityKey) -> bool {
|
||||
self.disable_websocket_at(key, Instant::now())
|
||||
pub(crate) fn post_send_failed(mut self) -> CodexSessionRecoveryTransition {
|
||||
let transition = self.recovery.record_post_send_failure(&self.key);
|
||||
self.armed = false;
|
||||
transition
|
||||
}
|
||||
|
||||
fn is_http_only_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
|
||||
pub(crate) fn fallback(
|
||||
mut self,
|
||||
fallback: SessionWebSocketFallback,
|
||||
) -> CodexSessionRecoveryTransition {
|
||||
let transition = self.recovery.record_websocket_fallback(&self.key, fallback);
|
||||
self.armed = false;
|
||||
transition
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CodexSessionWebSocketProbe {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
self.recovery.cancel_probe(&self.key, self.probe_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexSessionTransportRecovery {
|
||||
pub(crate) fn decide(&self, key: &ProviderSessionAffinityKey) -> CodexSessionTransportDecision {
|
||||
self.decide_at(key, Instant::now())
|
||||
}
|
||||
|
||||
pub(crate) fn record_post_send_failure(
|
||||
&self,
|
||||
key: &ProviderSessionAffinityKey,
|
||||
) -> CodexSessionRecoveryTransition {
|
||||
self.record_post_send_failure_at(key, Instant::now())
|
||||
}
|
||||
|
||||
pub(crate) fn record_websocket_fallback(
|
||||
&self,
|
||||
key: &ProviderSessionAffinityKey,
|
||||
fallback: SessionWebSocketFallback,
|
||||
) -> CodexSessionRecoveryTransition {
|
||||
self.record_websocket_fallback_at(key, fallback, Instant::now())
|
||||
}
|
||||
|
||||
fn decide_at(
|
||||
&self,
|
||||
key: &ProviderSessionAffinityKey,
|
||||
now: Instant,
|
||||
) -> CodexSessionTransportDecision {
|
||||
let mut state = self.lock();
|
||||
let Some(last_seen) = state.http_only.get_mut(key) else {
|
||||
return false;
|
||||
let stale = state.sessions.get(key).is_some_and(|circuit| {
|
||||
now.saturating_duration_since(circuit.last_failure_at()) >= self.retention
|
||||
});
|
||||
if stale {
|
||||
state.sessions.remove(key);
|
||||
return CodexSessionTransportDecision::Default;
|
||||
}
|
||||
|
||||
let Some(circuit) = state.sessions.get_mut(key) else {
|
||||
return CodexSessionTransportDecision::Default;
|
||||
};
|
||||
if now.saturating_duration_since(*last_seen) >= self.ttl {
|
||||
state.http_only.remove(key);
|
||||
return false;
|
||||
match circuit {
|
||||
SessionCircuit::Open {
|
||||
failure_count,
|
||||
until,
|
||||
..
|
||||
} if now < *until => CodexSessionTransportDecision::HttpSse {
|
||||
reason: SessionHttpFallbackReason::Cooldown,
|
||||
failure_count: *failure_count,
|
||||
retry_after: Some(until.saturating_duration_since(now)),
|
||||
},
|
||||
SessionCircuit::HalfOpen { failure_count, .. } => {
|
||||
CodexSessionTransportDecision::HttpSse {
|
||||
reason: SessionHttpFallbackReason::ProbeInFlight,
|
||||
failure_count: *failure_count,
|
||||
retry_after: None,
|
||||
}
|
||||
}
|
||||
SessionCircuit::FreshPending {
|
||||
failure_count,
|
||||
last_failure_at,
|
||||
}
|
||||
| SessionCircuit::Open {
|
||||
failure_count,
|
||||
last_failure_at,
|
||||
..
|
||||
} => {
|
||||
let failure_count = *failure_count;
|
||||
let last_failure_at = *last_failure_at;
|
||||
let probe_id = Uuid::new_v4();
|
||||
*circuit = SessionCircuit::HalfOpen {
|
||||
failure_count,
|
||||
last_failure_at,
|
||||
probe_id,
|
||||
};
|
||||
CodexSessionTransportDecision::FreshWebSocket {
|
||||
failure_count,
|
||||
probe: CodexSessionWebSocketProbe::new(self.clone(), key.clone(), probe_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
*last_seen = now;
|
||||
true
|
||||
}
|
||||
|
||||
fn disable_websocket_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
|
||||
fn record_post_send_failure_at(
|
||||
&self,
|
||||
key: &ProviderSessionAffinityKey,
|
||||
now: Instant,
|
||||
) -> CodexSessionRecoveryTransition {
|
||||
let mut state = self.lock();
|
||||
let already_disabled = state
|
||||
.http_only
|
||||
.get(key)
|
||||
.is_some_and(|last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
|
||||
if !already_disabled && !state.http_only.contains_key(key) {
|
||||
let failure_count = self.next_failure_count(state.sessions.get(key), now);
|
||||
if !state.sessions.contains_key(key) {
|
||||
self.make_room(&mut state, now);
|
||||
}
|
||||
state.http_only.insert(key.clone(), now);
|
||||
!already_disabled
|
||||
if failure_count == 1 {
|
||||
state.sessions.insert(
|
||||
key.clone(),
|
||||
SessionCircuit::FreshPending {
|
||||
failure_count,
|
||||
last_failure_at: now,
|
||||
},
|
||||
);
|
||||
return CodexSessionRecoveryTransition {
|
||||
failure_count,
|
||||
cooldown: None,
|
||||
};
|
||||
}
|
||||
|
||||
let cooldown = self.cooldown_for(failure_count);
|
||||
state.sessions.insert(
|
||||
key.clone(),
|
||||
SessionCircuit::Open {
|
||||
failure_count,
|
||||
last_failure_at: now,
|
||||
until: now + cooldown,
|
||||
},
|
||||
);
|
||||
CodexSessionRecoveryTransition {
|
||||
failure_count,
|
||||
cooldown: Some(cooldown),
|
||||
}
|
||||
}
|
||||
|
||||
fn make_room(&self, state: &mut SessionTransportFallbackState, now: Instant) {
|
||||
if state.http_only.len() < self.capacity {
|
||||
fn record_websocket_fallback_at(
|
||||
&self,
|
||||
key: &ProviderSessionAffinityKey,
|
||||
fallback: SessionWebSocketFallback,
|
||||
now: Instant,
|
||||
) -> CodexSessionRecoveryTransition {
|
||||
let mut state = self.lock();
|
||||
let failure_count = self
|
||||
.next_failure_count(state.sessions.get(key), now)
|
||||
.max(fallback.minimum_failure_count());
|
||||
if !state.sessions.contains_key(key) {
|
||||
self.make_room(&mut state, now);
|
||||
}
|
||||
let cooldown = self.cooldown_for(failure_count);
|
||||
state.sessions.insert(
|
||||
key.clone(),
|
||||
SessionCircuit::Open {
|
||||
failure_count,
|
||||
last_failure_at: now,
|
||||
until: now + cooldown,
|
||||
},
|
||||
);
|
||||
CodexSessionRecoveryTransition {
|
||||
failure_count,
|
||||
cooldown: Some(cooldown),
|
||||
}
|
||||
}
|
||||
|
||||
fn next_failure_count(&self, circuit: Option<&SessionCircuit>, now: Instant) -> u8 {
|
||||
circuit.map_or(1, |circuit| {
|
||||
if now.saturating_duration_since(circuit.last_failure_at()) < self.retention {
|
||||
circuit.failure_count().saturating_add(1)
|
||||
} else {
|
||||
1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn cooldown_for(&self, failure_count: u8) -> Duration {
|
||||
let index = usize::from(failure_count.saturating_sub(2)).min(self.http_cooldowns.len() - 1);
|
||||
self.http_cooldowns[index]
|
||||
}
|
||||
|
||||
fn record_probe_success(&self, key: &ProviderSessionAffinityKey, probe_id: Uuid) -> bool {
|
||||
let mut state = self.lock();
|
||||
let owns_probe = matches!(
|
||||
state.sessions.get(key),
|
||||
Some(SessionCircuit::HalfOpen {
|
||||
probe_id: active_probe_id,
|
||||
..
|
||||
}) if *active_probe_id == probe_id
|
||||
);
|
||||
if owns_probe {
|
||||
state.sessions.remove(key);
|
||||
}
|
||||
owns_probe
|
||||
}
|
||||
|
||||
fn cancel_probe(&self, key: &ProviderSessionAffinityKey, probe_id: Uuid) {
|
||||
let mut state = self.lock();
|
||||
let Some(SessionCircuit::HalfOpen {
|
||||
failure_count,
|
||||
last_failure_at,
|
||||
probe_id: active_probe_id,
|
||||
}) = state.sessions.get(key)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if *active_probe_id != probe_id {
|
||||
return;
|
||||
}
|
||||
state
|
||||
.http_only
|
||||
.retain(|_, last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
|
||||
if state.http_only.len() < self.capacity {
|
||||
let failure_count = *failure_count;
|
||||
let last_failure_at = *last_failure_at;
|
||||
state.sessions.insert(
|
||||
key.clone(),
|
||||
SessionCircuit::FreshPending {
|
||||
failure_count,
|
||||
last_failure_at,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn make_room(&self, state: &mut SessionTransportRecoveryState, now: Instant) {
|
||||
if state.sessions.len() < self.capacity {
|
||||
return;
|
||||
}
|
||||
state.sessions.retain(|_, circuit| {
|
||||
now.saturating_duration_since(circuit.last_failure_at()) < self.retention
|
||||
});
|
||||
if state.sessions.len() < self.capacity {
|
||||
return;
|
||||
}
|
||||
let oldest = state
|
||||
.http_only
|
||||
.sessions
|
||||
.iter()
|
||||
.min_by_key(|(_, last_seen)| **last_seen)
|
||||
.min_by_key(|(_, circuit)| circuit.last_failure_at())
|
||||
.map(|(key, _)| key.clone());
|
||||
if let Some(oldest) = oldest {
|
||||
state.http_only.remove(&oldest);
|
||||
state.sessions.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
fn lock(&self) -> MutexGuard<'_, SessionTransportFallbackState> {
|
||||
fn lock(&self) -> MutexGuard<'_, SessionTransportRecoveryState> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
|
||||
@@ -24,7 +24,9 @@ use serde_json::{Value, map::Map};
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::transport::protocol::responses::{CodexResponsesRequest, TransportRequirement};
|
||||
use crate::transport::protocol::responses::{
|
||||
CodexResponsesRequest, TransportRequirement, X_CODEX_TURN_STATE_CLIENT_METADATA_KEY,
|
||||
};
|
||||
|
||||
use super::diagnostics::{CodexUpstreamDiagnostics, CodexUpstreamFailure, CodexUpstreamSendPhase};
|
||||
use super::response_meta::CodexResponseMetadata;
|
||||
@@ -546,7 +548,7 @@ pub struct CodexBackendStreamingResponse {
|
||||
pub transport: CodexBackendTransport,
|
||||
/// WebSocket 响应所绑定的连接;HTTP transport 为 `None`。
|
||||
pub websocket_connection_id: Option<Uuid>,
|
||||
/// 响应头里的最新 turn state。
|
||||
/// 响应头或 metadata 事件里最先确认的 turn state。
|
||||
pub turn_state: Option<String>,
|
||||
/// 上游透传的 `set-cookie` 列表。
|
||||
pub set_cookie_headers: Vec<String>,
|
||||
@@ -760,32 +762,33 @@ pub(super) async fn read_error_response_body(
|
||||
|
||||
pub(super) fn websocket_upstream_request(request: &CodexResponsesRequest) -> CodexResponsesRequest {
|
||||
let mut request = request.clone();
|
||||
project_responses_lite_to_ws_metadata(&mut request);
|
||||
stamp_ws_stream_request_start_ms(&mut request);
|
||||
project_websocket_client_metadata(&mut request);
|
||||
request
|
||||
}
|
||||
|
||||
fn project_responses_lite_to_ws_metadata(request: &mut CodexResponsesRequest) {
|
||||
let Some(responses_lite) = request.responses_lite.clone() else {
|
||||
return;
|
||||
};
|
||||
let mut metadata = match request.client_metadata() {
|
||||
Some(Value::Object(metadata)) => metadata.clone(),
|
||||
None => Map::new(),
|
||||
Some(_) => return,
|
||||
};
|
||||
metadata
|
||||
.entry(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY.to_string())
|
||||
.or_insert(Value::String(responses_lite));
|
||||
request.set_client_metadata(Some(Value::Object(metadata)));
|
||||
}
|
||||
|
||||
fn stamp_ws_stream_request_start_ms(request: &mut CodexResponsesRequest) {
|
||||
fn project_websocket_client_metadata(request: &mut CodexResponsesRequest) {
|
||||
let mut metadata = match request.client_metadata() {
|
||||
Some(Value::Object(metadata)) => metadata.clone(),
|
||||
None => Map::new(),
|
||||
Some(_) => return,
|
||||
};
|
||||
if let Some(responses_lite) = request.responses_lite.clone() {
|
||||
metadata
|
||||
.entry(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY.to_owned())
|
||||
.or_insert(Value::String(responses_lite));
|
||||
}
|
||||
let turn_state = request.turn_state.clone();
|
||||
match turn_state {
|
||||
Some(turn_state) => {
|
||||
metadata.insert(
|
||||
X_CODEX_TURN_STATE_CLIENT_METADATA_KEY.to_owned(),
|
||||
Value::String(turn_state),
|
||||
);
|
||||
}
|
||||
None => {
|
||||
metadata.remove(X_CODEX_TURN_STATE_CLIENT_METADATA_KEY);
|
||||
}
|
||||
}
|
||||
metadata.insert(
|
||||
X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY.to_string(),
|
||||
Value::String(now_unix_timestamp_millis().to_string()),
|
||||
@@ -854,9 +857,6 @@ pub(super) fn http_fallback_decision(
|
||||
error: &CodexWebSocketExchangeError,
|
||||
) -> CodexTransportDecision {
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::FastPathTimeout { .. } => {
|
||||
CodexTransportDecision::Http2WebSocketBudgetExhausted
|
||||
}
|
||||
CodexWebSocketExchangeError::OriginCircuitOpen
|
||||
| CodexWebSocketExchangeError::OriginHalfOpenBusy => {
|
||||
CodexTransportDecision::Http2BreakerOpen
|
||||
|
||||
@@ -39,6 +39,11 @@ impl CodexBackendClient {
|
||||
context.installation_id,
|
||||
)?;
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-image-turn-id", image_turn_id);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-turn-metadata",
|
||||
context.turn_metadata,
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-client-request-id"),
|
||||
HeaderValue::from_str(context.request_id)?,
|
||||
|
||||
@@ -37,8 +37,9 @@ use crate::transport::{
|
||||
response_meta,
|
||||
websocket::{
|
||||
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
|
||||
CodexWebSocketPoolKey, CodexWebSocketStreamingExchange, DEFAULT_INITIAL_EVENT_TIMEOUT,
|
||||
WEBSOCKET_FAST_PATH_BUDGET, WebSocketOriginBreaker, WebSocketPoolDecision,
|
||||
CodexWebSocketPoolKey, CodexWebSocketRequest, CodexWebSocketStreamingExchange,
|
||||
DEFAULT_INITIAL_EVENT_TIMEOUT, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
|
||||
WebSocketOriginBreaker, WebSocketPoolDecision,
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool, websocket_audit_dir,
|
||||
write_websocket_audit_artifact_from_env,
|
||||
@@ -47,6 +48,16 @@ use crate::transport::{
|
||||
|
||||
use super::client::*;
|
||||
|
||||
/// 单次 Responses WebSocket opening 是否可以使用会话连接池。
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub(crate) enum WebSocketConnectionPreference {
|
||||
/// 优先复用匹配的空闲连接,否则建立并登记一条新连接。
|
||||
#[default]
|
||||
ReuseOrConnect,
|
||||
/// 绕过连接池建立独立连接,用于根会话恢复探针。
|
||||
Fresh,
|
||||
}
|
||||
|
||||
impl CodexBackendClient {
|
||||
/// 构造客户端。
|
||||
pub fn new(
|
||||
@@ -94,11 +105,26 @@ impl CodexBackendClient {
|
||||
upstream_body.insert("stream".to_owned(), serde_json::Value::Bool(true));
|
||||
let body =
|
||||
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
let endpoint = endpoint_url(&self.base_url, CODEX_RESPONSES_PATH);
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
request_direction = "upstream",
|
||||
request_transport = "http_sse",
|
||||
request_method = "POST",
|
||||
request_url = %endpoint,
|
||||
request_headers = ?headers,
|
||||
request_body = %String::from_utf8_lossy(&body),
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
|
||||
.map_err(CodexClientError::RequestCompression)?;
|
||||
let response = self
|
||||
.client
|
||||
.post(endpoint_url(&self.base_url, CODEX_RESPONSES_PATH))
|
||||
.post(endpoint)
|
||||
.headers(headers)
|
||||
.header(CONTENT_ENCODING, HeaderValue::from_static("zstd"))
|
||||
.body(body)
|
||||
@@ -188,9 +214,16 @@ impl CodexBackendClient {
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
connection_preference: WebSocketConnectionPreference,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
let prepared = self
|
||||
.prepare_response_transport(request, context, pool_account_id, false, true)
|
||||
.prepare_response_transport(
|
||||
request,
|
||||
context,
|
||||
pool_account_id,
|
||||
connection_preference,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
self.create_response_stream_with_prepared(request, context, prepared)
|
||||
.await
|
||||
@@ -204,18 +237,24 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
self.prepare_response_transport(request, context, pool_account_id, false, false)
|
||||
.await
|
||||
self.prepare_response_transport(
|
||||
request,
|
||||
context,
|
||||
pool_account_id,
|
||||
WebSocketConnectionPreference::ReuseOrConnect,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 与 [`Self::prepare_response_transport_with_pool_account`] 相同;
|
||||
/// `force_fresh` 时绕过连接池,为连接寿命限制重试强制新建连接。
|
||||
/// fresh 偏好会绕过连接池,为受控恢复探针强制新建连接。
|
||||
async fn prepare_response_transport(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
force_fresh: bool,
|
||||
connection_preference: WebSocketConnectionPreference,
|
||||
defer_websocket_recovery: bool,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
let requirement = transport_requirement(request);
|
||||
@@ -256,10 +295,11 @@ impl CodexBackendClient {
|
||||
let pool_key =
|
||||
self.websocket_pool_key(request, context, pool_account_id, &connection_profile);
|
||||
let pool_log_context = pool_key.as_ref().map(WebSocketPoolLogContext::from_key);
|
||||
let pool = if force_fresh {
|
||||
None
|
||||
} else {
|
||||
self.websocket_pool.as_deref().zip(pool_key)
|
||||
let pool = match connection_preference {
|
||||
WebSocketConnectionPreference::ReuseOrConnect => {
|
||||
self.websocket_pool.as_deref().zip(pool_key)
|
||||
}
|
||||
WebSocketConnectionPreference::Fresh => None,
|
||||
};
|
||||
let fast_path_budget = match requirement {
|
||||
TransportRequirement::PersistedContinuation | TransportRequirement::NewChain => {
|
||||
@@ -282,7 +322,31 @@ impl CodexBackendClient {
|
||||
)
|
||||
.await;
|
||||
let prepared = match prepared {
|
||||
Ok(prepared) => prepared,
|
||||
Ok(WebSocketFastPath::Ready(prepared)) => prepared,
|
||||
Ok(WebSocketFastPath::Missed) => {
|
||||
let decision = CodexTransportDecision::Http2WebSocketBudgetExhausted;
|
||||
let wait_ms = elapsed_duration_millis(prepare_started_at.elapsed());
|
||||
tracing::info!(
|
||||
request_id = %context.request_id,
|
||||
account_id = pool_account_id.or(context.account_id).unwrap_or_default(),
|
||||
transport_requirement = requirement.as_str(),
|
||||
transport_decision = decision.as_str(),
|
||||
transport_decision_wait_ms = wait_ms,
|
||||
websocket_preconnect_continues = pool_log_context.is_some(),
|
||||
"WebSocket fast path missed; using same-account HTTP"
|
||||
);
|
||||
return Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
route: PreparedResponseRoute::Http,
|
||||
metrics: CodexTransportMetrics {
|
||||
decision: Some(decision),
|
||||
ws_connect_ms: None,
|
||||
transport_decision_wait_ms: Some(wait_ms),
|
||||
..CodexTransportMetrics::default()
|
||||
},
|
||||
defer_websocket_recovery,
|
||||
});
|
||||
}
|
||||
Err(error)
|
||||
if (!defer_websocket_recovery || error.requires_immediate_pool_fallback())
|
||||
&& requirement.allows_pre_send_http_fallback()
|
||||
@@ -377,6 +441,7 @@ impl CodexBackendClient {
|
||||
prepared,
|
||||
} = *route;
|
||||
let delivery_wait_started_at = Instant::now();
|
||||
log_unredacted_websocket_request(&websocket_request, context);
|
||||
let mut exchange = match execute_prepared_response_create_request_stream(
|
||||
&websocket_request,
|
||||
prepared,
|
||||
@@ -434,13 +499,20 @@ impl CodexBackendClient {
|
||||
);
|
||||
drop(exchange);
|
||||
match self
|
||||
.prepare_response_transport(request, context, None, true, false)
|
||||
.prepare_response_transport(
|
||||
request,
|
||||
context,
|
||||
None,
|
||||
WebSocketConnectionPreference::Fresh,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(PreparedResponseTransport {
|
||||
route: PreparedResponseRoute::WebSocket(route),
|
||||
..
|
||||
}) => {
|
||||
log_unredacted_websocket_request(&route.request, context);
|
||||
match execute_prepared_response_create_request_stream(
|
||||
&route.request,
|
||||
route.prepared,
|
||||
@@ -628,10 +700,12 @@ impl CodexBackendClient {
|
||||
.local_conversation_id
|
||||
.as_deref()
|
||||
.or(request.previous_response_id())?;
|
||||
Some(
|
||||
CodexWebSocketPoolKey::new(&self.base_url, account_id, conversation_id)
|
||||
.with_connection_profile(connection_profile),
|
||||
)
|
||||
let mut key = CodexWebSocketPoolKey::new(&self.base_url, account_id, conversation_id)
|
||||
.with_connection_profile(connection_profile);
|
||||
if let Some(connection_id) = request.downstream_websocket_connection_id.as_deref() {
|
||||
key = key.with_downstream_connection_id(connection_id);
|
||||
}
|
||||
Some(key)
|
||||
}
|
||||
|
||||
/// 获取后端模型目录条目。
|
||||
@@ -835,6 +909,26 @@ impl CodexBackendClient {
|
||||
}
|
||||
}
|
||||
|
||||
fn log_unredacted_websocket_request(
|
||||
request: &CodexWebSocketRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) {
|
||||
tracing::info!(
|
||||
target: "request_dump",
|
||||
provider = "openai",
|
||||
request_id = %context.request_id,
|
||||
account_id = context.account_id.unwrap_or_default(),
|
||||
request_direction = "upstream",
|
||||
request_transport = "websocket",
|
||||
request_method = "GET + response.create",
|
||||
request_url = request.connection().endpoint(),
|
||||
request_headers = ?request.connection().headers(),
|
||||
request_payload = request.payload_text(),
|
||||
contains_sensitive_data = true,
|
||||
"request dump (unredacted)"
|
||||
);
|
||||
}
|
||||
|
||||
/// 首个可投递帧前的交付边界结果。
|
||||
enum DeliveryBoundary {
|
||||
/// 已越过边界,可开始向下游投递。
|
||||
|
||||
@@ -4,6 +4,8 @@ pub const CODEX_RESPONSES_PATH: &str = "/codex/responses";
|
||||
pub const CODEX_IMAGE_GENERATIONS_PATH: &str = "/codex/images/generations";
|
||||
/// `/codex/images/edits`
|
||||
pub const CODEX_IMAGE_EDITS_PATH: &str = "/codex/images/edits";
|
||||
/// `/codex/alpha/search`
|
||||
pub const CODEX_ALPHA_SEARCH_PATH: &str = "/codex/alpha/search";
|
||||
/// `/api/codex/usage`
|
||||
pub const CODEX_USAGE_API_PATH: &str = "/api/codex/usage";
|
||||
/// `/wham/usage`
|
||||
|
||||
@@ -21,6 +21,7 @@ pub mod tls;
|
||||
pub mod usage;
|
||||
pub mod websocket;
|
||||
|
||||
pub(crate) use self::client_sse::WebSocketConnectionPreference;
|
||||
pub use self::{
|
||||
canonical::{CodexCanonicalDecoder, CodexCanonicalError},
|
||||
catalog::{
|
||||
@@ -36,8 +37,8 @@ pub use self::{
|
||||
},
|
||||
diagnostics::{CodexUpstreamDiagnostics, CodexUpstreamSendPhase},
|
||||
endpoints::{
|
||||
CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH, CODEX_RESPONSES_PATH,
|
||||
CODEX_USAGE_API_PATH, WHAM_PROFILE_STATISTICS_PATH,
|
||||
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
|
||||
CODEX_RESPONSES_PATH, CODEX_USAGE_API_PATH, WHAM_PROFILE_STATISTICS_PATH,
|
||||
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
|
||||
WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url,
|
||||
},
|
||||
|
||||
@@ -10,6 +10,8 @@ use serde_json::{Map, Value};
|
||||
|
||||
/// 官方 Codex 客户端据此触发完整历史重放的稳定错误码。
|
||||
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_CODE: &str = "previous_response_not_found";
|
||||
/// Responses WebSocket 用于回传同一 turn 不透明状态的官方 client metadata 键。
|
||||
pub(crate) const X_CODEX_TURN_STATE_CLIENT_METADATA_KEY: &str = "x-codex-turn-state";
|
||||
/// 本地生成 history unavailable 错误时使用的官方提示文本。
|
||||
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE: &str =
|
||||
"Previous response was not found. Retrying the full request.";
|
||||
@@ -82,6 +84,8 @@ pub struct CodexResponsesRequest {
|
||||
pub memgen_request: Option<String>,
|
||||
/// Codex window ID。
|
||||
pub codex_window_id: Option<String>,
|
||||
/// 代理分配的下游 WebSocket 连接 ID,仅用于隔离本地连接池通道。
|
||||
pub downstream_websocket_connection_id: Option<String>,
|
||||
/// 父线程 ID。
|
||||
pub parent_thread_id: Option<String>,
|
||||
/// 已知 previous response 的持久化范围,仅用于本地 transport 校验。
|
||||
@@ -162,8 +166,8 @@ impl TransportRequirement {
|
||||
matches!(self, Self::NewChain)
|
||||
}
|
||||
|
||||
/// 会话级 sticky 状态是否可以把首选 WebSocket 改为 HTTP。
|
||||
pub fn allows_sticky_http_fallback(self) -> bool {
|
||||
/// 会话级恢复状态是否可以为请求选择 fresh WebSocket 或临时 HTTP。
|
||||
pub fn allows_session_transport_recovery(self) -> bool {
|
||||
matches!(self, Self::NewChain)
|
||||
}
|
||||
|
||||
@@ -519,6 +523,7 @@ impl CodexResponsesRequest {
|
||||
responses_lite: None,
|
||||
memgen_request: None,
|
||||
codex_window_id: None,
|
||||
downstream_websocket_connection_id: None,
|
||||
parent_thread_id: None,
|
||||
previous_response_scope: None,
|
||||
}
|
||||
@@ -649,10 +654,9 @@ impl CodexResponsesRequest {
|
||||
|
||||
/// 返回请求语义与 child transport 隔离所用的子代理区分值。
|
||||
///
|
||||
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;网关的
|
||||
/// `/responses/review` 等入口则通过 `client_metadata.x-openai-subagent`
|
||||
/// 传递给上游。它不拆分根会话的账号首选项;`thread_spawn` 仍用它派生独立
|
||||
/// WebSocket/continuation transport identity。
|
||||
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;兼容客户端也可通过
|
||||
/// `client_metadata.x-openai-subagent` 声明同一语义。它不拆分根会话的账号首选项;
|
||||
/// `thread_spawn` 仍用它派生独立 WebSocket/continuation transport identity。
|
||||
pub fn subagent_kind(&self) -> Option<String> {
|
||||
self.semantics()
|
||||
.subagent_kind
|
||||
|
||||
@@ -110,9 +110,9 @@ fn is_internal_websocket_event(event: &str) -> bool {
|
||||
event == "codex.rate_limits"
|
||||
}
|
||||
|
||||
/// 提取 `response.metadata` 帧中的字符串响应头。
|
||||
/// 提取 Responses WebSocket metadata 帧中的字符串响应头。
|
||||
pub fn websocket_metadata_headers(value: &Value) -> Vec<(String, String)> {
|
||||
if value.get("type").and_then(Value::as_str) != Some("response.metadata") {
|
||||
if !is_websocket_metadata_event(websocket_event_type(value)) {
|
||||
return Vec::new();
|
||||
}
|
||||
value
|
||||
@@ -124,24 +124,18 @@ pub fn websocket_metadata_headers(value: &Value) -> Vec<(String, String)> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 从 `response.metadata` 帧中提取 `x-codex-turn-state`。
|
||||
/// 从 Responses WebSocket metadata 帧中提取 `x-codex-turn-state`。
|
||||
pub fn websocket_metadata_turn_state(value: &Value) -> Option<String> {
|
||||
if value.get("type").and_then(Value::as_str) != Some("response.metadata") {
|
||||
return None;
|
||||
}
|
||||
websocket_metadata_headers(value)
|
||||
.into_iter()
|
||||
.find_map(|(name, value)| {
|
||||
name.eq_ignore_ascii_case("x-codex-turn-state")
|
||||
.then_some(value)
|
||||
})
|
||||
.or_else(|| {
|
||||
value
|
||||
.pointer("/metadata/turn_state")
|
||||
.or_else(|| value.get("turn_state"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string)
|
||||
})
|
||||
}
|
||||
|
||||
fn is_websocket_metadata_event(event: Option<&str>) -> bool {
|
||||
matches!(event, Some("response.metadata" | "codex.response.metadata"))
|
||||
}
|
||||
|
||||
/// 上游 WebSocket 连接寿命限制错误码。
|
||||
|
||||
@@ -7,9 +7,12 @@ use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
|
||||
use serde_json::{Map, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
use crate::transport::protocol::responses::{
|
||||
CodexResponsesRequest, X_CODEX_TURN_STATE_CLIENT_METADATA_KEY,
|
||||
};
|
||||
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
const TURN_ID_CLIENT_METADATA_KEY: &str = "turn_id";
|
||||
const THREAD_SPAWN_SUBAGENT_KIND: &str = "thread_spawn";
|
||||
const THREAD_SPAWN_CONVERSATION_PREFIX: &str = "thread-spawn:";
|
||||
const UNSUPPORTED_CODEX_RESPONSES_FIELDS: &[&str] = &["max_output_tokens", "temperature"];
|
||||
@@ -144,8 +147,16 @@ struct ExtractedRequestContext {
|
||||
|
||||
impl ExtractedRequestContext {
|
||||
fn from_body(body: &Map<String, Value>) -> Self {
|
||||
let client_metadata = body.get("client_metadata").and_then(Value::as_object);
|
||||
Self {
|
||||
turn_state: body_string(body, "turnState"),
|
||||
// 官方 downstream WebSocket 无法逐帧更新 HTTP header,因此把
|
||||
// response.metadata 返回的 turn state 放回下一帧 client_metadata。
|
||||
// Provider 仍会在账号/turn 归属确定后决定是否允许复用该状态。
|
||||
turn_state: body_string(body, "turnState").or_else(|| {
|
||||
client_metadata.and_then(|metadata| {
|
||||
string_value(metadata.get(X_CODEX_TURN_STATE_CLIENT_METADATA_KEY))
|
||||
})
|
||||
}),
|
||||
turn_metadata: body_string(body, "turnMetadata"),
|
||||
beta_features: body_string(body, "betaFeatures"),
|
||||
version: body_string(body, "version"),
|
||||
@@ -156,15 +167,15 @@ impl ExtractedRequestContext {
|
||||
session_id: body_string(body, "session_id"),
|
||||
thread_id: body_string(body, "thread_id"),
|
||||
client_request_id: body_string(body, "x-client-request-id"),
|
||||
turn_id: body_string(body, "turn_id"),
|
||||
turn_id: body_string(body, "turn_id").or_else(|| {
|
||||
client_metadata
|
||||
.and_then(|metadata| string_value(metadata.get(TURN_ID_CLIENT_METADATA_KEY)))
|
||||
}),
|
||||
// Responses Lite 在 WebSocket body 中的这个键是官方 header 投影,
|
||||
// 不是普通上下文字段的 metadata 回退。
|
||||
responses_lite: body
|
||||
.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| {
|
||||
string_value(metadata.get(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY))
|
||||
}),
|
||||
responses_lite: client_metadata.and_then(|metadata| {
|
||||
string_value(metadata.get(WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY))
|
||||
}),
|
||||
// Memory consolidation 只由官方请求头提供;body 中同名字段不参与
|
||||
// transport 事实提取。
|
||||
memgen_request: None,
|
||||
@@ -427,7 +438,11 @@ fn metadata_string(request: &CodexResponsesRequest, key: &str) -> Option<String>
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn scope_turn_metadata(raw: &str, installation_id: &str, cross_account: bool) -> Option<String> {
|
||||
pub(crate) fn scope_turn_metadata(
|
||||
raw: &str,
|
||||
installation_id: &str,
|
||||
cross_account: bool,
|
||||
) -> Option<String> {
|
||||
let Ok(Value::Object(mut metadata)) = serde_json::from_str::<Value>(raw) else {
|
||||
return (!cross_account).then(|| raw.to_owned());
|
||||
};
|
||||
@@ -507,6 +522,9 @@ fn apply_protocol_context(request: &mut CodexResponsesRequest, context: &Map<Str
|
||||
.or_else(|| request.include_timing_metrics.take());
|
||||
request.codex_window_id =
|
||||
context_string(context, "codex_window_id").or_else(|| request.codex_window_id.take());
|
||||
request.downstream_websocket_connection_id =
|
||||
context_string(context, "downstream_websocket_connection_id")
|
||||
.or_else(|| request.downstream_websocket_connection_id.take());
|
||||
request.parent_thread_id =
|
||||
context_string(context, "parent_thread_id").or_else(|| request.parent_thread_id.take());
|
||||
let prompt_cache_key = request.prompt_cache_key().map(ToOwned::to_owned);
|
||||
|
||||
@@ -29,6 +29,14 @@ use super::{
|
||||
|
||||
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
|
||||
|
||||
/// WebSocket 快路径的控制流结果;预算未命中不代表连接失败。
|
||||
pub(crate) enum WebSocketFastPath<T> {
|
||||
/// 在前台预算内获得结果。
|
||||
Ready(T),
|
||||
/// 前台停止等待;池化 opening 可以继续在后台完成。
|
||||
Missed,
|
||||
}
|
||||
|
||||
/// 尚未发送 `response.create` 的 WebSocket。只有该类型可以安全切换到 HTTP。
|
||||
pub(crate) struct PreparedWebSocket {
|
||||
connection: PooledWebSocketConnection,
|
||||
@@ -104,7 +112,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
|
||||
fast_path_budget: Option<Duration>,
|
||||
require_pool: bool,
|
||||
fallback_initial_event_timeout: Option<Duration>,
|
||||
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
|
||||
) -> Result<WebSocketFastPath<PreparedWebSocket>, CodexWebSocketExchangeError> {
|
||||
let decision_started_at = Instant::now();
|
||||
let Some((pool, key)) = pool else {
|
||||
if require_pool || !request.continuation().permits_fresh_connection() {
|
||||
@@ -142,7 +150,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
|
||||
PreviousResponseUnavailableReason::LatestResponseMismatch,
|
||||
));
|
||||
}
|
||||
Ok(PreparedWebSocket {
|
||||
Ok(WebSocketFastPath::Ready(PreparedWebSocket {
|
||||
connection: *connection,
|
||||
binding: PoolBinding::Pooled {
|
||||
lease: Box::new(lease),
|
||||
@@ -152,7 +160,7 @@ pub(crate) async fn prepare_response_create_request_with_pool(
|
||||
connect_elapsed: None,
|
||||
decision_wait_elapsed: decision_started_at.elapsed(),
|
||||
initial_event_timeout: pool.initial_event_timeout(),
|
||||
})
|
||||
}))
|
||||
}
|
||||
WebSocketPoolAcquire::Connect(connect_lease) => {
|
||||
if !request.continuation().permits_fresh_connection() {
|
||||
@@ -178,14 +186,14 @@ pub(crate) async fn prepare_response_create_request_with_pool(
|
||||
PreviousResponseUnavailableReason::ConnectionBusy,
|
||||
));
|
||||
}
|
||||
let outcome = wait_for_shared_connect(waiter, fast_path_budget).await?;
|
||||
match outcome {
|
||||
WebSocketPoolConnectOutcome::Ready => Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::ConnectionBusy,
|
||||
)),
|
||||
WebSocketPoolConnectOutcome::Failed | WebSocketPoolConnectOutcome::Pending => {
|
||||
Err(CodexWebSocketExchangeError::SharedConnectFailed)
|
||||
}
|
||||
match wait_for_shared_connect(waiter, fast_path_budget).await {
|
||||
WebSocketFastPath::Missed => Ok(WebSocketFastPath::Missed),
|
||||
WebSocketFastPath::Ready(WebSocketPoolConnectOutcome::Ready) => Err(
|
||||
continuation_unavailable(PreviousResponseUnavailableReason::ConnectionBusy),
|
||||
),
|
||||
WebSocketFastPath::Ready(
|
||||
WebSocketPoolConnectOutcome::Failed | WebSocketPoolConnectOutcome::Pending,
|
||||
) => Err(CodexWebSocketExchangeError::SharedConnectFailed),
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::ContinuationLost(observation) => Err(continuation_unavailable(
|
||||
@@ -203,7 +211,7 @@ async fn prepare_unpooled_websocket(
|
||||
fast_path_budget: Option<Duration>,
|
||||
initial_event_timeout: Option<Duration>,
|
||||
decision_started_at: Instant,
|
||||
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
|
||||
) -> Result<WebSocketFastPath<PreparedWebSocket>, CodexWebSocketExchangeError> {
|
||||
let permit = acquire_breaker_permit(breaker, origin_key)?;
|
||||
let context = pump_log_context_from_connection(request.connection());
|
||||
let connected = connect_with_budget(
|
||||
@@ -213,14 +221,22 @@ async fn prepare_unpooled_websocket(
|
||||
context,
|
||||
)
|
||||
.await;
|
||||
let connected = match connected {
|
||||
Ok(WebSocketFastPath::Ready(connection)) => Ok(connection),
|
||||
Ok(WebSocketFastPath::Missed) => {
|
||||
permit.fast_timeout();
|
||||
return Ok(WebSocketFastPath::Missed);
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
let (connection, connect_elapsed) = finish_breaker_attempt(permit, connected)?;
|
||||
Ok(PreparedWebSocket {
|
||||
Ok(WebSocketFastPath::Ready(PreparedWebSocket {
|
||||
connection,
|
||||
binding: PoolBinding::Unpooled,
|
||||
connect_elapsed: Some(connect_elapsed),
|
||||
decision_wait_elapsed: decision_started_at.elapsed(),
|
||||
initial_event_timeout,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
async fn prepare_pooled_websocket(
|
||||
@@ -231,7 +247,7 @@ async fn prepare_pooled_websocket(
|
||||
origin_key: &str,
|
||||
fast_path_budget: Option<Duration>,
|
||||
decision_started_at: Instant,
|
||||
) -> Result<PreparedWebSocket, CodexWebSocketExchangeError> {
|
||||
) -> Result<WebSocketFastPath<PreparedWebSocket>, CodexWebSocketExchangeError> {
|
||||
let permit = match acquire_breaker_permit(breaker, origin_key) {
|
||||
Ok(permit) => permit,
|
||||
Err(error) => {
|
||||
@@ -245,18 +261,20 @@ async fn prepare_pooled_websocket(
|
||||
connect_lease,
|
||||
permit,
|
||||
);
|
||||
let handoff = waiter.wait(fast_path_budget).await?;
|
||||
Ok(PreparedWebSocket {
|
||||
connection: *handoff.connection,
|
||||
binding: PoolBinding::Pooled {
|
||||
lease: Box::new(handoff.lease),
|
||||
reused: false,
|
||||
decision: WebSocketPoolDecision::new(),
|
||||
},
|
||||
connect_elapsed: Some(handoff.connect_elapsed),
|
||||
decision_wait_elapsed: decision_started_at.elapsed(),
|
||||
initial_event_timeout: pool.initial_event_timeout(),
|
||||
})
|
||||
match waiter.wait(fast_path_budget).await? {
|
||||
WebSocketFastPath::Ready(handoff) => Ok(WebSocketFastPath::Ready(PreparedWebSocket {
|
||||
connection: *handoff.connection,
|
||||
binding: PoolBinding::Pooled {
|
||||
lease: Box::new(handoff.lease),
|
||||
reused: false,
|
||||
decision: WebSocketPoolDecision::new(),
|
||||
},
|
||||
connect_elapsed: Some(handoff.connect_elapsed),
|
||||
decision_wait_elapsed: decision_started_at.elapsed(),
|
||||
initial_event_timeout: pool.initial_event_timeout(),
|
||||
})),
|
||||
WebSocketFastPath::Missed => Ok(WebSocketFastPath::Missed),
|
||||
}
|
||||
}
|
||||
|
||||
struct PooledWebSocketConnectWaiter {
|
||||
@@ -275,7 +293,7 @@ impl PooledWebSocketConnectWaiter {
|
||||
async fn wait(
|
||||
&mut self,
|
||||
fast_path_budget: Option<Duration>,
|
||||
) -> Result<PooledWebSocketHandoff, CodexWebSocketExchangeError> {
|
||||
) -> Result<WebSocketFastPath<PooledWebSocketHandoff>, CodexWebSocketExchangeError> {
|
||||
let received = match fast_path_budget {
|
||||
Some(budget) => {
|
||||
let remaining = budget.saturating_sub(self.started_at.elapsed());
|
||||
@@ -283,15 +301,15 @@ impl PooledWebSocketConnectWaiter {
|
||||
Ok(received) => received,
|
||||
Err(_) => {
|
||||
self.fast_path_reporter.missed();
|
||||
return Err(CodexWebSocketExchangeError::FastPathTimeout {
|
||||
timeout: budget,
|
||||
});
|
||||
return Ok(WebSocketFastPath::Missed);
|
||||
}
|
||||
}
|
||||
}
|
||||
None => (&mut self.receiver).await,
|
||||
};
|
||||
received.map_err(|_| CodexWebSocketExchangeError::SharedConnectFailed)?
|
||||
received
|
||||
.map_err(|_| CodexWebSocketExchangeError::SharedConnectFailed)?
|
||||
.map(WebSocketFastPath::Ready)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,7 +345,7 @@ fn start_pooled_websocket_connect(
|
||||
);
|
||||
return;
|
||||
}
|
||||
result = connect_with_budget(&connection, keepalive, None, context) => result,
|
||||
result = connect_websocket_connection(&connection, keepalive, context) => result,
|
||||
};
|
||||
match finish_breaker_attempt(permit, connected) {
|
||||
Ok((connection, connect_elapsed)) => {
|
||||
@@ -401,8 +419,7 @@ fn opening_failure_reason(error: &CodexWebSocketExchangeError) -> &'static str {
|
||||
CodexWebSocketExchangeError::InvalidRequest(_) => "request_build_failure",
|
||||
CodexWebSocketExchangeError::Connect(error)
|
||||
| CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
|
||||
CodexWebSocketExchangeError::ConnectTimeout { .. }
|
||||
| CodexWebSocketExchangeError::FastPathTimeout { .. } => "connect_timeout",
|
||||
CodexWebSocketExchangeError::ConnectTimeout { .. } => "connect_timeout",
|
||||
CodexWebSocketExchangeError::Upstream(_) => "upgrade_rejected",
|
||||
CodexWebSocketExchangeError::OriginCircuitOpen
|
||||
| CodexWebSocketExchangeError::OriginHalfOpenBusy => "connect_suppressed",
|
||||
@@ -432,18 +449,32 @@ async fn connect_with_budget(
|
||||
keepalive: PumpKeepalive,
|
||||
fast_path_budget: Option<Duration>,
|
||||
context: PumpLogContext,
|
||||
) -> Result<WebSocketFastPath<(PooledWebSocketConnection, Duration)>, CodexWebSocketExchangeError> {
|
||||
match fast_path_budget {
|
||||
Some(budget) => {
|
||||
let Ok(connected) = timeout(
|
||||
budget,
|
||||
connect_websocket_connection(connection, keepalive, context),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return Ok(WebSocketFastPath::Missed);
|
||||
};
|
||||
connected.map(WebSocketFastPath::Ready)
|
||||
}
|
||||
None => connect_websocket_connection(connection, keepalive, context)
|
||||
.await
|
||||
.map(WebSocketFastPath::Ready),
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect_websocket_connection(
|
||||
connection: &CodexWebSocketConnection,
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
) -> Result<(PooledWebSocketConnection, Duration), CodexWebSocketExchangeError> {
|
||||
let started_at = Instant::now();
|
||||
let connected = match fast_path_budget {
|
||||
Some(budget) => timeout(
|
||||
budget,
|
||||
connect_pumped_websocket(connection, keepalive, context),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })?,
|
||||
None => connect_pumped_websocket(connection, keepalive, context).await,
|
||||
}?;
|
||||
let (websocket, response) = connected;
|
||||
let (websocket, response) = connect_pumped_websocket(connection, keepalive, context).await?;
|
||||
Ok((
|
||||
PooledWebSocketConnection {
|
||||
websocket,
|
||||
@@ -477,10 +508,6 @@ fn finish_breaker_attempt(
|
||||
permit.succeed();
|
||||
Ok(connection)
|
||||
}
|
||||
Err(error @ CodexWebSocketExchangeError::FastPathTimeout { .. }) => {
|
||||
permit.fast_timeout();
|
||||
Err(error)
|
||||
}
|
||||
Err(CodexWebSocketExchangeError::Upstream(upstream)) if upstream.status_code < 500 => {
|
||||
// 账号或请求级 opening 响应证明 origin 可达,不得污染 transport 熔断器。
|
||||
permit.succeed();
|
||||
@@ -496,15 +523,16 @@ fn finish_breaker_attempt(
|
||||
async fn wait_for_shared_connect(
|
||||
waiter: pool::WebSocketPoolConnectWaiter,
|
||||
fast_path_budget: Option<Duration>,
|
||||
) -> Result<WebSocketPoolConnectOutcome, CodexWebSocketExchangeError> {
|
||||
) -> WebSocketFastPath<WebSocketPoolConnectOutcome> {
|
||||
match fast_path_budget {
|
||||
Some(budget) => {
|
||||
let remaining = waiter.remaining_budget(budget);
|
||||
timeout(remaining, waiter.wait())
|
||||
.await
|
||||
.map_err(|_| CodexWebSocketExchangeError::FastPathTimeout { timeout: budget })
|
||||
match timeout(remaining, waiter.wait()).await {
|
||||
Ok(outcome) => WebSocketFastPath::Ready(outcome),
|
||||
Err(_) => WebSocketFastPath::Missed,
|
||||
}
|
||||
}
|
||||
None => Ok(waiter.wait().await),
|
||||
None => WebSocketFastPath::Ready(waiter.wait().await),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,12 +31,6 @@ pub enum CodexWebSocketExchangeError {
|
||||
/// 建连超时时长。
|
||||
timeout: Duration,
|
||||
},
|
||||
/// 普通请求的 WebSocket 快路径预算耗尽,payload 尚未发送。
|
||||
#[error("websocket fast-path connect budget exhausted after {timeout:?}")]
|
||||
FastPathTimeout {
|
||||
/// 快路径等待时长。
|
||||
timeout: Duration,
|
||||
},
|
||||
/// origin WebSocket 冷建连熔断中。
|
||||
#[error("websocket origin circuit is open")]
|
||||
OriginCircuitOpen,
|
||||
@@ -341,7 +335,6 @@ impl CodexWebSocketExchangeError {
|
||||
self,
|
||||
Self::Connect(_)
|
||||
| Self::ConnectTimeout { .. }
|
||||
| Self::FastPathTimeout { .. }
|
||||
| Self::OriginCircuitOpen
|
||||
| Self::OriginHalfOpenBusy
|
||||
| Self::SharedConnectFailed
|
||||
|
||||
@@ -31,7 +31,7 @@ pub struct CodexWebSocketStreamingExchange {
|
||||
pub(crate) websocket_connection_id: Uuid,
|
||||
/// 由 WebSocket 事件转换出的 live SSE 字节流。
|
||||
pub body: CodexWebSocketSseStream,
|
||||
/// 上游 metadata 帧中的最新 turn state。
|
||||
/// 上游为本次响应返回的首个 turn state。
|
||||
pub turn_state: Option<String>,
|
||||
/// 上游握手响应里的 `set-cookie` 列表。
|
||||
pub set_cookie_headers: Vec<String>,
|
||||
@@ -39,7 +39,7 @@ pub struct CodexWebSocketStreamingExchange {
|
||||
pub rate_limit_headers: Vec<(String, String)>,
|
||||
/// 上游内部 `codex.rate_limits` 事件里的结构化动态更新。
|
||||
pub rate_limit_updates: CodexWebSocketRateLimitUpdates,
|
||||
/// 上游内部 metadata 事件里的动态 turn state。
|
||||
/// 上游内部 metadata 事件里的首个动态 turn state。
|
||||
pub turn_state_update: CodexWebSocketTurnStateUpdate,
|
||||
/// WebSocket 连接池决策。
|
||||
pub pool_decision: Option<WebSocketPoolDecision>,
|
||||
@@ -63,5 +63,10 @@ pub(super) fn reusable_websocket_metadata(
|
||||
mut metadata: CodexWebSocketConnectionMetadata,
|
||||
) -> CodexWebSocketConnectionMetadata {
|
||||
metadata.rate_limit_headers.clear();
|
||||
metadata.turn_state = None;
|
||||
metadata
|
||||
.response_metadata
|
||||
.client_headers
|
||||
.retain(|(name, _)| !name.eq_ignore_ascii_case("x-codex-turn-state"));
|
||||
metadata
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ pub(in crate::transport::websocket) enum WebSocketTerminalKind {
|
||||
|
||||
pub(super) enum ExchangeAction {
|
||||
RateLimits(events::ParsedRateLimits),
|
||||
TurnState(String),
|
||||
Forward {
|
||||
frame: String,
|
||||
terminal: Option<WebSocketTerminalKind>,
|
||||
@@ -32,6 +31,7 @@ pub(super) enum ExchangeAction {
|
||||
pub(super) struct ReducedWebSocketEvent {
|
||||
pub(super) action: ExchangeAction,
|
||||
pub(super) diagnostic_event_type: Option<String>,
|
||||
pub(super) turn_state_update: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn reduce_websocket_event(
|
||||
@@ -45,6 +45,7 @@ pub(super) fn reduce_websocket_event(
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::Ignore,
|
||||
diagnostic_event_type: None,
|
||||
turn_state_update: None,
|
||||
});
|
||||
};
|
||||
let diagnostic_event_type = diagnostic_event_type(websocket_event_type(&value));
|
||||
@@ -54,6 +55,7 @@ pub(super) fn reduce_websocket_event(
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::RateLimits(parsed),
|
||||
diagnostic_event_type,
|
||||
turn_state_update: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -61,13 +63,13 @@ pub(super) fn reduce_websocket_event(
|
||||
&mut metadata.response_metadata,
|
||||
websocket_metadata_headers(&value),
|
||||
);
|
||||
if let Some(turn_state) = websocket_metadata_turn_state(&value) {
|
||||
let turn_state_update = websocket_metadata_turn_state(&value).and_then(|turn_state| {
|
||||
if metadata.turn_state.is_some() {
|
||||
return None;
|
||||
}
|
||||
metadata.turn_state = Some(turn_state.clone());
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::TurnState(turn_state),
|
||||
diagnostic_event_type,
|
||||
});
|
||||
}
|
||||
Some(turn_state)
|
||||
});
|
||||
|
||||
let event = websocket_event_type(&value);
|
||||
if event == Some("response.completed")
|
||||
@@ -89,6 +91,7 @@ pub(super) fn reduce_websocket_event(
|
||||
Ok(ReducedWebSocketEvent {
|
||||
action,
|
||||
diagnostic_event_type,
|
||||
turn_state_update,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -271,15 +271,17 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
if let Some(event_type) = reduced.diagnostic_event_type {
|
||||
last_event_type = Some(event_type);
|
||||
}
|
||||
if let Some(turn_state) = reduced.turn_state_update {
|
||||
let mut pending = turn_state_update.lock().await;
|
||||
if pending.is_none() {
|
||||
*pending = Some(turn_state);
|
||||
}
|
||||
}
|
||||
let (frame, terminal) = match reduced.action {
|
||||
ExchangeAction::RateLimits(rate_limits) => {
|
||||
rate_limit_updates.lock().await.push(rate_limits);
|
||||
continue;
|
||||
}
|
||||
ExchangeAction::TurnState(turn_state) => {
|
||||
*turn_state_update.lock().await = Some(turn_state);
|
||||
continue;
|
||||
}
|
||||
ExchangeAction::Forward { frame, terminal } => (frame, terminal),
|
||||
ExchangeAction::Ignore => continue,
|
||||
};
|
||||
|
||||
@@ -37,7 +37,7 @@ pub use self::{
|
||||
};
|
||||
pub(crate) use self::{
|
||||
coordinator::{
|
||||
PreparedWebSocket, WEBSOCKET_FAST_PATH_BUDGET,
|
||||
PreparedWebSocket, WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath,
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool,
|
||||
},
|
||||
|
||||
@@ -23,6 +23,7 @@ pub struct CodexWebSocketPoolKey {
|
||||
account_id: String,
|
||||
conversation_id: String,
|
||||
connection_profile: String,
|
||||
downstream_connection_id: String,
|
||||
}
|
||||
|
||||
impl CodexWebSocketPoolKey {
|
||||
@@ -37,6 +38,7 @@ impl CodexWebSocketPoolKey {
|
||||
account_id: account_id.into(),
|
||||
conversation_id: conversation_id.into(),
|
||||
connection_profile: String::new(),
|
||||
downstream_connection_id: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +48,15 @@ impl CodexWebSocketPoolKey {
|
||||
self
|
||||
}
|
||||
|
||||
/// 隔离同一逻辑会话中由不同下游 WebSocket 驱动的并发响应链。
|
||||
pub(crate) fn with_downstream_connection_id(
|
||||
mut self,
|
||||
connection_id: impl Into<String>,
|
||||
) -> Self {
|
||||
self.downstream_connection_id = connection_id.into();
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn account_id(&self) -> &str {
|
||||
&self.account_id
|
||||
}
|
||||
@@ -60,6 +71,7 @@ impl CodexWebSocketPoolKey {
|
||||
self.account_id.as_str(),
|
||||
self.conversation_id.as_str(),
|
||||
self.connection_profile.as_str(),
|
||||
self.downstream_connection_id.as_str(),
|
||||
])
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -295,8 +295,8 @@ fn new_chain_should_allow_pre_delivery_http_fallback() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sticky_http_fallback_should_only_apply_to_new_chains() {
|
||||
assert!(TransportRequirement::NewChain.allows_sticky_http_fallback());
|
||||
fn session_transport_recovery_should_only_apply_to_new_chains() {
|
||||
assert!(TransportRequirement::NewChain.allows_session_transport_recovery());
|
||||
|
||||
for requirement in [
|
||||
TransportRequirement::HttpRequired,
|
||||
@@ -306,8 +306,8 @@ fn sticky_http_fallback_should_only_apply_to_new_chains() {
|
||||
TransportRequirement::ExternalUnknown,
|
||||
] {
|
||||
assert!(
|
||||
!requirement.allows_sticky_http_fallback(),
|
||||
"sticky HTTP must not override {}",
|
||||
!requirement.allows_session_transport_recovery(),
|
||||
"session recovery must not override {}",
|
||||
requirement.as_str()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -155,6 +155,7 @@ fn websocket_event_to_sse_should_forward_public_events_and_strip_internal_events
|
||||
);
|
||||
assert!(websocket_event_to_sse_frame(r#"{"type":"codex.rate_limits"}"#).is_none());
|
||||
assert!(websocket_event_to_sse_frame(r#"{"type":"response.metadata"}"#).is_none());
|
||||
assert!(websocket_event_to_sse_frame(r#"{"type":"codex.response.metadata"}"#).is_some());
|
||||
assert!(websocket_event_to_sse_frame(r#"{"response":{}}"#).is_none());
|
||||
assert!(websocket_event_to_sse_frame("not-json").is_none());
|
||||
}
|
||||
@@ -178,15 +179,48 @@ fn websocket_event_to_sse_should_add_missing_previous_response_recovery_code() {
|
||||
|
||||
#[test]
|
||||
fn websocket_metadata_turn_state_should_accept_case_insensitive_header() {
|
||||
let event = json!({
|
||||
"type": "response.metadata",
|
||||
"headers": {"X-Codex-Turn-State": ["turn-from-metadata"]}
|
||||
});
|
||||
for (event, expected) in [
|
||||
(
|
||||
json!({
|
||||
"type": "response.metadata",
|
||||
"headers": {"X-Codex-Turn-State": ["turn-from-legacy-metadata"]}
|
||||
}),
|
||||
"turn-from-legacy-metadata",
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "codex.response.metadata",
|
||||
"headers": {"X-Codex-Turn-State": "turn-from-codex-metadata"}
|
||||
}),
|
||||
"turn-from-codex-metadata",
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
websocket_metadata_turn_state(&event).as_deref(),
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
websocket_metadata_turn_state(&event).as_deref(),
|
||||
Some("turn-from-metadata")
|
||||
);
|
||||
#[test]
|
||||
fn websocket_metadata_turn_state_should_ignore_unverified_shapes() {
|
||||
for event in [
|
||||
json!({
|
||||
"type": "codex.response.metadata",
|
||||
"headers": {"x-codex-turn-state": {"future": "shape"}}
|
||||
}),
|
||||
json!({
|
||||
"type": "codex.response.metadata",
|
||||
"metadata": {"turn_state": "nested-value"},
|
||||
"turn_state": "top-level-value"
|
||||
}),
|
||||
json!({
|
||||
"type": "future.response.metadata",
|
||||
"headers": {"x-codex-turn-state": "unverified-event"}
|
||||
}),
|
||||
] {
|
||||
assert_eq!(websocket_metadata_turn_state(&event), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -227,6 +227,26 @@ fn encoder_should_ignore_legacy_context_aliases_and_metadata_fallbacks() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_extract_official_websocket_context_projection() {
|
||||
let request = request(Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), json!("prompt")),
|
||||
(
|
||||
"client_metadata".to_owned(),
|
||||
json!({
|
||||
"x-codex-turn-state": "metadata-turn-state",
|
||||
"turn_id": "metadata-turn-id"
|
||||
}),
|
||||
),
|
||||
]));
|
||||
|
||||
let encoded = encode_generate_request(&request, "gpt-test").expect("encode");
|
||||
|
||||
assert_eq!(encoded.turn_state.as_deref(), Some("metadata-turn-state"));
|
||||
assert_eq!(encoded.client_turn_id.as_deref(), Some("metadata-turn-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_context_should_win_over_body_topline_aliases() {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
@@ -236,6 +256,13 @@ fn header_context_should_win_over_body_topline_aliases() {
|
||||
("input".to_owned(), json!("prompt")),
|
||||
("turnState".to_owned(), json!("body-turn-state")),
|
||||
("turnMetadata".to_owned(), json!("body-turn-metadata")),
|
||||
(
|
||||
"client_metadata".to_owned(),
|
||||
json!({
|
||||
"x-codex-turn-state": "metadata-turn-state",
|
||||
"turn_id": "metadata-turn-id"
|
||||
}),
|
||||
),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload")
|
||||
@@ -248,6 +275,10 @@ fn header_context_should_win_over_body_topline_aliases() {
|
||||
"turn_metadata".to_owned(),
|
||||
Value::String("header-turn-metadata".to_owned()),
|
||||
),
|
||||
(
|
||||
"turn_id".to_owned(),
|
||||
Value::String("header-turn-id".to_owned()),
|
||||
),
|
||||
]));
|
||||
let request = GenerateRequest::from_protocol_payload(payload);
|
||||
|
||||
@@ -258,6 +289,36 @@ fn header_context_should_win_over_body_topline_aliases() {
|
||||
encoded.turn_metadata.as_deref(),
|
||||
Some("header-turn-metadata")
|
||||
);
|
||||
assert_eq!(encoded.client_turn_id.as_deref(), Some("header-turn-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_preserve_downstream_websocket_connection_identity_outside_wire_body() {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), json!("prompt")),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"downstream_websocket_connection_id".to_owned(),
|
||||
Value::String("ws_downstream_a".to_owned()),
|
||||
)]));
|
||||
let request = GenerateRequest::from_protocol_payload(payload);
|
||||
|
||||
let encoded = encode_generate_request(&request, "gpt-test").expect("encode");
|
||||
|
||||
assert_eq!(
|
||||
encoded.downstream_websocket_connection_id.as_deref(),
|
||||
Some("ws_downstream_a")
|
||||
);
|
||||
assert!(
|
||||
!encoded
|
||||
.body()
|
||||
.contains_key("downstream_websocket_connection_id")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -914,6 +914,191 @@ async fn websocket_execute_response_create_request_should_capture_internal_metad
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_codex_metadata_should_capture_first_turn_state_without_changing_forwarding() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _message = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "codex.response.metadata",
|
||||
"headers": {"x-codex-turn-state": "turn-first"}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.metadata",
|
||||
"headers": {"x-codex-turn-state": ["turn-second"]}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_metadata_alias", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let prepared = prepared_websocket_request(&format!("http://{addr}"));
|
||||
|
||||
let response = execute_response_create_request(&prepared)
|
||||
.await
|
||||
.expect("metadata aliases should preserve a completed response");
|
||||
server.await.unwrap();
|
||||
|
||||
assert_eq!(response.turn_state.as_deref(), Some("turn-first"));
|
||||
assert!(response.body.contains("event: codex.response.metadata"));
|
||||
assert!(!response.body.contains("event: response.metadata\n"));
|
||||
assert!(response.body.contains("event: response.completed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_websocket_should_not_leak_turn_state_into_the_next_response() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _first = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "codex.response.metadata",
|
||||
"headers": {"x-codex-turn-state": "turn-from-first-response"}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_turn_state_first", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let _second = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_turn_state_second", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(pool);
|
||||
let request = pooled_websocket_request("conversation-turn-state-reset");
|
||||
|
||||
let first = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_turn_state_first", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("first response should complete");
|
||||
let second = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_turn_state_second", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("second response should reuse the WebSocket");
|
||||
server.await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
first.turn_state.as_deref(),
|
||||
Some("turn-from-first-response")
|
||||
);
|
||||
assert!(
|
||||
second
|
||||
.websocket_pool_decision
|
||||
.is_some_and(WebSocketPoolDecision::is_reuse)
|
||||
);
|
||||
assert_eq!(second.turn_state, None);
|
||||
assert!(
|
||||
second
|
||||
.response_metadata
|
||||
.client_headers
|
||||
.iter()
|
||||
.all(|(name, _)| { !name.eq_ignore_ascii_case("x-codex-turn-state") })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_request_should_project_trusted_turn_state_and_remove_stale_metadata() {
|
||||
for (case, turn_state) in [("trusted", Some("trusted-turn-state")), ("cleared", None)] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let message = websocket.next().await.unwrap().unwrap();
|
||||
let payload = serde_json::from_str::<Value>(&message.into_text().unwrap())
|
||||
.expect("response.create payload");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_turn_state_projection", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
payload
|
||||
});
|
||||
let mut body = codex_request_body("gpt-5.5", "be brief", Vec::new());
|
||||
body.insert("generate".to_owned(), json!(false));
|
||||
body.insert("store".to_owned(), json!(false));
|
||||
let mut request = CodexResponsesRequest::from_body(body);
|
||||
request.turn_state = turn_state.map(str::to_owned);
|
||||
request.local_conversation_id = Some(format!("turn-state-projection-{case}"));
|
||||
request.set_client_metadata(Some(json!({
|
||||
"x-codex-turn-state": "stale-client-value",
|
||||
"keep": "preserved"
|
||||
})));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::new(CodexWebSocketPool::new(Duration::from_mins(1))));
|
||||
let mut context = request_context(case, Some("chatgpt-account"));
|
||||
context.turn_state = turn_state;
|
||||
|
||||
let response = backend
|
||||
.create_response(&request, context)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("required WebSocket request should complete: {error}"));
|
||||
let payload = server.await.unwrap();
|
||||
|
||||
assert_eq!(response.transport, CodexBackendTransport::WebSocket);
|
||||
assert_eq!(
|
||||
payload.pointer("/client_metadata/keep"),
|
||||
Some(&json!("preserved"))
|
||||
);
|
||||
assert_eq!(
|
||||
payload.pointer("/client_metadata/x-codex-turn-state"),
|
||||
turn_state.map(|turn_state| json!(turn_state)).as_ref(),
|
||||
"case {case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_not_reuse_websocket_rate_limit_headers() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -71,6 +71,138 @@ async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_pool_should_isolate_concurrent_downstream_lanes_for_one_conversation() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accepted_connections = Arc::new(AtomicUsize::new(0));
|
||||
let accepted_connections_for_server = Arc::clone(&accepted_connections);
|
||||
let (first_accepted_tx, first_accepted_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (first_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut first = accept_codex_test_websocket(first_stream).await;
|
||||
first_accepted_tx.send(()).unwrap();
|
||||
|
||||
let (second_stream, _) = timeout(Duration::from_secs(2), listener.accept())
|
||||
.await
|
||||
.expect("the second downstream lane should open its own upstream websocket")
|
||||
.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
let mut second = accept_codex_test_websocket(second_stream).await;
|
||||
|
||||
let (first_warmup, second_warmup) = tokio::join!(first.next(), second.next());
|
||||
first_warmup.unwrap().unwrap();
|
||||
second_warmup.unwrap().unwrap();
|
||||
first
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_lane_a", 1, 0).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
second
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_lane_b", 1, 0).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let _first_continuation = first.next().await.unwrap().unwrap();
|
||||
first
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_lane_a_next", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let _second_continuation = second.next().await.unwrap().unwrap();
|
||||
second
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_lane_b_next", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
first.close(None).await.unwrap();
|
||||
second.close(None).await.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let warmup = |connection_id: &str| {
|
||||
let mut body = Map::new();
|
||||
body.insert("model".to_owned(), json!("gpt-5.5"));
|
||||
body.insert("input".to_owned(), json!([]));
|
||||
body.insert("generate".to_owned(), json!(false));
|
||||
body.insert("store".to_owned(), json!(false));
|
||||
let mut request = CodexResponsesRequest::from_body(body);
|
||||
request.use_websocket = true;
|
||||
request.local_conversation_id = Some("conversation-lanes".to_owned());
|
||||
request.downstream_websocket_connection_id = Some(connection_id.to_owned());
|
||||
request
|
||||
};
|
||||
|
||||
let first_backend = backend.clone();
|
||||
let first_request = warmup("ws_downstream_a");
|
||||
let first = tokio::spawn(async move {
|
||||
first_backend
|
||||
.create_response(
|
||||
&first_request,
|
||||
request_context("req_lane_a_warmup", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
});
|
||||
first_accepted_rx.await.unwrap();
|
||||
let second = backend
|
||||
.create_response(
|
||||
&warmup("ws_downstream_b"),
|
||||
request_context("req_lane_b_warmup", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("the second downstream lane should not contend with the first warmup");
|
||||
let first = first.await.unwrap().unwrap();
|
||||
|
||||
let mut first_continuation = pooled_websocket_request("conversation-lanes");
|
||||
first_continuation.downstream_websocket_connection_id = Some("ws_downstream_a".to_owned());
|
||||
first_continuation.set_previous_response_id(Some("resp_lane_a".to_owned()));
|
||||
first_continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
let first_continuation = backend
|
||||
.create_response(
|
||||
&first_continuation,
|
||||
request_context("req_lane_a_next", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("the first lane should retain its connection-local response owner");
|
||||
|
||||
let mut second_continuation = pooled_websocket_request("conversation-lanes");
|
||||
second_continuation.downstream_websocket_connection_id = Some("ws_downstream_b".to_owned());
|
||||
second_continuation.set_previous_response_id(Some("resp_lane_b".to_owned()));
|
||||
second_continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
let second_continuation = backend
|
||||
.create_response(
|
||||
&second_continuation,
|
||||
request_context("req_lane_b_next", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("the second lane should retain its connection-local response owner");
|
||||
server.await.unwrap();
|
||||
|
||||
assert_eq!(first.websocket_pool_decision.unwrap().kind(), "new");
|
||||
assert_eq!(second.websocket_pool_decision.unwrap().kind(), "new");
|
||||
assert_eq!(
|
||||
first_continuation.websocket_pool_decision.unwrap().kind(),
|
||||
"reuse"
|
||||
);
|
||||
assert_eq!(
|
||||
second_continuation.websocket_pool_decision.unwrap().kind(),
|
||||
"reuse"
|
||||
);
|
||||
assert_eq!(accepted_connections.load(Ordering::SeqCst), 2);
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_continuation_reuses_owning_socket_after_connection_profile_updates() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -3,3 +3,4 @@
|
||||
1b6ffab434c754814aebb0eee3b650bd9770010059472bccf5bc856b319880ca 0003_client_min_versions.sql
|
||||
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
|
||||
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
|
||||
3b4bfd5e3f5135fa2fad1088b0c2a287861e1250d66af31dbd40dc890751b567 0006_openai_session_transport_recovery.sql
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
create index model_requests_pending_ws_transport_recovery_idx
|
||||
on model_requests (
|
||||
client_api_key_ref,
|
||||
continuation_affinity_hash,
|
||||
completed_at desc,
|
||||
id desc
|
||||
)
|
||||
where provider_kind = 'openai'
|
||||
and outcome = 'failed'
|
||||
and error_kind = 'upstream_unavailable'
|
||||
and upstream_transport = 'websocket'
|
||||
and upstream_send_state = 'ambiguous'
|
||||
and downstream_committed_at is null
|
||||
and recovered_at is null
|
||||
and continuation_affinity_hash is not null;
|
||||
|
||||
with session_transport_recoveries as (
|
||||
select
|
||||
prior.id as failed_request_id,
|
||||
recovered.id as recovery_request_id,
|
||||
recovered.started_at as recovery_started_at,
|
||||
recovered.completed_at as recovered_at
|
||||
from model_requests prior
|
||||
cross join lateral (
|
||||
select current.id, current.started_at, current.completed_at
|
||||
from model_requests current
|
||||
where current.client_api_key_ref = prior.client_api_key_ref
|
||||
and current.continuation_affinity_hash = prior.continuation_affinity_hash
|
||||
and current.provider_kind = prior.provider_kind
|
||||
and current.outcome = 'succeeded'
|
||||
and current.upstream_transport = 'http_sse'
|
||||
and current.completed_at is not null
|
||||
and current.started_at >= prior.completed_at
|
||||
and current.started_at <= prior.completed_at + interval '30 seconds'
|
||||
order by current.started_at, current.id
|
||||
limit 1
|
||||
) recovered
|
||||
where prior.provider_kind = 'openai'
|
||||
and prior.outcome = 'failed'
|
||||
and prior.error_kind = 'upstream_unavailable'
|
||||
and prior.upstream_transport = 'websocket'
|
||||
and prior.upstream_send_state = 'ambiguous'
|
||||
and prior.downstream_committed_at is null
|
||||
and prior.recovered_at is null
|
||||
and prior.continuation_affinity_hash is not null
|
||||
)
|
||||
update model_requests prior
|
||||
set recovery_attempt_count = prior.recovery_attempt_count + 1,
|
||||
recovery_request_id = recovery.recovery_request_id,
|
||||
recovered_at = recovery.recovered_at,
|
||||
recovery_retry_delay_ms = greatest(
|
||||
0,
|
||||
floor(extract(epoch from (recovery.recovery_started_at - prior.completed_at)) * 1000)
|
||||
)::bigint,
|
||||
recovery_total_latency_ms = greatest(
|
||||
0,
|
||||
floor(extract(epoch from (recovery.recovered_at - prior.completed_at)) * 1000)
|
||||
)::bigint
|
||||
from session_transport_recoveries recovery
|
||||
where prior.id = recovery.failed_request_id
|
||||
and prior.recovered_at is null;
|
||||
@@ -20,6 +20,8 @@ host:
|
||||
logging:
|
||||
level: 'info'
|
||||
stdout: true
|
||||
# 将完整请求头和正文写入独立文件;包含凭据,仅排障时开启。
|
||||
request_dump: false
|
||||
file:
|
||||
enabled: true
|
||||
# 相对路径以 deploy/config.yaml 所在目录为基准。
|
||||
|
||||
+11
-6
@@ -108,20 +108,22 @@ mutation 请求不要求客户端提供全局配置版本。会改变路由快
|
||||
|
||||
## 3. OpenAI 数据面与模型目录
|
||||
|
||||
Responses 和 Images HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;协议可接受性由上游决定。
|
||||
Responses、Images 和 standalone Search HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;
|
||||
协议可接受性由上游决定。
|
||||
|
||||
| 方法 | 路由 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `POST` | `/v1/responses` | OpenAI Responses JSON;`stream=true` 返回 SSE,否则返回完整 JSON |
|
||||
| `GET` | `/v1/responses` | 通过 HTTP Upgrade 建立 Responses WebSocket |
|
||||
| `POST` | `/v1/responses/review` | 使用同一 Responses 合同发起 review 子代理请求 |
|
||||
| `POST` | `/v1/alpha/search` | Codex standalone web search;JSON 请求与响应正文原样转发 |
|
||||
| `POST` | `/v1/images/generations` | 通过 OpenAI Provider 发起图像生成;JSON 请求与响应正文原样转发 |
|
||||
| `POST` | `/v1/images/edits` | 通过 OpenAI Provider 发起图像编辑;JSON 请求与响应正文原样转发 |
|
||||
| `GET` | `/v1/models` | 返回当前 Client Key 账号范围内各 Provider 的可用公开模型并集;有两种响应形态,见下 |
|
||||
| `GET` | `/v1/models/catalog` | 返回 Codex 客户端使用的模型目录 |
|
||||
| `GET` | `/v1/models/{model_id}/info` | 返回 Codex 客户端使用的单模型信息 |
|
||||
| `GET` | `/v1/models/{model_id}` | 返回 OpenAI 兼容的单模型详情 |
|
||||
|
||||
Codex 的 review 等子代理请求仍使用 `/v1/responses`,并通过 `x-openai-subagent` 请求头携带子代理类型;
|
||||
网关不提供独立的子代理请求路径。
|
||||
|
||||
`GET /v1/models` 默认返回 OpenAI 兼容列表 `{"object": "list", "data": [...]}`;请求携带非空
|
||||
`client_version` query 参数(Codex 客户端)时改为返回 Codex 专用目录合同 `{"models": [...]}`。
|
||||
|
||||
@@ -129,8 +131,11 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段
|
||||
映射除外),HTTP SSE 与 WebSocket 的上游业务事件字节原样转发,response ID 按 opaque 值处理而不
|
||||
假设 UUID 或固定长度;OpenAI 上游错误 envelope 和允许下发的 opaque header 值也不由 canonical
|
||||
观测结果重写。Images 请求不读取或重建 JSON,也不要求或映射模型字段;它固定使用 OpenAI Provider,
|
||||
只在原始字节之外完成账号选择、鉴权头替换和端点路由,成功与失败响应正文同样保持原始字节。xAI 是
|
||||
Grok wire 与 Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
|
||||
只在原始字节之外完成账号选择、鉴权头替换和端点路由,成功与失败响应正文同样保持原始字节。
|
||||
`/v1/alpha/search` 使用相同的 OpenAI Provider 原生端点边界:body(包括 `model`)不解析、不映射,
|
||||
`x-codex-turn-metadata` 在移除客户端账号身份并按当前 lease 重写 installation ID 后转发;上游账号
|
||||
Authorization、Cookie、account ID、originator 和 User-Agent 均由代理安全重建。xAI 是 Grok wire 与
|
||||
Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
|
||||
上游结构化错误的 message/code/type 会透传给客户端,其中内嵌的账号指纹 UUID 已脱敏。模型映射是
|
||||
全局精确映射,未命中时模型名原样交给候选 Provider;分组只限定账号集合,不参与模型改名。
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
Codex Proxy RS 是单进程、单副本运行的多 Provider AI 网关,同时提供:
|
||||
|
||||
- 面向客户端的 OpenAI Responses、Images 和模型目录协议;
|
||||
- 面向客户端的 OpenAI Responses、Images、standalone Search 和模型目录协议;
|
||||
- 面向管理员的 `/api/admin/*` 控制面和 Vue 管理端;
|
||||
- OpenAI 与 xAI 两个编译期 Provider;
|
||||
- PostgreSQL 持久化、Redis 协调状态以及 S3/R2 数据库备份。
|
||||
@@ -61,7 +61,7 @@ flowchart LR
|
||||
| `gateway-api` | HTTP/WS/SSE 解码与交付、Admin wire、静态管理端;不直接访问 Store 或具体 Provider |
|
||||
| `gateway-store` | PostgreSQL、Redis、S3/R2、`pg_dump` 适配器;不拥有业务策略 |
|
||||
| `gateway-host` | 配置加载、日志、HTTP 生命周期、Worker 监督和系统更新 |
|
||||
| `providers/openai` | OpenAI OAuth、账号选择、目录、额度、Responses/Images transport |
|
||||
| `providers/openai` | OpenAI OAuth、账号选择、目录、额度、Responses/Images/Search transport |
|
||||
| `providers/xai` | xAI OAuth session、账号选择、目录、额度和 Grok/Responses 转换 |
|
||||
| `frontend` | Vue 管理端,仅通过 Admin API 读写状态 |
|
||||
|
||||
@@ -151,16 +151,17 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别 Codex Desktop/
|
||||
- 可恢复观测写入失败不能替换已经确定的客户端协议结果。
|
||||
|
||||
Responses 按模型目录编译候选;全局模型映射是精确映射,未命中时模型名原样交给候选 Provider。
|
||||
Images 是 OpenAI Provider 自有端点:它不要求模型字段、不参与文本模型映射,只在 Client Key 的账号范围
|
||||
确实包含 OpenAI 账号时生成单一 OpenAI 候选。
|
||||
Images 与 standalone Search 是 OpenAI Provider 自有端点:两者都不参与文本模型映射,只在 Client Key
|
||||
的账号范围确实包含 OpenAI 账号时生成单一 OpenAI 候选。Images 不要求模型字段;Search body 中的模型
|
||||
及其他字段保持原始 bytes 并由上游解释。
|
||||
|
||||
## 5. Provider 与协议边界
|
||||
|
||||
Core 只理解 `Operation`、能力要求、Provider 候选、稳定错误和 canonical event,不读取 Provider SDK
|
||||
类型。Provider 独占 credential schema、OAuth、账号选择、模型目录、额度投影和上游 transport。
|
||||
|
||||
- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序;SSE、WebSocket 与 Images 的业务正文
|
||||
按原始字节转发。canonical facts 从同一数据旁路提取,只用于路由、观测和计费。
|
||||
- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序;SSE、WebSocket、Images 与 standalone
|
||||
Search 的业务正文按原始字节转发。canonical facts 从同一数据旁路提取,只用于路由、观测和计费。
|
||||
- xAI 是翻译边界。Provider 把 Grok wire 转换为 Responses wire;上游结构化错误的 message/code/type
|
||||
可以透出,但账号指纹会先脱敏。
|
||||
- response ID 是不透明 UTF-8 bytes,不假设 UUID、固定长度或跨 Provider 可复用。
|
||||
|
||||
@@ -83,6 +83,8 @@ export interface AccountUsage {
|
||||
outputTokensDisplay: string
|
||||
cachedTokens: number | null
|
||||
cachedTokensDisplay: string
|
||||
reasoningTokens: number | null
|
||||
reasoningTokensDisplay: string
|
||||
imageInputTokens: number | null
|
||||
imageInputTokensDisplay: string
|
||||
imageOutputTokens: number | null
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { Key, Openai, Xai } from '@boxicons/vue'
|
||||
import { Key, LinkAlt, Openai, Xai } from '@boxicons/vue'
|
||||
import { computed } from 'vue'
|
||||
import { formatProviderLabel } from '@/utils/providers'
|
||||
|
||||
@@ -38,6 +38,8 @@ const providerLabel = computed(() => formatProviderLabel(props.provider, '未知
|
||||
const authenticationLabel = computed(() => {
|
||||
if (normalizedAuthenticationKind.value === 'oauth')
|
||||
return 'OAuth'
|
||||
if (normalizedAuthenticationKind.value === 'api_key')
|
||||
return 'API Key'
|
||||
return props.authenticationKind?.trim() || '未知认证类型'
|
||||
})
|
||||
</script>
|
||||
@@ -60,7 +62,8 @@ const authenticationLabel = computed(() => {
|
||||
:class="iconContainerClass"
|
||||
:title="authenticationLabel"
|
||||
>
|
||||
<Key v-if="normalizedAuthenticationKind === 'oauth'" :class="iconClass" />
|
||||
<LinkAlt v-if="normalizedAuthenticationKind === 'oauth'" :class="iconClass" />
|
||||
<Key v-else-if="normalizedAuthenticationKind === 'api_key'" :class="iconClass" />
|
||||
<span v-else class="text-[10px] font-heavy text-cp-text-quaternary">?</span>
|
||||
</span>
|
||||
|
||||
|
||||
@@ -45,7 +45,8 @@ const thumbClasses = computed(() => [
|
||||
model.value ? 'left-[calc(100%-1.375rem)]' : 'left-0.5',
|
||||
])
|
||||
const inlineTextClasses = computed(() => [
|
||||
'pointer-events-none absolute inset-0 flex min-w-0 select-none items-center justify-center overflow-hidden text-[10px] leading-none font-heavy text-cp-white transition-[padding] duration-180 motion-reduce:transition-none',
|
||||
'pointer-events-none absolute inset-0 flex min-w-0 select-none items-center justify-center overflow-hidden text-[10px] leading-none font-heavy transition-[padding,color] duration-180 motion-reduce:transition-none',
|
||||
model.value ? 'text-cp-white' : 'text-cp-text-secondary',
|
||||
model.value ? 'pr-6 pl-1' : 'pr-1 pl-6',
|
||||
])
|
||||
const inactiveTextClasses = computed(() => [
|
||||
|
||||
@@ -6,6 +6,7 @@ import { groupedAccountQuotaWindows, visibleSummaryQuotaWindows } from '../../co
|
||||
import AccountUsageWindow from '../AccountUsageWindow/index.vue'
|
||||
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
|
||||
import AccountQuotaSummaryEntry from './Entry.vue'
|
||||
import { recentlyUsedQuotaEntry, representativeQuotaWindow } from './presenter'
|
||||
|
||||
const props = defineProps<{
|
||||
account: AccountRow
|
||||
@@ -15,26 +16,15 @@ const quotaWindows = computed(() => props.account.quota.windows)
|
||||
const visibleQuotaWindows = computed(() => visibleSummaryQuotaWindows(quotaWindows.value))
|
||||
const summaryEntries = computed(() => groupedAccountQuotaWindows(visibleQuotaWindows.value))
|
||||
const hasUsage = computed(() => (props.account.usage.requestCount ?? 0) > 0)
|
||||
const highestUsageWindow = computed(() => visibleQuotaWindows.value.reduce((highest, window) => {
|
||||
if (typeof window.usedPercent !== 'number')
|
||||
return highest
|
||||
if (!highest || typeof highest.usedPercent !== 'number' || window.usedPercent > highest.usedPercent)
|
||||
return window
|
||||
return highest
|
||||
}, undefined as (typeof visibleQuotaWindows.value)[number] | undefined))
|
||||
const highestUsageDisplay = computed(() => highestUsageWindow.value?.usedPercentDisplay ?? '—')
|
||||
const highestUsageTextClass = computed(() => highestUsageWindow.value
|
||||
? quotaWindowPresentation(highestUsageWindow.value, '2px').percentTextClass
|
||||
const recentUsageEntry = computed(() => recentlyUsedQuotaEntry(
|
||||
summaryEntries.value,
|
||||
props.account.usage.models,
|
||||
))
|
||||
const currentUsageWindow = computed(() => representativeQuotaWindow(recentUsageEntry.value))
|
||||
const currentUsageDisplay = computed(() => currentUsageWindow.value?.usedPercentDisplay ?? '—')
|
||||
const currentUsageTextClass = computed(() => currentUsageWindow.value
|
||||
? quotaWindowPresentation(currentUsageWindow.value, '2px').percentTextClass
|
||||
: 'text-cp-text-quaternary')
|
||||
const highestUsageEntry = computed(() => {
|
||||
const highestWindow = highestUsageWindow.value
|
||||
if (!highestWindow)
|
||||
return summaryEntries.value[0]
|
||||
|
||||
return summaryEntries.value.find(entry =>
|
||||
entry.windows.some(window => window.key === highestWindow.key),
|
||||
) ?? summaryEntries.value[0]
|
||||
})
|
||||
const additionalEntryCount = computed(() => Math.max(summaryEntries.value.length - 1, 0))
|
||||
</script>
|
||||
|
||||
@@ -58,20 +48,20 @@ const additionalEntryCount = computed(() => Math.max(summaryEntries.value.length
|
||||
</span>
|
||||
<span
|
||||
class="flex shrink-0 items-baseline gap-1 text-[9px] font-emphasis text-cp-text-quaternary"
|
||||
title="所有额度窗口中的最高已用比例"
|
||||
title="最近使用额度的当前已用比例"
|
||||
>
|
||||
<span>最高占用</span>
|
||||
<strong class="font-mono font-heavy tabular-nums" :class="highestUsageTextClass">
|
||||
{{ highestUsageDisplay }}
|
||||
<span>使用率</span>
|
||||
<strong class="font-mono font-heavy tabular-nums" :class="currentUsageTextClass">
|
||||
{{ currentUsageDisplay }}
|
||||
</strong>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div v-if="highestUsageEntry" class="flex min-w-0 items-end gap-2">
|
||||
<div v-if="recentUsageEntry" class="flex min-w-0 items-end gap-2">
|
||||
<div class="min-w-0 flex-1">
|
||||
<AccountQuotaSummaryEntry
|
||||
:label="highestUsageEntry.label"
|
||||
:windows="highestUsageEntry.windows"
|
||||
:label="recentUsageEntry.label"
|
||||
:windows="recentUsageEntry.windows"
|
||||
:show-percentage="false"
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { AccountQuotaWindow, AccountQuotaWindowEntry, AccountRow } from '../../constants'
|
||||
|
||||
type AccountModelUsage = AccountRow['usage']['models'][number]
|
||||
|
||||
/**
|
||||
* 根据账号最近一次模型请求选择对应额度组。
|
||||
* 没有专属额度的模型统一回退到 Codex 通用额度。
|
||||
*/
|
||||
export function recentlyUsedQuotaEntry(
|
||||
entries: readonly AccountQuotaWindowEntry[],
|
||||
models: readonly AccountModelUsage[],
|
||||
) {
|
||||
const recentModel = latestModelUsage(models)
|
||||
if (recentModel) {
|
||||
const modelKey = quotaIdentity(recentModel.model)
|
||||
const modelEntry = entries.find(entry => quotaEntryIdentities(entry).has(modelKey))
|
||||
if (modelEntry)
|
||||
return modelEntry
|
||||
}
|
||||
|
||||
return entries.find(entry => entry.windows.some(window => window.limitId === 'codex'))
|
||||
?? entries[0]
|
||||
}
|
||||
|
||||
/** 一个额度组可能包含多个滚动窗口,摘要使用其中当前占用最高的窗口。 */
|
||||
export function representativeQuotaWindow(entry: AccountQuotaWindowEntry | undefined) {
|
||||
return entry?.windows.reduce<AccountQuotaWindow | undefined>((representative, window) => {
|
||||
if (typeof window.usedPercent !== 'number')
|
||||
return representative
|
||||
if (
|
||||
!representative
|
||||
|| typeof representative.usedPercent !== 'number'
|
||||
|| window.usedPercent > representative.usedPercent
|
||||
) {
|
||||
return window
|
||||
}
|
||||
return representative
|
||||
}, undefined)
|
||||
}
|
||||
|
||||
function latestModelUsage(models: readonly AccountModelUsage[]) {
|
||||
let latest: AccountModelUsage | undefined
|
||||
let latestTimestamp = Number.NEGATIVE_INFINITY
|
||||
|
||||
for (const model of models) {
|
||||
const timestamp = Date.parse(model.lastUsedAt)
|
||||
if (!Number.isFinite(timestamp) || timestamp <= latestTimestamp)
|
||||
continue
|
||||
latest = model
|
||||
latestTimestamp = timestamp
|
||||
}
|
||||
|
||||
return latest
|
||||
}
|
||||
|
||||
function quotaEntryIdentities(entry: AccountQuotaWindowEntry) {
|
||||
const identities = new Set<string>()
|
||||
addQuotaIdentity(identities, entry.label)
|
||||
for (const window of entry.windows) {
|
||||
addQuotaIdentity(identities, window.limitId)
|
||||
addQuotaIdentity(identities, window.limitName)
|
||||
}
|
||||
return identities
|
||||
}
|
||||
|
||||
function addQuotaIdentity(identities: Set<string>, value: string | null) {
|
||||
const identity = quotaIdentity(value)
|
||||
if (identity)
|
||||
identities.add(identity)
|
||||
}
|
||||
|
||||
function quotaIdentity(value: string | null) {
|
||||
return value?.trim().toLowerCase().replace(/[^a-z0-9]+/g, '') ?? ''
|
||||
}
|
||||
@@ -35,33 +35,33 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
|
||||
<div class="xl:flex xl:min-h-0 xl:flex-col">
|
||||
<div class="mb-3 flex shrink-0 items-baseline justify-between gap-3">
|
||||
<h3 class="m-0 text-cp-lg font-heavy text-cp-text">
|
||||
Token 结构
|
||||
Tokens 结构
|
||||
</h3>
|
||||
<span class="text-cp-xs font-emphasis text-cp-text-quaternary">当前额度窗口</span>
|
||||
</div>
|
||||
<div class="grid gap-2 xl:min-h-0 xl:flex-1 xl:grid-rows-5">
|
||||
<div class="flex items-center justify-between rounded-lg bg-cp-green-bg px-3 py-2">
|
||||
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入 Tokens</span>
|
||||
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入</span>
|
||||
<strong class="font-mono text-cp text-cp-text">
|
||||
{{ account.usage.inputTokensDisplay }}
|
||||
</strong>
|
||||
</div>
|
||||
<div class="flex items-center justify-between rounded-lg bg-cp-orange-bg px-3 py-2">
|
||||
<span class="text-cp-sm font-bold text-cp-orange-text-on-bg">输出 Tokens</span>
|
||||
<span class="text-cp-sm font-bold text-cp-orange-text-on-bg">输出</span>
|
||||
<strong class="font-mono text-cp text-cp-text">
|
||||
{{ account.usage.outputTokensDisplay }}
|
||||
</strong>
|
||||
</div>
|
||||
<div class="flex items-center justify-between rounded-lg bg-cp-cyan-bg px-3 py-2">
|
||||
<span class="text-cp-sm font-bold text-cp-cyan-text-on-bg">缓存 Tokens</span>
|
||||
<span class="text-cp-sm font-bold text-cp-cyan-text-on-bg">缓存</span>
|
||||
<strong class="font-mono text-cp text-cp-text">
|
||||
{{ account.usage.cachedTokensDisplay }}
|
||||
</strong>
|
||||
</div>
|
||||
<div class="flex items-center justify-between rounded-lg bg-cp-blue-bg px-3 py-2">
|
||||
<span class="text-cp-sm font-bold text-cp-blue-text-on-bg">创建</span>
|
||||
<span class="text-cp-sm font-bold text-cp-blue-text-on-bg">推理</span>
|
||||
<strong class="font-mono text-cp text-cp-text">
|
||||
{{ account.usage.createdTokensDisplay }}
|
||||
{{ account.usage.reasoningTokensDisplay }}
|
||||
</strong>
|
||||
</div>
|
||||
<div class="flex items-center justify-between rounded-lg bg-cp-blue-bg px-3 py-2">
|
||||
|
||||
@@ -8,6 +8,10 @@ import BaseModal from '@/components/base/BaseModal/index.vue'
|
||||
import BaseScrollbar from '@/components/base/BaseScrollbar.vue'
|
||||
import BaseSegmented from '@/components/base/BaseSegmented.vue'
|
||||
import BaseSwitch from '@/components/base/BaseSwitch.vue'
|
||||
import {
|
||||
buildCodexConfigFiles,
|
||||
CODEX_WEBSOCKET_ENABLED_BY_DEFAULT,
|
||||
} from '../utils/codexConfig'
|
||||
|
||||
const props = defineProps<{
|
||||
apiKey: {
|
||||
@@ -24,7 +28,7 @@ const emit = defineEmits<{
|
||||
const open = defineModel<boolean>({ default: false })
|
||||
|
||||
const activePlatform = shallowRef('unix')
|
||||
const websocketEnabled = shallowRef(true)
|
||||
const websocketEnabled = shallowRef(CODEX_WEBSOCKET_ENABLED_BY_DEFAULT)
|
||||
|
||||
const platformOptions = [
|
||||
{ label: 'macOS / Linux', value: 'unix', icon: Apple },
|
||||
@@ -40,31 +44,15 @@ const configPath = computed(() =>
|
||||
const authPath = computed(() =>
|
||||
activePlatform.value === 'windows' ? '%userprofile%\\.codex\\auth.json' : '~/.codex/auth.json',
|
||||
)
|
||||
const codexAuthJson = computed(() => JSON.stringify({ OPENAI_API_KEY: keyValue.value }, null, 2))
|
||||
const defaultModel = 'gpt-5.6-terra'
|
||||
|
||||
const codexConfigToml = computed(
|
||||
() => `model_provider = "OpenAI"
|
||||
model = "${defaultModel}"
|
||||
review_model = "${defaultModel}"
|
||||
model_reasoning_effort = "max"
|
||||
service_tier = "default"
|
||||
disable_response_storage = true
|
||||
network_access = "enabled"
|
||||
|
||||
[model_providers.OpenAI]
|
||||
name = "OpenAI"
|
||||
base_url = "${props.apiBaseUrl}"
|
||||
wire_api = "responses"${websocketEnabled.value ? '\nsupports_websockets = true' : ''}
|
||||
requires_openai_auth = true
|
||||
|
||||
[features]${websocketEnabled.value ? '\nresponses_websockets_v2 = true' : ''}
|
||||
goals = true`,
|
||||
)
|
||||
const codexConfigFiles = computed(() => buildCodexConfigFiles({
|
||||
apiKey: keyValue.value,
|
||||
baseUrl: props.apiBaseUrl,
|
||||
websocketEnabled: websocketEnabled.value,
|
||||
}))
|
||||
|
||||
const visibleFiles = computed(() => [
|
||||
{ path: configPath.value, content: codexConfigToml.value, scrollbarHeight: '360px' },
|
||||
{ path: authPath.value, content: codexAuthJson.value, scrollbarHeight: undefined },
|
||||
{ path: configPath.value, content: codexConfigFiles.value.configToml, scrollbarHeight: '360px' },
|
||||
{ path: authPath.value, content: codexConfigFiles.value.authJson, scrollbarHeight: undefined },
|
||||
])
|
||||
</script>
|
||||
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
const DEFAULT_CC_SWITCH_MODEL = 'gpt-5.5'
|
||||
import {
|
||||
buildCodexConfigFiles,
|
||||
CODEX_DEFAULT_MODEL,
|
||||
CODEX_WEBSOCKET_ENABLED_BY_DEFAULT,
|
||||
} from './codexConfig'
|
||||
|
||||
export interface CodexCcSwitchImportInput {
|
||||
apiKey: string
|
||||
@@ -7,18 +11,37 @@ export interface CodexCcSwitchImportInput {
|
||||
}
|
||||
|
||||
export function buildCodexCcSwitchImportDeeplink(input: CodexCcSwitchImportInput): string {
|
||||
const baseUrl = input.baseUrl.replace(/\/+$/, '')
|
||||
const configFiles = buildCodexConfigFiles({
|
||||
apiKey: input.apiKey,
|
||||
baseUrl: input.baseUrl,
|
||||
websocketEnabled: CODEX_WEBSOCKET_ENABLED_BY_DEFAULT,
|
||||
})
|
||||
// Carry the complete files shown in the UI; the explicit fields below keep
|
||||
// compatibility with CCSwitch versions that only extract common settings.
|
||||
const config = encodeBase64(JSON.stringify({
|
||||
auth: configFiles.auth,
|
||||
config: configFiles.configToml,
|
||||
}))
|
||||
const entries: [string, string][] = [
|
||||
['resource', 'provider'],
|
||||
['app', 'codex'],
|
||||
['model', DEFAULT_CC_SWITCH_MODEL],
|
||||
['model', CODEX_DEFAULT_MODEL],
|
||||
['name', input.providerName],
|
||||
['homepage', baseUrl],
|
||||
['endpoint', baseUrl],
|
||||
['homepage', configFiles.baseUrl],
|
||||
['endpoint', configFiles.baseUrl],
|
||||
['apiKey', input.apiKey],
|
||||
['configFormat', 'json'],
|
||||
['config', config],
|
||||
['usageEnabled', 'false'],
|
||||
]
|
||||
|
||||
return `ccswitch://v1/import?${new URLSearchParams(entries).toString()}`
|
||||
}
|
||||
|
||||
function encodeBase64(value: string) {
|
||||
const bytes = new TextEncoder().encode(value)
|
||||
let binary = ''
|
||||
for (const byte of bytes)
|
||||
binary += String.fromCharCode(byte)
|
||||
return btoa(binary)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
export const CODEX_DEFAULT_MODEL = 'gpt-5.6-terra'
|
||||
export const CODEX_WEBSOCKET_ENABLED_BY_DEFAULT = false
|
||||
|
||||
export interface CodexConfigInput {
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
websocketEnabled?: boolean
|
||||
}
|
||||
|
||||
export function buildCodexConfigFiles(input: CodexConfigInput) {
|
||||
const baseUrl = input.baseUrl.replace(/\/+$/, '')
|
||||
const websocketEnabled = input.websocketEnabled ?? CODEX_WEBSOCKET_ENABLED_BY_DEFAULT
|
||||
const auth = { OPENAI_API_KEY: input.apiKey }
|
||||
const configToml = `model_provider = "OpenAI"
|
||||
model = "${CODEX_DEFAULT_MODEL}"
|
||||
review_model = "${CODEX_DEFAULT_MODEL}"
|
||||
model_reasoning_effort = "max"
|
||||
service_tier = "default"
|
||||
disable_response_storage = true
|
||||
network_access = "enabled"
|
||||
|
||||
[model_providers.OpenAI]
|
||||
name = "OpenAI"
|
||||
base_url = "${baseUrl}"
|
||||
wire_api = "responses"${websocketEnabled ? '\nsupports_websockets = true' : ''}
|
||||
requires_openai_auth = true
|
||||
|
||||
[features]${websocketEnabled ? '\nresponses_websockets_v2 = true' : ''}
|
||||
goals = true`
|
||||
|
||||
return {
|
||||
auth,
|
||||
authJson: JSON.stringify(auth, null, 2),
|
||||
baseUrl,
|
||||
configToml,
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,10 @@ import { CircleCheck, RefreshCw, ShieldAlert, TriangleAlert } from '@lucide/vue'
|
||||
import { computed } from 'vue'
|
||||
import BaseCard from '@/components/base/BaseCard.vue'
|
||||
import BaseEmpty from '@/components/base/BaseEmpty.vue'
|
||||
import ProviderIconGroup from '@/components/ProviderIconGroup.vue'
|
||||
import { formatCompactNumber } from '@/utils/number'
|
||||
import AccountIdentityCell from '@/views/accounts/components/AccountIdentityCell.vue'
|
||||
import AccountUsageWindow from '@/views/accounts/components/AccountUsageWindow/index.vue'
|
||||
import { metricToneIconClasses, metricToneValueClasses } from '../constants'
|
||||
import DashboardAccountIdentity from './DashboardAccountIdentity.vue'
|
||||
|
||||
type DashboardSnapshot = ReturnType<typeof dashboardSnapshotView>
|
||||
|
||||
@@ -233,24 +232,12 @@ const statusBars = computed(() => {
|
||||
<article
|
||||
v-for="account in accounts"
|
||||
:key="account.id"
|
||||
class="grid w-full shrink-0 grid-cols-1 gap-3 rounded-xl bg-cp-fill-alter/70 px-3.5 py-3.5 transition-colors xl:h-18.75 xl:grid-cols-[minmax(0,1fr)_minmax(70px,0.36fr)_minmax(74px,0.38fr)_minmax(82px,0.46fr)] xl:items-center xl:gap-4 xl:py-0"
|
||||
class="grid w-full shrink-0 grid-cols-1 gap-3 rounded-xl bg-cp-fill-alter/70 px-3.5 py-3.5 transition-colors xl:h-18.75 xl:grid-cols-[minmax(0,1.32fr)_minmax(64px,0.4fr)_minmax(66px,0.42fr)_minmax(96px,0.7fr)] xl:items-center xl:gap-3 xl:px-4 xl:py-0"
|
||||
>
|
||||
<AccountIdentityCell
|
||||
<DashboardAccountIdentity
|
||||
:account="account"
|
||||
show-plan
|
||||
title-mode="email"
|
||||
meta-position="secondary"
|
||||
meta-size="xs"
|
||||
class="min-w-0"
|
||||
>
|
||||
<template #meta>
|
||||
<ProviderIconGroup
|
||||
:provider="account.provider"
|
||||
:authentication-kind="account.authenticationKind"
|
||||
size="xs"
|
||||
/>
|
||||
</template>
|
||||
</AccountIdentityCell>
|
||||
class="min-w-0 xl:pr-4"
|
||||
/>
|
||||
|
||||
<span
|
||||
class="grid min-w-0 grid-cols-[minmax(0,0.82fr)_minmax(0,0.64fr)_minmax(104px,1fr)] items-start gap-3 xl:contents"
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
<script setup lang="ts">
|
||||
import type { dashboardSnapshotView } from '../composables/useDashboard'
|
||||
import { Key, LinkAlt, Openai, Xai } from '@boxicons/vue'
|
||||
import { computed } from 'vue'
|
||||
|
||||
import { formatProviderLabel } from '@/utils/providers'
|
||||
import AccountPlanBadge from '@/views/accounts/components/AccountPlanBadge.vue'
|
||||
import { stablePresetVisualToneClass } from '@/views/accounts/utils/visualTone'
|
||||
|
||||
type DashboardSnapshot = ReturnType<typeof dashboardSnapshotView>
|
||||
type DashboardAccount = DashboardSnapshot['accountUsage'][number]
|
||||
|
||||
const props = defineProps<{
|
||||
account: DashboardAccount
|
||||
}>()
|
||||
|
||||
const email = computed(() => props.account.email?.trim() || String(props.account.id))
|
||||
const displayTitle = computed(() => email.value.split('@')[0] || email.value)
|
||||
const normalizedProvider = computed(() => props.account.provider?.trim().toLowerCase() || '')
|
||||
const normalizedAuthenticationKind = computed(() =>
|
||||
props.account.authenticationKind?.trim().toLowerCase() || '',
|
||||
)
|
||||
const providerLabel = computed(() => formatProviderLabel(props.account.provider, '未知平台'))
|
||||
|
||||
const authenticationLabel = computed(() => {
|
||||
if (normalizedAuthenticationKind.value === 'oauth')
|
||||
return 'OAuth'
|
||||
if (normalizedAuthenticationKind.value === 'api_key')
|
||||
return 'API Key'
|
||||
return props.account.authenticationKind?.trim() || '未知认证类型'
|
||||
})
|
||||
|
||||
const avatarToneClass = computed(() =>
|
||||
stablePresetVisualToneClass(props.account.id || props.account.email || email.value),
|
||||
)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex min-w-0 items-center gap-3">
|
||||
<span class="relative inline-flex size-9 shrink-0">
|
||||
<span
|
||||
class="inline-flex size-9 items-center justify-center rounded-lg shadow-cp-tertiary"
|
||||
:class="avatarToneClass"
|
||||
:title="providerLabel"
|
||||
>
|
||||
<Openai v-if="normalizedProvider === 'openai'" class="size-3.5" />
|
||||
<Xai v-else-if="normalizedProvider === 'xai'" class="size-3.5" />
|
||||
<span v-else class="text-cp-xs font-heavy">?</span>
|
||||
<span class="sr-only">{{ providerLabel }}</span>
|
||||
</span>
|
||||
|
||||
<span
|
||||
class="absolute -right-1 -bottom-1 inline-flex size-4 items-center justify-center rounded-[5px] bg-cp-bg-container text-cp-text shadow-cp-tertiary"
|
||||
:title="authenticationLabel"
|
||||
>
|
||||
<LinkAlt v-if="normalizedAuthenticationKind === 'oauth'" class="size-2.5" />
|
||||
<Key v-else-if="normalizedAuthenticationKind === 'api_key'" class="size-2.5" />
|
||||
<span v-else class="text-[8px] font-heavy text-cp-text-quaternary">?</span>
|
||||
<span class="sr-only">{{ authenticationLabel }}</span>
|
||||
</span>
|
||||
</span>
|
||||
|
||||
<span class="min-w-0 flex-1">
|
||||
<span class="flex min-w-0 items-center gap-1.5">
|
||||
<strong class="min-w-0 truncate text-cp leading-[1.15] font-heavy text-cp-text">
|
||||
{{ displayTitle }}
|
||||
</strong>
|
||||
<AccountPlanBadge :plan-type="account.planType" size="xs" />
|
||||
</span>
|
||||
<span
|
||||
class="mt-0.5 block min-w-0 truncate font-mono text-cp-xs font-emphasis text-cp-text-quaternary"
|
||||
:title="email"
|
||||
>
|
||||
{{ email }}
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
@@ -31,6 +31,10 @@ const accountLabel = computed(() => props.record?.accountEmail
|
||||
|| props.record?.accountId)
|
||||
|
||||
const errorFields = computed(() => visibleFields([
|
||||
{
|
||||
label: '处理状态',
|
||||
value: props.record?.metadata.recoveredAt ? '已自动恢复' : null,
|
||||
},
|
||||
{ label: 'Provider 错误码', value: props.record?.providerErrorCode, mono: true },
|
||||
{ label: '失败分类', value: failureClassText(props.record?.failureClass) },
|
||||
{ label: '分类代码', value: props.record?.failureClass, mono: true },
|
||||
|
||||
@@ -12,7 +12,9 @@ import ProviderIconGroup from '@/components/ProviderIconGroup.vue'
|
||||
import { useOpsErrorsTable } from '../composables/useOpsErrorsTable'
|
||||
import { opsErrorColumns } from '../constants'
|
||||
import { opsErrorSummary } from '../utils/opsErrorPresentation'
|
||||
import { usageUserAgent } from '../utils/records'
|
||||
import OpsErrorDetailModal from './OpsErrorDetailModal.vue'
|
||||
import UsageClientIpCell from './UsageClientIpCell.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
timeRangeParams: UsageTimeRangeParams
|
||||
@@ -122,9 +124,17 @@ function upstreamSendStateText(value: string | null | undefined) {
|
||||
</template>
|
||||
<template #message="{ row }">
|
||||
<div class="min-w-0 py-0.5" :title="row.message || opsErrorSummary(row)">
|
||||
<code class="block max-w-full truncate font-mono text-cp-sm font-bold text-cp-error-text">
|
||||
{{ opsErrorSummary(row) }}
|
||||
</code>
|
||||
<div class="flex min-w-0 items-center gap-2">
|
||||
<code class="block min-w-0 flex-1 truncate font-mono text-cp-sm font-bold text-cp-error-text">
|
||||
{{ opsErrorSummary(row) }}
|
||||
</code>
|
||||
<span
|
||||
v-if="row.metadata.recoveredAt"
|
||||
class="inline-flex h-5 shrink-0 items-center rounded-full bg-cp-success-bg px-2 text-cp-xs leading-none font-heavy text-cp-success-text"
|
||||
>
|
||||
已自动恢复
|
||||
</span>
|
||||
</div>
|
||||
<p
|
||||
v-if="row.message"
|
||||
class="mt-1 mb-0 line-clamp-1 text-cp-xs leading-[1.45] font-emphasis text-cp-text-secondary"
|
||||
@@ -157,6 +167,14 @@ function upstreamSendStateText(value: string | null | undefined) {
|
||||
{{ modelText(row) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #clientIp="{ row }">
|
||||
<UsageClientIpCell :record="row" />
|
||||
</template>
|
||||
<template #userAgent="{ row }">
|
||||
<span class="block max-w-full wrap-break-word whitespace-normal font-mono text-cp-sm leading-[1.4] font-emphasis text-cp-text-secondary">
|
||||
{{ usageUserAgent(row) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #actions="{ row }">
|
||||
<BaseIconButton
|
||||
variant="ghost"
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
<script setup lang="ts">
|
||||
import type { UsageDisplayRecord } from '../utils/records'
|
||||
|
||||
import { computed } from 'vue'
|
||||
import { usageClientIp } from '../utils/records'
|
||||
|
||||
const props = defineProps<{
|
||||
record: UsageDisplayRecord
|
||||
record: { clientIp?: string | null }
|
||||
}>()
|
||||
|
||||
const clientIp = computed(() => usageClientIp(props.record))
|
||||
|
||||
@@ -19,7 +19,7 @@ const search = defineModel<string>('search', { required: true })
|
||||
<template>
|
||||
<div class="flex w-full items-center gap-3" role="group" aria-label="使用记录筛选与操作">
|
||||
<div class="min-w-0 flex-1 sm:w-96 sm:flex-none">
|
||||
<BaseInput v-model="search" placeholder="请求、Key、账号或模型 ID 前缀" class="w-full">
|
||||
<BaseInput v-model="search" placeholder="请求、API Key、账号或模型 ID 前缀" class="w-full">
|
||||
<template #prefix>
|
||||
<Search class="size-4.5 text-cp-text-tertiary" />
|
||||
</template>
|
||||
|
||||
@@ -112,8 +112,11 @@ const summaryMetrics = computed(() => {
|
||||
const hasData = computed(
|
||||
() =>
|
||||
!props.loading
|
||||
&& selectedPoints.value.some(
|
||||
point => point.first != null || point.second != null || point.third != null,
|
||||
&& performancePoints.value.some(
|
||||
point =>
|
||||
seriesValue(point, 'first') != null
|
||||
|| seriesValue(point, 'second') != null
|
||||
|| seriesValue(point, 'third') != null,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -267,7 +270,7 @@ function formatThroughput(value: number | null) {
|
||||
: activeView === 'throughput'
|
||||
? '当前范围没有吞吐样本'
|
||||
: activeView === 'scheduling'
|
||||
? '部署迁移后才会开始积累调度与容量样本'
|
||||
? '当前范围没有调度或容量样本'
|
||||
: '当前范围没有总耗时样本'
|
||||
"
|
||||
class="h-52.5 place-content-center"
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import type { UsageDisplayRecord } from '../utils/records'
|
||||
|
||||
import { Bot } from '@lucide/vue'
|
||||
import { usageIsSubagent, usageReasoningEffort } from '../utils/records'
|
||||
import { Bot, SquareDashedBottomCode } from '@lucide/vue'
|
||||
import { usageIsReview, usageIsSubagent, usageReasoningEffort } from '../utils/records'
|
||||
|
||||
defineProps<{
|
||||
record: UsageDisplayRecord
|
||||
@@ -20,7 +20,15 @@ defineProps<{
|
||||
title="子代理请求"
|
||||
aria-label="子代理请求"
|
||||
>
|
||||
<Bot class="size-3.25" stroke-width="2.2" />
|
||||
<Bot class="size-3.5" stroke-width="2.2" />
|
||||
</span>
|
||||
<span
|
||||
v-if="usageIsReview(record)"
|
||||
class="inline-flex shrink-0 text-cp-purple-text"
|
||||
title="Review 子代理请求"
|
||||
aria-label="Review 子代理请求"
|
||||
>
|
||||
<SquareDashedBottomCode class="size-3.25" stroke-width="2.2" />
|
||||
</span>
|
||||
</span>
|
||||
</template>
|
||||
|
||||
@@ -21,6 +21,8 @@ interface UseUsageRecordsTableOptions {
|
||||
|
||||
type UsageLoadScope = 'all' | 'table'
|
||||
|
||||
const CLIENT_API_KEY_VISIBLE_PREFIX_LENGTH = 10
|
||||
|
||||
interface UsageLoadOptions {
|
||||
scope?: UsageLoadScope
|
||||
background?: boolean
|
||||
@@ -49,7 +51,7 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
...(providerQuery.value ? { provider: providerQuery.value } : {}),
|
||||
})
|
||||
const filterParams = () => ({
|
||||
search: searchQuery.value || undefined,
|
||||
search: usageSearchParam(searchQuery.value),
|
||||
})
|
||||
const usagePagination = computed(() => ({
|
||||
currentPage: currentPage.value,
|
||||
@@ -244,6 +246,15 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
function usageSearchParam(value: string) {
|
||||
const search = value.trim()
|
||||
if (!search)
|
||||
return undefined
|
||||
if (search.startsWith('sk_'))
|
||||
return search.slice(0, CLIENT_API_KEY_VISIBLE_PREFIX_LENGTH)
|
||||
return search
|
||||
}
|
||||
|
||||
function emptySummary() {
|
||||
const summary: Awaited<ReturnType<typeof getUsageRecordSummary>> = {
|
||||
totalRequests: '0',
|
||||
|
||||
@@ -20,7 +20,7 @@ export const usageRecordColumns = defineTableColumns<UsageDisplayRecord>([
|
||||
format: (value: unknown) => formatProvider(typeof value === 'string' ? value : null),
|
||||
},
|
||||
{ key: 'model', label: '模型', kind: 'custom', size: 'xl' },
|
||||
{ key: 'reasoningEffort', label: '推理强度', kind: 'status' },
|
||||
{ key: 'reasoningEffort', label: '推理强度', kind: 'status', size: 'lg' },
|
||||
{ key: 'route', label: '端点', kind: 'mono' },
|
||||
{ key: 'upstreamTransport', label: '上游', kind: 'status', size: 'md' },
|
||||
{ key: 'clientTransport', label: '接入', kind: 'status', size: 'md' },
|
||||
@@ -42,8 +42,8 @@ export const opsErrorColumns = defineTableColumns<OpsErrorRow>([
|
||||
{ key: 'route', label: '端点', kind: 'mono', size: 'xl', emptyText: '未记录' },
|
||||
{ key: 'createdAtDisplay', label: '时间', kind: 'datetime' },
|
||||
{ key: 'requestId', label: '请求 ID', kind: 'mono', size: '2xl', emptyText: '未记录' },
|
||||
{ key: 'clientIp', label: 'IP', kind: 'mono', size: '3xl', emptyText: '未记录' },
|
||||
{ key: 'userAgent', label: 'User-Agent', kind: 'mono', size: '4xl', emptyText: '未记录' },
|
||||
{ key: 'clientIp', label: 'IP', kind: 'custom', size: '3xl', emptyText: '未记录' },
|
||||
{ key: 'userAgent', label: 'User-Agent', kind: 'custom', size: '4xl', emptyText: '未记录' },
|
||||
{ key: 'actions', label: '操作', kind: 'actions', size: 'sm' },
|
||||
])
|
||||
|
||||
|
||||
@@ -178,11 +178,11 @@ export function usageAuthenticationKind(record: UsageCommonRecord) {
|
||||
return typeof record.authenticationKind === 'string' ? record.authenticationKind : null
|
||||
}
|
||||
|
||||
export function usageClientIp(record: UsageCommonRecord) {
|
||||
export function usageClientIp(record: { clientIp?: string | null }) {
|
||||
return record.clientIp || '—'
|
||||
}
|
||||
|
||||
export function usageUserAgent(record: UsageCommonRecord) {
|
||||
export function usageUserAgent(record: { userAgent?: string | null }) {
|
||||
return record.userAgent || '—'
|
||||
}
|
||||
|
||||
@@ -197,6 +197,10 @@ export function usageIsSubagent(record: UsageCommonRecord) {
|
||||
return Boolean(record.subagentKind)
|
||||
}
|
||||
|
||||
export function usageIsReview(record: UsageCommonRecord) {
|
||||
return record.subagentKind === 'review'
|
||||
}
|
||||
|
||||
export function usageIsCompact(record: UsageCommonRecord) {
|
||||
return record.compact === true
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.3.16"
|
||||
version: "3.3.23"
|
||||
|
||||
@@ -44,7 +44,7 @@ README 保持用户导向;不要把上游 URL、重试常量、数据库字段
|
||||
|
||||
## 当前关键合同
|
||||
|
||||
- 数据面公开 Responses JSON/SSE/WS、review、Images generation/edit 和模型目录;不提供 Chat Completions。
|
||||
- 数据面公开 Responses JSON/SSE/WS(含 Codex 子代理语义)、Images generation/edit 和模型目录;不提供 Chat Completions。
|
||||
- OpenAI Responses/Images 保持业务 wire 透明;xAI 在 Provider 内完成 Grok/Responses 转换。
|
||||
- Images 固定走 OpenAI Provider 自有端点,不要求模型,也不参与文本模型映射。
|
||||
- OpenAI 账号支持 OAuth、AT/RT 和 OAuth JSON;xAI 只接受 OAuth session/账号 JSON,
|
||||
|
||||
@@ -41,12 +41,13 @@
|
||||
|
||||
公开路由由 `gateway-api/src/openai/router.rs` 固定:
|
||||
|
||||
- Responses:`POST /v1/responses`、WebSocket `GET /v1/responses`、`POST /v1/responses/review`;
|
||||
- Responses:`POST /v1/responses`、WebSocket `GET /v1/responses`;
|
||||
- Images:`POST /v1/images/generations`、`POST /v1/images/edits`;
|
||||
- 模型:`/v1/models`、catalog、info、detail。
|
||||
- 模型:`GET /v1/models`、`GET /v1/models/{model_id}`。
|
||||
|
||||
Responses adapter 解析参与路由的最少语义,并保留原始请求;Images adapter 不解析模型或重建 JSON,
|
||||
直接创建 `GenerateImage` operation。两者都进入同一 Core execution lifecycle。
|
||||
Responses adapter 解析参与路由的最少语义,并保留原始请求;review 等子代理仍走 `/v1/responses`,
|
||||
类型由 `x-openai-subagent` 携带。Images adapter 不解析模型或重建 JSON,直接创建
|
||||
`GenerateImage` operation。两者都进入同一 Core execution lifecycle。
|
||||
|
||||
定位请求问题时按以下顺序追踪:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user